Line data Source code
1 : /* SLP - Basic Block Vectorization
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #define INCLUDE_ALGORITHM
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "optabs-tree.h"
34 : #include "insn-config.h"
35 : #include "recog.h" /* FIXME: for insn_data */
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "gimple-iterator.h"
39 : #include "cfgloop.h"
40 : #include "tree-vectorizer.h"
41 : #include "langhooks.h"
42 : #include "gimple-walk.h"
43 : #include "dbgcnt.h"
44 : #include "tree-vector-builder.h"
45 : #include "vec-perm-indices.h"
46 : #include "gimple-fold.h"
47 : #include "internal-fn.h"
48 : #include "dump-context.h"
49 : #include "cfganal.h"
50 : #include "tree-eh.h"
51 : #include "tree-cfg.h"
52 : #include "alloc-pool.h"
53 : #include "sreal.h"
54 : #include "predict.h"
55 :
56 : #define REDUC_GROUP_FIRST_ELEMENT(S) \
57 : (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
58 :
59 : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
60 : load_permutation_t &,
61 : const vec<tree> &,
62 : gimple_stmt_iterator *,
63 : poly_uint64, bool, bool,
64 : unsigned *,
65 : unsigned * = nullptr,
66 : bool = false);
67 : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
68 : slp_tree, lane_permutation_t &,
69 : vec<slp_tree> &, bool);
70 : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
71 : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
72 :
73 : static object_allocator<_slp_tree> *slp_tree_pool;
74 : static slp_tree slp_first_node;
75 :
76 : void
77 1136857 : vect_slp_init (void)
78 : {
79 1136857 : slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
80 1136857 : }
81 :
82 : void
83 1136857 : vect_slp_fini (void)
84 : {
85 1829189 : while (slp_first_node)
86 692332 : delete slp_first_node;
87 2273714 : delete slp_tree_pool;
88 1136857 : slp_tree_pool = NULL;
89 1136857 : }
90 :
91 : void *
92 8273609 : _slp_tree::operator new (size_t n)
93 : {
94 8273609 : gcc_assert (n == sizeof (_slp_tree));
95 8273609 : return slp_tree_pool->allocate_raw ();
96 : }
97 :
98 : void
99 8273609 : _slp_tree::operator delete (void *node, size_t n)
100 : {
101 8273609 : gcc_assert (n == sizeof (_slp_tree));
102 8273609 : slp_tree_pool->remove_raw (node);
103 8273609 : }
104 :
105 :
106 : /* Initialize a SLP node. */
107 :
108 8273609 : _slp_tree::_slp_tree ()
109 : {
110 8273609 : this->prev_node = NULL;
111 8273609 : if (slp_first_node)
112 7240792 : slp_first_node->prev_node = this;
113 8273609 : this->next_node = slp_first_node;
114 8273609 : slp_first_node = this;
115 8273609 : SLP_TREE_SCALAR_STMTS (this) = vNULL;
116 8273609 : SLP_TREE_SCALAR_OPS (this) = vNULL;
117 8273609 : SLP_TREE_LIVE_LANES (this) = vNULL;
118 8273609 : SLP_TREE_VEC_DEFS (this) = vNULL;
119 8273609 : SLP_TREE_CHILDREN (this) = vNULL;
120 8273609 : SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
121 8273609 : SLP_TREE_LANE_PERMUTATION (this) = vNULL;
122 8273609 : SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
123 8273609 : SLP_TREE_CODE (this) = ERROR_MARK;
124 8273609 : SLP_TREE_GS_SCALE (this) = 0;
125 8273609 : SLP_TREE_GS_BASE (this) = NULL_TREE;
126 8273609 : this->ldst_lanes = false;
127 8273609 : this->avoid_stlf_fail = false;
128 8273609 : SLP_TREE_VECTYPE (this) = NULL_TREE;
129 8273609 : SLP_TREE_REPRESENTATIVE (this) = NULL;
130 8273609 : this->cycle_info.id = -1;
131 8273609 : this->cycle_info.reduc_idx = -1;
132 8273609 : SLP_TREE_REF_COUNT (this) = 1;
133 8273609 : this->failed = NULL;
134 8273609 : this->lanes = 0;
135 8273609 : SLP_TREE_TYPE (this) = undef_vec_info_type;
136 8273609 : this->data = NULL;
137 8273609 : this->si = NULL;
138 8273609 : }
139 :
140 : /* Tear down a SLP node. */
141 :
142 8273609 : _slp_tree::~_slp_tree ()
143 : {
144 8273609 : if (this->prev_node)
145 5055085 : this->prev_node->next_node = this->next_node;
146 : else
147 3218524 : slp_first_node = this->next_node;
148 8273609 : if (this->next_node)
149 6245989 : this->next_node->prev_node = this->prev_node;
150 8273609 : SLP_TREE_CHILDREN (this).release ();
151 8273609 : SLP_TREE_SCALAR_STMTS (this).release ();
152 8273609 : SLP_TREE_SCALAR_OPS (this).release ();
153 8273609 : SLP_TREE_LIVE_LANES (this).release ();
154 8273609 : SLP_TREE_VEC_DEFS (this).release ();
155 8273609 : SLP_TREE_LOAD_PERMUTATION (this).release ();
156 8273609 : SLP_TREE_LANE_PERMUTATION (this).release ();
157 8273609 : if (this->failed)
158 2217294 : free (failed);
159 8273609 : if (this->data)
160 1278600 : delete this->data;
161 8273609 : }
162 :
163 : /* Push the single SSA definition in DEF to the vector of vector defs. */
164 :
165 : void
166 531726 : _slp_tree::push_vec_def (gimple *def)
167 : {
168 531726 : if (gphi *phi = dyn_cast <gphi *> (def))
169 59484 : vec_defs.quick_push (gimple_phi_result (phi));
170 : else
171 : {
172 472242 : def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
173 472242 : vec_defs.quick_push (get_def_from_ptr (defop));
174 : }
175 531726 : }
176 :
177 : /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
178 :
179 : void
180 15601262 : vect_free_slp_tree (slp_tree node)
181 : {
182 15601262 : int i;
183 15601262 : slp_tree child;
184 :
185 15601262 : if (--SLP_TREE_REF_COUNT (node) != 0)
186 15601262 : return;
187 :
188 11752740 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
189 4171463 : if (child)
190 3789942 : vect_free_slp_tree (child);
191 :
192 7581277 : delete node;
193 : }
194 :
195 : /* Return a location suitable for dumpings related to the SLP instance. */
196 :
197 : dump_user_location_t
198 3551793 : _slp_instance::location () const
199 : {
200 3551793 : if (!root_stmts.is_empty ())
201 411346 : return root_stmts[0]->stmt;
202 : else
203 3140447 : return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
204 : }
205 :
206 :
207 : /* Free the memory allocated for the SLP instance. */
208 :
209 : void
210 1628477 : vect_free_slp_instance (slp_instance instance)
211 : {
212 1628477 : vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
213 1628477 : SLP_INSTANCE_LOADS (instance).release ();
214 1628477 : SLP_INSTANCE_ROOT_STMTS (instance).release ();
215 1628477 : SLP_INSTANCE_REMAIN_DEFS (instance).release ();
216 1628477 : instance->subgraph_entries.release ();
217 1628477 : instance->cost_vec.release ();
218 1628477 : free (instance);
219 1628477 : }
220 :
221 :
222 : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
223 : for a permute node or else ERROR_MARK. */
224 :
225 : slp_tree
226 97583 : vect_create_new_slp_node (unsigned nops, tree_code code)
227 : {
228 97583 : gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
229 97583 : slp_tree node = new _slp_tree;
230 97583 : SLP_TREE_SCALAR_STMTS (node) = vNULL;
231 97583 : SLP_TREE_CHILDREN (node).create (nops);
232 97583 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
233 97583 : SLP_TREE_CODE (node) = code;
234 97583 : return node;
235 : }
236 :
237 : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children. */
238 :
239 : static slp_tree
240 3979938 : vect_create_new_slp_node (slp_tree node,
241 : vec<stmt_vec_info> scalar_stmts, unsigned nops)
242 : {
243 3979938 : SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
244 3979938 : SLP_TREE_CHILDREN (node).create (nops);
245 3979938 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
246 3979938 : SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
247 3979938 : SLP_TREE_LANES (node) = scalar_stmts.length ();
248 3979938 : return node;
249 : }
250 :
251 : /* Create an SLP node for SCALAR_STMTS and NOPS children. */
252 :
253 : static slp_tree
254 8392 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
255 : {
256 8392 : return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
257 : }
258 :
259 : /* Create a vect_external_def SLP node inplace at NODE for scalar
260 : operands OPS. */
261 :
262 : static slp_tree
263 1966664 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
264 : {
265 1966664 : SLP_TREE_SCALAR_OPS (node) = ops;
266 1966664 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
267 0 : SLP_TREE_LANES (node) = ops.length ();
268 1966664 : return node;
269 : }
270 :
271 : /* Create a vect_external_def SLP node for scalar operands OPS. */
272 :
273 : static slp_tree
274 1966664 : vect_create_new_slp_node (vec<tree> ops)
275 : {
276 1966664 : return vect_create_new_slp_node (new _slp_tree, ops);
277 : }
278 :
279 :
280 : /* This structure is used in creation of an SLP tree. Each instance
281 : corresponds to the same operand in a group of scalar stmts in an SLP
282 : node. */
283 : typedef struct _slp_oprnd_info
284 : {
285 : /* Def-stmts for the operands. */
286 : vec<stmt_vec_info> def_stmts;
287 : /* Operands. */
288 : vec<tree> ops;
289 : /* Information about the first statement, its vector def-type, type, the
290 : operand itself in case it's constant, and an indication if it's a pattern
291 : stmt and gather/scatter info. */
292 : tree first_op_type;
293 : enum vect_def_type first_dt;
294 : bool any_pattern;
295 : bool first_gs_p;
296 : gather_scatter_info first_gs_info;
297 : } *slp_oprnd_info;
298 :
299 :
300 : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
301 : operand. */
302 : static vec<slp_oprnd_info>
303 3536849 : vect_create_oprnd_info (int nops, int group_size)
304 : {
305 3536849 : int i;
306 3536849 : slp_oprnd_info oprnd_info;
307 3536849 : vec<slp_oprnd_info> oprnds_info;
308 :
309 3536849 : oprnds_info.create (nops);
310 12720352 : for (i = 0; i < nops; i++)
311 : {
312 5646654 : oprnd_info = XNEW (struct _slp_oprnd_info);
313 5646654 : oprnd_info->def_stmts.create (group_size);
314 5646654 : oprnd_info->ops.create (group_size);
315 5646654 : oprnd_info->first_dt = vect_uninitialized_def;
316 5646654 : oprnd_info->first_op_type = NULL_TREE;
317 5646654 : oprnd_info->any_pattern = false;
318 5646654 : oprnd_info->first_gs_p = false;
319 5646654 : oprnds_info.quick_push (oprnd_info);
320 : }
321 :
322 3536849 : return oprnds_info;
323 : }
324 :
325 :
326 : /* Free operands info. */
327 :
328 : static void
329 3536849 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
330 : {
331 3536849 : int i;
332 3536849 : slp_oprnd_info oprnd_info;
333 :
334 9183503 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
335 : {
336 5646654 : oprnd_info->def_stmts.release ();
337 5646654 : oprnd_info->ops.release ();
338 5646654 : XDELETE (oprnd_info);
339 : }
340 :
341 3536849 : oprnds_info.release ();
342 3536849 : }
343 :
344 : /* Return the execution frequency of NODE (so that a higher value indicates
345 : a "more important" node when optimizing for speed). */
346 :
347 : static sreal
348 3671415 : vect_slp_node_weight (vec_info *vinfo, slp_tree node)
349 : {
350 3671415 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
351 3671415 : basic_block bb;
352 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
353 : but are laid out close to their children. */
354 3671415 : if (!stmt_info)
355 143093 : bb = vinfo->bbs[0];
356 : else
357 3998528 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
358 3671415 : return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
359 : }
360 :
361 : /* Return true if STMTS contains a pattern statement. */
362 :
363 : static bool
364 30550 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
365 : {
366 30550 : stmt_vec_info stmt_info;
367 30550 : unsigned int i;
368 89106 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
369 66080 : if (stmt_info && is_pattern_stmt_p (stmt_info))
370 : return true;
371 : return false;
372 : }
373 :
374 : /* Return true when all lanes in the external or constant NODE have
375 : the same value. */
376 :
377 : static bool
378 645985 : vect_slp_tree_uniform_p (slp_tree node)
379 : {
380 645985 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
381 : || SLP_TREE_DEF_TYPE (node) == vect_external_def);
382 :
383 : /* Pre-existing vectors. */
384 645985 : if (SLP_TREE_SCALAR_OPS (node).is_empty ())
385 : return false;
386 :
387 : unsigned i;
388 : tree op, first = NULL_TREE;
389 1482408 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
390 1323883 : if (!first)
391 : first = op;
392 677898 : else if (!operand_equal_p (first, op, 0))
393 : return false;
394 :
395 : return true;
396 : }
397 :
398 : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
399 : that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
400 : of the chain. */
401 :
402 : int
403 746670 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
404 : stmt_vec_info first_stmt_info)
405 : {
406 746670 : stmt_vec_info next_stmt_info = first_stmt_info;
407 746670 : int result = 0;
408 :
409 746670 : if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
410 : return -1;
411 :
412 2114957 : do
413 : {
414 2114957 : if (next_stmt_info == stmt_info)
415 : return result;
416 1368287 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
417 1368287 : if (next_stmt_info)
418 1368287 : result += DR_GROUP_GAP (next_stmt_info);
419 : }
420 1368287 : while (next_stmt_info);
421 :
422 : return -1;
423 : }
424 :
425 : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
426 : using the method implemented by duplicate_and_interleave. Return true
427 : if so, returning the number of intermediate vectors in *NVECTORS_OUT
428 : (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
429 : (if nonnull). */
430 :
431 : bool
432 0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
433 : tree elt_type, unsigned int *nvectors_out,
434 : tree *vector_type_out,
435 : tree *permutes)
436 : {
437 0 : tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
438 0 : if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
439 : return false;
440 :
441 0 : machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
442 0 : poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
443 0 : unsigned int nvectors = 1;
444 0 : for (;;)
445 : {
446 0 : scalar_int_mode int_mode;
447 0 : poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
448 0 : if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
449 : {
450 : /* Get the natural vector type for this SLP group size. */
451 0 : tree int_type = build_nonstandard_integer_type
452 0 : (GET_MODE_BITSIZE (int_mode), 1);
453 0 : tree vector_type
454 0 : = get_vectype_for_scalar_type (vinfo, int_type, count);
455 0 : poly_int64 half_nelts;
456 0 : if (vector_type
457 0 : && VECTOR_MODE_P (TYPE_MODE (vector_type))
458 0 : && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
459 : GET_MODE_SIZE (base_vector_mode))
460 0 : && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
461 : 2, &half_nelts))
462 : {
463 : /* Try fusing consecutive sequences of COUNT / NVECTORS elements
464 : together into elements of type INT_TYPE and using the result
465 : to build NVECTORS vectors. */
466 0 : poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
467 0 : vec_perm_builder sel1 (nelts, 2, 3);
468 0 : vec_perm_builder sel2 (nelts, 2, 3);
469 :
470 0 : for (unsigned int i = 0; i < 3; ++i)
471 : {
472 0 : sel1.quick_push (i);
473 0 : sel1.quick_push (i + nelts);
474 0 : sel2.quick_push (half_nelts + i);
475 0 : sel2.quick_push (half_nelts + i + nelts);
476 : }
477 0 : vec_perm_indices indices1 (sel1, 2, nelts);
478 0 : vec_perm_indices indices2 (sel2, 2, nelts);
479 0 : machine_mode vmode = TYPE_MODE (vector_type);
480 0 : if (can_vec_perm_const_p (vmode, vmode, indices1)
481 0 : && can_vec_perm_const_p (vmode, vmode, indices2))
482 : {
483 0 : if (nvectors_out)
484 0 : *nvectors_out = nvectors;
485 0 : if (vector_type_out)
486 0 : *vector_type_out = vector_type;
487 0 : if (permutes)
488 : {
489 0 : permutes[0] = vect_gen_perm_mask_checked (vector_type,
490 : indices1);
491 0 : permutes[1] = vect_gen_perm_mask_checked (vector_type,
492 : indices2);
493 : }
494 0 : return true;
495 : }
496 0 : }
497 : }
498 0 : if (!multiple_p (elt_bytes, 2, &elt_bytes))
499 : return false;
500 0 : nvectors *= 2;
501 : /* We need to be able to fuse COUNT / NVECTORS elements together. */
502 0 : if (!multiple_p (count, nvectors))
503 : return false;
504 : }
505 : }
506 :
507 : /* Return true if DTA and DTB match. */
508 :
509 : static bool
510 17179586 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
511 : {
512 17179586 : return (dta == dtb
513 366747 : || ((dta == vect_external_def || dta == vect_constant_def)
514 227094 : && (dtb == vect_external_def || dtb == vect_constant_def)));
515 : }
516 :
517 : #define GATHER_SCATTER_OFFSET (-3)
518 :
519 : /* For most SLP statements, there is a one-to-one mapping between
520 : gimple arguments and child nodes. If that is not true for STMT,
521 : return an array that contains:
522 :
523 : - the number of child nodes, followed by
524 : - for each child node, the index of the argument associated with that node.
525 : The special index -1 is the first operand of an embedded comparison and
526 : the special index -2 is the second operand of an embedded comparison.
527 : The special index -3 is the offset of a gather as analyzed by
528 : vect_check_gather_scatter.
529 :
530 : SWAP is as for vect_get_and_check_slp_defs. */
531 :
532 : static const int *
533 25275422 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
534 : unsigned char swap)
535 : {
536 25275422 : static const int no_arg_map[] = { 0 };
537 25275422 : static const int arg0_map[] = { 1, 0 };
538 25275422 : static const int arg2_map[] = { 1, 2 };
539 25275422 : static const int arg2_arg3_map[] = { 2, 2, 3 };
540 25275422 : static const int arg2_arg4_map[] = { 2, 2, 4 };
541 25275422 : static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
542 25275422 : static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
543 25275422 : static const int arg3_arg2_map[] = { 2, 3, 2 };
544 25275422 : static const int op00_map[] = { 1, -1 };
545 25275422 : static const int op1_op0_map[] = { 2, 1, 0 };
546 25275422 : static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
547 25275422 : static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
548 25275422 : static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
549 25275422 : static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
550 25275422 : static const int mask_call_maps[6][7] = {
551 : { 1, 1, },
552 : { 2, 1, 2, },
553 : { 3, 1, 2, 3, },
554 : { 4, 1, 2, 3, 4, },
555 : { 5, 1, 2, 3, 4, 5, },
556 : { 6, 1, 2, 3, 4, 5, 6 },
557 : };
558 :
559 25275422 : gcc_checking_assert (!swap
560 : || !is_gimple_assign (stmt)
561 : || TREE_CODE_CLASS
562 : (gimple_assign_rhs_code (stmt)) == tcc_comparison
563 : || commutative_tree_code
564 : (gimple_assign_rhs_code (stmt)));
565 :
566 25275422 : if (auto assign = dyn_cast<const gassign *> (stmt))
567 : {
568 23717989 : tree_code code = gimple_assign_rhs_code (assign);
569 23717989 : if (code == COND_EXPR
570 23717989 : && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
571 0 : gcc_unreachable ();
572 23717989 : else if ((TREE_CODE_CLASS (code) == tcc_comparison
573 22205212 : || commutative_tree_code (code))
574 32807569 : && swap)
575 : return op1_op0_map;
576 23674957 : else if (code == VIEW_CONVERT_EXPR)
577 : return op00_map;
578 23665780 : else if (gather_scatter_p)
579 46084 : return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
580 46084 : ? off_op0_map : off_map);
581 : }
582 1557433 : else if (auto call = dyn_cast<const gcall *> (stmt))
583 : {
584 176916 : if (gimple_call_internal_p (call))
585 93596 : switch (gimple_call_internal_fn (call))
586 : {
587 16137 : case IFN_MASK_LOAD:
588 16137 : return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
589 :
590 0 : case IFN_GATHER_LOAD:
591 0 : return arg2_map;
592 :
593 0 : case IFN_MASK_GATHER_LOAD:
594 0 : case IFN_MASK_LEN_GATHER_LOAD:
595 0 : return arg2_arg5_arg6_map;
596 :
597 0 : case IFN_SCATTER_STORE:
598 0 : return arg2_arg4_map;
599 :
600 0 : case IFN_MASK_SCATTER_STORE:
601 0 : case IFN_MASK_LEN_SCATTER_STORE:
602 0 : return arg2_arg4_arg5_map;
603 :
604 9125 : case IFN_MASK_STORE:
605 9125 : return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
606 :
607 996 : case IFN_MASK_CALL:
608 996 : {
609 996 : unsigned nargs = gimple_call_num_args (call);
610 996 : if (nargs >= 2 && nargs <= 7)
611 996 : return mask_call_maps[nargs-2];
612 : else
613 : return nullptr;
614 : }
615 :
616 278 : case IFN_CLZ:
617 278 : case IFN_CTZ:
618 278 : return arg0_map;
619 :
620 7302 : case IFN_GOMP_SIMD_LANE:
621 7302 : return no_arg_map;
622 :
623 : default:
624 : break;
625 : }
626 : }
627 : return nullptr;
628 : }
629 :
630 : static const int *
631 25254088 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
632 : {
633 0 : return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
634 0 : swap);
635 : }
636 :
637 : /* Return the SLP node child index for operand OP of STMT. */
638 :
639 : int
640 1398261 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
641 : {
642 1398261 : const int *opmap = vect_get_operand_map (stmt);
643 1398261 : if (!opmap)
644 : return op;
645 21835 : for (int i = 1; i < 1 + opmap[0]; ++i)
646 21835 : if (opmap[i] == op)
647 12192 : return i - 1;
648 0 : gcc_unreachable ();
649 : }
650 :
651 : /* Helper class for mapping of GIMPLE operands to SLP children. */
652 : /* ??? Add vect_slp_child_index_for_operand here and amend opmaps
653 : with the full reverse mapping and indicating the position of the
654 : first commutative operand index, eliding the swap_p argument from
655 : vect_get_operand_map. Adjust all consumers. */
656 :
657 : struct slp_oprnds {
658 : slp_oprnds (stmt_vec_info);
659 : tree get_op_for_slp_child (stmt_vec_info, unsigned);
660 : const int *opmap;
661 : const unsigned int num_slp_children;
662 : };
663 :
664 4617072 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
665 4617072 : : opmap (vect_get_operand_map (stmt_info)),
666 4617072 : num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
667 : {
668 4617072 : }
669 :
670 : /* For SLP child number N get the corresponding tree operand from GIMPLE
671 : statement described by STMT_INFO. */
672 :
673 : tree
674 5153940 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
675 : {
676 5153940 : gcc_assert (n < num_slp_children);
677 5153940 : int opno = opmap ? opmap[n + 1] : (int) n;
678 5153940 : if (opno == GATHER_SCATTER_OFFSET)
679 0 : gcc_unreachable (); // TODO
680 5153940 : else if (opno < 0)
681 2530 : return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
682 : else
683 5151410 : return gimple_arg (stmt_info->stmt, opno);
684 : }
685 :
686 : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
687 : they are of a valid type and that they match the defs of the first stmt of
688 : the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
689 : by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP
690 : indicates swap is required for cond_expr stmts. Specifically, SWAP
691 : is 1 if STMT is cond and operands of comparison need to be swapped;
692 : SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
693 :
694 : If there was a fatal error return -1; if the error could be corrected by
695 : swapping operands of father node of this one, return 1; if everything is
696 : ok return 0. */
697 : static int
698 13053497 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
699 : bool *skip_args,
700 : vec<stmt_vec_info> stmts, unsigned stmt_num,
701 : vec<slp_oprnd_info> *oprnds_info)
702 : {
703 13053497 : stmt_vec_info stmt_info = stmts[stmt_num];
704 13053497 : tree oprnd;
705 13053497 : unsigned int i, number_of_oprnds;
706 13053497 : enum vect_def_type dt = vect_uninitialized_def;
707 13053497 : slp_oprnd_info oprnd_info;
708 13053497 : gather_scatter_info gs_info;
709 13053497 : unsigned int gs_op = -1u;
710 13053497 : unsigned int commutative_op = -1U;
711 13053497 : bool first = stmt_num == 0;
712 :
713 13053497 : if (!stmt_info)
714 : {
715 0 : for (auto oi : *oprnds_info)
716 : {
717 0 : oi->def_stmts.quick_push (NULL);
718 0 : oi->ops.quick_push (NULL_TREE);
719 : }
720 : return 0;
721 : }
722 :
723 13053497 : if (!is_a<gcall *> (stmt_info->stmt)
724 : && !is_a<gassign *> (stmt_info->stmt)
725 : && !is_a<gphi *> (stmt_info->stmt))
726 : return -1;
727 :
728 13053497 : number_of_oprnds = gimple_num_args (stmt_info->stmt);
729 13053497 : const int *map = vect_get_operand_map (stmt_info, swap);
730 13053497 : if (map)
731 80279 : number_of_oprnds = *map++;
732 13053497 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
733 : {
734 54578 : if (gimple_call_internal_p (stmt))
735 : {
736 33290 : internal_fn ifn = gimple_call_internal_fn (stmt);
737 33290 : commutative_op = first_commutative_argument (ifn);
738 33290 : if (internal_gather_scatter_fn_p (ifn))
739 : {
740 0 : vect_describe_gather_scatter_call
741 0 : (stmt_info,
742 0 : first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
743 0 : if (first)
744 0 : (*oprnds_info)[0]->first_gs_p = true;
745 : gs_op = 0;
746 : }
747 : }
748 : }
749 12998919 : else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
750 : {
751 15147316 : if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
752 8495663 : commutative_op = 0;
753 : }
754 :
755 13053497 : bool swapped = (swap != 0);
756 13053497 : bool backedge = false;
757 13053497 : enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
758 36072729 : for (i = 0; i < number_of_oprnds; i++)
759 : {
760 23020537 : oprnd_info = (*oprnds_info)[i];
761 23020537 : int opno = map ? map[i] : int (i);
762 23020537 : if (opno == GATHER_SCATTER_OFFSET)
763 : {
764 24109 : gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
765 24109 : if (!is_a <loop_vec_info> (vinfo)
766 24109 : || !vect_check_gather_scatter (stmt_info, vectype,
767 : as_a <loop_vec_info> (vinfo),
768 : first ? &oprnd_info->first_gs_info
769 : : &gs_info))
770 1305 : return -1;
771 :
772 24109 : if (first)
773 : {
774 23835 : oprnd_info->first_gs_p = true;
775 23835 : oprnd = oprnd_info->first_gs_info.offset;
776 : }
777 : else
778 : {
779 274 : gs_op = i;
780 274 : oprnd = gs_info.offset;
781 : }
782 : }
783 22996428 : else if (opno < 0)
784 3083 : oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
785 : else
786 : {
787 22993345 : oprnd = gimple_arg (stmt_info->stmt, opno);
788 22993345 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
789 : {
790 1265347 : edge e = gimple_phi_arg_edge (stmt, opno);
791 2530694 : backedge = (is_a <bb_vec_info> (vinfo)
792 1960712 : ? e->flags & EDGE_DFS_BACK
793 695365 : : dominated_by_p (CDI_DOMINATORS, e->src,
794 695365 : gimple_bb (stmt_info->stmt)));
795 : }
796 : }
797 :
798 23020537 : stmt_vec_info def_stmt_info;
799 23020537 : if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
800 : {
801 1029 : if (dump_enabled_p ())
802 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
803 : "Build SLP failed: can't analyze def for %T\n",
804 : oprnd);
805 :
806 : return -1;
807 : }
808 :
809 23019508 : if (skip_args[i])
810 : {
811 558374 : oprnd_info->def_stmts.quick_push (NULL);
812 558374 : oprnd_info->ops.quick_push (NULL_TREE);
813 558374 : oprnd_info->first_dt = vect_uninitialized_def;
814 558374 : continue;
815 : }
816 :
817 22461134 : oprnd_info->def_stmts.quick_push (def_stmt_info);
818 22461134 : oprnd_info->ops.quick_push (oprnd);
819 :
820 22461134 : if (def_stmt_info
821 22461134 : && is_pattern_stmt_p (def_stmt_info))
822 : {
823 441851 : if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
824 : != def_stmt_info)
825 316667 : oprnd_info->any_pattern = true;
826 : else
827 : /* If we promote this to external use the original stmt def. */
828 125184 : oprnd_info->ops.last ()
829 250368 : = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
830 : }
831 :
832 : /* If there's a extern def on a backedge make sure we can
833 : code-generate at the region start.
834 : ??? This is another case that could be fixed by adjusting
835 : how we split the function but at the moment we'd have conflicting
836 : goals there. */
837 22461134 : if (backedge
838 173520 : && dts[i] == vect_external_def
839 297 : && is_a <bb_vec_info> (vinfo)
840 297 : && TREE_CODE (oprnd) == SSA_NAME
841 276 : && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
842 22461410 : && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
843 276 : gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
844 : {
845 276 : if (dump_enabled_p ())
846 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
847 : "Build SLP failed: extern def %T only defined "
848 : "on backedge\n", oprnd);
849 : return -1;
850 : }
851 :
852 22460858 : if (first)
853 : {
854 5127016 : tree type = TREE_TYPE (oprnd);
855 5127016 : dt = dts[i];
856 :
857 : /* For the swapping logic below force vect_reduction_def
858 : for the reduction op in a SLP reduction group. */
859 5127016 : if (!STMT_VINFO_DATA_REF (stmt_info)
860 3931784 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
861 5242 : && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
862 5129597 : && def_stmt_info)
863 2581 : dts[i] = dt = vect_reduction_def;
864 :
865 : /* Check the types of the definition. */
866 5127016 : switch (dt)
867 : {
868 5127016 : case vect_external_def:
869 5127016 : case vect_constant_def:
870 5127016 : case vect_internal_def:
871 5127016 : case vect_reduction_def:
872 5127016 : case vect_double_reduction_def:
873 5127016 : case vect_induction_def:
874 5127016 : case vect_nested_cycle:
875 5127016 : case vect_first_order_recurrence:
876 5127016 : break;
877 :
878 0 : default:
879 : /* FORNOW: Not supported. */
880 0 : if (dump_enabled_p ())
881 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
882 : "Build SLP failed: illegal type of def %T\n",
883 : oprnd);
884 : return -1;
885 : }
886 :
887 5127016 : oprnd_info->first_dt = dt;
888 5127016 : oprnd_info->first_op_type = type;
889 : }
890 : }
891 13052192 : if (first)
892 : return 0;
893 :
894 : /* Now match the operand definition types to that of the first stmt. */
895 26544557 : for (i = 0; i < number_of_oprnds;)
896 : {
897 17327268 : if (skip_args[i])
898 : {
899 44458 : ++i;
900 44458 : continue;
901 : }
902 :
903 17282810 : oprnd_info = (*oprnds_info)[i];
904 17282810 : dt = dts[i];
905 17282810 : stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
906 17282810 : oprnd = oprnd_info->ops[stmt_num];
907 17282810 : tree type = TREE_TYPE (oprnd);
908 :
909 17282810 : if (!types_compatible_p (oprnd_info->first_op_type, type))
910 : {
911 109757 : if (dump_enabled_p ())
912 93 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
913 : "Build SLP failed: different operand types\n");
914 : return 1;
915 : }
916 :
917 17173053 : if ((gs_op == i) != oprnd_info->first_gs_p)
918 : {
919 0 : if (dump_enabled_p ())
920 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
921 : "Build SLP failed: mixed gather and non-gather\n");
922 : return 1;
923 : }
924 17173053 : else if (gs_op == i)
925 : {
926 242 : if (!operand_equal_p (oprnd_info->first_gs_info.base,
927 242 : gs_info.base))
928 : {
929 16 : if (dump_enabled_p ())
930 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
931 : "Build SLP failed: different gather base\n");
932 : return 1;
933 : }
934 226 : if (oprnd_info->first_gs_info.scale != gs_info.scale)
935 : {
936 8 : if (dump_enabled_p ())
937 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
938 : "Build SLP failed: different gather scale\n");
939 : return 1;
940 : }
941 : }
942 :
943 : /* Not first stmt of the group, check that the def-stmt/s match
944 : the def-stmt/s of the first stmt. Allow different definition
945 : types for reduction chains: the first stmt must be a
946 : vect_reduction_def (a phi node), and the rest
947 : end in the reduction chain. */
948 17173029 : if ((!vect_def_types_match (oprnd_info->first_dt, dt)
949 307496 : && !(oprnd_info->first_dt == vect_reduction_def
950 4806 : && !STMT_VINFO_DATA_REF (stmt_info)
951 4806 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
952 4780 : && def_stmt_info
953 4778 : && !STMT_VINFO_DATA_REF (def_stmt_info)
954 4778 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
955 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
956 16870311 : || (!STMT_VINFO_DATA_REF (stmt_info)
957 15535169 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
958 9961 : && ((!def_stmt_info
959 9765 : || STMT_VINFO_DATA_REF (def_stmt_info)
960 17987 : || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
961 : != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
962 9961 : != (oprnd_info->first_dt != vect_reduction_def))))
963 : {
964 : /* Try swapping operands if we got a mismatch. For BB
965 : vectorization only in case it will clearly improve things. */
966 305248 : if (i == commutative_op && !swapped
967 302718 : && (!is_a <bb_vec_info> (vinfo)
968 5038 : || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
969 5038 : dts[i+1])
970 1294 : && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
971 : || vect_def_types_match
972 225 : ((*oprnds_info)[i+1]->first_dt, dts[i])))))
973 : {
974 2530 : if (dump_enabled_p ())
975 153 : dump_printf_loc (MSG_NOTE, vect_location,
976 : "trying swapped operands\n");
977 2530 : std::swap (dts[i], dts[i+1]);
978 2530 : std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
979 2530 : (*oprnds_info)[i+1]->def_stmts[stmt_num]);
980 2530 : std::swap ((*oprnds_info)[i]->ops[stmt_num],
981 2530 : (*oprnds_info)[i+1]->ops[stmt_num]);
982 : /* After swapping some operands we lost track whether an
983 : operand has any pattern defs so be conservative here. */
984 2530 : if ((*oprnds_info)[i]->any_pattern
985 2530 : || (*oprnds_info)[i+1]->any_pattern)
986 36 : (*oprnds_info)[i]->any_pattern
987 18 : = (*oprnds_info)[i+1]->any_pattern = true;
988 2530 : swapped = true;
989 2530 : continue;
990 : }
991 :
992 300188 : if (is_a <bb_vec_info> (vinfo)
993 284640 : && !oprnd_info->any_pattern
994 584572 : && number_of_oprnds > 1)
995 : {
996 : /* Now for commutative ops we should see whether we can
997 : make the other operand matching. */
998 106614 : if (dump_enabled_p ())
999 261 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1000 : "treating operand as external\n");
1001 : oprnd_info->first_dt = dt = vect_external_def;
1002 : }
1003 : else
1004 : {
1005 193574 : if (dump_enabled_p ())
1006 411 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1007 : "Build SLP failed: different types\n");
1008 : return 1;
1009 : }
1010 : }
1011 :
1012 : /* Make sure to demote the overall operand to external. */
1013 16870311 : if (dt == vect_external_def)
1014 352042 : oprnd_info->first_dt = vect_external_def;
1015 : /* For a SLP reduction chain we want to duplicate the reduction to
1016 : each of the chain members. That gets us a sane SLP graph (still
1017 : the stmts are not 100% correct wrt the initial values). */
1018 16624883 : else if ((dt == vect_internal_def
1019 16624883 : || dt == vect_reduction_def)
1020 15658083 : && oprnd_info->first_dt == vect_reduction_def
1021 101252 : && !STMT_VINFO_DATA_REF (stmt_info)
1022 101252 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
1023 4778 : && !STMT_VINFO_DATA_REF (def_stmt_info)
1024 16629661 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
1025 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
1026 : {
1027 4778 : oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
1028 4778 : oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
1029 : }
1030 :
1031 16976925 : ++i;
1032 : }
1033 :
1034 : /* Swap operands. */
1035 9217289 : if (swapped)
1036 : {
1037 43039 : if (dump_enabled_p ())
1038 457 : dump_printf_loc (MSG_NOTE, vect_location,
1039 : "swapped operands to match def types in %G",
1040 : stmt_info->stmt);
1041 : }
1042 :
1043 : return 0;
1044 : }
1045 :
1046 : /* Return true if call statements CALL1 and CALL2 are similar enough
1047 : to be combined into the same SLP group. */
1048 :
1049 : bool
1050 70780 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
1051 : {
1052 70780 : unsigned int nargs = gimple_call_num_args (call1);
1053 70780 : if (nargs != gimple_call_num_args (call2))
1054 : return false;
1055 :
1056 64419 : auto cfn1 = gimple_call_combined_fn (call1);
1057 64419 : auto cfn2 = gimple_call_combined_fn (call2);
1058 64419 : if (cfn1 != cfn2
1059 2 : && (!allow_two_operators
1060 2 : || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
1061 2 : && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
1062 : return false;
1063 :
1064 64419 : if (gimple_call_internal_p (call1))
1065 : {
1066 7241 : if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
1067 7241 : TREE_TYPE (gimple_call_lhs (call2))))
1068 : return false;
1069 14878 : for (unsigned int i = 0; i < nargs; ++i)
1070 7637 : if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
1071 7637 : TREE_TYPE (gimple_call_arg (call2, i))))
1072 : return false;
1073 : }
1074 : else
1075 : {
1076 57178 : if (!operand_equal_p (gimple_call_fn (call1),
1077 57178 : gimple_call_fn (call2), 0))
1078 : return false;
1079 :
1080 42279 : if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
1081 : return false;
1082 : }
1083 :
1084 : /* Check that any unvectorized arguments are equal. */
1085 21334 : if (const int *map = vect_get_operand_map (call1, false, false))
1086 : {
1087 15 : unsigned int nkept = *map++;
1088 15 : unsigned int mapi = 0;
1089 57 : for (unsigned int i = 0; i < nargs; ++i)
1090 42 : if (mapi < nkept && map[mapi] == int (i))
1091 27 : mapi += 1;
1092 15 : else if (!operand_equal_p (gimple_call_arg (call1, i),
1093 15 : gimple_call_arg (call2, i)))
1094 : return false;
1095 : }
1096 :
1097 : return true;
1098 : }
1099 :
1100 : /* A subroutine of vect_build_slp_tree for checking VECTYPE, which is the
1101 : caller's attempt to find the vector type in STMT_INFO with the narrowest
1102 : element type. Return true if VECTYPE is nonnull and if it is valid
1103 : for STMT_INFO. GROUP_SIZE is as for vect_build_slp_tree. */
1104 :
1105 : static bool
1106 5902381 : vect_record_vectype (vec_info *vinfo, stmt_vec_info stmt_info,
1107 : unsigned int group_size, tree vectype)
1108 : {
1109 5902381 : if (!vectype)
1110 : {
1111 4241 : if (dump_enabled_p ())
1112 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1113 : "Build SLP failed: unsupported data-type in %G\n",
1114 : stmt_info->stmt);
1115 : /* Fatal mismatch. */
1116 : return false;
1117 : }
1118 :
1119 : /* If populating the vector type requires unrolling then fail
1120 : for basic-block vectorization. */
1121 5898140 : if (is_a <bb_vec_info> (vinfo)
1122 5898140 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
1123 : {
1124 216055 : if (dump_enabled_p ())
1125 155 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1126 : "Build SLP failed: unrolling required "
1127 : "in basic block SLP\n");
1128 : /* Fatal mismatch. */
1129 : return false;
1130 : }
1131 :
1132 : return true;
1133 : }
1134 :
1135 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1136 : permutation or are of unsupported types of operation. Return
1137 : true if they are, otherwise return false and indicate in *MATCHES
1138 : which stmts are not isomorphic to the first one. If MATCHES[0]
1139 : is false then this indicates the comparison could not be
1140 : carried out or the stmts will never be vectorized by SLP.
1141 :
1142 : Note COND_EXPR is possibly isomorphic to another one after swapping its
1143 : operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
1144 : the first stmt by swapping the two operands of comparison; set SWAP[i]
1145 : to 2 if stmt I is isormorphic to the first stmt by inverting the code
1146 : of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
1147 : to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
1148 :
1149 : static bool
1150 6181011 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
1151 : vec<stmt_vec_info> stmts, bool *matches,
1152 : bool *two_operators, tree *node_vectype)
1153 : {
1154 6181011 : unsigned int group_size = stmts.length ();
1155 6181011 : unsigned int i;
1156 6181011 : stmt_vec_info first_stmt_info = stmts[0];
1157 6181011 : code_helper first_stmt_code = ERROR_MARK;
1158 6181011 : code_helper alt_stmt_code = ERROR_MARK;
1159 6181011 : code_helper first_cond_code = ERROR_MARK;
1160 6181011 : bool need_same_oprnds = false;
1161 6181011 : tree first_lhs = NULL_TREE;
1162 6181011 : tree first_op1 = NULL_TREE;
1163 6181011 : stmt_vec_info first_load = NULL, prev_first_load = NULL;
1164 6181011 : bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
1165 6181011 : bool first_stmt_phi_p = false;
1166 6181011 : int first_reduc_idx = -1;
1167 6181011 : bool maybe_soft_fail = false;
1168 6181011 : tree soft_fail_nunits_vectype = NULL_TREE;
1169 :
1170 6181011 : tree vectype, nunits_vectype;
1171 6181011 : if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
1172 : &nunits_vectype, group_size))
1173 : {
1174 : /* Fatal mismatch. */
1175 237129 : matches[0] = false;
1176 237129 : return false;
1177 : }
1178 5943882 : if (is_a <bb_vec_info> (vinfo)
1179 5943882 : && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
1180 : {
1181 358406 : if (dump_enabled_p ())
1182 301 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1183 : "Build SLP failed: not using single lane "
1184 : "vector type %T\n", vectype);
1185 358406 : matches[0] = false;
1186 358406 : return false;
1187 : }
1188 : /* Record nunits required but continue analysis, producing matches[]
1189 : as if nunits was not an issue. This allows splitting of groups
1190 : to happen. */
1191 5585476 : if (nunits_vectype
1192 5585476 : && !vect_record_vectype (vinfo, first_stmt_info, group_size,
1193 : nunits_vectype))
1194 : {
1195 216055 : gcc_assert (is_a <bb_vec_info> (vinfo));
1196 216055 : maybe_soft_fail = true;
1197 216055 : soft_fail_nunits_vectype = nunits_vectype;
1198 : }
1199 :
1200 5585476 : gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
1201 5585476 : *node_vectype = vectype;
1202 :
1203 5585476 : basic_block common_bb = gimple_bb (first_stmt_info->stmt);
1204 5585476 : gimple *trapping_stmt = NULL;
1205 :
1206 : /* For every stmt in NODE find its def stmt/s. */
1207 5585476 : stmt_vec_info stmt_info;
1208 23453710 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
1209 : {
1210 18046221 : bool ldst_p = false;
1211 18046221 : bool ldst_masklen_p = false;
1212 18046221 : bool phi_p = false;
1213 18046221 : code_helper rhs_code = ERROR_MARK;
1214 :
1215 18046221 : swap[i] = 0;
1216 18046221 : matches[i] = false;
1217 18046221 : if (!stmt_info)
1218 : {
1219 40999 : matches[i] = true;
1220 17868234 : continue;
1221 : }
1222 :
1223 18005222 : gimple *stmt = stmt_info->stmt;
1224 18005222 : if (dump_enabled_p ())
1225 227349 : dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
1226 :
1227 : /* Fail to vectorize statements marked as unvectorizable, throw
1228 : or are volatile. */
1229 18005222 : if (!STMT_VINFO_VECTORIZABLE (stmt_info)
1230 17753106 : || stmt_can_throw_internal (cfun, stmt)
1231 34893038 : || gimple_has_volatile_ops (stmt))
1232 : {
1233 259088 : if (dump_enabled_p ())
1234 248 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1235 : "Build SLP failed: unvectorizable statement %G",
1236 : stmt);
1237 : /* ??? For BB vectorization we want to commutate operands in a way
1238 : to shuffle all unvectorizable defs into one operand and have
1239 : the other still vectorized. The following doesn't reliably
1240 : work for this though but it's the easiest we can do here. */
1241 259088 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1242 106481 : continue;
1243 : /* Fatal mismatch. */
1244 152607 : matches[0] = false;
1245 152607 : return false;
1246 : }
1247 :
1248 17746134 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
1249 17746134 : tree lhs = gimple_get_lhs (stmt);
1250 17746134 : if (lhs == NULL_TREE && !call_stmt)
1251 : {
1252 34 : if (dump_enabled_p ())
1253 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1254 : "Build SLP failed: not GIMPLE_ASSIGN nor "
1255 : "GIMPLE_CALL %G", stmt);
1256 34 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1257 34 : continue;
1258 : /* Fatal mismatch. */
1259 0 : matches[0] = false;
1260 0 : return false;
1261 : }
1262 :
1263 17746100 : if (call_stmt)
1264 : {
1265 174643 : combined_fn cfn = gimple_call_combined_fn (call_stmt);
1266 174643 : if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
1267 59980 : rhs_code = cfn;
1268 : else
1269 : rhs_code = CALL_EXPR;
1270 :
1271 174643 : if (cfn == CFN_GATHER_LOAD
1272 174643 : || cfn == CFN_SCATTER_STORE)
1273 : ldst_p = true;
1274 : else if (cfn == CFN_MASK_LOAD
1275 : || cfn == CFN_MASK_GATHER_LOAD
1276 : || cfn == CFN_MASK_LEN_GATHER_LOAD
1277 : || cfn == CFN_MASK_SCATTER_STORE
1278 : || cfn == CFN_MASK_LEN_SCATTER_STORE)
1279 : {
1280 : ldst_p = true;
1281 : ldst_masklen_p = true;
1282 : }
1283 : else if (cfn == CFN_MASK_STORE)
1284 : {
1285 : ldst_p = true;
1286 : ldst_masklen_p = true;
1287 : rhs_code = CFN_MASK_STORE;
1288 : }
1289 : else if (cfn == CFN_GOMP_SIMD_LANE)
1290 : ;
1291 162763 : else if ((cfn != CFN_LAST
1292 : && cfn != CFN_MASK_CALL
1293 48100 : && internal_fn_p (cfn)
1294 37571 : && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
1295 162680 : || gimple_call_tail_p (call_stmt)
1296 162680 : || gimple_call_noreturn_p (call_stmt)
1297 325443 : || gimple_call_chain (call_stmt))
1298 : {
1299 443 : if (dump_enabled_p ())
1300 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1301 : "Build SLP failed: unsupported call type %G",
1302 : (gimple *) call_stmt);
1303 443 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1304 72 : continue;
1305 : /* Fatal mismatch. */
1306 371 : matches[0] = false;
1307 371 : return false;
1308 : }
1309 : }
1310 17571457 : else if (gimple_code (stmt) == GIMPLE_PHI)
1311 : {
1312 17745657 : rhs_code = ERROR_MARK;
1313 17745657 : phi_p = true;
1314 : }
1315 : else
1316 : {
1317 16706167 : rhs_code = gimple_assign_rhs_code (stmt);
1318 16706167 : ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
1319 : }
1320 :
1321 : /* Check the operation. */
1322 17745657 : if (i == 0)
1323 : {
1324 5432498 : first_lhs = lhs;
1325 5432498 : first_stmt_code = rhs_code;
1326 5432498 : first_stmt_ldst_p = ldst_p;
1327 5432498 : first_stmt_ldst_masklen_p = ldst_masklen_p;
1328 5432498 : first_stmt_phi_p = phi_p;
1329 5432498 : first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
1330 :
1331 : /* Shift arguments should be equal in all the packed stmts for a
1332 : vector shift with scalar shift operand. */
1333 5432498 : if (rhs_code == LSHIFT_EXPR
1334 5360918 : || rhs_code == RSHIFT_EXPR
1335 5289814 : || rhs_code == LROTATE_EXPR
1336 10722198 : || rhs_code == RROTATE_EXPR)
1337 : {
1338 : /* First see if we have a vector/vector shift. */
1339 143276 : if (!directly_supported_p (rhs_code, vectype, optab_vector))
1340 : {
1341 : /* No vector/vector shift, arrange for a vector/scalar
1342 : SLP layout. */
1343 131085 : need_same_oprnds = true;
1344 131085 : first_op1 = gimple_assign_rhs2 (stmt);
1345 : }
1346 : }
1347 5289222 : else if (rhs_code == WIDEN_LSHIFT_EXPR)
1348 : {
1349 0 : need_same_oprnds = true;
1350 0 : first_op1 = gimple_assign_rhs2 (stmt);
1351 : }
1352 5289222 : else if (!ldst_p
1353 5289222 : && rhs_code == BIT_FIELD_REF)
1354 : {
1355 8914 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
1356 8914 : if (!is_a <bb_vec_info> (vinfo)
1357 8788 : || TREE_CODE (vec) != SSA_NAME
1358 : /* When the element types are not compatible we pun the
1359 : source to the target vectype which requires equal size. */
1360 17690 : || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
1361 7979 : || !types_compatible_p (TREE_TYPE (vectype),
1362 7979 : TREE_TYPE (TREE_TYPE (vec))))
1363 1116 : && !operand_equal_p (TYPE_SIZE (vectype),
1364 1116 : TYPE_SIZE (TREE_TYPE (vec)))))
1365 : {
1366 853 : if (dump_enabled_p ())
1367 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1368 : "Build SLP failed: "
1369 : "BIT_FIELD_REF not supported\n");
1370 : /* Fatal mismatch. */
1371 853 : matches[0] = false;
1372 853 : return false;
1373 : }
1374 : }
1375 5280308 : else if (rhs_code == CFN_DIV_POW2)
1376 : {
1377 0 : need_same_oprnds = true;
1378 0 : first_op1 = gimple_call_arg (call_stmt, 1);
1379 : }
1380 5280308 : else if (rhs_code == CFN_GOMP_SIMD_LANE)
1381 : {
1382 3651 : need_same_oprnds = true;
1383 3651 : first_op1 = gimple_call_arg (call_stmt, 1);
1384 : }
1385 : }
1386 : else
1387 : {
1388 12313159 : int comm_arg;
1389 12313537 : if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1390 : /* For SLP reduction groups the index isn't necessarily
1391 : uniform but only that of the first stmt matters. */
1392 2382 : && !(first_reduc_idx != -1
1393 2382 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1394 2382 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
1395 12313159 : && !(first_reduc_idx != -1
1396 1085 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1397 1085 : && (comm_arg = first_commutative_argument
1398 1085 : (rhs_code, TREE_TYPE (lhs))) >= 0
1399 : && (first_reduc_idx
1400 851 : == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
1401 : {
1402 378 : if (dump_enabled_p ())
1403 : {
1404 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1405 : "Build SLP failed: different reduc_idx "
1406 : "%d instead of %d in %G",
1407 : STMT_VINFO_REDUC_IDX (stmt_info),
1408 : first_reduc_idx, stmt);
1409 : }
1410 : /* Mismatch. */
1411 378 : continue;
1412 : }
1413 12312781 : if (!ldst_p
1414 9670548 : && first_stmt_code != rhs_code
1415 13826456 : && alt_stmt_code == ERROR_MARK)
1416 : alt_stmt_code = rhs_code;
1417 13804046 : if ((!ldst_p
1418 9670548 : && first_stmt_code != rhs_code
1419 1513675 : && (first_stmt_code != IMAGPART_EXPR
1420 218 : || rhs_code != REALPART_EXPR)
1421 1513654 : && (first_stmt_code != REALPART_EXPR
1422 734 : || rhs_code != IMAGPART_EXPR)
1423 : /* Handle mismatches in plus/minus by computing both
1424 : and merging the results. */
1425 1513651 : && !((((first_stmt_code == PLUS_EXPR
1426 1404666 : || first_stmt_code == MINUS_EXPR)
1427 139324 : && (alt_stmt_code == PLUS_EXPR
1428 129459 : || alt_stmt_code == MINUS_EXPR))
1429 1482255 : || ((first_stmt_code == CFN_FMA
1430 1482253 : || first_stmt_code == CFN_FMS)
1431 2 : && (alt_stmt_code == CFN_FMA
1432 2 : || alt_stmt_code == CFN_FMS)))
1433 31398 : && rhs_code == alt_stmt_code)
1434 1526039 : && !(first_stmt_code.is_tree_code ()
1435 1384038 : && rhs_code.is_tree_code ()
1436 1267731 : && (TREE_CODE_CLASS (tree_code (first_stmt_code))
1437 : == tcc_comparison)
1438 140841 : && (swap_tree_comparison (tree_code (first_stmt_code))
1439 140841 : == tree_code (rhs_code))
1440 : && (first_reduc_idx == -1
1441 0 : || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
1442 : || (ldst_p
1443 5284466 : && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1444 2642233 : != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
1445 : || (ldst_p
1446 2590392 : && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1447 2590392 : != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
1448 10821688 : || first_stmt_ldst_p != ldst_p
1449 10821524 : || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
1450 23134297 : || first_stmt_phi_p != phi_p)
1451 : {
1452 1491265 : if (dump_enabled_p ())
1453 : {
1454 3375 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1455 : "Build SLP failed: different operation "
1456 : "in stmt %G", stmt);
1457 3375 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1458 : "original stmt %G", first_stmt_info->stmt);
1459 : }
1460 : /* Mismatch. */
1461 1491265 : continue;
1462 : }
1463 :
1464 10838351 : if (!ldst_p
1465 8231275 : && first_stmt_code == BIT_FIELD_REF
1466 10848129 : && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
1467 26613 : != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
1468 : {
1469 16835 : if (dump_enabled_p ())
1470 76 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1471 : "Build SLP failed: different BIT_FIELD_REF "
1472 : "arguments in %G", stmt);
1473 : /* Mismatch. */
1474 16835 : continue;
1475 : }
1476 :
1477 10804681 : if (call_stmt
1478 71493 : && first_stmt_code != CFN_MASK_LOAD
1479 10875772 : && first_stmt_code != CFN_MASK_STORE)
1480 : {
1481 70780 : if (!is_a <gcall *> (stmts[0]->stmt)
1482 70780 : || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
1483 : call_stmt, true))
1484 : {
1485 49446 : if (dump_enabled_p ())
1486 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1487 : "Build SLP failed: different calls in %G",
1488 : stmt);
1489 : /* Mismatch. */
1490 49446 : continue;
1491 : }
1492 : }
1493 :
1494 10796636 : if (phi_p
1495 10755235 : && (gimple_bb (first_stmt_info->stmt)
1496 208044 : != gimple_bb (stmt_info->stmt)))
1497 : {
1498 41401 : if (dump_enabled_p ())
1499 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1500 : "Build SLP failed: different BB for PHI %G",
1501 : stmt);
1502 : /* Mismatch. */
1503 41401 : continue;
1504 : }
1505 :
1506 10713834 : if (need_same_oprnds)
1507 : {
1508 55367 : tree other_op1 = gimple_arg (stmt, 1);
1509 55367 : if (!operand_equal_p (first_op1, other_op1, 0))
1510 : {
1511 7002 : if (dump_enabled_p ())
1512 133 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1513 : "Build SLP failed: different shift "
1514 : "arguments in %G", stmt);
1515 : /* Mismatch. */
1516 7002 : continue;
1517 : }
1518 : }
1519 :
1520 10707569 : if (first_lhs
1521 10706832 : && lhs
1522 10706832 : && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
1523 : {
1524 737 : if (dump_enabled_p ())
1525 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1526 : "Build SLP failed: different vector type "
1527 : "in %G", stmt);
1528 : /* Mismatch. */
1529 737 : continue;
1530 : }
1531 : }
1532 :
1533 : /* Grouped store or load. */
1534 16137740 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1535 : {
1536 4009059 : gcc_assert (ldst_p);
1537 4009059 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
1538 : {
1539 : /* Store. */
1540 3103737 : gcc_assert (rhs_code == CFN_MASK_STORE
1541 : || REFERENCE_CLASS_P (lhs)
1542 : || DECL_P (lhs));
1543 : }
1544 : else
1545 : {
1546 : /* Load. */
1547 905322 : first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
1548 905322 : if (prev_first_load)
1549 : {
1550 : /* Check that there are no loads from different interleaving
1551 : chains in the same node. */
1552 418766 : if (prev_first_load != first_load)
1553 : {
1554 62875 : if (dump_enabled_p ())
1555 2217 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
1556 : vect_location,
1557 : "Build SLP failed: different "
1558 : "interleaving chains in one node %G",
1559 : stmt);
1560 : /* Mismatch. */
1561 62875 : continue;
1562 : }
1563 : }
1564 : else
1565 : prev_first_load = first_load;
1566 : }
1567 : }
1568 : /* Non-grouped store or load. */
1569 12128681 : else if (ldst_p)
1570 : {
1571 923103 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1572 646140 : && rhs_code != CFN_GATHER_LOAD
1573 : && rhs_code != CFN_MASK_GATHER_LOAD
1574 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1575 : && rhs_code != CFN_SCATTER_STORE
1576 : && rhs_code != CFN_MASK_SCATTER_STORE
1577 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1578 646140 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1579 : /* Not grouped loads are handled as externals for BB
1580 : vectorization. For loop vectorization we can handle
1581 : splats the same we handle single element interleaving.
1582 : Likewise we can handle a collection of invariant refs. */
1583 1549371 : && (is_a <bb_vec_info> (vinfo)
1584 626268 : || (stmt_info != first_stmt_info
1585 68115 : && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
1586 241 : && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
1587 : (first_stmt_info)))))))
1588 : {
1589 : /* Not grouped load. */
1590 67633 : if (dump_enabled_p ())
1591 145 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1592 : "Build SLP failed: not grouped load %G", stmt);
1593 :
1594 67633 : if (i != 0)
1595 67633 : continue;
1596 : /* Fatal mismatch. */
1597 0 : matches[0] = false;
1598 0 : return false;
1599 : }
1600 : }
1601 : /* Not memory operation. */
1602 : else
1603 : {
1604 11205578 : if (!phi_p
1605 10501144 : && rhs_code.is_tree_code ()
1606 10451486 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
1607 1823434 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
1608 1130279 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
1609 1060618 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
1610 89685 : && rhs_code != VIEW_CONVERT_EXPR
1611 : && rhs_code != CALL_EXPR
1612 : && rhs_code != BIT_FIELD_REF
1613 11205578 : && rhs_code != SSA_NAME)
1614 : {
1615 24156 : if (dump_enabled_p ())
1616 17 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1617 : "Build SLP failed: operation unsupported %G",
1618 : stmt);
1619 24156 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1620 0 : continue;
1621 : /* Fatal mismatch. */
1622 24156 : matches[0] = false;
1623 24156 : return false;
1624 : }
1625 :
1626 11181422 : if (rhs_code == COND_EXPR)
1627 : {
1628 66734 : tree cond_expr = gimple_assign_rhs1 (stmt);
1629 66734 : enum tree_code cond_code = TREE_CODE (cond_expr);
1630 66734 : enum tree_code swap_code = ERROR_MARK;
1631 66734 : enum tree_code invert_code = ERROR_MARK;
1632 :
1633 66734 : if (i == 0)
1634 54600 : first_cond_code = TREE_CODE (cond_expr);
1635 12134 : else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
1636 : {
1637 0 : bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
1638 0 : swap_code = swap_tree_comparison (cond_code);
1639 0 : invert_code = invert_tree_comparison (cond_code, honor_nans);
1640 : }
1641 :
1642 66734 : if (first_cond_code == cond_code)
1643 : ;
1644 : /* Isomorphic can be achieved by swapping. */
1645 0 : else if (first_cond_code == swap_code)
1646 0 : swap[i] = 1;
1647 : /* Isomorphic can be achieved by inverting. */
1648 0 : else if (first_cond_code == invert_code)
1649 0 : swap[i] = 2;
1650 : else
1651 : {
1652 0 : if (dump_enabled_p ())
1653 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1654 : "Build SLP failed: different"
1655 : " operation %G", stmt);
1656 : /* Mismatch. */
1657 0 : continue;
1658 : }
1659 : }
1660 :
1661 11181422 : if (i != 0
1662 8116221 : && first_stmt_code != rhs_code
1663 74418 : && first_stmt_code.is_tree_code ()
1664 74416 : && rhs_code.is_tree_code ()
1665 74416 : && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
1666 11224825 : && (swap_tree_comparison ((tree_code)first_stmt_code)
1667 43403 : == (tree_code)rhs_code))
1668 43403 : swap[i] = 1;
1669 :
1670 11181422 : if (i != 0
1671 8116221 : && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1672 1716 : && first_reduc_idx != -1
1673 1716 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1674 1716 : && rhs_code.is_tree_code ()
1675 1708 : && commutative_tree_code (tree_code (rhs_code))
1676 11183128 : && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
1677 1706 : swap[i] = 1;
1678 : }
1679 :
1680 : /* We need to ensure all stmts are in the same BB when one stmt could
1681 : trap. Not matching stmts are not relevant, so exclude those. */
1682 15983076 : if (!trapping_stmt && gimple_could_trap_p (stmt))
1683 : trapping_stmt = stmt;
1684 15983076 : if (common_bb != gimple_bb (stmt))
1685 62327 : common_bb = NULL;
1686 :
1687 15983076 : matches[i] = true;
1688 : }
1689 :
1690 5407489 : if (trapping_stmt && common_bb == NULL)
1691 : {
1692 824 : if (dump_enabled_p ())
1693 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1694 : "Build SLP failed: not all stmts in same BB but "
1695 : "possibly trapping operation in %G", trapping_stmt);
1696 : /* Fatal mismatch. */
1697 824 : matches[0] = false;
1698 824 : return false;
1699 : }
1700 :
1701 21381919 : for (i = 0; i < group_size; ++i)
1702 16756435 : if (!matches[i])
1703 : return false;
1704 :
1705 : /* If we allowed a two-operation SLP node verify the target can cope
1706 : with the permute we are going to use. */
1707 4625484 : if (alt_stmt_code != ERROR_MARK
1708 4625484 : && (!alt_stmt_code.is_tree_code ()
1709 57597 : || (TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_reference
1710 57597 : && TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_comparison)))
1711 : {
1712 16401 : *two_operators = true;
1713 : }
1714 :
1715 4625484 : if (maybe_soft_fail)
1716 : {
1717 162841 : unsigned HOST_WIDE_INT const_nunits;
1718 162841 : if (!TYPE_VECTOR_SUBPARTS
1719 162841 : (soft_fail_nunits_vectype).is_constant (&const_nunits)
1720 162841 : || const_nunits > group_size)
1721 0 : matches[0] = false;
1722 : else
1723 : {
1724 : /* With constant vector elements simulate a mismatch at the
1725 : point we need to split. */
1726 162841 : unsigned tail = group_size & (const_nunits - 1);
1727 162841 : memset (&matches[group_size - tail], 0, sizeof (bool) * tail);
1728 : }
1729 : return false;
1730 : }
1731 :
1732 : return true;
1733 : }
1734 :
1735 : /* Traits for the hash_set to record failed SLP builds for a stmt set.
1736 : Note we never remove apart from at destruction time so we do not
1737 : need a special value for deleted that differs from empty. */
1738 : struct bst_traits
1739 : {
1740 : typedef vec <stmt_vec_info> value_type;
1741 : typedef vec <stmt_vec_info> compare_type;
1742 : static inline hashval_t hash (value_type);
1743 : static inline bool equal (value_type existing, value_type candidate);
1744 510115586 : static inline bool is_empty (value_type x) { return !x.exists (); }
1745 114579579 : static inline bool is_deleted (value_type x) { return !x.exists (); }
1746 : static const bool empty_zero_p = true;
1747 0 : static inline void mark_empty (value_type &x) { x.release (); }
1748 : static inline void mark_deleted (value_type &x) { x.release (); }
1749 9823221 : static inline void remove (value_type &x) { x.release (); }
1750 : };
1751 : inline hashval_t
1752 99722115 : bst_traits::hash (value_type x)
1753 : {
1754 99722115 : inchash::hash h;
1755 442660500 : for (unsigned i = 0; i < x.length (); ++i)
1756 342938385 : h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
1757 99722115 : return h.end ();
1758 : }
1759 : inline bool
1760 87360022 : bst_traits::equal (value_type existing, value_type candidate)
1761 : {
1762 262080066 : if (existing.length () != candidate.length ())
1763 : return false;
1764 87657007 : for (unsigned i = 0; i < existing.length (); ++i)
1765 83098272 : if (existing[i] != candidate[i])
1766 : return false;
1767 : return true;
1768 : }
1769 :
1770 : typedef hash_map <vec <stmt_vec_info>, slp_tree,
1771 : simple_hashmap_traits <bst_traits, slp_tree> >
1772 : scalar_stmts_to_slp_tree_map_t;
1773 :
1774 : /* Release BST_MAP. */
1775 :
1776 : static void
1777 1901275 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
1778 : {
1779 : /* The map keeps a reference on SLP nodes built, release that. */
1780 11724496 : for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
1781 11724496 : it != bst_map->end (); ++it)
1782 9823221 : if ((*it).second)
1783 9823221 : vect_free_slp_tree ((*it).second);
1784 3802550 : delete bst_map;
1785 1901275 : }
1786 :
1787 : /* ??? This was std::pair<std::pair<tree_code, vect_def_type>, tree>
1788 : but then vec::insert does memmove and that's not compatible with
1789 : std::pair. */
1790 : struct chain_op_t
1791 : {
1792 4293748 : chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
1793 4293748 : : code (code_), dt (dt_), op (op_) {}
1794 : tree_code code;
1795 : vect_def_type dt;
1796 : tree op;
1797 : };
1798 :
1799 : /* Comparator for sorting associatable chains. */
1800 :
1801 : static int
1802 12570305 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
1803 : {
1804 12570305 : auto *op1 = (const chain_op_t *) op1_;
1805 12570305 : auto *op2 = (const chain_op_t *) op2_;
1806 12570305 : if (op1->dt != op2->dt)
1807 1843818 : return (int)op1->dt - (int)op2->dt;
1808 10726487 : return (int)op1->code - (int)op2->code;
1809 : }
1810 :
1811 : /* Linearize the associatable expression chain at START with the
1812 : associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
1813 : filling CHAIN with the result and using WORKLIST as intermediate storage.
1814 : CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
1815 : or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation
1816 : stmts, starting with START. When ALLOW_ALT_CODE is false, do not
1817 : follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf). */
1818 :
1819 : static void
1820 1851645 : vect_slp_linearize_chain (vec_info *vinfo,
1821 : vec<std::pair<tree_code, gimple *> > &worklist,
1822 : vec<chain_op_t> &chain,
1823 : enum tree_code code, gimple *start,
1824 : gimple *&code_stmt, gimple *&alt_code_stmt,
1825 : vec<gimple *> *chain_stmts,
1826 : bool allow_alt_code = true)
1827 : {
1828 : /* For each lane linearize the addition/subtraction (or other
1829 : uniform associatable operation) expression tree. */
1830 1851645 : worklist.safe_push (std::make_pair (code, start));
1831 4293748 : while (!worklist.is_empty ())
1832 : {
1833 2442103 : auto entry = worklist.pop ();
1834 2442103 : gassign *stmt = as_a <gassign *> (entry.second);
1835 2442103 : enum tree_code in_code = entry.first;
1836 4884206 : enum tree_code this_code = gimple_assign_rhs_code (stmt);
1837 : /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE. */
1838 2442103 : if (!code_stmt
1839 2442103 : && gimple_assign_rhs_code (stmt) == code)
1840 1544979 : code_stmt = stmt;
1841 897124 : else if (!alt_code_stmt
1842 897124 : && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
1843 363478 : alt_code_stmt = stmt;
1844 2442103 : if (chain_stmts)
1845 2360873 : chain_stmts->safe_push (stmt);
1846 7326309 : for (unsigned opnum = 1; opnum <= 2; ++opnum)
1847 : {
1848 4884206 : tree op = gimple_op (stmt, opnum);
1849 4884206 : vect_def_type dt;
1850 4884206 : stmt_vec_info def_stmt_info;
1851 4884206 : bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
1852 4884206 : gcc_assert (res);
1853 4884206 : if (dt == vect_internal_def
1854 4884206 : && is_pattern_stmt_p (def_stmt_info))
1855 9280 : op = gimple_get_lhs (def_stmt_info->stmt);
1856 4884206 : gimple *use_stmt;
1857 4884206 : use_operand_p use_p;
1858 4884206 : if (dt == vect_internal_def
1859 4449189 : && single_imm_use (op, &use_p, &use_stmt)
1860 2870770 : && is_gimple_assign (def_stmt_info->stmt)
1861 7560460 : && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
1862 2086111 : || (allow_alt_code
1863 57362 : && code == PLUS_EXPR
1864 36310 : && (gimple_assign_rhs_code (def_stmt_info->stmt)
1865 : == MINUS_EXPR))))
1866 : {
1867 590458 : tree_code op_def_code = this_code;
1868 590458 : if (op_def_code == MINUS_EXPR && opnum == 1)
1869 55050 : op_def_code = PLUS_EXPR;
1870 590458 : if (in_code == MINUS_EXPR)
1871 222 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1872 590458 : worklist.safe_push (std::make_pair (op_def_code,
1873 590458 : def_stmt_info->stmt));
1874 : }
1875 : else
1876 : {
1877 4293748 : tree_code op_def_code = this_code;
1878 4293748 : if (op_def_code == MINUS_EXPR && opnum == 1)
1879 308545 : op_def_code = PLUS_EXPR;
1880 4293748 : if (in_code == MINUS_EXPR)
1881 4230 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1882 4293748 : chain.safe_push (chain_op_t (op_def_code, dt, op));
1883 : }
1884 : }
1885 : }
1886 1851645 : }
1887 :
1888 : /* Distance from the node currently being discovered to the closest upthread
1889 : commutative operation whose operand-zero discovery may still be fixed by
1890 : retrying with swapped operands, or -1U if there is none. */
1891 :
1892 : static unsigned least_upthread_swappable_op_distance = -1U;
1893 :
1894 : static slp_tree
1895 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
1896 : vec<stmt_vec_info> stmts,
1897 : bool *matches, unsigned *limit, unsigned *tree_size,
1898 : scalar_stmts_to_slp_tree_map_t *bst_map);
1899 :
1900 : static slp_tree
1901 6684862 : vect_build_slp_tree (vec_info *vinfo,
1902 : vec<stmt_vec_info> stmts,
1903 : bool *matches, unsigned *limit, unsigned *tree_size,
1904 : scalar_stmts_to_slp_tree_map_t *bst_map)
1905 : {
1906 6684862 : unsigned int group_size = stmts.length ();
1907 6684862 : if (slp_tree *leader = bst_map->get (stmts))
1908 : {
1909 498205 : if (dump_enabled_p ())
1910 17399 : dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
1911 17399 : !(*leader)->failed ? "" : "failed ",
1912 : (void *) *leader);
1913 498205 : if (!(*leader)->failed)
1914 : {
1915 452230 : SLP_TREE_REF_COUNT (*leader)++;
1916 452230 : stmts.release ();
1917 452230 : return *leader;
1918 : }
1919 45975 : memcpy (matches, (*leader)->failed, sizeof (bool) * group_size);
1920 45975 : return NULL;
1921 : }
1922 :
1923 : /* Single-lane SLP doesn't have the chance of run-away, do not account
1924 : it to the limit. */
1925 6186657 : if (stmts.length () > 1)
1926 : {
1927 3481478 : if (*limit == 0)
1928 : {
1929 1301 : if (dump_enabled_p ())
1930 15 : dump_printf_loc (MSG_NOTE, vect_location,
1931 : "SLP discovery limit exceeded\n");
1932 1301 : memset (matches, 0, sizeof (bool) * group_size);
1933 1301 : return NULL;
1934 : }
1935 3480177 : --*limit;
1936 : }
1937 :
1938 : /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
1939 : so we can pick up backedge destinations during discovery. */
1940 6185356 : slp_tree res = new _slp_tree;
1941 6185356 : SLP_TREE_DEF_TYPE (res) = vect_internal_def;
1942 6185356 : SLP_TREE_SCALAR_STMTS (res) = stmts;
1943 6185356 : bst_map->put (stmts.copy (), res);
1944 :
1945 6185356 : if (dump_enabled_p ())
1946 150679 : dump_printf_loc (MSG_NOTE, vect_location,
1947 : "starting SLP discovery for node %p\n", (void *) res);
1948 :
1949 6185356 : slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
1950 : matches, limit, tree_size, bst_map);
1951 6185356 : if (!res_)
1952 : {
1953 2217294 : if (dump_enabled_p ())
1954 8001 : dump_printf_loc (MSG_NOTE, vect_location,
1955 : "SLP discovery for node %p failed\n", (void *) res);
1956 : /* Mark the node invalid so we can detect those when still in use
1957 : as backedge destinations. */
1958 2217294 : SLP_TREE_SCALAR_STMTS (res) = vNULL;
1959 2217294 : SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
1960 2217294 : res->failed = XNEWVEC (bool, group_size);
1961 2217294 : if (flag_checking)
1962 : {
1963 : unsigned i;
1964 4017156 : for (i = 0; i < group_size; ++i)
1965 4017156 : if (!matches[i])
1966 : break;
1967 2217294 : gcc_assert (i < group_size);
1968 : }
1969 2217294 : memcpy (res->failed, matches, sizeof (bool) * group_size);
1970 : }
1971 : else
1972 : {
1973 3968062 : if (dump_enabled_p ())
1974 142678 : dump_printf_loc (MSG_NOTE, vect_location,
1975 : "SLP discovery for node %p succeeded\n",
1976 : (void *) res);
1977 3968062 : gcc_assert (res_ == res);
1978 : /* Keep a reference for the bst_map use. */
1979 3968062 : SLP_TREE_REF_COUNT (res)++;
1980 : }
1981 : return res_;
1982 : }
1983 :
1984 : /* Helper for building an associated SLP node chain. */
1985 :
1986 : static void
1987 158 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
1988 : slp_tree op0, slp_tree op1,
1989 : stmt_vec_info oper1, stmt_vec_info oper2,
1990 : vec<std::pair<unsigned, unsigned> > lperm)
1991 : {
1992 158 : unsigned group_size = SLP_TREE_LANES (op1);
1993 :
1994 158 : slp_tree child1 = new _slp_tree;
1995 158 : SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
1996 158 : SLP_TREE_VECTYPE (child1) = vectype;
1997 158 : SLP_TREE_LANES (child1) = group_size;
1998 158 : SLP_TREE_CHILDREN (child1).create (2);
1999 158 : SLP_TREE_CHILDREN (child1).quick_push (op0);
2000 158 : SLP_TREE_CHILDREN (child1).quick_push (op1);
2001 158 : SLP_TREE_REPRESENTATIVE (child1) = oper1;
2002 :
2003 158 : slp_tree child2 = new _slp_tree;
2004 158 : SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
2005 158 : SLP_TREE_VECTYPE (child2) = vectype;
2006 158 : SLP_TREE_LANES (child2) = group_size;
2007 158 : SLP_TREE_CHILDREN (child2).create (2);
2008 158 : SLP_TREE_CHILDREN (child2).quick_push (op0);
2009 158 : SLP_TREE_REF_COUNT (op0)++;
2010 158 : SLP_TREE_CHILDREN (child2).quick_push (op1);
2011 158 : SLP_TREE_REF_COUNT (op1)++;
2012 158 : SLP_TREE_REPRESENTATIVE (child2) = oper2;
2013 :
2014 158 : SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
2015 158 : SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
2016 158 : SLP_TREE_VECTYPE (perm) = vectype;
2017 158 : SLP_TREE_LANES (perm) = group_size;
2018 158 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
2019 158 : SLP_TREE_LANE_PERMUTATION (perm) = lperm;
2020 158 : SLP_TREE_CHILDREN (perm).quick_push (child1);
2021 158 : SLP_TREE_CHILDREN (perm).quick_push (child2);
2022 158 : }
2023 :
2024 : /* Recursively build an SLP tree starting from NODE.
2025 : Fail (and return a value not equal to zero) if def-stmts are not
2026 : isomorphic, require data permutation or are of unsupported types of
2027 : operation. Otherwise, return 0.
2028 : The value returned is the depth in the SLP tree where a mismatch
2029 : was found. */
2030 :
2031 : static slp_tree
2032 6185356 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
2033 : vec<stmt_vec_info> stmts,
2034 : bool *matches, unsigned *limit, unsigned *tree_size,
2035 : scalar_stmts_to_slp_tree_map_t *bst_map)
2036 : {
2037 6185356 : unsigned int group_size = stmts.length ();
2038 6185356 : unsigned nops, i, this_tree_size = 0;
2039 :
2040 6185356 : matches[0] = false;
2041 :
2042 6185356 : stmt_vec_info stmt_info = stmts[0];
2043 6185356 : if (!is_a<gcall *> (stmt_info->stmt)
2044 : && !is_a<gassign *> (stmt_info->stmt)
2045 : && !is_a<gphi *> (stmt_info->stmt))
2046 : return NULL;
2047 :
2048 6185258 : nops = gimple_num_args (stmt_info->stmt);
2049 6185258 : if (const int *map = vect_get_operand_map (stmt_info))
2050 37099 : nops = map[0];
2051 :
2052 : /* If the SLP node is a PHI (induction or reduction), terminate
2053 : the recursion. */
2054 6185258 : bool *skip_args = XALLOCAVEC (bool, nops);
2055 6185258 : memset (skip_args, 0, sizeof (bool) * nops);
2056 6185258 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2057 2903600 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
2058 : {
2059 316925 : tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
2060 316925 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
2061 : group_size);
2062 316925 : if (!vect_record_vectype (vinfo, stmt_info, group_size, vectype))
2063 : return NULL;
2064 :
2065 312684 : vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
2066 312684 : if (def_type == vect_induction_def)
2067 : {
2068 : /* Induction PHIs are not cycles but walk the initial
2069 : value. Only for inner loops through, for outer loops
2070 : we need to pick up the value from the actual PHIs
2071 : to more easily support peeling and epilogue vectorization. */
2072 204288 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2073 204288 : if (!nested_in_vect_loop_p (loop, stmt_info))
2074 203432 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2075 : else
2076 : loop = loop->inner;
2077 204288 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2078 : }
2079 108396 : else if (def_type == vect_reduction_def
2080 : || def_type == vect_double_reduction_def
2081 : || def_type == vect_nested_cycle
2082 108396 : || def_type == vect_first_order_recurrence)
2083 : {
2084 : /* Else def types have to match. */
2085 : stmt_vec_info other_info;
2086 : bool all_same = true;
2087 245015 : FOR_EACH_VEC_ELT (stmts, i, other_info)
2088 : {
2089 137942 : if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
2090 6185356 : return NULL;
2091 137936 : if (other_info != stmt_info)
2092 26235 : all_same = false;
2093 : }
2094 107073 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2095 : /* Reduction initial values are not explicitly represented. */
2096 107073 : if (def_type != vect_first_order_recurrence
2097 107073 : && gimple_bb (stmt_info->stmt) == loop->header)
2098 103826 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2099 : /* Reduction chain backedge defs are filled manually.
2100 : ??? Need a better way to identify a SLP reduction chain PHI.
2101 : Or a better overall way to SLP match those. */
2102 107073 : if (stmts.length () > 1
2103 107073 : && all_same && def_type == vect_reduction_def)
2104 2370 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2105 : }
2106 1317 : else if (def_type != vect_internal_def)
2107 : return NULL;
2108 : }
2109 :
2110 :
2111 6181011 : bool two_operators = false;
2112 6181011 : unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
2113 6181011 : tree vectype = NULL_TREE;
2114 6181011 : if (!vect_build_slp_tree_1 (vinfo, swap, stmts, matches, &two_operators,
2115 : &vectype))
2116 : return NULL;
2117 :
2118 : /* If the SLP node is a load, terminate the recursion unless masked. */
2119 4462643 : if (STMT_VINFO_DATA_REF (stmt_info)
2120 2108299 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
2121 : {
2122 949036 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2123 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
2124 : else
2125 : {
2126 929438 : (*tree_size)++;
2127 929438 : node = vect_create_new_slp_node (node, stmts, 0);
2128 929438 : SLP_TREE_VECTYPE (node) = vectype;
2129 : /* And compute the load permutation. Whether it is actually
2130 : a permutation depends on the unrolling factor which is
2131 : decided later. */
2132 929438 : vec<unsigned> load_permutation;
2133 929438 : int j;
2134 929438 : stmt_vec_info load_info;
2135 929438 : load_permutation.create (group_size);
2136 929438 : stmt_vec_info first_stmt_info
2137 929438 : = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2138 929438 : ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
2139 929438 : bool any_permute = false;
2140 2247363 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
2141 : {
2142 1317925 : int load_place;
2143 1317925 : if (! load_info)
2144 : {
2145 40759 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2146 : load_place = j;
2147 : else
2148 : load_place = 0;
2149 : }
2150 1277166 : else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2151 746670 : load_place = vect_get_place_in_interleaving_chain
2152 746670 : (load_info, first_stmt_info);
2153 : else
2154 : /* Recognize the splat case as { 0, 0, ... } but make
2155 : sure to use the appropriate refs for collections
2156 : of invariant refs. */
2157 530496 : load_place = (load_info == stmt_info) ? 0 : j;
2158 787670 : gcc_assert (load_place != -1);
2159 1317925 : any_permute |= load_place != j;
2160 1317925 : load_permutation.quick_push (load_place);
2161 : }
2162 :
2163 929438 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
2164 : {
2165 3500 : gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
2166 3500 : bool has_gaps = false;
2167 3500 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2168 189 : for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
2169 846 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2170 657 : if (DR_GROUP_GAP (si) != 1)
2171 80 : has_gaps = true;
2172 : /* We cannot handle permuted masked loads directly, see
2173 : PR114375. We cannot handle strided masked loads or masked
2174 : loads with gaps unless the mask is uniform. */
2175 3500 : if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
2176 189 : && (DR_GROUP_GAP (first_stmt_info) != 0
2177 129 : || (has_gaps
2178 35 : && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
2179 6920 : || STMT_VINFO_STRIDED_P (stmt_info))
2180 : {
2181 93 : load_permutation.release ();
2182 93 : matches[0] = false;
2183 926070 : return NULL;
2184 : }
2185 :
2186 : /* For permuted masked loads do an unpermuted masked load of
2187 : the whole group followed by a SLP permute node. */
2188 3407 : if (any_permute
2189 3407 : || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
2190 83 : && DR_GROUP_SIZE (first_stmt_info) != group_size))
2191 : {
2192 : /* Discover the whole unpermuted load. */
2193 39 : vec<stmt_vec_info> stmts2;
2194 39 : unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2195 68 : ? DR_GROUP_SIZE (first_stmt_info) : 1;
2196 39 : stmts2.create (dr_group_size);
2197 39 : stmts2.quick_grow_cleared (dr_group_size);
2198 39 : unsigned i = 0;
2199 39 : for (stmt_vec_info si = first_stmt_info;
2200 464 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2201 : {
2202 425 : if (si != first_stmt_info)
2203 1586 : for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
2204 1200 : stmts2[i++] = NULL;
2205 425 : stmts2[i++] = si;
2206 : }
2207 39 : bool *matches2 = XALLOCAVEC (bool, dr_group_size);
2208 39 : slp_tree unperm_load
2209 39 : = vect_build_slp_tree (vinfo, stmts2, matches2, limit,
2210 39 : &this_tree_size, bst_map);
2211 : /* When we are able to do the full masked load emit that
2212 : followed by 'node' being the desired final permutation. */
2213 39 : if (unperm_load)
2214 : {
2215 16 : gcc_assert
2216 : (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
2217 16 : lane_permutation_t lperm;
2218 16 : lperm.create (group_size);
2219 72 : for (unsigned j = 0; j < load_permutation.length (); ++j)
2220 40 : lperm.quick_push
2221 40 : (std::make_pair (0, load_permutation[j]));
2222 16 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2223 16 : SLP_TREE_CHILDREN (node).safe_push (unperm_load);
2224 16 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2225 16 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2226 16 : load_permutation.release ();
2227 16 : return node;
2228 : }
2229 23 : stmts2.release ();
2230 23 : load_permutation.release ();
2231 23 : matches[0] = false;
2232 23 : return NULL;
2233 : }
2234 3368 : load_permutation.release ();
2235 : }
2236 : else
2237 : {
2238 925938 : if (!any_permute
2239 797098 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2240 1218872 : && group_size == DR_GROUP_SIZE (first_stmt_info))
2241 128934 : load_permutation.release ();
2242 925938 : SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
2243 925938 : return node;
2244 : }
2245 : }
2246 : }
2247 3513607 : else if (gimple_assign_single_p (stmt_info->stmt)
2248 2325462 : && !gimple_vuse (stmt_info->stmt)
2249 3522519 : && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
2250 : {
2251 : /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
2252 : the same SSA name vector of a compatible type to vectype. */
2253 3397 : vec<std::pair<unsigned, unsigned> > lperm = vNULL;
2254 3397 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
2255 3397 : stmt_vec_info estmt_info;
2256 12015 : FOR_EACH_VEC_ELT (stmts, i, estmt_info)
2257 : {
2258 8765 : gassign *estmt = as_a <gassign *> (estmt_info->stmt);
2259 8765 : tree bfref = gimple_assign_rhs1 (estmt);
2260 8765 : HOST_WIDE_INT lane;
2261 8765 : if (!known_eq (bit_field_size (bfref),
2262 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
2263 17383 : || !constant_multiple_p (bit_field_offset (bfref),
2264 8765 : bit_field_size (bfref), &lane))
2265 : {
2266 147 : lperm.release ();
2267 147 : matches[0] = false;
2268 147 : return NULL;
2269 : }
2270 8618 : lperm.safe_push (std::make_pair (0, (unsigned)lane));
2271 : }
2272 3250 : slp_tree vnode = vect_create_new_slp_node (vNULL);
2273 3250 : if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
2274 : /* ??? We record vectype here but we hide eventually necessary
2275 : punning and instead rely on code generation to materialize
2276 : VIEW_CONVERT_EXPRs as necessary. We instead should make
2277 : this explicit somehow. */
2278 1456 : SLP_TREE_VECTYPE (vnode) = vectype;
2279 : else
2280 : {
2281 : /* For different size but compatible elements we can still
2282 : use VEC_PERM_EXPR without punning. */
2283 1794 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
2284 : && types_compatible_p (TREE_TYPE (vectype),
2285 : TREE_TYPE (TREE_TYPE (vec))));
2286 1794 : SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
2287 : }
2288 3250 : auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
2289 3250 : unsigned HOST_WIDE_INT const_nunits;
2290 3250 : if (nunits.is_constant (&const_nunits))
2291 3250 : SLP_TREE_LANES (vnode) = const_nunits;
2292 3250 : SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
2293 : /* We are always building a permutation node even if it is an identity
2294 : permute to shield the rest of the vectorizer from the odd node
2295 : representing an actual vector without any scalar ops.
2296 : ??? We could hide it completely with making the permute node
2297 : external? */
2298 3250 : node = vect_create_new_slp_node (node, stmts, 1);
2299 3250 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2300 3250 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2301 3250 : SLP_TREE_VECTYPE (node) = vectype;
2302 3250 : SLP_TREE_CHILDREN (node).quick_push (vnode);
2303 3250 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2304 3250 : return node;
2305 : }
2306 : /* When discovery reaches an associatable operation see whether we can
2307 : improve that to match up lanes in a way superior to the operand
2308 : swapping code which at most looks at two defs.
2309 : ??? For BB vectorization we cannot do the brute-force search
2310 : for matching as we can succeed by means of builds from scalars
2311 : and have no good way to "cost" one build against another. */
2312 3510210 : else if (is_a <loop_vec_info> (vinfo)
2313 : /* Do not bother for single-lane SLP. */
2314 2063047 : && group_size > 1
2315 : /* ??? We don't handle !vect_internal_def defs below. */
2316 113692 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
2317 : /* ??? Do not associate a reduction, this will wreck REDUC_IDX
2318 : mapping as long as that exists on the stmt_info level. */
2319 88114 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1
2320 79559 : && is_gimple_assign (stmt_info->stmt)
2321 79240 : && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
2322 52360 : || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
2323 3539042 : && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
2324 16577 : || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
2325 14039 : && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
2326 : {
2327 : /* See if we have a chain of (mixed) adds or subtracts or other
2328 : associatable ops. */
2329 21770 : enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
2330 21770 : if (code == MINUS_EXPR)
2331 848 : code = PLUS_EXPR;
2332 21770 : stmt_vec_info other_op_stmt_info = NULL;
2333 21770 : stmt_vec_info op_stmt_info = NULL;
2334 21770 : unsigned chain_len = 0;
2335 21770 : auto_vec<chain_op_t> chain;
2336 21770 : auto_vec<std::pair<tree_code, gimple *> > worklist;
2337 21770 : auto_vec<vec<chain_op_t> > chains (group_size);
2338 21770 : auto_vec<slp_tree, 4> children;
2339 21770 : bool hard_fail = true;
2340 22797 : for (unsigned lane = 0; lane < group_size; ++lane)
2341 : {
2342 22461 : if (!stmts[lane])
2343 : {
2344 : /* ??? Below we require lane zero is present. */
2345 0 : if (lane == 0)
2346 : {
2347 : hard_fail = false;
2348 21434 : break;
2349 : }
2350 0 : chains.quick_push (vNULL);
2351 0 : continue;
2352 : }
2353 : /* For each lane linearize the addition/subtraction (or other
2354 : uniform associatable operation) expression tree. */
2355 22461 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
2356 22461 : vect_slp_linearize_chain (vinfo, worklist, chain, code,
2357 22461 : stmts[lane]->stmt, op_stmt, other_op_stmt,
2358 : NULL);
2359 22461 : if (!op_stmt_info && op_stmt)
2360 21119 : op_stmt_info = vinfo->lookup_stmt (op_stmt);
2361 22461 : if (!other_op_stmt_info && other_op_stmt)
2362 884 : other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
2363 22461 : if (chain.length () == 2)
2364 : {
2365 : /* In a chain of just two elements resort to the regular
2366 : operand swapping scheme. Likewise if we run into a
2367 : length mismatch process regularly as well as we did not
2368 : process the other lanes we cannot report a good hint what
2369 : lanes to try swapping in the parent. */
2370 : hard_fail = false;
2371 : break;
2372 : }
2373 1030 : else if (chain_len == 0)
2374 376 : chain_len = chain.length ();
2375 1308 : else if (chain.length () != chain_len)
2376 : {
2377 : /* ??? Here we could slip in magic to compensate with
2378 : neutral operands. */
2379 3 : matches[lane] = false;
2380 3 : if (lane != group_size - 1)
2381 3 : matches[0] = false;
2382 : break;
2383 : }
2384 1027 : chains.quick_push (chain.copy ());
2385 1027 : chain.truncate (0);
2386 : }
2387 43540 : if (chains.length () == group_size)
2388 : {
2389 : /* We cannot yet use SLP_TREE_CODE to communicate the operation. */
2390 336 : if (!op_stmt_info)
2391 : {
2392 3 : hard_fail = false;
2393 3 : goto out;
2394 : }
2395 : /* Now we have a set of chains with the same length. */
2396 : /* 1. pre-sort according to def_type and operation. */
2397 1248 : for (unsigned lane = 0; lane < group_size; ++lane)
2398 1830 : chains[lane].stablesort (dt_sort_cmp, vinfo);
2399 333 : if (dump_enabled_p ())
2400 : {
2401 157 : dump_printf_loc (MSG_NOTE, vect_location,
2402 : "pre-sorted chains of %s\n",
2403 : get_tree_code_name (code));
2404 685 : for (unsigned lane = 0; lane < group_size; ++lane)
2405 : {
2406 528 : if (!stmts[lane])
2407 0 : dump_printf (MSG_NOTE, "--");
2408 : else
2409 2422 : for (unsigned opnum = 0; opnum < chain_len; ++opnum)
2410 3788 : dump_printf (MSG_NOTE, "%s %T ",
2411 1894 : get_tree_code_name (chains[lane][opnum].code),
2412 1894 : chains[lane][opnum].op);
2413 528 : dump_printf (MSG_NOTE, "\n");
2414 : }
2415 : }
2416 : /* 2. try to build children nodes, associating as necessary. */
2417 : /* 2a. prepare and perform early checks to avoid eating into
2418 : discovery limit unnecessarily. */
2419 333 : vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
2420 1407 : for (unsigned n = 0; n < chain_len; ++n)
2421 : {
2422 1074 : vect_def_type dt = chains[0][n].dt;
2423 1074 : unsigned lane;
2424 4177 : for (lane = 0; lane < group_size; ++lane)
2425 6206 : if (stmts[lane] && chains[lane][n].dt != dt)
2426 : {
2427 0 : if (dt == vect_constant_def
2428 0 : && chains[lane][n].dt == vect_external_def)
2429 : dt = vect_external_def;
2430 0 : else if (dt == vect_external_def
2431 0 : && chains[lane][n].dt == vect_constant_def)
2432 : ;
2433 : else
2434 : break;
2435 : }
2436 1074 : if (lane != group_size)
2437 : {
2438 0 : if (dump_enabled_p ())
2439 0 : dump_printf_loc (MSG_NOTE, vect_location,
2440 : "giving up on chain due to mismatched "
2441 : "def types\n");
2442 0 : matches[lane] = false;
2443 0 : if (lane != group_size - 1)
2444 0 : matches[0] = false;
2445 0 : goto out;
2446 : }
2447 1074 : dts[n] = dt;
2448 1074 : if (dt == vect_constant_def
2449 1074 : || dt == vect_external_def)
2450 : {
2451 : /* Check whether we can build the invariant. If we can't
2452 : we never will be able to. */
2453 93 : tree type = TREE_TYPE (chains[0][n].op);
2454 1074 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
2455 : && (TREE_CODE (type) == BOOLEAN_TYPE
2456 : || !can_duplicate_and_interleave_p (vinfo, group_size,
2457 : type)))
2458 : {
2459 : matches[0] = false;
2460 : goto out;
2461 : }
2462 : }
2463 981 : else if (dt != vect_internal_def)
2464 : {
2465 : /* Not sure, we might need sth special.
2466 : gcc.dg/vect/pr96854.c,
2467 : gfortran.dg/vect/fast-math-pr37021.f90
2468 : and gfortran.dg/vect/pr61171.f trigger. */
2469 : /* Soft-fail for now. */
2470 0 : hard_fail = false;
2471 0 : goto out;
2472 : }
2473 : }
2474 : /* 2b. do the actual build. */
2475 1353 : for (unsigned n = 0; n < chain_len; ++n)
2476 : {
2477 1039 : vect_def_type dt = dts[n];
2478 1039 : unsigned lane;
2479 1039 : if (dt == vect_constant_def
2480 1039 : || dt == vect_external_def)
2481 : {
2482 93 : vec<tree> ops;
2483 93 : ops.create (group_size);
2484 461 : for (lane = 0; lane < group_size; ++lane)
2485 275 : if (stmts[lane])
2486 275 : ops.quick_push (chains[lane][n].op);
2487 : else
2488 0 : ops.quick_push (NULL_TREE);
2489 93 : slp_tree child = vect_create_new_slp_node (ops);
2490 93 : SLP_TREE_DEF_TYPE (child) = dt;
2491 93 : children.safe_push (child);
2492 : }
2493 : else
2494 : {
2495 946 : vec<stmt_vec_info> op_stmts;
2496 946 : op_stmts.create (group_size);
2497 946 : slp_tree child = NULL;
2498 : /* Brute-force our way. We have to consider a lane
2499 : failing after fixing an earlier fail up in the
2500 : SLP discovery recursion. So track the current
2501 : permute per lane. */
2502 946 : unsigned *perms = XALLOCAVEC (unsigned, group_size);
2503 946 : memset (perms, 0, sizeof (unsigned) * group_size);
2504 1040 : do
2505 : {
2506 1040 : op_stmts.truncate (0);
2507 5092 : for (lane = 0; lane < group_size; ++lane)
2508 3012 : if (stmts[lane])
2509 3012 : op_stmts.quick_push
2510 3012 : (vinfo->lookup_def (chains[lane][n].op));
2511 : else
2512 0 : op_stmts.quick_push (NULL);
2513 1040 : child = vect_build_slp_tree (vinfo, op_stmts,
2514 : matches, limit,
2515 : &this_tree_size, bst_map);
2516 : /* ??? We're likely getting too many fatal mismatches
2517 : here so maybe we want to ignore them (but then we
2518 : have no idea which lanes fatally mismatched). */
2519 1040 : if (child || !matches[0])
2520 : break;
2521 : /* Swap another lane we have not yet matched up into
2522 : lanes that did not match. If we run out of
2523 : permute possibilities for a lane terminate the
2524 : search. */
2525 287 : bool term = false;
2526 287 : for (lane = 1; lane < group_size; ++lane)
2527 193 : if (!matches[lane])
2528 : {
2529 165 : if (n + perms[lane] + 1 == chain_len)
2530 : {
2531 : term = true;
2532 : break;
2533 : }
2534 146 : if (dump_enabled_p ())
2535 113 : dump_printf_loc (MSG_NOTE, vect_location,
2536 : "swapping operand %d and %d "
2537 : "of lane %d\n",
2538 : n, n + perms[lane] + 1, lane);
2539 292 : std::swap (chains[lane][n],
2540 146 : chains[lane][n + perms[lane] + 1]);
2541 146 : perms[lane]++;
2542 : }
2543 113 : if (term)
2544 : break;
2545 : }
2546 : while (1);
2547 946 : if (!child)
2548 : {
2549 19 : if (dump_enabled_p ())
2550 18 : dump_printf_loc (MSG_NOTE, vect_location,
2551 : "failed to match up op %d\n", n);
2552 19 : op_stmts.release ();
2553 19 : if (lane != group_size - 1)
2554 9 : matches[0] = false;
2555 : else
2556 10 : matches[lane] = false;
2557 19 : goto out;
2558 : }
2559 927 : if (dump_enabled_p ())
2560 : {
2561 421 : dump_printf_loc (MSG_NOTE, vect_location,
2562 : "matched up op %d to\n", n);
2563 421 : vect_print_slp_tree (MSG_NOTE, vect_location, child);
2564 : }
2565 927 : children.safe_push (child);
2566 : }
2567 : }
2568 : /* 3. build SLP nodes to combine the chain. */
2569 1156 : for (unsigned lane = 0; lane < group_size; ++lane)
2570 1696 : if (stmts[lane] && chains[lane][0].code != code)
2571 : {
2572 : /* See if there's any alternate all-PLUS entry. */
2573 : unsigned n;
2574 6 : for (n = 1; n < chain_len; ++n)
2575 : {
2576 30 : for (lane = 0; lane < group_size; ++lane)
2577 48 : if (stmts[lane] && chains[lane][n].code != code)
2578 : break;
2579 6 : if (lane == group_size)
2580 : break;
2581 : }
2582 6 : if (n != chain_len)
2583 : {
2584 : /* Swap that in at first position. */
2585 6 : std::swap (children[0], children[n]);
2586 30 : for (lane = 0; lane < group_size; ++lane)
2587 24 : if (stmts[lane])
2588 24 : std::swap (chains[lane][0], chains[lane][n]);
2589 : }
2590 : else
2591 : {
2592 : /* ??? When this triggers and we end up with two
2593 : vect_constant/external_def up-front things break (ICE)
2594 : spectacularly finding an insertion place for the
2595 : all-constant op. We should have a fully
2596 : vect_internal_def operand though(?) so we can swap
2597 : that into first place and then prepend the all-zero
2598 : constant. */
2599 0 : if (dump_enabled_p ())
2600 0 : dump_printf_loc (MSG_NOTE, vect_location,
2601 : "inserting constant zero to compensate "
2602 : "for (partially) negated first "
2603 : "operand\n");
2604 0 : chain_len++;
2605 0 : for (lane = 0; lane < group_size; ++lane)
2606 0 : if (stmts[lane])
2607 0 : chains[lane].safe_insert
2608 0 : (0, chain_op_t (code, vect_constant_def, NULL_TREE));
2609 0 : vec<tree> zero_ops;
2610 0 : zero_ops.create (group_size);
2611 0 : zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
2612 0 : for (lane = 1; lane < group_size; ++lane)
2613 0 : if (stmts[lane])
2614 0 : zero_ops.quick_push (zero_ops[0]);
2615 : else
2616 0 : zero_ops.quick_push (NULL_TREE);
2617 0 : slp_tree zero = vect_create_new_slp_node (zero_ops);
2618 0 : SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
2619 0 : children.safe_insert (0, zero);
2620 : }
2621 : break;
2622 : }
2623 1015 : for (unsigned i = 1; i < children.length (); ++i)
2624 : {
2625 701 : slp_tree op0 = children[i - 1];
2626 701 : slp_tree op1 = children[i];
2627 701 : bool this_two_op = false;
2628 2569 : for (unsigned lane = 0; lane < group_size; ++lane)
2629 4052 : if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
2630 : {
2631 : this_two_op = true;
2632 : break;
2633 : }
2634 701 : slp_tree child;
2635 701 : if (i == children.length () - 1)
2636 314 : child = vect_create_new_slp_node (node, stmts, 2);
2637 : else
2638 387 : child = vect_create_new_slp_node (2, ERROR_MARK);
2639 701 : if (this_two_op)
2640 : {
2641 158 : vec<std::pair<unsigned, unsigned> > lperm;
2642 158 : lperm.create (group_size);
2643 728 : for (unsigned lane = 0; lane < group_size; ++lane)
2644 824 : lperm.quick_push (std::make_pair
2645 412 : (chains[lane][i].code != chains[0][i].code, lane));
2646 316 : vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
2647 158 : (chains[0][i].code == code
2648 : ? op_stmt_info
2649 : : other_op_stmt_info),
2650 158 : (chains[0][i].code == code
2651 : ? other_op_stmt_info
2652 : : op_stmt_info),
2653 : lperm);
2654 : }
2655 : else
2656 : {
2657 543 : SLP_TREE_DEF_TYPE (child) = vect_internal_def;
2658 543 : SLP_TREE_VECTYPE (child) = vectype;
2659 543 : SLP_TREE_LANES (child) = group_size;
2660 543 : SLP_TREE_CHILDREN (child).quick_push (op0);
2661 543 : SLP_TREE_CHILDREN (child).quick_push (op1);
2662 543 : SLP_TREE_REPRESENTATIVE (child)
2663 1086 : = (chains[0][i].code == code
2664 543 : ? op_stmt_info : other_op_stmt_info);
2665 : }
2666 701 : children[i] = child;
2667 : }
2668 314 : *tree_size += this_tree_size + 1;
2669 1516 : while (!chains.is_empty ())
2670 866 : chains.pop ().release ();
2671 : return node;
2672 : }
2673 21434 : out:
2674 21456 : if (dump_enabled_p ())
2675 2817 : dump_printf_loc (MSG_NOTE, vect_location,
2676 : "failed to line up SLP graph by re-associating "
2677 : "operations in lanes%s\n",
2678 : !hard_fail ? " trying regular discovery" : "");
2679 21461 : while (!children.is_empty ())
2680 5 : vect_free_slp_tree (children.pop ());
2681 21617 : while (!chains.is_empty ())
2682 161 : chains.pop ().release ();
2683 : /* Hard-fail, otherwise we might run into quadratic processing of the
2684 : chains starting one stmt into the chain again. */
2685 21456 : if (hard_fail)
2686 : return NULL;
2687 : /* Fall thru to normal processing. */
2688 21770 : }
2689 :
2690 : /* Get at the operands, verifying they are compatible. */
2691 3532840 : vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
2692 3532840 : slp_oprnd_info oprnd_info;
2693 20117872 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
2694 : {
2695 26106994 : int res = vect_get_and_check_slp_defs (vinfo, vectype,
2696 13053497 : swap[i], skip_args,
2697 : stmts, i, &oprnds_info);
2698 13053497 : if (res != 0)
2699 608015 : matches[(res == -1) ? 0 : i] = false;
2700 13053497 : if (!matches[0])
2701 : break;
2702 : }
2703 16239251 : for (i = 0; i < group_size; ++i)
2704 12947677 : if (!matches[i])
2705 : {
2706 241266 : vect_free_oprnd_info (oprnds_info);
2707 241266 : return NULL;
2708 : }
2709 3291574 : swap = NULL;
2710 :
2711 3291574 : bool has_two_operators_perm = false;
2712 16457870 : auto_vec<unsigned> two_op_perm_indices[2];
2713 3291574 : vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
2714 :
2715 3307761 : if (two_operators && oprnds_info.length () == 2 && group_size > 2)
2716 : {
2717 4009 : unsigned idx = 0;
2718 4009 : hash_map<gimple *, unsigned> seen;
2719 4009 : vec<slp_oprnd_info> new_oprnds_info
2720 4009 : = vect_create_oprnd_info (1, group_size);
2721 4009 : bool success = true;
2722 :
2723 4009 : enum tree_code code = ERROR_MARK;
2724 4009 : if (oprnds_info[0]->def_stmts[0]
2725 4009 : && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
2726 3948 : code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
2727 4009 : basic_block bb = nullptr;
2728 :
2729 7707 : for (unsigned j = 0; j < group_size; ++j)
2730 : {
2731 17989 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2732 : {
2733 14291 : stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
2734 14291 : if (!stmt_info
2735 14125 : || !is_a<gassign *> (stmt_info->stmt)
2736 14122 : || gimple_assign_rhs_code (stmt_info->stmt) != code
2737 25055 : || skip_args[i])
2738 : {
2739 : success = false;
2740 3531 : break;
2741 : }
2742 : /* Avoid mixing lanes with defs in different basic-blocks. */
2743 10764 : if (!bb)
2744 4144 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
2745 8554 : else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
2746 : {
2747 : success = false;
2748 : break;
2749 : }
2750 :
2751 10760 : bool exists;
2752 10760 : unsigned &stmt_idx
2753 10760 : = seen.get_or_insert (stmt_info->stmt, &exists);
2754 :
2755 10760 : if (!exists)
2756 : {
2757 9383 : new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
2758 9383 : new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
2759 9383 : stmt_idx = idx;
2760 9383 : idx++;
2761 : }
2762 :
2763 10760 : two_op_perm_indices[i].safe_push (stmt_idx);
2764 : }
2765 :
2766 7229 : if (!success)
2767 : break;
2768 : }
2769 :
2770 4009 : if (success && idx == group_size)
2771 : {
2772 97 : if (dump_enabled_p ())
2773 : {
2774 0 : dump_printf_loc (MSG_NOTE, vect_location,
2775 : "Replace two_operators operands:\n");
2776 :
2777 0 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2778 : {
2779 0 : dump_printf_loc (MSG_NOTE, vect_location,
2780 : "Operand %u:\n", i);
2781 0 : for (unsigned j = 0; j < group_size; j++)
2782 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2783 0 : j, oprnd_info->def_stmts[j]->stmt);
2784 : }
2785 :
2786 0 : dump_printf_loc (MSG_NOTE, vect_location,
2787 : "With a single operand:\n");
2788 0 : for (unsigned j = 0; j < group_size; j++)
2789 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2790 0 : j, new_oprnds_info[0]->def_stmts[j]->stmt);
2791 : }
2792 :
2793 97 : two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
2794 97 : two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
2795 :
2796 97 : new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
2797 97 : new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
2798 97 : new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
2799 97 : new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
2800 97 : new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
2801 :
2802 97 : vect_free_oprnd_info (oprnds_info);
2803 97 : oprnds_info = new_oprnds_info;
2804 97 : nops = 1;
2805 97 : has_two_operators_perm = true;
2806 : }
2807 : else
2808 3912 : vect_free_oprnd_info (new_oprnds_info);
2809 4009 : }
2810 :
2811 6583148 : auto_vec<slp_tree, 4> children;
2812 :
2813 3291574 : stmt_info = stmts[0];
2814 :
2815 3291574 : int reduc_idx = -1;
2816 3291574 : int gs_scale = 0;
2817 3291574 : tree gs_base = NULL_TREE;
2818 :
2819 : /* Create SLP_TREE nodes for the definition node/s. */
2820 8464707 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2821 : {
2822 5286533 : slp_tree child = nullptr;
2823 5286533 : unsigned int j;
2824 5286533 : unsigned old_swap_distance;
2825 5286533 : bool can_swap;
2826 5286533 : bool can_swap_nonmatching;
2827 5286533 : bool *stmt_can_swap;
2828 :
2829 : /* We're skipping certain operands from processing, for example
2830 : outer loop reduction initial defs. */
2831 5286533 : if (skip_args[i])
2832 : {
2833 513916 : children.safe_push (NULL);
2834 5173133 : continue;
2835 : }
2836 :
2837 4772617 : if (oprnd_info->first_dt == vect_uninitialized_def)
2838 : {
2839 : /* COND_EXPR have one too many eventually if the condition
2840 : is a SSA name. */
2841 0 : gcc_assert (i == 3 && nops == 4);
2842 0 : continue;
2843 : }
2844 :
2845 4772617 : if (oprnd_info->first_gs_p)
2846 : {
2847 23785 : gs_scale = oprnd_info->first_gs_info.scale;
2848 23785 : gs_base = oprnd_info->first_gs_info.base;
2849 : }
2850 :
2851 4772617 : if (is_a <bb_vec_info> (vinfo)
2852 1699629 : && oprnd_info->first_dt == vect_internal_def
2853 5680604 : && !oprnd_info->any_pattern)
2854 : {
2855 : /* For BB vectorization, if all defs are the same do not
2856 : bother to continue the build along the single-lane
2857 : graph but use a splat of the scalar value. */
2858 854818 : stmt_vec_info first_def = oprnd_info->def_stmts[0];
2859 922648 : for (j = 1; j < group_size; ++j)
2860 872194 : if (oprnd_info->def_stmts[j] != first_def)
2861 : break;
2862 854818 : if (j == group_size
2863 : /* But avoid doing this for loads where we may be
2864 : able to CSE things, unless the stmt is not
2865 : vectorizable. */
2866 854818 : && (!STMT_VINFO_VECTORIZABLE (first_def)
2867 60542 : || !gimple_vuse (first_def->stmt)))
2868 : {
2869 40854 : if (dump_enabled_p ())
2870 119 : dump_printf_loc (MSG_NOTE, vect_location,
2871 : "Using a splat of the uniform operand %G",
2872 : first_def->stmt);
2873 40854 : oprnd_info->first_dt = vect_external_def;
2874 : }
2875 : }
2876 :
2877 4772617 : if (oprnd_info->first_dt == vect_external_def
2878 4772617 : || oprnd_info->first_dt == vect_constant_def)
2879 : {
2880 1572390 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
2881 : {
2882 : tree op0;
2883 : tree uniform_val = op0 = oprnd_info->ops[0];
2884 : for (j = 1; j < oprnd_info->ops.length (); ++j)
2885 : if (oprnd_info->ops[j]
2886 : && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
2887 : {
2888 : uniform_val = NULL_TREE;
2889 : break;
2890 : }
2891 : if (!uniform_val
2892 : && !can_duplicate_and_interleave_p (vinfo,
2893 : oprnd_info->ops.length (),
2894 : TREE_TYPE (op0)))
2895 : {
2896 : matches[j] = false;
2897 : if (dump_enabled_p ())
2898 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2899 : "Build SLP failed: invalid type of def "
2900 : "for variable-length SLP %T\n", op0);
2901 : goto fail;
2902 : }
2903 : }
2904 1572390 : slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
2905 1572390 : SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
2906 1572390 : oprnd_info->ops = vNULL;
2907 1572390 : children.safe_push (invnode);
2908 1572390 : continue;
2909 1572390 : }
2910 :
2911 : /* See which SLP operand a reduction chain continues on. We want
2912 : to chain even PHIs but not backedges. */
2913 3200227 : if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
2914 3200227 : || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
2915 : {
2916 237923 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
2917 : {
2918 776 : if (oprnd_info->first_dt == vect_double_reduction_def)
2919 388 : reduc_idx = i;
2920 : }
2921 237147 : else if (is_a <gphi *> (stmt_info->stmt)
2922 237147 : && gimple_phi_num_args
2923 101849 : (as_a <gphi *> (stmt_info->stmt)) != 1)
2924 : ;
2925 135691 : else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
2926 393 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
2927 : ;
2928 135691 : else if (reduc_idx == -1)
2929 127200 : reduc_idx = i;
2930 : else
2931 : /* For .COND_* reduction operations the else value can be the
2932 : same as one of the operation operands. The other def
2933 : stmts have been moved, so we can't check easily. Check
2934 : it's a call at least. */
2935 8491 : gcc_assert (is_a <gcall *> (stmt_info->stmt));
2936 : }
2937 :
2938 : /* When we have a masked load with uniform mask discover this
2939 : as a single-lane mask with a splat permute. This way we can
2940 : recognize this as a masked load-lane by stripping the splat. */
2941 3200227 : if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
2942 58311 : && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
2943 : IFN_MASK_LOAD)
2944 6193 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2945 3200304 : && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
2946 : {
2947 35 : vec<stmt_vec_info> def_stmts2;
2948 35 : def_stmts2.create (1);
2949 35 : def_stmts2.quick_push (oprnd_info->def_stmts[0]);
2950 35 : child = vect_build_slp_tree (vinfo, def_stmts2,
2951 : matches, limit,
2952 : &this_tree_size, bst_map);
2953 35 : if (child)
2954 : {
2955 35 : slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
2956 35 : SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
2957 35 : SLP_TREE_LANES (pnode) = group_size;
2958 35 : SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
2959 35 : SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
2960 245 : for (unsigned k = 0; k < group_size; ++k)
2961 : {
2962 175 : SLP_TREE_SCALAR_STMTS (pnode)
2963 175 : .quick_push (oprnd_info->def_stmts[0]);
2964 175 : SLP_TREE_LANE_PERMUTATION (pnode)
2965 175 : .quick_push (std::make_pair (0u, 0u));
2966 : }
2967 35 : SLP_TREE_CHILDREN (pnode).quick_push (child);
2968 35 : children.safe_push (pnode);
2969 35 : oprnd_info->def_stmts = vNULL;
2970 35 : continue;
2971 35 : }
2972 : else
2973 0 : def_stmts2.release ();
2974 : }
2975 :
2976 6400384 : can_swap = (i == 0
2977 2363226 : && (nops == 2 || nops == 3)
2978 1528388 : && oprnds_info.length () > 1
2979 1528388 : && oprnds_info[1]->first_dt == vect_internal_def
2980 624229 : && (is_gimple_assign (stmt_info->stmt)
2981 51244 : || is_gimple_call (stmt_info->stmt))
2982 : /* Swapping operands for reductions breaks assumptions
2983 : later on. */
2984 3778328 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
2985 3200192 : can_swap_nonmatching = can_swap;
2986 3200192 : stmt_can_swap = NULL;
2987 3200192 : if (can_swap)
2988 : {
2989 522492 : stmt_can_swap = XALLOCAVEC (bool, group_size);
2990 8197673 : for (j = 0; j < group_size; ++j)
2991 : {
2992 7675181 : stmt_can_swap[j] = false;
2993 7675181 : if (!stmts[j])
2994 : /* NULL lanes are gaps and have no stmt to swap. */
2995 0 : stmt_can_swap[j] = true;
2996 7675181 : else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
2997 : {
2998 7669593 : tree_code code = gimple_assign_rhs_code (stmt);
2999 15339186 : stmt_can_swap[j] = (commutative_tree_code (code)
3000 7669593 : || commutative_ternary_tree_code (code));
3001 : }
3002 5588 : else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
3003 : {
3004 5588 : internal_fn fn = (gimple_call_internal_p (call)
3005 5588 : ? gimple_call_internal_fn (call) : IFN_LAST);
3006 11176 : stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
3007 5272 : || commutative_ternary_fn_p (fn))
3008 5624 : && first_commutative_argument (fn) == 0);
3009 : }
3010 :
3011 7675181 : if (j != 0 && !stmt_can_swap[j])
3012 7675181 : can_swap_nonmatching = false;
3013 : }
3014 : }
3015 :
3016 3200192 : old_swap_distance = least_upthread_swappable_op_distance;
3017 3200192 : if (can_swap_nonmatching)
3018 480187 : least_upthread_swappable_op_distance = 1;
3019 2720005 : else if (least_upthread_swappable_op_distance != -1U)
3020 351457 : least_upthread_swappable_op_distance++;
3021 3200192 : child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3022 : matches, limit,
3023 : &this_tree_size, bst_map);
3024 3200192 : least_upthread_swappable_op_distance = old_swap_distance;
3025 3200192 : if (child != NULL)
3026 : {
3027 2691582 : oprnd_info->def_stmts = vNULL;
3028 2691582 : children.safe_push (child);
3029 2691582 : continue;
3030 : }
3031 :
3032 : /* If the SLP build for operand zero failed and operand zero
3033 : and one can be commuted try that for the scalar stmts
3034 : that failed the match. */
3035 508610 : if (/* A first scalar stmt mismatch signals a fatal mismatch. */
3036 508610 : matches[0]
3037 285934 : && can_swap)
3038 : {
3039 : /* See whether we can swap the matching or the non-matching
3040 : stmt operands. */
3041 : bool swap_not_matching = true;
3042 73995 : do
3043 : {
3044 7111931 : for (j = 0; j < group_size; ++j)
3045 : {
3046 7055805 : if (matches[j] != !swap_not_matching)
3047 87377 : continue;
3048 : /* Verify if we can swap operands of this stmt. */
3049 6968428 : if (!stmt_can_swap[j])
3050 : {
3051 17869 : if (!swap_not_matching)
3052 7875 : goto fail;
3053 : swap_not_matching = false;
3054 : break;
3055 : }
3056 : }
3057 : }
3058 66120 : while (j != group_size);
3059 :
3060 : /* Swap mismatched definition stmts. */
3061 56126 : if (dump_enabled_p ())
3062 401 : dump_printf_loc (MSG_NOTE, vect_location,
3063 : "Re-trying with swapped operands of stmts ");
3064 7083005 : for (j = 0; j < group_size; ++j)
3065 7026879 : if (matches[j] == !swap_not_matching)
3066 : {
3067 13900710 : std::swap (oprnds_info[0]->def_stmts[j],
3068 6950355 : oprnds_info[1]->def_stmts[j]);
3069 13900710 : std::swap (oprnds_info[0]->ops[j],
3070 6950355 : oprnds_info[1]->ops[j]);
3071 6950355 : if (dump_enabled_p ())
3072 1096 : dump_printf (MSG_NOTE, "%d ", j);
3073 : }
3074 56126 : if (dump_enabled_p ())
3075 401 : dump_printf (MSG_NOTE, "\n");
3076 : /* After swapping some operands we lost track whether an
3077 : operand has any pattern defs so be conservative here. */
3078 109087 : if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
3079 4240 : oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
3080 : /* And try again with scratch 'matches' ... */
3081 56126 : bool *tem = XALLOCAVEC (bool, group_size);
3082 56126 : if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3083 : tem, limit,
3084 : &this_tree_size, bst_map)) != NULL)
3085 : {
3086 7831 : oprnd_info->def_stmts = vNULL;
3087 7831 : children.safe_push (child);
3088 7831 : continue;
3089 : }
3090 : }
3091 444609 : fail:
3092 :
3093 : /* If the SLP build failed and we analyze a basic-block
3094 : simply treat nodes we fail to build as externally defined
3095 : (and thus build vectors from the scalar defs).
3096 : The cost model will reject outright expensive cases.
3097 : ??? This doesn't treat cases where permutation ultimatively
3098 : fails (or we don't try permutation below). Ideally we'd
3099 : even compute a permutation that will end up with the maximum
3100 : SLP tree size... */
3101 500779 : if (is_a <bb_vec_info> (vinfo)
3102 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3103 : do extra work to cancel the pattern so the uses see the
3104 : scalar version. */
3105 : /* Skip building vector operands from scalars while operand
3106 : discovery may still be fixed by retrying with swapped operands. */
3107 441957 : && (least_upthread_swappable_op_distance != 1
3108 : /* A first scalar stmt mismatch signals a fatal mismatch
3109 : that the parent commutative retry cannot recover. */
3110 27479 : || !matches[0])
3111 423013 : && !is_pattern_stmt_p (stmt_info)
3112 895260 : && !oprnd_info->any_pattern)
3113 : {
3114 : /* But if there's a leading vector sized set of matching stmts
3115 : fail here so we can split the group. This matches the condition
3116 : vect_analyze_slp_instance uses. */
3117 : /* ??? We might want to split here and combine the results to support
3118 : multiple vector sizes better. */
3119 612673 : for (j = 0; j < group_size; ++j)
3120 612673 : if (!matches[j])
3121 : break;
3122 394160 : if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
3123 394129 : && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
3124 : {
3125 387379 : if (dump_enabled_p ())
3126 706 : dump_printf_loc (MSG_NOTE, vect_location,
3127 : "Building vector operands from scalars\n");
3128 387379 : this_tree_size++;
3129 387379 : child = vect_create_new_slp_node (oprnd_info->ops);
3130 387379 : children.safe_push (child);
3131 387379 : oprnd_info->ops = vNULL;
3132 387379 : continue;
3133 : }
3134 : }
3135 :
3136 113400 : gcc_assert (child == NULL);
3137 247144 : FOR_EACH_VEC_ELT (children, j, child)
3138 20344 : if (child)
3139 20344 : vect_free_slp_tree (child);
3140 113400 : vect_free_oprnd_info (oprnds_info);
3141 113400 : return NULL;
3142 : }
3143 :
3144 3178174 : vect_free_oprnd_info (oprnds_info);
3145 :
3146 : /* If we have all children of a child built up from uniform scalars
3147 : or does more than one possibly expensive vector construction then
3148 : just throw that away, causing it built up from scalars.
3149 : The exception is the SLP node for the vector store. */
3150 3178174 : if (is_a <bb_vec_info> (vinfo)
3151 1157202 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
3152 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3153 : do extra work to cancel the pattern so the uses see the
3154 : scalar version. */
3155 3660032 : && !is_pattern_stmt_p (stmt_info))
3156 : {
3157 : slp_tree child;
3158 : unsigned j;
3159 : bool all_uniform_p = true;
3160 : unsigned n_vector_builds = 0;
3161 1344082 : FOR_EACH_VEC_ELT (children, j, child)
3162 : {
3163 896059 : if (!child)
3164 : ;
3165 896059 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
3166 : all_uniform_p = false;
3167 639813 : else if (!vect_slp_tree_uniform_p (child))
3168 : {
3169 483908 : all_uniform_p = false;
3170 483908 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
3171 443391 : n_vector_builds++;
3172 : }
3173 : }
3174 448023 : if (all_uniform_p
3175 448023 : || n_vector_builds > 1
3176 761304 : || (n_vector_builds == children.length ()
3177 35433 : && is_a <gphi *> (stmt_info->stmt)))
3178 : {
3179 : /* Roll back. */
3180 139630 : matches[0] = false;
3181 442254 : FOR_EACH_VEC_ELT (children, j, child)
3182 302624 : if (child)
3183 302624 : vect_free_slp_tree (child);
3184 :
3185 139630 : if (dump_enabled_p ())
3186 234 : dump_printf_loc (MSG_NOTE, vect_location,
3187 : "Building parent vector operands from "
3188 : "scalars instead\n");
3189 3291574 : return NULL;
3190 : }
3191 : }
3192 :
3193 3038544 : *tree_size += this_tree_size + 1;
3194 :
3195 3038544 : if (two_operators)
3196 : {
3197 : /* ??? We'd likely want to either cache in bst_map sth like
3198 : { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
3199 : the true { a+b, a+b, a+b, a+b } ... but there we don't have
3200 : explicit stmts to put in so the keying on 'stmts' doesn't
3201 : work (but we have the same issue with nodes that use 'ops'). */
3202 :
3203 7649 : if (has_two_operators_perm)
3204 : {
3205 45 : slp_tree child = children[0];
3206 45 : children.truncate (0);
3207 180 : for (i = 0; i < 2; i++)
3208 : {
3209 90 : slp_tree pnode
3210 90 : = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
3211 90 : SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
3212 90 : SLP_TREE_VECTYPE (pnode) = vectype;
3213 90 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3214 90 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3215 90 : SLP_TREE_REPRESENTATIVE (pnode) = NULL;
3216 90 : lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
3217 90 : children.safe_push (pnode);
3218 :
3219 796 : for (unsigned j = 0; j < stmts.length (); j++)
3220 616 : perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
3221 : }
3222 :
3223 45 : SLP_TREE_REF_COUNT (child) += 4;
3224 : }
3225 :
3226 7649 : slp_tree one = new _slp_tree;
3227 7649 : slp_tree two = new _slp_tree;
3228 7649 : SLP_TREE_DEF_TYPE (one) = vect_internal_def;
3229 7649 : SLP_TREE_DEF_TYPE (two) = vect_internal_def;
3230 7649 : SLP_TREE_VECTYPE (one) = vectype;
3231 7649 : SLP_TREE_VECTYPE (two) = vectype;
3232 7649 : SLP_TREE_CHILDREN (one).safe_splice (children);
3233 7649 : SLP_TREE_CHILDREN (two).safe_splice (children);
3234 7649 : slp_tree child;
3235 30598 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
3236 15300 : SLP_TREE_REF_COUNT (child)++;
3237 :
3238 : /* Here we record the original defs since this
3239 : node represents the final lane configuration. */
3240 7649 : node = vect_create_new_slp_node (node, stmts, 2);
3241 7649 : SLP_TREE_VECTYPE (node) = vectype;
3242 7649 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
3243 7649 : SLP_TREE_CHILDREN (node).quick_push (one);
3244 7649 : SLP_TREE_CHILDREN (node).quick_push (two);
3245 7649 : SLP_TREE_REPRESENTATIVE (node) = NULL;
3246 7649 : enum tree_code code0 = ERROR_MARK;
3247 7649 : enum tree_code ocode = ERROR_MARK;
3248 7649 : if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
3249 7647 : code0 = gimple_assign_rhs_code (stmt);
3250 7649 : stmt_vec_info ostmt_info;
3251 7649 : unsigned j = 0;
3252 27338 : FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
3253 : {
3254 19689 : int op = 0;
3255 19689 : if (gassign *ostmt = dyn_cast <gassign *> (ostmt_info->stmt))
3256 : {
3257 19685 : if (gimple_assign_rhs_code (ostmt) != code0)
3258 : {
3259 9863 : ocode = gimple_assign_rhs_code (ostmt);
3260 : op = 1;
3261 : j = i;
3262 : }
3263 : }
3264 : else
3265 : {
3266 8 : if (gimple_call_combined_fn (stmts[0]->stmt)
3267 4 : != gimple_call_combined_fn (ostmt_info->stmt))
3268 : {
3269 2 : op = 1;
3270 2 : j = i;
3271 : }
3272 : }
3273 19689 : SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
3274 : }
3275 7649 : SLP_TREE_CODE (one) = code0;
3276 7649 : SLP_TREE_CODE (two) = ocode;
3277 7649 : SLP_TREE_LANES (one) = stmts.length ();
3278 7649 : SLP_TREE_LANES (two) = stmts.length ();
3279 7649 : SLP_TREE_REPRESENTATIVE (one) = stmts[0];
3280 7649 : SLP_TREE_REPRESENTATIVE (two) = stmts[j];
3281 :
3282 7649 : return node;
3283 : }
3284 :
3285 3030895 : node = vect_create_new_slp_node (node, stmts, nops);
3286 3030895 : SLP_TREE_VECTYPE (node) = vectype;
3287 3030895 : SLP_TREE_CHILDREN (node).splice (children);
3288 3030895 : SLP_TREE_GS_SCALE (node) = gs_scale;
3289 3030895 : SLP_TREE_GS_BASE (node) = gs_base;
3290 3030895 : if (reduc_idx != -1)
3291 : {
3292 119065 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
3293 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3294 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
3295 119065 : SLP_TREE_REDUC_IDX (node) = reduc_idx;
3296 119065 : node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
3297 : }
3298 : /* When reaching the reduction PHI, create a vect_reduc_info. */
3299 2911830 : else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
3300 2911830 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3301 2911830 : && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
3302 : {
3303 103826 : loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
3304 103826 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3305 103826 : node->cycle_info.id = loop_vinfo->reduc_infos.length ();
3306 103826 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
3307 103826 : loop_vinfo->reduc_infos.safe_push (reduc_info);
3308 103826 : stmt_vec_info reduc_phi = stmt_info;
3309 : /* ??? For double reductions vect_is_simple_reduction stores the
3310 : reduction type and code on the inner loop header PHI. */
3311 103826 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3312 : {
3313 388 : use_operand_p use_p;
3314 388 : gimple *use_stmt;
3315 388 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
3316 : &use_p, &use_stmt);
3317 388 : gcc_assert (res);
3318 388 : reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
3319 : }
3320 103826 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
3321 103826 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
3322 103826 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
3323 103826 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
3324 : }
3325 : return node;
3326 9874722 : }
3327 :
3328 : /* Dump a single SLP tree NODE. */
3329 :
3330 : static void
3331 461940 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
3332 : slp_tree node)
3333 : {
3334 461940 : unsigned i, j;
3335 461940 : slp_tree child;
3336 461940 : stmt_vec_info stmt_info;
3337 461940 : tree op;
3338 :
3339 461940 : dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
3340 461940 : dump_user_location_t user_loc = loc.get_user_location ();
3341 461940 : dump_printf_loc (metadata, user_loc,
3342 : "node%s %p (refcnt=%u)",
3343 461940 : SLP_TREE_DEF_TYPE (node) == vect_external_def
3344 : ? " (external)"
3345 : : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
3346 445571 : ? " (constant)"
3347 : : ""), (void *) node,
3348 : SLP_TREE_REF_COUNT (node));
3349 461940 : if (SLP_TREE_VECTYPE (node))
3350 391429 : dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
3351 461940 : dump_printf (metadata, "%s",
3352 461940 : node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
3353 461940 : if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
3354 24270 : dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
3355 : node->cycle_info.reduc_idx);
3356 461940 : if (node->si)
3357 4896 : dump_printf (metadata, " @%G", node->si);
3358 : else
3359 457044 : dump_printf (metadata, "\n");
3360 461940 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3361 : {
3362 376133 : if (SLP_TREE_PERMUTE_P (node))
3363 17295 : dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
3364 : else
3365 358838 : dump_printf_loc (metadata, user_loc, "op template: %G",
3366 358838 : SLP_TREE_REPRESENTATIVE (node)->stmt);
3367 : }
3368 461940 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
3369 904172 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
3370 537075 : if (stmt_info)
3371 530953 : dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
3372 530953 : SLP_TREE_LIVE_LANES (node).contains (i)
3373 527260 : ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
3374 527260 : ? "[l] " : ""),
3375 : i, stmt_info->stmt);
3376 : else
3377 6122 : dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
3378 : else
3379 : {
3380 94843 : dump_printf_loc (metadata, user_loc, "\t{ ");
3381 208575 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
3382 113732 : dump_printf (metadata, "%T%s ", op,
3383 113732 : i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
3384 94843 : dump_printf (metadata, "}\n");
3385 : }
3386 461940 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
3387 : {
3388 66569 : dump_printf_loc (metadata, user_loc, "\tload permutation {");
3389 152290 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
3390 85721 : dump_printf (dump_kind, " %u", j);
3391 66569 : dump_printf (dump_kind, " }\n");
3392 : }
3393 461940 : if (SLP_TREE_LANE_PERMUTATION (node).exists ())
3394 : {
3395 17303 : dump_printf_loc (metadata, user_loc, "\tlane permutation {");
3396 64798 : for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
3397 47495 : dump_printf (dump_kind, " %u[%u]",
3398 47495 : SLP_TREE_LANE_PERMUTATION (node)[i].first,
3399 47495 : SLP_TREE_LANE_PERMUTATION (node)[i].second);
3400 17303 : dump_printf (dump_kind, " }%s\n",
3401 17303 : node->ldst_lanes ? " (load-lanes)" : "");
3402 : }
3403 461940 : if (SLP_TREE_CHILDREN (node).is_empty ())
3404 174749 : return;
3405 287191 : dump_printf_loc (metadata, user_loc, "\tchildren");
3406 756526 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3407 469335 : dump_printf (dump_kind, " %p", (void *)child);
3408 287191 : dump_printf (dump_kind, "%s\n",
3409 287191 : node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
3410 : ? " (store-lanes)" : "");
3411 : }
3412 :
3413 : DEBUG_FUNCTION void
3414 0 : debug (slp_tree node)
3415 : {
3416 0 : debug_dump_context ctx;
3417 0 : vect_print_slp_tree (MSG_NOTE,
3418 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3419 : node);
3420 0 : }
3421 :
3422 : /* Recursive helper for the dot producer below. */
3423 :
3424 : static void
3425 0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
3426 : {
3427 0 : if (visited.add (node))
3428 : return;
3429 :
3430 0 : fprintf (f, "\"%p\" [label=\"", (void *)node);
3431 0 : vect_print_slp_tree (MSG_NOTE,
3432 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3433 : node);
3434 0 : fprintf (f, "\"];\n");
3435 :
3436 :
3437 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3438 0 : fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
3439 :
3440 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3441 0 : if (child)
3442 0 : dot_slp_tree (f, child, visited);
3443 : }
3444 :
3445 : DEBUG_FUNCTION void
3446 0 : dot_slp_tree (const char *fname, slp_tree node)
3447 : {
3448 0 : FILE *f = fopen (fname, "w");
3449 0 : fprintf (f, "digraph {\n");
3450 0 : fflush (f);
3451 0 : {
3452 0 : debug_dump_context ctx (f);
3453 0 : hash_set<slp_tree> visited;
3454 0 : dot_slp_tree (f, node, visited);
3455 0 : }
3456 0 : fflush (f);
3457 0 : fprintf (f, "}\n");
3458 0 : fclose (f);
3459 0 : }
3460 :
3461 : DEBUG_FUNCTION void
3462 0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
3463 : {
3464 0 : FILE *f = fopen (fname, "w");
3465 0 : fprintf (f, "digraph {\n");
3466 0 : fflush (f);
3467 0 : {
3468 0 : debug_dump_context ctx (f);
3469 0 : hash_set<slp_tree> visited;
3470 0 : for (auto inst : slp_instances)
3471 0 : dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
3472 0 : }
3473 0 : fflush (f);
3474 0 : fprintf (f, "}\n");
3475 0 : fclose (f);
3476 0 : }
3477 :
3478 : /* Dump a slp tree NODE using flags specified in DUMP_KIND. */
3479 :
3480 : static void
3481 498023 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3482 : slp_tree node, hash_set<slp_tree> &visited)
3483 : {
3484 498023 : unsigned i;
3485 498023 : slp_tree child;
3486 :
3487 498023 : if (visited.add (node))
3488 498023 : return;
3489 :
3490 458108 : vect_print_slp_tree (dump_kind, loc, node);
3491 :
3492 1380435 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3493 464219 : if (child)
3494 420438 : vect_print_slp_graph (dump_kind, loc, child, visited);
3495 : }
3496 :
3497 : static void
3498 47523 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3499 : slp_tree entry)
3500 : {
3501 47523 : hash_set<slp_tree> visited;
3502 47523 : vect_print_slp_graph (dump_kind, loc, entry, visited);
3503 47523 : }
3504 :
3505 : DEBUG_FUNCTION void
3506 0 : debug (slp_instance instance)
3507 : {
3508 0 : debug_dump_context ctx;
3509 0 : vect_print_slp_graph (MSG_NOTE,
3510 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3511 : SLP_INSTANCE_TREE (instance));
3512 0 : }
3513 :
3514 :
3515 : /* Compute the set of scalar stmts participating in external nodes. */
3516 :
3517 : static void
3518 1701178 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
3519 : hash_set<slp_tree> &visited,
3520 : hash_set<stmt_vec_info> &estmts)
3521 : {
3522 1701178 : if (visited.add (node))
3523 : return;
3524 :
3525 1640142 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3526 : {
3527 : slp_tree child;
3528 : int i;
3529 1924156 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3530 984232 : if (child)
3531 984232 : vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
3532 : }
3533 : else
3534 3919008 : for (tree def : SLP_TREE_SCALAR_OPS (node))
3535 : {
3536 1820376 : stmt_vec_info def_stmt = vinfo->lookup_def (def);
3537 1820376 : if (def_stmt)
3538 390009 : estmts.add (def_stmt);
3539 : }
3540 : }
3541 :
3542 : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
3543 : with STMT_SLP_TYPE == pure_slp. */
3544 :
3545 : static void
3546 246835 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
3547 : {
3548 : /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
3549 : walk on. */
3550 246835 : hash_set<stmt_vec_info> scalar_stmts_in_externs;
3551 246835 : hash_set<slp_tree> visited;
3552 1457451 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3553 716946 : vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
3554 : visited, scalar_stmts_in_externs);
3555 :
3556 : /* DFS walk scalar stmts to compute the vectorized coverage indicated
3557 : by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
3558 : stmts. */
3559 1457451 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3560 : {
3561 903708 : for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
3562 84844 : if (!scalar_stmts_in_externs.contains (stmt))
3563 83590 : STMT_SLP_TYPE (stmt) = pure_slp;
3564 716946 : auto_vec<stmt_vec_info> worklist;
3565 4039368 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
3566 : {
3567 1888530 : stmt = vect_orig_stmt (stmt);
3568 1888530 : if (!scalar_stmts_in_externs.contains (stmt)
3569 1888530 : && STMT_SLP_TYPE (stmt) != pure_slp)
3570 : {
3571 1871641 : STMT_SLP_TYPE (stmt) = pure_slp;
3572 1871641 : worklist.safe_push (stmt);
3573 : }
3574 : }
3575 3769642 : while (!worklist.is_empty ())
3576 : {
3577 2340561 : stmt_vec_info stmt = worklist.pop ();
3578 :
3579 : /* Now walk relevant parts of the SSA use-def graph. */
3580 2340561 : slp_oprnds child_ops (stmt);
3581 7300904 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
3582 : {
3583 2619782 : tree op = child_ops.get_op_for_slp_child (stmt, i);
3584 2619782 : stmt_vec_info def = vinfo->lookup_def (op);
3585 2619782 : if (def
3586 941845 : && !scalar_stmts_in_externs.contains (def)
3587 3183766 : && STMT_SLP_TYPE (def) != pure_slp)
3588 : {
3589 468920 : STMT_SLP_TYPE (def) = pure_slp;
3590 468920 : worklist.safe_push (def);
3591 : }
3592 : }
3593 : }
3594 716946 : }
3595 246835 : }
3596 :
3597 : /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
3598 :
3599 : static void
3600 2607641 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
3601 : {
3602 2607641 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3603 : return;
3604 :
3605 1566865 : if (visited.add (node))
3606 : return;
3607 :
3608 7306031 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
3609 3261488 : if (stmt_info)
3610 : {
3611 3261488 : gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
3612 : || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
3613 3261488 : STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
3614 : }
3615 :
3616 5676979 : for (auto child: SLP_TREE_CHILDREN (node))
3617 1786476 : if (child)
3618 1786476 : vect_mark_slp_stmts_relevant (child, visited);
3619 : }
3620 :
3621 : static void
3622 821165 : vect_mark_slp_stmts_relevant (slp_tree node)
3623 : {
3624 821165 : hash_set<slp_tree> visited;
3625 821165 : vect_mark_slp_stmts_relevant (node, visited);
3626 821165 : }
3627 :
3628 :
3629 : /* Gather loads in the SLP graph NODE and populate the INST loads array. */
3630 :
3631 : static void
3632 11163541 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
3633 : hash_set<slp_tree> &visited)
3634 : {
3635 11163541 : if (!node || visited.add (node))
3636 : return;
3637 :
3638 9333850 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3639 : return;
3640 :
3641 6902527 : if (!SLP_TREE_PERMUTE_P (node))
3642 : {
3643 6688086 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
3644 6688086 : if (STMT_VINFO_DATA_REF (stmt_info)
3645 2867681 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
3646 1633851 : loads.safe_push (node);
3647 : }
3648 :
3649 6902527 : unsigned i;
3650 6902527 : slp_tree child;
3651 15736456 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3652 8833929 : vect_gather_slp_loads (loads, child, visited);
3653 : }
3654 :
3655 :
3656 : /* Find the last store in SLP INSTANCE. */
3657 :
3658 : stmt_vec_info
3659 2565180 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
3660 : {
3661 2565180 : stmt_vec_info last = NULL;
3662 14510627 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3663 6815087 : if (stmt_vinfo)
3664 : {
3665 6815087 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3666 6815087 : last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
3667 : }
3668 :
3669 2565180 : return last;
3670 : }
3671 :
3672 : /* Find the first stmt in NODE. */
3673 :
3674 : stmt_vec_info
3675 678102 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
3676 : {
3677 678102 : stmt_vec_info first = NULL;
3678 :
3679 3667093 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3680 1632787 : if (stmt_vinfo)
3681 : {
3682 1630099 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3683 1630099 : if (!first
3684 1630099 : || get_later_stmt (stmt_vinfo, first) == first)
3685 : first = stmt_vinfo;
3686 : }
3687 :
3688 678102 : return first;
3689 : }
3690 :
3691 : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
3692 : two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
3693 : (also containing the first GROUP1_SIZE stmts, since stores are
3694 : consecutive), the second containing the remainder.
3695 : Return the first stmt in the second group. */
3696 :
3697 : static stmt_vec_info
3698 163457 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
3699 : {
3700 163457 : gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
3701 163457 : gcc_assert (group1_size > 0);
3702 163457 : int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
3703 163457 : gcc_assert (group2_size > 0);
3704 163457 : DR_GROUP_SIZE (first_vinfo) = group1_size;
3705 :
3706 163457 : stmt_vec_info stmt_info = first_vinfo;
3707 545818 : for (unsigned i = group1_size; i > 1; i--)
3708 : {
3709 382361 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
3710 382361 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3711 : }
3712 : /* STMT is now the last element of the first group. */
3713 163457 : stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
3714 163457 : DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
3715 :
3716 163457 : DR_GROUP_SIZE (group2) = group2_size;
3717 456470 : for (stmt_info = group2; stmt_info;
3718 293013 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
3719 : {
3720 293013 : DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
3721 293013 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3722 : }
3723 :
3724 : /* For the second group, the DR_GROUP_GAP is that before the original group,
3725 : plus skipping over the first vector. */
3726 163457 : DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
3727 :
3728 : /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
3729 163457 : DR_GROUP_GAP (first_vinfo) += group2_size;
3730 :
3731 163457 : if (dump_enabled_p ())
3732 74 : dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
3733 : group1_size, group2_size);
3734 :
3735 163457 : return group2;
3736 : }
3737 :
3738 : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
3739 : statements and a vector of NUNITS elements. */
3740 :
3741 : static poly_uint64
3742 2866496 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
3743 : {
3744 2866496 : return exact_div (common_multiple (nunits, group_size), group_size);
3745 : }
3746 :
3747 : /* Helper that checks to see if a node is a load node. */
3748 :
3749 : static inline bool
3750 103 : vect_is_slp_load_node (slp_tree root)
3751 : {
3752 103 : return (!SLP_TREE_PERMUTE_P (root)
3753 103 : && SLP_TREE_DEF_TYPE (root) == vect_internal_def
3754 97 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
3755 167 : && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
3756 : }
3757 :
3758 :
3759 : /* Helper function of optimize_load_redistribution that performs the operation
3760 : recursively. */
3761 :
3762 : static slp_tree
3763 21799 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
3764 : vec_info *vinfo, unsigned int group_size,
3765 : hash_map<slp_tree, slp_tree> *load_map,
3766 : slp_tree root)
3767 : {
3768 21799 : if (slp_tree *leader = load_map->get (root))
3769 4549 : return *leader;
3770 :
3771 17250 : slp_tree node;
3772 17250 : unsigned i;
3773 :
3774 : /* For now, we don't know anything about externals so do not do anything. */
3775 17250 : if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
3776 : return NULL;
3777 12115 : else if (SLP_TREE_PERMUTE_P (root))
3778 : {
3779 : /* First convert this node into a load node and add it to the leaves
3780 : list and flatten the permute from a lane to a load one. If it's
3781 : unneeded it will be elided later. */
3782 71 : vec<stmt_vec_info> stmts;
3783 71 : stmts.create (SLP_TREE_LANES (root));
3784 71 : lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
3785 135 : for (unsigned j = 0; j < lane_perm.length (); j++)
3786 : {
3787 103 : std::pair<unsigned, unsigned> perm = lane_perm[j];
3788 103 : node = SLP_TREE_CHILDREN (root)[perm.first];
3789 :
3790 103 : if (!vect_is_slp_load_node (node)
3791 103 : || SLP_TREE_CHILDREN (node).exists ())
3792 : {
3793 39 : stmts.release ();
3794 39 : goto next;
3795 : }
3796 :
3797 64 : stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
3798 : }
3799 :
3800 32 : if (dump_enabled_p ())
3801 0 : dump_printf_loc (MSG_NOTE, vect_location,
3802 : "converting stmts on permute node %p\n",
3803 : (void *) root);
3804 :
3805 32 : bool *matches = XALLOCAVEC (bool, group_size);
3806 32 : unsigned tree_size = 0, limit = 1;
3807 32 : node = vect_build_slp_tree (vinfo, stmts,
3808 : matches, &limit, &tree_size, bst_map);
3809 32 : if (!node)
3810 0 : stmts.release ();
3811 :
3812 32 : load_map->put (root, node);
3813 32 : return node;
3814 : }
3815 :
3816 12044 : next:
3817 12083 : load_map->put (root, NULL);
3818 :
3819 27521 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3820 : {
3821 15438 : slp_tree value
3822 15438 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3823 : node);
3824 15438 : if (value)
3825 : {
3826 32 : SLP_TREE_REF_COUNT (value)++;
3827 32 : SLP_TREE_CHILDREN (root)[i] = value;
3828 : /* ??? We know the original leafs of the replaced nodes will
3829 : be referenced by bst_map, only the permutes created by
3830 : pattern matching are not. */
3831 32 : if (SLP_TREE_REF_COUNT (node) == 1)
3832 32 : load_map->remove (node);
3833 32 : vect_free_slp_tree (node);
3834 : }
3835 : }
3836 :
3837 : return NULL;
3838 : }
3839 :
3840 : /* Temporary workaround for loads not being CSEd during SLP build. This
3841 : function will traverse the SLP tree rooted in ROOT for INSTANCE and find
3842 : VEC_PERM nodes that blend vectors from multiple nodes that all read from the
3843 : same DR such that the final operation is equal to a permuted load. Such
3844 : NODES are then directly converted into LOADS themselves. The nodes are
3845 : CSEd using BST_MAP. */
3846 :
3847 : static void
3848 4179 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
3849 : vec_info *vinfo, unsigned int group_size,
3850 : hash_map<slp_tree, slp_tree> *load_map,
3851 : slp_tree root)
3852 : {
3853 4179 : slp_tree node;
3854 4179 : unsigned i;
3855 :
3856 10540 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3857 : {
3858 6361 : slp_tree value
3859 6361 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3860 : node);
3861 6361 : if (value)
3862 : {
3863 0 : SLP_TREE_REF_COUNT (value)++;
3864 0 : SLP_TREE_CHILDREN (root)[i] = value;
3865 : /* ??? We know the original leafs of the replaced nodes will
3866 : be referenced by bst_map, only the permutes created by
3867 : pattern matching are not. */
3868 0 : if (SLP_TREE_REF_COUNT (node) == 1)
3869 0 : load_map->remove (node);
3870 0 : vect_free_slp_tree (node);
3871 : }
3872 : }
3873 4179 : }
3874 :
3875 : /* Helper function of vect_match_slp_patterns.
3876 :
3877 : Attempts to match patterns against the slp tree rooted in REF_NODE using
3878 : VINFO. Patterns are matched in post-order traversal.
3879 :
3880 : If matching is successful the value in REF_NODE is updated and returned, if
3881 : not then it is returned unchanged. */
3882 :
3883 : static bool
3884 6483939 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
3885 : slp_tree_to_load_perm_map_t *perm_cache,
3886 : slp_compat_nodes_map_t *compat_cache,
3887 : hash_set<slp_tree> *visited)
3888 : {
3889 6483939 : unsigned i;
3890 6483939 : slp_tree node = *ref_node;
3891 6483939 : bool found_p = false;
3892 6483939 : if (!node || visited->add (node))
3893 : return false;
3894 :
3895 : slp_tree child;
3896 10415450 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3897 4857616 : found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
3898 : vinfo, perm_cache, compat_cache,
3899 : visited);
3900 :
3901 16673502 : for (unsigned x = 0; x < num__slp_patterns; x++)
3902 : {
3903 11115668 : vect_pattern *pattern
3904 11115668 : = slp_patterns[x] (perm_cache, compat_cache, ref_node);
3905 11115668 : if (pattern)
3906 : {
3907 1114 : pattern->build (vinfo);
3908 1114 : delete pattern;
3909 : found_p = true;
3910 : }
3911 : }
3912 :
3913 : return found_p;
3914 : }
3915 :
3916 : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
3917 : vec_info VINFO.
3918 :
3919 : The modified tree is returned. Patterns are tried in order and multiple
3920 : patterns may match. */
3921 :
3922 : static bool
3923 1626323 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
3924 : hash_set<slp_tree> *visited,
3925 : slp_tree_to_load_perm_map_t *perm_cache,
3926 : slp_compat_nodes_map_t *compat_cache)
3927 : {
3928 1626323 : DUMP_VECT_SCOPE ("vect_match_slp_patterns");
3929 1626323 : slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
3930 :
3931 1626323 : if (dump_enabled_p ())
3932 31329 : dump_printf_loc (MSG_NOTE, vect_location,
3933 : "Analyzing SLP tree %p for patterns\n",
3934 31329 : (void *) SLP_INSTANCE_TREE (instance));
3935 :
3936 1626323 : return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
3937 1626323 : visited);
3938 : }
3939 :
3940 : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
3941 : vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether
3942 : the stores are masked.
3943 : Return true if we could use IFN_STORE_LANES instead and if that appears
3944 : to be the better approach. */
3945 :
3946 : static bool
3947 6156 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
3948 : tree vectype, bool masked_p,
3949 : unsigned int group_size,
3950 : unsigned int new_group_size)
3951 : {
3952 6156 : if (!vectype)
3953 : {
3954 6156 : tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
3955 6156 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
3956 : }
3957 6156 : if (!vectype)
3958 : return false;
3959 : /* Allow the split if one of the two new groups would operate on full
3960 : vectors *within* rather than across one scalar loop iteration.
3961 : This is purely a heuristic, but it should work well for group
3962 : sizes of 3 and 4, where the possible splits are:
3963 :
3964 : 3->2+1: OK if the vector has exactly two elements
3965 : 4->2+2: Likewise
3966 : 4->3+1: Less clear-cut. */
3967 6156 : if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
3968 6156 : || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
3969 : return false;
3970 3480 : return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
3971 : }
3972 :
3973 : /* Analyze an SLP instance starting from a group of grouped stores. Call
3974 : vect_build_slp_tree to build a tree of packed stmts if possible.
3975 : Return FALSE if it's impossible to SLP any stmt in the loop. */
3976 :
3977 : static bool
3978 : vect_analyze_slp_instance (vec_info *vinfo,
3979 : scalar_stmts_to_slp_tree_map_t *bst_map,
3980 : stmt_vec_info stmt_info, slp_instance_kind kind,
3981 : unsigned max_tree_size, unsigned *limit,
3982 : bool force_single_lane);
3983 :
3984 : /* Build an interleaving scheme for the store sources RHS_NODES from
3985 : SCALAR_STMTS. */
3986 :
3987 : static slp_tree
3988 8259 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
3989 : vec<stmt_vec_info> &scalar_stmts)
3990 : {
3991 8259 : unsigned int group_size = scalar_stmts.length ();
3992 16518 : slp_tree node = vect_create_new_slp_node (scalar_stmts,
3993 8259 : SLP_TREE_CHILDREN
3994 : (rhs_nodes[0]).length ());
3995 8259 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
3996 8259 : for (unsigned l = 0;
3997 16545 : l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
3998 : {
3999 : /* And a permute merging all RHS SLP trees. */
4000 8286 : slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
4001 8286 : VEC_PERM_EXPR);
4002 8286 : SLP_TREE_CHILDREN (node).quick_push (perm);
4003 8286 : SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
4004 8286 : SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
4005 8286 : SLP_TREE_LANES (perm) = group_size;
4006 8286 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
4007 64412 : for (unsigned j = 0; j < rhs_nodes.length (); ++j)
4008 : {
4009 23920 : SLP_TREE_CHILDREN (perm)
4010 23920 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
4011 23920 : SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
4012 23920 : for (unsigned k = 0;
4013 50213 : k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
4014 : {
4015 : /* ??? We should populate SLP_TREE_SCALAR_STMTS
4016 : or SLP_TREE_SCALAR_OPS but then we might have
4017 : a mix of both in our children. */
4018 26293 : SLP_TREE_LANE_PERMUTATION (perm)
4019 26293 : .quick_push (std::make_pair (j, k));
4020 : }
4021 : }
4022 :
4023 : /* Now we have a single permute node but we cannot code-generate
4024 : the case with more than two inputs.
4025 : Perform pairwise reduction, reducing the two inputs
4026 : with the least number of lanes to one and then repeat until
4027 : we end up with two inputs. That scheme makes sure we end
4028 : up with permutes satisfying the restriction of requiring at
4029 : most two vector inputs to produce a single vector output
4030 : when the number of lanes is even. */
4031 15634 : while (SLP_TREE_CHILDREN (perm).length () > 2)
4032 : {
4033 : /* When we have three equal sized groups left the pairwise
4034 : reduction does not result in a scheme that avoids using
4035 : three vectors. Instead merge the first two groups
4036 : to the final size with do-not-care elements (chosen
4037 : from the first group) and then merge with the third.
4038 : { A0, B0, x, A1, B1, x, ... }
4039 : -> { A0, B0, C0, A1, B1, C1, ... }
4040 : This handles group size of three (and at least
4041 : power-of-two multiples of that). */
4042 7348 : if (SLP_TREE_CHILDREN (perm).length () == 3
4043 3390 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4044 3390 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
4045 7348 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4046 2515 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
4047 : {
4048 2209 : int ai = 0;
4049 2209 : int bi = 1;
4050 2209 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4051 2209 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4052 2209 : unsigned n = SLP_TREE_LANES (perm);
4053 :
4054 2209 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4055 2209 : SLP_TREE_LANES (permab) = n;
4056 2209 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4057 2209 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4058 : /* ??? Should be NULL but that's not expected. */
4059 2209 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4060 2209 : SLP_TREE_CHILDREN (permab).quick_push (a);
4061 6641 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4062 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4063 2223 : .quick_push (std::make_pair (0, k));
4064 2209 : SLP_TREE_CHILDREN (permab).quick_push (b);
4065 6641 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4066 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4067 2223 : .quick_push (std::make_pair (1, k));
4068 : /* Push the do-not-care lanes. */
4069 4432 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4070 2223 : SLP_TREE_LANE_PERMUTATION (permab)
4071 2223 : .quick_push (std::make_pair (0, k));
4072 :
4073 : /* Put the merged node into 'perm', in place of a. */
4074 2209 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4075 : /* Adjust the references to b in the permutation
4076 : of perm and to the later children which we'll
4077 : remove. */
4078 8878 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4079 : {
4080 6669 : std::pair<unsigned, unsigned> &p
4081 6669 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4082 6669 : if (p.first == (unsigned) bi)
4083 : {
4084 2223 : p.first = ai;
4085 2223 : p.second += SLP_TREE_LANES (a);
4086 : }
4087 4446 : else if (p.first > (unsigned) bi)
4088 2223 : p.first--;
4089 : }
4090 2209 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4091 2209 : break;
4092 : }
4093 :
4094 : /* Pick the two nodes with the least number of lanes,
4095 : prefer the earliest candidate and maintain ai < bi. */
4096 : int ai = -1;
4097 : int bi = -1;
4098 46264 : for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
4099 : {
4100 41125 : if (ai == -1)
4101 5139 : ai = ci;
4102 35986 : else if (bi == -1)
4103 5139 : bi = ci;
4104 30847 : else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4105 30847 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
4106 30847 : || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4107 25359 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
4108 : {
4109 11866 : if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
4110 5933 : <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
4111 2730 : bi = ci;
4112 : else
4113 : {
4114 3203 : ai = bi;
4115 3203 : bi = ci;
4116 : }
4117 : }
4118 : }
4119 :
4120 : /* Produce a merge of nodes ai and bi. */
4121 5139 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4122 5139 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4123 5139 : unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
4124 5139 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4125 5139 : SLP_TREE_LANES (permab) = n;
4126 5139 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4127 5139 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4128 : /* ??? Should be NULL but that's not expected. */
4129 5139 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4130 5139 : SLP_TREE_CHILDREN (permab).quick_push (a);
4131 18665 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4132 8387 : SLP_TREE_LANE_PERMUTATION (permab)
4133 8387 : .quick_push (std::make_pair (0, k));
4134 5139 : SLP_TREE_CHILDREN (permab).quick_push (b);
4135 17989 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4136 7711 : SLP_TREE_LANE_PERMUTATION (permab)
4137 7711 : .quick_push (std::make_pair (1, k));
4138 :
4139 : /* Put the merged node into 'perm', in place of a. */
4140 5139 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4141 : /* Adjust the references to b in the permutation
4142 : of perm and to the later children which we'll
4143 : remove. */
4144 73717 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4145 : {
4146 68578 : std::pair<unsigned, unsigned> &p
4147 68578 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4148 68578 : if (p.first == (unsigned) bi)
4149 : {
4150 7711 : p.first = ai;
4151 7711 : p.second += SLP_TREE_LANES (a);
4152 : }
4153 60867 : else if (p.first > (unsigned) bi)
4154 25554 : p.first--;
4155 : }
4156 5139 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4157 : }
4158 : }
4159 :
4160 8259 : return node;
4161 : }
4162 :
4163 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4164 : of KIND. Return true if successful. SCALAR_STMTS is owned by this
4165 : function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
4166 : the caller upon failure. */
4167 :
4168 : static bool
4169 2096777 : vect_build_slp_instance (vec_info *vinfo,
4170 : slp_instance_kind kind,
4171 : vec<stmt_vec_info> &scalar_stmts,
4172 : vec<stmt_vec_info> &root_stmt_infos,
4173 : vec<tree> &remain,
4174 : unsigned max_tree_size, unsigned *limit,
4175 : scalar_stmts_to_slp_tree_map_t *bst_map,
4176 : bool force_single_lane)
4177 : {
4178 : /* If there's no budget left bail out early. */
4179 2096777 : if (*limit == 0)
4180 : {
4181 16191 : scalar_stmts.release ();
4182 16191 : return false;
4183 : }
4184 :
4185 2080586 : if (kind == slp_inst_kind_ctor)
4186 : {
4187 14422 : if (dump_enabled_p ())
4188 98 : dump_printf_loc (MSG_NOTE, vect_location,
4189 : "Analyzing vectorizable constructor: %G\n",
4190 49 : root_stmt_infos[0]->stmt);
4191 : }
4192 2066164 : else if (kind == slp_inst_kind_gcond)
4193 : {
4194 292109 : if (dump_enabled_p ())
4195 5830 : dump_printf_loc (MSG_NOTE, vect_location,
4196 : "Analyzing vectorizable control flow: %G",
4197 2915 : root_stmt_infos[0]->stmt);
4198 : }
4199 1774055 : else if (kind == slp_inst_kind_bb_reduc)
4200 : {
4201 1384420 : if (dump_enabled_p ())
4202 7178 : dump_printf_loc (MSG_NOTE, vect_location,
4203 : "Analyzing vectorizable BB reduction: %G",
4204 3589 : root_stmt_infos[0]->stmt);
4205 : }
4206 :
4207 2080586 : if (dump_enabled_p ())
4208 : {
4209 26351 : dump_printf_loc (MSG_NOTE, vect_location,
4210 : "Starting SLP discovery for\n");
4211 56925 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4212 61148 : dump_printf_loc (MSG_NOTE, vect_location,
4213 30574 : " %G", scalar_stmts[i]->stmt);
4214 : }
4215 :
4216 : /* Build the tree for the SLP instance. */
4217 2080586 : unsigned int group_size = scalar_stmts.length ();
4218 2080586 : bool *matches = XALLOCAVEC (bool, group_size);
4219 2080586 : unsigned tree_size = 0;
4220 :
4221 2080586 : slp_tree node = NULL;
4222 2080586 : if (group_size > 1 && force_single_lane)
4223 : {
4224 0 : matches[0] = true;
4225 0 : matches[1] = false;
4226 : }
4227 : else
4228 2080586 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4229 : &tree_size, bst_map);
4230 2080586 : if (node != NULL)
4231 : {
4232 : /* Create a new SLP instance. */
4233 821916 : slp_instance new_instance = XNEW (class _slp_instance);
4234 821916 : SLP_INSTANCE_TREE (new_instance) = node;
4235 821916 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4236 821916 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4237 821916 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4238 821916 : SLP_INSTANCE_KIND (new_instance) = kind;
4239 821916 : new_instance->reduc_phis = NULL;
4240 821916 : new_instance->cost_vec = vNULL;
4241 821916 : new_instance->subgraph_entries = vNULL;
4242 :
4243 821916 : if (dump_enabled_p ())
4244 23144 : dump_printf_loc (MSG_NOTE, vect_location,
4245 : "SLP size %u vs. limit %u.\n",
4246 : tree_size, max_tree_size);
4247 :
4248 821916 : vinfo->slp_instances.safe_push (new_instance);
4249 :
4250 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4251 : the number of SLP lanes of the root in a few places.
4252 : Verify that assumption holds. */
4253 821916 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4254 : == group_size);
4255 :
4256 821916 : if (dump_enabled_p ())
4257 : {
4258 23144 : if (kind == slp_inst_kind_reduc_group)
4259 1470 : dump_printf_loc (MSG_NOTE, vect_location,
4260 : "SLP discovery of size %d reduction group "
4261 : "succeeded\n", group_size);
4262 23144 : dump_printf_loc (MSG_NOTE, vect_location,
4263 : "Final SLP tree for instance %p:\n",
4264 : (void *) new_instance);
4265 23144 : vect_print_slp_graph (MSG_NOTE, vect_location,
4266 : SLP_INSTANCE_TREE (new_instance));
4267 : }
4268 :
4269 821916 : return true;
4270 : }
4271 : /* Failed to SLP. */
4272 :
4273 : /* While we arrive here even with slp_inst_kind_store we should only
4274 : for group_size == 1. The code to split store groups is only in
4275 : vect_analyze_slp_instance now. */
4276 1258670 : gcc_assert (kind != slp_inst_kind_store || group_size == 1);
4277 :
4278 : /* For BB vectorization we get failures only in case of the need of
4279 : unrolling, as otherwise we'll simply get operands built from scalars.
4280 : Iff there is any mismatches in the toplevel stmts those will prevail,
4281 : otherwise we get the non-power-of-two tail of the lanes failed.
4282 : For BB reductions we mainly want to catch the first case so we pick
4283 : a more useful subset of lanes to reduce. */
4284 1258670 : if (kind == slp_inst_kind_bb_reduc && matches[0])
4285 : {
4286 : unsigned n_matching = 0;
4287 2197228 : for (unsigned i = 0; i < group_size; ++i)
4288 1553037 : if (matches[i])
4289 745123 : n_matching++;
4290 644191 : vec<stmt_vec_info> scalar_stmts2 = vNULL;
4291 : /* Try matched parts and put the rest to remain. */
4292 644191 : if (n_matching >= 2 && n_matching >= group_size / 2)
4293 : {
4294 : /* As we know the matches[] stmts match up, recursing for
4295 : non-power-of-two sizes will just force-fail the tail
4296 : for us at hopefully optimal vector size and succesfully
4297 : finish discovery. */
4298 45105 : scalar_stmts2.create (n_matching);
4299 282597 : for (unsigned i = 0; i < group_size; ++i)
4300 192387 : if (matches[i])
4301 138343 : scalar_stmts2.quick_push (scalar_stmts[i]);
4302 : else
4303 54044 : remain.safe_push
4304 57759 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4305 : }
4306 : /* Try the non-matching part. */
4307 599086 : else if (group_size - n_matching >= 2)
4308 : {
4309 : /* We do not know whether the !matches[] part matches, so avoid
4310 : cutting to a multiple of the vector size too early. We should
4311 : make progress by means of remain only growing and most of the
4312 : time prefering the matching[] part. */
4313 30105 : scalar_stmts2.create (scalar_stmts.length () - n_matching);
4314 282898 : for (unsigned i = 0; i < group_size; ++i)
4315 222688 : if (!matches[i])
4316 184889 : scalar_stmts2.quick_push (scalar_stmts[i]);
4317 : else
4318 37799 : remain.safe_push
4319 38794 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4320 : }
4321 644191 : if (scalar_stmts2.exists ())
4322 : {
4323 75210 : if (dump_enabled_p ())
4324 240 : dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d "
4325 : "non-matching lanes to scalar remains\n",
4326 120 : scalar_stmts.length () - scalar_stmts2.length ());
4327 75210 : scalar_stmts.release ();
4328 75210 : return vect_build_slp_instance (vinfo, kind, scalar_stmts2,
4329 : root_stmt_infos, remain,
4330 : max_tree_size, limit, bst_map,
4331 75210 : force_single_lane);
4332 : }
4333 : }
4334 :
4335 : /* Free the allocated memory. */
4336 1183460 : scalar_stmts.release ();
4337 :
4338 : /* Failed to SLP. */
4339 1183460 : if (dump_enabled_p ())
4340 3087 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4341 : return false;
4342 : }
4343 :
4344 : /* Analyze an SLP instance starting from a the start of a reduction chain.
4345 : Call vect_build_slp_tree to build a tree of packed stmts if possible.
4346 : Return FALSE if SLP build fails. */
4347 :
4348 : static bool
4349 73412 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
4350 : scalar_stmts_to_slp_tree_map_t *bst_map,
4351 : stmt_vec_info scalar_stmt,
4352 : unsigned max_tree_size, unsigned *limit)
4353 : {
4354 73412 : vec<stmt_vec_info> scalar_stmts = vNULL;
4355 :
4356 73412 : bool fail = false;
4357 : /* ??? We could leave operation code checking to SLP discovery. */
4358 73412 : code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
4359 : (vect_orig_stmt (scalar_stmt)));
4360 73412 : bool first = true;
4361 73412 : stmt_vec_info next_stmt = scalar_stmt;
4362 83402 : do
4363 : {
4364 83402 : stmt_vec_info stmt = next_stmt;
4365 83402 : gimple_match_op op, orig_op;
4366 83402 : if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
4367 0 : gcc_unreachable ();
4368 166804 : tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
4369 83402 : STMT_VINFO_REDUC_IDX (stmt));
4370 83402 : next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
4371 83402 : gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
4372 : || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
4373 89310 : if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)),
4374 : &orig_op))
4375 0 : gcc_unreachable ();
4376 83402 : if (CONVERT_EXPR_CODE_P (op.code)
4377 4989 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
4378 88379 : && (first
4379 2496 : || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
4380 : ;
4381 78485 : else if (code != orig_op.code)
4382 : {
4383 2728 : fail = true;
4384 2728 : break;
4385 : }
4386 : else
4387 75757 : scalar_stmts.safe_push (stmt);
4388 80674 : first = false;
4389 : }
4390 80674 : while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
4391 73412 : if (fail)
4392 2728 : return false;
4393 :
4394 : /* Remember a stmt with the actual reduction operation. */
4395 70684 : stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
4396 :
4397 : /* When the SSA def chain through reduc-idx does not form a natural
4398 : reduction chain try to linearize an associative operation manually. */
4399 70684 : if (scalar_stmts.length () == 1
4400 68061 : && code.is_tree_code ()
4401 61990 : && associative_tree_code ((tree_code)code)
4402 : /* We may not associate if a fold-left reduction is required. */
4403 131256 : && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
4404 : (reduc_scalar_stmt->stmt)),
4405 : code))
4406 : {
4407 57313 : auto_vec<chain_op_t> chain;
4408 57313 : auto_vec<std::pair<tree_code, gimple *> > worklist;
4409 57313 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
4410 57313 : if (is_a <gassign *> (scalar_stmts[0]->stmt)
4411 : /* We cannot linearize an operation that vect_slp_linearize_chain
4412 : would not put on its worklist. */
4413 57313 : && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
4414 : {
4415 56660 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4416 56660 : scalar_stmts[0]->stmt, op_stmt,
4417 : other_op_stmt,
4418 : NULL);
4419 :
4420 56660 : scalar_stmts.truncate (0);
4421 56660 : stmt_vec_info tail = NULL;
4422 283601 : for (auto el : chain)
4423 : {
4424 113915 : if (el.dt == vect_external_def
4425 113915 : || el.dt == vect_constant_def
4426 113915 : || el.code != (tree_code) code)
4427 : {
4428 294 : scalar_stmts.release ();
4429 294 : return false;
4430 : }
4431 113621 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4432 113621 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4433 111337 : || STMT_VINFO_REDUC_DEF (stmt))
4434 : {
4435 56577 : gcc_assert (tail == NULL);
4436 56577 : tail = stmt;
4437 56577 : continue;
4438 : }
4439 57044 : scalar_stmts.safe_push (stmt);
4440 : }
4441 56366 : gcc_assert (tail);
4442 : }
4443 :
4444 : /* When this linearization didn't produce a chain see if stripping
4445 : a wrapping sign conversion produces one. */
4446 57019 : if (scalar_stmts.length () == 1
4447 57019 : && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
4448 : || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
4449 : {
4450 55208 : gimple *stmt = scalar_stmts[0]->stmt;
4451 55208 : if (!is_gimple_assign (stmt)
4452 54041 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
4453 4724 : || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
4454 59932 : || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
4455 4724 : TREE_TYPE (gimple_assign_rhs1 (stmt))))
4456 : {
4457 53429 : scalar_stmts.release ();
4458 53429 : return false;
4459 : }
4460 1779 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
4461 1779 : if (!is_gimple_assign (stmt)
4462 1779 : || gimple_assign_rhs_code (stmt) != (tree_code)code)
4463 : {
4464 1760 : scalar_stmts.release ();
4465 1760 : return false;
4466 : }
4467 19 : chain.truncate (0);
4468 19 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4469 : stmt, op_stmt, other_op_stmt, NULL);
4470 :
4471 19 : scalar_stmts.truncate (0);
4472 19 : stmt_vec_info tail = NULL;
4473 93 : for (auto el : chain)
4474 : {
4475 44 : if (el.dt == vect_external_def
4476 44 : || el.dt == vect_constant_def
4477 44 : || el.code != (tree_code) code)
4478 : {
4479 8 : scalar_stmts.release ();
4480 8 : return false;
4481 : }
4482 36 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4483 36 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4484 36 : || STMT_VINFO_REDUC_DEF (stmt))
4485 : {
4486 0 : gcc_assert (tail == NULL);
4487 0 : tail = stmt;
4488 0 : continue;
4489 : }
4490 36 : scalar_stmts.safe_push (stmt);
4491 : }
4492 : /* Unlike the above this does not include the reduction SSA
4493 : cycle. */
4494 11 : gcc_assert (!tail);
4495 : }
4496 :
4497 1822 : if (scalar_stmts.length () < 2)
4498 : {
4499 1697 : scalar_stmts.release ();
4500 1697 : return false;
4501 : }
4502 :
4503 125 : if (dump_enabled_p ())
4504 : {
4505 34 : dump_printf_loc (MSG_NOTE, vect_location,
4506 : "Starting SLP discovery of reduction chain for\n");
4507 140 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4508 212 : dump_printf_loc (MSG_NOTE, vect_location,
4509 106 : " %G", scalar_stmts[i]->stmt);
4510 : }
4511 :
4512 125 : unsigned int group_size = scalar_stmts.length ();
4513 125 : bool *matches = XALLOCAVEC (bool, group_size);
4514 125 : unsigned tree_size = 0;
4515 125 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4516 125 : &tree_size, bst_map);
4517 125 : if (!node)
4518 : {
4519 47 : scalar_stmts.release ();
4520 47 : return false;
4521 : }
4522 :
4523 78 : unsigned cycle_id = vinfo->reduc_infos.length ();
4524 78 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
4525 78 : vinfo->reduc_infos.safe_push (reduc_info);
4526 78 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
4527 78 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
4528 78 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
4529 78 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
4530 78 : reduc_info->is_reduc_chain = true;
4531 :
4532 : /* Build the node for the PHI and possibly the conversions. */
4533 78 : slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
4534 78 : SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
4535 78 : phis->cycle_info.id = cycle_id;
4536 78 : SLP_TREE_LANES (phis) = group_size;
4537 78 : if (reduc_scalar_stmt == scalar_stmt)
4538 74 : SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
4539 : else
4540 4 : SLP_TREE_VECTYPE (phis)
4541 4 : = signed_or_unsigned_type_for (TYPE_UNSIGNED
4542 : (TREE_TYPE (gimple_get_lhs
4543 : (scalar_stmt->stmt))),
4544 : SLP_TREE_VECTYPE (node));
4545 : /* ??? vect_cse_slp_nodes cannot cope with cycles without any
4546 : SLP_TREE_SCALAR_STMTS. */
4547 78 : SLP_TREE_SCALAR_STMTS (phis).create (group_size);
4548 471 : for (unsigned i = 0; i < group_size; ++i)
4549 315 : SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
4550 :
4551 78 : slp_tree op_input = phis;
4552 78 : if (reduc_scalar_stmt != scalar_stmt)
4553 : {
4554 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4555 4 : SLP_TREE_REPRESENTATIVE (conv)
4556 4 : = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
4557 4 : STMT_VINFO_REDUC_IDX
4558 : (reduc_scalar_stmt)));
4559 4 : SLP_TREE_CHILDREN (conv).quick_push (phis);
4560 4 : conv->cycle_info.id = cycle_id;
4561 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4562 4 : SLP_TREE_LANES (conv) = group_size;
4563 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
4564 4 : SLP_TREE_SCALAR_STMTS (conv) = vNULL;
4565 4 : op_input = conv;
4566 : }
4567 :
4568 78 : slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
4569 78 : SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
4570 78 : SLP_TREE_CHILDREN (reduc).quick_push (op_input);
4571 78 : SLP_TREE_CHILDREN (reduc).quick_push (node);
4572 78 : reduc->cycle_info.id = cycle_id;
4573 78 : SLP_TREE_REDUC_IDX (reduc) = 0;
4574 78 : SLP_TREE_LANES (reduc) = group_size;
4575 78 : SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
4576 : /* ??? For the reduction epilogue we need a live lane. */
4577 78 : SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
4578 78 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
4579 393 : for (unsigned i = 1; i < group_size; ++i)
4580 237 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
4581 :
4582 78 : if (reduc_scalar_stmt != scalar_stmt)
4583 : {
4584 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4585 4 : SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
4586 4 : SLP_TREE_CHILDREN (conv).quick_push (reduc);
4587 4 : conv->cycle_info.id = cycle_id;
4588 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4589 4 : SLP_TREE_LANES (conv) = group_size;
4590 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
4591 : /* ??? For the reduction epilogue we need a live lane. */
4592 4 : SLP_TREE_SCALAR_STMTS (conv).create (group_size);
4593 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
4594 12 : for (unsigned i = 1; i < group_size; ++i)
4595 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
4596 4 : reduc = conv;
4597 : }
4598 :
4599 78 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
4600 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4601 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4602 78 : SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
4603 78 : SLP_TREE_REF_COUNT (reduc)++;
4604 :
4605 : /* Create a new SLP instance. */
4606 78 : slp_instance new_instance = XNEW (class _slp_instance);
4607 78 : SLP_INSTANCE_TREE (new_instance) = reduc;
4608 78 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4609 78 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4610 78 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4611 78 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4612 78 : new_instance->reduc_phis = NULL;
4613 78 : new_instance->cost_vec = vNULL;
4614 78 : new_instance->subgraph_entries = vNULL;
4615 :
4616 78 : vinfo->slp_instances.safe_push (new_instance);
4617 :
4618 78 : if (dump_enabled_p ())
4619 : {
4620 24 : dump_printf_loc (MSG_NOTE, vect_location,
4621 : "Final SLP tree for instance %p:\n",
4622 : (void *) new_instance);
4623 24 : vect_print_slp_graph (MSG_NOTE, vect_location,
4624 : SLP_INSTANCE_TREE (new_instance));
4625 : }
4626 :
4627 : return true;
4628 57313 : }
4629 :
4630 13371 : if (scalar_stmts.length () <= 1)
4631 : {
4632 10748 : scalar_stmts.release ();
4633 10748 : return false;
4634 : }
4635 :
4636 2623 : scalar_stmts.reverse ();
4637 2623 : stmt_vec_info reduc_phi_info = next_stmt;
4638 :
4639 : /* Build the tree for the SLP instance. */
4640 2623 : vec<stmt_vec_info> root_stmt_infos = vNULL;
4641 2623 : vec<tree> remain = vNULL;
4642 :
4643 2623 : if (dump_enabled_p ())
4644 : {
4645 196 : dump_printf_loc (MSG_NOTE, vect_location,
4646 : "Starting SLP discovery of reduction chain for\n");
4647 1038 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4648 1684 : dump_printf_loc (MSG_NOTE, vect_location,
4649 842 : " %G", scalar_stmts[i]->stmt);
4650 : }
4651 :
4652 : /* Build the tree for the SLP instance. */
4653 2623 : unsigned int group_size = scalar_stmts.length ();
4654 2623 : bool *matches = XALLOCAVEC (bool, group_size);
4655 2623 : unsigned tree_size = 0;
4656 :
4657 : /* ??? We need this only for SLP discovery. */
4658 10237 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4659 7614 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
4660 :
4661 2623 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4662 2623 : &tree_size, bst_map);
4663 :
4664 10237 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4665 7614 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
4666 :
4667 2623 : if (node != NULL)
4668 : {
4669 : /* Create a new SLP instance. */
4670 2339 : slp_instance new_instance = XNEW (class _slp_instance);
4671 2339 : SLP_INSTANCE_TREE (new_instance) = node;
4672 2339 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4673 2339 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4674 2339 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4675 2339 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4676 2339 : new_instance->reduc_phis = NULL;
4677 2339 : new_instance->cost_vec = vNULL;
4678 2339 : new_instance->subgraph_entries = vNULL;
4679 :
4680 2339 : vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
4681 2339 : reduc_info->is_reduc_chain = true;
4682 :
4683 2339 : if (dump_enabled_p ())
4684 147 : dump_printf_loc (MSG_NOTE, vect_location,
4685 : "SLP size %u vs. limit %u.\n",
4686 : tree_size, max_tree_size);
4687 :
4688 : /* Fixup SLP reduction chains. If this is a reduction chain with
4689 : a conversion in front amend the SLP tree with a node for that. */
4690 2339 : gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
4691 2339 : if (is_gimple_assign (scalar_def)
4692 2339 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
4693 : {
4694 43 : stmt_vec_info conv_info = vect_stmt_to_vectorize
4695 43 : (STMT_VINFO_REDUC_DEF (reduc_phi_info));
4696 43 : scalar_stmts = vNULL;
4697 43 : scalar_stmts.create (group_size);
4698 178 : for (unsigned i = 0; i < group_size; ++i)
4699 92 : scalar_stmts.quick_push (conv_info);
4700 43 : slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
4701 43 : SLP_TREE_VECTYPE (conv)
4702 43 : = get_vectype_for_scalar_type (vinfo,
4703 43 : TREE_TYPE
4704 : (gimple_assign_lhs (scalar_def)),
4705 : group_size);
4706 43 : SLP_TREE_REDUC_IDX (conv) = 0;
4707 43 : conv->cycle_info.id = node->cycle_info.id;
4708 43 : SLP_TREE_CHILDREN (conv).quick_push (node);
4709 43 : SLP_INSTANCE_TREE (new_instance) = conv;
4710 : }
4711 : /* Fill the backedge child of the PHI SLP node. The
4712 : general matching code cannot find it because the
4713 : scalar code does not reflect how we vectorize the
4714 : reduction. */
4715 2339 : use_operand_p use_p;
4716 2339 : imm_use_iterator imm_iter;
4717 2339 : class loop *loop = LOOP_VINFO_LOOP (vinfo);
4718 8896 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
4719 : gimple_get_lhs (scalar_def))
4720 : /* There are exactly two non-debug uses, the reduction
4721 : PHI and the loop-closed PHI node. */
4722 6557 : if (!is_gimple_debug (USE_STMT (use_p))
4723 6557 : && gimple_bb (USE_STMT (use_p)) == loop->header)
4724 : {
4725 2339 : auto_vec<stmt_vec_info, 64> phis (group_size);
4726 2339 : stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
4727 9223 : for (unsigned i = 0; i < group_size; ++i)
4728 6884 : phis.quick_push (phi_info);
4729 2339 : slp_tree *phi_node = bst_map->get (phis);
4730 2339 : unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
4731 4678 : SLP_TREE_CHILDREN (*phi_node)[dest_idx]
4732 2339 : = SLP_INSTANCE_TREE (new_instance);
4733 2339 : SLP_INSTANCE_TREE (new_instance)->refcnt++;
4734 2339 : }
4735 :
4736 2339 : vinfo->slp_instances.safe_push (new_instance);
4737 :
4738 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4739 : the number of SLP lanes of the root in a few places.
4740 : Verify that assumption holds. */
4741 2339 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4742 : == group_size);
4743 :
4744 2339 : if (dump_enabled_p ())
4745 : {
4746 147 : dump_printf_loc (MSG_NOTE, vect_location,
4747 : "Final SLP tree for instance %p:\n",
4748 : (void *) new_instance);
4749 147 : vect_print_slp_graph (MSG_NOTE, vect_location,
4750 : SLP_INSTANCE_TREE (new_instance));
4751 : }
4752 :
4753 2339 : return true;
4754 : }
4755 :
4756 : /* Failed to SLP. */
4757 284 : scalar_stmts.release ();
4758 284 : if (dump_enabled_p ())
4759 49 : dump_printf_loc (MSG_NOTE, vect_location,
4760 : "SLP discovery of reduction chain failed\n");
4761 : return false;
4762 : }
4763 :
4764 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4765 : of KIND. Return true if successful. */
4766 :
4767 : static bool
4768 100306 : vect_analyze_slp_reduction (loop_vec_info vinfo,
4769 : stmt_vec_info scalar_stmt,
4770 : unsigned max_tree_size, unsigned *limit,
4771 : scalar_stmts_to_slp_tree_map_t *bst_map,
4772 : bool force_single_lane)
4773 : {
4774 100306 : slp_instance_kind kind = slp_inst_kind_reduc_group;
4775 :
4776 : /* Try to gather a reduction chain. Only attempt if there's budget left
4777 : since chain analysis may build multi-lane trees that consume limit. */
4778 100306 : if (! force_single_lane
4779 73697 : && *limit != 0
4780 73697 : && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
4781 173718 : && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
4782 : max_tree_size, limit))
4783 : return true;
4784 :
4785 97889 : vec<stmt_vec_info> scalar_stmts;
4786 97889 : scalar_stmts.create (1);
4787 97889 : scalar_stmts.quick_push (scalar_stmt);
4788 :
4789 97889 : if (dump_enabled_p ())
4790 : {
4791 3904 : dump_printf_loc (MSG_NOTE, vect_location,
4792 : "Starting SLP discovery for\n");
4793 7808 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4794 7808 : dump_printf_loc (MSG_NOTE, vect_location,
4795 3904 : " %G", scalar_stmts[i]->stmt);
4796 : }
4797 :
4798 : /* Build the tree for the SLP instance. */
4799 97889 : unsigned int group_size = scalar_stmts.length ();
4800 97889 : bool *matches = XALLOCAVEC (bool, group_size);
4801 97889 : unsigned tree_size = 0;
4802 :
4803 97889 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4804 : &tree_size, bst_map);
4805 97889 : if (node != NULL)
4806 : {
4807 : /* Create a new SLP instance. */
4808 95989 : slp_instance new_instance = XNEW (class _slp_instance);
4809 95989 : SLP_INSTANCE_TREE (new_instance) = node;
4810 95989 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4811 95989 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4812 95989 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4813 95989 : SLP_INSTANCE_KIND (new_instance) = kind;
4814 95989 : new_instance->reduc_phis = NULL;
4815 95989 : new_instance->cost_vec = vNULL;
4816 95989 : new_instance->subgraph_entries = vNULL;
4817 :
4818 95989 : if (dump_enabled_p ())
4819 3823 : dump_printf_loc (MSG_NOTE, vect_location,
4820 : "SLP size %u vs. limit %u.\n",
4821 : tree_size, max_tree_size);
4822 :
4823 95989 : vinfo->slp_instances.safe_push (new_instance);
4824 :
4825 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4826 : the number of SLP lanes of the root in a few places.
4827 : Verify that assumption holds. */
4828 95989 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4829 : == group_size);
4830 :
4831 95989 : if (dump_enabled_p ())
4832 : {
4833 3823 : dump_printf_loc (MSG_NOTE, vect_location,
4834 : "Final SLP tree for instance %p:\n",
4835 : (void *) new_instance);
4836 3823 : vect_print_slp_graph (MSG_NOTE, vect_location,
4837 : SLP_INSTANCE_TREE (new_instance));
4838 : }
4839 :
4840 95989 : return true;
4841 : }
4842 : /* Failed to SLP. */
4843 :
4844 : /* Free the allocated memory. */
4845 1900 : scalar_stmts.release ();
4846 :
4847 : /* Failed to SLP. */
4848 1900 : if (dump_enabled_p ())
4849 81 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4850 : return false;
4851 : }
4852 :
4853 : /* Analyze a single SLP reduction group. If successful add a SLP instance
4854 : for it and return true, otherwise return false and have *MATCHES
4855 : populated. */
4856 :
4857 : static bool
4858 24238 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
4859 : vec<stmt_vec_info> scalar_stmts,
4860 : scalar_stmts_to_slp_tree_map_t *bst_map,
4861 : unsigned max_tree_size, unsigned *limit,
4862 : bool *matches)
4863 : {
4864 : /* Try to form a reduction group. Size-1 groups are not suitable
4865 : for SLP reduction and should fall back to single-lane reduction. */
4866 24238 : unsigned int group_size = scalar_stmts.length ();
4867 24238 : if (group_size <= 1)
4868 : return false;
4869 17511 : if (!matches)
4870 4550 : matches = XALLOCAVEC (bool, group_size);
4871 17511 : unsigned tree_size = 0;
4872 17511 : slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts, matches, limit,
4873 : &tree_size, bst_map);
4874 17511 : if (!node)
4875 : return false;
4876 :
4877 : /* Create a new SLP instance. */
4878 2756 : slp_instance new_instance = XNEW (class _slp_instance);
4879 2756 : SLP_INSTANCE_TREE (new_instance) = node;
4880 2756 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4881 2756 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4882 2756 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4883 2756 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
4884 2756 : new_instance->reduc_phis = NULL;
4885 2756 : new_instance->cost_vec = vNULL;
4886 2756 : new_instance->subgraph_entries = vNULL;
4887 :
4888 2756 : if (dump_enabled_p ())
4889 203 : dump_printf_loc (MSG_NOTE, vect_location,
4890 : "SLP size %u vs. limit %u.\n",
4891 : tree_size, max_tree_size);
4892 :
4893 2756 : loop_vinfo->slp_instances.safe_push (new_instance);
4894 :
4895 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4896 : the number of SLP lanes of the root in a few places.
4897 : Verify that assumption holds. */
4898 2756 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4899 : == group_size);
4900 :
4901 2756 : if (dump_enabled_p ())
4902 : {
4903 203 : dump_printf_loc (MSG_NOTE, vect_location,
4904 : "SLP discovery of size %d reduction group "
4905 : "succeeded\n", group_size);
4906 203 : dump_printf_loc (MSG_NOTE, vect_location,
4907 : "Final SLP tree for instance %p:\n",
4908 : (void *) new_instance);
4909 203 : vect_print_slp_graph (MSG_NOTE, vect_location,
4910 : SLP_INSTANCE_TREE (new_instance));
4911 : }
4912 :
4913 : return true;
4914 : }
4915 :
4916 : /* Analyze reductions in LOOP_VINFO and populate SLP instances
4917 : accordingly. Returns false if something fails. */
4918 :
4919 : static bool
4920 514644 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
4921 : unsigned max_tree_size, unsigned *limit,
4922 : scalar_stmts_to_slp_tree_map_t *bst_map,
4923 : bool force_single_lane)
4924 : {
4925 514644 : if (loop_vinfo->reductions.is_empty ())
4926 : return true;
4927 :
4928 : /* Collect reduction statements we can combine into
4929 : a SLP reduction. */
4930 74456 : vec<stmt_vec_info> scalar_stmts;
4931 74456 : scalar_stmts.create (loop_vinfo->reductions.length ());
4932 330054 : for (auto next_info : loop_vinfo->reductions)
4933 : {
4934 106686 : next_info = vect_stmt_to_vectorize (next_info);
4935 106686 : if ((STMT_VINFO_RELEVANT_P (next_info)
4936 14 : || STMT_VINFO_LIVE_P (next_info))
4937 : /* ??? Make sure we didn't skip a conversion around a
4938 : reduction path. In that case we'd have to reverse
4939 : engineer that conversion stmt following the chain using
4940 : reduc_idx and from the PHI using reduc_def. */
4941 106672 : && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
4942 106672 : || (STMT_VINFO_DEF_TYPE (next_info)
4943 : == vect_double_reduction_def)))
4944 : {
4945 : /* Do not discover SLP reductions combining lane-reducing
4946 : ops, that will fail later. */
4947 106672 : if (!force_single_lane
4948 106672 : && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
4949 79366 : scalar_stmts.quick_push (next_info);
4950 : /* Do SLP discovery for single-lane reductions. */
4951 27306 : else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
4952 : max_tree_size, limit,
4953 : bst_map,
4954 : force_single_lane))
4955 : {
4956 0 : scalar_stmts.release ();
4957 0 : return false;
4958 : }
4959 : }
4960 : }
4961 :
4962 74456 : if (scalar_stmts.length () > 1)
4963 : {
4964 : /* Try to form a reduction group. */
4965 4660 : unsigned int group_size = scalar_stmts.length ();
4966 4660 : bool *matches = XALLOCAVEC (bool, group_size);
4967 4660 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
4968 : max_tree_size, limit, matches))
4969 1571 : return true;
4970 :
4971 : /* When analysis as a single SLP reduction group failed try to
4972 : form sub-groups by collecting matching lanes. Do not recurse
4973 : that on failure (to limit compile-time costs), but recurse
4974 : for the initial non-matching parts. Everything not covered
4975 : by a sub-group gets single-reduction treatment. */
4976 3541 : vec<stmt_vec_info> cands = vNULL;
4977 11412 : while (matches[0])
4978 : {
4979 11277 : cands.truncate (0);
4980 11277 : cands.reserve (group_size, true);
4981 99784 : for (unsigned i = 0; i < group_size; ++i)
4982 77230 : if (matches[i])
4983 19610 : cands.quick_push (scalar_stmts[i]);
4984 :
4985 : /* Try to form a reduction group. */
4986 11277 : if (vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
4987 : max_tree_size, limit, NULL))
4988 1207 : cands = vNULL;
4989 : else
4990 : {
4991 : /* Do SLP discovery for single-lane reductions. */
4992 47356 : for (auto stmt_info : cands)
4993 17168 : if (! vect_analyze_slp_reduction (loop_vinfo,
4994 : vect_stmt_to_vectorize
4995 : (stmt_info),
4996 : max_tree_size, limit,
4997 : bst_map, force_single_lane))
4998 : {
4999 22 : scalar_stmts.release ();
5000 22 : cands.release ();
5001 22 : return false;
5002 : }
5003 : }
5004 : /* Remove the handled stmts from scalar_stmts and try again,
5005 : possibly repeating the above with updated matches[]. */
5006 11255 : unsigned j = 0;
5007 88423 : for (unsigned i = 0; i < group_size; ++i)
5008 77168 : if (!matches[i])
5009 : {
5010 57593 : scalar_stmts[j] = scalar_stmts[i];
5011 57593 : ++j;
5012 : }
5013 11255 : scalar_stmts.truncate (j);
5014 11390 : group_size = scalar_stmts.length ();
5015 11255 : if (group_size <= 1)
5016 : break;
5017 8301 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts,
5018 : bst_map, max_tree_size, limit,
5019 : matches))
5020 : return true;
5021 : }
5022 : }
5023 : /* Do SLP discovery for single-lane reductions. */
5024 272609 : for (auto stmt_info : scalar_stmts)
5025 55832 : if (! vect_analyze_slp_reduction (loop_vinfo,
5026 : vect_stmt_to_vectorize (stmt_info),
5027 : max_tree_size, limit,
5028 : bst_map, force_single_lane))
5029 : {
5030 1878 : scalar_stmts.release ();
5031 1878 : return false;
5032 : }
5033 :
5034 71007 : scalar_stmts.release ();
5035 71007 : return true;
5036 : }
5037 :
5038 : /* Analyze an SLP instance starting from a group of grouped stores. Call
5039 : vect_build_slp_tree to build a tree of packed stmts if possible.
5040 : Return FALSE if it's impossible to SLP any stmt in the group. */
5041 :
5042 : static bool
5043 1125000 : vect_analyze_slp_instance (vec_info *vinfo,
5044 : scalar_stmts_to_slp_tree_map_t *bst_map,
5045 : stmt_vec_info stmt_info,
5046 : slp_instance_kind kind,
5047 : unsigned max_tree_size, unsigned *limit,
5048 : bool force_single_lane)
5049 : {
5050 1125000 : vec<stmt_vec_info> scalar_stmts;
5051 :
5052 1125000 : if (is_a <bb_vec_info> (vinfo))
5053 1094816 : vect_location = stmt_info->stmt;
5054 :
5055 1125000 : gcc_assert (kind == slp_inst_kind_store);
5056 :
5057 : /* Collect the stores and store them in scalar_stmts. */
5058 1125000 : scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
5059 1125000 : stmt_vec_info next_info = stmt_info;
5060 5595552 : while (next_info)
5061 : {
5062 3345552 : scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
5063 3345552 : next_info = DR_GROUP_NEXT_ELEMENT (next_info);
5064 : }
5065 :
5066 1125000 : vec<stmt_vec_info> root_stmt_infos = vNULL;
5067 1125000 : vec<tree> remain = vNULL;
5068 :
5069 : /* Build the tree for the SLP instance. */
5070 :
5071 : /* If there's no budget left bail out early. */
5072 1125000 : if (*limit == 0)
5073 : return false;
5074 :
5075 1124980 : if (dump_enabled_p ())
5076 : {
5077 4183 : dump_printf_loc (MSG_NOTE, vect_location,
5078 : "Starting SLP discovery for\n");
5079 24392 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
5080 40418 : dump_printf_loc (MSG_NOTE, vect_location,
5081 20209 : " %G", scalar_stmts[i]->stmt);
5082 : }
5083 :
5084 : /* Build the tree for the SLP instance. */
5085 1124980 : unsigned int group_size = scalar_stmts.length ();
5086 1124980 : bool *matches = XALLOCAVEC (bool, group_size);
5087 1124980 : unsigned tree_size = 0;
5088 1124980 : unsigned i;
5089 :
5090 1124980 : slp_tree node = NULL;
5091 1124980 : if (group_size > 1 && force_single_lane)
5092 : {
5093 1774 : matches[0] = true;
5094 1774 : matches[1] = false;
5095 : }
5096 : else
5097 1123206 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
5098 : &tree_size, bst_map);
5099 1124980 : if (node != NULL)
5100 : {
5101 : /* Create a new SLP instance. */
5102 697140 : slp_instance new_instance = XNEW (class _slp_instance);
5103 697140 : SLP_INSTANCE_TREE (new_instance) = node;
5104 697140 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5105 697140 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5106 697140 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5107 697140 : SLP_INSTANCE_KIND (new_instance) = kind;
5108 697140 : new_instance->reduc_phis = NULL;
5109 697140 : new_instance->cost_vec = vNULL;
5110 697140 : new_instance->subgraph_entries = vNULL;
5111 :
5112 697140 : if (dump_enabled_p ())
5113 3166 : dump_printf_loc (MSG_NOTE, vect_location,
5114 : "SLP size %u vs. limit %u.\n",
5115 : tree_size, max_tree_size);
5116 :
5117 697140 : vinfo->slp_instances.safe_push (new_instance);
5118 :
5119 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5120 : the number of SLP lanes of the root in a few places.
5121 : Verify that assumption holds. */
5122 697140 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5123 : == group_size);
5124 :
5125 697140 : if (dump_enabled_p ())
5126 : {
5127 3166 : dump_printf_loc (MSG_NOTE, vect_location,
5128 : "Final SLP tree for instance %p:\n",
5129 : (void *) new_instance);
5130 3166 : vect_print_slp_graph (MSG_NOTE, vect_location,
5131 : SLP_INSTANCE_TREE (new_instance));
5132 : }
5133 697140 : return true;
5134 : }
5135 : /* Failed to SLP. */
5136 :
5137 : /* Try to break the group up into pieces. */
5138 427840 : if (*limit > 0 && kind == slp_inst_kind_store)
5139 : {
5140 : /* ??? We could delay all the actual splitting of store-groups
5141 : until after SLP discovery of the original group completed.
5142 : Then we can recurse to vect_build_slp_instance directly. */
5143 1119999 : for (i = 0; i < group_size; i++)
5144 1119999 : if (!matches[i])
5145 : break;
5146 :
5147 : /* For basic block SLP, try to break the group up into multiples of
5148 : a vector size. */
5149 427839 : if (is_a <bb_vec_info> (vinfo)
5150 427839 : && (i > 1 && i < group_size))
5151 : {
5152 : /* Free the allocated memory. */
5153 160826 : scalar_stmts.release ();
5154 :
5155 160826 : tree scalar_type
5156 160826 : = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
5157 321652 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
5158 160826 : 1 << floor_log2 (i));
5159 160826 : unsigned HOST_WIDE_INT const_nunits;
5160 160826 : if (vectype
5161 160826 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
5162 : {
5163 : /* Split into two groups at the first vector boundary. */
5164 160826 : gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
5165 160826 : unsigned group1_size = i & ~(const_nunits - 1);
5166 :
5167 160826 : if (dump_enabled_p ())
5168 70 : dump_printf_loc (MSG_NOTE, vect_location,
5169 : "Splitting SLP group at stmt %u\n", i);
5170 160826 : stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
5171 : group1_size);
5172 160826 : bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
5173 : kind, max_tree_size,
5174 : limit, false);
5175 : /* Split the rest at the failure point and possibly
5176 : re-analyze the remaining matching part if it has
5177 : at least two lanes. */
5178 160826 : if (group1_size < i
5179 5715 : && (i + 1 < group_size
5180 3112 : || i - group1_size > 1))
5181 : {
5182 2631 : stmt_vec_info rest2 = rest;
5183 2631 : rest = vect_split_slp_store_group (rest, i - group1_size);
5184 2631 : if (i - group1_size > 1)
5185 57 : res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
5186 : kind, max_tree_size,
5187 : limit, false);
5188 : }
5189 : /* Re-analyze the non-matching tail if it has at least
5190 : two lanes. */
5191 160826 : if (i + 1 < group_size)
5192 22872 : res |= vect_analyze_slp_instance (vinfo, bst_map,
5193 : rest, kind, max_tree_size,
5194 : limit, false);
5195 1125000 : return res;
5196 : }
5197 : }
5198 :
5199 : /* For loop vectorization split the RHS into arbitrary pieces of
5200 : size >= 1. */
5201 267013 : else if (is_a <loop_vec_info> (vinfo)
5202 267013 : && (group_size != 1 && i < group_size))
5203 : {
5204 8377 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
5205 28 : bool masked_p = call
5206 28 : && gimple_call_internal_p (call)
5207 28 : && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
5208 : /* There are targets that cannot do even/odd interleaving schemes
5209 : so they absolutely need to use load/store-lanes. For now
5210 : force single-lane SLP for them - they would be happy with
5211 : uniform power-of-two lanes (but depending on element size),
5212 : but even if we can use 'i' as indicator we would need to
5213 : backtrack when later lanes fail to discover with the same
5214 : granularity. We cannot turn any of strided or scatter store
5215 : into store-lanes. */
5216 : /* ??? If this is not in sync with what get_load_store_type
5217 : later decides the SLP representation is not good for other
5218 : store vectorization methods. */
5219 8377 : bool want_store_lanes
5220 8377 : = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5221 8377 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5222 6265 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5223 6261 : && compare_step_with_zero (vinfo, stmt_info) > 0
5224 14533 : && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
5225 16754 : masked_p, group_size, i));
5226 8377 : if (want_store_lanes || force_single_lane)
5227 : i = 1;
5228 :
5229 : /* A fatal discovery fail doesn't always mean single-lane SLP
5230 : isn't a possibility, so try. */
5231 6603 : if (i == 0)
5232 : i = 1;
5233 :
5234 8377 : if (dump_enabled_p ())
5235 891 : dump_printf_loc (MSG_NOTE, vect_location,
5236 : "Splitting SLP group at stmt %u\n", i);
5237 :
5238 : /* Analyze the stored values and pinch them together with
5239 : a permute node so we can preserve the whole store group. */
5240 8377 : auto_vec<slp_tree> rhs_nodes;
5241 :
5242 8377 : unsigned int rhs_common_nlanes = 0;
5243 8377 : unsigned int start = 0, end = i;
5244 37923 : while (start < group_size)
5245 : {
5246 29664 : gcc_assert (end - start >= 1);
5247 29664 : vec<stmt_vec_info> substmts;
5248 29664 : substmts.create (end - start);
5249 121529 : for (unsigned j = start; j < end; ++j)
5250 62201 : substmts.quick_push (scalar_stmts[j]);
5251 29664 : node = vect_build_slp_tree (vinfo, substmts,
5252 : matches, limit, &tree_size, bst_map);
5253 29664 : if (node)
5254 : {
5255 23857 : rhs_nodes.safe_push (node);
5256 23857 : if (start == 0)
5257 8259 : rhs_common_nlanes = SLP_TREE_LANES (node);
5258 15598 : else if (rhs_common_nlanes != SLP_TREE_LANES (node))
5259 1413 : rhs_common_nlanes = 0;
5260 23857 : start = end;
5261 23857 : if (want_store_lanes || force_single_lane)
5262 5327 : end = start + 1;
5263 : else
5264 : end = group_size;
5265 : }
5266 : else
5267 : {
5268 5807 : substmts.release ();
5269 5807 : if (end - start == 1)
5270 : {
5271 : /* Single-lane discovery failed. Free resources. */
5272 118 : for (auto node : rhs_nodes)
5273 0 : vect_free_slp_tree (node);
5274 118 : scalar_stmts.release ();
5275 118 : if (dump_enabled_p ())
5276 17 : dump_printf_loc (MSG_NOTE, vect_location,
5277 : "SLP discovery failed\n");
5278 118 : return false;
5279 : }
5280 :
5281 : /* ??? It really happens that we soft-fail SLP
5282 : build at a mismatch but the matching part hard-fails
5283 : later. As we know we arrived here with a group
5284 : larger than one try a group of size one! */
5285 5689 : if (!matches[0])
5286 12 : end = start + 1;
5287 : else
5288 12459 : for (unsigned j = start; j < end; j++)
5289 12459 : if (!matches[j - start])
5290 : {
5291 : end = j;
5292 : break;
5293 : }
5294 : }
5295 : }
5296 :
5297 : /* Now re-assess whether we want store lanes in case the
5298 : discovery ended up producing all single-lane RHSs. */
5299 8259 : if (! want_store_lanes
5300 8259 : && rhs_common_nlanes == 1
5301 7179 : && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5302 7179 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5303 5441 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5304 5438 : && compare_step_with_zero (vinfo, stmt_info) > 0
5305 13610 : && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
5306 : group_size, masked_p)
5307 : != IFN_LAST))
5308 : want_store_lanes = true;
5309 :
5310 : /* Now we assume we can build the root SLP node from all stores. */
5311 8259 : if (want_store_lanes)
5312 : {
5313 : /* For store-lanes feed the store node with all RHS nodes
5314 : in order. */
5315 0 : node = vect_create_new_slp_node (scalar_stmts,
5316 0 : SLP_TREE_CHILDREN
5317 : (rhs_nodes[0]).length ());
5318 0 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
5319 0 : node->ldst_lanes = true;
5320 0 : SLP_TREE_CHILDREN (node)
5321 0 : .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
5322 0 : + rhs_nodes.length () - 1);
5323 : /* First store value and possibly mask. */
5324 0 : SLP_TREE_CHILDREN (node)
5325 0 : .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
5326 : /* Rest of the store values. All mask nodes are the same,
5327 : this should be guaranteed by dataref group discovery. */
5328 0 : for (unsigned j = 1; j < rhs_nodes.length (); ++j)
5329 0 : SLP_TREE_CHILDREN (node)
5330 0 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
5331 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
5332 0 : child->refcnt++;
5333 : }
5334 : else
5335 8259 : node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts);
5336 :
5337 32116 : while (!rhs_nodes.is_empty ())
5338 23857 : vect_free_slp_tree (rhs_nodes.pop ());
5339 :
5340 : /* Create a new SLP instance. */
5341 8259 : slp_instance new_instance = XNEW (class _slp_instance);
5342 8259 : SLP_INSTANCE_TREE (new_instance) = node;
5343 8259 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5344 8259 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5345 8259 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5346 8259 : SLP_INSTANCE_KIND (new_instance) = kind;
5347 8259 : new_instance->reduc_phis = NULL;
5348 8259 : new_instance->cost_vec = vNULL;
5349 8259 : new_instance->subgraph_entries = vNULL;
5350 :
5351 8259 : if (dump_enabled_p ())
5352 874 : dump_printf_loc (MSG_NOTE, vect_location,
5353 : "SLP size %u vs. limit %u.\n",
5354 : tree_size, max_tree_size);
5355 :
5356 8259 : vinfo->slp_instances.safe_push (new_instance);
5357 :
5358 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5359 : the number of SLP lanes of the root in a few places.
5360 : Verify that assumption holds. */
5361 8259 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5362 : == group_size);
5363 :
5364 8259 : if (dump_enabled_p ())
5365 : {
5366 874 : dump_printf_loc (MSG_NOTE, vect_location,
5367 : "Final SLP tree for instance %p:\n",
5368 : (void *) new_instance);
5369 874 : vect_print_slp_graph (MSG_NOTE, vect_location,
5370 : SLP_INSTANCE_TREE (new_instance));
5371 : }
5372 : return true;
5373 8377 : }
5374 : else
5375 : /* Free the allocated memory. */
5376 258636 : scalar_stmts.release ();
5377 :
5378 : /* Even though the first vector did not all match, we might be able to SLP
5379 : (some) of the remainder. FORNOW ignore this possibility. */
5380 : }
5381 : else
5382 : /* Free the allocated memory. */
5383 1 : scalar_stmts.release ();
5384 :
5385 : /* Failed to SLP. */
5386 258637 : if (dump_enabled_p ())
5387 56 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
5388 : return false;
5389 : }
5390 :
5391 : /* qsort comparator ordering SLP load nodes. */
5392 :
5393 : static int
5394 2691208 : vllp_cmp (const void *a_, const void *b_)
5395 : {
5396 2691208 : const slp_tree a = *(const slp_tree *)a_;
5397 2691208 : const slp_tree b = *(const slp_tree *)b_;
5398 2691208 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
5399 2691208 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
5400 2691208 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5401 1545919 : && STMT_VINFO_GROUPED_ACCESS (b0)
5402 4175059 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5403 : {
5404 : /* Same group, order after lanes used. */
5405 348323 : if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
5406 : return 1;
5407 339350 : else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
5408 : return -1;
5409 : else
5410 : {
5411 : /* Try to order loads using the same lanes together, breaking
5412 : the tie with the lane number that first differs. */
5413 329613 : if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5414 329613 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5415 : return 0;
5416 329613 : else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
5417 329613 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5418 : return 1;
5419 325550 : else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5420 325550 : && SLP_TREE_LOAD_PERMUTATION (b).exists ())
5421 : return -1;
5422 : else
5423 : {
5424 318064 : for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
5425 318064 : if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5426 318064 : != SLP_TREE_LOAD_PERMUTATION (b)[i])
5427 : {
5428 : /* In-order lane first, that's what the above case for
5429 : no permutation does. */
5430 316752 : if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
5431 : return -1;
5432 194195 : else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
5433 : return 1;
5434 102102 : else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5435 102102 : < SLP_TREE_LOAD_PERMUTATION (b)[i])
5436 : return -1;
5437 : else
5438 38096 : return 1;
5439 : }
5440 : return 0;
5441 : }
5442 : }
5443 : }
5444 : else /* Different groups or non-groups. */
5445 : {
5446 : /* Order groups as their first element to keep them together. */
5447 2342885 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5448 2342885 : a0 = DR_GROUP_FIRST_ELEMENT (a0);
5449 2342885 : if (STMT_VINFO_GROUPED_ACCESS (b0))
5450 2342885 : b0 = DR_GROUP_FIRST_ELEMENT (b0);
5451 2342885 : if (a0 == b0)
5452 : return 0;
5453 : /* Tie using UID. */
5454 2342765 : else if (gimple_uid (STMT_VINFO_STMT (a0))
5455 2342765 : < gimple_uid (STMT_VINFO_STMT (b0)))
5456 : return -1;
5457 : else
5458 : {
5459 1041744 : gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
5460 : != gimple_uid (STMT_VINFO_STMT (b0)));
5461 : return 1;
5462 : }
5463 : }
5464 : }
5465 :
5466 : /* Return whether if the load permutation of NODE is consecutive starting
5467 : with value START_VAL in the first element. If START_VAL is not given
5468 : the first element's value is used. */
5469 :
5470 : bool
5471 647890 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
5472 : {
5473 647890 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
5474 :
5475 647890 : if (!perm.exists () || !perm.length ())
5476 : return false;
5477 :
5478 647890 : if (start_val == UINT_MAX)
5479 79797 : start_val = perm[0];
5480 :
5481 1279628 : for (unsigned int i = 0; i < perm.length (); i++)
5482 655226 : if (perm[i] != start_val + (unsigned int) i)
5483 : return false;
5484 :
5485 : return true;
5486 : }
5487 :
5488 : /* Process the set of LOADS that are all from the same dataref group. */
5489 :
5490 : static void
5491 161954 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5492 : scalar_stmts_to_slp_tree_map_t *bst_map,
5493 : const array_slice<slp_tree> &loads,
5494 : bool force_single_lane)
5495 : {
5496 : /* We at this point want to lower without a fixed VF or vector
5497 : size in mind which means we cannot actually compute whether we
5498 : need three or more vectors for a load permutation yet. So always
5499 : lower. */
5500 161954 : stmt_vec_info first
5501 161954 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
5502 161954 : unsigned group_lanes = DR_GROUP_SIZE (first);
5503 :
5504 : /* Verify if all load permutations can be implemented with a suitably
5505 : large element load-lanes operation. */
5506 161954 : unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
5507 161954 : if (STMT_VINFO_STRIDED_P (first)
5508 159501 : || compare_step_with_zero (loop_vinfo, first) <= 0
5509 156813 : || exact_log2 (ld_lanes_lanes) == -1
5510 : /* ??? For now only support the single-lane case as there is
5511 : missing support on the store-lane side and code generation
5512 : isn't up to the task yet. */
5513 154002 : || ld_lanes_lanes != 1
5514 304926 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
5515 : group_lanes / ld_lanes_lanes,
5516 : false) == IFN_LAST)
5517 : ld_lanes_lanes = 0;
5518 : else
5519 : /* Verify the loads access the same number of lanes aligned to
5520 : ld_lanes_lanes. */
5521 0 : for (slp_tree load : loads)
5522 : {
5523 0 : if (SLP_TREE_LANES (load) != ld_lanes_lanes)
5524 : {
5525 : ld_lanes_lanes = 0;
5526 : break;
5527 : }
5528 0 : unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
5529 0 : if (first % ld_lanes_lanes != 0)
5530 : {
5531 : ld_lanes_lanes = 0;
5532 : break;
5533 : }
5534 0 : if (!vect_load_perm_consecutive_p (load))
5535 : {
5536 : ld_lanes_lanes = 0;
5537 : break;
5538 : }
5539 : }
5540 :
5541 : /* Only a power-of-two number of lanes matches interleaving with N levels.
5542 : ??? An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
5543 : at each step. */
5544 262659 : if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
5545 : return;
5546 :
5547 266630 : for (slp_tree load : loads)
5548 : {
5549 : /* Leave masked or gather loads alone for now. */
5550 188168 : if (!SLP_TREE_CHILDREN (load).is_empty ())
5551 61040 : continue;
5552 :
5553 : /* For single-element interleaving spanning multiple vectors avoid
5554 : lowering, we want to use VMAT_ELEMENTWISE later. */
5555 188162 : if (ld_lanes_lanes == 0
5556 188162 : && SLP_TREE_LANES (load) == 1
5557 168676 : && !DR_GROUP_NEXT_ELEMENT (first)
5558 267949 : && maybe_gt (group_lanes,
5559 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
5560 51334 : return;
5561 :
5562 : /* We want to pattern-match special cases here and keep those
5563 : alone. Candidates are splats and load-lane. */
5564 :
5565 : /* We need to lower only loads of less than half of the groups
5566 : lanes, including duplicate lanes. Note this leaves nodes
5567 : with a non-1:1 load permutation around instead of canonicalizing
5568 : those into a load and a permute node. Removing this early
5569 : check would do such canonicalization. */
5570 136828 : if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
5571 57464 : && ld_lanes_lanes == 0)
5572 57464 : continue;
5573 :
5574 : /* Build the permute to get the original load permutation order. */
5575 79364 : bool contiguous = vect_load_perm_consecutive_p (load);
5576 79364 : lane_permutation_t final_perm;
5577 79364 : final_perm.create (SLP_TREE_LANES (load));
5578 239036 : for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
5579 160616 : final_perm.quick_push (
5580 80308 : std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
5581 :
5582 : /* When the load permutation accesses a contiguous unpermuted,
5583 : power-of-two aligned and sized chunk leave the load alone.
5584 : We can likely (re-)load it more efficiently rather than
5585 : extracting it from the larger load.
5586 : ??? Long-term some of the lowering should move to where
5587 : the vector types involved are fixed. */
5588 82934 : if (!force_single_lane
5589 79364 : && ld_lanes_lanes == 0
5590 53518 : && contiguous
5591 53258 : && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
5592 6566 : && pow2p_hwi (SLP_TREE_LANES (load))
5593 6530 : && pow2p_hwi (group_lanes)
5594 3570 : && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
5595 82934 : && group_lanes % SLP_TREE_LANES (load) == 0)
5596 : {
5597 3570 : final_perm.release ();
5598 3570 : continue;
5599 : }
5600 :
5601 : /* First build (and possibly re-use) a load node for the
5602 : unpermuted group. Gaps in the middle and on the end are
5603 : represented with NULL stmts. */
5604 75794 : vec<stmt_vec_info> stmts;
5605 75794 : stmts.create (group_lanes);
5606 270615 : for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
5607 : {
5608 194821 : if (s != first)
5609 124074 : for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
5610 5047 : stmts.quick_push (NULL);
5611 194821 : stmts.quick_push (s);
5612 : }
5613 139685 : for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
5614 63891 : stmts.quick_push (NULL);
5615 75794 : bool *matches = XALLOCAVEC (bool, group_lanes);
5616 75794 : unsigned limit = 1;
5617 75794 : unsigned tree_size = 0;
5618 75794 : slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts, matches, &limit,
5619 75794 : &tree_size, bst_map);
5620 75794 : gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
5621 :
5622 75794 : if (ld_lanes_lanes != 0)
5623 : {
5624 : /* ??? If this is not in sync with what get_load_store_type
5625 : later decides the SLP representation is not good for other
5626 : store vectorization methods. */
5627 0 : l0->ldst_lanes = true;
5628 0 : load->ldst_lanes = true;
5629 : }
5630 :
5631 236080 : while (1)
5632 : {
5633 155937 : unsigned group_lanes = SLP_TREE_LANES (l0);
5634 155937 : if (ld_lanes_lanes != 0
5635 155937 : || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
5636 : break;
5637 :
5638 : /* Try to lower by reducing the group to half its size using an
5639 : interleaving scheme. For this try to compute whether all
5640 : elements needed for this load are in even or odd elements of
5641 : an even/odd decomposition with N consecutive elements.
5642 : Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
5643 : with N == 2. */
5644 : /* ??? Only an even number of lanes can be handed this way, but the
5645 : fallback below could work for any number. We have to make sure
5646 : to round up in that case. */
5647 80143 : gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
5648 12134 : unsigned even = 0, odd = 0;
5649 12134 : if ((group_lanes & 1) == 0)
5650 : {
5651 12134 : even = (1 << ceil_log2 (group_lanes)) - 1;
5652 12134 : odd = even;
5653 49285 : for (auto l : final_perm)
5654 : {
5655 12883 : even &= ~l.second;
5656 12883 : odd &= l.second;
5657 : }
5658 : }
5659 :
5660 : /* Now build an even or odd extraction from the unpermuted load. */
5661 80143 : lane_permutation_t perm;
5662 80143 : perm.create ((group_lanes + 1) / 2);
5663 80143 : unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
5664 80143 : unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
5665 80143 : if (even_level
5666 11152 : && group_lanes % (2 * even_level) == 0
5667 : /* ??? When code generating permutes we do not try to pun
5668 : to larger component modes so level != 1 isn't a natural
5669 : even/odd extract. Prefer one if possible. */
5670 11152 : && (even_level == 1 || !odd_level || odd_level != 1))
5671 : {
5672 : /* { 0, 1, ... 4, 5 ..., } */
5673 39526 : for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
5674 62401 : for (unsigned j = 0; j < even_level; ++j)
5675 31396 : perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
5676 : }
5677 68991 : else if (odd_level)
5678 : {
5679 : /* { ..., 2, 3, ... 6, 7 } */
5680 3583 : gcc_assert (group_lanes % (2 * odd_level) == 0);
5681 15451 : for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
5682 23790 : for (unsigned j = 0; j < odd_level; ++j)
5683 11922 : perm.quick_push
5684 11922 : (std::make_pair (0, (2 * i + 1) * odd_level + j));
5685 : }
5686 : else
5687 : {
5688 : /* As fallback extract all used lanes and fill to half the
5689 : group size by repeating the last element.
5690 : ??? This is quite a bad strathegy for re-use - we could
5691 : brute force our way to find more optimal filling lanes to
5692 : maximize re-use when looking at all loads from the group. */
5693 68039 : auto_bitmap l;
5694 272212 : for (auto p : final_perm)
5695 68095 : bitmap_set_bit (l, p.second);
5696 68039 : unsigned i = 0;
5697 68039 : bitmap_iterator bi;
5698 136134 : EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
5699 68095 : perm.quick_push (std::make_pair (0, i));
5700 272308 : while (perm.length () < (group_lanes + 1) / 2)
5701 68115 : perm.quick_push (perm.last ());
5702 68039 : }
5703 :
5704 : /* Update final_perm with the intermediate permute. */
5705 161035 : for (unsigned i = 0; i < final_perm.length (); ++i)
5706 : {
5707 80892 : unsigned l = final_perm[i].second;
5708 80892 : unsigned j;
5709 90457 : for (j = 0; j < perm.length (); ++j)
5710 90457 : if (perm[j].second == l)
5711 : {
5712 80892 : final_perm[i].second = j;
5713 80892 : break;
5714 : }
5715 80892 : gcc_assert (j < perm.length ());
5716 : }
5717 :
5718 : /* And create scalar stmts. */
5719 80143 : vec<stmt_vec_info> perm_stmts;
5720 80143 : perm_stmts.create (perm.length ());
5721 339814 : for (unsigned i = 0; i < perm.length (); ++i)
5722 179528 : perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
5723 :
5724 80143 : slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
5725 80143 : SLP_TREE_CHILDREN (p).quick_push (l0);
5726 80143 : SLP_TREE_LANE_PERMUTATION (p) = perm;
5727 80143 : SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
5728 80143 : SLP_TREE_LANES (p) = perm.length ();
5729 : /* ??? As we have scalar stmts for this intermediate permute we
5730 : could CSE it via bst_map but we do not want to pick up
5731 : another SLP node with a load permutation. We instead should
5732 : have a "local" CSE map here. */
5733 80143 : SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
5734 :
5735 : /* We now have a node for (group_lanes + 1) / 2 lanes. */
5736 80143 : l0 = p;
5737 80143 : }
5738 :
5739 : /* And finally from the ordered reduction node create the
5740 : permute to shuffle the lanes into the original load-permutation
5741 : order. We replace the original load node with this. */
5742 75794 : SLP_TREE_CODE (load) = VEC_PERM_EXPR;
5743 75794 : SLP_TREE_LOAD_PERMUTATION (load).release ();
5744 75794 : SLP_TREE_LANE_PERMUTATION (load) = final_perm;
5745 75794 : SLP_TREE_CHILDREN (load).create (1);
5746 75794 : SLP_TREE_CHILDREN (load).quick_push (l0);
5747 75794 : SLP_TREE_REPRESENTATIVE (load) = NULL;
5748 : }
5749 : }
5750 :
5751 : /* Transform SLP loads in the SLP graph created by SLP discovery to
5752 : group loads from the same group and lower load permutations that
5753 : are unlikely to be supported into a series of permutes.
5754 : In the degenerate case of having only single-lane SLP instances
5755 : this should result in a series of permute nodes emulating an
5756 : interleaving scheme. */
5757 :
5758 : static void
5759 500241 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5760 : scalar_stmts_to_slp_tree_map_t *bst_map,
5761 : bool force_single_lane)
5762 : {
5763 : /* Gather and sort loads across all instances. */
5764 500241 : hash_set<slp_tree> visited;
5765 500241 : auto_vec<slp_tree> loads;
5766 2295214 : for (auto inst : loop_vinfo->slp_instances)
5767 796441 : vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
5768 500241 : if (loads.is_empty ())
5769 93287 : return;
5770 406954 : loads.qsort (vllp_cmp);
5771 :
5772 : /* Now process each dataref group separately. */
5773 406954 : unsigned firsti = 0;
5774 1158384 : for (unsigned i = 1; i < loads.length (); ++i)
5775 : {
5776 344476 : slp_tree first = loads[firsti];
5777 344476 : slp_tree next = loads[i];
5778 344476 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
5779 344476 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
5780 344476 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5781 158687 : && STMT_VINFO_GROUPED_ACCESS (b0)
5782 489981 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5783 63447 : continue;
5784 : /* Now we have one or multiple SLP loads of the same group from
5785 : firsti to i - 1. */
5786 281029 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5787 95240 : vect_lower_load_permutations (loop_vinfo, bst_map,
5788 95240 : make_array_slice (&loads[firsti],
5789 : i - firsti),
5790 : force_single_lane);
5791 : firsti = i;
5792 : }
5793 813908 : if (firsti < loads.length ()
5794 813908 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
5795 66714 : vect_lower_load_permutations (loop_vinfo, bst_map,
5796 66714 : make_array_slice (&loads[firsti],
5797 66714 : loads.length () - firsti),
5798 : force_single_lane);
5799 500241 : }
5800 :
5801 : /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
5802 : trees of packed scalar stmts if SLP is possible. */
5803 :
5804 : opt_result
5805 1186844 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
5806 : bool force_single_lane)
5807 : {
5808 1186844 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5809 1186844 : unsigned int i;
5810 1186844 : stmt_vec_info first_element;
5811 1186844 : slp_instance instance;
5812 :
5813 1186844 : DUMP_VECT_SCOPE ("vect_analyze_slp");
5814 :
5815 1186844 : unsigned limit = max_tree_size;
5816 :
5817 1186844 : scalar_stmts_to_slp_tree_map_t *bst_map
5818 1186844 : = new scalar_stmts_to_slp_tree_map_t ();
5819 :
5820 : /* Find SLP sequences starting from groups of grouped stores. */
5821 3314807 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
5822 941245 : if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
5823 : slp_inst_kind_store, max_tree_size, &limit,
5824 : force_single_lane)
5825 941245 : && loop_vinfo)
5826 : {
5827 126 : release_scalar_stmts_to_slp_tree_map (bst_map);
5828 126 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5829 : }
5830 :
5831 : /* For loops also start SLP discovery from non-grouped stores. */
5832 1186718 : if (loop_vinfo)
5833 : {
5834 : data_reference_p dr;
5835 1687316 : FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
5836 1172672 : if (DR_IS_WRITE (dr))
5837 : {
5838 379777 : stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
5839 : /* Grouped stores are already handled above. */
5840 379777 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5841 102814 : continue;
5842 276963 : vec<stmt_vec_info> stmts;
5843 276963 : vec<stmt_vec_info> roots = vNULL;
5844 276963 : vec<tree> remain = vNULL;
5845 276963 : stmts.create (1);
5846 276963 : stmts.quick_push (stmt_info);
5847 276963 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5848 : stmts, roots, remain, max_tree_size,
5849 : &limit, bst_map, force_single_lane))
5850 : {
5851 3577 : release_scalar_stmts_to_slp_tree_map (bst_map);
5852 3577 : return opt_result::failure_at (vect_location,
5853 : "SLP build failed.\n");
5854 : }
5855 : }
5856 :
5857 : stmt_vec_info stmt_info;
5858 514684 : FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
5859 : {
5860 20 : vec<stmt_vec_info> stmts;
5861 20 : vec<stmt_vec_info> roots = vNULL;
5862 20 : vec<tree> remain = vNULL;
5863 20 : stmts.create (1);
5864 20 : stmts.quick_push (stmt_info);
5865 20 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5866 : stmts, roots, remain, max_tree_size,
5867 : &limit, bst_map, force_single_lane))
5868 : {
5869 0 : release_scalar_stmts_to_slp_tree_map (bst_map);
5870 0 : return opt_result::failure_at (vect_location,
5871 : "SLP build failed.\n");
5872 : }
5873 : }
5874 : }
5875 :
5876 1183141 : if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
5877 : {
5878 2008320 : for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
5879 : {
5880 1339823 : vect_location = bb_vinfo->roots[i].roots[0]->stmt;
5881 : /* Apply patterns. */
5882 4389819 : for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
5883 6099992 : bb_vinfo->roots[i].stmts[j]
5884 3144280 : = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
5885 1339823 : if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
5886 1339823 : bb_vinfo->roots[i].stmts,
5887 1339823 : bb_vinfo->roots[i].roots,
5888 1339823 : bb_vinfo->roots[i].remain,
5889 : max_tree_size, &limit, bst_map, false))
5890 : {
5891 154541 : bb_vinfo->roots[i].roots = vNULL;
5892 154541 : bb_vinfo->roots[i].remain = vNULL;
5893 : }
5894 1339823 : bb_vinfo->roots[i].stmts = vNULL;
5895 : }
5896 : }
5897 :
5898 1183141 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
5899 : {
5900 : /* Find SLP sequences starting from groups of reductions. */
5901 514644 : if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
5902 : bst_map, force_single_lane))
5903 : {
5904 1900 : release_scalar_stmts_to_slp_tree_map (bst_map);
5905 1900 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5906 : }
5907 :
5908 : /* Make sure to vectorize only-live stmts, usually inductions. */
5909 2309646 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
5910 1492391 : for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
5911 711090 : gsi_next (&gsi))
5912 : {
5913 720977 : gphi *lc_phi = *gsi;
5914 720977 : tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
5915 720977 : stmt_vec_info stmt_info;
5916 720977 : if (TREE_CODE (def) == SSA_NAME
5917 604760 : && !virtual_operand_p (def)
5918 310225 : && (stmt_info = loop_vinfo->lookup_def (def))
5919 278010 : && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
5920 278010 : && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
5921 216842 : && STMT_VINFO_LIVE_P (stmt_info)
5922 216842 : && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
5923 833717 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
5924 : {
5925 112652 : vec<stmt_vec_info> stmts;
5926 112652 : vec<stmt_vec_info> roots = vNULL;
5927 112652 : vec<tree> remain = vNULL;
5928 112652 : stmts.create (1);
5929 112652 : stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
5930 112652 : if (! vect_build_slp_instance (vinfo,
5931 : slp_inst_kind_reduc_group,
5932 : stmts, roots, remain,
5933 : max_tree_size, &limit,
5934 : bst_map, force_single_lane))
5935 : {
5936 9887 : release_scalar_stmts_to_slp_tree_map (bst_map);
5937 9887 : return opt_result::failure_at (vect_location,
5938 : "SLP build failed.\n");
5939 : }
5940 : }
5941 9887 : }
5942 :
5943 : /* Find SLP sequences starting from gconds. */
5944 1259383 : for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
5945 : {
5946 293820 : auto cond_info = loop_vinfo->lookup_stmt (cond);
5947 :
5948 293820 : cond_info = vect_stmt_to_vectorize (cond_info);
5949 293820 : vec<stmt_vec_info> roots = vNULL;
5950 293820 : roots.safe_push (cond_info);
5951 293820 : gimple *stmt = STMT_VINFO_STMT (cond_info);
5952 293820 : tree args0 = gimple_cond_lhs (stmt);
5953 293820 : tree args1 = gimple_cond_rhs (stmt);
5954 :
5955 : /* These should be enforced by cond lowering, but if it failed
5956 : bail. */
5957 293820 : if (gimple_cond_code (stmt) != NE_EXPR
5958 292689 : || TREE_TYPE (args0) != boolean_type_node
5959 585929 : || !integer_zerop (args1))
5960 : {
5961 1711 : roots.release ();
5962 1711 : release_scalar_stmts_to_slp_tree_map (bst_map);
5963 1711 : return opt_result::failure_at (vect_location,
5964 : "SLP build failed.\n");
5965 : }
5966 :
5967 : /* An argument without a loop def will be codegened from vectorizing the
5968 : root gcond itself. As such we don't need to try to build an SLP tree
5969 : from them. It's highly likely that the resulting SLP tree here if both
5970 : arguments have a def will be incompatible, but we rely on it being split
5971 : later on. */
5972 292109 : auto varg = loop_vinfo->lookup_def (args0);
5973 292109 : vec<stmt_vec_info> stmts;
5974 292109 : vec<tree> remain = vNULL;
5975 292109 : stmts.create (1);
5976 292109 : stmts.quick_push (vect_stmt_to_vectorize (varg));
5977 :
5978 292109 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
5979 : stmts, roots, remain,
5980 : max_tree_size, &limit,
5981 : bst_map, force_single_lane))
5982 : {
5983 905 : roots.release ();
5984 905 : release_scalar_stmts_to_slp_tree_map (bst_map);
5985 905 : return opt_result::failure_at (vect_location,
5986 : "SLP build failed.\n");
5987 : }
5988 : }
5989 : }
5990 :
5991 1168738 : hash_set<slp_tree> visited_patterns;
5992 1168738 : slp_tree_to_load_perm_map_t perm_cache;
5993 1168738 : slp_compat_nodes_map_t compat_cache;
5994 :
5995 : /* See if any patterns can be found in the SLP tree. */
5996 1168738 : bool pattern_found = false;
5997 3963799 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
5998 1626323 : pattern_found |= vect_match_slp_patterns (instance, vinfo,
5999 : &visited_patterns, &perm_cache,
6000 : &compat_cache);
6001 :
6002 : /* If any were found optimize permutations of loads. */
6003 1168738 : if (pattern_found)
6004 : {
6005 267 : hash_map<slp_tree, slp_tree> load_map;
6006 4980 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6007 : {
6008 4179 : slp_tree root = SLP_INSTANCE_TREE (instance);
6009 4179 : optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
6010 : &load_map, root);
6011 : }
6012 267 : }
6013 :
6014 : /* Check whether we should force some SLP instances to use load/store-lanes
6015 : and do so by forcing SLP re-discovery with single lanes. We used
6016 : to cancel SLP when this applied to all instances in a loop but now
6017 : we decide this per SLP instance. It's important to do this only
6018 : after SLP pattern recognition. */
6019 1168738 : if (is_a <loop_vec_info> (vinfo))
6020 1296682 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6021 796441 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
6022 301976 : && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
6023 : {
6024 301976 : slp_tree slp_root = SLP_INSTANCE_TREE (instance);
6025 301976 : unsigned int group_size = SLP_TREE_LANES (slp_root);
6026 301976 : tree vectype = SLP_TREE_VECTYPE (slp_root);
6027 :
6028 301976 : stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
6029 301976 : gimple *rep = STMT_VINFO_STMT (rep_info);
6030 301976 : bool masked = (is_gimple_call (rep)
6031 2446 : && gimple_call_internal_p (rep)
6032 304402 : && internal_fn_mask_index
6033 2426 : (gimple_call_internal_fn (rep)) != -1);
6034 301956 : if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
6035 30023 : || slp_root->ldst_lanes
6036 331999 : || (vect_store_lanes_supported (vectype, group_size, masked)
6037 : == IFN_LAST))
6038 301976 : continue;
6039 :
6040 0 : auto_vec<slp_tree> loads;
6041 0 : hash_set<slp_tree> visited;
6042 0 : vect_gather_slp_loads (loads, slp_root, visited);
6043 :
6044 : /* Check whether any load in the SLP instance is possibly
6045 : permuted. */
6046 0 : bool loads_permuted = false;
6047 0 : slp_tree load_node;
6048 0 : unsigned j;
6049 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6050 : {
6051 0 : if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
6052 0 : continue;
6053 0 : for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
6054 0 : if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
6055 : {
6056 : loads_permuted = true;
6057 : break;
6058 : }
6059 : }
6060 :
6061 : /* If the loads and stores can use load/store-lanes force re-discovery
6062 : with single lanes. */
6063 0 : if (loads_permuted)
6064 : {
6065 0 : bool can_use_lanes = true;
6066 : bool prefer_load_lanes = false;
6067 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6068 0 : if (STMT_VINFO_GROUPED_ACCESS
6069 : (SLP_TREE_REPRESENTATIVE (load_node)))
6070 : {
6071 0 : stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
6072 : (SLP_TREE_REPRESENTATIVE (load_node));
6073 0 : rep = STMT_VINFO_STMT (stmt_vinfo);
6074 0 : masked = (is_gimple_call (rep)
6075 0 : && gimple_call_internal_p (rep)
6076 0 : && internal_fn_mask_index
6077 0 : (gimple_call_internal_fn (rep)));
6078 : /* Use SLP for strided accesses (or if we can't
6079 : load-lanes). */
6080 0 : if (STMT_VINFO_STRIDED_P (stmt_vinfo)
6081 0 : || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
6082 0 : || vect_load_lanes_supported
6083 0 : (SLP_TREE_VECTYPE (load_node),
6084 0 : DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
6085 : /* ??? During SLP re-discovery with a single lane
6086 : a masked grouped load will appear permuted and
6087 : discovery will fail. We have to rework this
6088 : on the discovery side - for now avoid ICEing. */
6089 0 : || masked)
6090 : {
6091 : can_use_lanes = false;
6092 : break;
6093 : }
6094 : /* Make sure that the target would prefer store-lanes
6095 : for at least one of the loads.
6096 :
6097 : ??? Perhaps we should instead require this for
6098 : all loads? */
6099 0 : prefer_load_lanes
6100 : = (prefer_load_lanes
6101 0 : || SLP_TREE_LANES (load_node) == group_size
6102 0 : || (vect_slp_prefer_store_lanes_p
6103 0 : (vinfo, stmt_vinfo,
6104 : SLP_TREE_VECTYPE (load_node), masked,
6105 : group_size, SLP_TREE_LANES (load_node))));
6106 : }
6107 :
6108 0 : if (can_use_lanes && prefer_load_lanes)
6109 : {
6110 0 : if (dump_enabled_p ())
6111 0 : dump_printf_loc (MSG_NOTE, vect_location,
6112 : "SLP instance %p can use load/store-lanes,"
6113 : " re-discovering with single-lanes\n",
6114 : (void *) instance);
6115 :
6116 0 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
6117 :
6118 0 : vect_free_slp_instance (instance);
6119 0 : limit = max_tree_size;
6120 0 : bool res = vect_analyze_slp_instance (vinfo, bst_map,
6121 : stmt_info,
6122 : slp_inst_kind_store,
6123 : max_tree_size, &limit,
6124 : true);
6125 0 : gcc_assert (res);
6126 0 : auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
6127 0 : LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
6128 : }
6129 : }
6130 0 : }
6131 :
6132 : /* When we end up with load permutations that we cannot possibly handle,
6133 : like those requiring three vector inputs, lower them using interleaving
6134 : like schemes. */
6135 1168738 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6136 : {
6137 500241 : vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
6138 500241 : if (dump_enabled_p ())
6139 : {
6140 20606 : dump_printf_loc (MSG_NOTE, vect_location,
6141 : "SLP graph after lowering permutations:\n");
6142 20606 : hash_set<slp_tree> visited;
6143 91680 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6144 29887 : vect_print_slp_graph (MSG_NOTE, vect_location,
6145 : SLP_INSTANCE_TREE (instance), visited);
6146 20606 : }
6147 : }
6148 :
6149 1168738 : release_scalar_stmts_to_slp_tree_map (bst_map);
6150 :
6151 1168738 : if (pattern_found && dump_enabled_p ())
6152 : {
6153 20 : dump_printf_loc (MSG_NOTE, vect_location,
6154 : "Pattern matched SLP tree\n");
6155 20 : hash_set<slp_tree> visited;
6156 101 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6157 41 : vect_print_slp_graph (MSG_NOTE, vect_location,
6158 : SLP_INSTANCE_TREE (instance), visited);
6159 20 : }
6160 :
6161 1168738 : return opt_result::success ();
6162 1168738 : }
6163 :
6164 : /* Estimates the cost of inserting layout changes into the SLP graph.
6165 : It can also say that the insertion is impossible. */
6166 :
6167 : struct slpg_layout_cost
6168 : {
6169 11211502 : slpg_layout_cost () = default;
6170 : slpg_layout_cost (sreal, bool);
6171 :
6172 530514 : static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
6173 5625312 : bool is_possible () const { return depth != sreal::max (); }
6174 :
6175 : bool operator== (const slpg_layout_cost &) const;
6176 : bool operator!= (const slpg_layout_cost &) const;
6177 :
6178 : bool is_better_than (const slpg_layout_cost &, bool) const;
6179 :
6180 : void add_parallel_cost (const slpg_layout_cost &);
6181 : void add_serial_cost (const slpg_layout_cost &);
6182 : void split (unsigned int);
6183 :
6184 : /* The longest sequence of layout changes needed during any traversal
6185 : of the partition dag, weighted by execution frequency.
6186 :
6187 : This is the most important metric when optimizing for speed, since
6188 : it helps to ensure that we keep the number of operations on
6189 : critical paths to a minimum. */
6190 : sreal depth = 0;
6191 :
6192 : /* An estimate of the total number of operations needed. It is weighted by
6193 : execution frequency when optimizing for speed but not when optimizing for
6194 : size. In order to avoid double-counting, a node with a fanout of N will
6195 : distribute 1/N of its total cost to each successor.
6196 :
6197 : This is the most important metric when optimizing for size, since
6198 : it helps to keep the total number of operations to a minimum, */
6199 : sreal total = 0;
6200 : };
6201 :
6202 : /* Construct costs for a node with weight WEIGHT. A higher weight
6203 : indicates more frequent execution. IS_FOR_SIZE is true if we are
6204 : optimizing for size rather than speed. */
6205 :
6206 1329255 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
6207 1337772 : : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
6208 : {
6209 1329255 : }
6210 :
6211 : bool
6212 0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
6213 : {
6214 0 : return depth == other.depth && total == other.total;
6215 : }
6216 :
6217 : bool
6218 0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
6219 : {
6220 0 : return !operator== (other);
6221 : }
6222 :
6223 : /* Return true if these costs are better than OTHER. IS_FOR_SIZE is
6224 : true if we are optimizing for size rather than speed. */
6225 :
6226 : bool
6227 320436 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
6228 : bool is_for_size) const
6229 : {
6230 320436 : if (is_for_size)
6231 : {
6232 3642 : if (total != other.total)
6233 1879 : return total < other.total;
6234 1763 : return depth < other.depth;
6235 : }
6236 : else
6237 : {
6238 316794 : if (depth != other.depth)
6239 129487 : return depth < other.depth;
6240 187307 : return total < other.total;
6241 : }
6242 : }
6243 :
6244 : /* Increase the costs to account for something with cost INPUT_COST
6245 : happening in parallel with the current costs. */
6246 :
6247 : void
6248 384176 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
6249 : {
6250 384176 : depth = std::max (depth, input_cost.depth);
6251 384176 : total += input_cost.total;
6252 384176 : }
6253 :
6254 : /* Increase the costs to account for something with cost INPUT_COST
6255 : happening in series with the current costs. */
6256 :
6257 : void
6258 1577465 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
6259 : {
6260 1577465 : depth += other.depth;
6261 1577465 : total += other.total;
6262 1577465 : }
6263 :
6264 : /* Split the total cost among TIMES successors or predecessors. */
6265 :
6266 : void
6267 1259003 : slpg_layout_cost::split (unsigned int times)
6268 : {
6269 1259003 : if (times > 1)
6270 605609 : total /= times;
6271 1259003 : }
6272 :
6273 : /* Information about one node in the SLP graph, for use during
6274 : vect_optimize_slp_pass. */
6275 :
6276 : struct slpg_vertex
6277 : {
6278 10470951 : slpg_vertex (slp_tree node_) : node (node_) {}
6279 :
6280 : /* The node itself. */
6281 : slp_tree node;
6282 :
6283 : /* Which partition the node belongs to, or -1 if none. Nodes outside of
6284 : partitions are flexible; they can have whichever layout consumers
6285 : want them to have. */
6286 : int partition = -1;
6287 :
6288 : /* The number of nodes that directly use the result of this one
6289 : (i.e. the number of nodes that count this one as a child). */
6290 : unsigned int out_degree = 0;
6291 :
6292 : /* The execution frequency of the node. */
6293 : sreal weight = 0;
6294 :
6295 : /* The total execution frequency of all nodes that directly use the
6296 : result of this one. */
6297 : sreal out_weight = 0;
6298 : };
6299 :
6300 : /* Information about one partition of the SLP graph, for use during
6301 : vect_optimize_slp_pass. */
6302 :
6303 : struct slpg_partition_info
6304 : {
6305 : /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
6306 : of m_partitioned_nodes. */
6307 : unsigned int node_begin = 0;
6308 : unsigned int node_end = 0;
6309 :
6310 : /* Which layout we've chosen to use for this partition, or -1 if
6311 : we haven't picked one yet. */
6312 : int layout = -1;
6313 :
6314 : /* The number of predecessors and successors in the partition dag.
6315 : The predecessors always have lower partition numbers and the
6316 : successors always have higher partition numbers.
6317 :
6318 : Note that the directions of these edges are not necessarily the
6319 : same as in the data flow graph. For example, if an SCC has separate
6320 : partitions for an inner loop and an outer loop, the inner loop's
6321 : partition will have at least two incoming edges from the outer loop's
6322 : partition: one for a live-in value and one for a live-out value.
6323 : In data flow terms, one of these edges would also be from the outer loop
6324 : to the inner loop, but the other would be in the opposite direction. */
6325 : unsigned int in_degree = 0;
6326 : unsigned int out_degree = 0;
6327 : };
6328 :
6329 : /* Information about the costs of using a particular layout for a
6330 : particular partition. It can also say that the combination is
6331 : impossible. */
6332 :
6333 : struct slpg_partition_layout_costs
6334 : {
6335 1652083 : bool is_possible () const { return internal_cost.is_possible (); }
6336 68313 : void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
6337 :
6338 : /* The costs inherited from predecessor partitions. */
6339 : slpg_layout_cost in_cost;
6340 :
6341 : /* The inherent cost of the layout within the node itself. For example,
6342 : this is nonzero for a load if choosing a particular layout would require
6343 : the load to permute the loaded elements. It is nonzero for a
6344 : VEC_PERM_EXPR if the permutation cannot be eliminated or converted
6345 : to full-vector moves. */
6346 : slpg_layout_cost internal_cost;
6347 :
6348 : /* The costs inherited from successor partitions. */
6349 : slpg_layout_cost out_cost;
6350 : };
6351 :
6352 : /* This class tries to optimize the layout of vectors in order to avoid
6353 : unnecessary shuffling. At the moment, the set of possible layouts are
6354 : restricted to bijective permutations.
6355 :
6356 : The goal of the pass depends on whether we're optimizing for size or
6357 : for speed. When optimizing for size, the goal is to reduce the overall
6358 : number of layout changes (including layout changes implied by things
6359 : like load permutations). When optimizing for speed, the goal is to
6360 : reduce the maximum latency attributable to layout changes on any
6361 : non-cyclical path through the data flow graph.
6362 :
6363 : For example, when optimizing a loop nest for speed, we will prefer
6364 : to make layout changes outside of a loop rather than inside of a loop,
6365 : and will prefer to make layout changes in parallel rather than serially,
6366 : even if that increases the overall number of layout changes.
6367 :
6368 : The high-level procedure is:
6369 :
6370 : (1) Build a graph in which edges go from uses (parents) to definitions
6371 : (children).
6372 :
6373 : (2) Divide the graph into a dag of strongly-connected components (SCCs).
6374 :
6375 : (3) When optimizing for speed, partition the nodes in each SCC based
6376 : on their containing cfg loop. When optimizing for size, treat
6377 : each SCC as a single partition.
6378 :
6379 : This gives us a dag of partitions. The goal is now to assign a
6380 : layout to each partition.
6381 :
6382 : (4) Construct a set of vector layouts that are worth considering.
6383 : Record which nodes must keep their current layout.
6384 :
6385 : (5) Perform a forward walk over the partition dag (from loads to stores)
6386 : accumulating the "forward" cost of using each layout. When visiting
6387 : each partition, assign a tentative choice of layout to the partition
6388 : and use that choice when calculating the cost of using a different
6389 : layout in successor partitions.
6390 :
6391 : (6) Perform a backward walk over the partition dag (from stores to loads),
6392 : accumulating the "backward" cost of using each layout. When visiting
6393 : each partition, make a final choice of layout for that partition based
6394 : on the accumulated forward costs (from (5)) and backward costs
6395 : (from (6)).
6396 :
6397 : (7) Apply the chosen layouts to the SLP graph.
6398 :
6399 : For example, consider the SLP statements:
6400 :
6401 : S1: a_1 = load
6402 : loop:
6403 : S2: a_2 = PHI<a_1, a_3>
6404 : S3: b_1 = load
6405 : S4: a_3 = a_2 + b_1
6406 : exit:
6407 : S5: a_4 = PHI<a_3>
6408 : S6: store a_4
6409 :
6410 : S2 and S4 form an SCC and are part of the same loop. Every other
6411 : statement is in a singleton SCC. In this example there is a one-to-one
6412 : mapping between SCCs and partitions and the partition dag looks like this;
6413 :
6414 : S1 S3
6415 : \ /
6416 : S2+S4
6417 : |
6418 : S5
6419 : |
6420 : S6
6421 :
6422 : S2, S3 and S4 will have a higher execution frequency than the other
6423 : statements, so when optimizing for speed, the goal is to avoid any
6424 : layout changes:
6425 :
6426 : - within S3
6427 : - within S2+S4
6428 : - on the S3->S2+S4 edge
6429 :
6430 : For example, if S3 was originally a reversing load, the goal of the
6431 : pass is to make it an unreversed load and change the layout on the
6432 : S1->S2+S4 and S2+S4->S5 edges to compensate. (Changing the layout
6433 : on S1->S2+S4 and S5->S6 would also be acceptable.)
6434 :
6435 : The difference between SCCs and partitions becomes important if we
6436 : add an outer loop:
6437 :
6438 : S1: a_1 = ...
6439 : loop1:
6440 : S2: a_2 = PHI<a_1, a_6>
6441 : S3: b_1 = load
6442 : S4: a_3 = a_2 + b_1
6443 : loop2:
6444 : S5: a_4 = PHI<a_3, a_5>
6445 : S6: c_1 = load
6446 : S7: a_5 = a_4 + c_1
6447 : exit2:
6448 : S8: a_6 = PHI<a_5>
6449 : S9: store a_6
6450 : exit1:
6451 :
6452 : Here, S2, S4, S5, S7 and S8 form a single SCC. However, when optimizing
6453 : for speed, we usually do not want restrictions in the outer loop to "infect"
6454 : the decision for the inner loop. For example, if an outer-loop node
6455 : in the SCC contains a statement with a fixed layout, that should not
6456 : prevent the inner loop from using a different layout. Conversely,
6457 : the inner loop should not dictate a layout to the outer loop: if the
6458 : outer loop does a lot of computation, then it may not be efficient to
6459 : do all of that computation in the inner loop's preferred layout.
6460 :
6461 : So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
6462 : and S5+S7 (inner). We also try to arrange partitions so that:
6463 :
6464 : - the partition for an outer loop comes before the partition for
6465 : an inner loop
6466 :
6467 : - if a sibling loop A dominates a sibling loop B, A's partition
6468 : comes before B's
6469 :
6470 : This gives the following partition dag for the example above:
6471 :
6472 : S1 S3
6473 : \ /
6474 : S2+S4+S8 S6
6475 : | \\ /
6476 : | S5+S7
6477 : |
6478 : S9
6479 :
6480 : There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
6481 : one for a reversal of the edge S7->S8.
6482 :
6483 : The backward walk picks a layout for S5+S7 before S2+S4+S8. The choice
6484 : for S2+S4+S8 therefore has to balance the cost of using the outer loop's
6485 : preferred layout against the cost of changing the layout on entry to the
6486 : inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
6487 :
6488 : Although this works well when optimizing for speed, it has the downside
6489 : when optimizing for size that the choice of layout for S5+S7 is completely
6490 : independent of S9, which lessens the chance of reducing the overall number
6491 : of permutations. We therefore do not partition SCCs when optimizing
6492 : for size.
6493 :
6494 : To give a concrete example of the difference between optimizing
6495 : for size and speed, consider:
6496 :
6497 : a[0] = (b[1] << c[3]) - d[1];
6498 : a[1] = (b[0] << c[2]) - d[0];
6499 : a[2] = (b[3] << c[1]) - d[3];
6500 : a[3] = (b[2] << c[0]) - d[2];
6501 :
6502 : There are three different layouts here: one for a, one for b and d,
6503 : and one for c. When optimizing for speed it is better to permute each
6504 : of b, c and d into the order required by a, since those permutations
6505 : happen in parallel. But when optimizing for size, it is better to:
6506 :
6507 : - permute c into the same order as b
6508 : - do the arithmetic
6509 : - permute the result into the order required by a
6510 :
6511 : This gives 2 permutations rather than 3. */
6512 :
6513 : class vect_optimize_slp_pass
6514 : {
6515 : public:
6516 714431 : vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
6517 : void run ();
6518 :
6519 : private:
6520 : /* Graph building. */
6521 : struct loop *containing_loop (slp_tree);
6522 : bool is_cfg_latch_edge (graph_edge *);
6523 : void build_vertices (hash_set<slp_tree> &, slp_tree);
6524 : void build_vertices ();
6525 : void build_graph ();
6526 :
6527 : /* Partitioning. */
6528 : void create_partitions ();
6529 : template<typename T> void for_each_partition_edge (unsigned int, T);
6530 :
6531 : /* Layout selection. */
6532 : bool is_compatible_layout (slp_tree, unsigned int);
6533 : bool is_compatible_layout (const slpg_partition_info &, unsigned int);
6534 : int change_layout_cost (slp_tree, unsigned int, unsigned int);
6535 : slpg_partition_layout_costs &partition_layout_costs (unsigned int,
6536 : unsigned int);
6537 : void change_vec_perm_layout (slp_tree, lane_permutation_t &,
6538 : int, unsigned int);
6539 : int internal_node_cost (slp_tree, int, unsigned int);
6540 : void start_choosing_layouts ();
6541 : bool legitimize ();
6542 :
6543 : /* Cost propagation. */
6544 : slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
6545 : unsigned int, unsigned int);
6546 : slpg_layout_cost total_in_cost (unsigned int);
6547 : slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
6548 : slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
6549 : void forward_pass ();
6550 : void backward_pass ();
6551 :
6552 : /* Rematerialization. */
6553 : slp_tree get_result_with_layout (slp_tree, unsigned int);
6554 : void materialize ();
6555 :
6556 : /* Clean-up. */
6557 : void remove_redundant_permutations ();
6558 :
6559 : /* Masked load lanes discovery. */
6560 : void decide_masked_load_lanes ();
6561 :
6562 : void dump ();
6563 :
6564 : vec_info *m_vinfo;
6565 :
6566 : /* True if we should optimize the graph for size, false if we should
6567 : optimize it for speed. (It wouldn't be easy to make this decision
6568 : more locally.) */
6569 : bool m_optimize_size;
6570 :
6571 : /* A graph of all SLP nodes, with edges leading from uses to definitions.
6572 : In other words, a node's predecessors are its slp_tree parents and
6573 : a node's successors are its slp_tree children. */
6574 : graph *m_slpg = nullptr;
6575 :
6576 : /* The vertices of M_SLPG, indexed by slp_tree::vertex. */
6577 : auto_vec<slpg_vertex> m_vertices;
6578 :
6579 : /* The list of all leaves of M_SLPG. such as external definitions, constants,
6580 : and loads. */
6581 : auto_vec<int> m_leafs;
6582 :
6583 : /* This array has one entry for every vector layout that we're considering.
6584 : Element 0 is null and indicates "no change". Other entries describe
6585 : permutations that are inherent in the current graph and that we would
6586 : like to reverse if possible.
6587 :
6588 : For example, a permutation { 1, 2, 3, 0 } means that something has
6589 : effectively been permuted in that way, such as a load group
6590 : { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
6591 : We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
6592 : in order to put things "back" in order. */
6593 : auto_vec<vec<unsigned> > m_perms;
6594 :
6595 : /* A partitioning of the nodes for which a layout must be chosen.
6596 : Each partition represents an <SCC, cfg loop> pair; that is,
6597 : nodes in different SCCs belong to different partitions, and nodes
6598 : within an SCC can be further partitioned according to a containing
6599 : cfg loop. Partition <SCC1, L1> comes before <SCC2, L2> if:
6600 :
6601 : - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
6602 : from leaves (such as loads) to roots (such as stores).
6603 :
6604 : - SCC1 == SCC2 and L1's header strictly dominates L2's header. */
6605 : auto_vec<slpg_partition_info> m_partitions;
6606 :
6607 : /* The list of all nodes for which a layout must be chosen. Nodes for
6608 : partition P come before the nodes for partition P+1. Nodes within a
6609 : partition are in reverse postorder. */
6610 : auto_vec<unsigned int> m_partitioned_nodes;
6611 :
6612 : /* Index P * num-layouts + L contains the cost of using layout L
6613 : for partition P. */
6614 : auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
6615 :
6616 : /* Index N * num-layouts + L, if nonnull, is a node that provides the
6617 : original output of node N adjusted to have layout L. */
6618 : auto_vec<slp_tree> m_node_layouts;
6619 : };
6620 :
6621 : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
6622 : Also record whether we should optimize anything for speed rather
6623 : than size. */
6624 :
6625 : void
6626 11313585 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
6627 : slp_tree node)
6628 : {
6629 11313585 : unsigned i;
6630 11313585 : slp_tree child;
6631 :
6632 11313585 : if (visited.add (node))
6633 11313585 : return;
6634 :
6635 10470951 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
6636 : {
6637 7997056 : basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
6638 7056644 : if (optimize_bb_for_speed_p (bb))
6639 6927472 : m_optimize_size = false;
6640 : }
6641 :
6642 10470951 : node->vertex = m_vertices.length ();
6643 10470951 : m_vertices.safe_push (slpg_vertex (node));
6644 :
6645 10470951 : bool leaf = true;
6646 10470951 : bool force_leaf = false;
6647 19632804 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
6648 9161853 : if (child)
6649 : {
6650 8247243 : leaf = false;
6651 8247243 : build_vertices (visited, child);
6652 : }
6653 : else
6654 : force_leaf = true;
6655 : /* Since SLP discovery works along use-def edges all cycles have an
6656 : entry - but there's the exception of cycles where we do not handle
6657 : the entry explicitly (but with a NULL SLP node), like some reductions
6658 : and inductions. Force those SLP PHIs to act as leafs to make them
6659 : backwards reachable. */
6660 10470951 : if (leaf || force_leaf)
6661 5185362 : m_leafs.safe_push (node->vertex);
6662 : }
6663 :
6664 : /* Fill the vertices and leafs vector with all nodes in the SLP graph. */
6665 :
6666 : void
6667 1428862 : vect_optimize_slp_pass::build_vertices ()
6668 : {
6669 1428862 : hash_set<slp_tree> visited;
6670 1428862 : unsigned i;
6671 1428862 : slp_instance instance;
6672 1428862 : m_vertices.truncate (0);
6673 1428862 : m_leafs.truncate (0);
6674 7352928 : FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
6675 3066342 : build_vertices (visited, SLP_INSTANCE_TREE (instance));
6676 1428862 : }
6677 :
6678 : /* Apply (reverse) bijectite PERM to VEC. */
6679 :
6680 : template <class T>
6681 : static void
6682 229615 : vect_slp_permute (vec<unsigned> perm,
6683 : vec<T> &vec, bool reverse)
6684 : {
6685 229615 : auto_vec<T, 64> saved;
6686 229615 : saved.create (vec.length ());
6687 761177 : for (unsigned i = 0; i < vec.length (); ++i)
6688 531562 : saved.quick_push (vec[i]);
6689 :
6690 229615 : if (reverse)
6691 : {
6692 1503891 : for (unsigned i = 0; i < vec.length (); ++i)
6693 529470 : vec[perm[i]] = saved[i];
6694 758119 : for (unsigned i = 0; i < vec.length (); ++i)
6695 905195 : gcc_assert (vec[perm[i]] == saved[i]);
6696 : }
6697 : else
6698 : {
6699 6116 : for (unsigned i = 0; i < vec.length (); ++i)
6700 2092 : vec[i] = saved[perm[i]];
6701 231707 : for (unsigned i = 0; i < vec.length (); ++i)
6702 3138 : gcc_assert (vec[i] == saved[perm[i]]);
6703 : }
6704 229615 : }
6705 :
6706 : /* Return the cfg loop that contains NODE. */
6707 :
6708 : struct loop *
6709 4076519 : vect_optimize_slp_pass::containing_loop (slp_tree node)
6710 : {
6711 4076519 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
6712 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
6713 : but are laid out close to their children. */
6714 4076519 : if (!rep)
6715 144760 : return m_vinfo->bbs[0]->loop_father;
6716 4415117 : return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
6717 : }
6718 :
6719 : /* Return true if UD (an edge from a use to a definition) is associated
6720 : with a loop latch edge in the cfg. */
6721 :
6722 : bool
6723 8247243 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
6724 : {
6725 8247243 : slp_tree use = m_vertices[ud->src].node;
6726 8247243 : slp_tree def = m_vertices[ud->dest].node;
6727 8247243 : if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
6728 8247243 : || SLP_TREE_PERMUTE_P (use))
6729 7927058 : || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
6730 : return false;
6731 :
6732 4802234 : stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
6733 4802234 : return (is_a<gphi *> (use_rep->stmt)
6734 385802 : && bb_loop_header_p (gimple_bb (use_rep->stmt))
6735 5020168 : && containing_loop (def) == containing_loop (use));
6736 : }
6737 :
6738 : /* Build the graph. Mark edges that correspond to cfg loop latch edges with
6739 : a nonnull data field. */
6740 :
6741 : void
6742 1428862 : vect_optimize_slp_pass::build_graph ()
6743 : {
6744 1428862 : m_optimize_size = true;
6745 1428862 : build_vertices ();
6746 :
6747 2857724 : m_slpg = new_graph (m_vertices.length ());
6748 14757537 : for (slpg_vertex &v : m_vertices)
6749 31303302 : for (slp_tree child : SLP_TREE_CHILDREN (v.node))
6750 9161853 : if (child)
6751 : {
6752 8247243 : graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
6753 8247243 : if (is_cfg_latch_edge (ud))
6754 208010 : ud->data = this;
6755 : }
6756 1428862 : }
6757 :
6758 : /* Return true if E corresponds to a loop latch edge in the cfg. */
6759 :
6760 : static bool
6761 4227072 : skip_cfg_latch_edges (graph_edge *e)
6762 : {
6763 4227072 : return e->data;
6764 : }
6765 :
6766 : /* Create the node partitions. */
6767 :
6768 : void
6769 714431 : vect_optimize_slp_pass::create_partitions ()
6770 : {
6771 : /* Calculate a postorder of the graph, ignoring edges that correspond
6772 : to natural latch edges in the cfg. Reading the vector from the end
6773 : to the beginning gives the reverse postorder. */
6774 714431 : auto_vec<int> initial_rpo;
6775 1428862 : graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
6776 : false, NULL, skip_cfg_latch_edges);
6777 2143293 : gcc_assert (initial_rpo.length () == m_vertices.length ());
6778 :
6779 : /* Calculate the strongly connected components of the graph. */
6780 714431 : auto_vec<int> scc_grouping;
6781 714431 : unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
6782 :
6783 : /* Create a new index order in which all nodes from the same SCC are
6784 : consecutive. Use scc_pos to record the index of the first node in
6785 : each SCC. */
6786 714431 : auto_vec<unsigned int> scc_pos (num_sccs);
6787 714431 : int last_component = -1;
6788 714431 : unsigned int node_count = 0;
6789 7378220 : for (unsigned int node_i : scc_grouping)
6790 : {
6791 5234927 : if (last_component != m_slpg->vertices[node_i].component)
6792 : {
6793 5104303 : last_component = m_slpg->vertices[node_i].component;
6794 10208606 : gcc_assert (last_component == int (scc_pos.length ()));
6795 5104303 : scc_pos.quick_push (node_count);
6796 : }
6797 5234927 : node_count += 1;
6798 : }
6799 1428862 : gcc_assert (node_count == initial_rpo.length ()
6800 : && last_component + 1 == int (num_sccs));
6801 :
6802 : /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
6803 : inside each SCC following the RPO we calculated above. The fact that
6804 : we ignored natural latch edges when calculating the RPO should ensure
6805 : that, for natural loop nests:
6806 :
6807 : - the first node that we encounter in a cfg loop is the loop header phi
6808 : - the loop header phis are in dominance order
6809 :
6810 : Arranging for this is an optimization (see below) rather than a
6811 : correctness issue. Unnatural loops with a tangled mess of backedges
6812 : will still work correctly, but might give poorer results.
6813 :
6814 : Also update scc_pos so that it gives 1 + the index of the last node
6815 : in the SCC. */
6816 714431 : m_partitioned_nodes.safe_grow (node_count);
6817 6663789 : for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
6818 : {
6819 5234927 : unsigned int node_i = initial_rpo[old_i];
6820 5234927 : unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
6821 5234927 : m_partitioned_nodes[new_i] = node_i;
6822 : }
6823 :
6824 : /* When optimizing for speed, partition each SCC based on the containing
6825 : cfg loop. The order we constructed above should ensure that, for natural
6826 : cfg loops, we'll create sub-SCC partitions for outer loops before
6827 : the corresponding sub-SCC partitions for inner loops. Similarly,
6828 : when one sibling loop A dominates another sibling loop B, we should
6829 : create a sub-SCC partition for A before a sub-SCC partition for B.
6830 :
6831 : As above, nothing depends for correctness on whether this achieves
6832 : a natural nesting, but we should get better results when it does. */
6833 1428862 : m_partitions.reserve (m_vertices.length ());
6834 714431 : unsigned int next_partition_i = 0;
6835 714431 : hash_map<struct loop *, int> loop_partitions;
6836 714431 : unsigned int rpo_begin = 0;
6837 714431 : unsigned int num_partitioned_nodes = 0;
6838 7247596 : for (unsigned int rpo_end : scc_pos)
6839 : {
6840 5104303 : loop_partitions.empty ();
6841 5104303 : unsigned int partition_i = next_partition_i;
6842 10339230 : for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
6843 : {
6844 : /* Handle externals and constants optimistically throughout.
6845 : But treat existing vectors as fixed since we do not handle
6846 : permuting them. */
6847 5234927 : unsigned int node_i = m_partitioned_nodes[rpo_i];
6848 5234927 : auto &vertex = m_vertices[node_i];
6849 5234927 : if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
6850 529895 : && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
6851 5238149 : || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
6852 1563512 : vertex.partition = -1;
6853 : else
6854 : {
6855 3671415 : bool existed;
6856 3671415 : if (m_optimize_size)
6857 30764 : existed = next_partition_i > partition_i;
6858 : else
6859 : {
6860 3640651 : struct loop *loop = containing_loop (vertex.node);
6861 3640651 : auto &entry = loop_partitions.get_or_insert (loop, &existed);
6862 3640651 : if (!existed)
6863 3511254 : entry = next_partition_i;
6864 3640651 : partition_i = entry;
6865 : }
6866 3671415 : if (!existed)
6867 : {
6868 3541926 : m_partitions.quick_push (slpg_partition_info ());
6869 3541926 : next_partition_i += 1;
6870 : }
6871 3671415 : vertex.partition = partition_i;
6872 3671415 : num_partitioned_nodes += 1;
6873 3671415 : m_partitions[partition_i].node_end += 1;
6874 : }
6875 : }
6876 5104303 : rpo_begin = rpo_end;
6877 : }
6878 :
6879 : /* Assign ranges of consecutive node indices to each partition,
6880 : in partition order. Start with node_end being the same as
6881 : node_begin so that the next loop can use it as a counter. */
6882 714431 : unsigned int node_begin = 0;
6883 5685219 : for (auto &partition : m_partitions)
6884 : {
6885 3541926 : partition.node_begin = node_begin;
6886 3541926 : node_begin += partition.node_end;
6887 3541926 : partition.node_end = partition.node_begin;
6888 : }
6889 714431 : gcc_assert (node_begin == num_partitioned_nodes);
6890 :
6891 : /* Finally build the list of nodes in partition order. */
6892 714431 : m_partitioned_nodes.truncate (num_partitioned_nodes);
6893 6663789 : for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
6894 : {
6895 5234927 : int partition_i = m_vertices[node_i].partition;
6896 5234927 : if (partition_i >= 0)
6897 : {
6898 3671415 : unsigned int order_i = m_partitions[partition_i].node_end++;
6899 3671415 : m_partitioned_nodes[order_i] = node_i;
6900 : }
6901 : }
6902 714431 : }
6903 :
6904 : /* Look for edges from earlier partitions into node NODE_I and edges from
6905 : node NODE_I into later partitions. Call:
6906 :
6907 : FN (ud, other_node_i)
6908 :
6909 : for each such use-to-def edge ud, where other_node_i is the node at the
6910 : other end of the edge. */
6911 :
6912 : template<typename T>
6913 : void
6914 4134220 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
6915 : {
6916 4134220 : int partition_i = m_vertices[node_i].partition;
6917 4134220 : for (graph_edge *pred = m_slpg->vertices[node_i].pred;
6918 7149007 : pred; pred = pred->pred_next)
6919 : {
6920 3014787 : int src_partition_i = m_vertices[pred->src].partition;
6921 3014787 : if (src_partition_i >= 0 && src_partition_i != partition_i)
6922 2685496 : fn (pred, pred->src);
6923 : }
6924 4134220 : for (graph_edge *succ = m_slpg->vertices[node_i].succ;
6925 8882095 : succ; succ = succ->succ_next)
6926 : {
6927 4747875 : int dest_partition_i = m_vertices[succ->dest].partition;
6928 4747875 : if (dest_partition_i >= 0 && dest_partition_i != partition_i)
6929 2722202 : fn (succ, succ->dest);
6930 : }
6931 4134220 : }
6932 :
6933 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
6934 : that NODE would operate on. This test is independent of NODE's actual
6935 : operation. */
6936 :
6937 : bool
6938 1779168 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
6939 : unsigned int layout_i)
6940 : {
6941 1779168 : if (layout_i == 0)
6942 : return true;
6943 :
6944 1044882 : if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
6945 18826 : return false;
6946 :
6947 : return true;
6948 : }
6949 :
6950 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
6951 : that NODE would operate on for each NODE in PARTITION.
6952 : This test is independent of NODE's actual operations. */
6953 :
6954 : bool
6955 24007 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
6956 : &partition,
6957 : unsigned int layout_i)
6958 : {
6959 48446 : for (unsigned int order_i = partition.node_begin;
6960 48446 : order_i < partition.node_end; ++order_i)
6961 : {
6962 24541 : unsigned int node_i = m_partitioned_nodes[order_i];
6963 24541 : auto &vertex = m_vertices[node_i];
6964 :
6965 : /* The layout is incompatible if it is individually incompatible
6966 : with any node in the partition. */
6967 24541 : if (!is_compatible_layout (vertex.node, layout_i))
6968 : return false;
6969 : }
6970 : return true;
6971 : }
6972 :
6973 : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
6974 : to layout TO_LAYOUT_I for a node like NODE. Return -1 if either of the
6975 : layouts is incompatible with NODE or if the change is not possible for
6976 : some other reason.
6977 :
6978 : The properties taken from NODE include the number of lanes and the
6979 : vector type. The actual operation doesn't matter. */
6980 :
6981 : int
6982 750877 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
6983 : unsigned int from_layout_i,
6984 : unsigned int to_layout_i)
6985 : {
6986 750877 : if (!is_compatible_layout (node, from_layout_i)
6987 750877 : || !is_compatible_layout (node, to_layout_i))
6988 : return -1;
6989 :
6990 750232 : if (from_layout_i == to_layout_i)
6991 : return 0;
6992 :
6993 311195 : auto_vec<slp_tree, 1> children (1);
6994 311195 : children.quick_push (node);
6995 311195 : auto_lane_permutation_t perm (SLP_TREE_LANES (node));
6996 311195 : if (from_layout_i > 0)
6997 876124 : for (unsigned int i : m_perms[from_layout_i])
6998 389161 : perm.quick_push ({ 0, i });
6999 : else
7000 488889 : for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
7001 340015 : perm.quick_push ({ 0, i });
7002 311195 : if (to_layout_i > 0)
7003 149611 : vect_slp_permute (m_perms[to_layout_i], perm, true);
7004 311195 : auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
7005 : children, false);
7006 311195 : if (count >= 0)
7007 305365 : return MAX (count, 1);
7008 :
7009 : /* ??? In principle we could try changing via layout 0, giving two
7010 : layout changes rather than 1. Doing that would require
7011 : corresponding support in get_result_with_layout. */
7012 : return -1;
7013 311195 : }
7014 :
7015 : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I. */
7016 :
7017 : inline slpg_partition_layout_costs &
7018 1123633 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
7019 : unsigned int layout_i)
7020 : {
7021 2247266 : return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
7022 : }
7023 :
7024 : /* Change PERM in one of two ways:
7025 :
7026 : - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
7027 : chosen for child I of NODE.
7028 :
7029 : - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
7030 :
7031 : In both cases, arrange for the output to have layout OUT_LAYOUT_I */
7032 :
7033 : void
7034 39214 : vect_optimize_slp_pass::
7035 : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
7036 : int in_layout_i, unsigned int out_layout_i)
7037 : {
7038 230420 : for (auto &entry : perm)
7039 : {
7040 112778 : int this_in_layout_i = in_layout_i;
7041 112778 : if (this_in_layout_i < 0)
7042 : {
7043 76859 : slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
7044 76859 : unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
7045 76859 : if (in_partition_i == -1u)
7046 329 : continue;
7047 76530 : this_in_layout_i = m_partitions[in_partition_i].layout;
7048 : }
7049 112449 : if (this_in_layout_i > 0)
7050 25601 : entry.second = m_perms[this_in_layout_i][entry.second];
7051 : }
7052 39214 : if (out_layout_i > 0)
7053 11094 : vect_slp_permute (m_perms[out_layout_i], perm, true);
7054 39214 : }
7055 :
7056 : /* Check whether the target allows NODE to be rearranged so that the node's
7057 : output has layout OUT_LAYOUT_I. Return the cost of the change if so,
7058 : in the same arbitrary units as for change_layout_cost. Return -1 otherwise.
7059 :
7060 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
7061 : NODE can adapt to the layout changes that have (perhaps provisionally)
7062 : been chosen for NODE's children, so that no extra permutations are
7063 : needed on either the input or the output of NODE.
7064 :
7065 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
7066 : that all inputs will be forced into layout IN_LAYOUT_I beforehand.
7067 :
7068 : IN_LAYOUT_I has no meaning for other types of node.
7069 :
7070 : Keeping the node as-is is always valid. If the target doesn't appear
7071 : to support the node as-is, but might realistically support other layouts,
7072 : then layout 0 instead has the cost of a worst-case permutation. On the
7073 : one hand, this ensures that every node has at least one valid layout,
7074 : avoiding what would otherwise be an awkward special case. On the other,
7075 : it still encourages the pass to change an invalid pre-existing layout
7076 : choice into a valid one. */
7077 :
7078 : int
7079 248357 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
7080 : unsigned int out_layout_i)
7081 : {
7082 248357 : const int fallback_cost = 1;
7083 :
7084 248357 : if (SLP_TREE_PERMUTE_P (node))
7085 : {
7086 32553 : auto_lane_permutation_t tmp_perm;
7087 32553 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7088 :
7089 : /* Check that the child nodes support the chosen layout. Checking
7090 : the first child is enough, since any second child would have the
7091 : same shape. */
7092 32553 : auto first_child = SLP_TREE_CHILDREN (node)[0];
7093 32553 : if (in_layout_i > 0
7094 32553 : && !is_compatible_layout (first_child, in_layout_i))
7095 : return -1;
7096 :
7097 31913 : change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
7098 63826 : int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
7099 : node, tmp_perm,
7100 31913 : SLP_TREE_CHILDREN (node),
7101 : false);
7102 31913 : if (count < 0)
7103 : {
7104 2609 : if (in_layout_i == 0 && out_layout_i == 0)
7105 : {
7106 : /* Use the fallback cost if the node could in principle support
7107 : some nonzero layout for both the inputs and the outputs.
7108 : Otherwise assume that the node will be rejected later
7109 : and rebuilt from scalars. */
7110 641 : if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
7111 : return fallback_cost;
7112 319 : return 0;
7113 : }
7114 : return -1;
7115 : }
7116 :
7117 : /* We currently have no way of telling whether the new layout is cheaper
7118 : or more expensive than the old one. But at least in principle,
7119 : it should be worth making zero permutations (whole-vector shuffles)
7120 : cheaper than real permutations, in case the pass is able to remove
7121 : the latter. */
7122 29304 : return count == 0 ? 0 : 1;
7123 32553 : }
7124 :
7125 215804 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
7126 215804 : if (rep
7127 213845 : && STMT_VINFO_DATA_REF (rep)
7128 71861 : && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
7129 263744 : && SLP_TREE_LOAD_PERMUTATION (node).exists ())
7130 : {
7131 39616 : auto_load_permutation_t tmp_perm;
7132 39616 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7133 39616 : if (out_layout_i > 0)
7134 15202 : vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
7135 :
7136 39616 : poly_uint64 vf = 1;
7137 39616 : if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
7138 12152 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7139 39616 : unsigned int n_perms;
7140 39616 : if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
7141 : nullptr, vf, true, false, &n_perms))
7142 : {
7143 2366 : auto rep = SLP_TREE_REPRESENTATIVE (node);
7144 2366 : if (out_layout_i == 0)
7145 : {
7146 : /* Use the fallback cost if the load is an N-to-N permutation.
7147 : Otherwise assume that the node will be rejected later
7148 : and rebuilt from scalars. */
7149 1733 : if (STMT_VINFO_GROUPED_ACCESS (rep)
7150 3466 : && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
7151 1733 : == SLP_TREE_LANES (node)))
7152 685 : return fallback_cost;
7153 : return 0;
7154 : }
7155 : return -1;
7156 : }
7157 :
7158 : /* See the comment above the corresponding VEC_PERM_EXPR handling. */
7159 37250 : return n_perms == 0 ? 0 : 1;
7160 39616 : }
7161 :
7162 : return 0;
7163 : }
7164 :
7165 : /* Decide which element layouts we should consider using. Calculate the
7166 : weights associated with inserting layout changes on partition edges.
7167 : Also mark partitions that cannot change layout, by setting their
7168 : layout to zero. */
7169 :
7170 : void
7171 714431 : vect_optimize_slp_pass::start_choosing_layouts ()
7172 : {
7173 : /* Used to assign unique permutation indices. */
7174 714431 : using perm_hash = unbounded_hashmap_traits<
7175 : vec_free_hash_base<int_hash_base<unsigned>>,
7176 : int_hash<int, -1, -2>
7177 : >;
7178 714431 : hash_map<vec<unsigned>, int, perm_hash> layout_ids;
7179 :
7180 : /* Layout 0 is "no change". */
7181 714431 : m_perms.safe_push (vNULL);
7182 :
7183 : /* Create layouts from existing permutations. */
7184 714431 : auto_load_permutation_t tmp_perm;
7185 5814708 : for (unsigned int node_i : m_partitioned_nodes)
7186 : {
7187 : /* Leafs also double as entries to the reverse graph. Allow the
7188 : layout of those to be changed. */
7189 3671415 : auto &vertex = m_vertices[node_i];
7190 3671415 : auto &partition = m_partitions[vertex.partition];
7191 3671415 : if (!m_slpg->vertices[node_i].succ)
7192 936814 : partition.layout = 0;
7193 :
7194 : /* Loads and VEC_PERM_EXPRs are the only things generating permutes. */
7195 3671415 : slp_tree node = vertex.node;
7196 3671415 : stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
7197 3671415 : slp_tree child;
7198 3671415 : unsigned HOST_WIDE_INT imin, imax = 0;
7199 3671415 : bool any_permute = false;
7200 3671415 : tmp_perm.truncate (0);
7201 3671415 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
7202 : {
7203 : /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
7204 : unpermuted, record a layout that reverses this permutation.
7205 :
7206 : We would need more work to cope with loads that are internally
7207 : permuted and also have inputs (such as masks for
7208 : IFN_MASK_LOADs). */
7209 630990 : gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
7210 630990 : if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
7211 : {
7212 446659 : partition.layout = -1;
7213 3650183 : continue;
7214 : }
7215 184331 : dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
7216 184331 : imin = DR_GROUP_SIZE (dr_stmt) + 1;
7217 184331 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7218 : }
7219 5960656 : else if (SLP_TREE_PERMUTE_P (node)
7220 139866 : && SLP_TREE_CHILDREN (node).length () == 1
7221 120194 : && (child = SLP_TREE_CHILDREN (node)[0])
7222 3180291 : && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
7223 120194 : .is_constant (&imin)))
7224 : {
7225 : /* If the child has the same vector size as this node,
7226 : reversing the permutation can make the permutation a no-op.
7227 : In other cases it can change a true permutation into a
7228 : full-vector extract. */
7229 120194 : tmp_perm.reserve (SLP_TREE_LANES (node));
7230 445327 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7231 204939 : tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
7232 : }
7233 : else
7234 2920231 : continue;
7235 :
7236 813765 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7237 : {
7238 509240 : unsigned idx = tmp_perm[j];
7239 509240 : imin = MIN (imin, idx);
7240 509240 : imax = MAX (imax, idx);
7241 509240 : if (idx - tmp_perm[0] != j)
7242 160812 : any_permute = true;
7243 : }
7244 : /* If the span doesn't match we'd disrupt VF computation, avoid
7245 : that for now. */
7246 304525 : if (imax - imin + 1 != SLP_TREE_LANES (node))
7247 91328 : continue;
7248 : /* If there's no permute no need to split one out. In this case
7249 : we can consider turning a load into a permuted load, if that
7250 : turns out to be cheaper than alternatives. */
7251 213197 : if (!any_permute)
7252 : {
7253 191810 : partition.layout = -1;
7254 191810 : continue;
7255 : }
7256 :
7257 : /* For now only handle true permutes, like
7258 : vect_attempt_slp_rearrange_stmts did. This allows us to be lazy
7259 : when permuting constants and invariants keeping the permute
7260 : bijective. */
7261 21387 : auto_sbitmap load_index (SLP_TREE_LANES (node));
7262 21387 : bitmap_clear (load_index);
7263 106668 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7264 63894 : bitmap_set_bit (load_index, tmp_perm[j] - imin);
7265 : unsigned j;
7266 84438 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
7267 63206 : if (!bitmap_bit_p (load_index, j))
7268 : break;
7269 21387 : if (j != SLP_TREE_LANES (node))
7270 155 : continue;
7271 :
7272 21232 : vec<unsigned> perm = vNULL;
7273 21232 : perm.safe_grow (SLP_TREE_LANES (node), true);
7274 105391 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7275 62927 : perm[j] = tmp_perm[j] - imin;
7276 :
7277 42464 : if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
7278 : {
7279 : /* Continue to use existing layouts, but don't add any more. */
7280 0 : int *entry = layout_ids.get (perm);
7281 0 : partition.layout = entry ? *entry : 0;
7282 0 : perm.release ();
7283 : }
7284 : else
7285 : {
7286 21232 : bool existed;
7287 21232 : int &layout_i = layout_ids.get_or_insert (perm, &existed);
7288 21232 : if (existed)
7289 7971 : perm.release ();
7290 : else
7291 : {
7292 13261 : layout_i = m_perms.length ();
7293 13261 : m_perms.safe_push (perm);
7294 : }
7295 21232 : partition.layout = layout_i;
7296 : }
7297 21387 : }
7298 :
7299 : /* Initially assume that every layout is possible and has zero cost
7300 : in every partition. */
7301 714431 : m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
7302 1428862 : * m_perms.length ());
7303 :
7304 : /* We have to mark outgoing permutations facing non-associating-reduction
7305 : graph entries that are not represented as to be materialized.
7306 : slp_inst_kind_bb_reduc currently only covers associatable reductions. */
7307 3676464 : for (slp_instance instance : m_vinfo->slp_instances)
7308 1533171 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
7309 : {
7310 7054 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7311 7054 : m_partitions[m_vertices[node_i].partition].layout = 0;
7312 : }
7313 1526117 : else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
7314 : {
7315 2306 : stmt_vec_info stmt_info
7316 2306 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
7317 2306 : vect_reduc_info reduc_info
7318 2306 : = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
7319 : SLP_INSTANCE_TREE (instance));
7320 2306 : if (needs_fold_left_reduction_p (TREE_TYPE
7321 : (gimple_get_lhs (stmt_info->stmt)),
7322 : VECT_REDUC_INFO_CODE (reduc_info)))
7323 : {
7324 100 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7325 100 : m_partitions[m_vertices[node_i].partition].layout = 0;
7326 : }
7327 : }
7328 :
7329 : /* Check which layouts each node and partition can handle. Calculate the
7330 : weights associated with inserting layout changes on edges. */
7331 5814708 : for (unsigned int node_i : m_partitioned_nodes)
7332 : {
7333 3671415 : auto &vertex = m_vertices[node_i];
7334 3671415 : auto &partition = m_partitions[vertex.partition];
7335 3671415 : slp_tree node = vertex.node;
7336 :
7337 3671415 : vertex.weight = vect_slp_node_weight (m_vinfo, node);
7338 :
7339 3671415 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
7340 : {
7341 : /* We do not handle stores with a permutation, so all
7342 : incoming permutations must have been materialized.
7343 :
7344 : We also don't handle masked grouped loads, which lack a
7345 : permutation vector. In this case the memory locations
7346 : form an implicit second input to the loads, on top of the
7347 : explicit mask input, and the memory input's layout cannot
7348 : be changed.
7349 :
7350 : On the other hand, we do support permuting gather loads and
7351 : masked gather loads, where each scalar load is independent
7352 : of the others. This can be useful if the address/index input
7353 : benefits from permutation. */
7354 3528322 : if (STMT_VINFO_DATA_REF (rep)
7355 1700275 : && STMT_VINFO_GROUPED_ACCESS (rep)
7356 4529941 : && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
7357 817288 : partition.layout = 0;
7358 :
7359 : /* We cannot change the layout of an operation that is
7360 : not independent on lanes. Note this is an explicit
7361 : negative list since that's much shorter than the respective
7362 : positive one but it's critical to keep maintaining it. */
7363 3528322 : if (is_gimple_call (STMT_VINFO_STMT (rep)))
7364 33412 : switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
7365 : {
7366 1098 : case CFN_COMPLEX_ADD_ROT90:
7367 1098 : case CFN_COMPLEX_ADD_ROT270:
7368 1098 : case CFN_COMPLEX_MUL:
7369 1098 : case CFN_COMPLEX_MUL_CONJ:
7370 1098 : case CFN_VEC_ADDSUB:
7371 1098 : case CFN_VEC_FMADDSUB:
7372 1098 : case CFN_VEC_FMSUBADD:
7373 1098 : partition.layout = 0;
7374 : default:;
7375 : }
7376 : }
7377 :
7378 8296583 : auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
7379 : {
7380 4625168 : auto &other_vertex = m_vertices[other_node_i];
7381 :
7382 : /* Count the number of edges from earlier partitions and the number
7383 : of edges to later partitions. */
7384 4625168 : if (other_vertex.partition < vertex.partition)
7385 2312584 : partition.in_degree += 1;
7386 : else
7387 2312584 : partition.out_degree += 1;
7388 :
7389 : /* If the current node uses the result of OTHER_NODE_I, accumulate
7390 : the effects of that. */
7391 4625168 : if (ud->src == int (node_i))
7392 : {
7393 2312584 : other_vertex.out_weight += vertex.weight;
7394 2312584 : other_vertex.out_degree += 1;
7395 : }
7396 8296583 : };
7397 3671415 : for_each_partition_edge (node_i, process_edge);
7398 : }
7399 714431 : }
7400 :
7401 : /* Return the incoming costs for node NODE_I, assuming that each input keeps
7402 : its current (provisional) choice of layout. The inputs do not necessarily
7403 : have the same layout as each other. */
7404 :
7405 : slpg_layout_cost
7406 4461 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
7407 : {
7408 4461 : auto &vertex = m_vertices[node_i];
7409 4461 : slpg_layout_cost cost;
7410 14732 : auto add_cost = [&](graph_edge *, unsigned int other_node_i)
7411 : {
7412 10271 : auto &other_vertex = m_vertices[other_node_i];
7413 10271 : if (other_vertex.partition < vertex.partition)
7414 : {
7415 6894 : auto &other_partition = m_partitions[other_vertex.partition];
7416 13788 : auto &other_costs = partition_layout_costs (other_vertex.partition,
7417 6894 : other_partition.layout);
7418 6894 : slpg_layout_cost this_cost = other_costs.in_cost;
7419 6894 : this_cost.add_serial_cost (other_costs.internal_cost);
7420 6894 : this_cost.split (other_partition.out_degree);
7421 6894 : cost.add_parallel_cost (this_cost);
7422 : }
7423 14732 : };
7424 4461 : for_each_partition_edge (node_i, add_cost);
7425 4461 : return cost;
7426 : }
7427 :
7428 : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
7429 : and layout LAYOUT2_I on cross-partition use-to-def edge UD. Return
7430 : slpg_layout_cost::impossible () if the change isn't possible. */
7431 :
7432 : slpg_layout_cost
7433 750877 : vect_optimize_slp_pass::
7434 : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
7435 : unsigned int layout2_i)
7436 : {
7437 750877 : auto &def_vertex = m_vertices[ud->dest];
7438 750877 : auto &use_vertex = m_vertices[ud->src];
7439 750877 : auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
7440 750877 : auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
7441 750877 : auto factor = change_layout_cost (def_vertex.node, def_layout_i,
7442 : use_layout_i);
7443 750877 : if (factor < 0)
7444 6475 : return slpg_layout_cost::impossible ();
7445 :
7446 : /* We have a choice of putting the layout change at the site of the
7447 : definition or at the site of the use. Prefer the former when
7448 : optimizing for size or when the execution frequency of the
7449 : definition is no greater than the combined execution frequencies of
7450 : the uses. When putting the layout change at the site of the definition,
7451 : divvy up the cost among all consumers. */
7452 744402 : if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
7453 : {
7454 701658 : slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
7455 701658 : cost.split (def_vertex.out_degree);
7456 701658 : return cost;
7457 : }
7458 42744 : return { use_vertex.weight * factor, m_optimize_size };
7459 : }
7460 :
7461 : /* UD represents a use-def link between FROM_NODE_I and a node in a later
7462 : partition; FROM_NODE_I could be the definition node or the use node.
7463 : The node at the other end of the link wants to use layout TO_LAYOUT_I.
7464 : Return the cost of any necessary fix-ups on edge UD, or return
7465 : slpg_layout_cost::impossible () if the change isn't possible.
7466 :
7467 : At this point, FROM_NODE_I's partition has chosen the cheapest
7468 : layout based on the information available so far, but this choice
7469 : is only provisional. */
7470 :
7471 : slpg_layout_cost
7472 200652 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
7473 : unsigned int to_layout_i)
7474 : {
7475 200652 : auto &from_vertex = m_vertices[from_node_i];
7476 200652 : unsigned int from_partition_i = from_vertex.partition;
7477 200652 : slpg_partition_info &from_partition = m_partitions[from_partition_i];
7478 200652 : gcc_assert (from_partition.layout >= 0);
7479 :
7480 : /* First calculate the cost on the assumption that FROM_PARTITION sticks
7481 : with its current layout preference. */
7482 200652 : slpg_layout_cost cost = slpg_layout_cost::impossible ();
7483 200652 : auto edge_cost = edge_layout_cost (ud, from_node_i,
7484 200652 : from_partition.layout, to_layout_i);
7485 200652 : if (edge_cost.is_possible ())
7486 : {
7487 394460 : auto &from_costs = partition_layout_costs (from_partition_i,
7488 197230 : from_partition.layout);
7489 197230 : cost = from_costs.in_cost;
7490 197230 : cost.add_serial_cost (from_costs.internal_cost);
7491 197230 : cost.split (from_partition.out_degree);
7492 197230 : cost.add_serial_cost (edge_cost);
7493 : }
7494 3422 : else if (from_partition.layout == 0)
7495 : /* We must allow the source partition to have layout 0 as a fallback,
7496 : in case all other options turn out to be impossible. */
7497 3422 : return cost;
7498 :
7499 : /* Take the minimum of that cost and the cost that applies if
7500 : FROM_PARTITION instead switches to TO_LAYOUT_I. */
7501 197230 : auto &direct_layout_costs = partition_layout_costs (from_partition_i,
7502 : to_layout_i);
7503 197230 : if (direct_layout_costs.is_possible ())
7504 : {
7505 173169 : slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
7506 173169 : direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
7507 173169 : direct_cost.split (from_partition.out_degree);
7508 173169 : if (!cost.is_possible ()
7509 173169 : || direct_cost.is_better_than (cost, m_optimize_size))
7510 34721 : cost = direct_cost;
7511 : }
7512 :
7513 197230 : return cost;
7514 : }
7515 :
7516 : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
7517 : partition; TO_NODE_I could be the definition node or the use node.
7518 : The node at the other end of the link wants to use layout FROM_LAYOUT_I;
7519 : return the cost of any necessary fix-ups on edge UD, or
7520 : slpg_layout_cost::impossible () if the choice cannot be made.
7521 :
7522 : At this point, TO_NODE_I's partition has a fixed choice of layout. */
7523 :
7524 : slpg_layout_cost
7525 180052 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
7526 : unsigned int from_layout_i)
7527 : {
7528 180052 : auto &to_vertex = m_vertices[to_node_i];
7529 180052 : unsigned int to_partition_i = to_vertex.partition;
7530 180052 : slpg_partition_info &to_partition = m_partitions[to_partition_i];
7531 180052 : gcc_assert (to_partition.layout >= 0);
7532 :
7533 : /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
7534 : adjusted for this input having layout FROM_LAYOUT_I. Assume that
7535 : any other inputs keep their current choice of layout. */
7536 180052 : auto &to_costs = partition_layout_costs (to_partition_i,
7537 : to_partition.layout);
7538 180052 : if (ud->src == int (to_node_i)
7539 179770 : && SLP_TREE_PERMUTE_P (to_vertex.node))
7540 : {
7541 11642 : auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
7542 11642 : auto old_layout = from_partition.layout;
7543 11642 : from_partition.layout = from_layout_i;
7544 23284 : int factor = internal_node_cost (to_vertex.node, -1,
7545 11642 : to_partition.layout);
7546 11642 : from_partition.layout = old_layout;
7547 11642 : if (factor >= 0)
7548 : {
7549 10738 : slpg_layout_cost cost = to_costs.out_cost;
7550 10738 : cost.add_serial_cost ({ to_vertex.weight * factor,
7551 : m_optimize_size });
7552 10738 : cost.split (to_partition.in_degree);
7553 10738 : return cost;
7554 : }
7555 : }
7556 :
7557 : /* Compute the cost if we insert any necessary layout change on edge UD. */
7558 169314 : auto edge_cost = edge_layout_cost (ud, to_node_i,
7559 169314 : to_partition.layout, from_layout_i);
7560 169314 : if (edge_cost.is_possible ())
7561 : {
7562 169314 : slpg_layout_cost cost = to_costs.out_cost;
7563 169314 : cost.add_serial_cost (to_costs.internal_cost);
7564 169314 : cost.split (to_partition.in_degree);
7565 169314 : cost.add_serial_cost (edge_cost);
7566 169314 : return cost;
7567 : }
7568 :
7569 0 : return slpg_layout_cost::impossible ();
7570 : }
7571 :
7572 : /* Make a forward pass through the partitions, accumulating input costs.
7573 : Make a tentative (provisional) choice of layout for each partition,
7574 : ensuring that this choice still allows later partitions to keep
7575 : their original layout. */
7576 :
7577 : void
7578 6621 : vect_optimize_slp_pass::forward_pass ()
7579 : {
7580 134158 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
7581 : ++partition_i)
7582 : {
7583 127537 : auto &partition = m_partitions[partition_i];
7584 :
7585 : /* If the partition consists of a single VEC_PERM_EXPR, precompute
7586 : the incoming cost that would apply if every predecessor partition
7587 : keeps its current layout. This is used within the loop below. */
7588 127537 : slpg_layout_cost in_cost;
7589 127537 : slp_tree single_node = nullptr;
7590 127537 : if (partition.node_end == partition.node_begin + 1)
7591 : {
7592 121146 : unsigned int node_i = m_partitioned_nodes[partition.node_begin];
7593 121146 : single_node = m_vertices[node_i].node;
7594 121146 : if (SLP_TREE_PERMUTE_P (single_node))
7595 4461 : in_cost = total_in_cost (node_i);
7596 : }
7597 :
7598 : /* Go through the possible layouts. Decide which ones are valid
7599 : for this partition and record which of the valid layouts has
7600 : the lowest cost. */
7601 127537 : unsigned int min_layout_i = 0;
7602 127537 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7603 393499 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7604 : {
7605 265962 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7606 265962 : if (!layout_costs.is_possible ())
7607 68313 : continue;
7608 :
7609 : /* If the recorded layout is already 0 then the layout cannot
7610 : change. */
7611 265962 : if (partition.layout == 0 && layout_i != 0)
7612 : {
7613 46065 : layout_costs.mark_impossible ();
7614 46065 : continue;
7615 : }
7616 :
7617 219897 : bool is_possible = true;
7618 447232 : for (unsigned int order_i = partition.node_begin;
7619 447232 : order_i < partition.node_end; ++order_i)
7620 : {
7621 246388 : unsigned int node_i = m_partitioned_nodes[order_i];
7622 246388 : auto &vertex = m_vertices[node_i];
7623 :
7624 : /* Reject the layout if it is individually incompatible
7625 : with any node in the partition. */
7626 246388 : if (!is_compatible_layout (vertex.node, layout_i))
7627 : {
7628 17439 : is_possible = false;
7629 19053 : break;
7630 : }
7631 :
7632 613754 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7633 : {
7634 384805 : auto &other_vertex = m_vertices[other_node_i];
7635 384805 : if (other_vertex.partition < vertex.partition)
7636 : {
7637 : /* Accumulate the incoming costs from earlier
7638 : partitions, plus the cost of any layout changes
7639 : on UD itself. */
7640 200652 : auto cost = forward_cost (ud, other_node_i, layout_i);
7641 200652 : if (!cost.is_possible ())
7642 3422 : is_possible = false;
7643 : else
7644 197230 : layout_costs.in_cost.add_parallel_cost (cost);
7645 : }
7646 : else
7647 : /* Reject the layout if it would make layout 0 impossible
7648 : for later partitions. This amounts to testing that the
7649 : target supports reversing the layout change on edges
7650 : to later partitions.
7651 :
7652 : In principle, it might be possible to push a layout
7653 : change all the way down a graph, so that it never
7654 : needs to be reversed and so that the target doesn't
7655 : need to support the reverse operation. But it would
7656 : be awkward to bail out if we hit a partition that
7657 : does not support the new layout, especially since
7658 : we are not dealing with a lattice. */
7659 184153 : is_possible &= edge_layout_cost (ud, other_node_i, 0,
7660 184153 : layout_i).is_possible ();
7661 613754 : };
7662 228949 : for_each_partition_edge (node_i, add_cost);
7663 :
7664 : /* Accumulate the cost of using LAYOUT_I within NODE,
7665 : both for the inputs and the outputs. */
7666 228949 : int factor = internal_node_cost (vertex.node, layout_i,
7667 : layout_i);
7668 228949 : if (factor < 0)
7669 : {
7670 1614 : is_possible = false;
7671 1614 : break;
7672 : }
7673 227335 : else if (factor)
7674 36558 : layout_costs.internal_cost.add_serial_cost
7675 36558 : ({ vertex.weight * factor, m_optimize_size });
7676 : }
7677 219897 : if (!is_possible)
7678 : {
7679 22248 : layout_costs.mark_impossible ();
7680 22248 : continue;
7681 : }
7682 :
7683 : /* Combine the incoming and partition-internal costs. */
7684 197649 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7685 197649 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7686 :
7687 : /* If this partition consists of a single VEC_PERM_EXPR, see
7688 : if the VEC_PERM_EXPR can be changed to support output layout
7689 : LAYOUT_I while keeping all the provisional choices of input
7690 : layout. */
7691 197649 : if (single_node && SLP_TREE_PERMUTE_P (single_node))
7692 : {
7693 7766 : int factor = internal_node_cost (single_node, -1, layout_i);
7694 7766 : if (factor >= 0)
7695 : {
7696 7043 : auto weight = m_vertices[single_node->vertex].weight;
7697 7043 : slpg_layout_cost internal_cost
7698 7043 : = { weight * factor, m_optimize_size };
7699 :
7700 7043 : slpg_layout_cost alt_cost = in_cost;
7701 7043 : alt_cost.add_serial_cost (internal_cost);
7702 7043 : if (alt_cost.is_better_than (combined_cost, m_optimize_size))
7703 : {
7704 1732 : combined_cost = alt_cost;
7705 1732 : layout_costs.in_cost = in_cost;
7706 1732 : layout_costs.internal_cost = internal_cost;
7707 : }
7708 : }
7709 : }
7710 :
7711 : /* Record the layout with the lowest cost. Prefer layout 0 in
7712 : the event of a tie between it and another layout. */
7713 197649 : if (!min_layout_cost.is_possible ()
7714 70112 : || combined_cost.is_better_than (min_layout_cost,
7715 : m_optimize_size))
7716 : {
7717 144822 : min_layout_i = layout_i;
7718 144822 : min_layout_cost = combined_cost;
7719 : }
7720 : }
7721 :
7722 : /* This loop's handling of earlier partitions should ensure that
7723 : choosing the original layout for the current partition is no
7724 : less valid than it was in the original graph, even with the
7725 : provisional layout choices for those earlier partitions. */
7726 127537 : gcc_assert (min_layout_cost.is_possible ());
7727 127537 : partition.layout = min_layout_i;
7728 : }
7729 6621 : }
7730 :
7731 : /* Make a backward pass through the partitions, accumulating output costs.
7732 : Make a final choice of layout for each partition. */
7733 :
7734 : void
7735 6621 : vect_optimize_slp_pass::backward_pass ()
7736 : {
7737 140779 : for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
7738 : {
7739 127537 : auto &partition = m_partitions[partition_i];
7740 :
7741 127537 : unsigned int min_layout_i = 0;
7742 127537 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7743 393499 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7744 : {
7745 265962 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7746 265962 : if (!layout_costs.is_possible ())
7747 68313 : continue;
7748 :
7749 : /* Accumulate the costs from successor partitions. */
7750 197649 : bool is_possible = true;
7751 421758 : for (unsigned int order_i = partition.node_begin;
7752 421758 : order_i < partition.node_end; ++order_i)
7753 : {
7754 224109 : unsigned int node_i = m_partitioned_nodes[order_i];
7755 224109 : auto &vertex = m_vertices[node_i];
7756 600919 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7757 : {
7758 376810 : auto &other_vertex = m_vertices[other_node_i];
7759 376810 : auto &other_partition = m_partitions[other_vertex.partition];
7760 376810 : if (other_vertex.partition > vertex.partition)
7761 : {
7762 : /* Accumulate the incoming costs from later
7763 : partitions, plus the cost of any layout changes
7764 : on UD itself. */
7765 180052 : auto cost = backward_cost (ud, other_node_i, layout_i);
7766 180052 : if (!cost.is_possible ())
7767 0 : is_possible = false;
7768 : else
7769 180052 : layout_costs.out_cost.add_parallel_cost (cost);
7770 : }
7771 : else
7772 : /* Make sure that earlier partitions can (if necessary
7773 : or beneficial) keep the layout that they chose in
7774 : the forward pass. This ensures that there is at
7775 : least one valid choice of layout. */
7776 196758 : is_possible &= edge_layout_cost (ud, other_node_i,
7777 196758 : other_partition.layout,
7778 196758 : layout_i).is_possible ();
7779 600919 : };
7780 224109 : for_each_partition_edge (node_i, add_cost);
7781 : }
7782 197649 : if (!is_possible)
7783 : {
7784 0 : layout_costs.mark_impossible ();
7785 0 : continue;
7786 : }
7787 :
7788 : /* Locally combine the costs from the forward and backward passes.
7789 : (This combined cost is not passed on, since that would lead
7790 : to double counting.) */
7791 197649 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7792 197649 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7793 197649 : combined_cost.add_serial_cost (layout_costs.out_cost);
7794 :
7795 : /* Record the layout with the lowest cost. Prefer layout 0 in
7796 : the event of a tie between it and another layout. */
7797 197649 : if (!min_layout_cost.is_possible ()
7798 70112 : || combined_cost.is_better_than (min_layout_cost,
7799 : m_optimize_size))
7800 : {
7801 138516 : min_layout_i = layout_i;
7802 138516 : min_layout_cost = combined_cost;
7803 : }
7804 : }
7805 :
7806 127537 : gcc_assert (min_layout_cost.is_possible ());
7807 127537 : partition.layout = min_layout_i;
7808 : }
7809 6621 : }
7810 :
7811 : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
7812 : NODE already has the layout that was selected for its partition. */
7813 :
7814 : slp_tree
7815 177819 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
7816 : unsigned int to_layout_i)
7817 : {
7818 177819 : unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
7819 177819 : slp_tree result = m_node_layouts[result_i];
7820 177819 : if (result)
7821 : return result;
7822 :
7823 176947 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
7824 176947 : || (SLP_TREE_DEF_TYPE (node) == vect_external_def
7825 : /* We can't permute vector defs in place. */
7826 21552 : && SLP_TREE_VEC_DEFS (node).is_empty ()))
7827 : {
7828 : /* If the vector is uniform or unchanged, there's nothing to do. */
7829 44913 : if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
7830 : result = node;
7831 : else
7832 : {
7833 3552 : auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
7834 3552 : result = vect_create_new_slp_node (scalar_ops);
7835 3552 : vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
7836 : }
7837 : }
7838 : else
7839 : {
7840 132034 : unsigned int partition_i = m_vertices[node->vertex].partition;
7841 132034 : unsigned int from_layout_i = m_partitions[partition_i].layout;
7842 132034 : if (from_layout_i == to_layout_i)
7843 130846 : return node;
7844 :
7845 : /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
7846 : permutation instead of a serial one. Leave the new permutation
7847 : in TMP_PERM on success. */
7848 1188 : auto_lane_permutation_t tmp_perm;
7849 1188 : unsigned int num_inputs = 1;
7850 1188 : if (SLP_TREE_PERMUTE_P (node))
7851 : {
7852 104 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7853 104 : if (from_layout_i != 0)
7854 44 : vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
7855 104 : if (to_layout_i != 0)
7856 64 : vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
7857 104 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7858 : tmp_perm,
7859 104 : SLP_TREE_CHILDREN (node),
7860 : false) >= 0)
7861 104 : num_inputs = SLP_TREE_CHILDREN (node).length ();
7862 : else
7863 0 : tmp_perm.truncate (0);
7864 : }
7865 :
7866 1188 : if (dump_enabled_p ())
7867 : {
7868 70 : if (tmp_perm.length () > 0)
7869 6 : dump_printf_loc (MSG_NOTE, vect_location,
7870 : "duplicating permutation node %p with"
7871 : " layout %d\n",
7872 : (void *) node, to_layout_i);
7873 : else
7874 64 : dump_printf_loc (MSG_NOTE, vect_location,
7875 : "inserting permutation node in place of %p\n",
7876 : (void *) node);
7877 : }
7878 :
7879 1188 : unsigned int num_lanes = SLP_TREE_LANES (node);
7880 1188 : result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
7881 1188 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
7882 : {
7883 1188 : auto &stmts = SLP_TREE_SCALAR_STMTS (result);
7884 1188 : stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
7885 1188 : if (from_layout_i != 0)
7886 483 : vect_slp_permute (m_perms[from_layout_i], stmts, false);
7887 1188 : if (to_layout_i != 0)
7888 715 : vect_slp_permute (m_perms[to_layout_i], stmts, true);
7889 : }
7890 1188 : SLP_TREE_LANES (result) = num_lanes;
7891 1188 : SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
7892 1188 : result->vertex = -1;
7893 :
7894 1188 : auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
7895 1188 : if (tmp_perm.length ())
7896 : {
7897 104 : lane_perm.safe_splice (tmp_perm);
7898 104 : SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
7899 : }
7900 : else
7901 : {
7902 1084 : lane_perm.create (num_lanes);
7903 4428 : for (unsigned j = 0; j < num_lanes; ++j)
7904 2260 : lane_perm.quick_push ({ 0, j });
7905 1084 : if (from_layout_i != 0)
7906 439 : vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
7907 1084 : if (to_layout_i != 0)
7908 651 : vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
7909 1084 : SLP_TREE_CHILDREN (result).safe_push (node);
7910 : }
7911 4824 : for (slp_tree child : SLP_TREE_CHILDREN (result))
7912 1260 : child->refcnt++;
7913 1188 : }
7914 46101 : m_node_layouts[result_i] = result;
7915 46101 : return result;
7916 : }
7917 :
7918 : /* Apply the chosen vector layouts to the SLP graph. */
7919 :
7920 : void
7921 12669 : vect_optimize_slp_pass::materialize ()
7922 : {
7923 : /* We no longer need the costs, so avoid having two O(N * P) arrays
7924 : live at the same time. */
7925 12669 : m_partition_layout_costs.release ();
7926 38007 : m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
7927 :
7928 25338 : auto_sbitmap fully_folded (m_vertices.length ());
7929 12669 : bitmap_clear (fully_folded);
7930 193807 : for (unsigned int node_i : m_partitioned_nodes)
7931 : {
7932 155800 : auto &vertex = m_vertices[node_i];
7933 155800 : slp_tree node = vertex.node;
7934 155800 : int layout_i = m_partitions[vertex.partition].layout;
7935 155800 : gcc_assert (layout_i >= 0);
7936 :
7937 : /* Rearrange the scalar statements to match the chosen layout. */
7938 155800 : if (layout_i > 0)
7939 24337 : vect_slp_permute (m_perms[layout_i],
7940 24337 : SLP_TREE_SCALAR_STMTS (node), true);
7941 :
7942 : /* Update load and lane permutations. */
7943 155800 : if (SLP_TREE_PERMUTE_P (node))
7944 : {
7945 : /* First try to absorb the input vector layouts. If that fails,
7946 : force the inputs to have layout LAYOUT_I too. We checked that
7947 : that was possible before deciding to use nonzero output layouts.
7948 : (Note that at this stage we don't really have any guarantee that
7949 : the target supports the original VEC_PERM_EXPR.) */
7950 6660 : auto &perm = SLP_TREE_LANE_PERMUTATION (node);
7951 6660 : auto_lane_permutation_t tmp_perm;
7952 6660 : tmp_perm.safe_splice (perm);
7953 6660 : change_vec_perm_layout (node, tmp_perm, -1, layout_i);
7954 6660 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7955 : tmp_perm,
7956 6660 : SLP_TREE_CHILDREN (node),
7957 : false) >= 0)
7958 : {
7959 6019 : if (dump_enabled_p ()
7960 6939 : && !std::equal (tmp_perm.begin (), tmp_perm.end (),
7961 : perm.begin ()))
7962 58 : dump_printf_loc (MSG_NOTE, vect_location,
7963 : "absorbing input layouts into %p\n",
7964 : (void *) node);
7965 34116 : std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
7966 6019 : bitmap_set_bit (fully_folded, node_i);
7967 : }
7968 : else
7969 : {
7970 : /* Not MSG_MISSED because it would make no sense to users. */
7971 641 : if (dump_enabled_p ())
7972 46 : dump_printf_loc (MSG_NOTE, vect_location,
7973 : "failed to absorb input layouts into %p\n",
7974 : (void *) node);
7975 641 : change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
7976 : }
7977 6660 : }
7978 : else
7979 : {
7980 149140 : gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
7981 149140 : auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
7982 149140 : if (layout_i > 0)
7983 : /* ??? When we handle non-bijective permutes the idea
7984 : is that we can force the load-permutation to be
7985 : { min, min + 1, min + 2, ... max }. But then the
7986 : scalar defs might no longer match the lane content
7987 : which means wrong-code with live lane vectorization.
7988 : So we possibly have to have NULL entries for those. */
7989 23423 : vect_slp_permute (m_perms[layout_i], load_perm, true);
7990 : }
7991 : }
7992 :
7993 : /* Do this before any nodes disappear, since it involves a walk
7994 : over the leaves. */
7995 12669 : remove_redundant_permutations ();
7996 :
7997 : /* Replace each child with a correctly laid-out version. */
7998 193807 : for (unsigned int node_i : m_partitioned_nodes)
7999 : {
8000 : /* Skip nodes that have already been handled above. */
8001 155800 : if (bitmap_bit_p (fully_folded, node_i))
8002 6019 : continue;
8003 :
8004 149781 : auto &vertex = m_vertices[node_i];
8005 149781 : int in_layout_i = m_partitions[vertex.partition].layout;
8006 149781 : gcc_assert (in_layout_i >= 0);
8007 :
8008 : unsigned j;
8009 : slp_tree child;
8010 447763 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
8011 : {
8012 183836 : if (!child)
8013 6017 : continue;
8014 :
8015 177819 : slp_tree new_child = get_result_with_layout (child, in_layout_i);
8016 177819 : if (new_child != child)
8017 : {
8018 5321 : vect_free_slp_tree (child);
8019 5321 : SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
8020 5321 : new_child->refcnt += 1;
8021 : }
8022 : }
8023 : }
8024 12669 : }
8025 :
8026 : /* Elide load permutations that are not necessary. Such permutations might
8027 : be pre-existing, rather than created by the layout optimizations. */
8028 :
8029 : void
8030 714431 : vect_optimize_slp_pass::remove_redundant_permutations ()
8031 : {
8032 4735974 : for (unsigned int node_i : m_leafs)
8033 : {
8034 2592681 : slp_tree node = m_vertices[node_i].node;
8035 2592681 : if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
8036 1961691 : continue;
8037 :
8038 : /* In basic block vectorization we allow any subchain of an interleaving
8039 : chain.
8040 : FORNOW: not in loop SLP because of realignment complications. */
8041 630990 : if (is_a <bb_vec_info> (m_vinfo))
8042 : {
8043 184018 : bool subchain_p = true;
8044 : stmt_vec_info next_load_info = NULL;
8045 : stmt_vec_info load_info;
8046 : unsigned j;
8047 184018 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
8048 : {
8049 152219 : if (j != 0
8050 152219 : && (next_load_info != load_info
8051 66339 : || ! load_info
8052 66339 : || DR_GROUP_GAP (load_info) != 1))
8053 : {
8054 : subchain_p = false;
8055 : break;
8056 : }
8057 121121 : next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
8058 : }
8059 62897 : if (subchain_p)
8060 : {
8061 31799 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8062 31799 : continue;
8063 : }
8064 : }
8065 : else
8066 : {
8067 568093 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
8068 568093 : bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
8069 : /* When this isn't a grouped access we know it's single element
8070 : and contiguous. */
8071 568093 : if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
8072 : {
8073 446659 : if (!this_load_permuted
8074 446659 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8075 445898 : || SLP_TREE_LANES (node) == 1))
8076 445900 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8077 446659 : continue;
8078 : }
8079 121434 : stmt_vec_info first_stmt_info
8080 121434 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
8081 121938 : if (!this_load_permuted
8082 : /* The load requires permutation when unrolling exposes
8083 : a gap either because the group is larger than the SLP
8084 : group-size or because there is a gap between the groups. */
8085 121434 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8086 98935 : || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
8087 136 : && DR_GROUP_GAP (first_stmt_info) == 0)))
8088 : {
8089 504 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8090 504 : continue;
8091 : }
8092 : }
8093 : }
8094 714431 : }
8095 :
8096 : /* Print the partition graph and layout information to the dump file. */
8097 :
8098 : void
8099 684 : vect_optimize_slp_pass::dump ()
8100 : {
8101 684 : dump_printf_loc (MSG_NOTE, vect_location,
8102 : "SLP optimize permutations:\n");
8103 1381 : for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
8104 : {
8105 697 : dump_printf_loc (MSG_NOTE, vect_location, " %d: { ", layout_i);
8106 697 : const char *sep = "";
8107 5938 : for (unsigned int idx : m_perms[layout_i])
8108 : {
8109 3847 : dump_printf (MSG_NOTE, "%s%d", sep, idx);
8110 3847 : sep = ", ";
8111 : }
8112 697 : dump_printf (MSG_NOTE, " }\n");
8113 : }
8114 684 : dump_printf_loc (MSG_NOTE, vect_location,
8115 : "SLP optimize partitions:\n");
8116 5736 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
8117 : ++partition_i)
8118 : {
8119 5052 : auto &partition = m_partitions[partition_i];
8120 5052 : dump_printf_loc (MSG_NOTE, vect_location, " -------------\n");
8121 5052 : dump_printf_loc (MSG_NOTE, vect_location,
8122 : " partition %d (layout %d):\n",
8123 : partition_i, partition.layout);
8124 5052 : dump_printf_loc (MSG_NOTE, vect_location, " nodes:\n");
8125 10338 : for (unsigned int order_i = partition.node_begin;
8126 10338 : order_i < partition.node_end; ++order_i)
8127 : {
8128 5286 : auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
8129 10572 : dump_printf_loc (MSG_NOTE, vect_location, " - %p:\n",
8130 5286 : (void *) vertex.node);
8131 5286 : dump_printf_loc (MSG_NOTE, vect_location,
8132 : " weight: %f\n",
8133 : vertex.weight.to_double ());
8134 5286 : if (vertex.out_degree)
8135 4134 : dump_printf_loc (MSG_NOTE, vect_location,
8136 : " out weight: %f (degree %d)\n",
8137 : vertex.out_weight.to_double (),
8138 : vertex.out_degree);
8139 5286 : if (SLP_TREE_PERMUTE_P (vertex.node))
8140 506 : dump_printf_loc (MSG_NOTE, vect_location,
8141 : " op: VEC_PERM_EXPR\n");
8142 4780 : else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
8143 4762 : dump_printf_loc (MSG_NOTE, vect_location,
8144 : " op template: %G", rep->stmt);
8145 : }
8146 5052 : dump_printf_loc (MSG_NOTE, vect_location, " edges:\n");
8147 10338 : for (unsigned int order_i = partition.node_begin;
8148 10338 : order_i < partition.node_end; ++order_i)
8149 : {
8150 5286 : unsigned int node_i = m_partitioned_nodes[order_i];
8151 5286 : auto &vertex = m_vertices[node_i];
8152 15930 : auto print_edge = [&](graph_edge *, unsigned int other_node_i)
8153 : {
8154 10644 : auto &other_vertex = m_vertices[other_node_i];
8155 10644 : if (other_vertex.partition < vertex.partition)
8156 5322 : dump_printf_loc (MSG_NOTE, vect_location,
8157 : " - %p [%d] --> %p\n",
8158 5322 : (void *) other_vertex.node,
8159 : other_vertex.partition,
8160 5322 : (void *) vertex.node);
8161 : else
8162 5322 : dump_printf_loc (MSG_NOTE, vect_location,
8163 : " - %p --> [%d] %p\n",
8164 5322 : (void *) vertex.node,
8165 : other_vertex.partition,
8166 5322 : (void *) other_vertex.node);
8167 15930 : };
8168 5286 : for_each_partition_edge (node_i, print_edge);
8169 : }
8170 :
8171 15355 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
8172 : {
8173 10303 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
8174 10303 : if (layout_costs.is_possible ())
8175 : {
8176 8514 : dump_printf_loc (MSG_NOTE, vect_location,
8177 : " layout %d:%s\n", layout_i,
8178 8514 : partition.layout == int (layout_i)
8179 : ? " (*)" : "");
8180 8514 : slpg_layout_cost combined_cost = layout_costs.in_cost;
8181 8514 : combined_cost.add_serial_cost (layout_costs.internal_cost);
8182 8514 : combined_cost.add_serial_cost (layout_costs.out_cost);
8183 : #define TEMPLATE "{depth: %f, total: %f}"
8184 8514 : dump_printf_loc (MSG_NOTE, vect_location,
8185 : " " TEMPLATE "\n",
8186 : layout_costs.in_cost.depth.to_double (),
8187 : layout_costs.in_cost.total.to_double ());
8188 8514 : dump_printf_loc (MSG_NOTE, vect_location,
8189 : " + " TEMPLATE "\n",
8190 : layout_costs.internal_cost.depth.to_double (),
8191 : layout_costs.internal_cost.total.to_double ());
8192 8514 : dump_printf_loc (MSG_NOTE, vect_location,
8193 : " + " TEMPLATE "\n",
8194 : layout_costs.out_cost.depth.to_double (),
8195 : layout_costs.out_cost.total.to_double ());
8196 8514 : dump_printf_loc (MSG_NOTE, vect_location,
8197 : " = " TEMPLATE "\n",
8198 : combined_cost.depth.to_double (),
8199 : combined_cost.total.to_double ());
8200 : #undef TEMPLATE
8201 : }
8202 : else
8203 1789 : dump_printf_loc (MSG_NOTE, vect_location,
8204 : " layout %d: rejected\n", layout_i);
8205 : }
8206 : }
8207 684 : }
8208 :
8209 : /* Masked load lanes discovery. */
8210 :
8211 : void
8212 714431 : vect_optimize_slp_pass::decide_masked_load_lanes ()
8213 : {
8214 7379317 : for (auto v : m_vertices)
8215 : {
8216 5236024 : slp_tree node = v.node;
8217 5236024 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8218 3669285 : || SLP_TREE_PERMUTE_P (node))
8219 1707702 : continue;
8220 3528322 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
8221 1700275 : if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
8222 : /* The mask has to be uniform. */
8223 1001619 : || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
8224 1001541 : || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
8225 3528407 : || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
8226 : IFN_MASK_LOAD))
8227 3528289 : continue;
8228 33 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8229 66 : if (STMT_VINFO_STRIDED_P (stmt_info)
8230 33 : || compare_step_with_zero (m_vinfo, stmt_info) <= 0
8231 63 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
8232 30 : DR_GROUP_SIZE (stmt_info),
8233 : true) == IFN_LAST)
8234 33 : continue;
8235 :
8236 : /* Uniform masks need to be suitably represented. */
8237 0 : slp_tree mask = SLP_TREE_CHILDREN (node)[0];
8238 0 : if (!SLP_TREE_PERMUTE_P (mask)
8239 0 : || SLP_TREE_CHILDREN (mask).length () != 1)
8240 0 : continue;
8241 0 : bool match = true;
8242 0 : for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
8243 0 : if (perm.first != 0 || perm.second != 0)
8244 : {
8245 : match = false;
8246 : break;
8247 : }
8248 0 : if (!match)
8249 0 : continue;
8250 :
8251 : /* Now see if the consumer side matches. */
8252 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8253 0 : pred; pred = pred->pred_next)
8254 : {
8255 0 : slp_tree pred_node = m_vertices[pred->src].node;
8256 : /* All consumers should be a permute with a single outgoing lane. */
8257 0 : if (!SLP_TREE_PERMUTE_P (pred_node)
8258 0 : || SLP_TREE_LANES (pred_node) != 1)
8259 : {
8260 : match = false;
8261 : break;
8262 : }
8263 0 : gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
8264 : }
8265 0 : if (!match)
8266 0 : continue;
8267 : /* Now we can mark the nodes as to use load lanes. */
8268 0 : node->ldst_lanes = true;
8269 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8270 0 : pred; pred = pred->pred_next)
8271 0 : m_vertices[pred->src].node->ldst_lanes = true;
8272 : /* The catch is we have to massage the mask. We have arranged
8273 : analyzed uniform masks to be represented by a splat VEC_PERM
8274 : which we can now simply elide as we cannot easily re-do SLP
8275 : discovery here. */
8276 0 : slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
8277 0 : SLP_TREE_REF_COUNT (new_mask)++;
8278 0 : SLP_TREE_CHILDREN (node)[0] = new_mask;
8279 0 : vect_free_slp_tree (mask);
8280 : }
8281 714431 : }
8282 :
8283 : /* Perform legitimizing attempts. This is intended to improve the
8284 : situation when layout 0 is not valid which is a situation the cost
8285 : based propagation does not handle well.
8286 : Return true if further layout optimization is possible, false if
8287 : the layout configuration should be considered final. */
8288 :
8289 : bool
8290 12669 : vect_optimize_slp_pass::legitimize ()
8291 : {
8292 : /* Perform a very simple legitimizing attempt by attempting to choose
8293 : a single layout for all partitions that will make all permutations
8294 : a noop. That should also be the optimal layout choice in case
8295 : layout zero is legitimate.
8296 : ??? Disconnected components of the SLP graph could have distinct
8297 : single layouts. */
8298 12669 : int single_layout_i = -1;
8299 12669 : unsigned deferred_up_to = -1U;
8300 40000 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8301 : ++partition_i)
8302 : {
8303 33935 : auto &partition = m_partitions[partition_i];
8304 33935 : if (single_layout_i == -1)
8305 : {
8306 16675 : single_layout_i = partition.layout;
8307 16675 : deferred_up_to = partition_i;
8308 : }
8309 17260 : else if (partition.layout == single_layout_i || partition.layout == -1)
8310 : ;
8311 : else
8312 : single_layout_i = 0;
8313 30660 : if (single_layout_i == 0)
8314 : return true;
8315 :
8316 27416 : if (single_layout_i != -1
8317 27416 : && !is_compatible_layout (partition, single_layout_i))
8318 : return true;
8319 : }
8320 :
8321 6065 : if (single_layout_i <= 0)
8322 : return true;
8323 :
8324 6645 : for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
8325 597 : if (!is_compatible_layout (m_partitions[partition_i],
8326 : single_layout_i))
8327 : return true;
8328 :
8329 18865 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8330 : ++partition_i)
8331 : {
8332 12817 : auto &partition = m_partitions[partition_i];
8333 12817 : partition.layout = single_layout_i;
8334 : }
8335 :
8336 : return false;
8337 : }
8338 :
8339 : /* Main entry point for the SLP graph optimization pass. */
8340 :
8341 : void
8342 714431 : vect_optimize_slp_pass::run ()
8343 : {
8344 714431 : build_graph ();
8345 714431 : create_partitions ();
8346 714431 : start_choosing_layouts ();
8347 714431 : if (m_perms.length () > 1)
8348 : {
8349 12669 : if (legitimize ())
8350 : {
8351 6621 : forward_pass ();
8352 6621 : backward_pass ();
8353 : }
8354 12669 : if (dump_enabled_p ())
8355 684 : dump ();
8356 12669 : materialize ();
8357 51268 : while (!m_perms.is_empty ())
8358 25930 : m_perms.pop ().release ();
8359 : }
8360 : else
8361 701762 : remove_redundant_permutations ();
8362 714431 : free_graph (m_slpg);
8363 714431 : build_graph ();
8364 714431 : decide_masked_load_lanes ();
8365 714431 : free_graph (m_slpg);
8366 714431 : }
8367 :
8368 : /* Apply CSE to NODE and its children using BST_MAP. */
8369 :
8370 : static void
8371 5653171 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
8372 : {
8373 5653171 : bool put_p = false;
8374 5653171 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
8375 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8376 : lack scalar stmts and thus CSE doesn't work via bst_map. Ideally
8377 : we'd have sth that works for all internal and external nodes. */
8378 5653171 : && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
8379 : {
8380 4058191 : slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
8381 4058191 : if (leader)
8382 : {
8383 : /* We've visited this node already. */
8384 420326 : if (!*leader || *leader == node)
8385 : return;
8386 :
8387 4320 : if (dump_enabled_p ())
8388 912 : dump_printf_loc (MSG_NOTE, vect_location,
8389 : "re-using SLP tree %p for %p\n",
8390 : (void *)*leader, (void *)node);
8391 4320 : vect_free_slp_tree (node);
8392 4320 : (*leader)->refcnt += 1;
8393 4320 : node = *leader;
8394 4320 : return;
8395 : }
8396 :
8397 : /* Avoid creating a cycle by populating the map only after recursion. */
8398 3637865 : bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
8399 3637865 : node->refcnt += 1;
8400 3637865 : put_p = true;
8401 : /* And recurse. */
8402 : }
8403 :
8404 15638322 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8405 4577299 : if (child)
8406 4120000 : vect_cse_slp_nodes (bst_map, child);
8407 :
8408 : /* Now record the node for CSE in other siblings. */
8409 5232845 : if (put_p)
8410 3637865 : *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
8411 : }
8412 :
8413 : /* Associate stmts with possible starts of a subset of lanes of NODE
8414 : in PART_STARTS. */
8415 :
8416 : static void
8417 2151585 : vect_cse_gather_part_starts (hash_set<slp_tree> &visited,
8418 : vec<vec<slp_tree>> part_starts, slp_tree node)
8419 : {
8420 : /* CSEing external nodes complicates scheduling since we materialize
8421 : those at the latest position, so avoid that. */
8422 2151585 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8423 2151585 : || visited.add (node))
8424 : return;
8425 :
8426 : /* Besides some VEC_PERM_EXPR, two-operator nodes also lack scalar stmts
8427 : and thus CSE doesn't work. For now gather two-lane aligned starts
8428 : of nodes with a multiple of two number of lanes. */
8429 1155140 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8430 1147570 : && SLP_TREE_LANES (node) > 2
8431 149595 : && (SLP_TREE_LANES (node) & 1) == 0)
8432 : {
8433 149595 : auto_vec<unsigned, 8> uids;
8434 622675 : for (unsigned i = 0; i < SLP_TREE_LANES (node); i += 2)
8435 : {
8436 473080 : stmt_vec_info s = SLP_TREE_SCALAR_STMTS (node)[i];
8437 473080 : if (!s)
8438 0 : continue;
8439 473080 : unsigned uid = gimple_uid (s->stmt);
8440 473080 : if (!uids.contains (uid))
8441 : {
8442 469079 : uids.safe_push (uid);
8443 469079 : part_starts[uid].safe_push (node);
8444 : }
8445 : }
8446 149595 : }
8447 :
8448 4501727 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8449 1321703 : if (child)
8450 1321703 : vect_cse_gather_part_starts (visited, part_starts, child);
8451 : }
8452 :
8453 : /* Apply CSE to NODE and its children using lowparts of nodes in BST_MAP. */
8454 :
8455 : static void
8456 2148020 : vect_cse_slp_node_parts (hash_set<slp_tree> &visited,
8457 : const vec<vec<slp_tree>> part_starts,
8458 : vec<slp_tree> &drops, slp_tree node)
8459 : {
8460 2148020 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8461 2148020 : || visited.add (node))
8462 : return;
8463 :
8464 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8465 : lack scalar stmts and thus CSE doesn't work. */
8466 1153498 : unsigned HOST_WIDE_INT c;
8467 1153498 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8468 1145932 : && SLP_TREE_SCALAR_STMTS (node)[0]
8469 : /* Avoid touching loads which need care with load permutations
8470 : and specialities like load-lane representations. */
8471 1145932 : && (SLP_TREE_PERMUTE_P (node)
8472 1138248 : || !STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node))))
8473 36828 : for (slp_tree cand
8474 431555 : : part_starts[gimple_uid (SLP_TREE_SCALAR_STMTS (node)[0]->stmt)])
8475 : /* ??? There is a possible ordering/optimality problem in that
8476 : the CSE then can keep a wider feeding live even though it itself
8477 : becomes dead by means of CSE. Which might be solvable by doing
8478 : the CSE in a wide-to-narrow order. */
8479 37290 : if (SLP_TREE_LANES (cand) > SLP_TREE_LANES (node)
8480 : /* We can do high/lo extracts and full vector copies. */
8481 : && constant_multiple_p
8482 4633 : (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (cand)),
8483 37780 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)), &c)
8484 41923 : && c <= 2)
8485 : {
8486 4143 : unsigned HOST_WIDE_INT s;
8487 4143 : bool const_p
8488 4143 : = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)).is_constant (&s);
8489 4143 : unsigned i;
8490 14870 : for (i = 0; i <= SLP_TREE_LANES (cand) - SLP_TREE_LANES (node);)
8491 : {
8492 : unsigned j;
8493 15833 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
8494 15371 : if (!SLP_TREE_SCALAR_STMTS (node)[j]
8495 15371 : || (SLP_TREE_SCALAR_STMTS (cand)[i+j]
8496 15371 : != SLP_TREE_SCALAR_STMTS (node)[j]))
8497 : break;
8498 11189 : if (j == SLP_TREE_LANES (node))
8499 : break;
8500 10727 : if (!const_p)
8501 : {
8502 : i = SLP_TREE_LANES (cand);
8503 : break;
8504 : }
8505 : /* We can extract only aligned on node vector type boundary. */
8506 10727 : i += s;
8507 : }
8508 4143 : if (i > SLP_TREE_LANES (cand) - SLP_TREE_LANES (node))
8509 3681 : continue;
8510 : /* Found node within cand at i. Put a permute in place
8511 : of it, selecting the subset from cand. */
8512 462 : if (dump_enabled_p ())
8513 26 : dump_printf (MSG_NOTE, "CSEd node %p as %spart of node %p\n",
8514 : (void *)node, i == 0 ? "low" : "high", (void *)cand);
8515 2277 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8516 : /* Delay SLP tree release since we might still reference a node
8517 : from the part_starts map. */
8518 891 : drops.safe_push (child);
8519 462 : SLP_TREE_CHILDREN (node).truncate (1);
8520 462 : SLP_TREE_REF_COUNT (cand)++;
8521 462 : SLP_TREE_CHILDREN (node)[0] = cand;
8522 462 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
8523 462 : SLP_TREE_REPRESENTATIVE (node) = NULL;
8524 462 : SLP_TREE_LANE_PERMUTATION (node).create (SLP_TREE_LANES (node));
8525 1988 : for (unsigned j = i; j < i + SLP_TREE_LANES (node); ++j)
8526 1064 : SLP_TREE_LANE_PERMUTATION (node).quick_push (std::make_pair (0, j));
8527 2148020 : return;
8528 : }
8529 :
8530 4492428 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8531 1318138 : if (child)
8532 1318138 : vect_cse_slp_node_parts (visited, part_starts, drops, child);
8533 : }
8534 :
8535 : /* Optimize the SLP graph of VINFO. */
8536 :
8537 : void
8538 1102221 : vect_optimize_slp (vec_info *vinfo)
8539 : {
8540 1102221 : if (vinfo->slp_instances.is_empty ())
8541 821514 : return;
8542 714431 : vect_optimize_slp_pass (vinfo).run ();
8543 :
8544 : /* Apply CSE again to nodes after permute optimization. */
8545 714431 : scalar_stmts_to_slp_tree_map_t *bst_map
8546 714431 : = new scalar_stmts_to_slp_tree_map_t ();
8547 :
8548 3676464 : for (auto inst : vinfo->slp_instances)
8549 1533171 : vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
8550 :
8551 714431 : release_scalar_stmts_to_slp_tree_map (bst_map);
8552 :
8553 714431 : if (!is_a <bb_vec_info> (vinfo))
8554 : return;
8555 :
8556 : /* Attempt to merge SLP sub-graphs that intersect in low or highparts of
8557 : each other. Build the reverse mapping from stmt to SLP node for
8558 : lanes starting at the low or high part.
8559 : ??? In the future we can extend this to do a two-step permute
8560 : and extract or extract and permute to put the high/low part in
8561 : place on the original vector or permute the hogh/low part to
8562 : match up the target lane order. */
8563 280707 : hash_set<slp_tree> visited;
8564 280707 : vec<vec<slp_tree>> start_for_part;
8565 561414 : start_for_part.create (vinfo->stmt_vec_infos.length () + 1);
8566 561414 : start_for_part.quick_grow_cleared (vinfo->stmt_vec_infos.length () + 1);
8567 1672003 : for (auto inst : vinfo->slp_instances)
8568 829882 : vect_cse_gather_part_starts (visited,
8569 : start_for_part, SLP_INSTANCE_TREE (inst));
8570 :
8571 : /* Now replace low/highpart copies with extracting permutes. */
8572 280707 : auto_vec<slp_tree> drops;
8573 280707 : visited.empty ();
8574 1672003 : for (auto inst : vinfo->slp_instances)
8575 829882 : vect_cse_slp_node_parts (visited, start_for_part, drops,
8576 : SLP_INSTANCE_TREE (inst));
8577 :
8578 : /* Now perform delayed releases of nodes. */
8579 282088 : for (slp_tree node : drops)
8580 891 : vect_free_slp_tree (node);
8581 :
8582 24202980 : for (auto v : start_for_part)
8583 23360859 : v.release ();
8584 280707 : start_for_part.release ();
8585 280707 : }
8586 :
8587 : /* Gather loads reachable from the individual SLP graph entries. */
8588 :
8589 : void
8590 1102221 : vect_gather_slp_loads (vec_info *vinfo)
8591 : {
8592 1102221 : unsigned i;
8593 1102221 : slp_instance instance;
8594 2635392 : FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
8595 : {
8596 1533171 : hash_set<slp_tree> visited;
8597 1533171 : vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
8598 : SLP_INSTANCE_TREE (instance), visited);
8599 1533171 : }
8600 1102221 : }
8601 :
8602 : /* For NODE update VF based on the number of lanes and the vector types
8603 : used. */
8604 :
8605 : static void
8606 4483906 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
8607 : hash_set<slp_tree> &visited)
8608 : {
8609 4483906 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
8610 1621017 : return;
8611 3237591 : if (visited.add (node))
8612 : return;
8613 :
8614 10893762 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8615 3687465 : vect_update_slp_vf_for_node (child, vf, visited);
8616 :
8617 : /* We do not visit SLP nodes for constants or externals - those neither
8618 : have a vector type set yet (vectorizable_* does this).
8619 : Note that when we stop using fixed size vectors externs and constants
8620 : shouldn't influence the (minimum) vectorization factor, instead
8621 : vectorizable_* should honor the vectorization factor when trying to
8622 : assign vector types to constants and externals and cause iteration
8623 : to a higher vectorization factor when required. */
8624 2862909 : tree vectype = SLP_TREE_VECTYPE (node);
8625 2862909 : if (!vectype)
8626 : /* OMP SIMD calls w/o LHS have no SLP_TREE_VECTYPE set. */
8627 : return;
8628 2862889 : poly_uint64 node_vf
8629 2862889 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8630 : SLP_TREE_LANES (node));
8631 2862889 : vf = force_common_multiple (vf, node_vf);
8632 :
8633 : /* For permute nodes that are fed from externs or constants we have to
8634 : consider their number of lanes as well. Likewise for store-lanes. */
8635 2862889 : if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
8636 716487 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8637 193002 : if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
8638 : {
8639 3607 : poly_uint64 child_vf
8640 3607 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8641 : SLP_TREE_LANES (child));
8642 3607 : vf = force_common_multiple (vf, child_vf);
8643 : }
8644 : }
8645 :
8646 : /* For each possible SLP instance decide whether to SLP it and calculate overall
8647 : unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at
8648 : least one instance. */
8649 :
8650 : bool
8651 500241 : vect_make_slp_decision (loop_vec_info loop_vinfo)
8652 : {
8653 500241 : unsigned int i;
8654 500241 : poly_uint64 unrolling_factor = 1;
8655 500241 : const vec<slp_instance> &slp_instances
8656 : = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
8657 500241 : slp_instance instance;
8658 500241 : int decided_to_slp = 0;
8659 :
8660 500241 : DUMP_VECT_SCOPE ("vect_make_slp_decision");
8661 :
8662 500241 : hash_set<slp_tree> visited;
8663 1796923 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
8664 : {
8665 796441 : slp_tree root = SLP_INSTANCE_TREE (instance);
8666 :
8667 : /* All unroll factors have the form:
8668 :
8669 : GET_MODE_SIZE (vinfo->vector_mode) * X
8670 :
8671 : for some rational X, so they must have a common multiple. */
8672 796441 : vect_update_slp_vf_for_node (root, unrolling_factor, visited);
8673 :
8674 : /* If all instances ended up with vector(1) T roots make sure to
8675 : not vectorize. RVV for example relies on loop vectorization
8676 : when some instances are essentially kept scalar. See PR121048. */
8677 796441 : if (SLP_TREE_VECTYPE (root)
8678 796441 : && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
8679 646970 : decided_to_slp++;
8680 : }
8681 :
8682 500241 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
8683 :
8684 500241 : if (decided_to_slp && dump_enabled_p ())
8685 : {
8686 19502 : dump_printf_loc (MSG_NOTE, vect_location,
8687 : "Decided to SLP %d instances. Unrolling factor ",
8688 : decided_to_slp);
8689 19502 : dump_dec (MSG_NOTE, unrolling_factor);
8690 19502 : dump_printf (MSG_NOTE, "\n");
8691 : }
8692 :
8693 500241 : return (decided_to_slp > 0);
8694 500241 : }
8695 :
8696 : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks. */
8697 :
8698 2296064 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
8699 : : vec_info (vec_info::bb, shared),
8700 2296064 : roots (vNULL)
8701 : {
8702 : /* The region we are operating on. bbs[0] is the entry, excluding
8703 : its PHI nodes. In the future we might want to track an explicit
8704 : entry edge to cover bbs[0] PHI nodes and have a region entry
8705 : insert location. */
8706 2296064 : bbs = _bbs.address ();
8707 2296064 : nbbs = _bbs.length ();
8708 :
8709 18371992 : for (unsigned i = 0; i < nbbs; ++i)
8710 : {
8711 16075928 : if (i != 0)
8712 20878321 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8713 7098457 : gsi_next (&si))
8714 : {
8715 7098457 : gphi *phi = si.phi ();
8716 7098457 : gimple_set_uid (phi, 0);
8717 7098457 : add_stmt (phi);
8718 : }
8719 32151856 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8720 148199252 : !gsi_end_p (gsi); gsi_next (&gsi))
8721 : {
8722 132123324 : gimple *stmt = gsi_stmt (gsi);
8723 132123324 : gimple_set_uid (stmt, 0);
8724 132123324 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
8725 85669211 : continue;
8726 46454113 : add_stmt (stmt);
8727 : }
8728 : }
8729 2296064 : }
8730 :
8731 :
8732 : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
8733 : stmts in the basic block. */
8734 :
8735 2296064 : _bb_vec_info::~_bb_vec_info ()
8736 : {
8737 : /* Reset region marker. */
8738 18371992 : for (unsigned i = 0; i < nbbs; ++i)
8739 : {
8740 16075928 : if (i != 0)
8741 20894498 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8742 7114634 : gsi_next (&si))
8743 : {
8744 7114634 : gphi *phi = si.phi ();
8745 7114634 : gimple_set_uid (phi, -1);
8746 : }
8747 32151856 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8748 148167506 : !gsi_end_p (gsi); gsi_next (&gsi))
8749 : {
8750 132091578 : gimple *stmt = gsi_stmt (gsi);
8751 132091578 : gimple_set_uid (stmt, -1);
8752 : }
8753 : }
8754 :
8755 3635887 : for (unsigned i = 0; i < roots.length (); ++i)
8756 : {
8757 1339823 : roots[i].stmts.release ();
8758 1339823 : roots[i].roots.release ();
8759 1339823 : roots[i].remain.release ();
8760 : }
8761 2296064 : roots.release ();
8762 2296064 : }
8763 :
8764 : /* Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE,
8765 : given then that child nodes have already been processed, and that
8766 : their def types currently match their SLP node's def type. */
8767 :
8768 : static bool
8769 2720677 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
8770 : slp_instance node_instance,
8771 : stmt_vector_for_cost *cost_vec)
8772 : {
8773 : /* Handle purely internal nodes. */
8774 2720677 : if (SLP_TREE_PERMUTE_P (node))
8775 : {
8776 82888 : gcc_checking_assert (!SLP_TREE_REPRESENTATIVE (node));
8777 82888 : if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
8778 : return false;
8779 :
8780 : stmt_vec_info slp_stmt_info;
8781 : unsigned int i;
8782 204526 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
8783 : {
8784 123267 : if (slp_stmt_info
8785 117597 : && STMT_VINFO_LIVE_P (slp_stmt_info)
8786 123267 : && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
8787 : node_instance, i,
8788 : false, cost_vec))
8789 : return false;
8790 : }
8791 81259 : SLP_TREE_TYPE (node) = permute_info_type;
8792 81259 : return true;
8793 : }
8794 :
8795 2637789 : return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
8796 : }
8797 :
8798 : static int
8799 1930613 : sort_ints (const void *a_, const void *b_)
8800 : {
8801 1930613 : int a = *(const int *)a_;
8802 1930613 : int b = *(const int *)b_;
8803 1930613 : return a - b;
8804 : }
8805 :
8806 : /* Verify if we can externalize a set of internal defs. */
8807 :
8808 : static bool
8809 410199 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
8810 : {
8811 : /* Constant generation uses get_later_stmt which can only handle
8812 : defs from the same BB or a set of defs that can be ordered
8813 : with a dominance query. */
8814 410199 : basic_block bb = NULL;
8815 410199 : bool all_same = true;
8816 410199 : auto_vec<int> bbs;
8817 820398 : bbs.reserve_exact (stmts.length ());
8818 2197217 : for (stmt_vec_info stmt : stmts)
8819 : {
8820 966620 : if (!stmt)
8821 : return false;
8822 966620 : else if (!bb)
8823 410199 : bb = gimple_bb (stmt->stmt);
8824 556421 : else if (gimple_bb (stmt->stmt) != bb)
8825 187555 : all_same = false;
8826 966620 : bbs.quick_push (gimple_bb (stmt->stmt)->index);
8827 : }
8828 410199 : if (all_same)
8829 : return true;
8830 :
8831 : /* Produce a vector of unique BB indexes for the defs. */
8832 140856 : bbs.qsort (sort_ints);
8833 140856 : unsigned i, j;
8834 340116 : for (i = 1, j = 1; i < bbs.length (); ++i)
8835 199260 : if (bbs[i] != bbs[j-1])
8836 150422 : bbs[j++] = bbs[i];
8837 140856 : gcc_assert (j >= 2);
8838 140856 : bbs.truncate (j);
8839 :
8840 281712 : if (bbs.length () == 2)
8841 137334 : return (dominated_by_p (CDI_DOMINATORS,
8842 137334 : BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
8843 137334 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
8844 268414 : || dominated_by_p (CDI_DOMINATORS,
8845 131080 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
8846 131080 : BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
8847 :
8848 : /* ??? For more than two BBs we can sort the vector and verify the
8849 : result is a total order. But we can't use vec::qsort with a
8850 : compare function using a dominance query since there's no way to
8851 : signal failure and any fallback for an unordered pair would
8852 : fail qsort_chk later.
8853 : For now simply hope that ordering after BB index provides the
8854 : best candidate total order. If required we can implement our
8855 : own mergesort or export an entry without checking. */
8856 426754 : for (unsigned i = 1; i < bbs.length (); ++i)
8857 13064 : if (!dominated_by_p (CDI_DOMINATORS,
8858 13064 : BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
8859 13064 : BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
8860 : return false;
8861 :
8862 : return true;
8863 410199 : }
8864 :
8865 : /* Try to build NODE from scalars, returning true on success.
8866 : NODE_INSTANCE is the SLP instance that contains NODE. */
8867 :
8868 : static bool
8869 639834 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
8870 : slp_instance node_instance)
8871 : {
8872 639834 : stmt_vec_info stmt_info;
8873 639834 : unsigned int i;
8874 :
8875 639834 : if (!is_a <bb_vec_info> (vinfo)
8876 90312 : || node == SLP_INSTANCE_TREE (node_instance)
8877 30591 : || !SLP_TREE_SCALAR_STMTS (node).exists ()
8878 30550 : || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
8879 : /* Force the mask use to be built from scalars instead. */
8880 23026 : || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
8881 662623 : || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
8882 : return false;
8883 :
8884 22789 : if (dump_enabled_p ())
8885 82 : dump_printf_loc (MSG_NOTE, vect_location,
8886 : "Building vector operands of %p from scalars instead\n",
8887 : (void *) node);
8888 :
8889 : /* Don't remove and free the child nodes here, since they could be
8890 : referenced by other structures. The analysis and scheduling phases
8891 : (need to) ignore child nodes of anything that isn't vect_internal_def. */
8892 22789 : unsigned int group_size = SLP_TREE_LANES (node);
8893 22789 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
8894 : /* Invariants get their vector type from the uses. */
8895 22789 : SLP_TREE_VECTYPE (node) = NULL_TREE;
8896 22789 : SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
8897 22789 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8898 103418 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
8899 : {
8900 57840 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
8901 57840 : SLP_TREE_SCALAR_OPS (node)[i] = lhs;
8902 : }
8903 : return true;
8904 : }
8905 :
8906 : /* Return true if all elements of the slice are the same. */
8907 : bool
8908 449109 : vect_scalar_ops_slice::all_same_p () const
8909 : {
8910 500892 : for (unsigned int i = 1; i < length; ++i)
8911 414141 : if (!operand_equal_p (op (0), op (i)))
8912 : return false;
8913 : return true;
8914 : }
8915 :
8916 : hashval_t
8917 423908 : vect_scalar_ops_slice_hash::hash (const value_type &s)
8918 : {
8919 423908 : hashval_t hash = 0;
8920 1625020 : for (unsigned i = 0; i < s.length; ++i)
8921 1201112 : hash = iterative_hash_expr (s.op (i), hash);
8922 423908 : return hash;
8923 : }
8924 :
8925 : bool
8926 232941 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
8927 : const compare_type &s2)
8928 : {
8929 232941 : if (s1.length != s2.length)
8930 : return false;
8931 403665 : for (unsigned i = 0; i < s1.length; ++i)
8932 351985 : if (!operand_equal_p (s1.op (i), s2.op (i)))
8933 : return false;
8934 : return true;
8935 : }
8936 :
8937 : /* Like vect_get_num_copies but N copies of the vector might have
8938 : excess elements in the last vector. Returns false if *NVECTORS
8939 : cannot be computed. */
8940 :
8941 : static bool
8942 1595393 : vect_get_num_copies_for_invariant (vec_info *vinfo, slp_tree node,
8943 : unsigned *nvectors)
8944 : {
8945 1595393 : poly_uint64 vf;
8946 :
8947 1595393 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
8948 372508 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
8949 : else
8950 1222885 : vf = 1;
8951 1595393 : vf *= SLP_TREE_LANES (node);
8952 :
8953 1595393 : tree vectype = SLP_TREE_VECTYPE (node);
8954 1595393 : bool res = can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
8955 : nvectors);
8956 1595393 : return res;
8957 : }
8958 :
8959 : /* Compute the prologue cost for invariant or constant operands represented
8960 : by NODE. */
8961 :
8962 : static void
8963 1093989 : vect_prologue_cost_for_slp (slp_tree node, unsigned nvectors,
8964 : stmt_vector_for_cost *cost_vec)
8965 : {
8966 : /* There's a special case of an existing vector, that costs nothing. */
8967 1093989 : if (SLP_TREE_SCALAR_OPS (node).length () == 0
8968 1093989 : && !SLP_TREE_VEC_DEFS (node).is_empty ())
8969 2277 : return;
8970 : /* Without looking at the actual initializer a vector of
8971 : constants can be implemented as load from the constant pool.
8972 : When all elements are the same we can use a splat. */
8973 1091712 : tree vectype = SLP_TREE_VECTYPE (node);
8974 1091712 : unsigned group_size = SLP_TREE_LANES (node);
8975 1091712 : unsigned HOST_WIDE_INT const_nunits;
8976 1091712 : unsigned nelt_limit;
8977 1091712 : auto ops = &SLP_TREE_SCALAR_OPS (node);
8978 1091712 : auto_vec<unsigned int> starts (nvectors);
8979 1091712 : if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
8980 1091712 : && ! multiple_p (const_nunits, group_size))
8981 : {
8982 67361 : nelt_limit = const_nunits;
8983 67361 : hash_set<vect_scalar_ops_slice_hash> vector_ops;
8984 345603 : for (unsigned int i = 0; i < nvectors; ++i)
8985 210881 : if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
8986 159201 : starts.quick_push (i * nelt_limit);
8987 67361 : }
8988 : else
8989 : {
8990 : /* If either the vector has variable length or the vectors
8991 : are composed of repeated whole groups we only need to
8992 : cost construction once. All vectors will be the same. */
8993 1024351 : nelt_limit = group_size;
8994 1024351 : starts.quick_push (0);
8995 : }
8996 : /* ??? We're just tracking whether vectors in a single node are the same.
8997 : Ideally we'd do something more global. */
8998 1091712 : bool passed = false;
8999 4458688 : for (unsigned int start : starts)
9000 : {
9001 1183552 : vect_cost_for_stmt kind;
9002 1183552 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
9003 : kind = vector_load;
9004 449109 : else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
9005 : kind = scalar_to_vec;
9006 : else
9007 362358 : kind = vec_construct;
9008 : /* The target cost hook has no idea which part of the SLP node
9009 : we are costing so avoid passing it down more than once. Pass
9010 : it to the first vec_construct or scalar_to_vec part since for those
9011 : the x86 backend tries to account for GPR to XMM register moves. */
9012 1183552 : record_stmt_cost (cost_vec, 1, kind, nullptr,
9013 1183552 : (kind != vector_load && !passed) ? node : nullptr,
9014 : vectype, 0, vect_prologue);
9015 1183552 : if (kind != vector_load)
9016 449109 : passed = true;
9017 : }
9018 1091712 : }
9019 :
9020 : /* Analyze statements contained in SLP tree NODE after recursively analyzing
9021 : the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
9022 :
9023 : Return true if the operations are supported. */
9024 :
9025 : static bool
9026 5050729 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
9027 : slp_instance node_instance,
9028 : hash_set<slp_tree> &visited_set,
9029 : vec<slp_tree> &visited_vec,
9030 : stmt_vector_for_cost *cost_vec)
9031 : {
9032 5050729 : int i, j;
9033 5050729 : slp_tree child;
9034 :
9035 : /* Costing and analysis of invariants is delayed. */
9036 5050729 : if (!node
9037 4657592 : || SLP_TREE_DEF_TYPE (node) == vect_constant_def
9038 3866256 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
9039 : return true;
9040 :
9041 3381038 : if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
9042 : {
9043 5 : if (dump_enabled_p ())
9044 0 : dump_printf_loc (MSG_NOTE, vect_location,
9045 : "Failed cyclic SLP reference in %p\n", (void *) node);
9046 : return false;
9047 : }
9048 3381033 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
9049 :
9050 : /* If we already analyzed the exact same set of scalar stmts we're done.
9051 : We share the generated vector stmts for those. */
9052 3381033 : if (visited_set.add (node))
9053 : return true;
9054 3082075 : visited_vec.safe_push (node);
9055 :
9056 3082075 : bool res = true;
9057 3082075 : unsigned visited_rec_start = visited_vec.length ();
9058 3082075 : unsigned cost_vec_rec_start = cost_vec->length ();
9059 3082075 : bool seen_non_constant_child = false;
9060 8925269 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9061 : {
9062 3684578 : res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
9063 : visited_set, visited_vec,
9064 : cost_vec);
9065 3684578 : if (!res)
9066 : break;
9067 3323394 : if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
9068 3323394 : seen_non_constant_child = true;
9069 : }
9070 : /* We're having difficulties scheduling nodes with just constant
9071 : operands and no scalar stmts since we then cannot compute a stmt
9072 : insertion place. */
9073 3082075 : if (res
9074 3082075 : && !seen_non_constant_child
9075 3082075 : && SLP_TREE_SCALAR_STMTS (node).is_empty ())
9076 : {
9077 214 : if (dump_enabled_p ())
9078 6 : dump_printf_loc (MSG_NOTE, vect_location,
9079 : "Cannot vectorize all-constant op node %p\n",
9080 : (void *) node);
9081 : res = false;
9082 : }
9083 :
9084 3081867 : if (res)
9085 2720677 : res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
9086 : cost_vec);
9087 : /* If analysis failed we have to pop all recursive visited nodes
9088 : plus ourselves. */
9089 3082075 : if (!res)
9090 : {
9091 3156782 : while (visited_vec.length () >= visited_rec_start)
9092 938557 : visited_set.remove (visited_vec.pop ());
9093 639834 : cost_vec->truncate (cost_vec_rec_start);
9094 : }
9095 :
9096 : /* When the node can be vectorized cost invariant nodes it references.
9097 : This is not done in DFS order to allow the referring node
9098 : vectorizable_* calls to nail down the invariant nodes vector type
9099 : and possibly unshare it if it needs a different vector type than
9100 : other referrers. */
9101 3082075 : if (res)
9102 9389088 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
9103 2968187 : if (child
9104 2647440 : && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
9105 2647440 : || SLP_TREE_DEF_TYPE (child) == vect_external_def)
9106 : /* Perform usual caching, note code-generation still
9107 : code-gens these nodes multiple times but we expect
9108 : to CSE them later. */
9109 4102878 : && !visited_set.add (child))
9110 : {
9111 1131442 : visited_vec.safe_push (child);
9112 : /* ??? After auditing more code paths make a "default"
9113 : and push the vector type from NODE to all children
9114 : if it is not already set. */
9115 : /* Compute the number of vectors to be generated. */
9116 1131442 : tree vector_type = SLP_TREE_VECTYPE (child);
9117 1131442 : if (!vector_type)
9118 : {
9119 : /* Masked loads can have an undefined (default SSA definition)
9120 : else operand. We do not need to cost it. */
9121 37453 : vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
9122 38884 : if (SLP_TREE_TYPE (node) == load_vec_info_type
9123 38884 : && ((ops.length ()
9124 1431 : && TREE_CODE (ops[0]) == SSA_NAME
9125 0 : && SSA_NAME_IS_DEFAULT_DEF (ops[0])
9126 0 : && VAR_P (SSA_NAME_VAR (ops[0])))
9127 1431 : || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
9128 1431 : continue;
9129 :
9130 : /* For shifts with a scalar argument we don't need
9131 : to cost or code-generate anything.
9132 : ??? Represent this more explicitly. */
9133 36022 : gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
9134 : && j == 1);
9135 36022 : continue;
9136 36022 : }
9137 :
9138 : /* Make sure we can generate them and then cost them. */
9139 1093989 : unsigned nvectors;
9140 1093989 : if (!vect_get_num_copies_for_invariant (vinfo, node, &nvectors))
9141 : return false;
9142 1093989 : vect_prologue_cost_for_slp (child, nvectors, cost_vec);
9143 : }
9144 :
9145 : /* If this node or any of its children can't be vectorized, try pruning
9146 : the tree here rather than felling the whole thing. */
9147 639834 : if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
9148 : {
9149 : /* We'll need to revisit this for invariant costing and number
9150 : of vectorized stmt setting. */
9151 : res = true;
9152 : }
9153 :
9154 : return res;
9155 : }
9156 :
9157 : /* Mark lanes of NODE that are live outside of the basic-block vectorized
9158 : region and that can be vectorized using vectorizable_live_operation
9159 : with STMT_VINFO_LIVE_P. Not handled live operations will cause the
9160 : scalar code computing it to be retained. */
9161 :
9162 : static void
9163 998275 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
9164 : slp_instance instance,
9165 : stmt_vector_for_cost *cost_vec,
9166 : hash_set<stmt_vec_info> &svisited,
9167 : hash_set<slp_tree> &visited)
9168 : {
9169 998275 : if (visited.add (node))
9170 58328 : return;
9171 :
9172 : unsigned i;
9173 : stmt_vec_info stmt_info;
9174 : gimple *last_stmt = NULL;
9175 3371521 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9176 : {
9177 2431574 : if (!stmt_info || svisited.contains (stmt_info))
9178 94739 : continue;
9179 2396034 : stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
9180 2396034 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
9181 35609 : && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
9182 : /* Only the pattern root stmt computes the original scalar value. */
9183 29649 : continue;
9184 2366385 : if (!PURE_SLP_STMT (orig_stmt_info))
9185 : /* Iff the stmt is not part of the vector coverage because it or
9186 : uses of it are used by SLP graph leafs as extern input there is
9187 : no point in trying to live code-generate from a vector stmt as
9188 : the scalar stmt will survive anyway. */
9189 29550 : continue;
9190 2336835 : bool mark_visited = true;
9191 2336835 : gimple *orig_stmt = orig_stmt_info->stmt;
9192 2336835 : ssa_op_iter op_iter;
9193 2336835 : def_operand_p def_p;
9194 5236927 : FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
9195 : {
9196 : /* We have to verify whether we can insert the lane extract
9197 : before all uses. The following is a conservative approximation.
9198 : We cannot put this into vectorizable_live_operation because
9199 : iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
9200 : doesn't work.
9201 : Note that while the fact that we emit code for loads at the
9202 : first load should make this a non-problem leafs we construct
9203 : from scalars are vectorized after the last scalar def.
9204 : ??? If we'd actually compute the insert location during
9205 : analysis we could use sth less conservative than the last
9206 : scalar stmt in the node for the dominance check. */
9207 : /* ??? What remains is "live" uses in vector CTORs in the same
9208 : SLP graph which is where those uses can end up code-generated
9209 : right after their definition instead of close to their original
9210 : use. But that would restrict us to code-generate lane-extracts
9211 : from the latest stmt in a node. So we compensate for this
9212 : during code-generation, simply not replacing uses for those
9213 : hopefully rare cases. */
9214 563257 : imm_use_iterator use_iter;
9215 :
9216 563257 : bool live_p = false;
9217 563257 : bool can_insert = true;
9218 563257 : use_operand_p use_p;
9219 1691009 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, DEF_FROM_PTR (def_p))
9220 : {
9221 1135891 : gimple *use_stmt = USE_STMT (use_p);
9222 1135891 : stmt_vec_info use_stmt_info;
9223 1135891 : if (!(!is_gimple_debug (use_stmt)
9224 872314 : && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
9225 861816 : || !PURE_SLP_STMT (use_stmt_info))))
9226 929256 : continue;
9227 206635 : live_p = true;
9228 206635 : if (!last_stmt)
9229 64279 : last_stmt
9230 64529 : = (node->si ? node->si
9231 250 : : vect_find_last_scalar_stmt_in_slp (node)->stmt);
9232 206635 : if (is_a <gphi *> (use_stmt))
9233 : {
9234 72560 : if (!dominated_by_p (CDI_DOMINATORS,
9235 36280 : phi_arg_edge_from_use (use_p)->src,
9236 36280 : gimple_bb (last_stmt)))
9237 : can_insert = false;
9238 : }
9239 : /* As we instert after last_stmt it may not be the use_stmt
9240 : itself. */
9241 170355 : else if (last_stmt == use_stmt
9242 170355 : || !vect_stmt_dominates_stmt_p (last_stmt, use_stmt))
9243 : can_insert = false;
9244 : if (!can_insert)
9245 : {
9246 8139 : if (dump_enabled_p ())
9247 66 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9248 : "Cannot determine insertion place for "
9249 : "lane extract of %T at node %p\n",
9250 : DEF_FROM_PTR (def_p), (void *)node);
9251 : can_insert = false;
9252 : break;
9253 : }
9254 563257 : }
9255 563257 : if (live_p && can_insert)
9256 : {
9257 : /* Only record a live stmt when we can replace all uses. We
9258 : record from which SLP tree we vectorize the uses, so we'll
9259 : cost once and can deal with the case that not all SLP nodes
9260 : may be suitable for code-generation of all live uses.
9261 : ??? But we never split up the work between multiple SLP
9262 : nodes. */
9263 101521 : STMT_VINFO_LIVE_P (stmt_info) = true;
9264 101521 : if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
9265 : instance, i, false, cost_vec))
9266 : {
9267 0 : STMT_VINFO_LIVE_P (stmt_info) = false;
9268 0 : mark_visited = false;
9269 : }
9270 : }
9271 : }
9272 2336835 : if (mark_visited)
9273 2336835 : svisited.add (stmt_info);
9274 : }
9275 :
9276 : slp_tree child;
9277 2751821 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9278 984250 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9279 281329 : vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
9280 : svisited, visited);
9281 : }
9282 :
9283 : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
9284 : are live outside of the basic-block vectorized region and that can be
9285 : vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. */
9286 :
9287 : static void
9288 246835 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
9289 : {
9290 246835 : if (bb_vinfo->slp_instances.is_empty ())
9291 0 : return;
9292 :
9293 246835 : hash_set<slp_tree> visited;
9294 246835 : hash_set<stmt_vec_info> svisited;
9295 1457451 : for (slp_instance instance : bb_vinfo->slp_instances)
9296 : {
9297 716946 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9298 50959 : STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
9299 716946 : vect_location = instance->location ();
9300 716946 : vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
9301 : instance, &instance->cost_vec,
9302 : svisited, visited);
9303 : }
9304 246835 : }
9305 :
9306 : /* Determine whether we can vectorize the reduction epilogue for INSTANCE. */
9307 :
9308 : static bool
9309 89164 : vectorizable_bb_reduc_epilogue (slp_instance instance,
9310 : stmt_vector_for_cost *cost_vec)
9311 : {
9312 89164 : gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
9313 89164 : enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
9314 89164 : if (reduc_code == MINUS_EXPR)
9315 0 : reduc_code = PLUS_EXPR;
9316 89164 : internal_fn reduc_fn;
9317 89164 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
9318 89164 : if (!vectype
9319 89152 : || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
9320 89152 : TREE_TYPE (vectype))
9321 52231 : || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
9322 1050 : && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
9323 1050 : || reduc_fn == IFN_LAST
9324 1050 : || !direct_internal_fn_supported_p (reduc_fn, vectype,
9325 : OPTIMIZE_FOR_BOTH)))
9326 : /* Two-element reductions do not need special-handling for fold-left,
9327 : other cases are not yet implemented. remain_defs also have to
9328 : be included here. */
9329 178364 : || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
9330 5739 : && (!instance->remain_defs.is_empty ()
9331 1857 : || SLP_TREE_LANES (SLP_INSTANCE_TREE (instance)) != 2)))
9332 : {
9333 38924 : if (dump_enabled_p ())
9334 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9335 : "not vectorized: basic block reduction epilogue "
9336 : "operation unsupported.\n");
9337 : return false;
9338 : }
9339 :
9340 : /* There's no way to cost a horizontal vector reduction via REDUC_FN so
9341 : cost log2 vector operations plus shuffles and one extraction. */
9342 50240 : unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
9343 50240 : record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
9344 : vectype, 0, vect_body);
9345 50240 : record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
9346 : vectype, 0, vect_body);
9347 50240 : record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
9348 : vectype, 0, vect_body);
9349 :
9350 : /* Since we replace all stmts of a possibly longer scalar reduction
9351 : chain account for the extra scalar stmts for that. */
9352 50240 : if (!instance->remain_defs.is_empty ())
9353 30340 : record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
9354 15170 : instance->root_stmts[0], 0, vect_body);
9355 : return true;
9356 : }
9357 :
9358 : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
9359 : and recurse to children. */
9360 :
9361 : static void
9362 361981 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
9363 : hash_set<slp_tree> &visited)
9364 : {
9365 361981 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
9366 361981 : || visited.add (node))
9367 : return;
9368 :
9369 1008848 : for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
9370 447119 : if (stmt)
9371 478571 : roots.remove (vect_orig_stmt (stmt));
9372 :
9373 767126 : for (auto child : SLP_TREE_CHILDREN (node))
9374 256781 : if (child)
9375 255275 : vect_slp_prune_covered_roots (child, roots, visited);
9376 : }
9377 :
9378 : /* Hand over COST_VEC to the target COSTS grouped by SLP node. */
9379 :
9380 : static void
9381 985416 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
9382 : {
9383 3776353 : for (unsigned start = 0; start < cost_vec.length ();)
9384 : {
9385 2790937 : unsigned end = start + 1;
9386 3448866 : while (end < cost_vec.length ()
9387 5921000 : && cost_vec[start].node == cost_vec[end].node)
9388 657929 : end++;
9389 2790937 : costs->add_slp_cost (cost_vec[start].node,
9390 2790937 : array_slice<stmt_info_for_cost>
9391 2790937 : (cost_vec.begin () + start, end - start));
9392 2790937 : start = end;
9393 : }
9394 985416 : }
9395 :
9396 : /* Analyze statements in SLP instances of VINFO. Return true if the
9397 : operations are supported. */
9398 :
9399 : bool
9400 694578 : vect_slp_analyze_operations (vec_info *vinfo)
9401 : {
9402 694578 : slp_instance instance;
9403 694578 : int i;
9404 :
9405 694578 : DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
9406 :
9407 694578 : hash_set<slp_tree> visited;
9408 2496079 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9409 : {
9410 1366151 : auto_vec<slp_tree> visited_vec;
9411 1366151 : stmt_vector_for_cost cost_vec;
9412 1366151 : cost_vec.create (2);
9413 1366151 : if (is_a <bb_vec_info> (vinfo))
9414 821165 : vect_location = instance->location ();
9415 1366151 : if (!vect_slp_analyze_node_operations (vinfo,
9416 : SLP_INSTANCE_TREE (instance),
9417 : instance, visited, visited_vec,
9418 : &cost_vec)
9419 : /* CTOR instances require vectorized defs for the SLP tree root. */
9420 1110285 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
9421 6291 : && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
9422 : != vect_internal_def
9423 : /* Make sure we vectorized with the expected type. */
9424 6291 : || !useless_type_conversion_p
9425 6291 : (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
9426 : (instance->root_stmts[0]->stmt))),
9427 6291 : TREE_TYPE (SLP_TREE_VECTYPE
9428 : (SLP_INSTANCE_TREE (instance))))))
9429 : /* Check we can vectorize the reduction. */
9430 1110270 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
9431 89164 : && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
9432 : /* Check we can vectorize the gcond. */
9433 2437497 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
9434 65744 : && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
9435 65744 : SLP_INSTANCE_ROOT_STMTS (instance)[0],
9436 : NULL,
9437 : SLP_INSTANCE_TREE (instance),
9438 : &cost_vec)))
9439 : {
9440 357888 : cost_vec.release ();
9441 357888 : slp_tree node = SLP_INSTANCE_TREE (instance);
9442 357888 : stmt_vec_info stmt_info;
9443 357888 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9444 267656 : stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9445 90232 : else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
9446 90232 : && SLP_TREE_SCALAR_STMTS (node)[0])
9447 : stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
9448 : else
9449 0 : stmt_info = SLP_TREE_REPRESENTATIVE (node);
9450 357888 : if (is_a <loop_vec_info> (vinfo))
9451 : {
9452 259228 : if (dump_enabled_p ())
9453 6845 : dump_printf_loc (MSG_NOTE, vect_location,
9454 : "unsupported SLP instance starting from: %G",
9455 : stmt_info->stmt);
9456 259228 : return false;
9457 : }
9458 98660 : if (dump_enabled_p ())
9459 110 : dump_printf_loc (MSG_NOTE, vect_location,
9460 : "removing SLP instance operations starting from: %G",
9461 : stmt_info->stmt);
9462 235227 : while (!visited_vec.is_empty ())
9463 : {
9464 136567 : slp_tree node = visited_vec.pop ();
9465 136567 : SLP_TREE_TYPE (node) = undef_vec_info_type;
9466 136567 : if (node->data)
9467 : {
9468 8555 : delete node->data;
9469 8555 : node->data = nullptr;
9470 : }
9471 136567 : visited.remove (node);
9472 : }
9473 98660 : vect_free_slp_instance (instance);
9474 98660 : vinfo->slp_instances.ordered_remove (i);
9475 : }
9476 : else
9477 : {
9478 1008263 : i++;
9479 1008263 : if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
9480 : {
9481 285758 : add_slp_costs (loop_vinfo->vector_costs, cost_vec);
9482 285758 : cost_vec.release ();
9483 : }
9484 : else
9485 : /* For BB vectorization remember the SLP graph entry
9486 : cost for later. */
9487 722505 : instance->cost_vec = cost_vec;
9488 : }
9489 1366151 : }
9490 :
9491 : /* Now look for SLP instances with a root that are covered by other
9492 : instances and remove them. */
9493 435350 : hash_set<stmt_vec_info> roots;
9494 1807953 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9495 996418 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9496 59165 : roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
9497 435350 : if (!roots.is_empty ())
9498 : {
9499 23418 : visited.empty ();
9500 130124 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9501 106706 : vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
9502 : visited);
9503 130124 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9504 106706 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
9505 59165 : && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
9506 : {
9507 5537 : stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9508 5537 : if (dump_enabled_p ())
9509 283 : dump_printf_loc (MSG_NOTE, vect_location,
9510 : "removing SLP instance operations starting "
9511 : "from: %G", root->stmt);
9512 5537 : vect_free_slp_instance (instance);
9513 5537 : vinfo->slp_instances.ordered_remove (i);
9514 : }
9515 : else
9516 101169 : ++i;
9517 : }
9518 :
9519 870700 : return !vinfo->slp_instances.is_empty ();
9520 1129928 : }
9521 :
9522 : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
9523 : closing the eventual chain. */
9524 :
9525 : static slp_instance
9526 807080 : get_ultimate_leader (slp_instance instance,
9527 : hash_map<slp_instance, slp_instance> &instance_leader)
9528 : {
9529 807080 : auto_vec<slp_instance *, 8> chain;
9530 807080 : slp_instance *tem;
9531 917085 : while (*(tem = instance_leader.get (instance)) != instance)
9532 : {
9533 110005 : chain.safe_push (tem);
9534 110005 : instance = *tem;
9535 : }
9536 917085 : while (!chain.is_empty ())
9537 110005 : *chain.pop () = instance;
9538 807080 : return instance;
9539 807080 : }
9540 :
9541 : namespace {
9542 : /* Subroutine of vect_bb_partition_graph_r. Map KEY to INSTANCE in
9543 : KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
9544 : for KEY. Return true if KEY was already in KEY_TO_INSTANCE.
9545 :
9546 : INSTANCE_LEADER is as for get_ultimate_leader. */
9547 :
9548 : template<typename T>
9549 : bool
9550 3567197 : vect_map_to_instance (slp_instance instance, T key,
9551 : hash_map<T, slp_instance> &key_to_instance,
9552 : hash_map<slp_instance, slp_instance> &instance_leader)
9553 : {
9554 : bool existed_p;
9555 3567197 : slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
9556 3567197 : if (!existed_p)
9557 : ;
9558 244262 : else if (key_instance != instance)
9559 : {
9560 : /* If we're running into a previously marked key make us the
9561 : leader of the current ultimate leader. This keeps the
9562 : leader chain acyclic and works even when the current instance
9563 : connects two previously independent graph parts. */
9564 90134 : slp_instance key_leader
9565 90134 : = get_ultimate_leader (key_instance, instance_leader);
9566 90134 : if (key_leader != instance)
9567 26949 : instance_leader.put (key_leader, instance);
9568 : }
9569 3567197 : key_instance = instance;
9570 3567197 : return existed_p;
9571 : }
9572 : }
9573 :
9574 : /* Worker of vect_bb_partition_graph, recurse on NODE. */
9575 :
9576 : static void
9577 998275 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
9578 : slp_instance instance, slp_tree node,
9579 : hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
9580 : hash_map<slp_tree, slp_instance> &node_to_instance,
9581 : hash_map<slp_instance, slp_instance> &instance_leader)
9582 : {
9583 5552367 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
9584 2568922 : if (stmt_info)
9585 2568922 : vect_map_to_instance (instance, stmt_info, stmt_to_instance,
9586 : instance_leader);
9587 :
9588 998275 : if (vect_map_to_instance (instance, node, node_to_instance,
9589 : instance_leader))
9590 : return;
9591 :
9592 3579445 : for (auto child : SLP_TREE_CHILDREN (node))
9593 984250 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9594 281329 : vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
9595 : node_to_instance, instance_leader);
9596 : }
9597 :
9598 : /* Partition the SLP graph into pieces that can be costed independently. */
9599 :
9600 : static void
9601 246835 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
9602 : {
9603 246835 : DUMP_VECT_SCOPE ("vect_bb_partition_graph");
9604 :
9605 : /* First walk the SLP graph assigning each involved scalar stmt a
9606 : corresponding SLP graph entry and upon visiting a previously
9607 : marked stmt, make the stmts leader the current SLP graph entry. */
9608 246835 : hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
9609 246835 : hash_map<slp_tree, slp_instance> node_to_instance;
9610 246835 : hash_map<slp_instance, slp_instance> instance_leader;
9611 246835 : slp_instance instance;
9612 1210616 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9613 : {
9614 716946 : instance_leader.put (instance, instance);
9615 716946 : vect_bb_partition_graph_r (bb_vinfo,
9616 : instance, SLP_INSTANCE_TREE (instance),
9617 : stmt_to_instance, node_to_instance,
9618 : instance_leader);
9619 : }
9620 :
9621 : /* Then collect entries to each independent subgraph. */
9622 1210616 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9623 : {
9624 716946 : slp_instance leader = get_ultimate_leader (instance, instance_leader);
9625 716946 : leader->subgraph_entries.safe_push (instance);
9626 716946 : if (dump_enabled_p ()
9627 716946 : && leader != instance)
9628 83 : dump_printf_loc (MSG_NOTE, vect_location,
9629 : "instance %p is leader of %p\n",
9630 : (void *) leader, (void *) instance);
9631 : }
9632 246835 : }
9633 :
9634 : /* Compute the scalar cost of the SLP node NODE and its children
9635 : and return it. Do not account defs that are marked in LIFE and
9636 : update LIFE according to uses of NODE. */
9637 :
9638 : static void
9639 712945 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
9640 : vec<stmt_vec_info> &worklist,
9641 : stmt_vector_for_cost *cost_vec,
9642 : hash_set<stmt_vec_info> &visited)
9643 : {
9644 3322645 : while (!worklist.is_empty ())
9645 : {
9646 2609700 : stmt_vec_info stmt = worklist.pop ();
9647 2934378 : if (!PURE_SLP_STMT (stmt))
9648 333189 : continue;
9649 :
9650 : /* When the stmt is live but not actually vectorized we have
9651 : to keep the feeding scalar defs. */
9652 2290672 : if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
9653 : {
9654 2185772 : bool live_p = false;
9655 2185772 : ssa_op_iter op_iter;
9656 2185772 : def_operand_p def_p;
9657 4806666 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
9658 : {
9659 435122 : imm_use_iterator use_iter;
9660 435122 : gimple *use_stmt;
9661 1115455 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9662 680333 : if (!is_gimple_debug (use_stmt))
9663 : {
9664 511946 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
9665 511946 : if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
9666 : {
9667 15736 : if (dump_enabled_p ())
9668 : {
9669 74 : dump_printf_loc (MSG_NOTE, vect_location,
9670 : "stmt considered live: %G",
9671 : stmt->stmt);
9672 74 : dump_printf_loc (MSG_NOTE, vect_location,
9673 : "because of use in: %G",
9674 : use_stmt);
9675 : }
9676 : live_p = true;
9677 : }
9678 435122 : }
9679 : }
9680 2185772 : if (live_p)
9681 8511 : continue;
9682 : }
9683 :
9684 : /* The following assert verifies that vect_bb_partition_graph
9685 : partitions the SLP graph in a way that each scalar stmt of
9686 : the coverage of the SLP graph belongs to exactly one subgraph.
9687 : ??? This is currently not guaranteed since the function
9688 : works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
9689 : tripping or scalar stmts costed multiple times, making vectorization
9690 : more profitable than it really is. */
9691 : /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
9692 :
9693 2276511 : if (vect_nop_conversion_p (stmt))
9694 : ;
9695 : /* For single-argument PHIs assume coalescing which means zero
9696 : cost for the scalar and the vector PHIs. This avoids
9697 : artificially favoring the vector path (but may pessimize it
9698 : in some cases). */
9699 2245747 : else if (is_a <gphi *> (stmt->stmt)
9700 2245747 : && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
9701 : ;
9702 : else
9703 : {
9704 2235203 : vect_cost_for_stmt kind;
9705 2235203 : if (STMT_VINFO_DATA_REF (stmt))
9706 : {
9707 2009719 : data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
9708 2009719 : tree base = get_base_address (DR_REF (dr));
9709 : /* When the scalar access is to a non-global not
9710 : address-taken decl that is not BLKmode assume we can
9711 : access it with a single non-load/store instruction. */
9712 2009719 : if (DECL_P (base)
9713 1546344 : && !is_global_var (base)
9714 1468955 : && !TREE_ADDRESSABLE (base)
9715 2566968 : && DECL_MODE (base) != BLKmode)
9716 : kind = scalar_stmt;
9717 1865411 : else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
9718 : kind = scalar_load;
9719 : else
9720 1623262 : kind = scalar_store;
9721 : }
9722 : else
9723 : kind = scalar_stmt;
9724 : /* Cost each scalar stmt only once. */
9725 2235203 : gimple_set_visited (stmt->stmt, true);
9726 2235203 : record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
9727 : }
9728 :
9729 : /* Now walk relevant parts of the SSA use-def graph. */
9730 2276511 : slp_oprnds child_ops (stmt);
9731 7087180 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
9732 : {
9733 2534158 : tree op = child_ops.get_op_for_slp_child (stmt, i);
9734 2534158 : stmt_vec_info def = vinfo->lookup_def (op);
9735 2534158 : if (def && !visited.add (def))
9736 761658 : worklist.safe_push (def);
9737 : }
9738 : }
9739 712945 : }
9740 :
9741 :
9742 : /* Comparator for the loop-index sorted cost vectors. */
9743 :
9744 : static int
9745 19792589 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
9746 : {
9747 19792589 : auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
9748 19792589 : auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
9749 19792589 : if (a->first < b->first)
9750 : return -1;
9751 18595505 : else if (a->first == b->first)
9752 17587928 : return 0;
9753 : return 1;
9754 : }
9755 :
9756 : /* Check if vectorization of the basic block is profitable for the
9757 : subgraph denoted by SLP_INSTANCES. */
9758 :
9759 : static bool
9760 686182 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
9761 : vec<slp_instance> slp_instances,
9762 : loop_p orig_loop)
9763 : {
9764 686182 : slp_instance instance;
9765 686182 : int i;
9766 686182 : unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
9767 686182 : unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
9768 :
9769 686182 : if (dump_enabled_p ())
9770 : {
9771 125 : dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph:\n");
9772 259 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9773 134 : dump_printf_loc (MSG_NOTE, vect_location, " entry instance %p -> "
9774 : "node %p\n", (void *)instance,
9775 134 : (void *)SLP_INSTANCE_TREE (instance));
9776 125 : hash_set<slp_tree> visited;
9777 509 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9778 134 : vect_print_slp_graph (MSG_NOTE, vect_location,
9779 : SLP_INSTANCE_TREE (instance), visited);
9780 125 : }
9781 :
9782 : /* Then DFS walk scalar stmts, performing costing and handling
9783 : still live scalar stmts via the previously computed vector coverage. */
9784 686182 : stmt_vector_for_cost scalar_costs = vNULL;
9785 686182 : stmt_vector_for_cost vector_costs = vNULL;
9786 686182 : hash_set<slp_tree> visited;
9787 686182 : hash_set<stmt_vec_info> svisited;
9788 2085309 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9789 : {
9790 712945 : auto_vec<stmt_vec_info> worklist;
9791 712945 : if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
9792 100418 : record_stmt_cost (&scalar_costs,
9793 50209 : SLP_INSTANCE_ROOT_STMTS (instance).length (),
9794 : scalar_stmt,
9795 50209 : SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
9796 4002121 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
9797 : {
9798 1863286 : stmt = vect_orig_stmt (stmt);
9799 1863286 : if (!svisited.add (stmt))
9800 1848042 : worklist.safe_push (stmt);
9801 : }
9802 712945 : vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
9803 712945 : vector_costs.safe_splice (instance->cost_vec);
9804 712945 : instance->cost_vec.release ();
9805 712945 : }
9806 :
9807 686182 : if (dump_enabled_p ())
9808 125 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
9809 :
9810 : /* When costing non-loop vectorization we need to consider each covered
9811 : loop independently and make sure vectorization is profitable. For
9812 : now we assume a loop may be not entered or executed an arbitrary
9813 : number of iterations (??? static information can provide more
9814 : precise info here) which means we can simply cost each containing
9815 : loops stmts separately. */
9816 :
9817 : /* First produce cost vectors sorted by loop index. */
9818 686182 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9819 1372364 : li_scalar_costs (scalar_costs.length ());
9820 686182 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9821 1372351 : li_vector_costs (vector_costs.length ());
9822 686182 : stmt_info_for_cost *cost;
9823 3657776 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9824 : {
9825 2285412 : unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9826 2285412 : li_scalar_costs.quick_push (std::make_pair (l, cost));
9827 : }
9828 : /* Use a random used loop as fallback in case the first vector_costs
9829 : entry does not have a stmt_info associated with it. */
9830 686182 : unsigned l = li_scalar_costs[0].first;
9831 2630468 : FOR_EACH_VEC_ELT (vector_costs, i, cost)
9832 : {
9833 : /* We inherit from the previous COST, invariants, externals and
9834 : extracts immediately follow the cost for the related stmt. */
9835 1944286 : if (cost->stmt_info)
9836 1175882 : l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9837 1944286 : li_vector_costs.quick_push (std::make_pair (l, cost));
9838 : }
9839 686182 : li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
9840 686182 : li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
9841 :
9842 : /* Now cost the portions individually. */
9843 686182 : unsigned vi = 0;
9844 686182 : unsigned si = 0;
9845 686182 : bool profitable = true;
9846 1387673 : while (si < li_scalar_costs.length ()
9847 2089206 : && vi < li_vector_costs.length ())
9848 : {
9849 701491 : unsigned sl = li_scalar_costs[si].first;
9850 701491 : unsigned vl = li_vector_costs[vi].first;
9851 701491 : if (sl != vl)
9852 : {
9853 1833 : if (dump_enabled_p ())
9854 2 : dump_printf_loc (MSG_NOTE, vect_location,
9855 : "Scalar %d and vector %d loop part do not "
9856 : "match up, skipping scalar part\n", sl, vl);
9857 : /* Skip the scalar part, assuming zero cost on the vector side. */
9858 2763 : do
9859 : {
9860 2763 : si++;
9861 : }
9862 2763 : while (si < li_scalar_costs.length ()
9863 6135 : && li_scalar_costs[si].first == sl);
9864 1833 : continue;
9865 : }
9866 :
9867 699658 : if (dump_enabled_p ())
9868 143 : dump_printf_loc (MSG_NOTE, vect_location,
9869 : "Scalar cost for part in loop %d\n", sl);
9870 699658 : class vector_costs *scalar_target_cost_data = init_cost (bb_vinfo, true);
9871 2282516 : do
9872 : {
9873 2282516 : add_stmt_cost (scalar_target_cost_data, li_scalar_costs[si].second);
9874 2282516 : si++;
9875 : }
9876 2282516 : while (si < li_scalar_costs.length ()
9877 4578857 : && li_scalar_costs[si].first == sl);
9878 699658 : scalar_target_cost_data->finish_cost (nullptr);
9879 699658 : scalar_cost = scalar_target_cost_data->body_cost ();
9880 :
9881 : /* Complete the target-specific vector cost calculation. */
9882 699658 : if (dump_enabled_p ())
9883 143 : dump_printf_loc (MSG_NOTE, vect_location,
9884 : "Vector cost for part in loop %d\n", vl);
9885 699658 : class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
9886 699658 : auto_vec<stmt_info_for_cost> tem;
9887 1941915 : do
9888 : {
9889 1941915 : tem.safe_push (*li_vector_costs[vi].second);
9890 1941915 : vi++;
9891 : }
9892 1941915 : while (vi < li_vector_costs.length ()
9893 3897651 : && li_vector_costs[vi].first == vl);
9894 699658 : add_slp_costs (vect_target_cost_data, tem);
9895 699658 : vect_target_cost_data->finish_cost (scalar_target_cost_data);
9896 699658 : vec_prologue_cost = vect_target_cost_data->prologue_cost ();
9897 699658 : vec_inside_cost = vect_target_cost_data->body_cost ();
9898 699658 : vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
9899 699658 : delete scalar_target_cost_data;
9900 699658 : delete vect_target_cost_data;
9901 :
9902 699658 : vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
9903 :
9904 699658 : if (dump_enabled_p ())
9905 : {
9906 143 : dump_printf_loc (MSG_NOTE, vect_location,
9907 : "Cost model analysis for part in loop %d:\n", sl);
9908 143 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
9909 : vec_inside_cost + vec_outside_cost);
9910 143 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", scalar_cost);
9911 : }
9912 :
9913 : /* Vectorization is profitable if its cost is more than the cost of scalar
9914 : version. Note that we err on the vector side for equal cost because
9915 : the cost estimate is otherwise quite pessimistic (constant uses are
9916 : free on the scalar side but cost a load on the vector side for
9917 : example). */
9918 699658 : if (vec_outside_cost + vec_inside_cost > scalar_cost)
9919 203861 : profitable = false;
9920 699658 : }
9921 1175591 : if (profitable && vi < li_vector_costs.length ())
9922 : {
9923 325 : if (dump_enabled_p ())
9924 0 : dump_printf_loc (MSG_NOTE, vect_location,
9925 : "Excess vector cost for part in loop %d:\n",
9926 0 : li_vector_costs[vi].first);
9927 : profitable = false;
9928 : }
9929 :
9930 : /* Unset visited flag. This is delayed when the subgraph is profitable
9931 : and we process the loop for remaining unvectorized if-converted code. */
9932 686182 : if (!orig_loop || !profitable)
9933 2969359 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9934 2283337 : gimple_set_visited (cost->stmt_info->stmt, false);
9935 :
9936 686182 : scalar_costs.release ();
9937 686182 : vector_costs.release ();
9938 :
9939 686182 : return profitable;
9940 686182 : }
9941 :
9942 : /* qsort comparator for lane defs. */
9943 :
9944 : static int
9945 128 : vld_cmp (const void *a_, const void *b_)
9946 : {
9947 128 : auto *a = (const std::pair<unsigned, tree> *)a_;
9948 128 : auto *b = (const std::pair<unsigned, tree> *)b_;
9949 128 : return a->first - b->first;
9950 : }
9951 :
9952 : /* Return true if USE_STMT is a vector lane insert into VEC and set
9953 : *THIS_LANE to the lane number that is set. */
9954 :
9955 : static bool
9956 303 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
9957 : {
9958 303 : gassign *use_ass = dyn_cast <gassign *> (use_stmt);
9959 110 : if (!use_ass
9960 110 : || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
9961 35 : || (vec
9962 35 : ? gimple_assign_rhs1 (use_ass) != vec
9963 26 : : ((vec = gimple_assign_rhs1 (use_ass)), false))
9964 61 : || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
9965 61 : TREE_TYPE (gimple_assign_rhs2 (use_ass)))
9966 61 : || !constant_multiple_p
9967 61 : (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
9968 364 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
9969 : this_lane))
9970 : return false;
9971 : return true;
9972 : }
9973 :
9974 : /* Find any vectorizable constructors and add them to the grouped_store
9975 : array. */
9976 :
9977 : static void
9978 2296064 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
9979 : {
9980 18371992 : for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
9981 32151856 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
9982 148199252 : !gsi_end_p (gsi); gsi_next (&gsi))
9983 : {
9984 132123324 : gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
9985 : /* This can be used to start SLP discovery for early breaks for BB early breaks
9986 : when we get that far. */
9987 132123324 : if (!assign)
9988 101500629 : continue;
9989 :
9990 32743016 : tree rhs = gimple_assign_rhs1 (assign);
9991 32743016 : enum tree_code code = gimple_assign_rhs_code (assign);
9992 32743016 : use_operand_p use_p;
9993 32743016 : gimple *use_stmt;
9994 32743016 : if (code == CONSTRUCTOR)
9995 : {
9996 1737610 : if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
9997 1782740 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
9998 98187 : CONSTRUCTOR_NELTS (rhs))
9999 45130 : || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
10000 1782720 : || uniform_vector_p (rhs))
10001 1723196 : continue;
10002 :
10003 : unsigned j;
10004 : tree val;
10005 70955 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10006 56541 : if (TREE_CODE (val) != SSA_NAME
10007 56541 : || !bb_vinfo->lookup_def (val))
10008 : break;
10009 35478 : if (j != CONSTRUCTOR_NELTS (rhs))
10010 3325 : continue;
10011 :
10012 14414 : vec<stmt_vec_info> roots = vNULL;
10013 14414 : roots.safe_push (bb_vinfo->lookup_stmt (assign));
10014 14414 : vec<stmt_vec_info> stmts;
10015 14414 : stmts.create (CONSTRUCTOR_NELTS (rhs));
10016 80364 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10017 51536 : stmts.quick_push
10018 51536 : (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
10019 14414 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10020 14414 : stmts, roots));
10021 : }
10022 31005406 : else if (code == BIT_INSERT_EXPR
10023 1045 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
10024 719 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
10025 719 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
10026 716 : && integer_zerop (gimple_assign_rhs3 (assign))
10027 400 : && useless_type_conversion_p
10028 400 : (TREE_TYPE (TREE_TYPE (rhs)),
10029 400 : TREE_TYPE (gimple_assign_rhs2 (assign)))
10030 31006146 : && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
10031 : {
10032 : /* We start to match on insert to lane zero but since the
10033 : inserts need not be ordered we'd have to search both
10034 : the def and the use chains. */
10035 265 : tree vectype = TREE_TYPE (rhs);
10036 265 : unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10037 265 : auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
10038 265 : auto_sbitmap lanes (nlanes);
10039 265 : bitmap_clear (lanes);
10040 265 : bitmap_set_bit (lanes, 0);
10041 265 : tree def = gimple_assign_lhs (assign);
10042 265 : lane_defs.quick_push
10043 265 : (std::make_pair (0, gimple_assign_rhs2 (assign)));
10044 265 : unsigned lanes_found = 1;
10045 : /* Start with the use chains, the last stmt will be the root. */
10046 265 : stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
10047 265 : vec<stmt_vec_info> roots = vNULL;
10048 265 : roots.safe_push (last);
10049 276 : do
10050 : {
10051 276 : use_operand_p use_p;
10052 276 : gimple *use_stmt;
10053 276 : if (!single_imm_use (def, &use_p, &use_stmt))
10054 : break;
10055 264 : unsigned this_lane;
10056 264 : if (!bb_vinfo->lookup_stmt (use_stmt)
10057 264 : || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
10058 299 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
10059 : break;
10060 35 : if (bitmap_bit_p (lanes, this_lane))
10061 : break;
10062 15 : lanes_found++;
10063 15 : bitmap_set_bit (lanes, this_lane);
10064 15 : gassign *use_ass = as_a <gassign *> (use_stmt);
10065 15 : lane_defs.quick_push (std::make_pair
10066 15 : (this_lane, gimple_assign_rhs2 (use_ass)));
10067 15 : last = bb_vinfo->lookup_stmt (use_ass);
10068 15 : roots.safe_push (last);
10069 15 : def = gimple_assign_lhs (use_ass);
10070 : }
10071 15 : while (lanes_found < nlanes);
10072 265 : if (roots.length () > 1)
10073 7 : std::swap(roots[0], roots[roots.length () - 1]);
10074 265 : if (lanes_found < nlanes)
10075 : {
10076 : /* Now search the def chain. */
10077 261 : def = gimple_assign_rhs1 (assign);
10078 263 : do
10079 : {
10080 263 : if (TREE_CODE (def) != SSA_NAME
10081 263 : || !has_single_use (def))
10082 : break;
10083 59 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
10084 59 : unsigned this_lane;
10085 59 : if (!bb_vinfo->lookup_stmt (def_stmt)
10086 39 : || !vect_slp_is_lane_insert (def_stmt,
10087 : NULL_TREE, &this_lane)
10088 85 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
10089 : break;
10090 26 : if (bitmap_bit_p (lanes, this_lane))
10091 : break;
10092 6 : lanes_found++;
10093 6 : bitmap_set_bit (lanes, this_lane);
10094 12 : lane_defs.quick_push (std::make_pair
10095 6 : (this_lane,
10096 6 : gimple_assign_rhs2 (def_stmt)));
10097 6 : roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
10098 6 : def = gimple_assign_rhs1 (def_stmt);
10099 : }
10100 6 : while (lanes_found < nlanes);
10101 : }
10102 265 : if (lanes_found == nlanes)
10103 : {
10104 : /* Sort lane_defs after the lane index and register the root. */
10105 8 : lane_defs.qsort (vld_cmp);
10106 8 : vec<stmt_vec_info> stmts;
10107 8 : stmts.create (nlanes);
10108 44 : for (unsigned i = 0; i < nlanes; ++i)
10109 28 : stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
10110 8 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10111 8 : stmts, roots));
10112 : }
10113 : else
10114 257 : roots.release ();
10115 265 : }
10116 31005141 : else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10117 29989323 : && (associative_tree_code (code) || code == MINUS_EXPR)
10118 : /* Ops with constants at the tail can be stripped here. */
10119 6596389 : && TREE_CODE (rhs) == SSA_NAME
10120 6522541 : && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
10121 : /* Should be the chain end. */
10122 33841683 : && (!single_imm_use (gimple_assign_lhs (assign),
10123 : &use_p, &use_stmt)
10124 2269418 : || !is_gimple_assign (use_stmt)
10125 1643608 : || (gimple_assign_rhs_code (use_stmt) != code
10126 1093278 : && ((code != PLUS_EXPR && code != MINUS_EXPR)
10127 615398 : || (gimple_assign_rhs_code (use_stmt)
10128 615398 : != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
10129 : {
10130 : /* We start the match at the end of a possible association
10131 : chain. */
10132 2169630 : auto_vec<chain_op_t> chain;
10133 2169630 : auto_vec<std::pair<tree_code, gimple *> > worklist;
10134 2169630 : auto_vec<gimple *> chain_stmts;
10135 2169630 : gimple *code_stmt = NULL, *alt_code_stmt = NULL;
10136 2169630 : if (code == MINUS_EXPR)
10137 362295 : code = PLUS_EXPR;
10138 2169630 : internal_fn reduc_fn;
10139 2566755 : if (!reduction_fn_for_scalar_code (code, &reduc_fn)
10140 2169630 : || reduc_fn == IFN_LAST)
10141 397125 : continue;
10142 1772505 : vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
10143 : /* ??? */
10144 : code_stmt, alt_code_stmt, &chain_stmts,
10145 : false);
10146 3545010 : if (chain.length () > 1)
10147 : {
10148 : /* Sort the chain according to def_type and operation. */
10149 1772505 : chain.sort (dt_sort_cmp, bb_vinfo);
10150 : /* ??? Now we'd want to strip externals and constants
10151 : but record those to be handled in the epilogue. */
10152 : /* ??? For now do not allow mixing ops or externs/constants. */
10153 1772505 : bool invalid = false;
10154 1772505 : unsigned remain_cnt = 0;
10155 7302854 : for (unsigned i = 0; i < chain.length (); ++i)
10156 : {
10157 4120139 : if (chain[i].code != code)
10158 : {
10159 : invalid = true;
10160 : break;
10161 : }
10162 3757844 : if (chain[i].dt != vect_internal_def
10163 : /* Avoid stmts where the def is not the LHS, like
10164 : ASMs. */
10165 7161459 : || (gimple_get_lhs (bb_vinfo->lookup_def
10166 3403615 : (chain[i].op)->stmt)
10167 3403615 : != chain[i].op))
10168 357181 : remain_cnt++;
10169 : }
10170 1772505 : if (!invalid && chain.length () - remain_cnt > 1)
10171 : {
10172 1325401 : vec<stmt_vec_info> stmts;
10173 1325401 : vec<tree> remain = vNULL;
10174 1325401 : stmts.create (chain.length ());
10175 1325401 : if (remain_cnt > 0)
10176 47279 : remain.create (remain_cnt);
10177 4412152 : for (unsigned i = 0; i < chain.length (); ++i)
10178 : {
10179 3086751 : stmt_vec_info stmt_info;
10180 3086751 : if (chain[i].dt == vect_internal_def
10181 3086751 : && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
10182 2998516 : gimple_get_lhs (stmt_info->stmt) == chain[i].op))
10183 2998432 : stmts.quick_push (stmt_info);
10184 : else
10185 88319 : remain.quick_push (chain[i].op);
10186 : }
10187 1325401 : vec<stmt_vec_info> roots;
10188 1325401 : roots.create (chain_stmts.length ());
10189 4412152 : for (unsigned i = 0; i < chain_stmts.length (); ++i)
10190 1761350 : roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
10191 1325401 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
10192 1325401 : stmts, roots, remain));
10193 : }
10194 : }
10195 2169630 : }
10196 : }
10197 2296064 : }
10198 :
10199 : /* Walk the grouped store chains and replace entries with their
10200 : pattern variant if any. */
10201 :
10202 : static void
10203 668497 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
10204 : {
10205 668497 : stmt_vec_info first_element;
10206 668497 : unsigned i;
10207 :
10208 1579558 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
10209 : {
10210 : /* We also have CTORs in this array. */
10211 911061 : if (!STMT_VINFO_GROUPED_ACCESS (first_element))
10212 0 : continue;
10213 911061 : if (STMT_VINFO_IN_PATTERN_P (first_element))
10214 : {
10215 252 : stmt_vec_info orig = first_element;
10216 252 : first_element = STMT_VINFO_RELATED_STMT (first_element);
10217 252 : DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
10218 252 : DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
10219 252 : DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
10220 252 : DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
10221 252 : vinfo->grouped_stores[i] = first_element;
10222 : }
10223 911061 : stmt_vec_info prev = first_element;
10224 2564093 : while (DR_GROUP_NEXT_ELEMENT (prev))
10225 : {
10226 1653032 : stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
10227 1653032 : if (STMT_VINFO_IN_PATTERN_P (elt))
10228 : {
10229 849 : stmt_vec_info orig = elt;
10230 849 : elt = STMT_VINFO_RELATED_STMT (elt);
10231 849 : DR_GROUP_NEXT_ELEMENT (prev) = elt;
10232 849 : DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
10233 849 : DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
10234 : }
10235 1653032 : DR_GROUP_FIRST_ELEMENT (elt) = first_element;
10236 1653032 : prev = elt;
10237 : }
10238 : }
10239 668497 : }
10240 :
10241 : /* Check if the region described by BB_VINFO can be vectorized, returning
10242 : true if so. When returning false, set FATAL to true if the same failure
10243 : would prevent vectorization at other vector sizes, false if it is still
10244 : worth trying other sizes. N_STMTS is the number of statements in the
10245 : region. */
10246 :
10247 : static bool
10248 2296064 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
10249 : vec<int> *dataref_groups)
10250 : {
10251 2296064 : DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
10252 :
10253 2296064 : slp_instance instance;
10254 2296064 : int i;
10255 :
10256 : /* The first group of checks is independent of the vector size. */
10257 2296064 : fatal = true;
10258 :
10259 : /* Analyze the data references. */
10260 :
10261 2296064 : if (!vect_analyze_data_refs (bb_vinfo, NULL))
10262 : {
10263 0 : if (dump_enabled_p ())
10264 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10265 : "not vectorized: unhandled data-ref in basic "
10266 : "block.\n");
10267 : return false;
10268 : }
10269 :
10270 2296064 : if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
10271 : {
10272 0 : if (dump_enabled_p ())
10273 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10274 : "not vectorized: unhandled data access in "
10275 : "basic block.\n");
10276 : return false;
10277 : }
10278 :
10279 2296064 : vect_slp_check_for_roots (bb_vinfo);
10280 :
10281 : /* If there are no grouped stores and no constructors in the region
10282 : there is no need to continue with pattern recog as vect_analyze_slp
10283 : will fail anyway. */
10284 2296064 : if (bb_vinfo->grouped_stores.is_empty ()
10285 1945053 : && bb_vinfo->roots.is_empty ())
10286 : {
10287 1627567 : if (dump_enabled_p ())
10288 1014 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10289 : "not vectorized: no grouped stores in "
10290 : "basic block.\n");
10291 : return false;
10292 : }
10293 :
10294 : /* While the rest of the analysis below depends on it in some way. */
10295 668497 : fatal = false;
10296 :
10297 668497 : vect_pattern_recog (bb_vinfo);
10298 :
10299 : /* Update store groups from pattern processing. */
10300 668497 : vect_fixup_store_groups_with_patterns (bb_vinfo);
10301 :
10302 : /* Check the SLP opportunities in the basic block, analyze and build SLP
10303 : trees. */
10304 668497 : if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
10305 : {
10306 0 : if (dump_enabled_p ())
10307 : {
10308 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10309 : "Failed to SLP the basic block.\n");
10310 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10311 : "not vectorized: failed to find SLP opportunities "
10312 : "in basic block.\n");
10313 : }
10314 : return false;
10315 : }
10316 :
10317 : /* Optimize permutations. */
10318 668497 : vect_optimize_slp (bb_vinfo);
10319 :
10320 : /* Gather the loads reachable from the SLP graph entries. */
10321 668497 : vect_gather_slp_loads (bb_vinfo);
10322 :
10323 668497 : vect_record_base_alignments (bb_vinfo);
10324 :
10325 : /* Analyze and verify the alignment of data references and the
10326 : dependence in the SLP instances. */
10327 2166876 : for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
10328 : {
10329 829882 : vect_location = instance->location ();
10330 829882 : if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
10331 829882 : || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
10332 : {
10333 8717 : slp_tree node = SLP_INSTANCE_TREE (instance);
10334 8717 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10335 8717 : if (dump_enabled_p ())
10336 4 : dump_printf_loc (MSG_NOTE, vect_location,
10337 : "removing SLP instance operations starting from: %G",
10338 : stmt_info->stmt);
10339 8717 : vect_free_slp_instance (instance);
10340 8717 : BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
10341 8717 : continue;
10342 8717 : }
10343 :
10344 : /* Mark all the statements that we want to vectorize as relevant. */
10345 821165 : vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
10346 :
10347 821165 : i++;
10348 : }
10349 2574850 : if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
10350 : return false;
10351 :
10352 278786 : if (!vect_slp_analyze_operations (bb_vinfo))
10353 : {
10354 31951 : if (dump_enabled_p ())
10355 69 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10356 : "not vectorized: bad operation in basic block.\n");
10357 : return false;
10358 : }
10359 :
10360 : /* Compute vector stmt placement. */
10361 246835 : if (!vect_schedule_slp (bb_vinfo, BB_VINFO_SLP_INSTANCES (bb_vinfo), true))
10362 : {
10363 0 : if (dump_enabled_p ())
10364 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10365 : "not vectorized: cannot schedule SLP graph\n");
10366 : return false;
10367 : }
10368 :
10369 : /* Mark all the statements that we vectorize. */
10370 246835 : vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
10371 :
10372 : /* Compute vectorizable live stmts. */
10373 246835 : vect_bb_slp_mark_live_stmts (bb_vinfo);
10374 :
10375 246835 : vect_bb_partition_graph (bb_vinfo);
10376 :
10377 246835 : return true;
10378 : }
10379 :
10380 : /* Subroutine of vect_slp_bb. Try to vectorize the statements for all
10381 : basic blocks in BBS, returning true on success.
10382 : The region has N_STMTS statements and has the datarefs given by DATAREFS. */
10383 :
10384 : static bool
10385 1939978 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
10386 : vec<int> *dataref_groups, unsigned int n_stmts,
10387 : loop_p orig_loop)
10388 : {
10389 1939978 : bb_vec_info bb_vinfo;
10390 1939978 : auto_vector_modes vector_modes;
10391 :
10392 : /* Autodetect first vector size we try. */
10393 1939978 : machine_mode next_vector_mode = VOIDmode;
10394 1939978 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
10395 1939978 : unsigned int mode_i = 0;
10396 :
10397 1939978 : vec_info_shared shared;
10398 :
10399 1939978 : machine_mode autodetected_vector_mode = VOIDmode;
10400 2652150 : while (1)
10401 : {
10402 2296064 : bool vectorized = false;
10403 2296064 : bool fatal = false;
10404 2296064 : bb_vinfo = new _bb_vec_info (bbs, &shared);
10405 :
10406 2296064 : bool first_time_p = shared.datarefs.is_empty ();
10407 2296064 : BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
10408 2296064 : if (first_time_p)
10409 1964630 : bb_vinfo->shared->save_datarefs ();
10410 : else
10411 331434 : bb_vinfo->shared->check_datarefs ();
10412 2296064 : bb_vinfo->vector_mode = next_vector_mode;
10413 :
10414 2296064 : if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
10415 : {
10416 246835 : if (dump_enabled_p ())
10417 : {
10418 1632 : dump_printf_loc (MSG_NOTE, vect_location,
10419 : "***** Analysis succeeded with vector mode"
10420 816 : " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
10421 816 : dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
10422 : }
10423 :
10424 246835 : bb_vinfo->shared->check_datarefs ();
10425 :
10426 246835 : bool force_clear = false;
10427 246835 : auto_vec<slp_instance> profitable_subgraphs;
10428 1457451 : for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
10429 : {
10430 716946 : if (instance->subgraph_entries.is_empty ())
10431 224034 : continue;
10432 :
10433 689997 : dump_user_location_t saved_vect_location = vect_location;
10434 689997 : vect_location = instance->location ();
10435 689997 : if (!unlimited_cost_model (NULL)
10436 686191 : && !param_vect_allow_possibly_not_worthwhile_vectorizations
10437 1376179 : && !vect_bb_vectorization_profitable_p
10438 686182 : (bb_vinfo, instance->subgraph_entries, orig_loop))
10439 : {
10440 197085 : if (dump_enabled_p ())
10441 51 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10442 : "not vectorized: vectorization is not "
10443 : "profitable.\n");
10444 197085 : vect_location = saved_vect_location;
10445 197085 : continue;
10446 : }
10447 :
10448 492912 : vect_location = saved_vect_location;
10449 492912 : if (!dbg_cnt (vect_slp))
10450 : {
10451 0 : force_clear = true;
10452 0 : continue;
10453 : }
10454 :
10455 492912 : profitable_subgraphs.safe_push (instance);
10456 : }
10457 :
10458 : /* When we're vectorizing an if-converted loop body make sure
10459 : we vectorized all if-converted code. */
10460 410107 : if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
10461 : {
10462 159 : gcc_assert (bb_vinfo->nbbs == 1);
10463 318 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
10464 6235 : !gsi_end_p (gsi); gsi_next (&gsi))
10465 : {
10466 : /* The costing above left us with DCEable vectorized scalar
10467 : stmts having the visited flag set on profitable
10468 : subgraphs. Do the delayed clearing of the flag here. */
10469 6076 : if (gimple_visited_p (gsi_stmt (gsi)))
10470 : {
10471 1936 : gimple_set_visited (gsi_stmt (gsi), false);
10472 1936 : continue;
10473 : }
10474 4140 : if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
10475 813 : continue;
10476 :
10477 9115 : if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
10478 3773 : if (gimple_assign_rhs_code (ass) == COND_EXPR)
10479 : {
10480 175 : if (!profitable_subgraphs.is_empty ()
10481 74 : && dump_enabled_p ())
10482 0 : dump_printf_loc (MSG_NOTE, vect_location,
10483 : "not profitable because of "
10484 : "unprofitable if-converted scalar "
10485 : "code\n");
10486 101 : profitable_subgraphs.truncate (0);
10487 : }
10488 : }
10489 : }
10490 :
10491 : /* Finally schedule the profitable subgraphs. */
10492 1066198 : for (slp_instance instance : profitable_subgraphs)
10493 : {
10494 492819 : if (!vectorized && dump_enabled_p ())
10495 777 : dump_printf_loc (MSG_NOTE, vect_location,
10496 : "Basic block will be vectorized "
10497 : "using SLP\n");
10498 492819 : vectorized = true;
10499 :
10500 : /* Dump before scheduling as store vectorization will remove
10501 : the original stores and mess with the instance tree
10502 : so querying its location will eventually ICE. */
10503 492819 : if (flag_checking)
10504 1983292 : for (slp_instance sub : instance->subgraph_entries)
10505 504835 : gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
10506 492819 : unsigned HOST_WIDE_INT bytes;
10507 492819 : if (dump_enabled_p ())
10508 3693 : for (slp_instance sub : instance->subgraph_entries)
10509 : {
10510 984 : tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
10511 1968 : if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
10512 984 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10513 984 : sub->location (),
10514 : "basic block part vectorized using %wu "
10515 : "byte vectors\n", bytes);
10516 : else
10517 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10518 : sub->location (),
10519 : "basic block part vectorized using "
10520 : "variable length vectors\n");
10521 : }
10522 :
10523 492819 : dump_user_location_t saved_vect_location = vect_location;
10524 492819 : vect_location = instance->location ();
10525 :
10526 492819 : vect_schedule_slp (bb_vinfo, instance->subgraph_entries, false);
10527 :
10528 492819 : vect_location = saved_vect_location;
10529 : }
10530 :
10531 : /* Generate the invariant statements. */
10532 246835 : if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
10533 : {
10534 27 : if (dump_enabled_p ())
10535 0 : dump_printf_loc (MSG_NOTE, vect_location,
10536 : "------>generating invariant statements\n");
10537 :
10538 27 : bb_vinfo->insert_seq_on_entry (NULL,
10539 : bb_vinfo->inv_pattern_def_seq);
10540 : }
10541 246835 : }
10542 : else
10543 : {
10544 2049229 : if (dump_enabled_p ())
10545 1317 : dump_printf_loc (MSG_NOTE, vect_location,
10546 : "***** Analysis failed with vector mode %s\n",
10547 1317 : GET_MODE_NAME (bb_vinfo->vector_mode));
10548 : }
10549 :
10550 2296064 : if (mode_i == 0)
10551 1939978 : autodetected_vector_mode = bb_vinfo->vector_mode;
10552 :
10553 2296064 : if (!fatal)
10554 3321846 : while (mode_i < vector_modes.length ()
10555 1902302 : && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
10556 : {
10557 357285 : if (dump_enabled_p ())
10558 1772 : dump_printf_loc (MSG_NOTE, vect_location,
10559 : "***** The result for vector mode %s would"
10560 : " be the same\n",
10561 886 : GET_MODE_NAME (vector_modes[mode_i]));
10562 357285 : mode_i += 1;
10563 : }
10564 :
10565 2296064 : delete bb_vinfo;
10566 :
10567 2296064 : if (mode_i < vector_modes.length ()
10568 2100483 : && VECTOR_MODE_P (autodetected_vector_mode)
10569 2093758 : && (related_vector_mode (vector_modes[mode_i],
10570 : GET_MODE_INNER (autodetected_vector_mode))
10571 1046879 : == autodetected_vector_mode)
10572 4396547 : && (related_vector_mode (autodetected_vector_mode,
10573 527499 : GET_MODE_INNER (vector_modes[mode_i]))
10574 1054998 : == vector_modes[mode_i]))
10575 : {
10576 527499 : if (dump_enabled_p ())
10577 195 : dump_printf_loc (MSG_NOTE, vect_location,
10578 : "***** Skipping vector mode %s, which would"
10579 : " repeat the analysis for %s\n",
10580 195 : GET_MODE_NAME (vector_modes[mode_i]),
10581 195 : GET_MODE_NAME (autodetected_vector_mode));
10582 527499 : mode_i += 1;
10583 : }
10584 :
10585 2296064 : if (vectorized
10586 2132866 : || mode_i == vector_modes.length ()
10587 1937334 : || autodetected_vector_mode == VOIDmode
10588 : /* If vect_slp_analyze_bb_1 signaled that analysis for all
10589 : vector sizes will fail do not bother iterating. */
10590 3179794 : || fatal)
10591 3879956 : return vectorized;
10592 :
10593 : /* Try the next biggest vector size. */
10594 356086 : next_vector_mode = vector_modes[mode_i++];
10595 356086 : if (dump_enabled_p ())
10596 235 : dump_printf_loc (MSG_NOTE, vect_location,
10597 : "***** Re-trying analysis with vector mode %s\n",
10598 235 : GET_MODE_NAME (next_vector_mode));
10599 356086 : }
10600 1939978 : }
10601 :
10602 :
10603 : /* Main entry for the BB vectorizer. Analyze and transform BBS, returns
10604 : true if anything in the basic-block was vectorized. */
10605 :
10606 : static bool
10607 1939978 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
10608 : {
10609 1939978 : vec<data_reference_p> datarefs = vNULL;
10610 1939978 : auto_vec<int> dataref_groups;
10611 1939978 : int insns = 0;
10612 1939978 : int current_group = 0;
10613 :
10614 12845974 : for (unsigned i = 0; i < bbs.length (); i++)
10615 : {
10616 10905996 : basic_block bb = bbs[i];
10617 95562770 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
10618 84656774 : gsi_next (&gsi))
10619 : {
10620 84656774 : gimple *stmt = gsi_stmt (gsi);
10621 84656774 : if (is_gimple_debug (stmt))
10622 53909113 : continue;
10623 :
10624 30747661 : insns++;
10625 :
10626 30747661 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
10627 27593989 : vect_location = stmt;
10628 :
10629 30747661 : if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
10630 : &dataref_groups, current_group))
10631 5245041 : ++current_group;
10632 : }
10633 : /* New BBs always start a new DR group. */
10634 10905996 : ++current_group;
10635 : }
10636 :
10637 1939978 : return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
10638 1939978 : }
10639 :
10640 : /* Special entry for the BB vectorizer. Analyze and transform a single
10641 : if-converted BB with ORIG_LOOPs body being the not if-converted
10642 : representation. Returns true if anything in the basic-block was
10643 : vectorized. */
10644 :
10645 : bool
10646 19486 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
10647 : {
10648 19486 : auto_vec<basic_block> bbs;
10649 19486 : bbs.safe_push (bb);
10650 19486 : return vect_slp_bbs (bbs, orig_loop);
10651 19486 : }
10652 :
10653 : /* Main entry for the BB vectorizer. Analyze and transform BB, returns
10654 : true if anything in the basic-block was vectorized. */
10655 :
10656 : bool
10657 925486 : vect_slp_function (function *fun)
10658 : {
10659 925486 : bool r = false;
10660 925486 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
10661 925486 : auto_bitmap exit_bbs;
10662 925486 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
10663 925486 : edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
10664 925486 : unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
10665 925486 : true, rpo, NULL);
10666 :
10667 : /* For the moment split the function into pieces to avoid making
10668 : the iteration on the vector mode moot. Split at points we know
10669 : to not handle well which is CFG merges (SLP discovery doesn't
10670 : handle non-loop-header PHIs) and loop exits. Since pattern
10671 : recog requires reverse iteration to visit uses before defs
10672 : simply chop RPO into pieces. */
10673 925486 : auto_vec<basic_block> bbs;
10674 11842947 : for (unsigned i = 0; i < n; i++)
10675 : {
10676 10917461 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
10677 10917461 : bool split = false;
10678 :
10679 : /* Split when a BB is not dominated by the first block. */
10680 20594532 : if (!bbs.is_empty ()
10681 9677071 : && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
10682 : {
10683 699895 : if (dump_enabled_p ())
10684 146 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10685 : "splitting region at dominance boundary bb%d\n",
10686 : bb->index);
10687 : split = true;
10688 : }
10689 : /* Split when the loop determined by the first block
10690 : is exited. This is because we eventually insert
10691 : invariants at region begin. */
10692 19194742 : else if (!bbs.is_empty ()
10693 8977176 : && bbs[0]->loop_father != bb->loop_father
10694 2326370 : && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
10695 : {
10696 3852 : if (dump_enabled_p ())
10697 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10698 : "splitting region at loop %d exit at bb%d\n",
10699 3 : bbs[0]->loop_father->num, bb->index);
10700 : split = true;
10701 : }
10702 10213714 : else if (!bbs.is_empty ()
10703 8973324 : && bb->loop_father->header == bb
10704 476235 : && bb->loop_father->dont_vectorize)
10705 : {
10706 7306 : if (dump_enabled_p ())
10707 72 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10708 : "splitting region at dont-vectorize loop %d "
10709 : "entry at bb%d\n",
10710 : bb->loop_father->num, bb->index);
10711 : split = true;
10712 : }
10713 :
10714 221 : if (split && !bbs.is_empty ())
10715 : {
10716 711053 : r |= vect_slp_bbs (bbs, NULL);
10717 711053 : bbs.truncate (0);
10718 : }
10719 :
10720 10917461 : if (bbs.is_empty ())
10721 : {
10722 : /* We need to be able to insert at the head of the region which
10723 : we cannot for region starting with a returns-twice call. */
10724 1951443 : if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
10725 412068 : if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
10726 : {
10727 306 : if (dump_enabled_p ())
10728 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10729 : "skipping bb%d as start of region as it "
10730 : "starts with returns-twice call\n",
10731 : bb->index);
10732 30951 : continue;
10733 : }
10734 : /* If the loop this BB belongs to is marked as not to be vectorized
10735 : honor that also for BB vectorization. */
10736 1951137 : if (bb->loop_father->dont_vectorize)
10737 30645 : continue;
10738 : }
10739 :
10740 10886510 : bbs.safe_push (bb);
10741 :
10742 : /* When we have a stmt ending this block and defining a
10743 : value we have to insert on edges when inserting after it for
10744 : a vector containing its definition. Avoid this for now. */
10745 21773020 : if (gimple *last = *gsi_last_bb (bb))
10746 8865377 : if (gimple_get_lhs (last)
10747 8865377 : && is_ctrl_altering_stmt (last))
10748 : {
10749 283960 : if (dump_enabled_p ())
10750 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10751 : "splitting region at control altering "
10752 : "definition %G", last);
10753 283960 : r |= vect_slp_bbs (bbs, NULL);
10754 283960 : bbs.truncate (0);
10755 : }
10756 : }
10757 :
10758 925486 : if (!bbs.is_empty ())
10759 925479 : r |= vect_slp_bbs (bbs, NULL);
10760 :
10761 925486 : free (rpo);
10762 :
10763 925486 : return r;
10764 925486 : }
10765 :
10766 : /* Build a variable-length vector in which the elements in ELTS are repeated
10767 : to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
10768 : RESULTS and add any new instructions to SEQ.
10769 :
10770 : The approach we use is:
10771 :
10772 : (1) Find a vector mode VM with integer elements of mode IM.
10773 :
10774 : (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10775 : ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
10776 : from small vectors to IM.
10777 :
10778 : (3) Duplicate each ELTS'[I] into a vector of mode VM.
10779 :
10780 : (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
10781 : correct byte contents.
10782 :
10783 : (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
10784 :
10785 : We try to find the largest IM for which this sequence works, in order
10786 : to cut down on the number of interleaves. */
10787 :
10788 : void
10789 0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
10790 : const vec<tree> &elts, unsigned int nresults,
10791 : vec<tree> &results)
10792 : {
10793 0 : unsigned int nelts = elts.length ();
10794 0 : tree element_type = TREE_TYPE (vector_type);
10795 :
10796 : /* (1) Find a vector mode VM with integer elements of mode IM. */
10797 0 : unsigned int nvectors = 1;
10798 0 : tree new_vector_type;
10799 0 : tree permutes[2];
10800 0 : if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
10801 : &nvectors, &new_vector_type,
10802 : permutes))
10803 0 : gcc_unreachable ();
10804 :
10805 : /* Get a vector type that holds ELTS[0:NELTS/NELTS']. */
10806 0 : unsigned int partial_nelts = nelts / nvectors;
10807 0 : tree partial_vector_type = build_vector_type (element_type, partial_nelts);
10808 :
10809 0 : tree_vector_builder partial_elts;
10810 0 : auto_vec<tree, 32> pieces (nvectors * 2);
10811 0 : pieces.quick_grow_cleared (nvectors * 2);
10812 0 : for (unsigned int i = 0; i < nvectors; ++i)
10813 : {
10814 : /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10815 : ELTS' has mode IM. */
10816 0 : partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
10817 0 : for (unsigned int j = 0; j < partial_nelts; ++j)
10818 0 : partial_elts.quick_push (elts[i * partial_nelts + j]);
10819 0 : tree t = gimple_build_vector (seq, &partial_elts);
10820 0 : t = gimple_build (seq, VIEW_CONVERT_EXPR,
10821 0 : TREE_TYPE (new_vector_type), t);
10822 :
10823 : /* (3) Duplicate each ELTS'[I] into a vector of mode VM. */
10824 0 : pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
10825 : }
10826 :
10827 : /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
10828 : correct byte contents.
10829 :
10830 : Conceptually, we need to repeat the following operation log2(nvectors)
10831 : times, where hi_start = nvectors / 2:
10832 :
10833 : out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
10834 : out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
10835 :
10836 : However, if each input repeats every N elements and the VF is
10837 : a multiple of N * 2, the HI result is the same as the LO result.
10838 : This will be true for the first N1 iterations of the outer loop,
10839 : followed by N2 iterations for which both the LO and HI results
10840 : are needed. I.e.:
10841 :
10842 : N1 + N2 = log2(nvectors)
10843 :
10844 : Each "N1 iteration" doubles the number of redundant vectors and the
10845 : effect of the process as a whole is to have a sequence of nvectors/2**N1
10846 : vectors that repeats 2**N1 times. Rather than generate these redundant
10847 : vectors, we halve the number of vectors for each N1 iteration. */
10848 : unsigned int in_start = 0;
10849 : unsigned int out_start = nvectors;
10850 : unsigned int new_nvectors = nvectors;
10851 0 : for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
10852 : {
10853 0 : unsigned int hi_start = new_nvectors / 2;
10854 0 : unsigned int out_i = 0;
10855 0 : for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
10856 : {
10857 0 : if ((in_i & 1) != 0
10858 0 : && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
10859 : 2 * in_repeat))
10860 0 : continue;
10861 :
10862 0 : tree output = make_ssa_name (new_vector_type);
10863 0 : tree input1 = pieces[in_start + (in_i / 2)];
10864 0 : tree input2 = pieces[in_start + (in_i / 2) + hi_start];
10865 0 : gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
10866 : input1, input2,
10867 : permutes[in_i & 1]);
10868 0 : gimple_seq_add_stmt (seq, stmt);
10869 0 : pieces[out_start + out_i] = output;
10870 0 : out_i += 1;
10871 : }
10872 0 : std::swap (in_start, out_start);
10873 0 : new_nvectors = out_i;
10874 : }
10875 :
10876 : /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type. */
10877 0 : results.reserve (nresults);
10878 0 : for (unsigned int i = 0; i < nresults; ++i)
10879 0 : if (i < new_nvectors)
10880 0 : results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
10881 0 : pieces[in_start + i]));
10882 : else
10883 0 : results.quick_push (results[i - new_nvectors]);
10884 0 : }
10885 :
10886 :
10887 : /* For constant and loop invariant defs in OP_NODE this function creates
10888 : vector defs that will be used in the vectorized stmts and stores them
10889 : to SLP_TREE_VEC_DEFS of OP_NODE. */
10890 :
10891 : static void
10892 501404 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
10893 : {
10894 501404 : unsigned HOST_WIDE_INT nunits;
10895 501404 : tree vec_cst;
10896 501404 : unsigned j, number_of_places_left_in_vector;
10897 501404 : tree vector_type;
10898 501404 : tree vop;
10899 501404 : int group_size = op_node->ops.length ();
10900 501404 : unsigned int vec_num, i;
10901 501404 : unsigned number_of_copies = 1;
10902 501404 : bool constant_p;
10903 501404 : gimple_seq ctor_seq = NULL;
10904 501404 : auto_vec<tree, 16> permute_results;
10905 :
10906 : /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
10907 501404 : vector_type = SLP_TREE_VECTYPE (op_node);
10908 :
10909 501404 : unsigned int number_of_vectors;
10910 501404 : bool res = vect_get_num_copies_for_invariant (vinfo, op_node,
10911 : &number_of_vectors);
10912 501404 : gcc_assert (res);
10913 501404 : SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
10914 501404 : auto_vec<tree> voprnds (number_of_vectors);
10915 :
10916 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
10917 : created vectors. It is greater than 1 if unrolling is performed.
10918 :
10919 : For example, we have two scalar operands, s1 and s2 (e.g., group of
10920 : strided accesses of size two), while NUNITS is four (i.e., four scalars
10921 : of this type can be packed in a vector). The output vector will contain
10922 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
10923 : will be 2).
10924 :
10925 : If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
10926 : containing the operands.
10927 :
10928 : For example, NUNITS is four as before, and the group size is 8
10929 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
10930 : {s5, s6, s7, s8}. */
10931 :
10932 : /* When using duplicate_and_interleave, we just need one element for
10933 : each scalar statement. */
10934 501404 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
10935 : nunits = group_size;
10936 :
10937 501404 : number_of_copies = nunits * number_of_vectors / group_size;
10938 :
10939 501404 : number_of_places_left_in_vector = nunits;
10940 501404 : constant_p = true;
10941 501404 : tree uniform_elt = NULL_TREE;
10942 501404 : tree_vector_builder elts (vector_type, nunits, 1);
10943 501404 : elts.quick_grow (nunits);
10944 501404 : stmt_vec_info insert_after = NULL;
10945 1988254 : for (j = 0; j < number_of_copies; j++)
10946 : {
10947 985446 : tree op;
10948 3786056 : for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
10949 : {
10950 : /* Create 'vect_ = {op0,op1,...,opn}'. */
10951 1815164 : tree orig_op = op;
10952 1815164 : if (number_of_places_left_in_vector == nunits)
10953 : uniform_elt = op;
10954 1181691 : else if (uniform_elt && operand_equal_p (uniform_elt, op))
10955 748073 : op = elts[number_of_places_left_in_vector];
10956 : else
10957 : uniform_elt = NULL_TREE;
10958 1815164 : number_of_places_left_in_vector--;
10959 1815164 : if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
10960 : {
10961 281222 : if (CONSTANT_CLASS_P (op))
10962 : {
10963 102487 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10964 : {
10965 : /* Can't use VIEW_CONVERT_EXPR for booleans because
10966 : of possibly different sizes of scalar value and
10967 : vector element. */
10968 66 : if (integer_zerop (op))
10969 66 : op = build_int_cst (TREE_TYPE (vector_type), 0);
10970 0 : else if (integer_onep (op))
10971 0 : op = build_all_ones_cst (TREE_TYPE (vector_type));
10972 : else
10973 0 : gcc_unreachable ();
10974 : }
10975 : else
10976 102421 : op = fold_unary (VIEW_CONVERT_EXPR,
10977 : TREE_TYPE (vector_type), op);
10978 102487 : gcc_assert (op && CONSTANT_CLASS_P (op));
10979 : }
10980 : else
10981 : {
10982 178735 : tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
10983 178735 : gimple *init_stmt;
10984 178735 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10985 : {
10986 427 : tree true_val
10987 427 : = build_all_ones_cst (TREE_TYPE (vector_type));
10988 427 : tree false_val
10989 427 : = build_zero_cst (TREE_TYPE (vector_type));
10990 427 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
10991 427 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
10992 : op, true_val,
10993 : false_val);
10994 : }
10995 : else
10996 : {
10997 178308 : op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
10998 : op);
10999 178308 : init_stmt
11000 178308 : = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
11001 : op);
11002 : }
11003 178735 : gimple_seq_add_stmt (&ctor_seq, init_stmt);
11004 178735 : op = new_temp;
11005 : }
11006 : }
11007 1815164 : elts[number_of_places_left_in_vector] = op;
11008 1815164 : if (!CONSTANT_CLASS_P (op))
11009 324102 : constant_p = false;
11010 : /* For BB vectorization we have to compute an insert location
11011 : when a def is inside the analyzed region since we cannot
11012 : simply insert at the BB start in this case. */
11013 1815164 : stmt_vec_info opdef;
11014 1815164 : if (TREE_CODE (orig_op) == SSA_NAME
11015 186530 : && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
11016 166258 : && is_a <bb_vec_info> (vinfo)
11017 1923821 : && (opdef = vinfo->lookup_def (orig_op)))
11018 : {
11019 88956 : if (!insert_after)
11020 : insert_after = opdef;
11021 : else
11022 48829 : insert_after = get_later_stmt (insert_after, opdef);
11023 : }
11024 :
11025 1815164 : if (number_of_places_left_in_vector == 0)
11026 : {
11027 633473 : auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
11028 633473 : if (uniform_elt)
11029 657624 : vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
11030 328812 : elts[0]);
11031 609322 : else if (constant_p
11032 609322 : ? multiple_p (type_nunits, nunits)
11033 112697 : : known_eq (type_nunits, nunits))
11034 304661 : vec_cst = gimple_build_vector (&ctor_seq, &elts);
11035 : else
11036 : {
11037 0 : if (permute_results.is_empty ())
11038 0 : duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
11039 : elts, number_of_vectors,
11040 : permute_results);
11041 0 : vec_cst = permute_results[number_of_vectors - j - 1];
11042 : }
11043 633473 : if (!gimple_seq_empty_p (ctor_seq))
11044 : {
11045 140612 : if (insert_after)
11046 : {
11047 40127 : gimple_stmt_iterator gsi;
11048 40127 : if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
11049 : {
11050 749 : gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
11051 749 : gsi_insert_seq_before (&gsi, ctor_seq,
11052 : GSI_CONTINUE_LINKING);
11053 : }
11054 39378 : else if (!stmt_ends_bb_p (insert_after->stmt))
11055 : {
11056 39378 : gsi = gsi_for_stmt (insert_after->stmt);
11057 39378 : gsi_insert_seq_after (&gsi, ctor_seq,
11058 : GSI_CONTINUE_LINKING);
11059 : }
11060 : else
11061 : {
11062 : /* When we want to insert after a def where the
11063 : defining stmt throws then insert on the fallthru
11064 : edge. */
11065 0 : edge e = find_fallthru_edge
11066 0 : (gimple_bb (insert_after->stmt)->succs);
11067 0 : basic_block new_bb
11068 0 : = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
11069 0 : gcc_assert (!new_bb);
11070 : }
11071 : }
11072 : else
11073 100485 : vinfo->insert_seq_on_entry (NULL, ctor_seq);
11074 140612 : ctor_seq = NULL;
11075 : }
11076 633473 : voprnds.quick_push (vec_cst);
11077 633473 : insert_after = NULL;
11078 633473 : number_of_places_left_in_vector = nunits;
11079 633473 : constant_p = true;
11080 633473 : elts.new_vector (vector_type, nunits, 1);
11081 633473 : elts.quick_grow (nunits);
11082 : }
11083 : }
11084 : }
11085 :
11086 : /* Since the vectors are created in the reverse order, we should invert
11087 : them. */
11088 501404 : vec_num = voprnds.length ();
11089 1134877 : for (j = vec_num; j != 0; j--)
11090 : {
11091 633473 : vop = voprnds[j - 1];
11092 633473 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11093 : }
11094 :
11095 : /* In case that VF is greater than the unrolling factor needed for the SLP
11096 : group of stmts, NUMBER_OF_VECTORS to be created is greater than
11097 : NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
11098 : to replicate the vectors. */
11099 1002808 : while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
11100 0 : for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
11101 : i++)
11102 0 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11103 501404 : }
11104 :
11105 : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
11106 : if there is no definition for it in the scalar IL or it is not known. */
11107 :
11108 : tree
11109 2617 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
11110 : {
11111 2617 : if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
11112 : {
11113 2605 : if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
11114 : return NULL_TREE;
11115 2605 : stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
11116 2605 : if (!def)
11117 : return NULL_TREE;
11118 2605 : return gimple_get_lhs (STMT_VINFO_STMT (def));
11119 : }
11120 : else
11121 12 : return SLP_TREE_SCALAR_OPS (slp_node)[n];
11122 : }
11123 :
11124 : /* Get the Ith vectorized definition from SLP_NODE. */
11125 :
11126 : tree
11127 145650 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
11128 : {
11129 145650 : return SLP_TREE_VEC_DEFS (slp_node)[i];
11130 : }
11131 :
11132 : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
11133 :
11134 : void
11135 951364 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
11136 : {
11137 1902728 : vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
11138 951364 : vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
11139 951364 : }
11140 :
11141 : /* Get N vectorized definitions for SLP_NODE. */
11142 :
11143 : void
11144 2953 : vect_get_slp_defs (vec_info *,
11145 : slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
11146 : {
11147 2953 : if (n == -1U)
11148 2953 : n = SLP_TREE_CHILDREN (slp_node).length ();
11149 :
11150 10648 : for (unsigned i = 0; i < n; ++i)
11151 : {
11152 7695 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
11153 7695 : vec<tree> vec_defs = vNULL;
11154 7695 : vect_get_slp_defs (child, &vec_defs);
11155 7695 : vec_oprnds->quick_push (vec_defs);
11156 : }
11157 2953 : }
11158 :
11159 : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
11160 : - PERM gives the permutation that the caller wants to use for NODE,
11161 : which might be different from SLP_LOAD_PERMUTATION.
11162 : - DUMP_P controls whether the function dumps information. */
11163 :
11164 : static bool
11165 138456 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
11166 : load_permutation_t &perm,
11167 : const vec<tree> &dr_chain,
11168 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11169 : bool analyze_only, bool dump_p,
11170 : unsigned *n_perms, unsigned int *n_loads,
11171 : bool dce_chain)
11172 : {
11173 138456 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
11174 138456 : int vec_index = 0;
11175 138456 : tree vectype = SLP_TREE_VECTYPE (node);
11176 138456 : unsigned int group_size = SLP_TREE_LANES (node);
11177 138456 : unsigned int mask_element;
11178 138456 : unsigned dr_group_size;
11179 138456 : machine_mode mode;
11180 :
11181 138456 : if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
11182 : {
11183 : /* We have both splats of the same non-grouped load and groups
11184 : of distinct invariant loads entering here. */
11185 1491 : unsigned max_idx = 0;
11186 8261 : for (auto idx : perm)
11187 3788 : max_idx = idx > max_idx ? idx : max_idx;
11188 1491 : dr_group_size = max_idx + 1;
11189 : }
11190 : else
11191 : {
11192 136965 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
11193 136965 : dr_group_size = DR_GROUP_SIZE (stmt_info);
11194 : }
11195 :
11196 138456 : mode = TYPE_MODE (vectype);
11197 138456 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11198 138456 : unsigned int nstmts = vect_get_num_copies (vinfo, node);
11199 :
11200 : /* Initialize the vect stmts of NODE to properly insert the generated
11201 : stmts later. */
11202 138456 : if (! analyze_only)
11203 59522 : for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
11204 22874 : SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
11205 :
11206 : /* Generate permutation masks for every NODE. Number of masks for each NODE
11207 : is equal to GROUP_SIZE.
11208 : E.g., we have a group of three nodes with three loads from the same
11209 : location in each node, and the vector size is 4. I.e., we have a
11210 : a0b0c0a1b1c1... sequence and we need to create the following vectors:
11211 : for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
11212 : for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
11213 : ...
11214 :
11215 : The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
11216 : The last mask is illegal since we assume two operands for permute
11217 : operation, and the mask element values can't be outside that range.
11218 : Hence, the last mask must be converted into {2,5,5,5}.
11219 : For the first two permutations we need the first and the second input
11220 : vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
11221 : we need the second and the third vectors: {b1,c1,a2,b2} and
11222 : {c2,a3,b3,c3}. */
11223 :
11224 138456 : int vect_stmts_counter = 0;
11225 138456 : unsigned int index = 0;
11226 138456 : int first_vec_index = -1;
11227 138456 : int second_vec_index = -1;
11228 138456 : bool noop_p = true;
11229 138456 : *n_perms = 0;
11230 :
11231 138456 : vec_perm_builder mask;
11232 138456 : unsigned int nelts_to_build;
11233 138456 : unsigned int nvectors_per_build;
11234 138456 : unsigned int in_nlanes;
11235 138456 : bool repeating_p = (group_size == dr_group_size
11236 174311 : && multiple_p (nunits, group_size));
11237 138456 : if (repeating_p)
11238 : {
11239 : /* A single vector contains a whole number of copies of the node, so:
11240 : (a) all permutes can use the same mask; and
11241 : (b) the permutes only need a single vector input. */
11242 33447 : mask.new_vector (nunits, group_size, 3);
11243 33447 : nelts_to_build = mask.encoded_nelts ();
11244 : /* It's possible to obtain zero nstmts during analyze_only, so make
11245 : it at least one to ensure the later computation for n_perms
11246 : proceed. */
11247 33447 : nvectors_per_build = nstmts > 0 ? nstmts : 1;
11248 33447 : in_nlanes = dr_group_size * 3;
11249 : }
11250 : else
11251 : {
11252 : /* We need to construct a separate mask for each vector statement. */
11253 105009 : unsigned HOST_WIDE_INT const_nunits, const_vf;
11254 105009 : if (!nunits.is_constant (&const_nunits)
11255 105009 : || !vf.is_constant (&const_vf))
11256 : return false;
11257 105009 : mask.new_vector (const_nunits, const_nunits, 1);
11258 105009 : nelts_to_build = const_vf * group_size;
11259 105009 : nvectors_per_build = 1;
11260 105009 : in_nlanes = const_vf * dr_group_size;
11261 : }
11262 138456 : auto_sbitmap used_in_lanes (in_nlanes);
11263 138456 : bitmap_clear (used_in_lanes);
11264 138456 : auto_bitmap used_defs;
11265 :
11266 138456 : unsigned int count = mask.encoded_nelts ();
11267 138456 : mask.quick_grow (count);
11268 138456 : vec_perm_indices indices;
11269 :
11270 727241 : for (unsigned int j = 0; j < nelts_to_build; j++)
11271 : {
11272 600136 : unsigned int iter_num = j / group_size;
11273 600136 : unsigned int stmt_num = j % group_size;
11274 600136 : unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
11275 600136 : bitmap_set_bit (used_in_lanes, i);
11276 600136 : if (repeating_p)
11277 : {
11278 : first_vec_index = 0;
11279 : mask_element = i;
11280 : }
11281 : else
11282 : {
11283 : /* Enforced before the loop when !repeating_p. */
11284 382372 : unsigned int const_nunits = nunits.to_constant ();
11285 382372 : vec_index = i / const_nunits;
11286 382372 : mask_element = i % const_nunits;
11287 382372 : if (vec_index == first_vec_index
11288 382372 : || first_vec_index == -1)
11289 : {
11290 : first_vec_index = vec_index;
11291 : }
11292 153077 : else if (vec_index == second_vec_index
11293 153077 : || second_vec_index == -1)
11294 : {
11295 146799 : second_vec_index = vec_index;
11296 146799 : mask_element += const_nunits;
11297 : }
11298 : else
11299 : {
11300 6278 : if (dump_p)
11301 280 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11302 : "permutation requires at "
11303 : "least three vectors %G",
11304 : stmt_info->stmt);
11305 6278 : gcc_assert (analyze_only);
11306 : return false;
11307 : }
11308 :
11309 376094 : gcc_assert (mask_element < 2 * const_nunits);
11310 : }
11311 :
11312 593858 : if (mask_element != index)
11313 381789 : noop_p = false;
11314 593858 : mask[index++] = mask_element;
11315 :
11316 593858 : if (index == count)
11317 : {
11318 163037 : if (!noop_p)
11319 : {
11320 132832 : indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
11321 132832 : if (!can_vec_perm_const_p (mode, mode, indices))
11322 : {
11323 5073 : if (dump_p)
11324 : {
11325 79 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11326 : "unsupported vect permute { ");
11327 673 : for (i = 0; i < count; ++i)
11328 : {
11329 594 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11330 594 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11331 : }
11332 79 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11333 : }
11334 5073 : gcc_assert (analyze_only);
11335 : return false;
11336 : }
11337 :
11338 127759 : tree mask_vec = NULL_TREE;
11339 127759 : if (!analyze_only)
11340 21151 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11341 :
11342 127759 : if (second_vec_index == -1)
11343 36882 : second_vec_index = first_vec_index;
11344 :
11345 258383 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11346 : {
11347 130624 : ++*n_perms;
11348 130624 : if (analyze_only)
11349 109190 : continue;
11350 : /* Generate the permute statement if necessary. */
11351 21434 : tree first_vec = dr_chain[first_vec_index + ri];
11352 21434 : tree second_vec = dr_chain[second_vec_index + ri];
11353 21434 : gassign *stmt = as_a<gassign *> (stmt_info->stmt);
11354 21434 : tree perm_dest
11355 21434 : = vect_create_destination_var (gimple_assign_lhs (stmt),
11356 : vectype);
11357 21434 : perm_dest = make_ssa_name (perm_dest);
11358 21434 : gimple *perm_stmt
11359 21434 : = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
11360 : second_vec, mask_vec);
11361 21434 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
11362 : gsi);
11363 21434 : if (dce_chain)
11364 : {
11365 20511 : bitmap_set_bit (used_defs, first_vec_index + ri);
11366 20511 : bitmap_set_bit (used_defs, second_vec_index + ri);
11367 : }
11368 :
11369 : /* Store the vector statement in NODE. */
11370 21434 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
11371 : }
11372 : }
11373 30205 : else if (!analyze_only)
11374 : {
11375 2880 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11376 : {
11377 1440 : tree first_vec = dr_chain[first_vec_index + ri];
11378 : /* If mask was NULL_TREE generate the requested
11379 : identity transform. */
11380 1440 : if (dce_chain)
11381 1433 : bitmap_set_bit (used_defs, first_vec_index + ri);
11382 :
11383 : /* Store the vector statement in NODE. */
11384 1440 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
11385 : }
11386 : }
11387 :
11388 : index = 0;
11389 : first_vec_index = -1;
11390 : second_vec_index = -1;
11391 : noop_p = true;
11392 : }
11393 : }
11394 :
11395 127105 : if (n_loads)
11396 : {
11397 89062 : if (repeating_p)
11398 10860 : *n_loads = nstmts;
11399 : else
11400 : {
11401 : /* Enforced above when !repeating_p. */
11402 78202 : unsigned int const_nunits = nunits.to_constant ();
11403 78202 : *n_loads = 0;
11404 78202 : bool load_seen = false;
11405 1051385 : for (unsigned i = 0; i < in_nlanes; ++i)
11406 : {
11407 973183 : if (i % const_nunits == 0)
11408 : {
11409 415634 : if (load_seen)
11410 123702 : *n_loads += 1;
11411 : load_seen = false;
11412 : }
11413 973183 : if (bitmap_bit_p (used_in_lanes, i))
11414 270874 : load_seen = true;
11415 : }
11416 78202 : if (load_seen)
11417 52247 : *n_loads += 1;
11418 : }
11419 : }
11420 :
11421 127105 : if (dce_chain)
11422 213082 : for (unsigned i = 0; i < dr_chain.length (); ++i)
11423 74626 : if (!bitmap_bit_p (used_defs, i))
11424 : {
11425 40903 : tree def = dr_chain[i];
11426 41315 : do
11427 : {
11428 41315 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11429 41315 : if (is_gimple_assign (stmt)
11430 41315 : && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
11431 41315 : || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
11432 4981 : def = single_ssa_tree_operand (stmt, SSA_OP_USE);
11433 : else
11434 : def = NULL;
11435 41315 : gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
11436 41315 : gsi_remove (&rgsi, true);
11437 41315 : release_defs (stmt);
11438 : }
11439 41315 : while (def);
11440 : }
11441 :
11442 : return true;
11443 138456 : }
11444 :
11445 : /* Generate vector permute statements from a list of loads in DR_CHAIN.
11446 : If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
11447 : permute statements for the SLP node NODE. Store the number of vector
11448 : permute instructions in *N_PERMS and the number of vector load
11449 : instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions
11450 : that were not needed. */
11451 :
11452 : bool
11453 98840 : vect_transform_slp_perm_load (vec_info *vinfo,
11454 : slp_tree node, const vec<tree> &dr_chain,
11455 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11456 : bool analyze_only, unsigned *n_perms,
11457 : unsigned int *n_loads, bool dce_chain)
11458 : {
11459 98840 : return vect_transform_slp_perm_load_1 (vinfo, node,
11460 98840 : SLP_TREE_LOAD_PERMUTATION (node),
11461 : dr_chain, gsi, vf, analyze_only,
11462 : dump_enabled_p (), n_perms, n_loads,
11463 98840 : dce_chain);
11464 : }
11465 :
11466 : /* Produce the next vector result for SLP permutation NODE by adding a vector
11467 : statement at GSI. If MASK_VEC is nonnull, add:
11468 :
11469 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
11470 :
11471 : otherwise add:
11472 :
11473 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
11474 : { N, N+1, N+2, ... }>
11475 :
11476 : where N == IDENTITY_OFFSET which is either zero or equal to the
11477 : number of elements of the result. */
11478 :
11479 : static void
11480 31651 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11481 : slp_tree node, tree first_def, tree second_def,
11482 : tree mask_vec, poly_uint64 identity_offset)
11483 : {
11484 31651 : tree vectype = SLP_TREE_VECTYPE (node);
11485 :
11486 : /* ??? We SLP match existing vector element extracts but
11487 : allow punning which we need to re-instantiate at uses
11488 : but have no good way of explicitly representing. */
11489 31651 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
11490 31651 : && !types_compatible_p (TREE_TYPE (first_def), vectype))
11491 : {
11492 20 : gassign *conv_stmt
11493 20 : = gimple_build_assign (make_ssa_name (vectype),
11494 : build1 (VIEW_CONVERT_EXPR, vectype, first_def));
11495 20 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11496 20 : first_def = gimple_assign_lhs (conv_stmt);
11497 : }
11498 31651 : gassign *perm_stmt;
11499 31651 : if (mask_vec)
11500 : {
11501 28129 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
11502 28129 : TYPE_SIZE (vectype))
11503 28129 : && !types_compatible_p (TREE_TYPE (second_def), vectype))
11504 : {
11505 8 : gassign *conv_stmt
11506 8 : = gimple_build_assign (make_ssa_name (vectype),
11507 : build1 (VIEW_CONVERT_EXPR,
11508 : vectype, second_def));
11509 8 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11510 8 : second_def = gimple_assign_lhs (conv_stmt);
11511 : }
11512 28129 : tree perm_dest = make_ssa_name (vectype);
11513 28129 : perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
11514 : first_def, second_def,
11515 : mask_vec);
11516 : }
11517 : else
11518 : {
11519 3522 : auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
11520 3522 : unsigned HOST_WIDE_INT vecno;
11521 3522 : poly_uint64 eltno;
11522 3522 : if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
11523 : &vecno, &eltno))
11524 : gcc_unreachable ();
11525 3522 : tree def = vecno & 1 ? second_def : first_def;
11526 3522 : if (!types_compatible_p (TREE_TYPE (def), vectype))
11527 : {
11528 : /* For identity permutes we still need to handle the case
11529 : of offsetted extracts or concats. */
11530 397 : tree perm_dest = make_ssa_name (vectype);
11531 397 : unsigned HOST_WIDE_INT c;
11532 397 : if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
11533 : {
11534 393 : unsigned HOST_WIDE_INT elsz
11535 393 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
11536 786 : tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
11537 393 : TYPE_SIZE (vectype),
11538 393 : bitsize_int (eltno * elsz));
11539 393 : perm_stmt = gimple_build_assign (perm_dest, lowpart);
11540 : }
11541 4 : else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
11542 4 : def_nunits, &c) && c == 2)
11543 : {
11544 4 : gcc_assert (known_eq (identity_offset, 0U));
11545 4 : tree ctor = build_constructor_va (vectype, 2,
11546 : NULL_TREE, first_def,
11547 : NULL_TREE, second_def);
11548 4 : perm_stmt = gimple_build_assign (perm_dest, ctor);
11549 : }
11550 : else
11551 0 : gcc_unreachable ();
11552 : }
11553 : else
11554 : {
11555 3125 : gcc_assert (known_eq (eltno, 0U));
11556 3125 : node->push_vec_def (def);
11557 3125 : return;
11558 : }
11559 : }
11560 28526 : vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
11561 : /* Store the vector statement in NODE. */
11562 28526 : node->push_vec_def (perm_stmt);
11563 : }
11564 :
11565 : /* Subroutine of vectorizable_slp_permutation. Check whether the target
11566 : can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
11567 : If GSI is nonnull, emit the permutation there.
11568 :
11569 : When GSI is null, the only purpose of NODE is to give properties
11570 : of the result, such as the vector type and number of SLP lanes.
11571 : The node does not need to be a VEC_PERM_EXPR.
11572 :
11573 : If the target supports the operation, return the number of individual
11574 : VEC_PERM_EXPRs needed, otherwise return -1. Print information to the
11575 : dump file if DUMP_P is true. */
11576 :
11577 : static int
11578 449560 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
11579 : slp_tree node, lane_permutation_t &perm,
11580 : vec<slp_tree> &children, bool dump_p)
11581 : {
11582 449560 : tree vectype = SLP_TREE_VECTYPE (node);
11583 :
11584 : /* ??? We currently only support all same vector input types
11585 : while the SLP IL should really do a concat + select and thus accept
11586 : arbitrary mismatches. */
11587 449560 : slp_tree child;
11588 449560 : unsigned i;
11589 449560 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11590 449560 : bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
11591 : /* True if we're permuting a single input of 2N vectors down
11592 : to N vectors. This case doesn't generalize beyond 2 since
11593 : VEC_PERM_EXPR only takes 2 inputs. */
11594 449560 : bool pack_p = false;
11595 : /* If we're permuting inputs of N vectors each into X*N outputs,
11596 : this is the value of X, otherwise it is 1. */
11597 449560 : unsigned int unpack_factor = 1;
11598 449560 : tree op_vectype = NULL_TREE;
11599 451138 : FOR_EACH_VEC_ELT (children, i, child)
11600 451045 : if (SLP_TREE_VECTYPE (child))
11601 : {
11602 : op_vectype = SLP_TREE_VECTYPE (child);
11603 : break;
11604 : }
11605 449560 : if (!op_vectype)
11606 93 : op_vectype = vectype;
11607 944458 : FOR_EACH_VEC_ELT (children, i, child)
11608 : {
11609 494898 : if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
11610 19120 : && !vect_maybe_update_slp_op_vectype (child, op_vectype))
11611 494898 : || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
11612 989796 : || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
11613 : {
11614 0 : if (dump_p)
11615 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11616 : "Unsupported vector types in lane permutation\n");
11617 449560 : return -1;
11618 : }
11619 494898 : auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
11620 494898 : unsigned int this_unpack_factor;
11621 : /* Detect permutations of external, pre-existing vectors. The external
11622 : node's SLP_TREE_LANES stores the total number of units in the vector,
11623 : or zero if the vector has variable length.
11624 :
11625 : We are expected to keep the original VEC_PERM_EXPR for such cases.
11626 : There is no repetition to model. */
11627 494898 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def
11628 494898 : && SLP_TREE_SCALAR_OPS (child).is_empty ())
11629 : repeating_p = false;
11630 : /* Check whether the input has twice as many lanes per vector. */
11631 478396 : else if (children.length () == 1
11632 478396 : && known_eq (SLP_TREE_LANES (child) * nunits,
11633 : SLP_TREE_LANES (node) * op_nunits * 2))
11634 : pack_p = true;
11635 : /* Check whether the output has N times as many lanes per vector. */
11636 494898 : else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
11637 434332 : SLP_TREE_LANES (child) * nunits,
11638 : &this_unpack_factor)
11639 399457 : && (i == 0 || unpack_factor == this_unpack_factor))
11640 : unpack_factor = this_unpack_factor;
11641 : else
11642 : repeating_p = false;
11643 : }
11644 :
11645 899120 : gcc_assert (perm.length () == SLP_TREE_LANES (node));
11646 :
11647 : /* Load-lanes permute. This permute only acts as a forwarder to
11648 : select the correct vector def of the load-lanes load which
11649 : has the permuted vectors in its vector defs like
11650 : { v0, w0, r0, v1, w1, r1 ... } for a ld3. All costs are
11651 : accounted for in the costing for the actual load so we
11652 : return zero here. */
11653 449560 : if (node->ldst_lanes)
11654 : {
11655 0 : gcc_assert (children.length () == 1);
11656 0 : if (!gsi)
11657 : /* This is a trivial op always supported. */
11658 : return 0;
11659 0 : slp_tree child = children[0];
11660 0 : unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
11661 0 : / SLP_TREE_LANES (node));
11662 0 : unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
11663 0 : unsigned nvectors = vect_get_num_copies (vinfo, node);
11664 0 : for (unsigned i = 0; i < nvectors; ++i)
11665 : {
11666 0 : tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num + vec_idx];
11667 0 : node->push_vec_def (def);
11668 : }
11669 : return 0;
11670 : }
11671 :
11672 : /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
11673 : and if we can generate the vectors in a vector-length agnostic way.
11674 : This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
11675 : compile time.
11676 :
11677 : The significance of UNPACK_STEP is that, when PACK_P is false,
11678 : output vector I operates on a window of UNPACK_STEP elements from each
11679 : input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR). For example,
11680 : when UNPACK_FACTOR is 2, the first output vector operates on lanes
11681 : [0, NUNITS / 2 - 1] of each input vector and the second output vector
11682 : operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
11683 :
11684 : When REPEATING_P is true, NOUTPUTS holds the total number of outputs
11685 : that we actually need to generate. */
11686 449560 : uint64_t noutputs = 0;
11687 449560 : poly_uint64 unpack_step = 0;
11688 449560 : loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
11689 182839 : if (!linfo
11690 488542 : || !multiple_p (nunits, unpack_factor, &unpack_step)
11691 181896 : || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
11692 181896 : * SLP_TREE_LANES (node), nunits, &noutputs))
11693 : repeating_p = false;
11694 :
11695 : /* We can handle the conditions described for REPEATING_P above for
11696 : both variable- and constant-length vectors. The fallback requires
11697 : us to generate every element of every permute vector explicitly,
11698 : which is only possible for constant-length permute vectors.
11699 :
11700 : Set:
11701 :
11702 : - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
11703 : mask vectors that we want to build.
11704 :
11705 : - NCOPIES to the number of copies of PERM that we need in order
11706 : to build the necessary permute mask vectors. */
11707 181896 : uint64_t npatterns;
11708 181896 : unsigned nelts_per_pattern;
11709 181896 : uint64_t ncopies;
11710 181896 : if (repeating_p)
11711 : {
11712 : /* We need permute mask vectors that have the form:
11713 :
11714 : { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
11715 :
11716 : In other words, the original n-element permute in PERM is
11717 : "unrolled" to fill a full vector. The stepped vector encoding
11718 : that we use for permutes requires 3n elements. */
11719 142914 : npatterns = SLP_TREE_LANES (node);
11720 142914 : nelts_per_pattern = ncopies = 3;
11721 : }
11722 : else
11723 : {
11724 : /* Calculate every element of every permute mask vector explicitly,
11725 : instead of relying on the pattern described above. */
11726 306646 : if (!nunits.is_constant (&npatterns)
11727 306646 : || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
11728 : {
11729 : if (dump_p)
11730 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11731 : "unsupported permutation %p on variable-length"
11732 : " vectors\n", (void *) node);
11733 : return -1;
11734 : }
11735 306646 : nelts_per_pattern = ncopies = 1;
11736 306646 : if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
11737 : {
11738 : if (dump_p)
11739 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11740 : "unsupported permutation %p for variable VF\n",
11741 : (void *) node);
11742 : return -1;
11743 : }
11744 : pack_p = false;
11745 : unpack_factor = 1;
11746 : }
11747 449560 : unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
11748 449560 : gcc_assert (repeating_p || multiple_p (olanes, nunits));
11749 :
11750 : /* Compute the { { SLP operand, vector index}, lane } permutation sequence
11751 : from the { SLP operand, scalar lane } permutation as recorded in the
11752 : SLP node as intermediate step. This part should already work
11753 : with SLP children with arbitrary number of lanes. */
11754 449560 : auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
11755 449560 : auto_vec<poly_uint64> active_lane;
11756 449560 : vperm.create (olanes);
11757 449560 : active_lane.safe_grow_cleared (children.length (), true);
11758 1356968 : for (unsigned int ui = 0; ui < unpack_factor; ++ui)
11759 : {
11760 1938596 : for (unsigned j = 0; j < children.length (); ++j)
11761 511450 : active_lane[j] = ui * unpack_step;
11762 1315902 : for (unsigned i = 0; i < ncopies; ++i)
11763 : {
11764 5411944 : for (unsigned pi = 0; pi < perm.length (); ++pi)
11765 : {
11766 1847918 : std::pair<unsigned, unsigned> p = perm[pi];
11767 1847918 : tree vtype = SLP_TREE_VECTYPE (children[p.first]);
11768 1847918 : if (repeating_p)
11769 834096 : vperm.quick_push ({{p.first, 0},
11770 834096 : p.second + active_lane[p.first]});
11771 : else
11772 : {
11773 : /* We checked above that the vectors are constant-length. */
11774 1013822 : unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
11775 1013822 : .to_constant ();
11776 1013822 : unsigned lane = active_lane[p.first].to_constant ();
11777 1013822 : unsigned vi = (lane + p.second) / vnunits;
11778 1013822 : unsigned vl = (lane + p.second) % vnunits;
11779 1013822 : vperm.quick_push ({{p.first, vi}, vl});
11780 : }
11781 : }
11782 : /* Advance to the next group. */
11783 1834161 : for (unsigned j = 0; j < children.length (); ++j)
11784 976107 : active_lane[j] += SLP_TREE_LANES (children[j]);
11785 : }
11786 : }
11787 :
11788 449560 : if (dump_p)
11789 : {
11790 9003 : dump_printf_loc (MSG_NOTE, vect_location,
11791 : "vectorizing permutation %p", (void *)node);
11792 32594 : for (unsigned i = 0; i < perm.length (); ++i)
11793 23591 : dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
11794 9003 : if (repeating_p)
11795 7566 : dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
11796 9003 : dump_printf (MSG_NOTE, "\n");
11797 9003 : dump_printf_loc (MSG_NOTE, vect_location, "as");
11798 90582 : for (unsigned i = 0; i < vperm.length (); ++i)
11799 : {
11800 81579 : if (i != 0
11801 81579 : && (repeating_p
11802 55311 : ? multiple_p (i, npatterns)
11803 81579 : : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
11804 24361 : dump_printf (MSG_NOTE, ",");
11805 81579 : dump_printf (MSG_NOTE, " vops%u[%u][",
11806 81579 : vperm[i].first.first, vperm[i].first.second);
11807 81579 : dump_dec (MSG_NOTE, vperm[i].second);
11808 81579 : dump_printf (MSG_NOTE, "]");
11809 : }
11810 9003 : dump_printf (MSG_NOTE, "\n");
11811 : }
11812 :
11813 : /* We can only handle two-vector permutes, everything else should
11814 : be lowered on the SLP level. The following is closely inspired
11815 : by vect_transform_slp_perm_load and is supposed to eventually
11816 : replace it.
11817 : ??? As intermediate step do code-gen in the SLP tree representation
11818 : somehow? */
11819 449560 : std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
11820 449560 : std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
11821 449560 : unsigned int index = 0;
11822 449560 : poly_uint64 mask_element;
11823 449560 : vec_perm_builder mask;
11824 449560 : mask.new_vector (nunits, npatterns, nelts_per_pattern);
11825 449560 : unsigned int count = mask.encoded_nelts ();
11826 449560 : mask.quick_grow (count);
11827 449560 : vec_perm_indices indices;
11828 449560 : unsigned nperms = 0;
11829 : /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
11830 : vectors to check during analysis, but we need to generate NOUTPUTS
11831 : vectors during transformation. */
11832 449560 : unsigned total_nelts = olanes;
11833 449560 : unsigned process_nelts = olanes;
11834 449560 : if (repeating_p)
11835 : {
11836 142914 : total_nelts = (total_nelts / unpack_factor) * noutputs;
11837 142914 : if (gsi)
11838 9799 : process_nelts = total_nelts;
11839 : }
11840 449560 : unsigned last_ei = (total_nelts - 1) % process_nelts;
11841 2299940 : for (unsigned i = 0; i < process_nelts; ++i)
11842 : {
11843 : /* VI is the input vector index when generating code for REPEATING_P. */
11844 1861089 : unsigned vi = i / olanes * (pack_p ? 2 : 1);
11845 1861089 : unsigned ei = i % olanes;
11846 1861089 : mask_element = vperm[ei].second;
11847 1861089 : if (pack_p)
11848 : {
11849 : /* In this case, we have N outputs and the single child provides 2N
11850 : inputs. Output X permutes inputs 2X and 2X+1.
11851 :
11852 : The mask indices are taken directly from the SLP permutation node.
11853 : Index X selects from the first vector if (X / NUNITS) % 2 == 0;
11854 : X selects from the second vector otherwise. These conditions
11855 : are only known at compile time for constant-length vectors. */
11856 : first_vec = std::make_pair (0, 0);
11857 : second_vec = std::make_pair (0, 1);
11858 : }
11859 1691934 : else if (first_vec.first == -1U
11860 1691934 : || first_vec == vperm[ei].first)
11861 1496113 : first_vec = vperm[ei].first;
11862 195821 : else if (second_vec.first == -1U
11863 195821 : || second_vec == vperm[ei].first)
11864 : {
11865 195417 : second_vec = vperm[ei].first;
11866 195417 : mask_element += nunits;
11867 : }
11868 : else
11869 : {
11870 404 : if (dump_p)
11871 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11872 : "permutation requires at "
11873 : "least three vectors\n");
11874 404 : gcc_assert (!gsi);
11875 : return -1;
11876 : }
11877 :
11878 1860685 : mask[index++] = mask_element;
11879 :
11880 1860685 : if (index == count)
11881 : {
11882 593193 : indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
11883 : TYPE_VECTOR_SUBPARTS (op_vectype));
11884 593193 : bool identity_p = (indices.series_p (0, 1, mask[0], 1)
11885 923896 : && constant_multiple_p (mask[0], nunits));
11886 593193 : machine_mode vmode = TYPE_MODE (vectype);
11887 593193 : machine_mode op_vmode = TYPE_MODE (op_vectype);
11888 593193 : unsigned HOST_WIDE_INT c;
11889 593193 : if ((!identity_p
11890 544747 : && !can_vec_perm_const_p (vmode, op_vmode, indices))
11891 593193 : || (identity_p
11892 48446 : && !known_le (nunits,
11893 : TYPE_VECTOR_SUBPARTS (op_vectype))
11894 8 : && (!constant_multiple_p (nunits,
11895 10313 : TYPE_VECTOR_SUBPARTS (op_vectype),
11896 8 : &c) || c != 2)))
11897 : {
11898 10305 : if (dump_p)
11899 : {
11900 154 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
11901 : vect_location,
11902 : "unsupported vect permute { ");
11903 1596 : for (i = 0; i < count; ++i)
11904 : {
11905 1442 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11906 1442 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11907 : }
11908 154 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11909 : }
11910 10305 : gcc_assert (!gsi);
11911 449560 : return -1;
11912 : }
11913 :
11914 582888 : if (!identity_p)
11915 534442 : nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
11916 582888 : if (gsi)
11917 : {
11918 31651 : if (second_vec.first == -1U)
11919 7391 : second_vec = first_vec;
11920 :
11921 31651 : slp_tree
11922 31651 : first_node = children[first_vec.first],
11923 31651 : second_node = children[second_vec.first];
11924 :
11925 31651 : tree mask_vec = NULL_TREE;
11926 31651 : if (!identity_p)
11927 28129 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11928 :
11929 31651 : tree first_def
11930 31651 : = vect_get_slp_vect_def (first_node, first_vec.second + vi);
11931 31651 : tree second_def
11932 31651 : = vect_get_slp_vect_def (second_node, second_vec.second + vi);
11933 31651 : vect_add_slp_permutation (vinfo, gsi, node, first_def,
11934 31651 : second_def, mask_vec, mask[0]);
11935 : }
11936 :
11937 : index = 0;
11938 : first_vec = std::make_pair (-1U, -1U);
11939 : second_vec = std::make_pair (-1U, -1U);
11940 : }
11941 : }
11942 :
11943 438851 : return nperms;
11944 449560 : }
11945 :
11946 : /* Vectorize the SLP permutations in NODE as specified
11947 : in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
11948 : child number and lane number.
11949 : Interleaving of two two-lane two-child SLP subtrees (not supported):
11950 : [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
11951 : A blend of two four-lane two-child SLP subtrees:
11952 : [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
11953 : Highpart of a four-lane one-child SLP subtree (not supported):
11954 : [ { 0, 2 }, { 0, 3 } ]
11955 : Where currently only a subset is supported by code generating below. */
11956 :
11957 : bool
11958 99688 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11959 : slp_tree node, stmt_vector_for_cost *cost_vec)
11960 : {
11961 99688 : tree vectype = SLP_TREE_VECTYPE (node);
11962 99688 : lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
11963 99688 : int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
11964 99688 : SLP_TREE_CHILDREN (node),
11965 : dump_enabled_p ());
11966 99688 : if (nperms < 0)
11967 : return false;
11968 :
11969 98059 : if (!gsi && nperms != 0)
11970 74424 : record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
11971 :
11972 : return true;
11973 : }
11974 :
11975 : /* Vectorize SLP NODE. Only compute the vector insertion places when
11976 : PLACE_ONLY is true. When placing, return false if there is no possible
11977 : schedule. */
11978 :
11979 : static bool
11980 3142029 : vect_schedule_slp_node (vec_info *vinfo,
11981 : slp_tree node, slp_instance instance, bool place_only)
11982 : {
11983 3142029 : int i;
11984 3142029 : slp_tree child;
11985 :
11986 : /* Vectorize externals and constants. */
11987 3142029 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
11988 3142029 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
11989 : {
11990 1209796 : if (place_only)
11991 : {
11992 700218 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
11993 : return true;
11994 319773 : gimple *last_stmt = NULL;
11995 319773 : vec<tree> &defs = (!SLP_TREE_SCALAR_OPS (node).is_empty ()
11996 : ? SLP_TREE_SCALAR_OPS (node)
11997 319773 : : SLP_TREE_VEC_DEFS (node));
11998 1712044 : for (tree def : defs)
11999 : /* If the stmt is not inside the region do not
12000 : use it as possible insertion point. */
12001 752725 : if (auto stmt_info = vinfo->lookup_def (def))
12002 : {
12003 390885 : gimple *stmt = stmt_info->stmt;
12004 390885 : if (!last_stmt)
12005 : last_stmt = stmt;
12006 220342 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12007 : last_stmt = stmt;
12008 67523 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12009 : ;
12010 : else
12011 0 : gcc_unreachable ();
12012 : }
12013 319773 : node->si = last_stmt;
12014 319773 : return true;
12015 : }
12016 :
12017 : /* ??? vectorizable_shift can end up using a scalar operand which is
12018 : currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
12019 : node in this case. */
12020 509578 : if (!SLP_TREE_VECTYPE (node))
12021 : return true;
12022 :
12023 : /* There are two reasons vector defs might already exist. The first
12024 : is that we are vectorizing an existing vector def. The second is
12025 : when performing BB vectorization shared constant/external nodes
12026 : are not split apart during partitioning so during the code-gen
12027 : DFS walk we can end up visiting them twice. */
12028 502344 : if (! SLP_TREE_VEC_DEFS (node).exists ())
12029 501404 : vect_create_constant_vectors (vinfo, node);
12030 : return true;
12031 : }
12032 :
12033 1932233 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
12034 :
12035 1932233 : gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
12036 1932233 : if (!place_only && SLP_TREE_VECTYPE (node))
12037 992277 : SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
12038 :
12039 1932233 : gimple *last_stmt;
12040 1932233 : gimple_stmt_iterator si;
12041 : /* ??? When !place_only we'd like to re-use place_only computed info,
12042 : but this is a bit awkward due to using gsi_insert_before and the
12043 : requirement to insert after vector defs. So we compute last_stmt
12044 : during pre-scheduling and si during scheduling. */
12045 1932233 : if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
12046 : {
12047 : /* Vectorized loads go before the first scalar load to make it
12048 : ready early, vectorized stores go before the last scalar
12049 : stmt which is where all uses are ready.
12050 : In theory, if we delay dependence checking until after
12051 : placing, we can schedule at other points, but then
12052 : dependence checking would need to honor that. On the
12053 : other hand dependence checking could request a different
12054 : scheduling point as well, if dependences require that. */
12055 1504081 : stmt_vec_info last_stmt_info = NULL;
12056 1504081 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
12057 282096 : last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
12058 : else /* DR_IS_WRITE */
12059 : {
12060 1221985 : last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
12061 1221985 : if (place_only)
12062 1331976 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12063 : {
12064 665989 : if (child->si
12065 665989 : && !vect_stmt_dominates_stmt_p (child->si,
12066 : last_stmt_info->stmt))
12067 : return false;
12068 : }
12069 : }
12070 1504079 : last_stmt = last_stmt_info->stmt;
12071 1504079 : si = gsi_for_stmt (last_stmt);
12072 1504079 : }
12073 428152 : else if (!SLP_TREE_PERMUTE_P (node)
12074 406606 : && (SLP_TREE_TYPE (node) == cycle_phi_info_type
12075 : || SLP_TREE_TYPE (node) == induc_vec_info_type
12076 : || SLP_TREE_TYPE (node) == phi_info_type))
12077 : {
12078 : /* For PHI node vectorization we do not use the insertion iterator. */
12079 108163 : last_stmt = SLP_TREE_SCALAR_STMTS (node)[0]->stmt;
12080 108163 : if (place_only)
12081 182197 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12082 : {
12083 128900 : if (child->si
12084 224726 : && !dominated_by_p (CDI_DOMINATORS,
12085 : gimple_phi_arg_edge
12086 95826 : (as_a <gphi *> (last_stmt), i)->src,
12087 95826 : gimple_bb (child->si)))
12088 : return false;
12089 : }
12090 108161 : si = gsi_none ();
12091 : }
12092 : else
12093 : {
12094 : /* Emit other stmts after the children vectorized defs which is
12095 : earliest possible. */
12096 : last_stmt = NULL;
12097 885955 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12098 565966 : if (place_only)
12099 : {
12100 189369 : gimple *vstmt = child->si;
12101 189369 : if (!vstmt)
12102 : {
12103 : /* vect_constant_def and defs at region boundary do not
12104 : constrain placement. */
12105 : gcc_assert (SLP_TREE_DEF_TYPE (child) == vect_constant_def
12106 : /* ??? Region boundary is not representated
12107 : by a NULL stmt. */
12108 : || true);
12109 : }
12110 148215 : else if (!last_stmt)
12111 : last_stmt = vstmt;
12112 42996 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12113 : last_stmt = vstmt;
12114 10950 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12115 : ;
12116 : else
12117 : /* Non-trapping stmts from different BBs might be combined,
12118 : and if we later CSE a low/high part we can run into this. */
12119 : return false;
12120 : }
12121 376597 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
12122 : {
12123 : /* For fold-left reductions we are retaining the scalar
12124 : reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
12125 : set so the representation isn't perfect. Resort to the
12126 : last scalar def here. */
12127 300654 : if (SLP_TREE_VEC_DEFS (child).is_empty ())
12128 : {
12129 942 : gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
12130 942 : gphi *phi = as_a <gphi *>
12131 942 : (vect_find_last_scalar_stmt_in_slp (child)->stmt);
12132 942 : if (!last_stmt)
12133 : last_stmt = phi;
12134 722 : else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
12135 : last_stmt = phi;
12136 711 : else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
12137 : ;
12138 : else
12139 0 : gcc_unreachable ();
12140 : }
12141 : /* We are emitting all vectorized stmts in the same place and
12142 : the last one is the last.
12143 : ??? Unless we have a load permutation applied and that
12144 : figures to re-use an earlier generated load. */
12145 300654 : unsigned j;
12146 300654 : tree vdef;
12147 709663 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12148 409009 : if (TREE_CODE (vdef) == SSA_NAME
12149 409009 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12150 : {
12151 408957 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12152 408957 : if (!last_stmt)
12153 : last_stmt = vstmt;
12154 209127 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12155 : last_stmt = vstmt;
12156 46719 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12157 : ;
12158 : else
12159 0 : gcc_unreachable ();
12160 : }
12161 : }
12162 75943 : else if (!SLP_TREE_VECTYPE (child))
12163 : {
12164 : /* For externals we use unvectorized at all scalar defs. */
12165 : unsigned j;
12166 : tree def;
12167 15857 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
12168 : /* If the stmt is not inside the region do not
12169 : use it as possible insertion point. */
12170 9251 : if (auto stmt_info = vinfo->lookup_def (def))
12171 : {
12172 250 : gimple *stmt = stmt_info->stmt;
12173 250 : if (!last_stmt)
12174 : last_stmt = stmt;
12175 230 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12176 : last_stmt = stmt;
12177 26 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12178 : ;
12179 : else
12180 0 : gcc_unreachable ();
12181 : }
12182 : }
12183 : else
12184 : {
12185 : /* For externals we have to look at all defs since their
12186 : insertion place is decided per vector. But beware
12187 : of pre-existing vectors where we need to make sure
12188 : we do not insert before the region boundary. */
12189 138543 : if (SLP_TREE_SCALAR_OPS (child).is_empty ()
12190 708 : && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
12191 : ;
12192 : else
12193 : {
12194 : unsigned j;
12195 : tree vdef;
12196 732534 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12197 97362 : if (TREE_CODE (vdef) == SSA_NAME
12198 97362 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12199 : {
12200 21452 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12201 21452 : if (!last_stmt)
12202 : last_stmt = vstmt;
12203 11652 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12204 : last_stmt = vstmt;
12205 9089 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12206 : ;
12207 : else
12208 0 : gcc_unreachable ();
12209 : }
12210 : }
12211 : }
12212 :
12213 : /* We split regions to vectorize at control altering stmts
12214 : with a definition so this can only be an external. */
12215 319989 : gcc_checking_assert (!last_stmt
12216 : || !is_ctrl_altering_stmt (last_stmt));
12217 :
12218 319989 : if (is_a <bb_vec_info> (vinfo)
12219 129748 : && !SLP_TREE_PERMUTE_P (node)
12220 123171 : && (!last_stmt
12221 120155 : || gimple_bb (last_stmt) != gimple_bb (stmt_info->stmt))
12222 353887 : && gimple_could_trap_p (stmt_info->stmt))
12223 : {
12224 : /* We've constrained possibly trapping operations to all come
12225 : from the same basic-block, if vectorized defs would allow earlier
12226 : scheduling still force vectorized stmts to the original block.
12227 : This is only necessary for BB vectorization since for loop vect
12228 : all operations are in a single BB and scalar stmt based
12229 : placement doesn't play well with epilogue vectorization. */
12230 659 : gcc_assert (!last_stmt
12231 : || dominated_by_p (CDI_DOMINATORS,
12232 : gimple_bb (stmt_info->stmt),
12233 : gimple_bb (last_stmt)));
12234 659 : si = gsi_after_labels (gimple_bb (stmt_info->stmt));
12235 659 : last_stmt = gsi_stmt (si);
12236 : }
12237 : /* When there is no in-region child def to guide placement, insert
12238 : at region boundary. */
12239 319330 : else if (!last_stmt)
12240 : {
12241 4806 : si = gsi_after_labels (vinfo->bbs[0]);
12242 : /* last_stmt NULL marks the region start. */
12243 : }
12244 314524 : else if (is_a <gphi *> (last_stmt))
12245 31189 : si = gsi_after_labels (gimple_bb (last_stmt));
12246 : else
12247 : {
12248 283335 : si = gsi_for_stmt (last_stmt);
12249 : /* We use gsi_insert_before, so when last_stmt is a vector
12250 : def we have to advance (or use gsi_insert_after). */
12251 283335 : gsi_next (&si);
12252 :
12253 283335 : if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
12254 : {
12255 : /* We'll have to fix this up for loop vect. */
12256 174336 : gcc_assert (!place_only);
12257 : /* Avoid scheduling stmts to random places in the CFG, any
12258 : stmt dominance check we performed is possibly wrong as UIDs
12259 : are not initialized for all of the function for loop
12260 : vectorization. Instead append to the loop preheader. */
12261 174336 : if ((LOOP_VINFO_LOOP (loop_vinfo)->header
12262 174336 : != gimple_bb (last_stmt))
12263 177498 : && dominated_by_p (CDI_DOMINATORS,
12264 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12265 3162 : gimple_bb (last_stmt)))
12266 1278 : si = gsi_end_bb (loop_preheader_edge
12267 639 : (LOOP_VINFO_LOOP (loop_vinfo))->src);
12268 : /* Avoid scheduling internal defs outside of the loop when
12269 : we might have only implicitly tracked loop mask/len defs. */
12270 76 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
12271 174336 : || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12272 : {
12273 76 : gimple_stmt_iterator si2
12274 76 : = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
12275 76 : if ((gsi_end_p (si2)
12276 0 : && (LOOP_VINFO_LOOP (loop_vinfo)->header
12277 0 : != gimple_bb (last_stmt))
12278 0 : && dominated_by_p (CDI_DOMINATORS,
12279 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12280 0 : gimple_bb (last_stmt)))
12281 76 : || (!gsi_end_p (si2)
12282 76 : && last_stmt != *si2
12283 73 : && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
12284 3 : si = si2;
12285 : }
12286 : }
12287 : }
12288 : }
12289 :
12290 1932229 : if (place_only)
12291 : {
12292 939946 : if (dump_enabled_p () && last_stmt)
12293 3713 : dump_printf_loc (MSG_NOTE, vect_location,
12294 : "placing node %p at %G:", (void *)node, last_stmt);
12295 : /* Verify we either get a stmt anchor or region start. */
12296 939946 : gcc_assert ((last_stmt && gimple_bb (last_stmt))
12297 : || (!last_stmt && gsi_bb (si)));
12298 939946 : node->si = last_stmt;
12299 939946 : return true;
12300 : }
12301 :
12302 : /* ??? Asserting vect_stmt_dominates_stmt_p (gsi_stmt (si), node->si)
12303 : does not work because in some cases we advance si from last_stmt (as
12304 : we want to insert after vector stmts) and because vector stmts of
12305 : children have been inserted possibly at the same location constraint,
12306 : moving si even further. */
12307 992283 : if (flag_checking && node->si && gimple_bb (node->si) && !gsi_end_p (si))
12308 : {
12309 626859 : auto gsi2 = si;
12310 626859 : while (1)
12311 : {
12312 626859 : if (vect_stmt_dominates_stmt_p (gsi_stmt (gsi2), node->si))
12313 : break;
12314 : /* As we have possibly advanced si it might now point to the
12315 : scalar stmt immediately following node->si. That's OK. */
12316 25168 : if (gsi_stmt (gsi2) != gsi_stmt (si)
12317 25168 : && gimple_uid (gsi_stmt (gsi2)) != 0)
12318 0 : gcc_unreachable ();
12319 25168 : gsi_prev (&gsi2);
12320 25168 : if (gsi_end_p (gsi2))
12321 : {
12322 540 : if (is_a <gphi *> (node->si)
12323 540 : && gimple_bb (node->si) == gsi_bb (si))
12324 : break;
12325 0 : gcc_unreachable ();
12326 : }
12327 : }
12328 : }
12329 :
12330 992283 : if (dump_enabled_p ())
12331 : {
12332 72022 : if (stmt_info)
12333 68611 : dump_printf_loc (MSG_NOTE, vect_location,
12334 : "------>vectorizing SLP node starting from: %G",
12335 : stmt_info->stmt);
12336 : else
12337 : {
12338 3411 : dump_printf_loc (MSG_NOTE, vect_location,
12339 : "------>vectorizing SLP node:\n");
12340 3411 : vect_print_slp_tree (MSG_NOTE, vect_location, node);
12341 : }
12342 : }
12343 992283 : vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
12344 992283 : return true;
12345 : }
12346 :
12347 : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
12348 : For loop vectorization this is done in vectorizable_call, but for SLP
12349 : it needs to be deferred until end of vect_schedule_slp, because multiple
12350 : SLP instances may refer to the same scalar stmt. */
12351 :
12352 : static void
12353 604450 : vect_remove_slp_scalar_calls (vec_info *vinfo,
12354 : slp_tree node, hash_set<slp_tree> &visited)
12355 : {
12356 604450 : gimple *new_stmt;
12357 604450 : gimple_stmt_iterator gsi;
12358 604450 : tree lhs;
12359 :
12360 604450 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12361 189138 : return;
12362 :
12363 458993 : if (visited.add (node))
12364 : return;
12365 :
12366 1547727 : for (auto child : SLP_TREE_CHILDREN (node))
12367 513979 : vect_remove_slp_scalar_calls (vinfo, child, visited);
12368 :
12369 1725755 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
12370 : {
12371 488097 : if (!stmt_info)
12372 3962 : continue;
12373 484135 : stmt_info = vect_orig_stmt (stmt_info);
12374 484135 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
12375 5144 : if (!stmt || gimple_bb (stmt) == NULL)
12376 479033 : continue;
12377 5102 : lhs = gimple_call_lhs (stmt);
12378 5102 : if (lhs)
12379 4531 : new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
12380 : else
12381 571 : new_stmt = gimple_build_nop ();
12382 5102 : unlink_stmt_vdef (stmt_info->stmt);
12383 5102 : gsi = gsi_for_stmt (stmt);
12384 5102 : vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
12385 5102 : if (lhs)
12386 4531 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12387 : }
12388 : }
12389 :
12390 : static void
12391 90471 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
12392 : {
12393 90471 : hash_set<slp_tree> visited;
12394 90471 : vect_remove_slp_scalar_calls (vinfo, node, visited);
12395 90471 : }
12396 :
12397 : /* Vectorize the instance root. */
12398 :
12399 : void
12400 13846 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
12401 : {
12402 13846 : if (instance->kind == slp_inst_kind_ctor)
12403 : {
12404 5637 : tree new_def;
12405 5637 : if (SLP_TREE_VEC_DEFS (node).length () == 1)
12406 : {
12407 5592 : new_def = SLP_TREE_VEC_DEFS (node)[0];
12408 5592 : tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12409 5592 : if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
12410 5592 : TREE_TYPE (new_def)))
12411 0 : new_def = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
12412 : new_def);
12413 : }
12414 : else
12415 : {
12416 45 : gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
12417 45 : tree child_def;
12418 45 : int j;
12419 45 : vec<constructor_elt, va_gc> *v;
12420 45 : vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
12421 :
12422 : /* A CTOR can handle V16HI composition from VNx8HI so we
12423 : do not need to convert vector elements if the types
12424 : do not match. */
12425 180 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
12426 90 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
12427 45 : tree rtype
12428 45 : = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
12429 45 : new_def = build_constructor (rtype, v);
12430 : }
12431 :
12432 5637 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12433 5637 : gimple_assign_set_rhs_from_tree (&rgsi, new_def);
12434 5637 : update_stmt (gsi_stmt (rgsi));
12435 5637 : return;
12436 : }
12437 8209 : else if (instance->kind == slp_inst_kind_bb_reduc)
12438 : {
12439 : /* Largely inspired by reduction chain epilogue handling in
12440 : vect_create_epilog_for_reduction. */
12441 6590 : vec<tree> vec_defs = vNULL;
12442 6590 : vect_get_slp_defs (node, &vec_defs);
12443 6590 : enum tree_code reduc_code
12444 6590 : = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
12445 : /* ??? We actually have to reflect signs somewhere. */
12446 6590 : if (reduc_code == MINUS_EXPR)
12447 0 : reduc_code = PLUS_EXPR;
12448 6590 : gimple_seq epilogue = NULL;
12449 : /* We may end up with more than one vector result, reduce them
12450 : to one vector. */
12451 6590 : tree vec_def = vec_defs[0];
12452 6590 : tree vectype = TREE_TYPE (vec_def);
12453 6590 : tree compute_vectype = vectype;
12454 6590 : bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
12455 5449 : && TYPE_OVERFLOW_UNDEFINED (vectype)
12456 10224 : && operation_can_overflow (reduc_code));
12457 3079 : if (pun_for_overflow_p)
12458 : {
12459 3079 : compute_vectype = unsigned_type_for (vectype);
12460 3079 : vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12461 : compute_vectype, vec_def);
12462 : }
12463 9089 : for (unsigned i = 1; i < vec_defs.length (); ++i)
12464 : {
12465 2499 : tree def = vec_defs[i];
12466 2499 : if (pun_for_overflow_p)
12467 2291 : def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12468 : compute_vectype, def);
12469 2499 : vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
12470 : vec_def, def);
12471 : }
12472 6590 : vec_defs.release ();
12473 : /* ??? Support other schemes than direct internal fn or two
12474 : element vectors. */
12475 6590 : tree scalar_def;
12476 6590 : internal_fn reduc_fn;
12477 8239 : if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
12478 6590 : || reduc_fn == IFN_LAST
12479 13180 : || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
12480 : OPTIMIZE_FOR_BOTH))
12481 : {
12482 1649 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
12483 1649 : tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
12484 1649 : TREE_TYPE (compute_vectype), vec_def,
12485 1649 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12486 1649 : bitsize_zero_node);
12487 1649 : tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
12488 1649 : TREE_TYPE (compute_vectype), vec_def,
12489 1649 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12490 1649 : TYPE_SIZE (TREE_TYPE (compute_vectype)));
12491 1649 : scalar_def = gimple_build (&epilogue, reduc_code,
12492 1649 : TREE_TYPE (compute_vectype), tem0, tem1);
12493 : }
12494 : else
12495 4941 : scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
12496 4941 : TREE_TYPE (compute_vectype), vec_def);
12497 6590 : if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
12498 : {
12499 3522 : tree rem_def = NULL_TREE;
12500 14510 : for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
12501 : {
12502 10988 : def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
12503 10988 : if (!rem_def)
12504 : rem_def = def;
12505 : else
12506 7466 : rem_def = gimple_build (&epilogue, reduc_code,
12507 7466 : TREE_TYPE (scalar_def),
12508 : rem_def, def);
12509 : }
12510 3522 : scalar_def = gimple_build (&epilogue, reduc_code,
12511 3522 : TREE_TYPE (scalar_def),
12512 : scalar_def, rem_def);
12513 : }
12514 6590 : scalar_def = gimple_convert (&epilogue,
12515 6590 : TREE_TYPE (vectype), scalar_def);
12516 6590 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12517 6590 : gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
12518 6590 : gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
12519 6590 : update_stmt (gsi_stmt (rgsi));
12520 6590 : return;
12521 : }
12522 1619 : else if (instance->kind == slp_inst_kind_gcond)
12523 : {
12524 : /* Only support a single root for now as we can't codegen CFG yet and so we
12525 : can't support lane > 1 at this time. */
12526 1619 : gcc_assert (instance->root_stmts.length () == 1);
12527 1619 : auto root_stmt_info = instance->root_stmts[0];
12528 1619 : auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
12529 1619 : gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
12530 1619 : gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
12531 1619 : bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
12532 : root_stmt_info, &rgsi, node, NULL);
12533 1619 : gcc_assert (res);
12534 1619 : return;
12535 : }
12536 : else
12537 0 : gcc_unreachable ();
12538 : }
12539 :
12540 : struct slp_scc_info
12541 : {
12542 : bool on_stack;
12543 : int dfs;
12544 : int lowlink;
12545 : };
12546 :
12547 : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs.
12548 : When PLACE_ONLY, return false if there is no possible schedule. */
12549 :
12550 : static bool
12551 3142029 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
12552 : hash_map<slp_tree, slp_scc_info> &scc_info,
12553 : int &maxdfs, vec<slp_tree> &stack, bool place_only)
12554 : {
12555 3142029 : bool existed_p;
12556 3142029 : slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
12557 3142029 : gcc_assert (!existed_p);
12558 3142029 : info->dfs = maxdfs;
12559 3142029 : info->lowlink = maxdfs;
12560 3142029 : maxdfs++;
12561 :
12562 : /* Leaf. */
12563 3142029 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12564 : {
12565 1209796 : info->on_stack = false;
12566 1209796 : bool res = vect_schedule_slp_node (vinfo, node, instance, place_only);
12567 1209796 : gcc_assert (res);
12568 : return true;
12569 : }
12570 :
12571 1932233 : info->on_stack = true;
12572 1932233 : stack.safe_push (node);
12573 :
12574 1932233 : bool res = true;
12575 1932233 : unsigned i;
12576 1932233 : slp_tree child;
12577 : /* DFS recurse. */
12578 7552602 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12579 : {
12580 2038328 : if (!child)
12581 55566 : continue;
12582 1982762 : slp_scc_info *child_info = scc_info.get (child);
12583 1982762 : if (!child_info)
12584 : {
12585 1833627 : res &= vect_schedule_scc (vinfo, child, instance, scc_info,
12586 : maxdfs, stack, place_only);
12587 : /* Recursion might have re-allocated the node. */
12588 1833627 : info = scc_info.get (node);
12589 1833627 : child_info = scc_info.get (child);
12590 1833627 : info->lowlink = MIN (info->lowlink, child_info->lowlink);
12591 : }
12592 149135 : else if (child_info->on_stack)
12593 35568 : info->lowlink = MIN (info->lowlink, child_info->dfs);
12594 : }
12595 1932233 : if (info->lowlink != info->dfs)
12596 : return res;
12597 :
12598 1892728 : auto_vec<slp_tree, 4> phis_to_fixup;
12599 :
12600 : /* Singleton. */
12601 1892728 : if (stack.last () == node)
12602 : {
12603 1863684 : stack.pop ();
12604 1863684 : info->on_stack = false;
12605 1863684 : res &= vect_schedule_slp_node (vinfo, node, instance, place_only);
12606 1863684 : if (!SLP_TREE_PERMUTE_P (node)
12607 1863684 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
12608 73340 : phis_to_fixup.quick_push (node);
12609 : }
12610 : else
12611 : {
12612 : /* SCC. */
12613 29044 : int last_idx = stack.length () - 1;
12614 68549 : while (stack[last_idx] != node)
12615 39505 : last_idx--;
12616 : /* We can break the cycle at PHIs who have at least one child
12617 : code generated. Then we could re-start the DFS walk until
12618 : all nodes in the SCC are covered (we might have new entries
12619 : for only back-reachable nodes). But it's simpler to just
12620 : iterate and schedule those that are ready. */
12621 29044 : unsigned todo = stack.length () - last_idx;
12622 29912 : do
12623 : {
12624 132769 : for (int idx = stack.length () - 1; idx >= last_idx; --idx)
12625 : {
12626 72945 : slp_tree entry = stack[idx];
12627 72945 : if (!entry)
12628 2598 : continue;
12629 70347 : bool phi = (!SLP_TREE_PERMUTE_P (entry)
12630 70347 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
12631 70347 : bool ready = !phi;
12632 214003 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
12633 133367 : if (!child)
12634 : {
12635 23027 : gcc_assert (phi);
12636 : ready = true;
12637 : break;
12638 : }
12639 110340 : else if (scc_info.get (child)->on_stack)
12640 : {
12641 30978 : if (!phi)
12642 : {
12643 : ready = false;
12644 : break;
12645 : }
12646 : }
12647 : else
12648 : {
12649 79362 : if (phi)
12650 : {
12651 : ready = true;
12652 : break;
12653 : }
12654 : }
12655 47320 : if (ready)
12656 : {
12657 68549 : vect_schedule_slp_node (vinfo, entry, instance, place_only);
12658 68549 : scc_info.get (entry)->on_stack = false;
12659 68549 : stack[idx] = NULL;
12660 68549 : todo--;
12661 68549 : if (phi)
12662 35397 : phis_to_fixup.safe_push (entry);
12663 : }
12664 : }
12665 : }
12666 29912 : while (todo != 0);
12667 :
12668 : /* Pop the SCC. */
12669 29044 : stack.truncate (last_idx);
12670 : }
12671 :
12672 1892728 : if (place_only)
12673 : return res;
12674 :
12675 : /* Now fixup the backedge def of the vectorized PHIs in this SCC. */
12676 : slp_tree phi_node;
12677 1015832 : FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
12678 : {
12679 55438 : gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
12680 55438 : edge_iterator ei;
12681 55438 : edge e;
12682 174512 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
12683 : {
12684 119074 : unsigned dest_idx = e->dest_idx;
12685 119074 : child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
12686 119074 : if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
12687 66979 : continue;
12688 52095 : unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
12689 : /* Simply fill all args. */
12690 52095 : if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
12691 : != vect_first_order_recurrence)
12692 110216 : for (unsigned i = 0; i < n; ++i)
12693 : {
12694 58166 : tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
12695 58166 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
12696 58166 : add_phi_arg (phi, vect_get_slp_vect_def (child, i),
12697 : e, gimple_phi_arg_location (phi, dest_idx));
12698 : }
12699 : else
12700 : {
12701 : /* Unless it is a first order recurrence which needs
12702 : args filled in for both the PHI node and the permutes. */
12703 45 : gimple *perm
12704 45 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
12705 45 : gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
12706 45 : add_phi_arg (as_a <gphi *> (rphi),
12707 : vect_get_slp_vect_def (child, n - 1),
12708 : e, gimple_phi_arg_location (phi, dest_idx));
12709 172 : for (unsigned i = 0; i < n; ++i)
12710 : {
12711 82 : gimple *perm
12712 82 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
12713 82 : if (i > 0)
12714 37 : gimple_assign_set_rhs1 (perm,
12715 : vect_get_slp_vect_def (child, i - 1));
12716 82 : gimple_assign_set_rhs2 (perm,
12717 : vect_get_slp_vect_def (child, i));
12718 82 : update_stmt (perm);
12719 : }
12720 : }
12721 : }
12722 : }
12723 :
12724 960394 : gcc_assert (res);
12725 : return true;
12726 1892728 : }
12727 :
12728 : /* Generate vector code for SLP_INSTANCES in the loop/basic block. Perform
12729 : vector stmt placement only when PLACE_ONLY is true, removing SLP graph
12730 : entries that cannot be scheduled. If placing, return false if a schedule
12731 : cannot be computed for any entry. */
12732 :
12733 : bool
12734 801798 : vect_schedule_slp (vec_info *vinfo, vec<slp_instance> &slp_instances,
12735 : bool place_only)
12736 : {
12737 801798 : slp_instance instance;
12738 801798 : unsigned int i;
12739 :
12740 801798 : hash_map<slp_tree, slp_scc_info> scc_info;
12741 801798 : int maxdfs = 0;
12742 2114072 : for (i = 0; slp_instances.iterate (i, &instance); )
12743 : {
12744 1312274 : slp_tree node = SLP_INSTANCE_TREE (instance);
12745 1312274 : if (!place_only && dump_enabled_p ())
12746 : {
12747 16142 : dump_printf_loc (MSG_NOTE, vect_location,
12748 : "Vectorizing SLP tree:\n");
12749 : /* ??? Dump all? */
12750 16142 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12751 510 : dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
12752 510 : SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
12753 16142 : vect_print_slp_graph (MSG_NOTE, vect_location,
12754 : SLP_INSTANCE_TREE (instance));
12755 : }
12756 : /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
12757 : have a PHI be the node breaking the cycle. */
12758 1312274 : bool res = true;
12759 1312274 : auto_vec<slp_tree> stack;
12760 1312274 : if (!scc_info.get (node))
12761 1308402 : res &= vect_schedule_scc (vinfo, node, instance, scc_info,
12762 : maxdfs, stack, place_only);
12763 :
12764 1312274 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12765 : {
12766 64825 : if (place_only)
12767 : {
12768 50979 : gimple *root_stmt = instance->root_stmts[0]->stmt;
12769 50979 : res &= (!node->si
12770 : /* As we instert after node->si it may not be the
12771 : root_stmt itself. */
12772 50979 : || (node->si != root_stmt
12773 50134 : && vect_stmt_dominates_stmt_p (node->si, root_stmt)));
12774 : }
12775 : else
12776 13846 : vectorize_slp_instance_root_stmt (vinfo, node, instance);
12777 : }
12778 :
12779 1312274 : if (!place_only && dump_enabled_p ())
12780 16142 : dump_printf_loc (MSG_NOTE, vect_location,
12781 : "vectorizing stmts using SLP.\n");
12782 :
12783 1312274 : if (!res)
12784 : {
12785 22 : gcc_assert (place_only);
12786 22 : if (dump_enabled_p ())
12787 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12788 : "not vectorized: cannot schedule SLP graph "
12789 : "entry %p\n", (void *)instance);
12790 22 : vect_free_slp_instance (instance);
12791 22 : slp_instances.ordered_remove (i);
12792 22 : continue;
12793 : }
12794 1312252 : ++i;
12795 1312274 : }
12796 :
12797 801798 : if (place_only)
12798 493670 : return !slp_instances.is_empty ();
12799 :
12800 1952067 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12801 : {
12802 595306 : slp_tree root = SLP_INSTANCE_TREE (instance);
12803 595306 : stmt_vec_info store_info;
12804 595306 : unsigned int j;
12805 :
12806 : /* Remove scalar call stmts. Do not do this for basic-block
12807 : vectorization as not all uses may be vectorized.
12808 : ??? Why should this be necessary? DCE should be able to
12809 : remove the stmts itself.
12810 : ??? For BB vectorization we can as well remove scalar
12811 : stmts starting from the SLP tree root if they have no
12812 : uses. */
12813 595306 : if (is_a <loop_vec_info> (vinfo))
12814 90471 : vect_remove_slp_scalar_calls (vinfo, root);
12815 :
12816 : /* Remove vectorized stores original scalar stmts. */
12817 2651479 : for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
12818 : {
12819 1500177 : if (!store_info
12820 1500163 : || !STMT_VINFO_DATA_REF (store_info)
12821 1470677 : || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
12822 : break;
12823 :
12824 1460867 : store_info = vect_orig_stmt (store_info);
12825 : /* Free the attached stmt_vec_info and remove the stmt. */
12826 1460867 : vinfo->remove_stmt (store_info);
12827 :
12828 : /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
12829 : to not crash in vect_free_slp_tree later. */
12830 1460867 : if (SLP_TREE_REPRESENTATIVE (root) == store_info)
12831 555658 : SLP_TREE_REPRESENTATIVE (root) = NULL;
12832 : }
12833 : }
12834 :
12835 : return true;
12836 801798 : }
|
