Line data Source code
1 : /* SLP - Basic Block Vectorization
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #define INCLUDE_ALGORITHM
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "optabs-tree.h"
34 : #include "insn-config.h"
35 : #include "recog.h" /* FIXME: for insn_data */
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "gimple-iterator.h"
39 : #include "cfgloop.h"
40 : #include "tree-vectorizer.h"
41 : #include "langhooks.h"
42 : #include "gimple-walk.h"
43 : #include "dbgcnt.h"
44 : #include "tree-vector-builder.h"
45 : #include "vec-perm-indices.h"
46 : #include "gimple-fold.h"
47 : #include "internal-fn.h"
48 : #include "dump-context.h"
49 : #include "cfganal.h"
50 : #include "tree-eh.h"
51 : #include "tree-cfg.h"
52 : #include "alloc-pool.h"
53 : #include "sreal.h"
54 : #include "predict.h"
55 :
56 : #define REDUC_GROUP_FIRST_ELEMENT(S) \
57 : (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
58 :
59 : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
60 : load_permutation_t &,
61 : const vec<tree> &,
62 : gimple_stmt_iterator *,
63 : poly_uint64, bool, bool,
64 : unsigned *,
65 : unsigned * = nullptr,
66 : bool = false);
67 : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
68 : slp_tree, lane_permutation_t &,
69 : vec<slp_tree> &, bool);
70 : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
71 : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
72 :
73 : static object_allocator<_slp_tree> *slp_tree_pool;
74 : static slp_tree slp_first_node;
75 :
76 : void
77 1140025 : vect_slp_init (void)
78 : {
79 1140025 : slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
80 1140025 : }
81 :
82 : void
83 1140025 : vect_slp_fini (void)
84 : {
85 1835342 : while (slp_first_node)
86 695317 : delete slp_first_node;
87 2280050 : delete slp_tree_pool;
88 1140025 : slp_tree_pool = NULL;
89 1140025 : }
90 :
91 : void *
92 8391890 : _slp_tree::operator new (size_t n)
93 : {
94 8391890 : gcc_assert (n == sizeof (_slp_tree));
95 8391890 : return slp_tree_pool->allocate_raw ();
96 : }
97 :
98 : void
99 8391890 : _slp_tree::operator delete (void *node, size_t n)
100 : {
101 8391890 : gcc_assert (n == sizeof (_slp_tree));
102 8391890 : slp_tree_pool->remove_raw (node);
103 8391890 : }
104 :
105 :
106 : /* Initialize a SLP node. */
107 :
108 8391890 : _slp_tree::_slp_tree ()
109 : {
110 8391890 : this->prev_node = NULL;
111 8391890 : if (slp_first_node)
112 7335874 : slp_first_node->prev_node = this;
113 8391890 : this->next_node = slp_first_node;
114 8391890 : slp_first_node = this;
115 8391890 : SLP_TREE_SCALAR_STMTS (this) = vNULL;
116 8391890 : SLP_TREE_SCALAR_OPS (this) = vNULL;
117 8391890 : SLP_TREE_LIVE_LANES (this) = vNULL;
118 8391890 : SLP_TREE_VEC_DEFS (this) = vNULL;
119 8391890 : SLP_TREE_CHILDREN (this) = vNULL;
120 8391890 : SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
121 8391890 : SLP_TREE_LANE_PERMUTATION (this) = vNULL;
122 8391890 : SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
123 8391890 : SLP_TREE_CODE (this) = ERROR_MARK;
124 8391890 : SLP_TREE_GS_SCALE (this) = 0;
125 8391890 : SLP_TREE_GS_BASE (this) = NULL_TREE;
126 8391890 : this->ldst_lanes = false;
127 8391890 : this->avoid_stlf_fail = false;
128 8391890 : SLP_TREE_VECTYPE (this) = NULL_TREE;
129 8391890 : SLP_TREE_REPRESENTATIVE (this) = NULL;
130 8391890 : this->cycle_info.id = -1;
131 8391890 : this->cycle_info.reduc_idx = -1;
132 8391890 : SLP_TREE_REF_COUNT (this) = 1;
133 8391890 : this->failed = NULL;
134 8391890 : this->lanes = 0;
135 8391890 : SLP_TREE_TYPE (this) = undef_vec_info_type;
136 8391890 : this->data = NULL;
137 8391890 : this->si = NULL;
138 8391890 : }
139 :
140 : /* Tear down a SLP node. */
141 :
142 8391890 : _slp_tree::~_slp_tree ()
143 : {
144 8391890 : if (this->prev_node)
145 5127097 : this->prev_node->next_node = this->next_node;
146 : else
147 3264793 : slp_first_node = this->next_node;
148 8391890 : if (this->next_node)
149 6329361 : this->next_node->prev_node = this->prev_node;
150 8391890 : SLP_TREE_CHILDREN (this).release ();
151 8391890 : SLP_TREE_SCALAR_STMTS (this).release ();
152 8391890 : SLP_TREE_SCALAR_OPS (this).release ();
153 8391890 : SLP_TREE_LIVE_LANES (this).release ();
154 8391890 : SLP_TREE_VEC_DEFS (this).release ();
155 8391890 : SLP_TREE_LOAD_PERMUTATION (this).release ();
156 8391890 : SLP_TREE_LANE_PERMUTATION (this).release ();
157 8391890 : if (this->failed)
158 2188564 : free (failed);
159 8391890 : if (this->data)
160 1293306 : delete this->data;
161 8391890 : }
162 :
163 : /* Push the single SSA definition in DEF to the vector of vector defs. */
164 :
165 : void
166 518811 : _slp_tree::push_vec_def (gimple *def)
167 : {
168 518811 : if (gphi *phi = dyn_cast <gphi *> (def))
169 57696 : vec_defs.quick_push (gimple_phi_result (phi));
170 : else
171 : {
172 461115 : def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
173 461115 : vec_defs.quick_push (get_def_from_ptr (defop));
174 : }
175 518811 : }
176 :
177 : /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
178 :
179 : void
180 15926858 : vect_free_slp_tree (slp_tree node)
181 : {
182 15926858 : int i;
183 15926858 : slp_tree child;
184 :
185 15926858 : if (--SLP_TREE_REF_COUNT (node) != 0)
186 15926858 : return;
187 :
188 12008787 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
189 4312214 : if (child)
190 3911332 : vect_free_slp_tree (child);
191 :
192 7696573 : delete node;
193 : }
194 :
195 : /* Return a location suitable for dumpings related to the SLP instance. */
196 :
197 : dump_user_location_t
198 3560101 : _slp_instance::location () const
199 : {
200 3560101 : if (!root_stmts.is_empty ())
201 421891 : return root_stmts[0]->stmt;
202 : else
203 3138210 : return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
204 : }
205 :
206 :
207 : /* Free the memory allocated for the SLP instance. */
208 :
209 : void
210 1662560 : vect_free_slp_instance (slp_instance instance)
211 : {
212 1662560 : vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
213 1662560 : SLP_INSTANCE_LOADS (instance).release ();
214 1662560 : SLP_INSTANCE_ROOT_STMTS (instance).release ();
215 1662560 : SLP_INSTANCE_REMAIN_DEFS (instance).release ();
216 1662560 : instance->subgraph_entries.release ();
217 1662560 : instance->cost_vec.release ();
218 1662560 : free (instance);
219 1662560 : }
220 :
221 :
222 : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
223 : for a permute node or else ERROR_MARK. */
224 :
225 : slp_tree
226 101611 : vect_create_new_slp_node (unsigned nops, tree_code code)
227 : {
228 101611 : gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
229 101611 : slp_tree node = new _slp_tree;
230 101611 : SLP_TREE_SCALAR_STMTS (node) = vNULL;
231 101611 : SLP_TREE_CHILDREN (node).create (nops);
232 101611 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
233 101611 : SLP_TREE_CODE (node) = code;
234 101611 : return node;
235 : }
236 :
237 : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children. */
238 :
239 : static slp_tree
240 4083342 : vect_create_new_slp_node (slp_tree node,
241 : vec<stmt_vec_info> scalar_stmts, unsigned nops)
242 : {
243 4083342 : SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
244 4083342 : SLP_TREE_CHILDREN (node).create (nops);
245 4083342 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
246 4083342 : SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
247 4083342 : SLP_TREE_LANES (node) = scalar_stmts.length ();
248 4083342 : return node;
249 : }
250 :
251 : /* Create an SLP node for SCALAR_STMTS and NOPS children. */
252 :
253 : static slp_tree
254 11988 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
255 : {
256 11988 : return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
257 : }
258 :
259 : /* Create a vect_external_def SLP node inplace at NODE for scalar
260 : operands OPS. */
261 :
262 : static slp_tree
263 2002734 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
264 : {
265 2002734 : SLP_TREE_SCALAR_OPS (node) = ops;
266 2002734 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
267 0 : SLP_TREE_LANES (node) = ops.length ();
268 2002734 : return node;
269 : }
270 :
271 : /* Create a vect_external_def SLP node for scalar operands OPS. */
272 :
273 : static slp_tree
274 2002734 : vect_create_new_slp_node (vec<tree> ops)
275 : {
276 2002734 : return vect_create_new_slp_node (new _slp_tree, ops);
277 : }
278 :
279 :
280 : /* This structure is used in creation of an SLP tree. Each instance
281 : corresponds to the same operand in a group of scalar stmts in an SLP
282 : node. */
283 : typedef struct _slp_oprnd_info
284 : {
285 : /* Def-stmts for the operands. */
286 : vec<stmt_vec_info> def_stmts;
287 : /* Operands. */
288 : vec<tree> ops;
289 : /* Information about the first statement, its vector def-type, type, the
290 : operand itself in case it's constant, and an indication if it's a pattern
291 : stmt and gather/scatter info. */
292 : tree first_op_type;
293 : enum vect_def_type first_dt;
294 : bool any_pattern;
295 : bool first_gs_p;
296 : gather_scatter_info first_gs_info;
297 : } *slp_oprnd_info;
298 :
299 :
300 : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
301 : operand. */
302 : static vec<slp_oprnd_info>
303 3741087 : vect_create_oprnd_info (int nops, int group_size)
304 : {
305 3741087 : int i;
306 3741087 : slp_oprnd_info oprnd_info;
307 3741087 : vec<slp_oprnd_info> oprnds_info;
308 :
309 3741087 : oprnds_info.create (nops);
310 13376821 : for (i = 0; i < nops; i++)
311 : {
312 5894647 : oprnd_info = XNEW (struct _slp_oprnd_info);
313 5894647 : oprnd_info->def_stmts.create (group_size);
314 5894647 : oprnd_info->ops.create (group_size);
315 5894647 : oprnd_info->first_dt = vect_uninitialized_def;
316 5894647 : oprnd_info->first_op_type = NULL_TREE;
317 5894647 : oprnd_info->any_pattern = false;
318 5894647 : oprnd_info->first_gs_p = false;
319 5894647 : oprnds_info.quick_push (oprnd_info);
320 : }
321 :
322 3741087 : return oprnds_info;
323 : }
324 :
325 :
326 : /* Free operands info. */
327 :
328 : static void
329 3618734 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
330 : {
331 3618734 : int i;
332 3618734 : slp_oprnd_info oprnd_info;
333 :
334 9382546 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
335 : {
336 5763812 : oprnd_info->def_stmts.release ();
337 5763812 : oprnd_info->ops.release ();
338 5763812 : XDELETE (oprnd_info);
339 : }
340 :
341 3618734 : oprnds_info.release ();
342 3618734 : }
343 :
344 : /* Return the execution frequency of NODE (so that a higher value indicates
345 : a "more important" node when optimizing for speed). */
346 :
347 : static sreal
348 3772972 : vect_slp_node_weight (vec_info *vinfo, slp_tree node)
349 : {
350 3772972 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
351 3772972 : basic_block bb;
352 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
353 : but are laid out close to their children. */
354 3772972 : if (!stmt_info)
355 148959 : bb = vinfo->bbs[0];
356 : else
357 4121596 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
358 3772972 : return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
359 : }
360 :
361 : /* Return true if STMTS contains a pattern statement. */
362 :
363 : static bool
364 25010 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
365 : {
366 25010 : stmt_vec_info stmt_info;
367 25010 : unsigned int i;
368 83484 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
369 60531 : if (stmt_info && is_pattern_stmt_p (stmt_info))
370 : return true;
371 : return false;
372 : }
373 :
374 : /* Return true when all lanes in the external or constant NODE have
375 : the same value. */
376 :
377 : static bool
378 650049 : vect_slp_tree_uniform_p (slp_tree node)
379 : {
380 650049 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
381 : || SLP_TREE_DEF_TYPE (node) == vect_external_def);
382 :
383 : /* Pre-existing vectors. */
384 650049 : if (SLP_TREE_SCALAR_OPS (node).is_empty ())
385 : return false;
386 :
387 : unsigned i;
388 : tree op, first = NULL_TREE;
389 1489936 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
390 1330854 : if (!first)
391 : first = op;
392 680805 : else if (!operand_equal_p (first, op, 0))
393 : return false;
394 :
395 : return true;
396 : }
397 :
398 : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
399 : that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
400 : of the chain. */
401 :
402 : int
403 754762 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
404 : stmt_vec_info first_stmt_info)
405 : {
406 754762 : stmt_vec_info next_stmt_info = first_stmt_info;
407 754762 : int result = 0;
408 :
409 754762 : if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
410 : return -1;
411 :
412 2137967 : do
413 : {
414 2137967 : if (next_stmt_info == stmt_info)
415 : return result;
416 1383205 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
417 1383205 : if (next_stmt_info)
418 1383205 : result += DR_GROUP_GAP (next_stmt_info);
419 : }
420 1383205 : while (next_stmt_info);
421 :
422 : return -1;
423 : }
424 :
425 : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
426 : using the method implemented by duplicate_and_interleave. Return true
427 : if so, returning the number of intermediate vectors in *NVECTORS_OUT
428 : (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
429 : (if nonnull). */
430 :
431 : bool
432 0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
433 : tree elt_type, unsigned int *nvectors_out,
434 : tree *vector_type_out,
435 : tree *permutes)
436 : {
437 0 : tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
438 0 : if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
439 : return false;
440 :
441 0 : machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
442 0 : poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
443 0 : unsigned int nvectors = 1;
444 0 : for (;;)
445 : {
446 0 : scalar_int_mode int_mode;
447 0 : poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
448 0 : if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
449 : {
450 : /* Get the natural vector type for this SLP group size. */
451 0 : tree int_type = build_nonstandard_integer_type
452 0 : (GET_MODE_BITSIZE (int_mode), 1);
453 0 : tree vector_type
454 0 : = get_vectype_for_scalar_type (vinfo, int_type, count);
455 0 : poly_int64 half_nelts;
456 0 : if (vector_type
457 0 : && VECTOR_MODE_P (TYPE_MODE (vector_type))
458 0 : && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
459 : GET_MODE_SIZE (base_vector_mode))
460 0 : && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
461 : 2, &half_nelts))
462 : {
463 : /* Try fusing consecutive sequences of COUNT / NVECTORS elements
464 : together into elements of type INT_TYPE and using the result
465 : to build NVECTORS vectors. */
466 0 : poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
467 0 : vec_perm_builder sel1 (nelts, 2, 3);
468 0 : vec_perm_builder sel2 (nelts, 2, 3);
469 :
470 0 : for (unsigned int i = 0; i < 3; ++i)
471 : {
472 0 : sel1.quick_push (i);
473 0 : sel1.quick_push (i + nelts);
474 0 : sel2.quick_push (half_nelts + i);
475 0 : sel2.quick_push (half_nelts + i + nelts);
476 : }
477 0 : vec_perm_indices indices1 (sel1, 2, nelts);
478 0 : vec_perm_indices indices2 (sel2, 2, nelts);
479 0 : machine_mode vmode = TYPE_MODE (vector_type);
480 0 : if (can_vec_perm_const_p (vmode, vmode, indices1)
481 0 : && can_vec_perm_const_p (vmode, vmode, indices2))
482 : {
483 0 : if (nvectors_out)
484 0 : *nvectors_out = nvectors;
485 0 : if (vector_type_out)
486 0 : *vector_type_out = vector_type;
487 0 : if (permutes)
488 : {
489 0 : permutes[0] = vect_gen_perm_mask_checked (vector_type,
490 : indices1);
491 0 : permutes[1] = vect_gen_perm_mask_checked (vector_type,
492 : indices2);
493 : }
494 0 : return true;
495 : }
496 0 : }
497 : }
498 0 : if (!multiple_p (elt_bytes, 2, &elt_bytes))
499 : return false;
500 0 : nvectors *= 2;
501 : /* We need to be able to fuse COUNT / NVECTORS elements together. */
502 0 : if (!multiple_p (count, nvectors))
503 : return false;
504 : }
505 : }
506 :
507 : /* Return true if DTA and DTB match. */
508 :
509 : static bool
510 17502462 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
511 : {
512 17502462 : return (dta == dtb
513 416170 : || ((dta == vect_external_def || dta == vect_constant_def)
514 266638 : && (dtb == vect_external_def || dtb == vect_constant_def)));
515 : }
516 :
517 : #define GATHER_SCATTER_OFFSET (-3)
518 :
519 : /* For most SLP statements, there is a one-to-one mapping between
520 : gimple arguments and child nodes. If that is not true for STMT,
521 : return an array that contains:
522 :
523 : - the number of child nodes, followed by
524 : - for each child node, the index of the argument associated with that node.
525 : The special index -1 is the first operand of an embedded comparison and
526 : the special index -2 is the second operand of an embedded comparison.
527 : The special index -3 is the offset of a gather as analyzed by
528 : vect_check_gather_scatter.
529 :
530 : SWAP is as for vect_get_and_check_slp_defs. */
531 :
532 : static const int *
533 26014995 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
534 : unsigned char swap)
535 : {
536 26014995 : static const int no_arg_map[] = { 0 };
537 26014995 : static const int arg0_map[] = { 1, 0 };
538 26014995 : static const int arg2_map[] = { 1, 2 };
539 26014995 : static const int arg2_arg3_map[] = { 2, 2, 3 };
540 26014995 : static const int arg2_arg4_map[] = { 2, 2, 4 };
541 26014995 : static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
542 26014995 : static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
543 26014995 : static const int arg3_arg2_map[] = { 2, 3, 2 };
544 26014995 : static const int op00_map[] = { 1, -1 };
545 26014995 : static const int op1_op0_map[] = { 2, 1, 0 };
546 26014995 : static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
547 26014995 : static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
548 26014995 : static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
549 26014995 : static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
550 26014995 : static const int mask_call_maps[6][7] = {
551 : { 1, 1, },
552 : { 2, 1, 2, },
553 : { 3, 1, 2, 3, },
554 : { 4, 1, 2, 3, 4, },
555 : { 5, 1, 2, 3, 4, 5, },
556 : { 6, 1, 2, 3, 4, 5, 6 },
557 : };
558 :
559 26014995 : gcc_checking_assert (!swap
560 : || !is_gimple_assign (stmt)
561 : || TREE_CODE_CLASS
562 : (gimple_assign_rhs_code (stmt)) == tcc_comparison
563 : || commutative_tree_code
564 : (gimple_assign_rhs_code (stmt)));
565 :
566 26014995 : if (auto assign = dyn_cast<const gassign *> (stmt))
567 : {
568 24464987 : tree_code code = gimple_assign_rhs_code (assign);
569 24464987 : if (code == COND_EXPR
570 24464987 : && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
571 0 : gcc_unreachable ();
572 24464987 : else if ((TREE_CODE_CLASS (code) == tcc_comparison
573 22917696 : || commutative_tree_code (code))
574 33525614 : && swap)
575 : return op1_op0_map;
576 24422705 : else if (code == VIEW_CONVERT_EXPR)
577 : return op00_map;
578 24413217 : else if (gather_scatter_p)
579 54752 : return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
580 54752 : ? off_op0_map : off_map);
581 : }
582 1550008 : else if (auto call = dyn_cast<const gcall *> (stmt))
583 : {
584 174117 : if (gimple_call_internal_p (call))
585 95098 : switch (gimple_call_internal_fn (call))
586 : {
587 13996 : case IFN_MASK_LOAD:
588 13996 : return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
589 :
590 0 : case IFN_GATHER_LOAD:
591 0 : return arg2_map;
592 :
593 0 : case IFN_MASK_GATHER_LOAD:
594 0 : case IFN_MASK_LEN_GATHER_LOAD:
595 0 : return arg2_arg5_arg6_map;
596 :
597 0 : case IFN_SCATTER_STORE:
598 0 : return arg2_arg4_map;
599 :
600 0 : case IFN_MASK_SCATTER_STORE:
601 0 : case IFN_MASK_LEN_SCATTER_STORE:
602 0 : return arg2_arg4_arg5_map;
603 :
604 9326 : case IFN_MASK_STORE:
605 9326 : return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
606 :
607 1090 : case IFN_MASK_CALL:
608 1090 : {
609 1090 : unsigned nargs = gimple_call_num_args (call);
610 1090 : if (nargs >= 2 && nargs <= 7)
611 1090 : return mask_call_maps[nargs-2];
612 : else
613 : return nullptr;
614 : }
615 :
616 437 : case IFN_CLZ:
617 437 : case IFN_CTZ:
618 437 : return arg0_map;
619 :
620 7308 : case IFN_GOMP_SIMD_LANE:
621 7308 : return no_arg_map;
622 :
623 : default:
624 : break;
625 : }
626 : }
627 : return nullptr;
628 : }
629 :
630 : static const int *
631 25995787 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
632 : {
633 0 : return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
634 0 : swap);
635 : }
636 :
637 : /* Return the SLP node child index for operand OP of STMT. */
638 :
639 : int
640 1398238 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
641 : {
642 1398238 : const int *opmap = vect_get_operand_map (stmt);
643 1398238 : if (!opmap)
644 : return op;
645 20109 : for (int i = 1; i < 1 + opmap[0]; ++i)
646 20109 : if (opmap[i] == op)
647 11495 : return i - 1;
648 0 : gcc_unreachable ();
649 : }
650 :
651 : /* Helper class for mapping of GIMPLE operands to SLP children. */
652 : /* ??? Add vect_slp_child_index_for_operand here and amend opmaps
653 : with the full reverse mapping and indicating the position of the
654 : first commutative operand index, eliding the swap_p argument from
655 : vect_get_operand_map. Adjust all consumers. */
656 :
657 : struct slp_oprnds {
658 : slp_oprnds (stmt_vec_info);
659 : tree get_op_for_slp_child (stmt_vec_info, unsigned);
660 : const int *opmap;
661 : const unsigned int num_slp_children;
662 : };
663 :
664 4677206 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
665 4677206 : : opmap (vect_get_operand_map (stmt_info)),
666 4677206 : num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
667 : {
668 4677206 : }
669 :
670 : /* For SLP child number N get the corresponding tree operand from GIMPLE
671 : statement described by STMT_INFO. */
672 :
673 : tree
674 5236105 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
675 : {
676 5236105 : gcc_assert (n < num_slp_children);
677 5236105 : int opno = opmap ? opmap[n + 1] : (int) n;
678 5236105 : if (opno == GATHER_SCATTER_OFFSET)
679 0 : gcc_unreachable (); // TODO
680 5236105 : else if (opno < 0)
681 2602 : return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
682 : else
683 5233503 : return gimple_arg (stmt_info->stmt, opno);
684 : }
685 :
686 : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
687 : they are of a valid type and that they match the defs of the first stmt of
688 : the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
689 : by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP
690 : indicates swap is required for cond_expr stmts. Specifically, SWAP
691 : is 1 if STMT is cond and operands of comparison need to be swapped;
692 : SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
693 :
694 : If there was a fatal error return -1; if the error could be corrected by
695 : swapping operands of father node of this one, return 1; if everything is
696 : ok return 0. */
697 : static int
698 13663956 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
699 : bool *skip_args,
700 : vec<stmt_vec_info> stmts, unsigned stmt_num,
701 : vec<slp_oprnd_info> *oprnds_info,
702 : bool soft_fail)
703 : {
704 13663956 : stmt_vec_info stmt_info = stmts[stmt_num];
705 13663956 : tree oprnd;
706 13663956 : unsigned int i, number_of_oprnds;
707 13663956 : enum vect_def_type dt = vect_uninitialized_def;
708 13663956 : slp_oprnd_info oprnd_info;
709 13663956 : gather_scatter_info gs_info;
710 13663956 : unsigned int gs_op = -1u;
711 13663956 : unsigned int commutative_op = -1U;
712 13663956 : bool first = stmt_num == 0;
713 :
714 13663956 : if (!stmt_info)
715 : {
716 0 : for (auto oi : *oprnds_info)
717 : {
718 0 : oi->def_stmts.quick_push (NULL);
719 0 : oi->ops.quick_push (NULL_TREE);
720 : }
721 : return 0;
722 : }
723 :
724 13663956 : if (!is_a<gcall *> (stmt_info->stmt)
725 : && !is_a<gassign *> (stmt_info->stmt)
726 : && !is_a<gphi *> (stmt_info->stmt))
727 : return -1;
728 :
729 13663956 : number_of_oprnds = gimple_num_args (stmt_info->stmt);
730 13663956 : const int *map = vect_get_operand_map (stmt_info, swap);
731 13663956 : if (map)
732 84035 : number_of_oprnds = *map++;
733 13663956 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
734 : {
735 56035 : if (gimple_call_internal_p (stmt))
736 : {
737 34324 : internal_fn ifn = gimple_call_internal_fn (stmt);
738 34324 : commutative_op = first_commutative_argument (ifn);
739 34324 : if (internal_gather_scatter_fn_p (ifn))
740 : {
741 0 : vect_describe_gather_scatter_call
742 0 : (stmt_info,
743 0 : first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
744 0 : if (first)
745 0 : (*oprnds_info)[0]->first_gs_p = true;
746 : gs_op = 0;
747 : }
748 : }
749 : }
750 13607921 : else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
751 : {
752 16379941 : if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
753 8459527 : commutative_op = 0;
754 : }
755 :
756 13663956 : bool swapped = (swap != 0);
757 13663956 : bool backedge = false;
758 13663956 : enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
759 37255039 : for (i = 0; i < number_of_oprnds; i++)
760 : {
761 23592471 : oprnd_info = (*oprnds_info)[i];
762 23592471 : int opno = map ? map[i] : int (i);
763 23592471 : if (opno == GATHER_SCATTER_OFFSET)
764 : {
765 28351 : gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
766 28351 : if (!is_a <loop_vec_info> (vinfo)
767 28351 : || !vect_check_gather_scatter (stmt_info, vectype,
768 : as_a <loop_vec_info> (vinfo),
769 : first ? &oprnd_info->first_gs_info
770 : : &gs_info))
771 1388 : return -1;
772 :
773 28351 : if (first)
774 : {
775 27170 : oprnd_info->first_gs_p = true;
776 27170 : oprnd = oprnd_info->first_gs_info.offset;
777 : }
778 : else
779 : {
780 1181 : gs_op = i;
781 1181 : oprnd = gs_info.offset;
782 : }
783 : }
784 23564120 : else if (opno < 0)
785 3267 : oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
786 : else
787 : {
788 23560853 : oprnd = gimple_arg (stmt_info->stmt, opno);
789 23560853 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
790 : {
791 1263511 : edge e = gimple_phi_arg_edge (stmt, opno);
792 2527022 : backedge = (is_a <bb_vec_info> (vinfo)
793 1950793 : ? e->flags & EDGE_DFS_BACK
794 687282 : : dominated_by_p (CDI_DOMINATORS, e->src,
795 687282 : gimple_bb (stmt_info->stmt)));
796 : }
797 : }
798 :
799 23592471 : stmt_vec_info def_stmt_info;
800 23592471 : if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
801 : {
802 1064 : if (dump_enabled_p ())
803 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
804 : "Build SLP failed: can't analyze def for %T\n",
805 : oprnd);
806 :
807 : return -1;
808 : }
809 :
810 23591407 : if (skip_args[i])
811 : {
812 561462 : oprnd_info->def_stmts.quick_push (NULL);
813 561462 : oprnd_info->ops.quick_push (NULL_TREE);
814 561462 : oprnd_info->first_dt = vect_uninitialized_def;
815 561462 : continue;
816 : }
817 :
818 23029945 : oprnd_info->def_stmts.quick_push (def_stmt_info);
819 23029945 : oprnd_info->ops.quick_push (oprnd);
820 :
821 23029945 : if (def_stmt_info
822 23029945 : && is_pattern_stmt_p (def_stmt_info))
823 : {
824 473498 : if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
825 : != def_stmt_info)
826 326473 : oprnd_info->any_pattern = true;
827 : else
828 : /* If we promote this to external use the original stmt def. */
829 147025 : oprnd_info->ops.last ()
830 294050 : = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
831 : }
832 :
833 : /* If there's a extern def on a backedge make sure we can
834 : code-generate at the region start.
835 : ??? This is another case that could be fixed by adjusting
836 : how we split the function but at the moment we'd have conflicting
837 : goals there. */
838 23029945 : if (backedge
839 163836 : && dts[i] == vect_external_def
840 345 : && is_a <bb_vec_info> (vinfo)
841 345 : && TREE_CODE (oprnd) == SSA_NAME
842 324 : && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
843 23030269 : && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
844 324 : gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
845 : {
846 324 : if (dump_enabled_p ())
847 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
848 : "Build SLP failed: extern def %T only defined "
849 : "on backedge\n", oprnd);
850 : return -1;
851 : }
852 :
853 23029621 : if (first)
854 : {
855 5358901 : tree type = TREE_TYPE (oprnd);
856 5358901 : dt = dts[i];
857 :
858 : /* For the swapping logic below force vect_reduction_def
859 : for the reduction op in a SLP reduction group. */
860 5358901 : if (!STMT_VINFO_DATA_REF (stmt_info)
861 4026653 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
862 1804 : && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
863 5359760 : && def_stmt_info)
864 859 : dts[i] = dt = vect_reduction_def;
865 :
866 : /* Check the types of the definition. */
867 5358901 : switch (dt)
868 : {
869 5358901 : case vect_external_def:
870 5358901 : case vect_constant_def:
871 5358901 : case vect_internal_def:
872 5358901 : case vect_reduction_def:
873 5358901 : case vect_double_reduction_def:
874 5358901 : case vect_induction_def:
875 5358901 : case vect_nested_cycle:
876 5358901 : case vect_first_order_recurrence:
877 5358901 : break;
878 :
879 0 : default:
880 : /* FORNOW: Not supported. */
881 0 : if (dump_enabled_p ())
882 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
883 : "Build SLP failed: illegal type of def %T\n",
884 : oprnd);
885 : return -1;
886 : }
887 :
888 5358901 : oprnd_info->first_dt = dt;
889 5358901 : oprnd_info->first_op_type = type;
890 : }
891 : }
892 13662568 : if (first)
893 : return 0;
894 :
895 : /* Now match the operand definition types to that of the first stmt. */
896 27231155 : for (i = 0; i < number_of_oprnds;)
897 : {
898 17646202 : if (skip_args[i])
899 : {
900 30337 : ++i;
901 30337 : continue;
902 : }
903 :
904 17615865 : oprnd_info = (*oprnds_info)[i];
905 17615865 : dt = dts[i];
906 17615865 : stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
907 17615865 : oprnd = oprnd_info->ops[stmt_num];
908 17615865 : tree type = TREE_TYPE (oprnd);
909 :
910 17615865 : if (!types_compatible_p (oprnd_info->first_op_type, type))
911 : {
912 120735 : if (dump_enabled_p ())
913 93 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
914 : "Build SLP failed: different operand types\n");
915 : return 1;
916 : }
917 :
918 17495130 : if ((gs_op == i) != oprnd_info->first_gs_p)
919 : {
920 0 : if (dump_enabled_p ())
921 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
922 : "Build SLP failed: mixed gather and non-gather\n");
923 : return 1;
924 : }
925 17495130 : else if (gs_op == i)
926 : {
927 1149 : if (!operand_equal_p (oprnd_info->first_gs_info.base,
928 1149 : gs_info.base))
929 : {
930 16 : if (dump_enabled_p ())
931 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
932 : "Build SLP failed: different gather base\n");
933 : return 1;
934 : }
935 1133 : if (oprnd_info->first_gs_info.scale != gs_info.scale)
936 : {
937 8 : if (dump_enabled_p ())
938 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
939 : "Build SLP failed: different gather scale\n");
940 : return 1;
941 : }
942 : }
943 :
944 : /* Not first stmt of the group, check that the def-stmt/s match
945 : the def-stmt/s of the first stmt. Allow different definition
946 : types for reduction chains: the first stmt must be a
947 : vect_reduction_def (a phi node), and the rest
948 : end in the reduction chain. */
949 17495106 : if ((!vect_def_types_match (oprnd_info->first_dt, dt)
950 334013 : && !(oprnd_info->first_dt == vect_reduction_def
951 3269 : && !STMT_VINFO_DATA_REF (stmt_info)
952 3269 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
953 3243 : && def_stmt_info
954 3241 : && !STMT_VINFO_DATA_REF (def_stmt_info)
955 3241 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
956 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
957 17164334 : || (!STMT_VINFO_DATA_REF (stmt_info)
958 15376091 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
959 6927 : && ((!def_stmt_info
960 6691 : || STMT_VINFO_DATA_REF (def_stmt_info)
961 11656 : || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
962 : != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
963 6927 : != (oprnd_info->first_dt != vect_reduction_def))))
964 : {
965 : /* Try swapping operands if we got a mismatch. For BB
966 : vectorization only in case it will clearly improve things. */
967 334371 : if (i == commutative_op && !swapped
968 330772 : && (!is_a <bb_vec_info> (vinfo)
969 5655 : || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
970 5655 : dts[i+1])
971 1468 : && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
972 : || vect_def_types_match
973 233 : ((*oprnds_info)[i+1]->first_dt, dts[i])))))
974 : {
975 3599 : if (dump_enabled_p ())
976 194 : dump_printf_loc (MSG_NOTE, vect_location,
977 : "trying swapped operands\n");
978 3599 : std::swap (dts[i], dts[i+1]);
979 3599 : std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
980 3599 : (*oprnds_info)[i+1]->def_stmts[stmt_num]);
981 3599 : std::swap ((*oprnds_info)[i]->ops[stmt_num],
982 3599 : (*oprnds_info)[i+1]->ops[stmt_num]);
983 : /* After swapping some operands we lost track whether an
984 : operand has any pattern defs so be conservative here. */
985 3599 : if ((*oprnds_info)[i]->any_pattern
986 3599 : || (*oprnds_info)[i+1]->any_pattern)
987 36 : (*oprnds_info)[i]->any_pattern
988 18 : = (*oprnds_info)[i+1]->any_pattern = true;
989 3599 : swapped = true;
990 3599 : continue;
991 : }
992 :
993 327173 : if (is_a <bb_vec_info> (vinfo)
994 308935 : && !oprnd_info->any_pattern
995 308667 : && number_of_oprnds > 1
996 635840 : && !soft_fail)
997 : {
998 : /* Now for commutative ops we should see whether we can
999 : make the other operand matching. */
1000 107227 : if (dump_enabled_p ())
1001 336 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1002 : "treating operand as external\n");
1003 : oprnd_info->first_dt = dt = vect_external_def;
1004 : }
1005 : else
1006 : {
1007 219946 : if (dump_enabled_p ())
1008 412 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1009 : "Build SLP failed: different types\n");
1010 : return 1;
1011 : }
1012 : }
1013 :
1014 : /* Make sure to demote the overall operand to external. */
1015 17164334 : if (dt == vect_external_def)
1016 422267 : oprnd_info->first_dt = vect_external_def;
1017 : /* For a SLP reduction chain we want to duplicate the reduction to
1018 : each of the chain members. That gets us a sane SLP graph (still
1019 : the stmts are not 100% correct wrt the initial values). */
1020 16849294 : else if ((dt == vect_internal_def
1021 16849294 : || dt == vect_reduction_def)
1022 15589967 : && oprnd_info->first_dt == vect_reduction_def
1023 49214 : && !STMT_VINFO_DATA_REF (stmt_info)
1024 49214 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
1025 3241 : && !STMT_VINFO_DATA_REF (def_stmt_info)
1026 16852535 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
1027 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
1028 : {
1029 3241 : oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
1030 3241 : oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
1031 : }
1032 :
1033 17271561 : ++i;
1034 : }
1035 :
1036 : /* Swap operands. */
1037 9584953 : if (swapped)
1038 : {
1039 42001 : if (dump_enabled_p ())
1040 526 : dump_printf_loc (MSG_NOTE, vect_location,
1041 : "swapped operands to match def types in %G",
1042 : stmt_info->stmt);
1043 : }
1044 :
1045 : return 0;
1046 : }
1047 :
1048 : /* Return true if call statements CALL1 and CALL2 are similar enough
1049 : to be combined into the same SLP group. */
1050 :
1051 : bool
1052 65765 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
1053 : {
1054 65765 : unsigned int nargs = gimple_call_num_args (call1);
1055 65765 : if (nargs != gimple_call_num_args (call2))
1056 : return false;
1057 :
1058 48666 : auto cfn1 = gimple_call_combined_fn (call1);
1059 48666 : auto cfn2 = gimple_call_combined_fn (call2);
1060 48666 : if (cfn1 != cfn2
1061 39 : && (!allow_two_operators
1062 39 : || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
1063 2 : && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
1064 : return false;
1065 :
1066 48629 : if (gimple_call_internal_p (call1))
1067 : {
1068 7777 : if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
1069 7777 : TREE_TYPE (gimple_call_lhs (call2))))
1070 : return false;
1071 16495 : for (unsigned int i = 0; i < nargs; ++i)
1072 8718 : if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
1073 8718 : TREE_TYPE (gimple_call_arg (call2, i))))
1074 : return false;
1075 : }
1076 : else
1077 : {
1078 40852 : if (!operand_equal_p (gimple_call_fn (call1),
1079 40852 : gimple_call_fn (call2), 0))
1080 : return false;
1081 :
1082 34293 : if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
1083 : return false;
1084 : }
1085 :
1086 : /* Check that any unvectorized arguments are equal. */
1087 19208 : if (const int *map = vect_get_operand_map (call1, false, false))
1088 : {
1089 65 : unsigned int nkept = *map++;
1090 65 : unsigned int mapi = 0;
1091 207 : for (unsigned int i = 0; i < nargs; ++i)
1092 142 : if (mapi < nkept && map[mapi] == int (i))
1093 77 : mapi += 1;
1094 65 : else if (!operand_equal_p (gimple_call_arg (call1, i),
1095 65 : gimple_call_arg (call2, i)))
1096 : return false;
1097 : }
1098 :
1099 : return true;
1100 : }
1101 :
1102 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1103 : permutation or are of unsupported types of operation.
1104 : Return false if at least one (or all in case of BB vectorization)
1105 : stmt is unvectorizable or the comparison could not be carried out.
1106 : Return true if they all (or at least one in case of BB vectorization)
1107 : are and indicate in MATCHES[] which stmts are not isomorphic to the
1108 : stmt at *START_I.
1109 :
1110 : This function is designed to be invoked repeatedly on the same
1111 : set of STMTS with increasing *START_I.
1112 :
1113 : When *START_I is zero and the function returns true MATCHES[] will be
1114 : initialized with, in case of BB vectorization, unvectorizable stmts
1115 : marked with -2, stmts isomorphic to *START_I as *START_I and
1116 : other vectorizable stmts with -1 (not isomorphic to *START_I).
1117 : *START_I, when initially zero, is updated to the first vectorizable
1118 : statement, so MATCHES[] can have a prefix with entries valued -2.
1119 :
1120 : When *START_I is not zero MATCHES[] is expected to be pre-initialized
1121 : by a former call with *START_I zero and MATCHES[*START_I] should be -1.
1122 : The function will return true and have the stmts isomorphic to
1123 : MATCHES[*START_I] marked with *START_I.
1124 :
1125 : *TWO_OPERATORS indicates whether the group of isomorphic statements
1126 : uses two related operations like PLUS_EXPR and MINUS_EXPR. If
1127 : TWO_OPERATORS is NULL such case is not considered isomorphic.
1128 :
1129 : SWAP[] indicates whether for a stmt to be isomorphic to its group
1130 : leader, has to have its operands swapped (1) or its predicate inverted (2).
1131 : COND_EXPR is possibly isomorphic to another one after swapping its
1132 : operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
1133 : the first stmt by swapping the two operands of comparison; set SWAP[i]
1134 : to 2 if stmt I is isormorphic to the first stmt by inverting the code
1135 : of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
1136 : to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
1137 :
1138 : static bool
1139 6346506 : vect_build_slp_tree_3 (vec_info *vinfo, vec<stmt_vec_info> stmts,
1140 : match_elt_t *matches, unsigned char *swap,
1141 : bool *two_operators, tree vectype, unsigned *start_i)
1142 : {
1143 6346506 : stmt_vec_info first_stmt_info = NULL;
1144 6346506 : code_helper first_stmt_code = ERROR_MARK;
1145 6346506 : code_helper alt_stmt_code = ERROR_MARK;
1146 6346506 : code_helper first_cond_code = ERROR_MARK;
1147 6346506 : bool need_same_oprnds = false;
1148 6346506 : tree first_lhs = NULL_TREE;
1149 6346506 : tree first_op1 = NULL_TREE;
1150 6346506 : stmt_vec_info first_load = NULL, prev_first_load = NULL;
1151 6346506 : bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
1152 6346506 : bool first_stmt_phi_p = false;
1153 6346506 : int first_reduc_idx = -1;
1154 :
1155 6346506 : basic_block common_bb = NULL;
1156 6346506 : gimple *trapping_stmt = NULL;
1157 6346506 : int first_match = -1;
1158 :
1159 26163365 : for (unsigned i = *start_i; i < stmts.length (); ++i)
1160 : {
1161 19817130 : stmt_vec_info stmt_info = stmts[i];
1162 19817130 : bool ldst_p = false;
1163 19817130 : bool ldst_masklen_p = false;
1164 19817130 : bool phi_p = false;
1165 19817130 : code_helper rhs_code = ERROR_MARK;
1166 :
1167 19817130 : if (*start_i == 0)
1168 : {
1169 17960904 : swap[i] = 0;
1170 17960904 : matches[i] = -1;
1171 : }
1172 1856226 : else if (matches[i] != -1)
1173 19816859 : continue;
1174 :
1175 19758099 : if (!stmt_info)
1176 : {
1177 : /* ??? We shouldn't run into this. */
1178 41367 : gcc_assert (first_match != -1);
1179 41367 : matches[i] = first_match;
1180 41367 : continue;
1181 : }
1182 :
1183 19716732 : gimple *stmt = stmt_info->stmt;
1184 19716732 : if (dump_enabled_p ())
1185 244575 : dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
1186 :
1187 : /* Fail to vectorize statements marked as unvectorizable, throw
1188 : or are volatile. */
1189 19716732 : if (!STMT_VINFO_VECTORIZABLE (stmt_info)
1190 19328531 : || stmt_can_throw_internal (cfun, stmt)
1191 38038536 : || gimple_has_volatile_ops (stmt))
1192 : {
1193 405548 : if (dump_enabled_p ())
1194 474 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1195 : "Build SLP failed: unvectorizable statement %G",
1196 : stmt);
1197 405548 : if (is_a <bb_vec_info> (vinfo))
1198 : {
1199 405548 : matches[i] = -2;
1200 405548 : continue;
1201 : }
1202 6346506 : return false;
1203 : }
1204 :
1205 19311184 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
1206 19311184 : tree lhs = gimple_get_lhs (stmt);
1207 19311184 : if (lhs == NULL_TREE && !call_stmt)
1208 : {
1209 34 : if (dump_enabled_p ())
1210 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1211 : "Build SLP failed: not GIMPLE_ASSIGN nor "
1212 : "GIMPLE_CALL %G", stmt);
1213 34 : if (is_a <bb_vec_info> (vinfo))
1214 : {
1215 34 : matches[i] = -2;
1216 34 : continue;
1217 : }
1218 : return false;
1219 : }
1220 :
1221 19311150 : if (call_stmt)
1222 : {
1223 207594 : combined_fn cfn = gimple_call_combined_fn (call_stmt);
1224 207594 : if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
1225 65358 : rhs_code = cfn;
1226 : else
1227 : rhs_code = CALL_EXPR;
1228 :
1229 207594 : if (cfn == CFN_GATHER_LOAD
1230 207594 : || cfn == CFN_SCATTER_STORE)
1231 : ldst_p = true;
1232 : else if (cfn == CFN_MASK_LOAD
1233 : || cfn == CFN_MASK_GATHER_LOAD
1234 : || cfn == CFN_MASK_LEN_GATHER_LOAD
1235 : || cfn == CFN_MASK_SCATTER_STORE
1236 : || cfn == CFN_MASK_LEN_SCATTER_STORE)
1237 : {
1238 : ldst_p = true;
1239 : ldst_masklen_p = true;
1240 : }
1241 : else if (cfn == CFN_MASK_STORE)
1242 : {
1243 : ldst_p = true;
1244 : ldst_masklen_p = true;
1245 : rhs_code = CFN_MASK_STORE;
1246 : }
1247 : else if (cfn == CFN_GOMP_SIMD_LANE)
1248 : ;
1249 196435 : else if ((cfn != CFN_LAST
1250 : && cfn != CFN_MASK_CALL
1251 54199 : && internal_fn_p (cfn)
1252 40347 : && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
1253 196324 : || gimple_call_tail_p (call_stmt)
1254 196324 : || gimple_call_noreturn_p (call_stmt)
1255 392759 : || gimple_call_chain (call_stmt))
1256 : {
1257 1394 : if (dump_enabled_p ())
1258 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1259 : "Build SLP failed: unsupported call type %G",
1260 : (gimple *) call_stmt);
1261 1394 : if (is_a <bb_vec_info> (vinfo))
1262 : {
1263 1394 : matches[i] = -2;
1264 1394 : continue;
1265 : }
1266 : return false;
1267 : }
1268 : }
1269 19103556 : else if (gimple_code (stmt) == GIMPLE_PHI)
1270 : {
1271 19261690 : rhs_code = ERROR_MARK;
1272 : phi_p = true;
1273 : }
1274 : else
1275 : {
1276 18096829 : rhs_code = gimple_assign_rhs_code (stmt);
1277 18096829 : ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
1278 : }
1279 :
1280 18096829 : if (!ldst_p
1281 : && !phi_p
1282 13226157 : && rhs_code.is_tree_code ()
1283 13168415 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
1284 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
1285 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
1286 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
1287 241049 : && rhs_code != VIEW_CONVERT_EXPR
1288 : && rhs_code != CALL_EXPR
1289 : && rhs_code != BIT_FIELD_REF
1290 18096829 : && rhs_code != SSA_NAME)
1291 : {
1292 48066 : if (dump_enabled_p ())
1293 98 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1294 : "Build SLP failed: operation unsupported %G",
1295 : stmt);
1296 48066 : if (is_a <bb_vec_info> (vinfo))
1297 : {
1298 47921 : matches[i] = -2;
1299 47921 : continue;
1300 : }
1301 : return false;
1302 : }
1303 :
1304 : /* Non-grouped store or load. */
1305 19261690 : if (ldst_p
1306 5076872 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
1307 960454 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1308 677161 : && rhs_code != CFN_GATHER_LOAD
1309 677161 : && rhs_code != CFN_MASK_GATHER_LOAD
1310 677161 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1311 677161 : && rhs_code != CFN_SCATTER_STORE
1312 677161 : && rhs_code != CFN_MASK_SCATTER_STORE
1313 677161 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1314 677161 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1315 : /* Not grouped loads are handled as externals for BB
1316 : vectorization. Treat them as not vectorizable. */
1317 19914396 : && is_a <bb_vec_info> (vinfo))
1318 : {
1319 : /* Not grouped load. */
1320 0 : if (dump_enabled_p ())
1321 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1322 : "Build SLP failed: not grouped load %G", stmt);
1323 0 : matches[i] = -2;
1324 0 : continue;
1325 : }
1326 :
1327 19261690 : if (!ldst_p && rhs_code == BIT_FIELD_REF)
1328 : {
1329 45285 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
1330 45285 : if (!is_a <bb_vec_info> (vinfo)
1331 45159 : || TREE_CODE (vec) != SSA_NAME
1332 45135 : || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (vec)
1333 : /* When the element types are not compatible we pun the
1334 : source to the target vectype which requires equal size. */
1335 90364 : || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
1336 42345 : || !types_compatible_p (TREE_TYPE (vectype),
1337 42345 : TREE_TYPE (TREE_TYPE (vec))))
1338 4101 : && !operand_equal_p (TYPE_SIZE (vectype),
1339 4101 : TYPE_SIZE (TREE_TYPE (vec)))))
1340 : {
1341 2440 : if (dump_enabled_p ())
1342 32 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1343 : "Build SLP failed: "
1344 : "BIT_FIELD_REF not supported\n");
1345 2440 : if (is_a <bb_vec_info> (vinfo))
1346 : {
1347 2314 : matches[i] = -2;
1348 2314 : continue;
1349 : }
1350 : return false;
1351 : }
1352 : }
1353 :
1354 : /* Check the operation. */
1355 19259250 : if (first_match == -1)
1356 : {
1357 6245045 : first_match = i;
1358 6245045 : first_stmt_info = stmt_info;
1359 6245045 : common_bb = gimple_bb (stmt_info->stmt);
1360 6245045 : first_lhs = lhs;
1361 6245045 : first_stmt_code = rhs_code;
1362 6245045 : first_stmt_ldst_p = ldst_p;
1363 6245045 : first_stmt_ldst_masklen_p = ldst_masklen_p;
1364 6245045 : first_stmt_phi_p = phi_p;
1365 6245045 : first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
1366 :
1367 : /* Shift arguments should be equal in all the packed stmts for a
1368 : vector shift with scalar shift operand. */
1369 6245045 : if (rhs_code == LSHIFT_EXPR
1370 6144771 : || rhs_code == RSHIFT_EXPR
1371 6042617 : || rhs_code == LROTATE_EXPR
1372 12287548 : || rhs_code == RROTATE_EXPR)
1373 : {
1374 : /* First see if we have a vector/vector shift. */
1375 203027 : if (!directly_supported_p (rhs_code, vectype, optab_vector))
1376 : {
1377 : /* No vector/vector shift, arrange for a vector/scalar
1378 : SLP layout. */
1379 189573 : need_same_oprnds = true;
1380 189573 : first_op1 = gimple_assign_rhs2 (stmt);
1381 : }
1382 : }
1383 6042018 : else if (rhs_code == WIDEN_LSHIFT_EXPR)
1384 : {
1385 0 : need_same_oprnds = true;
1386 0 : first_op1 = gimple_assign_rhs2 (stmt);
1387 : }
1388 6042018 : else if (rhs_code == CFN_DIV_POW2)
1389 : {
1390 0 : need_same_oprnds = true;
1391 0 : first_op1 = gimple_call_arg (call_stmt, 1);
1392 : }
1393 6042018 : else if (rhs_code == CFN_GOMP_SIMD_LANE)
1394 : {
1395 3654 : need_same_oprnds = true;
1396 3654 : first_op1 = gimple_call_arg (call_stmt, 1);
1397 : }
1398 : }
1399 : else
1400 : {
1401 13014205 : int comm_arg;
1402 13014582 : if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1403 : /* For SLP reduction groups the index isn't necessarily
1404 : uniform but only that of the first stmt matters. */
1405 2468 : && !(first_reduc_idx != -1
1406 2468 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1407 2468 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
1408 13014205 : && !(first_reduc_idx != -1
1409 1121 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1410 1121 : && (comm_arg = first_commutative_argument
1411 1121 : (rhs_code, TREE_TYPE (lhs))) >= 0
1412 : && (first_reduc_idx
1413 890 : == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
1414 : {
1415 377 : if (dump_enabled_p ())
1416 : {
1417 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1418 : "Build SLP failed: different reduc_idx "
1419 : "%d instead of %d in %G",
1420 : STMT_VINFO_REDUC_IDX (stmt_info),
1421 : first_reduc_idx, stmt);
1422 : }
1423 : /* Mismatch. */
1424 377 : continue;
1425 : }
1426 13013828 : if (!ldst_p
1427 13013828 : && two_operators
1428 10379838 : && first_stmt_code != rhs_code
1429 14531577 : && alt_stmt_code == ERROR_MARK)
1430 : alt_stmt_code = rhs_code;
1431 14518330 : if ((!ldst_p
1432 10379838 : && first_stmt_code != rhs_code
1433 1517749 : && (first_stmt_code != IMAGPART_EXPR
1434 202 : || rhs_code != REALPART_EXPR)
1435 1517749 : && (first_stmt_code != REALPART_EXPR
1436 735 : || rhs_code != IMAGPART_EXPR)
1437 : /* Handle mismatches in plus/minus by computing both
1438 : and merging the results. */
1439 1517749 : && !((((first_stmt_code == PLUS_EXPR
1440 1415605 : || first_stmt_code == MINUS_EXPR)
1441 126441 : && (alt_stmt_code == PLUS_EXPR
1442 116496 : || alt_stmt_code == MINUS_EXPR))
1443 1491089 : || ((first_stmt_code == CFN_FMA
1444 1491087 : || first_stmt_code == CFN_FMS)
1445 2 : && (alt_stmt_code == CFN_FMA
1446 2 : || alt_stmt_code == CFN_FMS)))
1447 26662 : && rhs_code == alt_stmt_code)
1448 1533180 : && !(first_stmt_code.is_tree_code ()
1449 1401842 : && rhs_code.is_tree_code ()
1450 1287085 : && (TREE_CODE_CLASS (tree_code (first_stmt_code))
1451 : == tcc_comparison)
1452 141564 : && (swap_tree_comparison (tree_code (first_stmt_code))
1453 141564 : == tree_code (rhs_code))
1454 : && (first_reduc_idx == -1
1455 0 : || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
1456 : || (ldst_p
1457 5267980 : && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1458 2633990 : != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
1459 : || (ldst_p
1460 2579733 : && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1461 2579733 : != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
1462 11509472 : || first_stmt_ldst_p != ldst_p
1463 11509334 : || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
1464 24523154 : || first_stmt_phi_p != phi_p)
1465 : {
1466 1504502 : if (dump_enabled_p ())
1467 : {
1468 3672 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1469 : "Build SLP failed: different operation "
1470 : "in stmt %G", stmt);
1471 3672 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1472 : "original stmt %G", first_stmt_info->stmt);
1473 : }
1474 : /* Mismatch. */
1475 1504502 : continue;
1476 : }
1477 :
1478 11528509 : if (!ldst_p
1479 8929754 : && first_stmt_code == BIT_FIELD_REF
1480 11539455 : && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
1481 30129 : != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
1482 : {
1483 19183 : if (dump_enabled_p ())
1484 116 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1485 : "Build SLP failed: different BIT_FIELD_REF "
1486 : "arguments in %G", stmt);
1487 : /* Mismatch. */
1488 19183 : continue;
1489 : }
1490 :
1491 11490143 : if (call_stmt
1492 89121 : && first_stmt_code != CFN_MASK_LOAD
1493 11578862 : && first_stmt_code != CFN_MASK_STORE)
1494 : {
1495 88408 : if (!is_a <gcall *> (stmts[0]->stmt)
1496 88408 : || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
1497 : call_stmt, true))
1498 : {
1499 69200 : if (dump_enabled_p ())
1500 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1501 : "Build SLP failed: different calls in %G",
1502 : stmt);
1503 : /* Mismatch. */
1504 69200 : continue;
1505 : }
1506 : }
1507 :
1508 11463566 : if (phi_p
1509 11420943 : && (gimple_bb (first_stmt_info->stmt)
1510 200574 : != gimple_bb (stmt_info->stmt)))
1511 : {
1512 42623 : if (dump_enabled_p ())
1513 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1514 : "Build SLP failed: different BB for PHI %G",
1515 : stmt);
1516 : /* Mismatch. */
1517 42623 : continue;
1518 : }
1519 :
1520 11378320 : if (need_same_oprnds)
1521 : {
1522 63773 : tree other_op1 = gimple_arg (stmt, 1);
1523 63773 : if (!operand_equal_p (first_op1, other_op1, 0))
1524 : {
1525 11349 : if (dump_enabled_p ())
1526 195 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1527 : "Build SLP failed: different shift "
1528 : "arguments in %G", stmt);
1529 : /* Mismatch. */
1530 11349 : continue;
1531 : }
1532 : }
1533 :
1534 11367713 : if (first_lhs
1535 11366971 : && lhs
1536 11366971 : && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
1537 : {
1538 742 : if (dump_enabled_p ())
1539 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1540 : "Build SLP failed: different vector type "
1541 : "in %G", stmt);
1542 : /* Mismatch. */
1543 742 : continue;
1544 : }
1545 :
1546 : /* We need to ensure all stmts are in the same BB when one stmt could
1547 : trap. */
1548 11366229 : if (trapping_stmt || gimple_could_trap_p (stmt))
1549 : {
1550 953963 : gcc_assert (!trapping_stmt || common_bb);
1551 953963 : if (gimple_bb (stmt) != common_bb)
1552 : {
1553 6259 : if (dump_enabled_p ())
1554 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1555 : "Build SLP failed: not all stmts in same "
1556 : "BB but possibly trapping operation in %G",
1557 : trapping_stmt ? trapping_stmt : stmt);
1558 6259 : continue;
1559 : }
1560 : }
1561 : }
1562 :
1563 : /* Grouped store or load. */
1564 17605015 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1565 : {
1566 4056548 : gcc_assert (ldst_p);
1567 4056548 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
1568 : {
1569 : /* Store. */
1570 3036375 : gcc_assert (rhs_code == CFN_MASK_STORE
1571 : || REFERENCE_CLASS_P (lhs)
1572 : || DECL_P (lhs));
1573 : }
1574 : else
1575 : {
1576 : /* Load. */
1577 1020173 : first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
1578 1020173 : if (prev_first_load)
1579 : {
1580 : /* Check that there are no loads from different interleaving
1581 : chains in the same node. */
1582 450507 : if (prev_first_load != first_load)
1583 : {
1584 68110 : if (dump_enabled_p ())
1585 2296 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
1586 : vect_location,
1587 : "Build SLP failed: different "
1588 : "interleaving chains in one node %G",
1589 : stmt);
1590 : /* Mismatch. */
1591 68110 : continue;
1592 : }
1593 : }
1594 : else
1595 : prev_first_load = first_load;
1596 : }
1597 : }
1598 : /* Non-grouped store or load. */
1599 13548467 : else if (ldst_p)
1600 : {
1601 960135 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1602 676842 : && rhs_code != CFN_GATHER_LOAD
1603 : && rhs_code != CFN_MASK_GATHER_LOAD
1604 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1605 : && rhs_code != CFN_SCATTER_STORE
1606 : && rhs_code != CFN_MASK_SCATTER_STORE
1607 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1608 1636977 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1609 : {
1610 652406 : gcc_assert (!is_a <bb_vec_info> (vinfo));
1611 : /* For loop vectorization we can handle splats the same we
1612 : handle single element interleaving. Likewise we can handle
1613 : a collection of invariant refs. */
1614 727256 : if (stmt_info != first_stmt_info
1615 652692 : && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
1616 286 : && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
1617 : (first_stmt_info)))))
1618 : {
1619 : /* Not grouped load. */
1620 74850 : if (dump_enabled_p ())
1621 160 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1622 : "Build SLP failed: not grouped load %G",
1623 : stmt);
1624 74850 : continue;
1625 : }
1626 : }
1627 : }
1628 : /* Not memory operation. */
1629 : else
1630 : {
1631 12588332 : if (rhs_code == COND_EXPR)
1632 : {
1633 65751 : tree cond_expr = gimple_assign_rhs1 (stmt);
1634 65751 : enum tree_code cond_code = TREE_CODE (cond_expr);
1635 65751 : enum tree_code swap_code = ERROR_MARK;
1636 65751 : enum tree_code invert_code = ERROR_MARK;
1637 :
1638 65751 : if (i == (unsigned)first_match)
1639 55382 : first_cond_code = TREE_CODE (cond_expr);
1640 10369 : else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
1641 : {
1642 0 : bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
1643 0 : swap_code = swap_tree_comparison (cond_code);
1644 0 : invert_code = invert_tree_comparison (cond_code, honor_nans);
1645 : }
1646 :
1647 65751 : if (first_cond_code == cond_code)
1648 : ;
1649 : /* Isomorphic can be achieved by swapping. */
1650 0 : else if (first_cond_code == swap_code)
1651 0 : swap[i] = 1;
1652 : /* Isomorphic can be achieved by inverting. */
1653 0 : else if (first_cond_code == invert_code)
1654 0 : swap[i] = 2;
1655 : else
1656 : {
1657 0 : if (dump_enabled_p ())
1658 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1659 : "Build SLP failed: different"
1660 : " operation %G", stmt);
1661 : /* Mismatch. */
1662 0 : continue;
1663 : }
1664 : }
1665 :
1666 12588332 : if (i != (unsigned) first_match
1667 8786169 : && first_stmt_code != rhs_code
1668 67663 : && first_stmt_code.is_tree_code ()
1669 67661 : && rhs_code.is_tree_code ()
1670 67661 : && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
1671 12629882 : && (swap_tree_comparison ((tree_code)first_stmt_code)
1672 41550 : == (tree_code)rhs_code))
1673 41550 : swap[i] = 1;
1674 :
1675 12588332 : if (i != (unsigned) first_match
1676 8786169 : && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1677 1791 : && first_reduc_idx != -1
1678 1791 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1679 1791 : && rhs_code.is_tree_code ()
1680 1787 : && commutative_tree_code (tree_code (rhs_code))
1681 12590117 : && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
1682 1785 : swap[i] = 1;
1683 : }
1684 :
1685 : /* We need to ensure all stmts are in the same BB when one stmt could
1686 : trap. Not matching stmts are not relevant, so exclude those. */
1687 17462055 : if (!trapping_stmt && gimple_could_trap_p (stmt))
1688 : trapping_stmt = stmt;
1689 17462055 : if (common_bb != gimple_bb (stmt))
1690 : {
1691 63423 : common_bb = NULL;
1692 63423 : gcc_assert (!trapping_stmt);
1693 : }
1694 :
1695 17462055 : matches[i] = first_match;
1696 : }
1697 :
1698 : /* Record if we allowed two distinct operations for the SLP node. */
1699 6346235 : if (((first_stmt_code == PLUS_EXPR
1700 5829095 : || first_stmt_code == MINUS_EXPR)
1701 626770 : && (alt_stmt_code == PLUS_EXPR
1702 619468 : || alt_stmt_code == MINUS_EXPR))
1703 12674876 : || ((first_stmt_code == CFN_FMA
1704 6328592 : || first_stmt_code == CFN_FMS)
1705 49 : && (alt_stmt_code == CFN_FMA
1706 49 : || alt_stmt_code == CFN_FMS)))
1707 17596 : *two_operators = true;
1708 :
1709 6346235 : if (first_match == -1)
1710 : {
1711 : /* If there was no useful stmt, fail. Can only happen during
1712 : the first sweep. */
1713 101190 : gcc_assert (*start_i == 0);
1714 101190 : *start_i = stmts.length () - 1;
1715 101190 : return false;
1716 : }
1717 :
1718 6245045 : *start_i = first_match;
1719 6245045 : return true;
1720 : }
1721 :
1722 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1723 : permutation or are of unsupported types of operation. Return
1724 : true if they are, otherwise return false and indicate in *MATCHES
1725 : the groups of isomorphic stmts. See vect_build_slp_tree_3 for
1726 : details. *TWO_OPERATORS is for the first isomorphic group,
1727 : knowledge whether following isomorphic groups have one or two operators
1728 : is not retained. */
1729 :
1730 : static bool
1731 6252099 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
1732 : vec<stmt_vec_info> stmts, match_elt_t *matches,
1733 : bool *two_operators, tree *node_vectype)
1734 : {
1735 6252099 : stmt_vec_info first_stmt_info = stmts[0];
1736 6252099 : unsigned int group_size = stmts.length ();
1737 6252099 : tree vectype;
1738 6252099 : if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
1739 : group_size))
1740 : {
1741 : /* Fatal mismatch. */
1742 257784 : matches[0] = -1;
1743 257784 : return false;
1744 : }
1745 5994315 : if (is_a <bb_vec_info> (vinfo)
1746 5994315 : && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
1747 : {
1748 375474 : if (dump_enabled_p ())
1749 310 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1750 : "Build SLP failed: not using single lane "
1751 : "vector type %T\n", vectype);
1752 375474 : matches[0] = -1;
1753 375474 : return false;
1754 : }
1755 : /* Check nunits required but continue analysis, producing matches[]
1756 : as if nunits was not an issue. This allows splitting of groups
1757 : to happen. */
1758 5618841 : unsigned HOST_WIDE_INT const_nunits = 0;
1759 5618841 : if (vectype
1760 5618811 : && is_a <bb_vec_info> (vinfo)
1761 11237652 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
1762 : {
1763 210390 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
1764 210390 : || const_nunits > group_size)
1765 : {
1766 0 : if (dump_enabled_p ())
1767 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1768 : "Build SLP failed: unrolling required "
1769 : "in basic block SLP\n");
1770 : /* Fatal mismatch. */
1771 0 : matches[0] = -1;
1772 0 : return false;
1773 : }
1774 : }
1775 :
1776 5618841 : gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
1777 5618841 : *node_vectype = vectype;
1778 :
1779 5618841 : unsigned start_i = 0;
1780 5618841 : if (!vect_build_slp_tree_3 (vinfo, stmts, matches, swap, two_operators,
1781 : vectype, &start_i))
1782 : {
1783 : /* Fatal mismatch. */
1784 101461 : matches[0] = -1;
1785 101461 : return false;
1786 : }
1787 5517380 : gcc_assert (matches[start_i] == (int)start_i);
1788 : /* Discover further isomorphic groups. */
1789 17615511 : for (start_i = start_i + 1; start_i < group_size; ++start_i)
1790 12098131 : if (matches[start_i] == -1)
1791 : {
1792 727665 : unsigned prev_start_i = start_i;
1793 727665 : bool tem_two_operators;
1794 727665 : bool res = vect_build_slp_tree_3 (vinfo, stmts, matches, swap,
1795 : &tem_two_operators,
1796 : vectype, &start_i);
1797 727665 : gcc_assert (res && matches[start_i] == (int)prev_start_i);
1798 : }
1799 :
1800 : bool res = true;
1801 23210581 : for (unsigned i = 0; i < group_size; ++i)
1802 17693201 : if (matches[i] != 0)
1803 1845102 : res = false;
1804 : if (!res)
1805 : return false;
1806 :
1807 : return true;
1808 : }
1809 :
1810 : /* Traits for the hash_set to record failed SLP builds for a stmt set.
1811 : Note we never remove apart from at destruction time so we do not
1812 : need a special value for deleted that differs from empty. */
1813 : struct bst_traits
1814 : {
1815 : typedef vec <stmt_vec_info> value_type;
1816 : typedef vec <stmt_vec_info> compare_type;
1817 : static inline hashval_t hash (value_type);
1818 : static inline bool equal (value_type existing, value_type candidate);
1819 517810852 : static inline bool is_empty (value_type x) { return !x.exists (); }
1820 115822826 : static inline bool is_deleted (value_type x) { return !x.exists (); }
1821 : static const bool empty_zero_p = true;
1822 0 : static inline void mark_empty (value_type &x) { x.release (); }
1823 : static inline void mark_deleted (value_type &x) { x.release (); }
1824 9988706 : static inline void remove (value_type &x) { x.release (); }
1825 : };
1826 : inline hashval_t
1827 100959004 : bst_traits::hash (value_type x)
1828 : {
1829 100959004 : inchash::hash h;
1830 439013620 : for (unsigned i = 0; i < x.length (); ++i)
1831 338054616 : h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
1832 100959004 : return h.end ();
1833 : }
1834 : inline bool
1835 88305405 : bst_traits::equal (value_type existing, value_type candidate)
1836 : {
1837 264916215 : if (existing.length () != candidate.length ())
1838 : return false;
1839 89572227 : for (unsigned i = 0; i < existing.length (); ++i)
1840 84900639 : if (existing[i] != candidate[i])
1841 : return false;
1842 : return true;
1843 : }
1844 :
1845 : typedef hash_map <vec <stmt_vec_info>, slp_tree,
1846 : simple_hashmap_traits <bst_traits, slp_tree> >
1847 : scalar_stmts_to_slp_tree_map_t;
1848 :
1849 : /* Release BST_MAP. */
1850 :
1851 : static void
1852 1947323 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
1853 : {
1854 : /* The map keeps a reference on SLP nodes built, release that. */
1855 11936029 : for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
1856 11936029 : it != bst_map->end (); ++it)
1857 9988706 : if ((*it).second)
1858 9988706 : vect_free_slp_tree ((*it).second);
1859 3894646 : delete bst_map;
1860 1947323 : }
1861 :
1862 : /* ??? This was std::pair<std::pair<tree_code, vect_def_type>, tree>
1863 : but then vec::insert does memmove and that's not compatible with
1864 : std::pair. */
1865 : struct chain_op_t
1866 : {
1867 4391948 : chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
1868 4391948 : : code (code_), dt (dt_), op (op_) {}
1869 : tree_code code;
1870 : vect_def_type dt;
1871 : tree op;
1872 : };
1873 :
1874 : /* Comparator for sorting associatable chains. */
1875 :
1876 : static int
1877 12883570 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
1878 : {
1879 12883570 : auto *op1 = (const chain_op_t *) op1_;
1880 12883570 : auto *op2 = (const chain_op_t *) op2_;
1881 12883570 : if (op1->dt != op2->dt)
1882 2032039 : return (int)op1->dt - (int)op2->dt;
1883 10851531 : return (int)op1->code - (int)op2->code;
1884 : }
1885 :
1886 : /* Linearize the associatable expression chain at START with the
1887 : associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
1888 : filling CHAIN with the result and using WORKLIST as intermediate storage.
1889 : CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
1890 : or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation
1891 : stmts, starting with START. When ALLOW_ALT_CODE is false, do not
1892 : follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf). */
1893 :
1894 : static void
1895 1891233 : vect_slp_linearize_chain (vec_info *vinfo,
1896 : vec<std::pair<tree_code, gimple *> > &worklist,
1897 : vec<chain_op_t> &chain,
1898 : enum tree_code code, gimple *start,
1899 : gimple *&code_stmt, gimple *&alt_code_stmt,
1900 : vec<gimple *> *chain_stmts,
1901 : bool allow_alt_code = true)
1902 : {
1903 : /* For each lane linearize the addition/subtraction (or other
1904 : uniform associatable operation) expression tree. */
1905 1891233 : worklist.safe_push (std::make_pair (code, start));
1906 4391948 : while (!worklist.is_empty ())
1907 : {
1908 2500715 : auto entry = worklist.pop ();
1909 2500715 : gassign *stmt = as_a <gassign *> (entry.second);
1910 2500715 : enum tree_code in_code = entry.first;
1911 5001430 : enum tree_code this_code = gimple_assign_rhs_code (stmt);
1912 : /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE. */
1913 2500715 : if (!code_stmt
1914 2500715 : && gimple_assign_rhs_code (stmt) == code)
1915 1584679 : code_stmt = stmt;
1916 916036 : else if (!alt_code_stmt
1917 916036 : && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
1918 363297 : alt_code_stmt = stmt;
1919 2500715 : if (chain_stmts)
1920 2420293 : chain_stmts->safe_push (stmt);
1921 7502145 : for (unsigned opnum = 1; opnum <= 2; ++opnum)
1922 : {
1923 5001430 : tree op = gimple_op (stmt, opnum);
1924 5001430 : vect_def_type dt;
1925 5001430 : stmt_vec_info def_stmt_info;
1926 5001430 : bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
1927 5001430 : gcc_assert (res);
1928 5001430 : if (dt == vect_internal_def
1929 5001430 : && is_pattern_stmt_p (def_stmt_info))
1930 11045 : op = gimple_get_lhs (def_stmt_info->stmt);
1931 5001430 : gimple *use_stmt;
1932 5001430 : use_operand_p use_p;
1933 5001430 : if (dt == vect_internal_def
1934 4545693 : && single_imm_use (op, &use_p, &use_stmt)
1935 2898485 : && is_gimple_assign (def_stmt_info->stmt)
1936 7704238 : && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
1937 2093663 : || (allow_alt_code
1938 56815 : && code == PLUS_EXPR
1939 36956 : && (gimple_assign_rhs_code (def_stmt_info->stmt)
1940 : == MINUS_EXPR))))
1941 : {
1942 609482 : tree_code op_def_code = this_code;
1943 609482 : if (op_def_code == MINUS_EXPR && opnum == 1)
1944 54870 : op_def_code = PLUS_EXPR;
1945 609482 : if (in_code == MINUS_EXPR)
1946 222 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1947 609482 : worklist.safe_push (std::make_pair (op_def_code,
1948 609482 : def_stmt_info->stmt));
1949 : }
1950 : else
1951 : {
1952 4391948 : tree_code op_def_code = this_code;
1953 4391948 : if (op_def_code == MINUS_EXPR && opnum == 1)
1954 308546 : op_def_code = PLUS_EXPR;
1955 4391948 : if (in_code == MINUS_EXPR)
1956 4410 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1957 4391948 : chain.safe_push (chain_op_t (op_def_code, dt, op));
1958 : }
1959 : }
1960 : }
1961 1891233 : }
1962 :
1963 : /* Distance from the node currently being discovered to the closest upthread
1964 : commutative operation whose operand-zero discovery may still be fixed by
1965 : retrying with swapped operands, or -1U if there is none. */
1966 :
1967 : static unsigned least_upthread_swappable_op_distance = -1U;
1968 :
1969 : static slp_tree
1970 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
1971 : vec<stmt_vec_info> stmts,
1972 : match_elt_t *matches, unsigned *limit,
1973 : unsigned *tree_size,
1974 : scalar_stmts_to_slp_tree_map_t *bst_map);
1975 :
1976 : static slp_tree
1977 6766436 : vect_build_slp_tree (vec_info *vinfo,
1978 : vec<stmt_vec_info> stmts,
1979 : match_elt_t *matches, unsigned *limit, unsigned *tree_size,
1980 : scalar_stmts_to_slp_tree_map_t *bst_map)
1981 : {
1982 6766436 : unsigned int group_size = stmts.length ();
1983 6766436 : if (slp_tree *leader = bst_map->get (stmts))
1984 : {
1985 508524 : if (dump_enabled_p ())
1986 18853 : dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
1987 18853 : !(*leader)->failed ? "" : "failed ",
1988 : (void *) *leader);
1989 508524 : if (!(*leader)->failed)
1990 : {
1991 459853 : SLP_TREE_REF_COUNT (*leader)++;
1992 459853 : stmts.release ();
1993 459853 : return *leader;
1994 : }
1995 48671 : memcpy (matches, (*leader)->failed, sizeof (match_elt_t) * group_size);
1996 48671 : return NULL;
1997 : }
1998 :
1999 : /* Single-lane SLP doesn't have the chance of run-away, do not account
2000 : it to the limit. */
2001 6257912 : if (stmts.length () > 1)
2002 : {
2003 3450749 : if (*limit == 0)
2004 : {
2005 1355 : if (dump_enabled_p ())
2006 15 : dump_printf_loc (MSG_NOTE, vect_location,
2007 : "SLP discovery limit exceeded\n");
2008 1355 : memset (matches, -1, sizeof (match_elt_t) * group_size);
2009 1355 : return NULL;
2010 : }
2011 3449394 : --*limit;
2012 : }
2013 :
2014 : /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
2015 : so we can pick up backedge destinations during discovery. */
2016 6256557 : slp_tree res = new _slp_tree;
2017 6256557 : SLP_TREE_DEF_TYPE (res) = vect_internal_def;
2018 6256557 : SLP_TREE_SCALAR_STMTS (res) = stmts;
2019 6256557 : bst_map->put (stmts.copy (), res);
2020 :
2021 6256557 : if (dump_enabled_p ())
2022 160196 : dump_printf_loc (MSG_NOTE, vect_location,
2023 : "starting SLP discovery for node %p\n", (void *) res);
2024 :
2025 6256557 : slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
2026 : matches, limit, tree_size, bst_map);
2027 6256557 : if (!res_)
2028 : {
2029 2188564 : if (dump_enabled_p ())
2030 8267 : dump_printf_loc (MSG_NOTE, vect_location,
2031 : "SLP discovery for node %p failed\n", (void *) res);
2032 : /* Mark the node invalid so we can detect those when still in use
2033 : as backedge destinations. */
2034 2188564 : SLP_TREE_SCALAR_STMTS (res) = vNULL;
2035 2188564 : SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
2036 2188564 : res->failed = XNEWVEC (match_elt_t, group_size);
2037 2188564 : if (flag_checking)
2038 : {
2039 : unsigned i;
2040 3821239 : for (i = 0; i < group_size; ++i)
2041 3821239 : if (matches[i] != 0)
2042 : break;
2043 2188564 : gcc_assert (i < group_size);
2044 : }
2045 2188564 : memcpy (res->failed, matches, sizeof (match_elt_t) * group_size);
2046 : }
2047 : else
2048 : {
2049 4067993 : if (dump_enabled_p ())
2050 151929 : dump_printf_loc (MSG_NOTE, vect_location,
2051 : "SLP discovery for node %p succeeded\n",
2052 : (void *) res);
2053 4067993 : gcc_assert (res_ == res);
2054 : /* Keep a reference for the bst_map use. */
2055 4067993 : SLP_TREE_REF_COUNT (res)++;
2056 : /* For BB vectorization nodes that need splitting should be failed. */
2057 4067993 : gcc_assert (!is_a <bb_vec_info> (vinfo)
2058 : || multiple_p (group_size,
2059 : TYPE_VECTOR_SUBPARTS (res->vectype)));
2060 : }
2061 : return res_;
2062 : }
2063 :
2064 : /* Helper for building an associated SLP node chain. */
2065 :
2066 : static void
2067 162 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
2068 : slp_tree op0, slp_tree op1,
2069 : stmt_vec_info oper1, stmt_vec_info oper2,
2070 : vec<std::pair<unsigned, unsigned> > lperm)
2071 : {
2072 162 : unsigned group_size = SLP_TREE_LANES (op1);
2073 :
2074 162 : slp_tree child1 = new _slp_tree;
2075 162 : SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
2076 162 : SLP_TREE_VECTYPE (child1) = vectype;
2077 162 : SLP_TREE_LANES (child1) = group_size;
2078 162 : SLP_TREE_CHILDREN (child1).create (2);
2079 162 : SLP_TREE_CHILDREN (child1).quick_push (op0);
2080 162 : SLP_TREE_CHILDREN (child1).quick_push (op1);
2081 162 : SLP_TREE_REPRESENTATIVE (child1) = oper1;
2082 :
2083 162 : slp_tree child2 = new _slp_tree;
2084 162 : SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
2085 162 : SLP_TREE_VECTYPE (child2) = vectype;
2086 162 : SLP_TREE_LANES (child2) = group_size;
2087 162 : SLP_TREE_CHILDREN (child2).create (2);
2088 162 : SLP_TREE_CHILDREN (child2).quick_push (op0);
2089 162 : SLP_TREE_REF_COUNT (op0)++;
2090 162 : SLP_TREE_CHILDREN (child2).quick_push (op1);
2091 162 : SLP_TREE_REF_COUNT (op1)++;
2092 162 : SLP_TREE_REPRESENTATIVE (child2) = oper2;
2093 :
2094 162 : SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
2095 162 : SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
2096 162 : SLP_TREE_VECTYPE (perm) = vectype;
2097 162 : SLP_TREE_LANES (perm) = group_size;
2098 162 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
2099 162 : SLP_TREE_LANE_PERMUTATION (perm) = lperm;
2100 162 : SLP_TREE_CHILDREN (perm).quick_push (child1);
2101 162 : SLP_TREE_CHILDREN (perm).quick_push (child2);
2102 162 : }
2103 :
2104 : /* For isomorphic matches[], indicate a splitting point according to
2105 : CONST_NUNITS. */
2106 :
2107 : static void
2108 123354 : force_split_matches (match_elt_t *matches, unsigned group_size,
2109 : unsigned const_nunits)
2110 : {
2111 123354 : if (dump_enabled_p ())
2112 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2113 : "Build SLP failed: unrolling required "
2114 : "in basic block SLP\n");
2115 :
2116 : /* With constant vector elements simulate a mismatch at the
2117 : point we need to split. But indicate the tail is isomorphic. */
2118 123354 : unsigned tail = group_size & (const_nunits - 1);
2119 271730 : for (unsigned i = group_size - tail; i < group_size; ++i)
2120 148376 : matches[i] = (int)(group_size - tail);
2121 123354 : }
2122 :
2123 : /* Recursively build an SLP tree starting from NODE.
2124 : Fail (and return a value not equal to zero) if def-stmts are not
2125 : isomorphic, require data permutation or are of unsupported types of
2126 : operation. Otherwise, return 0.
2127 : The value returned is the depth in the SLP tree where a mismatch
2128 : was found. */
2129 :
2130 : static slp_tree
2131 6256557 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
2132 : vec<stmt_vec_info> stmts,
2133 : match_elt_t *matches, unsigned *limit,
2134 : unsigned *tree_size,
2135 : scalar_stmts_to_slp_tree_map_t *bst_map)
2136 : {
2137 6256557 : unsigned int group_size = stmts.length ();
2138 6256557 : unsigned nops, i, this_tree_size = 0;
2139 :
2140 6256557 : matches[0] = -1;
2141 :
2142 6256557 : stmt_vec_info stmt_info = stmts[0];
2143 6256557 : if (!is_a<gcall *> (stmt_info->stmt)
2144 : && !is_a<gassign *> (stmt_info->stmt)
2145 : && !is_a<gphi *> (stmt_info->stmt))
2146 : return NULL;
2147 :
2148 6256387 : nops = gimple_num_args (stmt_info->stmt);
2149 6256387 : if (const int *map = vect_get_operand_map (stmt_info))
2150 40466 : nops = map[0];
2151 :
2152 : /* If the SLP node is a PHI (induction or reduction), terminate
2153 : the recursion. */
2154 6256387 : bool *skip_args = XALLOCAVEC (bool, nops);
2155 6256387 : memset (skip_args, 0, sizeof (bool) * nops);
2156 6256387 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2157 2988535 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
2158 : {
2159 325537 : tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
2160 325537 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
2161 : group_size);
2162 325537 : if (!vectype)
2163 : {
2164 4282 : if (dump_enabled_p ())
2165 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2166 : "Build SLP failed: unsupported data-type in %G\n",
2167 : stmt_info->stmt);
2168 : return NULL;
2169 : }
2170 :
2171 321255 : vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
2172 321255 : if (def_type == vect_induction_def)
2173 : {
2174 : /* Induction PHIs are not cycles but walk the initial
2175 : value. Only for inner loops through, for outer loops
2176 : we need to pick up the value from the actual PHIs
2177 : to more easily support peeling and epilogue vectorization. */
2178 214606 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2179 214606 : if (!nested_in_vect_loop_p (loop, stmt_info))
2180 213764 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2181 : else
2182 : loop = loop->inner;
2183 214606 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2184 : }
2185 106649 : else if (def_type == vect_reduction_def
2186 : || def_type == vect_double_reduction_def
2187 : || def_type == vect_nested_cycle
2188 106649 : || def_type == vect_first_order_recurrence)
2189 : {
2190 : /* Else def types have to match. */
2191 : stmt_vec_info other_info;
2192 : bool all_same = true;
2193 228896 : FOR_EACH_VEC_ELT (stmts, i, other_info)
2194 : {
2195 123557 : if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
2196 6256557 : return NULL;
2197 123551 : if (other_info != stmt_info)
2198 15110 : all_same = false;
2199 : }
2200 105339 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2201 : /* Reduction initial values are not explicitly represented. */
2202 105339 : if (def_type != vect_first_order_recurrence
2203 105339 : && gimple_bb (stmt_info->stmt) == loop->header)
2204 102095 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2205 : /* Reduction chain backedge defs are filled manually.
2206 : ??? Need a better way to identify a SLP reduction chain PHI.
2207 : Or a better overall way to SLP match those. */
2208 105339 : if (stmts.length () > 1
2209 105339 : && all_same && def_type == vect_reduction_def)
2210 660 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2211 : }
2212 1304 : else if (def_type != vect_internal_def)
2213 : return NULL;
2214 : }
2215 :
2216 :
2217 6252099 : bool two_operators = false;
2218 6252099 : unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
2219 6252099 : tree vectype = NULL_TREE;
2220 6252099 : if (!vect_build_slp_tree_1 (vinfo, swap, stmts, matches, &two_operators,
2221 : &vectype))
2222 : return NULL;
2223 :
2224 4687785 : bool soft_fail
2225 4687785 : = (is_a <bb_vec_info> (vinfo)
2226 4687785 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)));
2227 :
2228 : /* If the SLP node is a load, terminate the recursion unless masked. */
2229 4687785 : if (STMT_VINFO_DATA_REF (stmt_info)
2230 2266668 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
2231 : {
2232 972208 : if (soft_fail)
2233 : {
2234 979 : force_split_matches (matches, group_size,
2235 979 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2236 979 : return NULL;
2237 : }
2238 971229 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2239 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
2240 : else
2241 : {
2242 947993 : (*tree_size)++;
2243 947993 : node = vect_create_new_slp_node (node, stmts, 0);
2244 947993 : SLP_TREE_VECTYPE (node) = vectype;
2245 : /* And compute the load permutation. Whether it is actually
2246 : a permutation depends on the unrolling factor which is
2247 : decided later. */
2248 947993 : vec<unsigned> load_permutation;
2249 947993 : int j;
2250 947993 : stmt_vec_info load_info;
2251 947993 : load_permutation.create (group_size);
2252 947993 : stmt_vec_info first_stmt_info
2253 947993 : = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2254 947993 : ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
2255 947993 : bool any_permute = false;
2256 2285873 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
2257 : {
2258 1337880 : int load_place;
2259 1337880 : if (! load_info)
2260 : {
2261 41127 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2262 : load_place = j;
2263 : else
2264 : load_place = 0;
2265 : }
2266 1296753 : else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2267 754762 : load_place = vect_get_place_in_interleaving_chain
2268 754762 : (load_info, first_stmt_info);
2269 : else
2270 : /* Recognize the splat case as { 0, 0, ... } but make
2271 : sure to use the appropriate refs for collections
2272 : of invariant refs. */
2273 541991 : load_place = (load_info == stmt_info) ? 0 : j;
2274 796166 : gcc_assert (load_place != -1);
2275 1337880 : any_permute |= load_place != j;
2276 1337880 : load_permutation.quick_push (load_place);
2277 : }
2278 :
2279 947993 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
2280 : {
2281 3377 : gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
2282 3377 : bool has_gaps = false;
2283 3377 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2284 189 : for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
2285 846 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2286 657 : if (DR_GROUP_GAP (si) != 1)
2287 80 : has_gaps = true;
2288 : /* We cannot handle permuted masked loads directly, see
2289 : PR114375. We cannot handle strided masked loads or masked
2290 : loads with gaps unless the mask is uniform. */
2291 3377 : if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
2292 189 : && (DR_GROUP_GAP (first_stmt_info) != 0
2293 129 : || (has_gaps
2294 35 : && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
2295 6674 : || STMT_VINFO_STRIDED_P (stmt_info))
2296 : {
2297 108 : load_permutation.release ();
2298 108 : matches[0] = -1;
2299 944763 : return NULL;
2300 : }
2301 :
2302 : /* For permuted masked loads do an unpermuted masked load of
2303 : the whole group followed by a SLP permute node. */
2304 3269 : if (any_permute
2305 3269 : || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
2306 83 : && DR_GROUP_SIZE (first_stmt_info) != group_size))
2307 : {
2308 : /* Discover the whole unpermuted load. */
2309 39 : vec<stmt_vec_info> stmts2;
2310 39 : unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2311 68 : ? DR_GROUP_SIZE (first_stmt_info) : 1;
2312 39 : stmts2.create (dr_group_size);
2313 39 : stmts2.quick_grow_cleared (dr_group_size);
2314 39 : unsigned i = 0;
2315 39 : for (stmt_vec_info si = first_stmt_info;
2316 464 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2317 : {
2318 425 : if (si != first_stmt_info)
2319 1586 : for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
2320 1200 : stmts2[i++] = NULL;
2321 425 : stmts2[i++] = si;
2322 : }
2323 39 : match_elt_t *matches2
2324 39 : = XALLOCAVEC (match_elt_t, dr_group_size);
2325 39 : slp_tree unperm_load
2326 39 : = vect_build_slp_tree (vinfo, stmts2, matches2, limit,
2327 39 : &this_tree_size, bst_map);
2328 : /* When we are able to do the full masked load emit that
2329 : followed by 'node' being the desired final permutation. */
2330 39 : if (unperm_load)
2331 : {
2332 16 : gcc_assert
2333 : (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
2334 16 : lane_permutation_t lperm;
2335 16 : lperm.create (group_size);
2336 72 : for (unsigned j = 0; j < load_permutation.length (); ++j)
2337 40 : lperm.quick_push
2338 40 : (std::make_pair (0, load_permutation[j]));
2339 16 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2340 16 : SLP_TREE_CHILDREN (node).safe_push (unperm_load);
2341 16 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2342 16 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2343 16 : load_permutation.release ();
2344 16 : return node;
2345 : }
2346 23 : stmts2.release ();
2347 23 : load_permutation.release ();
2348 23 : matches[0] = -1;
2349 23 : return NULL;
2350 : }
2351 3230 : load_permutation.release ();
2352 : }
2353 : else
2354 : {
2355 944616 : if (!any_permute
2356 816582 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2357 1238756 : && group_size == DR_GROUP_SIZE (first_stmt_info))
2358 129616 : load_permutation.release ();
2359 944616 : SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
2360 944616 : return node;
2361 : }
2362 : }
2363 : }
2364 3715577 : else if (gimple_assign_single_p (stmt_info->stmt)
2365 2596008 : && !gimple_vuse (stmt_info->stmt)
2366 3724686 : && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
2367 : {
2368 3427 : if (soft_fail)
2369 : {
2370 22 : force_split_matches (matches, group_size,
2371 22 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2372 22 : return NULL;
2373 : }
2374 : /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
2375 : the same SSA name vector of a compatible type to vectype. */
2376 3405 : vec<std::pair<unsigned, unsigned> > lperm = vNULL;
2377 3405 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
2378 3405 : stmt_vec_info estmt_info;
2379 11867 : FOR_EACH_VEC_ELT (stmts, i, estmt_info)
2380 : {
2381 8609 : gassign *estmt = as_a <gassign *> (estmt_info->stmt);
2382 8609 : tree bfref = gimple_assign_rhs1 (estmt);
2383 8609 : HOST_WIDE_INT lane;
2384 8609 : if (!known_eq (bit_field_size (bfref),
2385 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
2386 17071 : || !constant_multiple_p (bit_field_offset (bfref),
2387 8609 : bit_field_size (bfref), &lane))
2388 : {
2389 147 : lperm.release ();
2390 147 : matches[0] = -1;
2391 147 : return NULL;
2392 : }
2393 8462 : lperm.safe_push (std::make_pair (0, (unsigned)lane));
2394 : }
2395 3258 : slp_tree vnode = vect_create_new_slp_node (vNULL);
2396 3258 : if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
2397 : /* ??? We record vectype here but we hide eventually necessary
2398 : punning and instead rely on code generation to materialize
2399 : VIEW_CONVERT_EXPRs as necessary. We instead should make
2400 : this explicit somehow. */
2401 1456 : SLP_TREE_VECTYPE (vnode) = vectype;
2402 : else
2403 : {
2404 : /* For different size but compatible elements we can still
2405 : use VEC_PERM_EXPR without punning. */
2406 1802 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
2407 : && types_compatible_p (TREE_TYPE (vectype),
2408 : TREE_TYPE (TREE_TYPE (vec))));
2409 1802 : SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
2410 : }
2411 3258 : auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
2412 3258 : unsigned HOST_WIDE_INT const_nunits;
2413 3258 : if (nunits.is_constant (&const_nunits))
2414 3258 : SLP_TREE_LANES (vnode) = const_nunits;
2415 3258 : SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
2416 : /* We are always building a permutation node even if it is an identity
2417 : permute to shield the rest of the vectorizer from the odd node
2418 : representing an actual vector without any scalar ops.
2419 : ??? We could hide it completely with making the permute node
2420 : external? */
2421 3258 : node = vect_create_new_slp_node (node, stmts, 1);
2422 3258 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2423 3258 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2424 3258 : SLP_TREE_VECTYPE (node) = vectype;
2425 3258 : SLP_TREE_CHILDREN (node).quick_push (vnode);
2426 3258 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2427 3258 : return node;
2428 : }
2429 : /* When discovery reaches an associatable operation see whether we can
2430 : improve that to match up lanes in a way superior to the operand
2431 : swapping code which at most looks at two defs.
2432 : ??? For BB vectorization we cannot do the brute-force search
2433 : for matching as we can succeed by means of builds from scalars
2434 : and have no good way to "cost" one build against another. */
2435 3712150 : else if (is_a <loop_vec_info> (vinfo)
2436 : /* Do not bother for single-lane SLP. */
2437 2138242 : && group_size > 1
2438 : /* ??? We don't handle !vect_internal_def defs below. */
2439 108683 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
2440 : /* ??? Do not associate a reduction, this will wreck REDUC_IDX
2441 : mapping as long as that exists on the stmt_info level. */
2442 89452 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1
2443 82617 : && is_gimple_assign (stmt_info->stmt)
2444 82290 : && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
2445 56721 : || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
2446 3739682 : && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
2447 16121 : || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
2448 13582 : && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
2449 : {
2450 : /* See if we have a chain of (mixed) adds or subtracts or other
2451 : associatable ops. */
2452 20664 : enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
2453 20664 : if (code == MINUS_EXPR)
2454 845 : code = PLUS_EXPR;
2455 20664 : stmt_vec_info other_op_stmt_info = NULL;
2456 20664 : stmt_vec_info op_stmt_info = NULL;
2457 20664 : unsigned chain_len = 0;
2458 20664 : auto_vec<chain_op_t> chain;
2459 20664 : auto_vec<std::pair<tree_code, gimple *> > worklist;
2460 20664 : auto_vec<vec<chain_op_t> > chains (group_size);
2461 20664 : auto_vec<slp_tree, 4> children;
2462 20664 : bool hard_fail = true;
2463 21667 : for (unsigned lane = 0; lane < group_size; ++lane)
2464 : {
2465 21336 : if (!stmts[lane])
2466 : {
2467 : /* ??? Below we require lane zero is present. */
2468 0 : if (lane == 0)
2469 : {
2470 : hard_fail = false;
2471 20333 : break;
2472 : }
2473 0 : chains.quick_push (vNULL);
2474 0 : continue;
2475 : }
2476 : /* For each lane linearize the addition/subtraction (or other
2477 : uniform associatable operation) expression tree. */
2478 21336 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
2479 21336 : vect_slp_linearize_chain (vinfo, worklist, chain, code,
2480 21336 : stmts[lane]->stmt, op_stmt, other_op_stmt,
2481 : NULL);
2482 21336 : if (!op_stmt_info && op_stmt)
2483 20010 : op_stmt_info = vinfo->lookup_stmt (op_stmt);
2484 21336 : if (!other_op_stmt_info && other_op_stmt)
2485 886 : other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
2486 21336 : if (chain.length () == 2)
2487 : {
2488 : /* In a chain of just two elements resort to the regular
2489 : operand swapping scheme. Likewise if we run into a
2490 : length mismatch process regularly as well as we did not
2491 : process the other lanes we cannot report a good hint what
2492 : lanes to try swapping in the parent. */
2493 : hard_fail = false;
2494 : break;
2495 : }
2496 1006 : else if (chain_len == 0)
2497 371 : chain_len = chain.length ();
2498 1270 : else if (chain.length () != chain_len)
2499 : {
2500 : /* ??? Here we could slip in magic to compensate with
2501 : neutral operands. */
2502 3 : matches[lane] = -1;
2503 3 : if (lane != group_size - 1)
2504 3 : matches[0] = -1;
2505 : break;
2506 : }
2507 1003 : chains.quick_push (chain.copy ());
2508 1003 : chain.truncate (0);
2509 : }
2510 41328 : if (chains.length () == group_size)
2511 : {
2512 : /* We cannot yet use SLP_TREE_CODE to communicate the operation. */
2513 331 : if (!op_stmt_info)
2514 : {
2515 3 : hard_fail = false;
2516 3 : goto out;
2517 : }
2518 : /* Now we have a set of chains with the same length. */
2519 : /* 1. pre-sort according to def_type and operation. */
2520 1219 : for (unsigned lane = 0; lane < group_size; ++lane)
2521 1782 : chains[lane].stablesort (dt_sort_cmp, vinfo);
2522 328 : if (dump_enabled_p ())
2523 : {
2524 155 : dump_printf_loc (MSG_NOTE, vect_location,
2525 : "pre-sorted chains of %s\n",
2526 : get_tree_code_name (code));
2527 655 : for (unsigned lane = 0; lane < group_size; ++lane)
2528 : {
2529 500 : if (!stmts[lane])
2530 0 : dump_printf (MSG_NOTE, "--");
2531 : else
2532 2250 : for (unsigned opnum = 0; opnum < chain_len; ++opnum)
2533 3500 : dump_printf (MSG_NOTE, "%s %T ",
2534 1750 : get_tree_code_name (chains[lane][opnum].code),
2535 1750 : chains[lane][opnum].op);
2536 500 : dump_printf (MSG_NOTE, "\n");
2537 : }
2538 : }
2539 : /* 2. try to build children nodes, associating as necessary. */
2540 : /* 2a. prepare and perform early checks to avoid eating into
2541 : discovery limit unnecessarily. */
2542 328 : vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
2543 1383 : for (unsigned n = 0; n < chain_len; ++n)
2544 : {
2545 1055 : vect_def_type dt = chains[0][n].dt;
2546 1055 : unsigned lane;
2547 4034 : for (lane = 0; lane < group_size; ++lane)
2548 5958 : if (stmts[lane] && chains[lane][n].dt != dt)
2549 : {
2550 0 : if (dt == vect_constant_def
2551 0 : && chains[lane][n].dt == vect_external_def)
2552 : dt = vect_external_def;
2553 0 : else if (dt == vect_external_def
2554 0 : && chains[lane][n].dt == vect_constant_def)
2555 : ;
2556 : else
2557 : break;
2558 : }
2559 1055 : if (lane != group_size)
2560 : {
2561 0 : if (dump_enabled_p ())
2562 0 : dump_printf_loc (MSG_NOTE, vect_location,
2563 : "giving up on chain due to mismatched "
2564 : "def types\n");
2565 0 : matches[lane] = -1;
2566 0 : if (lane != group_size - 1)
2567 0 : matches[0] = -1;
2568 0 : goto out;
2569 : }
2570 1055 : dts[n] = dt;
2571 1055 : if (dt == vect_constant_def
2572 1055 : || dt == vect_external_def)
2573 : {
2574 : /* Check whether we can build the invariant. If we can't
2575 : we never will be able to. */
2576 77 : tree type = TREE_TYPE (chains[0][n].op);
2577 1055 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
2578 : && (TREE_CODE (type) == BOOLEAN_TYPE
2579 : || !can_duplicate_and_interleave_p (vinfo, group_size,
2580 : type)))
2581 : {
2582 : matches[0] = -1;
2583 : goto out;
2584 : }
2585 : }
2586 978 : else if (dt != vect_internal_def)
2587 : {
2588 : /* Not sure, we might need sth special.
2589 : gcc.dg/vect/pr96854.c,
2590 : gfortran.dg/vect/fast-math-pr37021.f90
2591 : and gfortran.dg/vect/pr61171.f trigger. */
2592 : /* Soft-fail for now. */
2593 0 : hard_fail = false;
2594 0 : goto out;
2595 : }
2596 : }
2597 : /* 2b. do the actual build. */
2598 1328 : for (unsigned n = 0; n < chain_len; ++n)
2599 : {
2600 1020 : vect_def_type dt = dts[n];
2601 1020 : unsigned lane;
2602 1020 : if (dt == vect_constant_def
2603 1020 : || dt == vect_external_def)
2604 : {
2605 77 : vec<tree> ops;
2606 77 : ops.create (group_size);
2607 397 : for (lane = 0; lane < group_size; ++lane)
2608 243 : if (stmts[lane])
2609 243 : ops.quick_push (chains[lane][n].op);
2610 : else
2611 0 : ops.quick_push (NULL_TREE);
2612 77 : slp_tree child = vect_create_new_slp_node (ops);
2613 77 : SLP_TREE_DEF_TYPE (child) = dt;
2614 77 : children.safe_push (child);
2615 : }
2616 : else
2617 : {
2618 943 : vec<stmt_vec_info> op_stmts;
2619 943 : op_stmts.create (group_size);
2620 943 : slp_tree child = NULL;
2621 : /* Brute-force our way. We have to consider a lane
2622 : failing after fixing an earlier fail up in the
2623 : SLP discovery recursion. So track the current
2624 : permute per lane. */
2625 943 : unsigned *perms = XALLOCAVEC (unsigned, group_size);
2626 943 : memset (perms, 0, sizeof (unsigned) * group_size);
2627 1039 : do
2628 : {
2629 1039 : op_stmts.truncate (0);
2630 5002 : for (lane = 0; lane < group_size; ++lane)
2631 2924 : if (stmts[lane])
2632 2924 : op_stmts.quick_push
2633 2924 : (vinfo->lookup_def (chains[lane][n].op));
2634 : else
2635 0 : op_stmts.quick_push (NULL);
2636 1039 : child = vect_build_slp_tree (vinfo, op_stmts,
2637 : matches, limit,
2638 : &this_tree_size, bst_map);
2639 : /* ??? We're likely getting too many fatal mismatches
2640 : here so maybe we want to ignore them (but then we
2641 : have no idea which lanes fatally mismatched). */
2642 : /* ??? Revisit this with matches[] improvements. */
2643 1039 : if (child || matches[0] != 0)
2644 : break;
2645 : /* Swap another lane we have not yet matched up into
2646 : lanes that did not match. If we run out of
2647 : permute possibilities for a lane terminate the
2648 : search. */
2649 291 : bool term = false;
2650 291 : for (lane = 1; lane < group_size; ++lane)
2651 195 : if (matches[lane] != 0)
2652 : {
2653 167 : if (n + perms[lane] + 1 == chain_len)
2654 : {
2655 : term = true;
2656 : break;
2657 : }
2658 148 : if (dump_enabled_p ())
2659 115 : dump_printf_loc (MSG_NOTE, vect_location,
2660 : "swapping operand %d and %d "
2661 : "of lane %d\n",
2662 : n, n + perms[lane] + 1, lane);
2663 296 : std::swap (chains[lane][n],
2664 148 : chains[lane][n + perms[lane] + 1]);
2665 148 : perms[lane]++;
2666 : }
2667 115 : if (term)
2668 : break;
2669 : }
2670 : while (1);
2671 943 : if (!child)
2672 : {
2673 20 : if (dump_enabled_p ())
2674 18 : dump_printf_loc (MSG_NOTE, vect_location,
2675 : "failed to match up op %d\n", n);
2676 20 : op_stmts.release ();
2677 20 : if (lane != group_size - 1)
2678 10 : matches[0] = -1;
2679 : else
2680 10 : matches[lane] = -1;
2681 20 : goto out;
2682 : }
2683 923 : if (dump_enabled_p ())
2684 : {
2685 409 : dump_printf_loc (MSG_NOTE, vect_location,
2686 : "matched up op %d to\n", n);
2687 409 : vect_print_slp_tree (MSG_NOTE, vect_location, child);
2688 : }
2689 923 : children.safe_push (child);
2690 : }
2691 : }
2692 : /* 3. build SLP nodes to combine the chain. */
2693 1122 : for (unsigned lane = 0; lane < group_size; ++lane)
2694 1640 : if (stmts[lane] && chains[lane][0].code != code)
2695 : {
2696 : /* See if there's any alternate all-PLUS entry. */
2697 : unsigned n;
2698 6 : for (n = 1; n < chain_len; ++n)
2699 : {
2700 30 : for (lane = 0; lane < group_size; ++lane)
2701 48 : if (stmts[lane] && chains[lane][n].code != code)
2702 : break;
2703 6 : if (lane == group_size)
2704 : break;
2705 : }
2706 6 : if (n != chain_len)
2707 : {
2708 : /* Swap that in at first position. */
2709 6 : std::swap (children[0], children[n]);
2710 30 : for (lane = 0; lane < group_size; ++lane)
2711 24 : if (stmts[lane])
2712 24 : std::swap (chains[lane][0], chains[lane][n]);
2713 : }
2714 : else
2715 : {
2716 : /* ??? When this triggers and we end up with two
2717 : vect_constant/external_def up-front things break (ICE)
2718 : spectacularly finding an insertion place for the
2719 : all-constant op. We should have a fully
2720 : vect_internal_def operand though(?) so we can swap
2721 : that into first place and then prepend the all-zero
2722 : constant. */
2723 0 : if (dump_enabled_p ())
2724 0 : dump_printf_loc (MSG_NOTE, vect_location,
2725 : "inserting constant zero to compensate "
2726 : "for (partially) negated first "
2727 : "operand\n");
2728 0 : chain_len++;
2729 0 : for (lane = 0; lane < group_size; ++lane)
2730 0 : if (stmts[lane])
2731 0 : chains[lane].safe_insert
2732 0 : (0, chain_op_t (code, vect_constant_def, NULL_TREE));
2733 0 : vec<tree> zero_ops;
2734 0 : zero_ops.create (group_size);
2735 0 : zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
2736 0 : for (lane = 1; lane < group_size; ++lane)
2737 0 : if (stmts[lane])
2738 0 : zero_ops.quick_push (zero_ops[0]);
2739 : else
2740 0 : zero_ops.quick_push (NULL_TREE);
2741 0 : slp_tree zero = vect_create_new_slp_node (zero_ops);
2742 0 : SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
2743 0 : children.safe_insert (0, zero);
2744 : }
2745 : break;
2746 : }
2747 993 : for (unsigned i = 1; i < children.length (); ++i)
2748 : {
2749 685 : slp_tree op0 = children[i - 1];
2750 685 : slp_tree op1 = children[i];
2751 685 : bool this_two_op = false;
2752 2441 : for (unsigned lane = 0; lane < group_size; ++lane)
2753 3836 : if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
2754 : {
2755 : this_two_op = true;
2756 : break;
2757 : }
2758 685 : slp_tree child;
2759 685 : if (i == children.length () - 1)
2760 308 : child = vect_create_new_slp_node (node, stmts, 2);
2761 : else
2762 377 : child = vect_create_new_slp_node (2, ERROR_MARK);
2763 685 : if (this_two_op)
2764 : {
2765 162 : vec<std::pair<unsigned, unsigned> > lperm;
2766 162 : lperm.create (group_size);
2767 744 : for (unsigned lane = 0; lane < group_size; ++lane)
2768 840 : lperm.quick_push (std::make_pair
2769 420 : (chains[lane][i].code != chains[0][i].code, lane));
2770 324 : vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
2771 162 : (chains[0][i].code == code
2772 : ? op_stmt_info
2773 : : other_op_stmt_info),
2774 162 : (chains[0][i].code == code
2775 : ? other_op_stmt_info
2776 : : op_stmt_info),
2777 : lperm);
2778 : }
2779 : else
2780 : {
2781 523 : SLP_TREE_DEF_TYPE (child) = vect_internal_def;
2782 523 : SLP_TREE_VECTYPE (child) = vectype;
2783 523 : SLP_TREE_LANES (child) = group_size;
2784 523 : SLP_TREE_CHILDREN (child).quick_push (op0);
2785 523 : SLP_TREE_CHILDREN (child).quick_push (op1);
2786 523 : SLP_TREE_REPRESENTATIVE (child)
2787 1046 : = (chains[0][i].code == code
2788 523 : ? op_stmt_info : other_op_stmt_info);
2789 : }
2790 685 : children[i] = child;
2791 : }
2792 308 : *tree_size += this_tree_size + 1;
2793 1477 : while (!chains.is_empty ())
2794 838 : chains.pop ().release ();
2795 : return node;
2796 : }
2797 20333 : out:
2798 20356 : if (dump_enabled_p ())
2799 2825 : dump_printf_loc (MSG_NOTE, vect_location,
2800 : "failed to line up SLP graph by re-associating "
2801 : "operations in lanes%s\n",
2802 : !hard_fail ? " trying regular discovery" : "");
2803 20363 : while (!children.is_empty ())
2804 7 : vect_free_slp_tree (children.pop ());
2805 20521 : while (!chains.is_empty ())
2806 165 : chains.pop ().release ();
2807 : /* Hard-fail, otherwise we might run into quadratic processing of the
2808 : chains starting one stmt into the chain again. */
2809 20356 : if (hard_fail)
2810 : return NULL;
2811 : /* Fall thru to normal processing. */
2812 20664 : }
2813 :
2814 : /* Get at the operands, verifying they are compatible. */
2815 3738285 : vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
2816 3738285 : slp_oprnd_info oprnd_info;
2817 21139138 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
2818 : {
2819 27327912 : int res = vect_get_and_check_slp_defs (vinfo, vectype,
2820 13663956 : swap[i], skip_args,
2821 : stmts, i, &oprnds_info,
2822 : soft_fail);
2823 13663956 : if (res != 0)
2824 : /* ??? This puts -1 back into matches[] and the cache. */
2825 342093 : matches[(res == -1) ? 0 : i] = -1;
2826 13663956 : if (matches[0] == -1)
2827 : break;
2828 : }
2829 17006507 : for (i = 0; i < group_size; ++i)
2830 13520085 : if (matches[i] != 0)
2831 : {
2832 251863 : vect_free_oprnd_info (oprnds_info);
2833 251863 : return NULL;
2834 : }
2835 3486422 : swap = NULL;
2836 :
2837 : /* Perform delayed soft-failing only here so we can factor in mismatches
2838 : determined by vect_get_and_check_slp_defs. */
2839 3486422 : if (soft_fail)
2840 : {
2841 122353 : force_split_matches (matches, group_size,
2842 122353 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2843 122353 : return NULL;
2844 : }
2845 :
2846 3364069 : bool has_two_operators_perm = false;
2847 16820345 : auto_vec<unsigned> two_op_perm_indices[2];
2848 3364069 : vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
2849 :
2850 3380844 : if (two_operators && oprnds_info.length () == 2 && group_size > 2)
2851 : {
2852 2802 : unsigned idx = 0;
2853 2802 : hash_map<gimple *, unsigned> seen;
2854 2802 : vec<slp_oprnd_info> new_oprnds_info
2855 2802 : = vect_create_oprnd_info (1, group_size);
2856 2802 : bool success = true;
2857 :
2858 2802 : enum tree_code code = ERROR_MARK;
2859 2802 : if (oprnds_info[0]->def_stmts[0]
2860 2802 : && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
2861 2742 : code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
2862 2802 : basic_block bb = nullptr;
2863 :
2864 6504 : for (unsigned j = 0; j < group_size; ++j)
2865 : {
2866 15572 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2867 : {
2868 11870 : stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
2869 11870 : if (!stmt_info
2870 11697 : || !is_a<gassign *> (stmt_info->stmt)
2871 11697 : || gimple_assign_rhs_code (stmt_info->stmt) != code
2872 21421 : || skip_args[i])
2873 : {
2874 : success = false;
2875 2323 : break;
2876 : }
2877 : /* Avoid mixing lanes with defs in different basic-blocks. */
2878 9551 : if (!bb)
2879 2962 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
2880 8547 : else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
2881 : {
2882 : success = false;
2883 : break;
2884 : }
2885 :
2886 9547 : bool exists;
2887 9547 : unsigned &stmt_idx
2888 9547 : = seen.get_or_insert (stmt_info->stmt, &exists);
2889 :
2890 9547 : if (!exists)
2891 : {
2892 8166 : new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
2893 8166 : new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
2894 8166 : stmt_idx = idx;
2895 8166 : idx++;
2896 : }
2897 :
2898 9547 : two_op_perm_indices[i].safe_push (stmt_idx);
2899 : }
2900 :
2901 6025 : if (!success)
2902 : break;
2903 : }
2904 :
2905 2802 : if (success && idx == group_size)
2906 : {
2907 98 : if (dump_enabled_p ())
2908 : {
2909 0 : dump_printf_loc (MSG_NOTE, vect_location,
2910 : "Replace two_operators operands:\n");
2911 :
2912 0 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2913 : {
2914 0 : dump_printf_loc (MSG_NOTE, vect_location,
2915 : "Operand %u:\n", i);
2916 0 : for (unsigned j = 0; j < group_size; j++)
2917 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2918 0 : j, oprnd_info->def_stmts[j]->stmt);
2919 : }
2920 :
2921 0 : dump_printf_loc (MSG_NOTE, vect_location,
2922 : "With a single operand:\n");
2923 0 : for (unsigned j = 0; j < group_size; j++)
2924 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2925 0 : j, new_oprnds_info[0]->def_stmts[j]->stmt);
2926 : }
2927 :
2928 98 : two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
2929 98 : two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
2930 :
2931 98 : new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
2932 98 : new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
2933 98 : new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
2934 98 : new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
2935 98 : new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
2936 :
2937 98 : vect_free_oprnd_info (oprnds_info);
2938 98 : oprnds_info = new_oprnds_info;
2939 98 : nops = 1;
2940 98 : has_two_operators_perm = true;
2941 : }
2942 : else
2943 2704 : vect_free_oprnd_info (new_oprnds_info);
2944 2802 : }
2945 :
2946 6728138 : auto_vec<slp_tree, 4> children;
2947 :
2948 3364069 : stmt_info = stmts[0];
2949 :
2950 3364069 : int reduc_idx = -1;
2951 3364069 : int gs_scale = 0;
2952 3364069 : tree gs_base = NULL_TREE;
2953 :
2954 : /* Create SLP_TREE nodes for the definition node/s. */
2955 8659920 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2956 : {
2957 5399741 : slp_tree child = nullptr;
2958 5399741 : unsigned int j;
2959 5399741 : unsigned old_swap_distance;
2960 5399741 : bool can_swap;
2961 5399741 : bool can_swap_nonmatching;
2962 5399741 : bool *stmt_can_swap;
2963 :
2964 : /* We're skipping certain operands from processing, for example
2965 : outer loop reduction initial defs. */
2966 5399741 : if (skip_args[i])
2967 : {
2968 531125 : children.safe_push (NULL);
2969 5295851 : continue;
2970 : }
2971 :
2972 4868616 : if (oprnd_info->first_dt == vect_uninitialized_def)
2973 : {
2974 : /* COND_EXPR have one too many eventually if the condition
2975 : is a SSA name. */
2976 0 : gcc_assert (i == 3 && nops == 4);
2977 0 : continue;
2978 : }
2979 :
2980 4868616 : if (oprnd_info->first_gs_p)
2981 : {
2982 27120 : gs_scale = oprnd_info->first_gs_info.scale;
2983 27120 : gs_base = oprnd_info->first_gs_info.base;
2984 : }
2985 :
2986 4868616 : if (is_a <bb_vec_info> (vinfo)
2987 1691433 : && oprnd_info->first_dt == vect_internal_def
2988 5772085 : && !oprnd_info->any_pattern)
2989 : {
2990 : /* For BB vectorization, if all defs are the same do not
2991 : bother to continue the build along the single-lane
2992 : graph but use a splat of the scalar value. */
2993 857411 : stmt_vec_info first_def = oprnd_info->def_stmts[0];
2994 924133 : for (j = 1; j < group_size; ++j)
2995 873196 : if (oprnd_info->def_stmts[j] != first_def)
2996 : break;
2997 857411 : if (j == group_size
2998 : /* But avoid doing this for loads where we may be
2999 : able to CSE things, unless the stmt is not
3000 : vectorizable. */
3001 857411 : && (!STMT_VINFO_VECTORIZABLE (first_def)
3002 61122 : || !gimple_vuse (first_def->stmt)))
3003 : {
3004 41211 : if (dump_enabled_p ())
3005 119 : dump_printf_loc (MSG_NOTE, vect_location,
3006 : "Using a splat of the uniform operand %G",
3007 : first_def->stmt);
3008 41211 : oprnd_info->first_dt = vect_external_def;
3009 : }
3010 : }
3011 :
3012 4868616 : if (oprnd_info->first_dt == vect_external_def
3013 4868616 : || oprnd_info->first_dt == vect_constant_def)
3014 : {
3015 1605919 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
3016 : {
3017 : tree op0;
3018 : tree uniform_val = op0 = oprnd_info->ops[0];
3019 : for (j = 1; j < oprnd_info->ops.length (); ++j)
3020 : if (oprnd_info->ops[j]
3021 : && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
3022 : {
3023 : uniform_val = NULL_TREE;
3024 : break;
3025 : }
3026 : if (!uniform_val
3027 : && !can_duplicate_and_interleave_p (vinfo,
3028 : oprnd_info->ops.length (),
3029 : TREE_TYPE (op0)))
3030 : {
3031 : for (unsigned k = 0; k < j; ++k)
3032 : matches[k] = 0;
3033 : matches[j] = -1;
3034 : for (unsigned k = j + 1; k < oprnd_info->ops.length (); ++k)
3035 : if (oprnd_info->ops[k]
3036 : && !operand_equal_p (op0, oprnd_info->ops[k]))
3037 : matches[k] = -1;
3038 : else
3039 : matches[k] = 0;
3040 : if (dump_enabled_p ())
3041 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3042 : "Build SLP failed: invalid type of def "
3043 : "for variable-length SLP %T\n", op0);
3044 : goto fail;
3045 : }
3046 : }
3047 1605919 : slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
3048 1605919 : SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
3049 1605919 : oprnd_info->ops = vNULL;
3050 1605919 : children.safe_push (invnode);
3051 1605919 : continue;
3052 1605919 : }
3053 :
3054 : /* See which SLP operand a reduction chain continues on. We want
3055 : to chain even PHIs but not backedges. */
3056 3262697 : if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
3057 3262697 : || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
3058 : {
3059 234481 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
3060 : {
3061 760 : if (oprnd_info->first_dt == vect_double_reduction_def)
3062 380 : reduc_idx = i;
3063 : }
3064 233721 : else if (is_a <gphi *> (stmt_info->stmt)
3065 233721 : && gimple_phi_num_args
3066 101820 : (as_a <gphi *> (stmt_info->stmt)) != 1)
3067 : ;
3068 132286 : else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
3069 385 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
3070 : ;
3071 132286 : else if (reduc_idx == -1)
3072 123794 : reduc_idx = i;
3073 : else
3074 : /* For .COND_* reduction operations the else value can be the
3075 : same as one of the operation operands. The other def
3076 : stmts have been moved, so we can't check easily. Check
3077 : it's a call at least. */
3078 8492 : gcc_assert (is_a <gcall *> (stmt_info->stmt));
3079 : }
3080 :
3081 : /* When we have a masked load with uniform mask discover this
3082 : as a single-lane mask with a splat permute. This way we can
3083 : recognize this as a masked load-lane by stripping the splat. */
3084 3262697 : if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
3085 60305 : && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
3086 : IFN_MASK_LOAD)
3087 4763 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
3088 3262774 : && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
3089 : {
3090 35 : vec<stmt_vec_info> def_stmts2;
3091 35 : def_stmts2.create (1);
3092 35 : def_stmts2.quick_push (oprnd_info->def_stmts[0]);
3093 35 : child = vect_build_slp_tree (vinfo, def_stmts2,
3094 : matches, limit,
3095 : &this_tree_size, bst_map);
3096 35 : if (child)
3097 : {
3098 35 : slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
3099 35 : SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
3100 35 : SLP_TREE_LANES (pnode) = group_size;
3101 35 : SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
3102 35 : SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
3103 245 : for (unsigned k = 0; k < group_size; ++k)
3104 : {
3105 175 : SLP_TREE_SCALAR_STMTS (pnode)
3106 175 : .quick_push (oprnd_info->def_stmts[0]);
3107 175 : SLP_TREE_LANE_PERMUTATION (pnode)
3108 175 : .quick_push (std::make_pair (0u, 0u));
3109 : }
3110 35 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3111 35 : children.safe_push (pnode);
3112 35 : oprnd_info->def_stmts = vNULL;
3113 35 : continue;
3114 35 : }
3115 : else
3116 0 : def_stmts2.release ();
3117 : }
3118 :
3119 6525324 : can_swap = (i == 0
3120 2424738 : && (nops == 2 || nops == 3)
3121 1553109 : && oprnds_info.length () > 1
3122 1553109 : && oprnds_info[1]->first_dt == vect_internal_def
3123 617093 : && (is_gimple_assign (stmt_info->stmt)
3124 51897 : || is_gimple_call (stmt_info->stmt))
3125 : /* Swapping operands for reductions breaks assumptions
3126 : later on. */
3127 3833277 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3128 3262662 : can_swap_nonmatching = can_swap;
3129 3262662 : stmt_can_swap = NULL;
3130 3262662 : if (can_swap)
3131 : {
3132 519362 : stmt_can_swap = XALLOCAVEC (bool, group_size);
3133 8140022 : for (j = 0; j < group_size; ++j)
3134 : {
3135 7620660 : stmt_can_swap[j] = false;
3136 7620660 : if (!stmts[j])
3137 : /* NULL lanes are gaps and have no stmt to swap. */
3138 0 : stmt_can_swap[j] = true;
3139 7620660 : else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
3140 : {
3141 7614379 : tree_code code = gimple_assign_rhs_code (stmt);
3142 15228758 : stmt_can_swap[j] = (commutative_tree_code (code)
3143 7614379 : || commutative_ternary_tree_code (code));
3144 : }
3145 6281 : else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
3146 : {
3147 6281 : internal_fn fn = (gimple_call_internal_p (call)
3148 6281 : ? gimple_call_internal_fn (call) : IFN_LAST);
3149 12562 : stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
3150 5026 : || commutative_ternary_fn_p (fn))
3151 6317 : && first_commutative_argument (fn) == 0);
3152 : }
3153 :
3154 7620660 : if (j != 0 && !stmt_can_swap[j])
3155 7620660 : can_swap_nonmatching = false;
3156 : }
3157 : }
3158 :
3159 3262662 : old_swap_distance = least_upthread_swappable_op_distance;
3160 3262662 : if (can_swap_nonmatching)
3161 477425 : least_upthread_swappable_op_distance = 1;
3162 2785237 : else if (least_upthread_swappable_op_distance != -1U)
3163 345832 : least_upthread_swappable_op_distance++;
3164 3262662 : child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3165 : matches, limit,
3166 : &this_tree_size, bst_map);
3167 3262662 : least_upthread_swappable_op_distance = old_swap_distance;
3168 3262662 : if (child != NULL)
3169 : {
3170 2762692 : oprnd_info->def_stmts = vNULL;
3171 2762692 : children.safe_push (child);
3172 2762692 : continue;
3173 : }
3174 :
3175 : /* If the SLP build for operand zero failed and operand zero
3176 : and one can be commuted try that for the scalar stmts
3177 : that failed the match. */
3178 499970 : if (/* A first scalar stmt mismatch signals a fatal mismatch. */
3179 499970 : matches[0] == 0
3180 499970 : && can_swap)
3181 : {
3182 : /* See whether we can swap the matching or the non-matching
3183 : stmt operands. */
3184 : bool swap_not_matching = true;
3185 69760 : do
3186 : {
3187 7072780 : for (j = 0; j < group_size; ++j)
3188 : {
3189 7021225 : if ((matches[j] == 0) != !swap_not_matching)
3190 76951 : continue;
3191 : /* Verify if we can swap operands of this stmt. */
3192 6944274 : if (!stmt_can_swap[j])
3193 : {
3194 18205 : if (!swap_not_matching)
3195 8020 : goto fail;
3196 : swap_not_matching = false;
3197 : break;
3198 : }
3199 : }
3200 : }
3201 61740 : while (j != group_size);
3202 :
3203 : /* Swap mismatched definition stmts. */
3204 51555 : if (dump_enabled_p ())
3205 432 : dump_printf_loc (MSG_NOTE, vect_location,
3206 : "Re-trying with swapped operands of stmts ");
3207 7043333 : for (j = 0; j < group_size; ++j)
3208 6991778 : if ((matches[j] == 0) == !swap_not_matching)
3209 : {
3210 13851730 : std::swap (oprnds_info[0]->def_stmts[j],
3211 6925865 : oprnds_info[1]->def_stmts[j]);
3212 13851730 : std::swap (oprnds_info[0]->ops[j],
3213 6925865 : oprnds_info[1]->ops[j]);
3214 6925865 : if (dump_enabled_p ())
3215 1101 : dump_printf (MSG_NOTE, "%d ", j);
3216 : }
3217 51555 : if (dump_enabled_p ())
3218 432 : dump_printf (MSG_NOTE, "\n");
3219 : /* After swapping some operands we lost track whether an
3220 : operand has any pattern defs so be conservative here. */
3221 99961 : if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
3222 3212 : oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
3223 : /* And try again with scratch 'matches' ... */
3224 51555 : match_elt_t *tem = XALLOCAVEC (match_elt_t, group_size);
3225 51555 : if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3226 : tem, limit,
3227 : &this_tree_size, bst_map)) != NULL)
3228 : {
3229 6277 : oprnd_info->def_stmts = vNULL;
3230 6277 : children.safe_push (child);
3231 6277 : continue;
3232 : }
3233 : }
3234 493693 : fail:
3235 :
3236 : /* If the SLP build failed and we analyze a basic-block
3237 : simply treat nodes we fail to build as externally defined
3238 : (and thus build vectors from the scalar defs).
3239 : The cost model will reject outright expensive cases.
3240 : ??? This doesn't treat cases where permutation ultimatively
3241 : fails (or we don't try permutation below). Ideally we'd
3242 : even compute a permutation that will end up with the maximum
3243 : SLP tree size... */
3244 493693 : if (is_a <bb_vec_info> (vinfo)
3245 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3246 : do extra work to cancel the pattern so the uses see the
3247 : scalar version. */
3248 : /* Skip building vector operands from scalars while operand
3249 : discovery may still be fixed by retrying with swapped operands. */
3250 440829 : && (least_upthread_swappable_op_distance != 1
3251 : /* A first scalar stmt mismatch signals a fatal mismatch
3252 : that the parent commutative retry cannot recover.
3253 : ??? Possibly revisit this with the matches[] improvements
3254 : as we can have matches[0] == -2 here. */
3255 26894 : || matches[0] != 0)
3256 422568 : && !is_pattern_stmt_p (stmt_info)
3257 890357 : && !oprnd_info->any_pattern)
3258 : {
3259 : /* But if there's a leading vector sized set of matching stmts
3260 : fail here so we can split the group. This matches the condition
3261 : vect_analyze_slp_instance uses. */
3262 : /* ??? We might want to split here and combine the results to support
3263 : multiple vector sizes better. */
3264 616313 : for (j = 0; j < group_size; ++j)
3265 616313 : if (matches[j] != 0)
3266 : break;
3267 396291 : if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
3268 396257 : && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
3269 : {
3270 389803 : if (dump_enabled_p ())
3271 809 : dump_printf_loc (MSG_NOTE, vect_location,
3272 : "Building vector operands from scalars\n");
3273 389803 : this_tree_size++;
3274 389803 : child = vect_create_new_slp_node (oprnd_info->ops);
3275 389803 : children.safe_push (child);
3276 389803 : oprnd_info->ops = vNULL;
3277 389803 : continue;
3278 : }
3279 : }
3280 :
3281 103890 : gcc_assert (child == NULL);
3282 223451 : FOR_EACH_VEC_ELT (children, j, child)
3283 15671 : if (child)
3284 15671 : vect_free_slp_tree (child);
3285 103890 : vect_free_oprnd_info (oprnds_info);
3286 103890 : return NULL;
3287 : }
3288 :
3289 3260179 : vect_free_oprnd_info (oprnds_info);
3290 :
3291 : /* If we have all children of a child built up from uniform scalars
3292 : or does more than one possibly expensive vector construction then
3293 : just throw that away, causing it built up from scalars.
3294 : The exception is the SLP node for the vector store. */
3295 3260179 : if (is_a <bb_vec_info> (vinfo)
3296 1156609 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
3297 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3298 : do extra work to cancel the pattern so the uses see the
3299 : scalar version. */
3300 3740091 : && !is_pattern_stmt_p (stmt_info))
3301 : {
3302 : slp_tree child;
3303 : unsigned j;
3304 : bool all_uniform_p = true;
3305 : unsigned n_vector_builds = 0;
3306 1351992 : FOR_EACH_VEC_ELT (children, j, child)
3307 : {
3308 901236 : if (!child)
3309 : ;
3310 901236 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
3311 : all_uniform_p = false;
3312 643549 : else if (!vect_slp_tree_uniform_p (child))
3313 : {
3314 487290 : all_uniform_p = false;
3315 487290 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
3316 446609 : n_vector_builds++;
3317 : }
3318 : }
3319 450756 : if (all_uniform_p
3320 450756 : || n_vector_builds > 1
3321 765616 : || (n_vector_builds == children.length ()
3322 35452 : && is_a <gphi *> (stmt_info->stmt)))
3323 : {
3324 : /* Roll back. */
3325 140384 : matches[0] = -1;
3326 444730 : FOR_EACH_VEC_ELT (children, j, child)
3327 304346 : if (child)
3328 304346 : vect_free_slp_tree (child);
3329 :
3330 140384 : if (dump_enabled_p ())
3331 262 : dump_printf_loc (MSG_NOTE, vect_location,
3332 : "Building parent vector operands from "
3333 : "scalars instead\n");
3334 3364069 : return NULL;
3335 : }
3336 : }
3337 :
3338 3119795 : *tree_size += this_tree_size + 1;
3339 :
3340 3119795 : if (two_operators)
3341 : {
3342 : /* ??? We'd likely want to either cache in bst_map sth like
3343 : { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
3344 : the true { a+b, a+b, a+b, a+b } ... but there we don't have
3345 : explicit stmts to put in so the keying on 'stmts' doesn't
3346 : work (but we have the same issue with nodes that use 'ops'). */
3347 :
3348 9338 : if (has_two_operators_perm)
3349 : {
3350 46 : slp_tree child = children[0];
3351 46 : children.truncate (0);
3352 184 : for (i = 0; i < 2; i++)
3353 : {
3354 92 : slp_tree pnode
3355 92 : = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
3356 92 : SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
3357 92 : SLP_TREE_VECTYPE (pnode) = vectype;
3358 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3359 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3360 92 : SLP_TREE_REPRESENTATIVE (pnode) = NULL;
3361 92 : lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
3362 92 : children.safe_push (pnode);
3363 :
3364 808 : for (unsigned j = 0; j < stmts.length (); j++)
3365 624 : perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
3366 : }
3367 :
3368 46 : SLP_TREE_REF_COUNT (child) += 4;
3369 : }
3370 :
3371 9338 : slp_tree one = new _slp_tree;
3372 9338 : slp_tree two = new _slp_tree;
3373 9338 : SLP_TREE_DEF_TYPE (one) = vect_internal_def;
3374 9338 : SLP_TREE_DEF_TYPE (two) = vect_internal_def;
3375 9338 : SLP_TREE_VECTYPE (one) = vectype;
3376 9338 : SLP_TREE_VECTYPE (two) = vectype;
3377 9338 : SLP_TREE_CHILDREN (one).safe_splice (children);
3378 9338 : SLP_TREE_CHILDREN (two).safe_splice (children);
3379 9338 : slp_tree child;
3380 37354 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
3381 18678 : SLP_TREE_REF_COUNT (child)++;
3382 :
3383 : /* Here we record the original defs since this
3384 : node represents the final lane configuration. */
3385 9338 : node = vect_create_new_slp_node (node, stmts, 2);
3386 9338 : SLP_TREE_VECTYPE (node) = vectype;
3387 9338 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
3388 9338 : SLP_TREE_CHILDREN (node).quick_push (one);
3389 9338 : SLP_TREE_CHILDREN (node).quick_push (two);
3390 9338 : SLP_TREE_REPRESENTATIVE (node) = NULL;
3391 9338 : enum tree_code code0 = ERROR_MARK;
3392 9338 : if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
3393 9336 : code0 = gimple_assign_rhs_code (stmt);
3394 9338 : stmt_vec_info ostmt_info;
3395 9338 : unsigned j = 0;
3396 32421 : FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
3397 : {
3398 23083 : int op = 0;
3399 23083 : gassign *ostmt = dyn_cast<gassign *> (ostmt_info->stmt);
3400 23079 : if ((ostmt && gimple_assign_rhs_code (ostmt) != code0)
3401 : || (!ostmt
3402 8 : && gimple_call_combined_fn (stmts[0]->stmt)
3403 4 : != gimple_call_combined_fn (ostmt_info->stmt)))
3404 : {
3405 : op = 1;
3406 : j = i;
3407 : }
3408 23083 : SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
3409 : }
3410 9338 : SLP_TREE_LANES (one) = stmts.length ();
3411 9338 : SLP_TREE_LANES (two) = stmts.length ();
3412 9338 : SLP_TREE_REPRESENTATIVE (one) = stmts[0];
3413 9338 : SLP_TREE_REPRESENTATIVE (two) = stmts[j];
3414 :
3415 9338 : return node;
3416 : }
3417 :
3418 3110457 : node = vect_create_new_slp_node (node, stmts, nops);
3419 3110457 : SLP_TREE_VECTYPE (node) = vectype;
3420 3110457 : SLP_TREE_CHILDREN (node).splice (children);
3421 3110457 : SLP_TREE_GS_SCALE (node) = gs_scale;
3422 3110457 : SLP_TREE_GS_BASE (node) = gs_base;
3423 3110457 : if (reduc_idx != -1)
3424 : {
3425 116899 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
3426 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3427 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
3428 116899 : SLP_TREE_REDUC_IDX (node) = reduc_idx;
3429 116899 : node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
3430 : }
3431 : /* When reaching the reduction PHI, create a vect_reduc_info. */
3432 2993558 : else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
3433 2993558 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3434 2993558 : && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
3435 : {
3436 102095 : loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
3437 102095 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3438 102095 : node->cycle_info.id = loop_vinfo->reduc_infos.length ();
3439 102095 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
3440 102095 : loop_vinfo->reduc_infos.safe_push (reduc_info);
3441 102095 : stmt_vec_info reduc_phi = stmt_info;
3442 : /* ??? For double reductions vect_is_simple_reduction stores the
3443 : reduction type and code on the inner loop header PHI. */
3444 102095 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3445 : {
3446 380 : use_operand_p use_p;
3447 380 : gimple *use_stmt;
3448 380 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
3449 : &use_p, &use_stmt);
3450 380 : gcc_assert (res);
3451 380 : reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
3452 : }
3453 102095 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
3454 102095 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
3455 102095 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
3456 102095 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
3457 : }
3458 : return node;
3459 10092207 : }
3460 :
3461 : /* Dump a single SLP tree NODE. */
3462 :
3463 : static void
3464 492228 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
3465 : slp_tree node)
3466 : {
3467 492228 : unsigned i, j;
3468 492228 : slp_tree child;
3469 492228 : stmt_vec_info stmt_info;
3470 492228 : tree op;
3471 :
3472 492228 : dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
3473 492228 : dump_user_location_t user_loc = loc.get_user_location ();
3474 492228 : dump_printf_loc (metadata, user_loc,
3475 : "node%s %p (refcnt=%u)",
3476 492228 : SLP_TREE_DEF_TYPE (node) == vect_external_def
3477 : ? " (external)"
3478 : : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
3479 475222 : ? " (constant)"
3480 : : ""), (void *) node,
3481 : SLP_TREE_REF_COUNT (node));
3482 492228 : if (SLP_TREE_VECTYPE (node))
3483 414875 : dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
3484 492228 : dump_printf (metadata, "%s",
3485 492228 : node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
3486 492228 : if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
3487 25083 : dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
3488 : node->cycle_info.reduc_idx);
3489 492228 : if (node->si)
3490 5460 : dump_printf (metadata, " @%G", node->si);
3491 : else
3492 486768 : dump_printf (metadata, "\n");
3493 492228 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3494 : {
3495 398631 : if (SLP_TREE_PERMUTE_P (node))
3496 17326 : dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
3497 : else
3498 381305 : dump_printf_loc (metadata, user_loc, "op template: %G",
3499 381305 : SLP_TREE_REPRESENTATIVE (node)->stmt);
3500 : }
3501 492228 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
3502 951097 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
3503 561482 : if (stmt_info)
3504 555324 : dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
3505 555324 : SLP_TREE_LIVE_LANES (node).contains (i)
3506 551497 : ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
3507 551497 : ? "[l] " : ""),
3508 : i, stmt_info->stmt);
3509 : else
3510 6158 : dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
3511 : else
3512 : {
3513 102613 : dump_printf_loc (metadata, user_loc, "\t{ ");
3514 226185 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
3515 123572 : dump_printf (metadata, "%T%s ", op,
3516 123572 : i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
3517 102613 : dump_printf (metadata, "}\n");
3518 : }
3519 492228 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
3520 : {
3521 70067 : dump_printf_loc (metadata, user_loc, "\tload permutation {");
3522 159332 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
3523 89265 : dump_printf (dump_kind, " %u", j);
3524 70067 : dump_printf (dump_kind, " }\n");
3525 : }
3526 492228 : if (SLP_TREE_LANE_PERMUTATION (node).exists ())
3527 : {
3528 17334 : dump_printf_loc (metadata, user_loc, "\tlane permutation {");
3529 64890 : for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
3530 47556 : dump_printf (dump_kind, " %u[%u]",
3531 47556 : SLP_TREE_LANE_PERMUTATION (node)[i].first,
3532 47556 : SLP_TREE_LANE_PERMUTATION (node)[i].second);
3533 17334 : dump_printf (dump_kind, " }%s\n",
3534 17334 : node->ldst_lanes ? " (load-lanes)" : "");
3535 : }
3536 492228 : if (SLP_TREE_CHILDREN (node).is_empty ())
3537 186583 : return;
3538 305645 : dump_printf_loc (metadata, user_loc, "\tchildren");
3539 806618 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3540 500973 : dump_printf (dump_kind, " %p", (void *)child);
3541 305645 : dump_printf (dump_kind, "%s\n",
3542 305645 : node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
3543 : ? " (store-lanes)" : "");
3544 : }
3545 :
3546 : DEBUG_FUNCTION void
3547 0 : debug (slp_tree node)
3548 : {
3549 0 : debug_dump_context ctx;
3550 0 : vect_print_slp_tree (MSG_NOTE,
3551 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3552 : node);
3553 0 : }
3554 :
3555 : /* Recursive helper for the dot producer below. */
3556 :
3557 : static void
3558 0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
3559 : {
3560 0 : if (visited.add (node))
3561 : return;
3562 :
3563 0 : fprintf (f, "\"%p\" [label=\"", (void *)node);
3564 0 : vect_print_slp_tree (MSG_NOTE,
3565 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3566 : node);
3567 0 : fprintf (f, "\"];\n");
3568 :
3569 :
3570 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3571 0 : fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
3572 :
3573 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3574 0 : if (child)
3575 0 : dot_slp_tree (f, child, visited);
3576 : }
3577 :
3578 : DEBUG_FUNCTION void
3579 0 : dot_slp_tree (const char *fname, slp_tree node)
3580 : {
3581 0 : FILE *f = fopen (fname, "w");
3582 0 : fprintf (f, "digraph {\n");
3583 0 : fflush (f);
3584 0 : {
3585 0 : debug_dump_context ctx (f);
3586 0 : hash_set<slp_tree> visited;
3587 0 : dot_slp_tree (f, node, visited);
3588 0 : }
3589 0 : fflush (f);
3590 0 : fprintf (f, "}\n");
3591 0 : fclose (f);
3592 0 : }
3593 :
3594 : DEBUG_FUNCTION void
3595 0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
3596 : {
3597 0 : FILE *f = fopen (fname, "w");
3598 0 : fprintf (f, "digraph {\n");
3599 0 : fflush (f);
3600 0 : {
3601 0 : debug_dump_context ctx (f);
3602 0 : hash_set<slp_tree> visited;
3603 0 : for (auto inst : slp_instances)
3604 0 : dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
3605 0 : }
3606 0 : fflush (f);
3607 0 : fprintf (f, "}\n");
3608 0 : fclose (f);
3609 0 : }
3610 :
3611 : /* Dump a slp tree NODE using flags specified in DUMP_KIND. */
3612 :
3613 : static void
3614 531443 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3615 : slp_tree node, hash_set<slp_tree> &visited)
3616 : {
3617 531443 : unsigned i;
3618 531443 : slp_tree child;
3619 :
3620 531443 : if (visited.add (node))
3621 531443 : return;
3622 :
3623 488397 : vect_print_slp_tree (dump_kind, loc, node);
3624 :
3625 1472640 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3626 495846 : if (child)
3627 448779 : vect_print_slp_graph (dump_kind, loc, child, visited);
3628 : }
3629 :
3630 : static void
3631 50458 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3632 : slp_tree entry)
3633 : {
3634 50458 : hash_set<slp_tree> visited;
3635 50458 : vect_print_slp_graph (dump_kind, loc, entry, visited);
3636 50458 : }
3637 :
3638 : DEBUG_FUNCTION void
3639 0 : debug (slp_instance instance)
3640 : {
3641 0 : debug_dump_context ctx;
3642 0 : vect_print_slp_graph (MSG_NOTE,
3643 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3644 : SLP_INSTANCE_TREE (instance));
3645 0 : }
3646 :
3647 :
3648 : /* Compute the set of scalar stmts participating in external nodes. */
3649 :
3650 : static void
3651 1717758 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
3652 : hash_set<slp_tree> &visited,
3653 : hash_set<stmt_vec_info> &estmts)
3654 : {
3655 1717758 : if (visited.add (node))
3656 : return;
3657 :
3658 1656321 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3659 : {
3660 : slp_tree child;
3661 : int i;
3662 1948096 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3663 995263 : if (child)
3664 995263 : vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
3665 : }
3666 : else
3667 3936870 : for (tree def : SLP_TREE_SCALAR_OPS (node))
3668 : {
3669 1828640 : stmt_vec_info def_stmt = vinfo->lookup_def (def);
3670 1828640 : if (def_stmt)
3671 392060 : estmts.add (def_stmt);
3672 : }
3673 : }
3674 :
3675 : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
3676 : with STMT_SLP_TYPE == pure_slp. */
3677 :
3678 : static void
3679 248546 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
3680 : {
3681 : /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
3682 : walk on. */
3683 248546 : hash_set<stmt_vec_info> scalar_stmts_in_externs;
3684 248546 : hash_set<slp_tree> visited;
3685 1468133 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3686 722495 : vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
3687 : visited, scalar_stmts_in_externs);
3688 :
3689 : /* DFS walk scalar stmts to compute the vectorized coverage indicated
3690 : by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
3691 : stmts. */
3692 1468133 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3693 : {
3694 928500 : for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
3695 95623 : if (!scalar_stmts_in_externs.contains (stmt))
3696 94358 : STMT_SLP_TYPE (stmt) = pure_slp;
3697 722495 : auto_vec<stmt_vec_info> worklist;
3698 4070343 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
3699 : {
3700 1902858 : stmt = vect_orig_stmt (stmt);
3701 1902858 : if (!scalar_stmts_in_externs.contains (stmt)
3702 1902858 : && STMT_SLP_TYPE (stmt) != pure_slp)
3703 : {
3704 1885796 : STMT_SLP_TYPE (stmt) = pure_slp;
3705 1885796 : worklist.safe_push (stmt);
3706 : }
3707 : }
3708 3811532 : while (!worklist.is_empty ())
3709 : {
3710 2371462 : stmt_vec_info stmt = worklist.pop ();
3711 :
3712 : /* Now walk relevant parts of the SSA use-def graph. */
3713 2371462 : slp_oprnds child_ops (stmt);
3714 7404880 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
3715 : {
3716 2661956 : tree op = child_ops.get_op_for_slp_child (stmt, i);
3717 2661956 : stmt_vec_info def = vinfo->lookup_def (op);
3718 2661956 : if (def
3719 963592 : && !scalar_stmts_in_externs.contains (def)
3720 3244592 : && STMT_SLP_TYPE (def) != pure_slp)
3721 : {
3722 485666 : STMT_SLP_TYPE (def) = pure_slp;
3723 485666 : worklist.safe_push (def);
3724 : }
3725 : }
3726 : }
3727 722495 : }
3728 248546 : }
3729 :
3730 : /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
3731 :
3732 : static void
3733 2604858 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
3734 : {
3735 2604858 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3736 : return;
3737 :
3738 1567507 : if (visited.add (node))
3739 : return;
3740 :
3741 7323344 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
3742 3272436 : if (stmt_info)
3743 : {
3744 3272436 : gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
3745 : || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
3746 3272436 : STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
3747 : }
3748 :
3749 5666978 : for (auto child: SLP_TREE_CHILDREN (node))
3750 1779176 : if (child)
3751 1779176 : vect_mark_slp_stmts_relevant (child, visited);
3752 : }
3753 :
3754 : static void
3755 825682 : vect_mark_slp_stmts_relevant (slp_tree node)
3756 : {
3757 825682 : hash_set<slp_tree> visited;
3758 825682 : vect_mark_slp_stmts_relevant (node, visited);
3759 825682 : }
3760 :
3761 :
3762 : /* Gather loads in the SLP graph NODE and populate the INST loads array. */
3763 :
3764 : static void
3765 11514987 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
3766 : hash_set<slp_tree> &visited)
3767 : {
3768 11514987 : if (!node || visited.add (node))
3769 : return;
3770 :
3771 9617296 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3772 : return;
3773 :
3774 7116632 : if (!SLP_TREE_PERMUTE_P (node))
3775 : {
3776 6891127 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
3777 6891127 : if (STMT_VINFO_DATA_REF (stmt_info)
3778 2930435 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
3779 1676655 : loads.safe_push (node);
3780 : }
3781 :
3782 7116632 : unsigned i;
3783 7116632 : slp_tree child;
3784 16241335 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3785 9124703 : vect_gather_slp_loads (loads, child, visited);
3786 : }
3787 :
3788 :
3789 : /* Find the last store in SLP INSTANCE. */
3790 :
3791 : stmt_vec_info
3792 2563721 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
3793 : {
3794 2563721 : stmt_vec_info last = NULL;
3795 14503297 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3796 6812134 : if (stmt_vinfo)
3797 : {
3798 6812134 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3799 6812134 : last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
3800 : }
3801 :
3802 2563721 : return last;
3803 : }
3804 :
3805 : /* Find the first stmt in NODE. */
3806 :
3807 : stmt_vec_info
3808 664862 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
3809 : {
3810 664862 : stmt_vec_info first = NULL;
3811 :
3812 3595039 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3813 1600453 : if (stmt_vinfo)
3814 : {
3815 1597861 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3816 1597861 : if (!first
3817 1597861 : || get_later_stmt (stmt_vinfo, first) == first)
3818 : first = stmt_vinfo;
3819 : }
3820 :
3821 664862 : return first;
3822 : }
3823 :
3824 : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
3825 : two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
3826 : (also containing the first GROUP1_SIZE stmts, since stores are
3827 : consecutive), the second containing the remainder.
3828 : Return the first stmt in the second group. */
3829 :
3830 : static stmt_vec_info
3831 137135 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
3832 : {
3833 137135 : gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
3834 137135 : gcc_assert (group1_size > 0);
3835 137135 : int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
3836 137135 : gcc_assert (group2_size > 0);
3837 137135 : DR_GROUP_SIZE (first_vinfo) = group1_size;
3838 :
3839 137135 : stmt_vec_info stmt_info = first_vinfo;
3840 459875 : for (unsigned i = group1_size; i > 1; i--)
3841 : {
3842 322740 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
3843 322740 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3844 : }
3845 : /* STMT is now the last element of the first group. */
3846 137135 : stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
3847 137135 : DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
3848 :
3849 137135 : DR_GROUP_SIZE (group2) = group2_size;
3850 407719 : for (stmt_info = group2; stmt_info;
3851 270584 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
3852 : {
3853 270584 : DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
3854 270584 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3855 : }
3856 :
3857 : /* For the second group, the DR_GROUP_GAP is that before the original group,
3858 : plus skipping over the first vector. */
3859 137135 : DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
3860 :
3861 : /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
3862 137135 : DR_GROUP_GAP (first_vinfo) += group2_size;
3863 :
3864 137135 : if (dump_enabled_p ())
3865 88 : dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
3866 : group1_size, group2_size);
3867 :
3868 137135 : return group2;
3869 : }
3870 :
3871 : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
3872 : statements and a vector of NUNITS elements. */
3873 :
3874 : static poly_uint64
3875 2973401 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
3876 : {
3877 2973401 : return exact_div (common_multiple (nunits, group_size), group_size);
3878 : }
3879 :
3880 : /* Helper that checks to see if a node is a load node. */
3881 :
3882 : static inline bool
3883 103 : vect_is_slp_load_node (slp_tree root)
3884 : {
3885 103 : return (!SLP_TREE_PERMUTE_P (root)
3886 103 : && SLP_TREE_DEF_TYPE (root) == vect_internal_def
3887 97 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
3888 167 : && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
3889 : }
3890 :
3891 :
3892 : /* Helper function of optimize_load_redistribution that performs the operation
3893 : recursively. */
3894 :
3895 : static slp_tree
3896 21801 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
3897 : vec_info *vinfo, unsigned int group_size,
3898 : hash_map<slp_tree, slp_tree> *load_map,
3899 : slp_tree root)
3900 : {
3901 21801 : if (slp_tree *leader = load_map->get (root))
3902 4549 : return *leader;
3903 :
3904 17252 : slp_tree node;
3905 17252 : unsigned i;
3906 :
3907 : /* For now, we don't know anything about externals so do not do anything. */
3908 17252 : if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
3909 : return NULL;
3910 12116 : else if (SLP_TREE_PERMUTE_P (root))
3911 : {
3912 : /* First convert this node into a load node and add it to the leaves
3913 : list and flatten the permute from a lane to a load one. If it's
3914 : unneeded it will be elided later. */
3915 71 : vec<stmt_vec_info> stmts;
3916 71 : stmts.create (SLP_TREE_LANES (root));
3917 71 : lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
3918 135 : for (unsigned j = 0; j < lane_perm.length (); j++)
3919 : {
3920 103 : std::pair<unsigned, unsigned> perm = lane_perm[j];
3921 103 : node = SLP_TREE_CHILDREN (root)[perm.first];
3922 :
3923 103 : if (!vect_is_slp_load_node (node)
3924 103 : || SLP_TREE_CHILDREN (node).exists ())
3925 : {
3926 39 : stmts.release ();
3927 39 : goto next;
3928 : }
3929 :
3930 64 : stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
3931 : }
3932 :
3933 32 : if (dump_enabled_p ())
3934 0 : dump_printf_loc (MSG_NOTE, vect_location,
3935 : "converting stmts on permute node %p\n",
3936 : (void *) root);
3937 :
3938 32 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
3939 32 : unsigned tree_size = 0, limit = 1;
3940 32 : node = vect_build_slp_tree (vinfo, stmts,
3941 : matches, &limit, &tree_size, bst_map);
3942 32 : if (!node)
3943 0 : stmts.release ();
3944 :
3945 32 : load_map->put (root, node);
3946 32 : return node;
3947 : }
3948 :
3949 12045 : next:
3950 12084 : load_map->put (root, NULL);
3951 :
3952 27523 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3953 : {
3954 15439 : slp_tree value
3955 15439 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3956 : node);
3957 15439 : if (value)
3958 : {
3959 32 : SLP_TREE_REF_COUNT (value)++;
3960 32 : SLP_TREE_CHILDREN (root)[i] = value;
3961 : /* ??? We know the original leafs of the replaced nodes will
3962 : be referenced by bst_map, only the permutes created by
3963 : pattern matching are not. */
3964 32 : if (SLP_TREE_REF_COUNT (node) == 1)
3965 32 : load_map->remove (node);
3966 32 : vect_free_slp_tree (node);
3967 : }
3968 : }
3969 :
3970 : return NULL;
3971 : }
3972 :
3973 : /* Temporary workaround for loads not being CSEd during SLP build. This
3974 : function will traverse the SLP tree rooted in ROOT for INSTANCE and find
3975 : VEC_PERM nodes that blend vectors from multiple nodes that all read from the
3976 : same DR such that the final operation is equal to a permuted load. Such
3977 : NODES are then directly converted into LOADS themselves. The nodes are
3978 : CSEd using BST_MAP. */
3979 :
3980 : static void
3981 4180 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
3982 : vec_info *vinfo, unsigned int group_size,
3983 : hash_map<slp_tree, slp_tree> *load_map,
3984 : slp_tree root)
3985 : {
3986 4180 : slp_tree node;
3987 4180 : unsigned i;
3988 :
3989 10542 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3990 : {
3991 6362 : slp_tree value
3992 6362 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3993 : node);
3994 6362 : if (value)
3995 : {
3996 0 : SLP_TREE_REF_COUNT (value)++;
3997 0 : SLP_TREE_CHILDREN (root)[i] = value;
3998 : /* ??? We know the original leafs of the replaced nodes will
3999 : be referenced by bst_map, only the permutes created by
4000 : pattern matching are not. */
4001 0 : if (SLP_TREE_REF_COUNT (node) == 1)
4002 0 : load_map->remove (node);
4003 0 : vect_free_slp_tree (node);
4004 : }
4005 : }
4006 4180 : }
4007 :
4008 : /* Helper function of vect_match_slp_patterns.
4009 :
4010 : Attempts to match patterns against the slp tree rooted in REF_NODE using
4011 : VINFO. Patterns are matched in post-order traversal.
4012 :
4013 : If matching is successful the value in REF_NODE is updated and returned, if
4014 : not then it is returned unchanged. */
4015 :
4016 : static bool
4017 6656351 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
4018 : slp_tree_to_load_perm_map_t *perm_cache,
4019 : slp_compat_nodes_map_t *compat_cache,
4020 : hash_set<slp_tree> *visited)
4021 : {
4022 6656351 : unsigned i;
4023 6656351 : slp_tree node = *ref_node;
4024 6656351 : bool found_p = false;
4025 6656351 : if (!node || visited->add (node))
4026 : return false;
4027 :
4028 : slp_tree child;
4029 10695611 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
4030 4995970 : found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
4031 : vinfo, perm_cache, compat_cache,
4032 : visited);
4033 :
4034 17098923 : for (unsigned x = 0; x < num__slp_patterns; x++)
4035 : {
4036 11399282 : vect_pattern *pattern
4037 11399282 : = slp_patterns[x] (perm_cache, compat_cache, ref_node);
4038 11399282 : if (pattern)
4039 : {
4040 1114 : pattern->build (vinfo);
4041 1114 : delete pattern;
4042 : found_p = true;
4043 : }
4044 : }
4045 :
4046 : return found_p;
4047 : }
4048 :
4049 : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
4050 : vec_info VINFO.
4051 :
4052 : The modified tree is returned. Patterns are tried in order and multiple
4053 : patterns may match. */
4054 :
4055 : static bool
4056 1660381 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
4057 : hash_set<slp_tree> *visited,
4058 : slp_tree_to_load_perm_map_t *perm_cache,
4059 : slp_compat_nodes_map_t *compat_cache)
4060 : {
4061 1660381 : DUMP_VECT_SCOPE ("vect_match_slp_patterns");
4062 1660381 : slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
4063 :
4064 1660381 : if (dump_enabled_p ())
4065 33590 : dump_printf_loc (MSG_NOTE, vect_location,
4066 : "Analyzing SLP tree %p for patterns\n",
4067 33590 : (void *) SLP_INSTANCE_TREE (instance));
4068 :
4069 1660381 : return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
4070 1660381 : visited);
4071 : }
4072 :
4073 : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
4074 : vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether
4075 : the stores are masked.
4076 : Return true if we could use IFN_STORE_LANES instead and if that appears
4077 : to be the better approach. */
4078 :
4079 : static bool
4080 6147 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
4081 : tree vectype, bool masked_p,
4082 : unsigned int group_size,
4083 : unsigned int new_group_size)
4084 : {
4085 6147 : if (!vectype)
4086 : {
4087 6147 : tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
4088 6147 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
4089 : }
4090 6147 : if (!vectype)
4091 : return false;
4092 : /* Allow the split if one of the two new groups would operate on full
4093 : vectors *within* rather than across one scalar loop iteration.
4094 : This is purely a heuristic, but it should work well for group
4095 : sizes of 3 and 4, where the possible splits are:
4096 :
4097 : 3->2+1: OK if the vector has exactly two elements
4098 : 4->2+2: Likewise
4099 : 4->3+1: Less clear-cut. */
4100 6147 : if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
4101 6147 : || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
4102 : return false;
4103 3500 : return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
4104 : }
4105 :
4106 : /* Analyze an SLP instance starting from a group of grouped stores. Call
4107 : vect_build_slp_tree to build a tree of packed stmts if possible.
4108 : Return FALSE if it's impossible to SLP any stmt in the loop. */
4109 :
4110 : static bool
4111 : vect_analyze_slp_instance (vec_info *vinfo,
4112 : scalar_stmts_to_slp_tree_map_t *bst_map,
4113 : stmt_vec_info stmt_info, slp_instance_kind kind,
4114 : unsigned max_tree_size, unsigned *limit,
4115 : bool force_single_lane);
4116 :
4117 : /* Build an interleaving scheme for the store sources RHS_NODES from
4118 : SCALAR_STMTS. */
4119 :
4120 : static slp_tree
4121 11853 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
4122 : vec<stmt_vec_info> &scalar_stmts)
4123 : {
4124 11853 : unsigned int group_size = scalar_stmts.length ();
4125 23706 : slp_tree node = vect_create_new_slp_node (scalar_stmts,
4126 11853 : SLP_TREE_CHILDREN
4127 : (rhs_nodes[0]).length ());
4128 11853 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
4129 11853 : for (unsigned l = 0;
4130 23733 : l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
4131 : {
4132 : /* And a permute merging all RHS SLP trees. */
4133 11880 : slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
4134 11880 : VEC_PERM_EXPR);
4135 11880 : SLP_TREE_CHILDREN (node).quick_push (perm);
4136 11880 : SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
4137 11880 : SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
4138 11880 : SLP_TREE_LANES (perm) = group_size;
4139 11880 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
4140 86356 : for (unsigned j = 0; j < rhs_nodes.length (); ++j)
4141 : {
4142 31298 : SLP_TREE_CHILDREN (perm)
4143 31298 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
4144 31298 : SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
4145 31298 : for (unsigned k = 0;
4146 64999 : k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
4147 : {
4148 : /* ??? We should populate SLP_TREE_SCALAR_STMTS
4149 : or SLP_TREE_SCALAR_OPS but then we might have
4150 : a mix of both in our children. */
4151 33701 : SLP_TREE_LANE_PERMUTATION (perm)
4152 33701 : .quick_push (std::make_pair (j, k));
4153 : }
4154 : }
4155 :
4156 : /* Now we have a single permute node but we cannot code-generate
4157 : the case with more than two inputs.
4158 : Perform pairwise reduction, reducing the two inputs
4159 : with the least number of lanes to one and then repeat until
4160 : we end up with two inputs. That scheme makes sure we end
4161 : up with permutes satisfying the restriction of requiring at
4162 : most two vector inputs to produce a single vector output
4163 : when the number of lanes is even. */
4164 19418 : while (SLP_TREE_CHILDREN (perm).length () > 2)
4165 : {
4166 : /* When we have three equal sized groups left the pairwise
4167 : reduction does not result in a scheme that avoids using
4168 : three vectors. Instead merge the first two groups
4169 : to the final size with do-not-care elements (chosen
4170 : from the first group) and then merge with the third.
4171 : { A0, B0, x, A1, B1, x, ... }
4172 : -> { A0, B0, C0, A1, B1, C1, ... }
4173 : This handles group size of three (and at least
4174 : power-of-two multiples of that). */
4175 7538 : if (SLP_TREE_CHILDREN (perm).length () == 3
4176 3376 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4177 3376 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
4178 7538 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4179 2489 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
4180 : {
4181 2183 : int ai = 0;
4182 2183 : int bi = 1;
4183 2183 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4184 2183 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4185 2183 : unsigned n = SLP_TREE_LANES (perm);
4186 :
4187 2183 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4188 2183 : SLP_TREE_LANES (permab) = n;
4189 2183 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4190 2183 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4191 : /* ??? Should be NULL but that's not expected. */
4192 2183 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4193 2183 : SLP_TREE_CHILDREN (permab).quick_push (a);
4194 6563 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4195 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4196 2197 : .quick_push (std::make_pair (0, k));
4197 2183 : SLP_TREE_CHILDREN (permab).quick_push (b);
4198 6563 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4199 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4200 2197 : .quick_push (std::make_pair (1, k));
4201 : /* Push the do-not-care lanes. */
4202 4380 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4203 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4204 2197 : .quick_push (std::make_pair (0, k));
4205 :
4206 : /* Put the merged node into 'perm', in place of a. */
4207 2183 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4208 : /* Adjust the references to b in the permutation
4209 : of perm and to the later children which we'll
4210 : remove. */
4211 8774 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4212 : {
4213 6591 : std::pair<unsigned, unsigned> &p
4214 6591 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4215 6591 : if (p.first == (unsigned) bi)
4216 : {
4217 2197 : p.first = ai;
4218 2197 : p.second += SLP_TREE_LANES (a);
4219 : }
4220 4394 : else if (p.first > (unsigned) bi)
4221 2197 : p.first--;
4222 : }
4223 2183 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4224 2183 : break;
4225 : }
4226 :
4227 : /* Pick the two nodes with the least number of lanes,
4228 : prefer the earliest candidate and maintain ai < bi. */
4229 : int ai = -1;
4230 : int bi = -1;
4231 50500 : for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
4232 : {
4233 45145 : if (ai == -1)
4234 5355 : ai = ci;
4235 39790 : else if (bi == -1)
4236 5355 : bi = ci;
4237 34435 : else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4238 34435 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
4239 34435 : || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4240 28605 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
4241 : {
4242 12550 : if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
4243 6275 : <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
4244 2892 : bi = ci;
4245 : else
4246 : {
4247 3383 : ai = bi;
4248 3383 : bi = ci;
4249 : }
4250 : }
4251 : }
4252 :
4253 : /* Produce a merge of nodes ai and bi. */
4254 5355 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4255 5355 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4256 5355 : unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
4257 5355 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4258 5355 : SLP_TREE_LANES (permab) = n;
4259 5355 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4260 5355 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4261 : /* ??? Should be NULL but that's not expected. */
4262 5355 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4263 5355 : SLP_TREE_CHILDREN (permab).quick_push (a);
4264 19589 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4265 8879 : SLP_TREE_LANE_PERMUTATION (permab)
4266 8879 : .quick_push (std::make_pair (0, k));
4267 5355 : SLP_TREE_CHILDREN (permab).quick_push (b);
4268 18865 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4269 8155 : SLP_TREE_LANE_PERMUTATION (permab)
4270 8155 : .quick_push (std::make_pair (1, k));
4271 :
4272 : /* Put the merged node into 'perm', in place of a. */
4273 5355 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4274 : /* Adjust the references to b in the permutation
4275 : of perm and to the later children which we'll
4276 : remove. */
4277 81325 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4278 : {
4279 75970 : std::pair<unsigned, unsigned> &p
4280 75970 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4281 75970 : if (p.first == (unsigned) bi)
4282 : {
4283 8155 : p.first = ai;
4284 8155 : p.second += SLP_TREE_LANES (a);
4285 : }
4286 67815 : else if (p.first > (unsigned) bi)
4287 28698 : p.first--;
4288 : }
4289 5355 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4290 : }
4291 : }
4292 :
4293 11853 : return node;
4294 : }
4295 :
4296 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4297 : of KIND. Return true if successful. SCALAR_STMTS is owned by this
4298 : function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
4299 : the caller upon failure. */
4300 :
4301 : static bool
4302 2140649 : vect_build_slp_instance (vec_info *vinfo,
4303 : slp_instance_kind kind,
4304 : vec<stmt_vec_info> &scalar_stmts,
4305 : vec<stmt_vec_info> &root_stmt_infos,
4306 : vec<tree> &remain,
4307 : unsigned max_tree_size, unsigned *limit,
4308 : scalar_stmts_to_slp_tree_map_t *bst_map,
4309 : bool force_single_lane)
4310 : {
4311 : /* If there's no budget left bail out early. */
4312 2140649 : if (*limit == 0)
4313 : {
4314 16098 : scalar_stmts.release ();
4315 16098 : return false;
4316 : }
4317 :
4318 2124551 : if (kind == slp_inst_kind_ctor)
4319 : {
4320 15570 : if (dump_enabled_p ())
4321 102 : dump_printf_loc (MSG_NOTE, vect_location,
4322 : "Analyzing vectorizable constructor: %G\n",
4323 51 : root_stmt_infos[0]->stmt);
4324 : }
4325 2108981 : else if (kind == slp_inst_kind_gcond)
4326 : {
4327 305120 : if (dump_enabled_p ())
4328 6730 : dump_printf_loc (MSG_NOTE, vect_location,
4329 : "Analyzing vectorizable control flow: %G",
4330 3365 : root_stmt_infos[0]->stmt);
4331 : }
4332 1803861 : else if (kind == slp_inst_kind_bb_reduc)
4333 : {
4334 1399712 : if (dump_enabled_p ())
4335 7538 : dump_printf_loc (MSG_NOTE, vect_location,
4336 : "Analyzing vectorizable BB reduction: %G",
4337 3769 : root_stmt_infos[0]->stmt);
4338 : }
4339 :
4340 2124551 : if (dump_enabled_p ())
4341 : {
4342 28546 : dump_printf_loc (MSG_NOTE, vect_location,
4343 : "Starting SLP discovery for\n");
4344 61577 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4345 66062 : dump_printf_loc (MSG_NOTE, vect_location,
4346 33031 : " %G", scalar_stmts[i]->stmt);
4347 : }
4348 :
4349 : /* Build the tree for the SLP instance. */
4350 2124551 : unsigned int group_size = scalar_stmts.length ();
4351 2124551 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4352 2124551 : unsigned tree_size = 0;
4353 :
4354 2124551 : slp_tree node = NULL;
4355 2124551 : if (group_size > 1 && force_single_lane)
4356 : {
4357 0 : matches[0] = 0;
4358 0 : matches[1] = -1;
4359 : }
4360 : else
4361 2124551 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4362 : &tree_size, bst_map);
4363 2124551 : if (node != NULL)
4364 : {
4365 : /* Create a new SLP instance. */
4366 851298 : slp_instance new_instance = XNEW (class _slp_instance);
4367 851298 : SLP_INSTANCE_TREE (new_instance) = node;
4368 851298 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4369 851298 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4370 851298 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4371 851298 : SLP_INSTANCE_KIND (new_instance) = kind;
4372 851298 : new_instance->reduc_phis = NULL;
4373 851298 : new_instance->cost_vec = vNULL;
4374 851298 : new_instance->subgraph_entries = vNULL;
4375 :
4376 851298 : if (dump_enabled_p ())
4377 25203 : dump_printf_loc (MSG_NOTE, vect_location,
4378 : "SLP size %u vs. limit %u.\n",
4379 : tree_size, max_tree_size);
4380 :
4381 851298 : vinfo->slp_instances.safe_push (new_instance);
4382 :
4383 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4384 : the number of SLP lanes of the root in a few places.
4385 : Verify that assumption holds. */
4386 851298 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4387 : == group_size);
4388 :
4389 851298 : if (dump_enabled_p ())
4390 : {
4391 25203 : if (kind == slp_inst_kind_reduc_group)
4392 1779 : dump_printf_loc (MSG_NOTE, vect_location,
4393 : "SLP discovery of size %d reduction group "
4394 : "succeeded\n", group_size);
4395 25203 : dump_printf_loc (MSG_NOTE, vect_location,
4396 : "Final SLP tree for instance %p:\n",
4397 : (void *) new_instance);
4398 25203 : vect_print_slp_graph (MSG_NOTE, vect_location,
4399 : SLP_INSTANCE_TREE (new_instance));
4400 : }
4401 :
4402 851298 : return true;
4403 : }
4404 : /* Failed to SLP. */
4405 :
4406 : /* While we arrive here even with slp_inst_kind_store we should only
4407 : for group_size == 1. The code to split store groups is only in
4408 : vect_analyze_slp_instance now. */
4409 1273253 : gcc_assert (kind != slp_inst_kind_store || group_size == 1);
4410 :
4411 : /* For BB vectorization we get failures only in case of the need of
4412 : unrolling, as otherwise we'll simply get operands built from scalars.
4413 : Iff there is any mismatches in the toplevel stmts those will prevail,
4414 : otherwise we get the non-power-of-two tail of the lanes failed.
4415 : For BB reductions we mainly want to catch the first case so we pick
4416 : a more useful subset of lanes to reduce. Pick the largest matching
4417 : subset of that covers half of the group or more. */
4418 1273253 : if (kind == slp_inst_kind_bb_reduc && matches[0] != -1)
4419 : {
4420 680321 : unsigned *n_matching = XALLOCAVEC (unsigned, group_size);
4421 680321 : memset (n_matching, 0, sizeof (unsigned) * group_size);
4422 2201933 : for (unsigned i = 0; i < group_size; ++i)
4423 1521612 : if (matches[i] != -2 && matches[i] != -1)
4424 1282647 : n_matching[matches[i].v]++;
4425 : unsigned largest_i = 0;
4426 1521612 : for (unsigned i = 1; i < group_size; ++i)
4427 841291 : if (n_matching[i] > n_matching[largest_i])
4428 64017 : largest_i = i;
4429 : /* Pick the largest matching part and put the rest to remain. */
4430 680321 : if (n_matching[largest_i] >= 2
4431 51159 : && n_matching[largest_i] >= group_size / 2)
4432 : {
4433 : /* As we know the matches[] stmts match up, recursing for
4434 : non-power-of-two sizes will just force-fail the tail
4435 : for us at hopefully optimal vector size and succesfully
4436 : finish discovery. */
4437 49797 : vec<stmt_vec_info> scalar_stmts2;
4438 49797 : scalar_stmts2.create (n_matching[largest_i]);
4439 294775 : for (unsigned i = 0; i < group_size; ++i)
4440 195181 : if (matches[i] == (int)largest_i)
4441 133058 : scalar_stmts2.quick_push (scalar_stmts[i]);
4442 : else
4443 62123 : remain.safe_push
4444 66106 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4445 49797 : if (dump_enabled_p ())
4446 228 : dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d "
4447 : "non-matching lanes to scalar remains\n",
4448 228 : scalar_stmts.length () - scalar_stmts2.length ());
4449 49797 : scalar_stmts.release ();
4450 49797 : return vect_build_slp_instance (vinfo, kind, scalar_stmts2,
4451 : root_stmt_infos, remain,
4452 : max_tree_size, limit, bst_map,
4453 : force_single_lane);
4454 : }
4455 : }
4456 :
4457 : /* Free the allocated memory. */
4458 1223456 : scalar_stmts.release ();
4459 :
4460 : /* Failed to SLP. */
4461 1223456 : if (dump_enabled_p ())
4462 3229 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4463 : return false;
4464 : }
4465 :
4466 : /* Analyze an SLP instance starting from a the start of a reduction chain.
4467 : Call vect_build_slp_tree to build a tree of packed stmts if possible.
4468 : Return FALSE if SLP build fails. */
4469 :
4470 : static bool
4471 70486 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
4472 : scalar_stmts_to_slp_tree_map_t *bst_map,
4473 : stmt_vec_info scalar_stmt,
4474 : unsigned max_tree_size, unsigned *limit)
4475 : {
4476 70486 : vec<stmt_vec_info> scalar_stmts = vNULL;
4477 :
4478 70486 : bool fail = false;
4479 : /* ??? We could leave operation code checking to SLP discovery. */
4480 70486 : code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
4481 : (vect_orig_stmt (scalar_stmt)));
4482 70486 : bool first = true;
4483 70486 : stmt_vec_info next_stmt = scalar_stmt;
4484 78937 : do
4485 : {
4486 78937 : stmt_vec_info stmt = next_stmt;
4487 78937 : gimple_match_op op, orig_op;
4488 78937 : if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
4489 0 : gcc_unreachable ();
4490 157874 : tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
4491 78937 : STMT_VINFO_REDUC_IDX (stmt));
4492 78937 : next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
4493 78937 : gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
4494 : || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
4495 84898 : if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)),
4496 : &orig_op))
4497 0 : gcc_unreachable ();
4498 78937 : if (CONVERT_EXPR_CODE_P (op.code)
4499 4999 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
4500 83924 : && (first
4501 2498 : || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
4502 : ;
4503 74010 : else if (code != orig_op.code)
4504 : {
4505 907 : fail = true;
4506 907 : break;
4507 : }
4508 : else
4509 73103 : scalar_stmts.safe_push (stmt);
4510 78030 : first = false;
4511 : }
4512 78030 : while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
4513 70486 : if (fail)
4514 907 : return false;
4515 :
4516 : /* When the SSA def chain through reduc-idx does not form a natural
4517 : reduction chain try to linearize an associative operation manually. */
4518 69579 : if (scalar_stmts.length () == 1
4519 68665 : && code.is_tree_code ()
4520 62593 : && associative_tree_code ((tree_code)code)
4521 : /* We may not associate if a fold-left reduction is required. */
4522 130751 : && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
4523 : (scalar_stmts[0]->stmt)),
4524 : code))
4525 : {
4526 : /* Remember a stmt with the actual reduction operation. */
4527 57695 : stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
4528 57695 : auto_vec<chain_op_t> chain;
4529 57695 : auto_vec<std::pair<tree_code, gimple *> > worklist;
4530 57695 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
4531 57695 : if (is_a <gassign *> (scalar_stmts[0]->stmt)
4532 : /* We cannot linearize an operation that vect_slp_linearize_chain
4533 : would not put on its worklist. */
4534 57695 : && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
4535 : {
4536 57039 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4537 57039 : scalar_stmts[0]->stmt, op_stmt,
4538 : other_op_stmt,
4539 : NULL);
4540 :
4541 57039 : scalar_stmts.truncate (0);
4542 57039 : stmt_vec_info tail = NULL;
4543 285493 : for (auto el : chain)
4544 : {
4545 114675 : if (el.dt == vect_external_def
4546 114675 : || el.dt == vect_constant_def
4547 114675 : || el.code != (tree_code) code)
4548 : {
4549 299 : scalar_stmts.release ();
4550 299 : return false;
4551 : }
4552 114376 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4553 114376 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4554 112090 : || STMT_VINFO_REDUC_DEF (stmt))
4555 : {
4556 56943 : gcc_assert (tail == NULL);
4557 56943 : tail = stmt;
4558 56943 : continue;
4559 : }
4560 57433 : scalar_stmts.safe_push (stmt);
4561 : }
4562 56740 : gcc_assert (tail);
4563 : }
4564 :
4565 : /* When this linearization didn't produce a chain see if stripping
4566 : a wrapping sign conversion produces one. */
4567 57396 : if (scalar_stmts.length () == 1
4568 57396 : && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
4569 : || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
4570 : {
4571 55595 : gimple *stmt = scalar_stmts[0]->stmt;
4572 55595 : if (!is_gimple_assign (stmt)
4573 54435 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
4574 4772 : || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
4575 60367 : || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
4576 4772 : TREE_TYPE (gimple_assign_rhs1 (stmt))))
4577 : {
4578 53815 : scalar_stmts.release ();
4579 53815 : return false;
4580 : }
4581 1780 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
4582 1780 : if (!is_gimple_assign (stmt)
4583 1780 : || gimple_assign_rhs_code (stmt) != (tree_code)code)
4584 : {
4585 1762 : scalar_stmts.release ();
4586 1762 : return false;
4587 : }
4588 18 : chain.truncate (0);
4589 18 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4590 : stmt, op_stmt, other_op_stmt, NULL);
4591 :
4592 18 : scalar_stmts.truncate (0);
4593 18 : stmt_vec_info tail = NULL;
4594 88 : for (auto el : chain)
4595 : {
4596 42 : if (el.dt == vect_external_def
4597 42 : || el.dt == vect_constant_def
4598 42 : || el.code != (tree_code) code)
4599 : {
4600 8 : scalar_stmts.release ();
4601 8 : return false;
4602 : }
4603 34 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4604 34 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4605 34 : || STMT_VINFO_REDUC_DEF (stmt))
4606 : {
4607 0 : gcc_assert (tail == NULL);
4608 0 : tail = stmt;
4609 0 : continue;
4610 : }
4611 34 : scalar_stmts.safe_push (stmt);
4612 : }
4613 : /* Unlike the above this does not include the reduction SSA
4614 : cycle. */
4615 10 : gcc_assert (!tail);
4616 : }
4617 :
4618 1811 : if (scalar_stmts.length () < 2)
4619 : {
4620 1678 : scalar_stmts.release ();
4621 1678 : return false;
4622 : }
4623 :
4624 133 : if (dump_enabled_p ())
4625 : {
4626 43 : dump_printf_loc (MSG_NOTE, vect_location,
4627 : "Starting SLP discovery of reduction chain for\n");
4628 173 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4629 260 : dump_printf_loc (MSG_NOTE, vect_location,
4630 130 : " %G", scalar_stmts[i]->stmt);
4631 : }
4632 :
4633 133 : unsigned int group_size = scalar_stmts.length ();
4634 133 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4635 133 : unsigned tree_size = 0;
4636 133 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4637 133 : &tree_size, bst_map);
4638 133 : if (!node)
4639 : {
4640 46 : scalar_stmts.release ();
4641 46 : return false;
4642 : }
4643 :
4644 87 : unsigned cycle_id = vinfo->reduc_infos.length ();
4645 87 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
4646 87 : vinfo->reduc_infos.safe_push (reduc_info);
4647 87 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
4648 87 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
4649 87 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
4650 87 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
4651 87 : reduc_info->is_reduc_chain = true;
4652 :
4653 : /* Build the node for the PHI and possibly the conversions. */
4654 87 : slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
4655 87 : SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
4656 87 : phis->cycle_info.id = cycle_id;
4657 87 : SLP_TREE_LANES (phis) = group_size;
4658 87 : if (reduc_scalar_stmt == scalar_stmt)
4659 83 : SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
4660 : else
4661 4 : SLP_TREE_VECTYPE (phis)
4662 4 : = signed_or_unsigned_type_for (TYPE_UNSIGNED
4663 : (TREE_TYPE (gimple_get_lhs
4664 : (scalar_stmt->stmt))),
4665 : SLP_TREE_VECTYPE (node));
4666 : /* ??? vect_cse_slp_nodes cannot cope with cycles without any
4667 : SLP_TREE_SCALAR_STMTS. */
4668 87 : SLP_TREE_SCALAR_STMTS (phis).create (group_size);
4669 513 : for (unsigned i = 0; i < group_size; ++i)
4670 339 : SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
4671 :
4672 87 : slp_tree op_input = phis;
4673 87 : if (reduc_scalar_stmt != scalar_stmt)
4674 : {
4675 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4676 4 : SLP_TREE_REPRESENTATIVE (conv)
4677 4 : = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
4678 4 : STMT_VINFO_REDUC_IDX
4679 : (reduc_scalar_stmt)));
4680 4 : SLP_TREE_CHILDREN (conv).quick_push (phis);
4681 4 : conv->cycle_info.id = cycle_id;
4682 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4683 4 : SLP_TREE_LANES (conv) = group_size;
4684 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
4685 4 : SLP_TREE_SCALAR_STMTS (conv) = vNULL;
4686 4 : op_input = conv;
4687 : }
4688 :
4689 87 : slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
4690 87 : SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
4691 87 : SLP_TREE_CHILDREN (reduc).quick_push (op_input);
4692 87 : SLP_TREE_CHILDREN (reduc).quick_push (node);
4693 87 : reduc->cycle_info.id = cycle_id;
4694 87 : SLP_TREE_REDUC_IDX (reduc) = 0;
4695 87 : SLP_TREE_LANES (reduc) = group_size;
4696 87 : SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
4697 : /* ??? For the reduction epilogue we need a live lane. */
4698 87 : SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
4699 87 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
4700 426 : for (unsigned i = 1; i < group_size; ++i)
4701 252 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
4702 :
4703 87 : if (reduc_scalar_stmt != scalar_stmt)
4704 : {
4705 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4706 4 : SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
4707 4 : SLP_TREE_CHILDREN (conv).quick_push (reduc);
4708 4 : conv->cycle_info.id = cycle_id;
4709 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4710 4 : SLP_TREE_LANES (conv) = group_size;
4711 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
4712 : /* ??? For the reduction epilogue we need a live lane. */
4713 4 : SLP_TREE_SCALAR_STMTS (conv).create (group_size);
4714 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
4715 12 : for (unsigned i = 1; i < group_size; ++i)
4716 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
4717 4 : reduc = conv;
4718 : }
4719 :
4720 87 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
4721 87 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4722 87 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4723 87 : SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
4724 87 : SLP_TREE_REF_COUNT (reduc)++;
4725 :
4726 : /* Create a new SLP instance. */
4727 87 : slp_instance new_instance = XNEW (class _slp_instance);
4728 87 : SLP_INSTANCE_TREE (new_instance) = reduc;
4729 87 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4730 87 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4731 87 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4732 87 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4733 87 : new_instance->reduc_phis = NULL;
4734 87 : new_instance->cost_vec = vNULL;
4735 87 : new_instance->subgraph_entries = vNULL;
4736 :
4737 87 : vinfo->slp_instances.safe_push (new_instance);
4738 :
4739 87 : if (dump_enabled_p ())
4740 : {
4741 33 : dump_printf_loc (MSG_NOTE, vect_location,
4742 : "Final SLP tree for instance %p:\n",
4743 : (void *) new_instance);
4744 33 : vect_print_slp_graph (MSG_NOTE, vect_location,
4745 : SLP_INSTANCE_TREE (new_instance));
4746 : }
4747 :
4748 : return true;
4749 57695 : }
4750 :
4751 11884 : if (scalar_stmts.length () <= 1)
4752 : {
4753 10976 : scalar_stmts.release ();
4754 10976 : return false;
4755 : }
4756 :
4757 908 : scalar_stmts.reverse ();
4758 908 : stmt_vec_info reduc_phi_info = next_stmt;
4759 :
4760 : /* Build the tree for the SLP instance. */
4761 908 : vec<stmt_vec_info> root_stmt_infos = vNULL;
4762 908 : vec<tree> remain = vNULL;
4763 :
4764 908 : if (dump_enabled_p ())
4765 : {
4766 239 : dump_printf_loc (MSG_NOTE, vect_location,
4767 : "Starting SLP discovery of reduction chain for\n");
4768 1207 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4769 1936 : dump_printf_loc (MSG_NOTE, vect_location,
4770 968 : " %G", scalar_stmts[i]->stmt);
4771 : }
4772 :
4773 : /* Build the tree for the SLP instance. */
4774 908 : unsigned int group_size = scalar_stmts.length ();
4775 908 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4776 908 : unsigned tree_size = 0;
4777 :
4778 : /* ??? We need this only for SLP discovery. */
4779 5264 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4780 4356 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
4781 :
4782 908 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4783 908 : &tree_size, bst_map);
4784 :
4785 5264 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4786 4356 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
4787 :
4788 908 : if (node != NULL)
4789 : {
4790 : /* Create a new SLP instance. */
4791 611 : slp_instance new_instance = XNEW (class _slp_instance);
4792 611 : SLP_INSTANCE_TREE (new_instance) = node;
4793 611 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4794 611 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4795 611 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4796 611 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4797 611 : new_instance->reduc_phis = NULL;
4798 611 : new_instance->cost_vec = vNULL;
4799 611 : new_instance->subgraph_entries = vNULL;
4800 :
4801 611 : vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
4802 611 : reduc_info->is_reduc_chain = true;
4803 :
4804 611 : if (dump_enabled_p ())
4805 153 : dump_printf_loc (MSG_NOTE, vect_location,
4806 : "SLP size %u vs. limit %u.\n",
4807 : tree_size, max_tree_size);
4808 :
4809 : /* Fixup SLP reduction chains. If this is a reduction chain with
4810 : a conversion in front amend the SLP tree with a node for that. */
4811 611 : gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
4812 611 : if (is_gimple_assign (scalar_def)
4813 611 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
4814 : {
4815 43 : stmt_vec_info conv_info = vect_stmt_to_vectorize
4816 43 : (STMT_VINFO_REDUC_DEF (reduc_phi_info));
4817 43 : scalar_stmts = vNULL;
4818 43 : scalar_stmts.create (group_size);
4819 178 : for (unsigned i = 0; i < group_size; ++i)
4820 92 : scalar_stmts.quick_push (conv_info);
4821 43 : slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
4822 43 : SLP_TREE_VECTYPE (conv)
4823 43 : = get_vectype_for_scalar_type (vinfo,
4824 43 : TREE_TYPE
4825 : (gimple_assign_lhs (scalar_def)),
4826 : group_size);
4827 43 : SLP_TREE_REDUC_IDX (conv) = 0;
4828 43 : conv->cycle_info.id = node->cycle_info.id;
4829 43 : SLP_TREE_CHILDREN (conv).quick_push (node);
4830 43 : SLP_INSTANCE_TREE (new_instance) = conv;
4831 : }
4832 : /* Fill the backedge child of the PHI SLP node. The
4833 : general matching code cannot find it because the
4834 : scalar code does not reflect how we vectorize the
4835 : reduction. */
4836 611 : use_operand_p use_p;
4837 611 : imm_use_iterator imm_iter;
4838 611 : class loop *loop = LOOP_VINFO_LOOP (vinfo);
4839 1968 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
4840 : gimple_get_lhs (scalar_def))
4841 : /* There are exactly two non-debug uses, the reduction
4842 : PHI and the loop-closed PHI node. */
4843 1357 : if (!is_gimple_debug (USE_STMT (use_p))
4844 1357 : && gimple_bb (USE_STMT (use_p)) == loop->header)
4845 : {
4846 611 : auto_vec<stmt_vec_info, 64> phis (group_size);
4847 611 : stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
4848 4213 : for (unsigned i = 0; i < group_size; ++i)
4849 3602 : phis.quick_push (phi_info);
4850 611 : slp_tree *phi_node = bst_map->get (phis);
4851 611 : unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
4852 1222 : SLP_TREE_CHILDREN (*phi_node)[dest_idx]
4853 611 : = SLP_INSTANCE_TREE (new_instance);
4854 611 : SLP_INSTANCE_TREE (new_instance)->refcnt++;
4855 611 : }
4856 :
4857 611 : vinfo->slp_instances.safe_push (new_instance);
4858 :
4859 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4860 : the number of SLP lanes of the root in a few places.
4861 : Verify that assumption holds. */
4862 611 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4863 : == group_size);
4864 :
4865 611 : if (dump_enabled_p ())
4866 : {
4867 153 : dump_printf_loc (MSG_NOTE, vect_location,
4868 : "Final SLP tree for instance %p:\n",
4869 : (void *) new_instance);
4870 153 : vect_print_slp_graph (MSG_NOTE, vect_location,
4871 : SLP_INSTANCE_TREE (new_instance));
4872 : }
4873 :
4874 611 : return true;
4875 : }
4876 :
4877 : /* Failed to SLP. */
4878 297 : scalar_stmts.release ();
4879 297 : if (dump_enabled_p ())
4880 86 : dump_printf_loc (MSG_NOTE, vect_location,
4881 : "SLP discovery of reduction chain failed\n");
4882 : return false;
4883 : }
4884 :
4885 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4886 : of KIND. Return true if successful. */
4887 :
4888 : static bool
4889 97892 : vect_analyze_slp_reduction (loop_vec_info vinfo,
4890 : stmt_vec_info scalar_stmt,
4891 : unsigned max_tree_size, unsigned *limit,
4892 : scalar_stmts_to_slp_tree_map_t *bst_map,
4893 : bool force_single_lane)
4894 : {
4895 97892 : slp_instance_kind kind = slp_inst_kind_reduc_group;
4896 :
4897 : /* Try to gather a reduction chain. Only attempt if there's budget left
4898 : since chain analysis may build multi-lane trees that consume limit. */
4899 97892 : if (! force_single_lane
4900 70765 : && *limit != 0
4901 70765 : && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
4902 168378 : && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
4903 : max_tree_size, limit))
4904 : return true;
4905 :
4906 97194 : vec<stmt_vec_info> scalar_stmts;
4907 97194 : scalar_stmts.create (1);
4908 97194 : scalar_stmts.quick_push (scalar_stmt);
4909 :
4910 97194 : if (dump_enabled_p ())
4911 : {
4912 4028 : dump_printf_loc (MSG_NOTE, vect_location,
4913 : "Starting SLP discovery for\n");
4914 8056 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4915 8056 : dump_printf_loc (MSG_NOTE, vect_location,
4916 4028 : " %G", scalar_stmts[i]->stmt);
4917 : }
4918 :
4919 : /* Build the tree for the SLP instance. */
4920 97194 : unsigned int group_size = scalar_stmts.length ();
4921 97194 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4922 97194 : unsigned tree_size = 0;
4923 :
4924 97194 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4925 : &tree_size, bst_map);
4926 97194 : if (node != NULL)
4927 : {
4928 : /* Create a new SLP instance. */
4929 95397 : slp_instance new_instance = XNEW (class _slp_instance);
4930 95397 : SLP_INSTANCE_TREE (new_instance) = node;
4931 95397 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4932 95397 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4933 95397 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4934 95397 : SLP_INSTANCE_KIND (new_instance) = kind;
4935 95397 : new_instance->reduc_phis = NULL;
4936 95397 : new_instance->cost_vec = vNULL;
4937 95397 : new_instance->subgraph_entries = vNULL;
4938 :
4939 95397 : if (dump_enabled_p ())
4940 3945 : dump_printf_loc (MSG_NOTE, vect_location,
4941 : "SLP size %u vs. limit %u.\n",
4942 : tree_size, max_tree_size);
4943 :
4944 95397 : vinfo->slp_instances.safe_push (new_instance);
4945 :
4946 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4947 : the number of SLP lanes of the root in a few places.
4948 : Verify that assumption holds. */
4949 95397 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4950 : == group_size);
4951 :
4952 95397 : if (dump_enabled_p ())
4953 : {
4954 3945 : dump_printf_loc (MSG_NOTE, vect_location,
4955 : "Final SLP tree for instance %p:\n",
4956 : (void *) new_instance);
4957 3945 : vect_print_slp_graph (MSG_NOTE, vect_location,
4958 : SLP_INSTANCE_TREE (new_instance));
4959 : }
4960 :
4961 95397 : return true;
4962 : }
4963 : /* Failed to SLP. */
4964 :
4965 : /* Free the allocated memory. */
4966 1797 : scalar_stmts.release ();
4967 :
4968 : /* Failed to SLP. */
4969 1797 : if (dump_enabled_p ())
4970 83 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4971 : return false;
4972 : }
4973 :
4974 : /* Analyze SCALAR_STMTS for SLP reduction groups. If successful add
4975 : SLP instances for them and return true. Any groups of size one
4976 : are handled appropriately. Otherwise return false. */
4977 :
4978 : static bool
4979 12140 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
4980 : vec<stmt_vec_info> scalar_stmts,
4981 : scalar_stmts_to_slp_tree_map_t *bst_map,
4982 : unsigned max_tree_size, unsigned *limit)
4983 : {
4984 : /* Try to form a reduction group. Size-1 groups are not suitable
4985 : for SLP reduction and should fall back to single-lane reduction. */
4986 12140 : unsigned int group_size = scalar_stmts.length ();
4987 12140 : gcc_assert (group_size > 1);
4988 12140 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4989 12140 : unsigned tree_size = 0;
4990 12140 : slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts, matches, limit,
4991 : &tree_size, bst_map);
4992 12140 : if (node)
4993 : {
4994 : /* Create a new SLP instance. */
4995 4561 : slp_instance new_instance = XNEW (class _slp_instance);
4996 4561 : SLP_INSTANCE_TREE (new_instance) = node;
4997 4561 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4998 4561 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4999 4561 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
5000 4561 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
5001 4561 : new_instance->reduc_phis = NULL;
5002 4561 : new_instance->cost_vec = vNULL;
5003 4561 : new_instance->subgraph_entries = vNULL;
5004 :
5005 4561 : if (dump_enabled_p ())
5006 203 : dump_printf_loc (MSG_NOTE, vect_location,
5007 : "SLP size %u vs. limit %u.\n",
5008 : tree_size, max_tree_size);
5009 :
5010 4561 : loop_vinfo->slp_instances.safe_push (new_instance);
5011 :
5012 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5013 : the number of SLP lanes of the root in a few places.
5014 : Verify that assumption holds. */
5015 4561 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5016 : == group_size);
5017 :
5018 4561 : if (dump_enabled_p ())
5019 : {
5020 203 : dump_printf_loc (MSG_NOTE, vect_location,
5021 : "SLP discovery of size %d reduction group "
5022 : "succeeded\n", group_size);
5023 203 : dump_printf_loc (MSG_NOTE, vect_location,
5024 : "Final SLP tree for instance %p:\n",
5025 : (void *) new_instance);
5026 203 : vect_print_slp_graph (MSG_NOTE, vect_location,
5027 : SLP_INSTANCE_TREE (new_instance));
5028 : }
5029 :
5030 4561 : return true;
5031 : }
5032 :
5033 : /* Bail on fatal error. */
5034 7579 : if (matches[0] == -1)
5035 : return false;
5036 :
5037 : /* When analysis as a single SLP reduction group failed try to form
5038 : sub-groups by collecting matching lanes. Everything not
5039 : covered by a sub-group gets single-reduction treatment. */
5040 7455 : vec<stmt_vec_info> cands = vNULL;
5041 7455 : int first_cand = -1;
5042 31332 : do
5043 : {
5044 31332 : cands.truncate (0);
5045 31332 : cands.reserve (group_size, true);
5046 31332 : int new_first_cand = -1;
5047 264782 : for (unsigned i = 0; i < group_size; ++i)
5048 202118 : if (matches[i] == first_cand)
5049 : {
5050 36347 : cands.quick_push (scalar_stmts[i]);
5051 36347 : scalar_stmts[i] = NULL;
5052 : }
5053 165771 : else if (new_first_cand == -1
5054 165771 : && matches[i] > first_cand)
5055 : {
5056 23910 : gcc_assert (matches[i] == (int)i);
5057 : new_first_cand = i;
5058 : }
5059 :
5060 31332 : if (cands.is_empty ())
5061 : ;
5062 24016 : else if (cands.length () == 1)
5063 : {
5064 33092 : if (! vect_analyze_slp_reduction (loop_vinfo,
5065 16546 : vect_stmt_to_vectorize (cands[0]),
5066 : max_tree_size, limit,
5067 : bst_map, false))
5068 : {
5069 36 : cands.release ();
5070 36 : return false;
5071 : }
5072 16510 : cands.release ();
5073 : }
5074 : /* Try to form a reduction group. */
5075 7470 : else if (!vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
5076 : max_tree_size, limit))
5077 : {
5078 14 : cands.release ();
5079 14 : return false;
5080 : }
5081 : else
5082 7456 : cands = vNULL;
5083 31282 : first_cand = new_first_cand;
5084 : }
5085 31282 : while (first_cand >= 0);
5086 7405 : cands.release ();
5087 :
5088 7405 : scalar_stmts.release ();
5089 7405 : return true;
5090 : }
5091 :
5092 : /* Analyze reductions in LOOP_VINFO and populate SLP instances
5093 : accordingly. Returns false if something fails. */
5094 :
5095 : static bool
5096 533325 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
5097 : unsigned max_tree_size, unsigned *limit,
5098 : scalar_stmts_to_slp_tree_map_t *bst_map,
5099 : bool force_single_lane)
5100 : {
5101 533325 : if (loop_vinfo->reductions.is_empty ())
5102 : return true;
5103 :
5104 : /* Collect reduction statements we can combine into
5105 : a SLP reduction. */
5106 75687 : vec<stmt_vec_info> scalar_stmts;
5107 75687 : scalar_stmts.create (loop_vinfo->reductions.length ());
5108 335067 : for (auto next_info : loop_vinfo->reductions)
5109 : {
5110 108006 : next_info = vect_stmt_to_vectorize (next_info);
5111 108006 : if ((STMT_VINFO_RELEVANT_P (next_info)
5112 14 : || STMT_VINFO_LIVE_P (next_info))
5113 : /* ??? Make sure we didn't skip a conversion around a
5114 : reduction path. In that case we'd have to reverse
5115 : engineer that conversion stmt following the chain using
5116 : reduc_idx and from the PHI using reduc_def. */
5117 107992 : && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
5118 107992 : || (STMT_VINFO_DEF_TYPE (next_info)
5119 : == vect_double_reduction_def)))
5120 : {
5121 : /* Do not discover SLP reductions combining lane-reducing
5122 : ops, that will fail later. */
5123 107992 : if (!force_single_lane
5124 107992 : && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
5125 80150 : scalar_stmts.quick_push (next_info);
5126 : /* Do SLP discovery for single-lane reductions. */
5127 27842 : else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
5128 : max_tree_size, limit,
5129 : bst_map,
5130 : force_single_lane))
5131 : {
5132 0 : scalar_stmts.release ();
5133 0 : return false;
5134 : }
5135 : }
5136 : }
5137 :
5138 : /* Try to form reduction groups. */
5139 75687 : if (scalar_stmts.length () > 1)
5140 : {
5141 4670 : if (!vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
5142 : max_tree_size, limit))
5143 : {
5144 160 : scalar_stmts.release ();
5145 160 : return false;
5146 : }
5147 : return true;
5148 : }
5149 :
5150 : /* Do SLP discovery for a single reduction. */
5151 71017 : if (scalar_stmts.length () == 1
5152 124521 : && !vect_analyze_slp_reduction (loop_vinfo,
5153 53504 : vect_stmt_to_vectorize (scalar_stmts[0]),
5154 : max_tree_size, limit,
5155 : bst_map, force_single_lane))
5156 : {
5157 1761 : scalar_stmts.release ();
5158 1761 : return false;
5159 : }
5160 :
5161 69256 : scalar_stmts.release ();
5162 69256 : return true;
5163 : }
5164 :
5165 : /* Analyze an SLP instance starting from a group of grouped stores. Call
5166 : vect_build_slp_tree to build a tree of packed stmts if possible.
5167 : Return FALSE if it's impossible to SLP any stmt in the group. */
5168 :
5169 : static bool
5170 1106300 : vect_analyze_slp_instance (vec_info *vinfo,
5171 : scalar_stmts_to_slp_tree_map_t *bst_map,
5172 : stmt_vec_info stmt_info,
5173 : slp_instance_kind kind,
5174 : unsigned max_tree_size, unsigned *limit,
5175 : bool force_single_lane)
5176 : {
5177 1106300 : vec<stmt_vec_info> scalar_stmts;
5178 :
5179 1106300 : if (is_a <bb_vec_info> (vinfo))
5180 1072046 : vect_location = stmt_info->stmt;
5181 :
5182 1106300 : gcc_assert (kind == slp_inst_kind_store);
5183 :
5184 : /* Collect the stores and store them in scalar_stmts. */
5185 1106300 : scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
5186 1106300 : stmt_vec_info next_info = stmt_info;
5187 5491928 : while (next_info)
5188 : {
5189 3279328 : scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
5190 3279328 : next_info = DR_GROUP_NEXT_ELEMENT (next_info);
5191 : }
5192 :
5193 1106300 : vec<stmt_vec_info> root_stmt_infos = vNULL;
5194 1106300 : vec<tree> remain = vNULL;
5195 :
5196 : /* Build the tree for the SLP instance. */
5197 :
5198 : /* If there's no budget left bail out early. */
5199 1106300 : if (*limit == 0)
5200 : return false;
5201 :
5202 1106280 : if (dump_enabled_p ())
5203 : {
5204 4252 : dump_printf_loc (MSG_NOTE, vect_location,
5205 : "Starting SLP discovery for\n");
5206 24675 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
5207 40846 : dump_printf_loc (MSG_NOTE, vect_location,
5208 20423 : " %G", scalar_stmts[i]->stmt);
5209 : }
5210 :
5211 : /* Build the tree for the SLP instance. */
5212 1106280 : unsigned int group_size = scalar_stmts.length ();
5213 1106280 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
5214 1106280 : unsigned tree_size = 0;
5215 1106280 : unsigned i;
5216 :
5217 1106280 : slp_tree node = NULL;
5218 1106280 : if (group_size > 1 && force_single_lane)
5219 : {
5220 3575 : matches[0] = 0;
5221 3575 : matches[1] = -1;
5222 : }
5223 : else
5224 1102705 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
5225 : &tree_size, bst_map);
5226 1106280 : if (node != NULL)
5227 : {
5228 : /* Create a new SLP instance. */
5229 698753 : slp_instance new_instance = XNEW (class _slp_instance);
5230 698753 : SLP_INSTANCE_TREE (new_instance) = node;
5231 698753 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5232 698753 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5233 698753 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5234 698753 : SLP_INSTANCE_KIND (new_instance) = kind;
5235 698753 : new_instance->reduc_phis = NULL;
5236 698753 : new_instance->cost_vec = vNULL;
5237 698753 : new_instance->subgraph_entries = vNULL;
5238 :
5239 698753 : if (dump_enabled_p ())
5240 3244 : dump_printf_loc (MSG_NOTE, vect_location,
5241 : "SLP size %u vs. limit %u.\n",
5242 : tree_size, max_tree_size);
5243 :
5244 698753 : vinfo->slp_instances.safe_push (new_instance);
5245 :
5246 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5247 : the number of SLP lanes of the root in a few places.
5248 : Verify that assumption holds. */
5249 698753 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5250 : == group_size);
5251 :
5252 698753 : if (dump_enabled_p ())
5253 : {
5254 3244 : dump_printf_loc (MSG_NOTE, vect_location,
5255 : "Final SLP tree for instance %p:\n",
5256 : (void *) new_instance);
5257 3244 : vect_print_slp_graph (MSG_NOTE, vect_location,
5258 : SLP_INSTANCE_TREE (new_instance));
5259 : }
5260 698753 : return true;
5261 : }
5262 : /* Failed to SLP. */
5263 :
5264 : /* Try to break the group up into pieces. */
5265 : /* ??? Improve this with the matches[] improvements. */
5266 407527 : if (*limit > 0 && kind == slp_inst_kind_store)
5267 : {
5268 : /* ??? We could delay all the actual splitting of store-groups
5269 : until after SLP discovery of the original group completed.
5270 : Then we can recurse to vect_build_slp_instance directly. */
5271 1015204 : for (i = 0; i < group_size; i++)
5272 1015204 : if (matches[i] != 0)
5273 : break;
5274 :
5275 : /* For basic block SLP, try to break the group up into multiples of
5276 : a vector size. */
5277 407526 : if (is_a <bb_vec_info> (vinfo)
5278 407526 : && (i > 1 && i < group_size))
5279 : {
5280 : /* Free the allocated memory. */
5281 133883 : scalar_stmts.release ();
5282 :
5283 133883 : tree scalar_type
5284 133883 : = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
5285 267766 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
5286 133883 : 1 << floor_log2 (i));
5287 133883 : unsigned HOST_WIDE_INT const_nunits;
5288 133883 : if (vectype
5289 133883 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
5290 : {
5291 : /* Split into two groups at the first vector boundary. */
5292 133883 : gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
5293 133883 : unsigned group1_size = i & ~(const_nunits - 1);
5294 :
5295 133883 : if (dump_enabled_p ())
5296 83 : dump_printf_loc (MSG_NOTE, vect_location,
5297 : "Splitting SLP group at stmt %u\n", i);
5298 133883 : stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
5299 : group1_size);
5300 133883 : bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
5301 : kind, max_tree_size,
5302 : limit, false);
5303 : /* Split the rest at the failure point and possibly
5304 : re-analyze the remaining matching part if it has
5305 : at least two lanes. */
5306 133883 : if (group1_size < i
5307 5797 : && (i + 1 < group_size
5308 2579 : || i - group1_size > 1))
5309 : {
5310 3252 : stmt_vec_info rest2 = rest;
5311 3252 : rest = vect_split_slp_store_group (rest, i - group1_size);
5312 3252 : if (i - group1_size > 1)
5313 71 : res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
5314 : kind, max_tree_size,
5315 : limit, false);
5316 : }
5317 : /* Re-analyze the non-matching tail if it has at least
5318 : two lanes. */
5319 133883 : if (i + 1 < group_size)
5320 22440 : res |= vect_analyze_slp_instance (vinfo, bst_map,
5321 : rest, kind, max_tree_size,
5322 : limit, false);
5323 1106300 : return res;
5324 : }
5325 : }
5326 :
5327 : /* For loop vectorization split the RHS into arbitrary pieces of
5328 : size >= 1. */
5329 273643 : else if (is_a <loop_vec_info> (vinfo)
5330 273643 : && (group_size != 1 && i < group_size))
5331 : {
5332 12187 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
5333 28 : bool masked_p = call
5334 28 : && gimple_call_internal_p (call)
5335 28 : && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
5336 : /* There are targets that cannot do even/odd interleaving schemes
5337 : so they absolutely need to use load/store-lanes. For now
5338 : force single-lane SLP for them - they would be happy with
5339 : uniform power-of-two lanes (but depending on element size),
5340 : but even if we can use 'i' as indicator we would need to
5341 : backtrack when later lanes fail to discover with the same
5342 : granularity. We cannot turn any of strided or scatter store
5343 : into store-lanes. */
5344 : /* ??? If this is not in sync with what get_load_store_type
5345 : later decides the SLP representation is not good for other
5346 : store vectorization methods. */
5347 12187 : bool want_store_lanes
5348 12187 : = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5349 12187 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5350 10064 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5351 10060 : && compare_step_with_zero (vinfo, stmt_info) > 0
5352 18334 : && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
5353 24374 : masked_p, group_size, i));
5354 12187 : if (want_store_lanes || force_single_lane)
5355 : i = 1;
5356 :
5357 : /* A fatal discovery fail doesn't always mean single-lane SLP
5358 : isn't a possibility, so try. */
5359 8612 : if (i == 0)
5360 : i = 1;
5361 :
5362 12187 : if (dump_enabled_p ())
5363 875 : dump_printf_loc (MSG_NOTE, vect_location,
5364 : "Splitting SLP group at stmt %u\n", i);
5365 :
5366 : /* Analyze the stored values and pinch them together with
5367 : a permute node so we can preserve the whole store group. */
5368 12187 : auto_vec<slp_tree> rhs_nodes;
5369 :
5370 12187 : unsigned int rhs_common_nlanes = 0;
5371 12187 : unsigned int start = 0, end = i;
5372 49367 : while (start < group_size)
5373 : {
5374 37514 : gcc_assert (end - start >= 1);
5375 37514 : vec<stmt_vec_info> substmts;
5376 37514 : substmts.create (end - start);
5377 148731 : for (unsigned j = start; j < end; ++j)
5378 73703 : substmts.quick_push (scalar_stmts[j]);
5379 37514 : node = vect_build_slp_tree (vinfo, substmts,
5380 : matches, limit, &tree_size, bst_map);
5381 37514 : if (node)
5382 : {
5383 31235 : rhs_nodes.safe_push (node);
5384 31235 : if (start == 0)
5385 11853 : rhs_common_nlanes = SLP_TREE_LANES (node);
5386 19382 : else if (rhs_common_nlanes != SLP_TREE_LANES (node))
5387 1425 : rhs_common_nlanes = 0;
5388 31235 : start = end;
5389 31235 : if (want_store_lanes || force_single_lane)
5390 8937 : end = start + 1;
5391 : else
5392 : end = group_size;
5393 : }
5394 : else
5395 : {
5396 6279 : substmts.release ();
5397 6279 : if (end - start == 1)
5398 : {
5399 : /* Single-lane discovery failed. Free resources. */
5400 334 : for (auto node : rhs_nodes)
5401 0 : vect_free_slp_tree (node);
5402 334 : scalar_stmts.release ();
5403 334 : if (dump_enabled_p ())
5404 14 : dump_printf_loc (MSG_NOTE, vect_location,
5405 : "SLP discovery failed\n");
5406 334 : return false;
5407 : }
5408 :
5409 : /* ??? It really happens that we soft-fail SLP
5410 : build at a mismatch but the matching part hard-fails
5411 : later. As we know we arrived here with a group
5412 : larger than one try a group of size one! */
5413 5945 : if (matches[0] != 0)
5414 12 : end = start + 1;
5415 : else
5416 13037 : for (unsigned j = start; j < end; j++)
5417 13037 : if (matches[j - start] != 0)
5418 : {
5419 : end = j;
5420 : break;
5421 : }
5422 : }
5423 : }
5424 :
5425 : /* Now re-assess whether we want store lanes in case the
5426 : discovery ended up producing all single-lane RHSs. */
5427 11853 : if (! want_store_lanes
5428 11853 : && rhs_common_nlanes == 1
5429 10761 : && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5430 10761 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5431 9009 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5432 9006 : && compare_step_with_zero (vinfo, stmt_info) > 0
5433 17182 : && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
5434 : group_size, masked_p)
5435 : != IFN_LAST))
5436 : want_store_lanes = true;
5437 :
5438 : /* Now we assume we can build the root SLP node from all stores. */
5439 11853 : if (want_store_lanes)
5440 : {
5441 : /* For store-lanes feed the store node with all RHS nodes
5442 : in order. */
5443 0 : node = vect_create_new_slp_node (scalar_stmts,
5444 0 : SLP_TREE_CHILDREN
5445 : (rhs_nodes[0]).length ());
5446 0 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
5447 0 : node->ldst_lanes = true;
5448 0 : SLP_TREE_CHILDREN (node)
5449 0 : .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
5450 0 : + rhs_nodes.length () - 1);
5451 : /* First store value and possibly mask. */
5452 0 : SLP_TREE_CHILDREN (node)
5453 0 : .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
5454 : /* Rest of the store values. All mask nodes are the same,
5455 : this should be guaranteed by dataref group discovery. */
5456 0 : for (unsigned j = 1; j < rhs_nodes.length (); ++j)
5457 0 : SLP_TREE_CHILDREN (node)
5458 0 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
5459 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
5460 0 : child->refcnt++;
5461 : }
5462 : else
5463 11853 : node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts);
5464 :
5465 43088 : while (!rhs_nodes.is_empty ())
5466 31235 : vect_free_slp_tree (rhs_nodes.pop ());
5467 :
5468 : /* Create a new SLP instance. */
5469 11853 : slp_instance new_instance = XNEW (class _slp_instance);
5470 11853 : SLP_INSTANCE_TREE (new_instance) = node;
5471 11853 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5472 11853 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5473 11853 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5474 11853 : SLP_INSTANCE_KIND (new_instance) = kind;
5475 11853 : new_instance->reduc_phis = NULL;
5476 11853 : new_instance->cost_vec = vNULL;
5477 11853 : new_instance->subgraph_entries = vNULL;
5478 :
5479 11853 : if (dump_enabled_p ())
5480 861 : dump_printf_loc (MSG_NOTE, vect_location,
5481 : "SLP size %u vs. limit %u.\n",
5482 : tree_size, max_tree_size);
5483 :
5484 11853 : vinfo->slp_instances.safe_push (new_instance);
5485 :
5486 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5487 : the number of SLP lanes of the root in a few places.
5488 : Verify that assumption holds. */
5489 11853 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5490 : == group_size);
5491 :
5492 11853 : if (dump_enabled_p ())
5493 : {
5494 861 : dump_printf_loc (MSG_NOTE, vect_location,
5495 : "Final SLP tree for instance %p:\n",
5496 : (void *) new_instance);
5497 861 : vect_print_slp_graph (MSG_NOTE, vect_location,
5498 : SLP_INSTANCE_TREE (new_instance));
5499 : }
5500 : return true;
5501 12187 : }
5502 : else
5503 : /* Free the allocated memory. */
5504 261456 : scalar_stmts.release ();
5505 :
5506 : /* Even though the first vector did not all match, we might be able to SLP
5507 : (some) of the remainder. FORNOW ignore this possibility. */
5508 : }
5509 : else
5510 : /* Free the allocated memory. */
5511 1 : scalar_stmts.release ();
5512 :
5513 : /* Failed to SLP. */
5514 261457 : if (dump_enabled_p ())
5515 50 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
5516 : return false;
5517 : }
5518 :
5519 : /* qsort comparator ordering SLP load nodes. */
5520 :
5521 : static int
5522 2696294 : vllp_cmp (const void *a_, const void *b_)
5523 : {
5524 2696294 : const slp_tree a = *(const slp_tree *)a_;
5525 2696294 : const slp_tree b = *(const slp_tree *)b_;
5526 2696294 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
5527 2696294 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
5528 2696294 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5529 1518131 : && STMT_VINFO_GROUPED_ACCESS (b0)
5530 4151961 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5531 : {
5532 : /* Same group, order after lanes used. */
5533 343508 : if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
5534 : return 1;
5535 340519 : else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
5536 : return -1;
5537 : else
5538 : {
5539 : /* Try to order loads using the same lanes together, breaking
5540 : the tie with the lane number that first differs. */
5541 337038 : if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5542 337038 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5543 : return 0;
5544 337038 : else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
5545 337038 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5546 : return 1;
5547 331611 : else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5548 331611 : && SLP_TREE_LOAD_PERMUTATION (b).exists ())
5549 : return -1;
5550 : else
5551 : {
5552 323849 : for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
5553 323849 : if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5554 323849 : != SLP_TREE_LOAD_PERMUTATION (b)[i])
5555 : {
5556 : /* In-order lane first, that's what the above case for
5557 : no permutation does. */
5558 320361 : if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
5559 : return -1;
5560 197085 : else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
5561 : return 1;
5562 103456 : else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5563 103456 : < SLP_TREE_LOAD_PERMUTATION (b)[i])
5564 : return -1;
5565 : else
5566 38798 : return 1;
5567 : }
5568 : return 0;
5569 : }
5570 : }
5571 : }
5572 : else /* Different groups or non-groups. */
5573 : {
5574 : /* Order groups as their first element to keep them together. */
5575 2352786 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5576 2352786 : a0 = DR_GROUP_FIRST_ELEMENT (a0);
5577 2352786 : if (STMT_VINFO_GROUPED_ACCESS (b0))
5578 2352786 : b0 = DR_GROUP_FIRST_ELEMENT (b0);
5579 2352786 : if (a0 == b0)
5580 : return 0;
5581 : /* Tie using UID. */
5582 2352666 : else if (gimple_uid (STMT_VINFO_STMT (a0))
5583 2352666 : < gimple_uid (STMT_VINFO_STMT (b0)))
5584 : return -1;
5585 : else
5586 : {
5587 1042915 : gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
5588 : != gimple_uid (STMT_VINFO_STMT (b0)));
5589 : return 1;
5590 : }
5591 : }
5592 : }
5593 :
5594 : /* Return whether if the load permutation of NODE is consecutive starting
5595 : with value START_VAL in the first element. If START_VAL is not given
5596 : the first element's value is used. */
5597 :
5598 : bool
5599 664804 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
5600 : {
5601 664804 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
5602 :
5603 664804 : if (!perm.exists () || !perm.length ())
5604 : return false;
5605 :
5606 664804 : if (start_val == UINT_MAX)
5607 80019 : start_val = perm[0];
5608 :
5609 1313944 : for (unsigned int i = 0; i < perm.length (); i++)
5610 673131 : if (perm[i] != start_val + (unsigned int) i)
5611 : return false;
5612 :
5613 : return true;
5614 : }
5615 :
5616 : /* Process the set of LOADS that are all from the same dataref group. */
5617 :
5618 : static void
5619 163371 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5620 : scalar_stmts_to_slp_tree_map_t *bst_map,
5621 : const array_slice<slp_tree> &loads,
5622 : bool force_single_lane)
5623 : {
5624 : /* We at this point want to lower without a fixed VF or vector
5625 : size in mind which means we cannot actually compute whether we
5626 : need three or more vectors for a load permutation yet. So always
5627 : lower. */
5628 163371 : stmt_vec_info first
5629 163371 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
5630 163371 : unsigned group_lanes = DR_GROUP_SIZE (first);
5631 :
5632 : /* Verify if all load permutations can be implemented with a suitably
5633 : large element load-lanes operation. */
5634 163371 : unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
5635 163371 : if (STMT_VINFO_STRIDED_P (first)
5636 160893 : || compare_step_with_zero (loop_vinfo, first) <= 0
5637 158236 : || exact_log2 (ld_lanes_lanes) == -1
5638 : /* ??? For now only support the single-lane case as there is
5639 : missing support on the store-lane side and code generation
5640 : isn't up to the task yet. */
5641 155417 : || ld_lanes_lanes != 1
5642 307710 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
5643 : group_lanes / ld_lanes_lanes,
5644 : false) == IFN_LAST)
5645 : ld_lanes_lanes = 0;
5646 : else
5647 : /* Verify the loads access the same number of lanes aligned to
5648 : ld_lanes_lanes. */
5649 0 : for (slp_tree load : loads)
5650 : {
5651 0 : if (SLP_TREE_LANES (load) != ld_lanes_lanes)
5652 : {
5653 : ld_lanes_lanes = 0;
5654 : break;
5655 : }
5656 0 : unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
5657 0 : if (first % ld_lanes_lanes != 0)
5658 : {
5659 : ld_lanes_lanes = 0;
5660 : break;
5661 : }
5662 0 : if (!vect_load_perm_consecutive_p (load))
5663 : {
5664 : ld_lanes_lanes = 0;
5665 : break;
5666 : }
5667 : }
5668 :
5669 : /* Only a power-of-two number of lanes matches interleaving with N levels.
5670 : ??? An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
5671 : at each step. */
5672 264739 : if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
5673 : return;
5674 :
5675 267379 : for (slp_tree load : loads)
5676 : {
5677 : /* Leave masked or gather loads alone for now. */
5678 188420 : if (!SLP_TREE_CHILDREN (load).is_empty ())
5679 60667 : continue;
5680 :
5681 : /* For single-element interleaving spanning multiple vectors avoid
5682 : lowering, we want to use VMAT_ELEMENTWISE later. */
5683 188414 : if (ld_lanes_lanes == 0
5684 188414 : && SLP_TREE_LANES (load) == 1
5685 167526 : && !DR_GROUP_NEXT_ELEMENT (first)
5686 269099 : && maybe_gt (group_lanes,
5687 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
5688 51824 : return;
5689 :
5690 : /* We want to pattern-match special cases here and keep those
5691 : alone. Candidates are splats and load-lane. */
5692 :
5693 : /* We need to lower only loads of less than half of the groups
5694 : lanes, including duplicate lanes. Note this leaves nodes
5695 : with a non-1:1 load permutation around instead of canonicalizing
5696 : those into a load and a permute node. Removing this early
5697 : check would do such canonicalization. */
5698 136590 : if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
5699 57060 : && ld_lanes_lanes == 0)
5700 57060 : continue;
5701 :
5702 : /* Build the permute to get the original load permutation order. */
5703 79530 : bool contiguous = vect_load_perm_consecutive_p (load);
5704 79530 : lane_permutation_t final_perm;
5705 79530 : final_perm.create (SLP_TREE_LANES (load));
5706 239534 : for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
5707 160948 : final_perm.quick_push (
5708 80474 : std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
5709 :
5710 : /* When the load permutation accesses a contiguous unpermuted,
5711 : power-of-two aligned and sized chunk leave the load alone.
5712 : We can likely (re-)load it more efficiently rather than
5713 : extracting it from the larger load.
5714 : ??? Long-term some of the lowering should move to where
5715 : the vector types involved are fixed. */
5716 83131 : if (!force_single_lane
5717 79530 : && ld_lanes_lanes == 0
5718 53617 : && contiguous
5719 53357 : && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
5720 6653 : && pow2p_hwi (SLP_TREE_LANES (load))
5721 6617 : && pow2p_hwi (group_lanes)
5722 3601 : && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
5723 83131 : && group_lanes % SLP_TREE_LANES (load) == 0)
5724 : {
5725 3601 : final_perm.release ();
5726 3601 : continue;
5727 : }
5728 :
5729 : /* First build (and possibly re-use) a load node for the
5730 : unpermuted group. Gaps in the middle and on the end are
5731 : represented with NULL stmts. */
5732 75929 : vec<stmt_vec_info> stmts;
5733 75929 : stmts.create (group_lanes);
5734 271167 : for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
5735 : {
5736 195238 : if (s != first)
5737 124994 : for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
5738 5685 : stmts.quick_push (NULL);
5739 195238 : stmts.quick_push (s);
5740 : }
5741 142961 : for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
5742 67032 : stmts.quick_push (NULL);
5743 75929 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_lanes);
5744 75929 : unsigned limit = 1;
5745 75929 : unsigned tree_size = 0;
5746 75929 : slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts, matches, &limit,
5747 75929 : &tree_size, bst_map);
5748 75929 : gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
5749 :
5750 75929 : if (ld_lanes_lanes != 0)
5751 : {
5752 : /* ??? If this is not in sync with what get_load_store_type
5753 : later decides the SLP representation is not good for other
5754 : store vectorization methods. */
5755 0 : l0->ldst_lanes = true;
5756 0 : load->ldst_lanes = true;
5757 : }
5758 :
5759 236629 : while (1)
5760 : {
5761 156279 : unsigned group_lanes = SLP_TREE_LANES (l0);
5762 156279 : if (ld_lanes_lanes != 0
5763 156279 : || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
5764 : break;
5765 :
5766 : /* Try to lower by reducing the group to half its size using an
5767 : interleaving scheme. For this try to compute whether all
5768 : elements needed for this load are in even or odd elements of
5769 : an even/odd decomposition with N consecutive elements.
5770 : Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
5771 : with N == 2. */
5772 : /* ??? Only an even number of lanes can be handed this way, but the
5773 : fallback below could work for any number. We have to make sure
5774 : to round up in that case. */
5775 80350 : gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
5776 12289 : unsigned even = 0, odd = 0;
5777 12289 : if ((group_lanes & 1) == 0)
5778 : {
5779 12289 : even = (1 << ceil_log2 (group_lanes)) - 1;
5780 12289 : odd = even;
5781 49905 : for (auto l : final_perm)
5782 : {
5783 13038 : even &= ~l.second;
5784 13038 : odd &= l.second;
5785 : }
5786 : }
5787 :
5788 : /* Now build an even or odd extraction from the unpermuted load. */
5789 80350 : lane_permutation_t perm;
5790 80350 : perm.create ((group_lanes + 1) / 2);
5791 80350 : unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
5792 80350 : unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
5793 80350 : if (even_level
5794 11307 : && group_lanes % (2 * even_level) == 0
5795 : /* ??? When code generating permutes we do not try to pun
5796 : to larger component modes so level != 1 isn't a natural
5797 : even/odd extract. Prefer one if possible. */
5798 11307 : && (even_level == 1 || !odd_level || odd_level != 1))
5799 : {
5800 : /* { 0, 1, ... 4, 5 ..., } */
5801 43060 : for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
5802 69241 : for (unsigned j = 0; j < even_level; ++j)
5803 34816 : perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
5804 : }
5805 69043 : else if (odd_level)
5806 : {
5807 : /* { ..., 2, 3, ... 6, 7 } */
5808 3624 : gcc_assert (group_lanes % (2 * odd_level) == 0);
5809 15946 : for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
5810 24698 : for (unsigned j = 0; j < odd_level; ++j)
5811 12376 : perm.quick_push
5812 12376 : (std::make_pair (0, (2 * i + 1) * odd_level + j));
5813 : }
5814 : else
5815 : {
5816 : /* As fallback extract all used lanes and fill to half the
5817 : group size by repeating the last element.
5818 : ??? This is quite a bad strathegy for re-use - we could
5819 : brute force our way to find more optimal filling lanes to
5820 : maximize re-use when looking at all loads from the group. */
5821 68091 : auto_bitmap l;
5822 272420 : for (auto p : final_perm)
5823 68147 : bitmap_set_bit (l, p.second);
5824 68091 : unsigned i = 0;
5825 68091 : bitmap_iterator bi;
5826 136238 : EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
5827 68147 : perm.quick_push (std::make_pair (0, i));
5828 272516 : while (perm.length () < (group_lanes + 1) / 2)
5829 68167 : perm.quick_push (perm.last ());
5830 68091 : }
5831 :
5832 : /* Update final_perm with the intermediate permute. */
5833 161449 : for (unsigned i = 0; i < final_perm.length (); ++i)
5834 : {
5835 81099 : unsigned l = final_perm[i].second;
5836 81099 : unsigned j;
5837 91073 : for (j = 0; j < perm.length (); ++j)
5838 91073 : if (perm[j].second == l)
5839 : {
5840 81099 : final_perm[i].second = j;
5841 81099 : break;
5842 : }
5843 81099 : gcc_assert (j < perm.length ());
5844 : }
5845 :
5846 : /* And create scalar stmts. */
5847 80350 : vec<stmt_vec_info> perm_stmts;
5848 80350 : perm_stmts.create (perm.length ());
5849 344206 : for (unsigned i = 0; i < perm.length (); ++i)
5850 183506 : perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
5851 :
5852 80350 : slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
5853 80350 : SLP_TREE_CHILDREN (p).quick_push (l0);
5854 80350 : SLP_TREE_LANE_PERMUTATION (p) = perm;
5855 80350 : SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
5856 80350 : SLP_TREE_LANES (p) = perm.length ();
5857 : /* ??? As we have scalar stmts for this intermediate permute we
5858 : could CSE it via bst_map but we do not want to pick up
5859 : another SLP node with a load permutation. We instead should
5860 : have a "local" CSE map here. */
5861 80350 : SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
5862 :
5863 : /* We now have a node for (group_lanes + 1) / 2 lanes. */
5864 80350 : l0 = p;
5865 80350 : }
5866 :
5867 : /* And finally from the ordered reduction node create the
5868 : permute to shuffle the lanes into the original load-permutation
5869 : order. We replace the original load node with this. */
5870 75929 : SLP_TREE_CODE (load) = VEC_PERM_EXPR;
5871 75929 : SLP_TREE_LOAD_PERMUTATION (load).release ();
5872 75929 : SLP_TREE_LANE_PERMUTATION (load) = final_perm;
5873 75929 : SLP_TREE_CHILDREN (load).create (1);
5874 75929 : SLP_TREE_CHILDREN (load).quick_push (l0);
5875 75929 : SLP_TREE_REPRESENTATIVE (load) = NULL;
5876 : }
5877 : }
5878 :
5879 : /* Transform SLP loads in the SLP graph created by SLP discovery to
5880 : group loads from the same group and lower load permutations that
5881 : are unlikely to be supported into a series of permutes.
5882 : In the degenerate case of having only single-lane SLP instances
5883 : this should result in a series of permute nodes emulating an
5884 : interleaving scheme. */
5885 :
5886 : static void
5887 517993 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5888 : scalar_stmts_to_slp_tree_map_t *bst_map,
5889 : bool force_single_lane)
5890 : {
5891 : /* Gather and sort loads across all instances. */
5892 517993 : hash_set<slp_tree> visited;
5893 517993 : auto_vec<slp_tree> loads;
5894 2377811 : for (auto inst : loop_vinfo->slp_instances)
5895 825900 : vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
5896 517993 : if (loads.is_empty ())
5897 97472 : return;
5898 420521 : loads.qsort (vllp_cmp);
5899 :
5900 : /* Now process each dataref group separately. */
5901 420521 : unsigned firsti = 0;
5902 1193552 : for (unsigned i = 1; i < loads.length (); ++i)
5903 : {
5904 352510 : slp_tree first = loads[firsti];
5905 352510 : slp_tree next = loads[i];
5906 352510 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
5907 352510 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
5908 352510 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5909 158558 : && STMT_VINFO_GROUPED_ACCESS (b0)
5910 497763 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5911 62828 : continue;
5912 : /* Now we have one or multiple SLP loads of the same group from
5913 : firsti to i - 1. */
5914 289682 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5915 95730 : vect_lower_load_permutations (loop_vinfo, bst_map,
5916 95730 : make_array_slice (&loads[firsti],
5917 : i - firsti),
5918 : force_single_lane);
5919 : firsti = i;
5920 : }
5921 841042 : if (firsti < loads.length ()
5922 841042 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
5923 67641 : vect_lower_load_permutations (loop_vinfo, bst_map,
5924 67641 : make_array_slice (&loads[firsti],
5925 67641 : loads.length () - firsti),
5926 : force_single_lane);
5927 517993 : }
5928 :
5929 : /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
5930 : trees of packed scalar stmts if SLP is possible. */
5931 :
5932 : opt_result
5933 1215145 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
5934 : bool force_single_lane)
5935 : {
5936 1215145 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5937 1215145 : unsigned int i;
5938 1215145 : stmt_vec_info first_element;
5939 1215145 : slp_instance instance;
5940 :
5941 1215145 : DUMP_VECT_SCOPE ("vect_analyze_slp");
5942 :
5943 1215145 : unsigned limit = max_tree_size;
5944 :
5945 1215145 : scalar_stmts_to_slp_tree_map_t *bst_map
5946 1215145 : = new scalar_stmts_to_slp_tree_map_t ();
5947 :
5948 : /* Find SLP sequences starting from groups of grouped stores. */
5949 3379854 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
5950 949906 : if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
5951 : slp_inst_kind_store, max_tree_size, &limit,
5952 : force_single_lane)
5953 949906 : && loop_vinfo)
5954 : {
5955 342 : release_scalar_stmts_to_slp_tree_map (bst_map);
5956 342 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5957 : }
5958 :
5959 : /* For loops also start SLP discovery from non-grouped stores. */
5960 1214803 : if (loop_vinfo)
5961 : {
5962 : data_reference_p dr;
5963 1745448 : FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
5964 1212123 : if (DR_IS_WRITE (dr))
5965 : {
5966 394362 : stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
5967 : /* Grouped stores are already handled above. */
5968 394362 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5969 111069 : continue;
5970 283293 : vec<stmt_vec_info> stmts;
5971 283293 : vec<stmt_vec_info> roots = vNULL;
5972 283293 : vec<tree> remain = vNULL;
5973 283293 : stmts.create (1);
5974 283293 : stmts.quick_push (stmt_info);
5975 283293 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5976 : stmts, roots, remain, max_tree_size,
5977 : &limit, bst_map, force_single_lane))
5978 : {
5979 4111 : release_scalar_stmts_to_slp_tree_map (bst_map);
5980 4111 : return opt_result::failure_at (vect_location,
5981 : "SLP build failed.\n");
5982 : }
5983 : }
5984 :
5985 : stmt_vec_info stmt_info;
5986 533385 : FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
5987 : {
5988 30 : vec<stmt_vec_info> stmts;
5989 30 : vec<stmt_vec_info> roots = vNULL;
5990 30 : vec<tree> remain = vNULL;
5991 30 : stmts.create (1);
5992 30 : stmts.quick_push (stmt_info);
5993 30 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5994 : stmts, roots, remain, max_tree_size,
5995 : &limit, bst_map, force_single_lane))
5996 : {
5997 0 : release_scalar_stmts_to_slp_tree_map (bst_map);
5998 0 : return opt_result::failure_at (vect_location,
5999 : "SLP build failed.\n");
6000 : }
6001 : }
6002 : }
6003 :
6004 1210692 : if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
6005 : {
6006 2058950 : for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
6007 : {
6008 1381583 : vect_location = bb_vinfo->roots[i].roots[0]->stmt;
6009 : /* Apply patterns. */
6010 4517605 : for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
6011 6272044 : bb_vinfo->roots[i].stmts[j]
6012 3244637 : = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
6013 1381583 : if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
6014 1381583 : bb_vinfo->roots[i].stmts,
6015 1381583 : bb_vinfo->roots[i].roots,
6016 1381583 : bb_vinfo->roots[i].remain,
6017 : max_tree_size, &limit, bst_map, false))
6018 : {
6019 157787 : bb_vinfo->roots[i].roots = vNULL;
6020 157787 : bb_vinfo->roots[i].remain = vNULL;
6021 : }
6022 1381583 : bb_vinfo->roots[i].stmts = vNULL;
6023 : }
6024 : }
6025 :
6026 1210692 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6027 : {
6028 : /* Find SLP sequences starting from groups of reductions. */
6029 533325 : if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
6030 : bst_map, force_single_lane))
6031 : {
6032 1921 : release_scalar_stmts_to_slp_tree_map (bst_map);
6033 1921 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
6034 : }
6035 :
6036 : /* Make sure to vectorize only-live stmts, usually inductions. */
6037 2394018 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
6038 1556592 : for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
6039 746064 : gsi_next (&gsi))
6040 : {
6041 756786 : gphi *lc_phi = *gsi;
6042 756786 : tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
6043 756786 : stmt_vec_info stmt_info;
6044 756786 : if (TREE_CODE (def) == SSA_NAME
6045 639361 : && !virtual_operand_p (def)
6046 331262 : && (stmt_info = loop_vinfo->lookup_def (def))
6047 294180 : && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
6048 294180 : && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
6049 226323 : && STMT_VINFO_LIVE_P (stmt_info)
6050 226323 : && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
6051 877700 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
6052 : {
6053 120826 : vec<stmt_vec_info> stmts;
6054 120826 : vec<stmt_vec_info> roots = vNULL;
6055 120826 : vec<tree> remain = vNULL;
6056 120826 : stmts.create (1);
6057 120826 : stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
6058 120826 : if (! vect_build_slp_instance (vinfo,
6059 : slp_inst_kind_reduc_group,
6060 : stmts, roots, remain,
6061 : max_tree_size, &limit,
6062 : bst_map, force_single_lane))
6063 : {
6064 10722 : release_scalar_stmts_to_slp_tree_map (bst_map);
6065 10722 : return opt_result::failure_at (vect_location,
6066 : "SLP build failed.\n");
6067 : }
6068 : }
6069 10722 : }
6070 :
6071 : /* Find SLP sequences starting from gconds. */
6072 1313719 : for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
6073 : {
6074 306884 : auto cond_info = loop_vinfo->lookup_stmt (cond);
6075 :
6076 306884 : cond_info = vect_stmt_to_vectorize (cond_info);
6077 306884 : vec<stmt_vec_info> roots = vNULL;
6078 306884 : roots.safe_push (cond_info);
6079 306884 : gimple *stmt = STMT_VINFO_STMT (cond_info);
6080 306884 : tree args0 = gimple_cond_lhs (stmt);
6081 306884 : tree args1 = gimple_cond_rhs (stmt);
6082 :
6083 : /* These should be enforced by cond lowering, but if it failed
6084 : bail. */
6085 306884 : if (gimple_cond_code (stmt) != NE_EXPR
6086 305725 : || TREE_TYPE (args0) != boolean_type_node
6087 612004 : || !integer_zerop (args1))
6088 : {
6089 1764 : roots.release ();
6090 1764 : release_scalar_stmts_to_slp_tree_map (bst_map);
6091 1764 : return opt_result::failure_at (vect_location,
6092 : "SLP build failed.\n");
6093 : }
6094 :
6095 : /* An argument without a loop def will be codegened from vectorizing the
6096 : root gcond itself. As such we don't need to try to build an SLP tree
6097 : from them. It's highly likely that the resulting SLP tree here if both
6098 : arguments have a def will be incompatible, but we rely on it being split
6099 : later on. */
6100 305120 : auto varg = loop_vinfo->lookup_def (args0);
6101 305120 : vec<stmt_vec_info> stmts;
6102 305120 : vec<tree> remain = vNULL;
6103 305120 : stmts.create (1);
6104 305120 : stmts.quick_push (vect_stmt_to_vectorize (varg));
6105 :
6106 305120 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
6107 : stmts, roots, remain,
6108 : max_tree_size, &limit,
6109 : bst_map, force_single_lane))
6110 : {
6111 925 : roots.release ();
6112 925 : release_scalar_stmts_to_slp_tree_map (bst_map);
6113 925 : return opt_result::failure_at (vect_location,
6114 : "SLP build failed.\n");
6115 : }
6116 : }
6117 : }
6118 :
6119 1195360 : hash_set<slp_tree> visited_patterns;
6120 1195360 : slp_tree_to_load_perm_map_t perm_cache;
6121 1195360 : slp_compat_nodes_map_t compat_cache;
6122 :
6123 : /* See if any patterns can be found in the SLP tree. */
6124 1195360 : bool pattern_found = false;
6125 4051101 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6126 1660381 : pattern_found |= vect_match_slp_patterns (instance, vinfo,
6127 : &visited_patterns, &perm_cache,
6128 : &compat_cache);
6129 :
6130 : /* If any were found optimize permutations of loads. */
6131 1195360 : if (pattern_found)
6132 : {
6133 267 : hash_map<slp_tree, slp_tree> load_map;
6134 4981 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6135 : {
6136 4180 : slp_tree root = SLP_INSTANCE_TREE (instance);
6137 4180 : optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
6138 : &load_map, root);
6139 : }
6140 267 : }
6141 :
6142 : /* Check whether we should force some SLP instances to use load/store-lanes
6143 : and do so by forcing SLP re-discovery with single lanes. We used
6144 : to cancel SLP when this applied to all instances in a loop but now
6145 : we decide this per SLP instance. It's important to do this only
6146 : after SLP pattern recognition. */
6147 1195360 : if (is_a <loop_vec_info> (vinfo))
6148 1343893 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6149 825900 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
6150 311628 : && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
6151 : {
6152 311628 : slp_tree slp_root = SLP_INSTANCE_TREE (instance);
6153 311628 : unsigned int group_size = SLP_TREE_LANES (slp_root);
6154 311628 : tree vectype = SLP_TREE_VECTYPE (slp_root);
6155 :
6156 311628 : stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
6157 311628 : gimple *rep = STMT_VINFO_STMT (rep_info);
6158 311628 : bool masked = (is_gimple_call (rep)
6159 2493 : && gimple_call_internal_p (rep)
6160 314091 : && internal_fn_mask_index
6161 2463 : (gimple_call_internal_fn (rep)) != -1);
6162 311598 : if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
6163 33877 : || slp_root->ldst_lanes
6164 345505 : || (vect_store_lanes_supported (vectype, group_size, masked)
6165 : == IFN_LAST))
6166 311628 : continue;
6167 :
6168 0 : auto_vec<slp_tree> loads;
6169 0 : hash_set<slp_tree> visited;
6170 0 : vect_gather_slp_loads (loads, slp_root, visited);
6171 :
6172 : /* Check whether any load in the SLP instance is possibly
6173 : permuted. */
6174 0 : bool loads_permuted = false;
6175 0 : slp_tree load_node;
6176 0 : unsigned j;
6177 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6178 : {
6179 0 : if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
6180 0 : continue;
6181 0 : for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
6182 0 : if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
6183 : {
6184 : loads_permuted = true;
6185 : break;
6186 : }
6187 : }
6188 :
6189 : /* If the loads and stores can use load/store-lanes force re-discovery
6190 : with single lanes. */
6191 0 : if (loads_permuted)
6192 : {
6193 0 : bool can_use_lanes = true;
6194 : bool prefer_load_lanes = false;
6195 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6196 0 : if (STMT_VINFO_GROUPED_ACCESS
6197 : (SLP_TREE_REPRESENTATIVE (load_node)))
6198 : {
6199 0 : stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
6200 : (SLP_TREE_REPRESENTATIVE (load_node));
6201 0 : rep = STMT_VINFO_STMT (stmt_vinfo);
6202 0 : masked = (is_gimple_call (rep)
6203 0 : && gimple_call_internal_p (rep)
6204 0 : && internal_fn_mask_index
6205 0 : (gimple_call_internal_fn (rep)));
6206 : /* Use SLP for strided accesses (or if we can't
6207 : load-lanes). */
6208 0 : if (STMT_VINFO_STRIDED_P (stmt_vinfo)
6209 0 : || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
6210 0 : || vect_load_lanes_supported
6211 0 : (SLP_TREE_VECTYPE (load_node),
6212 0 : DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
6213 : /* ??? During SLP re-discovery with a single lane
6214 : a masked grouped load will appear permuted and
6215 : discovery will fail. We have to rework this
6216 : on the discovery side - for now avoid ICEing. */
6217 0 : || masked)
6218 : {
6219 : can_use_lanes = false;
6220 : break;
6221 : }
6222 : /* Make sure that the target would prefer store-lanes
6223 : for at least one of the loads.
6224 :
6225 : ??? Perhaps we should instead require this for
6226 : all loads? */
6227 0 : prefer_load_lanes
6228 : = (prefer_load_lanes
6229 0 : || SLP_TREE_LANES (load_node) == group_size
6230 0 : || (vect_slp_prefer_store_lanes_p
6231 0 : (vinfo, stmt_vinfo,
6232 : SLP_TREE_VECTYPE (load_node), masked,
6233 : group_size, SLP_TREE_LANES (load_node))));
6234 : }
6235 :
6236 0 : if (can_use_lanes && prefer_load_lanes)
6237 : {
6238 0 : if (dump_enabled_p ())
6239 0 : dump_printf_loc (MSG_NOTE, vect_location,
6240 : "SLP instance %p can use load/store-lanes,"
6241 : " re-discovering with single-lanes\n",
6242 : (void *) instance);
6243 :
6244 0 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
6245 :
6246 0 : vect_free_slp_instance (instance);
6247 0 : limit = max_tree_size;
6248 0 : bool res = vect_analyze_slp_instance (vinfo, bst_map,
6249 : stmt_info,
6250 : slp_inst_kind_store,
6251 : max_tree_size, &limit,
6252 : true);
6253 0 : gcc_assert (res);
6254 0 : auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
6255 0 : LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
6256 : }
6257 : }
6258 0 : }
6259 :
6260 : /* When we end up with load permutations that we cannot possibly handle,
6261 : like those requiring three vector inputs, lower them using interleaving
6262 : like schemes. */
6263 1195360 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6264 : {
6265 517993 : vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
6266 517993 : if (dump_enabled_p ())
6267 : {
6268 22128 : dump_printf_loc (MSG_NOTE, vect_location,
6269 : "SLP graph after lowering permutations:\n");
6270 22128 : hash_set<slp_tree> visited;
6271 98387 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6272 32030 : vect_print_slp_graph (MSG_NOTE, vect_location,
6273 : SLP_INSTANCE_TREE (instance), visited);
6274 22128 : }
6275 : }
6276 :
6277 1195360 : release_scalar_stmts_to_slp_tree_map (bst_map);
6278 :
6279 1195360 : if (pattern_found && dump_enabled_p ())
6280 : {
6281 20 : dump_printf_loc (MSG_NOTE, vect_location,
6282 : "Pattern matched SLP tree\n");
6283 20 : hash_set<slp_tree> visited;
6284 101 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6285 41 : vect_print_slp_graph (MSG_NOTE, vect_location,
6286 : SLP_INSTANCE_TREE (instance), visited);
6287 20 : }
6288 :
6289 1195360 : return opt_result::success ();
6290 1195360 : }
6291 :
6292 : /* Estimates the cost of inserting layout changes into the SLP graph.
6293 : It can also say that the insertion is impossible. */
6294 :
6295 : struct slpg_layout_cost
6296 : {
6297 11501233 : slpg_layout_cost () = default;
6298 : slpg_layout_cost (sreal, bool);
6299 :
6300 521518 : static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
6301 5589299 : bool is_possible () const { return depth != sreal::max (); }
6302 :
6303 : bool operator== (const slpg_layout_cost &) const;
6304 : bool operator!= (const slpg_layout_cost &) const;
6305 :
6306 : bool is_better_than (const slpg_layout_cost &, bool) const;
6307 :
6308 : void add_parallel_cost (const slpg_layout_cost &);
6309 : void add_serial_cost (const slpg_layout_cost &);
6310 : void split (unsigned int);
6311 :
6312 : /* The longest sequence of layout changes needed during any traversal
6313 : of the partition dag, weighted by execution frequency.
6314 :
6315 : This is the most important metric when optimizing for speed, since
6316 : it helps to ensure that we keep the number of operations on
6317 : critical paths to a minimum. */
6318 : sreal depth = 0;
6319 :
6320 : /* An estimate of the total number of operations needed. It is weighted by
6321 : execution frequency when optimizing for speed but not when optimizing for
6322 : size. In order to avoid double-counting, a node with a fanout of N will
6323 : distribute 1/N of its total cost to each successor.
6324 :
6325 : This is the most important metric when optimizing for size, since
6326 : it helps to keep the total number of operations to a minimum, */
6327 : sreal total = 0;
6328 : };
6329 :
6330 : /* Construct costs for a node with weight WEIGHT. A higher weight
6331 : indicates more frequent execution. IS_FOR_SIZE is true if we are
6332 : optimizing for size rather than speed. */
6333 :
6334 1324397 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
6335 1332428 : : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
6336 : {
6337 1324397 : }
6338 :
6339 : bool
6340 0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
6341 : {
6342 0 : return depth == other.depth && total == other.total;
6343 : }
6344 :
6345 : bool
6346 0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
6347 : {
6348 0 : return !operator== (other);
6349 : }
6350 :
6351 : /* Return true if these costs are better than OTHER. IS_FOR_SIZE is
6352 : true if we are optimizing for size rather than speed. */
6353 :
6354 : bool
6355 321646 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
6356 : bool is_for_size) const
6357 : {
6358 321646 : if (is_for_size)
6359 : {
6360 3477 : if (total != other.total)
6361 1720 : return total < other.total;
6362 1757 : return depth < other.depth;
6363 : }
6364 : else
6365 : {
6366 318169 : if (depth != other.depth)
6367 127265 : return depth < other.depth;
6368 190904 : return total < other.total;
6369 : }
6370 : }
6371 :
6372 : /* Increase the costs to account for something with cost INPUT_COST
6373 : happening in parallel with the current costs. */
6374 :
6375 : void
6376 385083 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
6377 : {
6378 385083 : depth = std::max (depth, input_cost.depth);
6379 385083 : total += input_cost.total;
6380 385083 : }
6381 :
6382 : /* Increase the costs to account for something with cost INPUT_COST
6383 : happening in series with the current costs. */
6384 :
6385 : void
6386 1576581 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
6387 : {
6388 1576581 : depth += other.depth;
6389 1576581 : total += other.total;
6390 1576581 : }
6391 :
6392 : /* Split the total cost among TIMES successors or predecessors. */
6393 :
6394 : void
6395 1261052 : slpg_layout_cost::split (unsigned int times)
6396 : {
6397 1261052 : if (times > 1)
6398 638429 : total /= times;
6399 1261052 : }
6400 :
6401 : /* Information about one node in the SLP graph, for use during
6402 : vect_optimize_slp_pass. */
6403 :
6404 : struct slpg_vertex
6405 : {
6406 10738300 : slpg_vertex (slp_tree node_) : node (node_) {}
6407 :
6408 : /* The node itself. */
6409 : slp_tree node;
6410 :
6411 : /* Which partition the node belongs to, or -1 if none. Nodes outside of
6412 : partitions are flexible; they can have whichever layout consumers
6413 : want them to have. */
6414 : int partition = -1;
6415 :
6416 : /* The number of nodes that directly use the result of this one
6417 : (i.e. the number of nodes that count this one as a child). */
6418 : unsigned int out_degree = 0;
6419 :
6420 : /* The execution frequency of the node. */
6421 : sreal weight = 0;
6422 :
6423 : /* The total execution frequency of all nodes that directly use the
6424 : result of this one. */
6425 : sreal out_weight = 0;
6426 : };
6427 :
6428 : /* Information about one partition of the SLP graph, for use during
6429 : vect_optimize_slp_pass. */
6430 :
6431 : struct slpg_partition_info
6432 : {
6433 : /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
6434 : of m_partitioned_nodes. */
6435 : unsigned int node_begin = 0;
6436 : unsigned int node_end = 0;
6437 :
6438 : /* Which layout we've chosen to use for this partition, or -1 if
6439 : we haven't picked one yet. */
6440 : int layout = -1;
6441 :
6442 : /* The number of predecessors and successors in the partition dag.
6443 : The predecessors always have lower partition numbers and the
6444 : successors always have higher partition numbers.
6445 :
6446 : Note that the directions of these edges are not necessarily the
6447 : same as in the data flow graph. For example, if an SCC has separate
6448 : partitions for an inner loop and an outer loop, the inner loop's
6449 : partition will have at least two incoming edges from the outer loop's
6450 : partition: one for a live-in value and one for a live-out value.
6451 : In data flow terms, one of these edges would also be from the outer loop
6452 : to the inner loop, but the other would be in the opposite direction. */
6453 : unsigned int in_degree = 0;
6454 : unsigned int out_degree = 0;
6455 : };
6456 :
6457 : /* Information about the costs of using a particular layout for a
6458 : particular partition. It can also say that the combination is
6459 : impossible. */
6460 :
6461 : struct slpg_partition_layout_costs
6462 : {
6463 1631269 : bool is_possible () const { return internal_cost.is_possible (); }
6464 63679 : void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
6465 :
6466 : /* The costs inherited from predecessor partitions. */
6467 : slpg_layout_cost in_cost;
6468 :
6469 : /* The inherent cost of the layout within the node itself. For example,
6470 : this is nonzero for a load if choosing a particular layout would require
6471 : the load to permute the loaded elements. It is nonzero for a
6472 : VEC_PERM_EXPR if the permutation cannot be eliminated or converted
6473 : to full-vector moves. */
6474 : slpg_layout_cost internal_cost;
6475 :
6476 : /* The costs inherited from successor partitions. */
6477 : slpg_layout_cost out_cost;
6478 : };
6479 :
6480 : /* This class tries to optimize the layout of vectors in order to avoid
6481 : unnecessary shuffling. At the moment, the set of possible layouts are
6482 : restricted to bijective permutations.
6483 :
6484 : The goal of the pass depends on whether we're optimizing for size or
6485 : for speed. When optimizing for size, the goal is to reduce the overall
6486 : number of layout changes (including layout changes implied by things
6487 : like load permutations). When optimizing for speed, the goal is to
6488 : reduce the maximum latency attributable to layout changes on any
6489 : non-cyclical path through the data flow graph.
6490 :
6491 : For example, when optimizing a loop nest for speed, we will prefer
6492 : to make layout changes outside of a loop rather than inside of a loop,
6493 : and will prefer to make layout changes in parallel rather than serially,
6494 : even if that increases the overall number of layout changes.
6495 :
6496 : The high-level procedure is:
6497 :
6498 : (1) Build a graph in which edges go from uses (parents) to definitions
6499 : (children).
6500 :
6501 : (2) Divide the graph into a dag of strongly-connected components (SCCs).
6502 :
6503 : (3) When optimizing for speed, partition the nodes in each SCC based
6504 : on their containing cfg loop. When optimizing for size, treat
6505 : each SCC as a single partition.
6506 :
6507 : This gives us a dag of partitions. The goal is now to assign a
6508 : layout to each partition.
6509 :
6510 : (4) Construct a set of vector layouts that are worth considering.
6511 : Record which nodes must keep their current layout.
6512 :
6513 : (5) Perform a forward walk over the partition dag (from loads to stores)
6514 : accumulating the "forward" cost of using each layout. When visiting
6515 : each partition, assign a tentative choice of layout to the partition
6516 : and use that choice when calculating the cost of using a different
6517 : layout in successor partitions.
6518 :
6519 : (6) Perform a backward walk over the partition dag (from stores to loads),
6520 : accumulating the "backward" cost of using each layout. When visiting
6521 : each partition, make a final choice of layout for that partition based
6522 : on the accumulated forward costs (from (5)) and backward costs
6523 : (from (6)).
6524 :
6525 : (7) Apply the chosen layouts to the SLP graph.
6526 :
6527 : For example, consider the SLP statements:
6528 :
6529 : S1: a_1 = load
6530 : loop:
6531 : S2: a_2 = PHI<a_1, a_3>
6532 : S3: b_1 = load
6533 : S4: a_3 = a_2 + b_1
6534 : exit:
6535 : S5: a_4 = PHI<a_3>
6536 : S6: store a_4
6537 :
6538 : S2 and S4 form an SCC and are part of the same loop. Every other
6539 : statement is in a singleton SCC. In this example there is a one-to-one
6540 : mapping between SCCs and partitions and the partition dag looks like this;
6541 :
6542 : S1 S3
6543 : \ /
6544 : S2+S4
6545 : |
6546 : S5
6547 : |
6548 : S6
6549 :
6550 : S2, S3 and S4 will have a higher execution frequency than the other
6551 : statements, so when optimizing for speed, the goal is to avoid any
6552 : layout changes:
6553 :
6554 : - within S3
6555 : - within S2+S4
6556 : - on the S3->S2+S4 edge
6557 :
6558 : For example, if S3 was originally a reversing load, the goal of the
6559 : pass is to make it an unreversed load and change the layout on the
6560 : S1->S2+S4 and S2+S4->S5 edges to compensate. (Changing the layout
6561 : on S1->S2+S4 and S5->S6 would also be acceptable.)
6562 :
6563 : The difference between SCCs and partitions becomes important if we
6564 : add an outer loop:
6565 :
6566 : S1: a_1 = ...
6567 : loop1:
6568 : S2: a_2 = PHI<a_1, a_6>
6569 : S3: b_1 = load
6570 : S4: a_3 = a_2 + b_1
6571 : loop2:
6572 : S5: a_4 = PHI<a_3, a_5>
6573 : S6: c_1 = load
6574 : S7: a_5 = a_4 + c_1
6575 : exit2:
6576 : S8: a_6 = PHI<a_5>
6577 : S9: store a_6
6578 : exit1:
6579 :
6580 : Here, S2, S4, S5, S7 and S8 form a single SCC. However, when optimizing
6581 : for speed, we usually do not want restrictions in the outer loop to "infect"
6582 : the decision for the inner loop. For example, if an outer-loop node
6583 : in the SCC contains a statement with a fixed layout, that should not
6584 : prevent the inner loop from using a different layout. Conversely,
6585 : the inner loop should not dictate a layout to the outer loop: if the
6586 : outer loop does a lot of computation, then it may not be efficient to
6587 : do all of that computation in the inner loop's preferred layout.
6588 :
6589 : So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
6590 : and S5+S7 (inner). We also try to arrange partitions so that:
6591 :
6592 : - the partition for an outer loop comes before the partition for
6593 : an inner loop
6594 :
6595 : - if a sibling loop A dominates a sibling loop B, A's partition
6596 : comes before B's
6597 :
6598 : This gives the following partition dag for the example above:
6599 :
6600 : S1 S3
6601 : \ /
6602 : S2+S4+S8 S6
6603 : | \\ /
6604 : | S5+S7
6605 : |
6606 : S9
6607 :
6608 : There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
6609 : one for a reversal of the edge S7->S8.
6610 :
6611 : The backward walk picks a layout for S5+S7 before S2+S4+S8. The choice
6612 : for S2+S4+S8 therefore has to balance the cost of using the outer loop's
6613 : preferred layout against the cost of changing the layout on entry to the
6614 : inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
6615 :
6616 : Although this works well when optimizing for speed, it has the downside
6617 : when optimizing for size that the choice of layout for S5+S7 is completely
6618 : independent of S9, which lessens the chance of reducing the overall number
6619 : of permutations. We therefore do not partition SCCs when optimizing
6620 : for size.
6621 :
6622 : To give a concrete example of the difference between optimizing
6623 : for size and speed, consider:
6624 :
6625 : a[0] = (b[1] << c[3]) - d[1];
6626 : a[1] = (b[0] << c[2]) - d[0];
6627 : a[2] = (b[3] << c[1]) - d[3];
6628 : a[3] = (b[2] << c[0]) - d[2];
6629 :
6630 : There are three different layouts here: one for a, one for b and d,
6631 : and one for c. When optimizing for speed it is better to permute each
6632 : of b, c and d into the order required by a, since those permutations
6633 : happen in parallel. But when optimizing for size, it is better to:
6634 :
6635 : - permute c into the same order as b
6636 : - do the arithmetic
6637 : - permute the result into the order required by a
6638 :
6639 : This gives 2 permutations rather than 3. */
6640 :
6641 : class vect_optimize_slp_pass
6642 : {
6643 : public:
6644 732178 : vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
6645 : void run ();
6646 :
6647 : private:
6648 : /* Graph building. */
6649 : struct loop *containing_loop (slp_tree);
6650 : bool is_cfg_latch_edge (graph_edge *);
6651 : void build_vertices (hash_set<slp_tree> &, slp_tree);
6652 : void build_vertices ();
6653 : void build_graph ();
6654 :
6655 : /* Partitioning. */
6656 : void create_partitions ();
6657 : template<typename T> void for_each_partition_edge (unsigned int, T);
6658 :
6659 : /* Layout selection. */
6660 : bool is_compatible_layout (slp_tree, unsigned int);
6661 : bool is_compatible_layout (const slpg_partition_info &, unsigned int);
6662 : int change_layout_cost (slp_tree, unsigned int, unsigned int);
6663 : slpg_partition_layout_costs &partition_layout_costs (unsigned int,
6664 : unsigned int);
6665 : void change_vec_perm_layout (slp_tree, lane_permutation_t &,
6666 : int, unsigned int);
6667 : int internal_node_cost (slp_tree, int, unsigned int);
6668 : void start_choosing_layouts ();
6669 : bool legitimize ();
6670 :
6671 : /* Cost propagation. */
6672 : slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
6673 : unsigned int, unsigned int);
6674 : slpg_layout_cost total_in_cost (unsigned int);
6675 : slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
6676 : slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
6677 : void forward_pass ();
6678 : void backward_pass ();
6679 :
6680 : /* Rematerialization. */
6681 : slp_tree get_result_with_layout (slp_tree, unsigned int);
6682 : void materialize ();
6683 :
6684 : /* Clean-up. */
6685 : void remove_redundant_permutations ();
6686 :
6687 : /* Masked load lanes discovery. */
6688 : void decide_masked_load_lanes ();
6689 :
6690 : void dump ();
6691 :
6692 : vec_info *m_vinfo;
6693 :
6694 : /* True if we should optimize the graph for size, false if we should
6695 : optimize it for speed. (It wouldn't be easy to make this decision
6696 : more locally.) */
6697 : bool m_optimize_size;
6698 :
6699 : /* A graph of all SLP nodes, with edges leading from uses to definitions.
6700 : In other words, a node's predecessors are its slp_tree parents and
6701 : a node's successors are its slp_tree children. */
6702 : graph *m_slpg = nullptr;
6703 :
6704 : /* The vertices of M_SLPG, indexed by slp_tree::vertex. */
6705 : auto_vec<slpg_vertex> m_vertices;
6706 :
6707 : /* The list of all leaves of M_SLPG. such as external definitions, constants,
6708 : and loads. */
6709 : auto_vec<int> m_leafs;
6710 :
6711 : /* This array has one entry for every vector layout that we're considering.
6712 : Element 0 is null and indicates "no change". Other entries describe
6713 : permutations that are inherent in the current graph and that we would
6714 : like to reverse if possible.
6715 :
6716 : For example, a permutation { 1, 2, 3, 0 } means that something has
6717 : effectively been permuted in that way, such as a load group
6718 : { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
6719 : We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
6720 : in order to put things "back" in order. */
6721 : auto_vec<vec<unsigned> > m_perms;
6722 :
6723 : /* A partitioning of the nodes for which a layout must be chosen.
6724 : Each partition represents an <SCC, cfg loop> pair; that is,
6725 : nodes in different SCCs belong to different partitions, and nodes
6726 : within an SCC can be further partitioned according to a containing
6727 : cfg loop. Partition <SCC1, L1> comes before <SCC2, L2> if:
6728 :
6729 : - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
6730 : from leaves (such as loads) to roots (such as stores).
6731 :
6732 : - SCC1 == SCC2 and L1's header strictly dominates L2's header. */
6733 : auto_vec<slpg_partition_info> m_partitions;
6734 :
6735 : /* The list of all nodes for which a layout must be chosen. Nodes for
6736 : partition P come before the nodes for partition P+1. Nodes within a
6737 : partition are in reverse postorder. */
6738 : auto_vec<unsigned int> m_partitioned_nodes;
6739 :
6740 : /* Index P * num-layouts + L contains the cost of using layout L
6741 : for partition P. */
6742 : auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
6743 :
6744 : /* Index N * num-layouts + L, if nonnull, is a node that provides the
6745 : original output of node N adjusted to have layout L. */
6746 : auto_vec<slp_tree> m_node_layouts;
6747 : };
6748 :
6749 : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
6750 : Also record whether we should optimize anything for speed rather
6751 : than size. */
6752 :
6753 : void
6754 11600796 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
6755 : slp_tree node)
6756 : {
6757 11600796 : unsigned i;
6758 11600796 : slp_tree child;
6759 :
6760 11600796 : if (visited.add (node))
6761 11600796 : return;
6762 :
6763 10738300 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
6764 : {
6765 8243192 : basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
6766 7248026 : if (optimize_bb_for_speed_p (bb))
6767 7118780 : m_optimize_size = false;
6768 : }
6769 :
6770 10738300 : node->vertex = m_vertices.length ();
6771 10738300 : m_vertices.safe_push (slpg_vertex (node));
6772 :
6773 10738300 : bool leaf = true;
6774 10738300 : bool force_leaf = false;
6775 20161236 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
6776 9422936 : if (child)
6777 : {
6778 8472028 : leaf = false;
6779 8472028 : build_vertices (visited, child);
6780 : }
6781 : else
6782 : force_leaf = true;
6783 : /* Since SLP discovery works along use-def edges all cycles have an
6784 : entry - but there's the exception of cycles where we do not handle
6785 : the entry explicitly (but with a NULL SLP node), like some reductions
6786 : and inductions. Force those SLP PHIs to act as leafs to make them
6787 : backwards reachable. */
6788 10738300 : if (leaf || force_leaf)
6789 5306898 : m_leafs.safe_push (node->vertex);
6790 : }
6791 :
6792 : /* Fill the vertices and leafs vector with all nodes in the SLP graph. */
6793 :
6794 : void
6795 1464356 : vect_optimize_slp_pass::build_vertices ()
6796 : {
6797 1464356 : hash_set<slp_tree> visited;
6798 1464356 : unsigned i;
6799 1464356 : slp_instance instance;
6800 1464356 : m_vertices.truncate (0);
6801 1464356 : m_leafs.truncate (0);
6802 7521836 : FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
6803 3128768 : build_vertices (visited, SLP_INSTANCE_TREE (instance));
6804 1464356 : }
6805 :
6806 : /* Apply (reverse) bijectite PERM to VEC. */
6807 :
6808 : template <class T>
6809 : static void
6810 248360 : vect_slp_permute (vec<unsigned> perm,
6811 : vec<T> &vec, bool reverse)
6812 : {
6813 248360 : auto_vec<T, 64> saved;
6814 248360 : saved.create (vec.length ());
6815 818672 : for (unsigned i = 0; i < vec.length (); ++i)
6816 570312 : saved.quick_push (vec[i]);
6817 :
6818 248360 : if (reverse)
6819 : {
6820 1617703 : for (unsigned i = 0; i < vec.length (); ++i)
6821 568192 : vec[perm[i]] = saved[i];
6822 815552 : for (unsigned i = 0; i < vec.length (); ++i)
6823 970541 : gcc_assert (vec[perm[i]] == saved[i]);
6824 : }
6825 : else
6826 : {
6827 6240 : for (unsigned i = 0; i < vec.length (); ++i)
6828 2120 : vec[i] = saved[perm[i]];
6829 250480 : for (unsigned i = 0; i < vec.length (); ++i)
6830 3180 : gcc_assert (vec[i] == saved[perm[i]]);
6831 : }
6832 248360 : }
6833 :
6834 : /* Return the cfg loop that contains NODE. */
6835 :
6836 : struct loop *
6837 4176324 : vect_optimize_slp_pass::containing_loop (slp_tree node)
6838 : {
6839 4176324 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
6840 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
6841 : but are laid out close to their children. */
6842 4176324 : if (!rep)
6843 154289 : return m_vinfo->bbs[0]->loop_father;
6844 4533003 : return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
6845 : }
6846 :
6847 : /* Return true if UD (an edge from a use to a definition) is associated
6848 : with a loop latch edge in the cfg. */
6849 :
6850 : bool
6851 8472028 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
6852 : {
6853 8472028 : slp_tree use = m_vertices[ud->src].node;
6854 8472028 : slp_tree def = m_vertices[ud->dest].node;
6855 8472028 : if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
6856 8472028 : || SLP_TREE_PERMUTE_P (use))
6857 8129090 : || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
6858 : return false;
6859 :
6860 4940280 : stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
6861 4940280 : return (is_a<gphi *> (use_rep->stmt)
6862 385824 : && bb_loop_header_p (gimple_bb (use_rep->stmt))
6863 5157034 : && containing_loop (def) == containing_loop (use));
6864 : }
6865 :
6866 : /* Build the graph. Mark edges that correspond to cfg loop latch edges with
6867 : a nonnull data field. */
6868 :
6869 : void
6870 1464356 : vect_optimize_slp_pass::build_graph ()
6871 : {
6872 1464356 : m_optimize_size = true;
6873 1464356 : build_vertices ();
6874 :
6875 2928712 : m_slpg = new_graph (m_vertices.length ());
6876 15131368 : for (slpg_vertex &v : m_vertices)
6877 32158348 : for (slp_tree child : SLP_TREE_CHILDREN (v.node))
6878 9422936 : if (child)
6879 : {
6880 8472028 : graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
6881 8472028 : if (is_cfg_latch_edge (ud))
6882 206806 : ud->data = this;
6883 : }
6884 1464356 : }
6885 :
6886 : /* Return true if E corresponds to a loop latch edge in the cfg. */
6887 :
6888 : static bool
6889 4338848 : skip_cfg_latch_edges (graph_edge *e)
6890 : {
6891 4338848 : return e->data;
6892 : }
6893 :
6894 : /* Create the node partitions. */
6895 :
6896 : void
6897 732178 : vect_optimize_slp_pass::create_partitions ()
6898 : {
6899 : /* Calculate a postorder of the graph, ignoring edges that correspond
6900 : to natural latch edges in the cfg. Reading the vector from the end
6901 : to the beginning gives the reverse postorder. */
6902 732178 : auto_vec<int> initial_rpo;
6903 1464356 : graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
6904 : false, NULL, skip_cfg_latch_edges);
6905 2196534 : gcc_assert (initial_rpo.length () == m_vertices.length ());
6906 :
6907 : /* Calculate the strongly connected components of the graph. */
6908 732178 : auto_vec<int> scc_grouping;
6909 732178 : unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
6910 :
6911 : /* Create a new index order in which all nodes from the same SCC are
6912 : consecutive. Use scc_pos to record the index of the first node in
6913 : each SCC. */
6914 732178 : auto_vec<unsigned int> scc_pos (num_sccs);
6915 732178 : int last_component = -1;
6916 732178 : unsigned int node_count = 0;
6917 7565121 : for (unsigned int node_i : scc_grouping)
6918 : {
6919 5368587 : if (last_component != m_slpg->vertices[node_i].component)
6920 : {
6921 5235030 : last_component = m_slpg->vertices[node_i].component;
6922 10470060 : gcc_assert (last_component == int (scc_pos.length ()));
6923 5235030 : scc_pos.quick_push (node_count);
6924 : }
6925 5368587 : node_count += 1;
6926 : }
6927 1464356 : gcc_assert (node_count == initial_rpo.length ()
6928 : && last_component + 1 == int (num_sccs));
6929 :
6930 : /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
6931 : inside each SCC following the RPO we calculated above. The fact that
6932 : we ignored natural latch edges when calculating the RPO should ensure
6933 : that, for natural loop nests:
6934 :
6935 : - the first node that we encounter in a cfg loop is the loop header phi
6936 : - the loop header phis are in dominance order
6937 :
6938 : Arranging for this is an optimization (see below) rather than a
6939 : correctness issue. Unnatural loops with a tangled mess of backedges
6940 : will still work correctly, but might give poorer results.
6941 :
6942 : Also update scc_pos so that it gives 1 + the index of the last node
6943 : in the SCC. */
6944 732178 : m_partitioned_nodes.safe_grow (node_count);
6945 6832943 : for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
6946 : {
6947 5368587 : unsigned int node_i = initial_rpo[old_i];
6948 5368587 : unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
6949 5368587 : m_partitioned_nodes[new_i] = node_i;
6950 : }
6951 :
6952 : /* When optimizing for speed, partition each SCC based on the containing
6953 : cfg loop. The order we constructed above should ensure that, for natural
6954 : cfg loops, we'll create sub-SCC partitions for outer loops before
6955 : the corresponding sub-SCC partitions for inner loops. Similarly,
6956 : when one sibling loop A dominates another sibling loop B, we should
6957 : create a sub-SCC partition for A before a sub-SCC partition for B.
6958 :
6959 : As above, nothing depends for correctness on whether this achieves
6960 : a natural nesting, but we should get better results when it does. */
6961 1464356 : m_partitions.reserve (m_vertices.length ());
6962 732178 : unsigned int next_partition_i = 0;
6963 732178 : hash_map<struct loop *, int> loop_partitions;
6964 732178 : unsigned int rpo_begin = 0;
6965 732178 : unsigned int num_partitioned_nodes = 0;
6966 7431564 : for (unsigned int rpo_end : scc_pos)
6967 : {
6968 5235030 : loop_partitions.empty ();
6969 5235030 : unsigned int partition_i = next_partition_i;
6970 10603617 : for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
6971 : {
6972 : /* Handle externals and constants optimistically throughout.
6973 : But treat existing vectors as fixed since we do not handle
6974 : permuting them. */
6975 5368587 : unsigned int node_i = m_partitioned_nodes[rpo_i];
6976 5368587 : auto &vertex = m_vertices[node_i];
6977 5368587 : if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
6978 533050 : && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
6979 5371817 : || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
6980 1595615 : vertex.partition = -1;
6981 : else
6982 : {
6983 3772972 : bool existed;
6984 3772972 : if (m_optimize_size)
6985 30156 : existed = next_partition_i > partition_i;
6986 : else
6987 : {
6988 3742816 : struct loop *loop = containing_loop (vertex.node);
6989 3742816 : auto &entry = loop_partitions.get_or_insert (loop, &existed);
6990 3742816 : if (!existed)
6991 3610500 : entry = next_partition_i;
6992 3742816 : partition_i = entry;
6993 : }
6994 3772972 : if (!existed)
6995 : {
6996 3640564 : m_partitions.quick_push (slpg_partition_info ());
6997 3640564 : next_partition_i += 1;
6998 : }
6999 3772972 : vertex.partition = partition_i;
7000 3772972 : num_partitioned_nodes += 1;
7001 3772972 : m_partitions[partition_i].node_end += 1;
7002 : }
7003 : }
7004 5235030 : rpo_begin = rpo_end;
7005 : }
7006 :
7007 : /* Assign ranges of consecutive node indices to each partition,
7008 : in partition order. Start with node_end being the same as
7009 : node_begin so that the next loop can use it as a counter. */
7010 732178 : unsigned int node_begin = 0;
7011 5837098 : for (auto &partition : m_partitions)
7012 : {
7013 3640564 : partition.node_begin = node_begin;
7014 3640564 : node_begin += partition.node_end;
7015 3640564 : partition.node_end = partition.node_begin;
7016 : }
7017 732178 : gcc_assert (node_begin == num_partitioned_nodes);
7018 :
7019 : /* Finally build the list of nodes in partition order. */
7020 732178 : m_partitioned_nodes.truncate (num_partitioned_nodes);
7021 6832943 : for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
7022 : {
7023 5368587 : int partition_i = m_vertices[node_i].partition;
7024 5368587 : if (partition_i >= 0)
7025 : {
7026 3772972 : unsigned int order_i = m_partitions[partition_i].node_end++;
7027 3772972 : m_partitioned_nodes[order_i] = node_i;
7028 : }
7029 : }
7030 732178 : }
7031 :
7032 : /* Look for edges from earlier partitions into node NODE_I and edges from
7033 : node NODE_I into later partitions. Call:
7034 :
7035 : FN (ud, other_node_i)
7036 :
7037 : for each such use-to-def edge ud, where other_node_i is the node at the
7038 : other end of the edge. */
7039 :
7040 : template<typename T>
7041 : void
7042 4246650 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
7043 : {
7044 4246650 : int partition_i = m_vertices[node_i].partition;
7045 4246650 : for (graph_edge *pred = m_slpg->vertices[node_i].pred;
7046 7357288 : pred; pred = pred->pred_next)
7047 : {
7048 3110638 : int src_partition_i = m_vertices[pred->src].partition;
7049 3110638 : if (src_partition_i >= 0 && src_partition_i != partition_i)
7050 2759242 : fn (pred, pred->src);
7051 : }
7052 4246650 : for (graph_edge *succ = m_slpg->vertices[node_i].succ;
7053 9133155 : succ; succ = succ->succ_next)
7054 : {
7055 4886505 : int dest_partition_i = m_vertices[succ->dest].partition;
7056 4886505 : if (dest_partition_i >= 0 && dest_partition_i != partition_i)
7057 2803005 : fn (succ, succ->dest);
7058 : }
7059 4246650 : }
7060 :
7061 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7062 : that NODE would operate on. This test is independent of NODE's actual
7063 : operation. */
7064 :
7065 : bool
7066 1783211 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
7067 : unsigned int layout_i)
7068 : {
7069 1783211 : if (layout_i == 0)
7070 : return true;
7071 :
7072 1087766 : if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
7073 11080 : return false;
7074 :
7075 : return true;
7076 : }
7077 :
7078 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7079 : that NODE would operate on for each NODE in PARTITION.
7080 : This test is independent of NODE's actual operations. */
7081 :
7082 : bool
7083 25822 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
7084 : &partition,
7085 : unsigned int layout_i)
7086 : {
7087 52077 : for (unsigned int order_i = partition.node_begin;
7088 52077 : order_i < partition.node_end; ++order_i)
7089 : {
7090 26357 : unsigned int node_i = m_partitioned_nodes[order_i];
7091 26357 : auto &vertex = m_vertices[node_i];
7092 :
7093 : /* The layout is incompatible if it is individually incompatible
7094 : with any node in the partition. */
7095 26357 : if (!is_compatible_layout (vertex.node, layout_i))
7096 : return false;
7097 : }
7098 : return true;
7099 : }
7100 :
7101 : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
7102 : to layout TO_LAYOUT_I for a node like NODE. Return -1 if either of the
7103 : layouts is incompatible with NODE or if the change is not possible for
7104 : some other reason.
7105 :
7106 : The properties taken from NODE include the number of lanes and the
7107 : vector type. The actual operation doesn't matter. */
7108 :
7109 : int
7110 752488 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
7111 : unsigned int from_layout_i,
7112 : unsigned int to_layout_i)
7113 : {
7114 752488 : if (!is_compatible_layout (node, from_layout_i)
7115 752488 : || !is_compatible_layout (node, to_layout_i))
7116 : return -1;
7117 :
7118 751807 : if (from_layout_i == to_layout_i)
7119 : return 0;
7120 :
7121 329683 : auto_vec<slp_tree, 1> children (1);
7122 329683 : children.quick_push (node);
7123 329683 : auto_lane_permutation_t perm (SLP_TREE_LANES (node));
7124 329683 : if (from_layout_i > 0)
7125 905950 : for (unsigned int i : m_perms[from_layout_i])
7126 400897 : perm.quick_push ({ 0, i });
7127 : else
7128 525543 : for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
7129 364211 : perm.quick_push ({ 0, i });
7130 329683 : if (to_layout_i > 0)
7131 162081 : vect_slp_permute (m_perms[to_layout_i], perm, true);
7132 329683 : auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
7133 : children, false);
7134 329683 : if (count >= 0)
7135 324338 : return MAX (count, 1);
7136 :
7137 : /* ??? In principle we could try changing via layout 0, giving two
7138 : layout changes rather than 1. Doing that would require
7139 : corresponding support in get_result_with_layout. */
7140 : return -1;
7141 329683 : }
7142 :
7143 : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I. */
7144 :
7145 : inline slpg_partition_layout_costs &
7146 1114787 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
7147 : unsigned int layout_i)
7148 : {
7149 2229574 : return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
7150 : }
7151 :
7152 : /* Change PERM in one of two ways:
7153 :
7154 : - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
7155 : chosen for child I of NODE.
7156 :
7157 : - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
7158 :
7159 : In both cases, arrange for the output to have layout OUT_LAYOUT_I */
7160 :
7161 : void
7162 43878 : vect_optimize_slp_pass::
7163 : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
7164 : int in_layout_i, unsigned int out_layout_i)
7165 : {
7166 252696 : for (auto &entry : perm)
7167 : {
7168 121062 : int this_in_layout_i = in_layout_i;
7169 121062 : if (this_in_layout_i < 0)
7170 : {
7171 78867 : slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
7172 78867 : unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
7173 78867 : if (in_partition_i == -1u)
7174 329 : continue;
7175 78538 : this_in_layout_i = m_partitions[in_partition_i].layout;
7176 : }
7177 120733 : if (this_in_layout_i > 0)
7178 28645 : entry.second = m_perms[this_in_layout_i][entry.second];
7179 : }
7180 43878 : if (out_layout_i > 0)
7181 12531 : vect_slp_permute (m_perms[out_layout_i], perm, true);
7182 43878 : }
7183 :
7184 : /* Check whether the target allows NODE to be rearranged so that the node's
7185 : output has layout OUT_LAYOUT_I. Return the cost of the change if so,
7186 : in the same arbitrary units as for change_layout_cost. Return -1 otherwise.
7187 :
7188 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
7189 : NODE can adapt to the layout changes that have (perhaps provisionally)
7190 : been chosen for NODE's children, so that no extra permutations are
7191 : needed on either the input or the output of NODE.
7192 :
7193 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
7194 : that all inputs will be forced into layout IN_LAYOUT_I beforehand.
7195 :
7196 : IN_LAYOUT_I has no meaning for other types of node.
7197 :
7198 : Keeping the node as-is is always valid. If the target doesn't appear
7199 : to support the node as-is, but might realistically support other layouts,
7200 : then layout 0 instead has the cost of a worst-case permutation. On the
7201 : one hand, this ensures that every node has at least one valid layout,
7202 : avoiding what would otherwise be an awkward special case. On the other,
7203 : it still encourages the pass to change an invalid pre-existing layout
7204 : choice into a valid one. */
7205 :
7206 : int
7207 253121 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
7208 : unsigned int out_layout_i)
7209 : {
7210 253121 : const int fallback_cost = 1;
7211 :
7212 253121 : if (SLP_TREE_PERMUTE_P (node))
7213 : {
7214 35564 : auto_lane_permutation_t tmp_perm;
7215 35564 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7216 :
7217 : /* Check that the child nodes support the chosen layout. Checking
7218 : the first child is enough, since any second child would have the
7219 : same shape. */
7220 35564 : auto first_child = SLP_TREE_CHILDREN (node)[0];
7221 35564 : if (in_layout_i > 0
7222 35564 : && !is_compatible_layout (first_child, in_layout_i))
7223 : return -1;
7224 :
7225 34888 : change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
7226 69776 : int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
7227 : node, tmp_perm,
7228 34888 : SLP_TREE_CHILDREN (node),
7229 : false);
7230 34888 : if (count < 0)
7231 : {
7232 2585 : if (in_layout_i == 0 && out_layout_i == 0)
7233 : {
7234 : /* Use the fallback cost if the node could in principle support
7235 : some nonzero layout for both the inputs and the outputs.
7236 : Otherwise assume that the node will be rejected later
7237 : and rebuilt from scalars. */
7238 644 : if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
7239 : return fallback_cost;
7240 355 : return 0;
7241 : }
7242 : return -1;
7243 : }
7244 :
7245 : /* We currently have no way of telling whether the new layout is cheaper
7246 : or more expensive than the old one. But at least in principle,
7247 : it should be worth making zero permutations (whole-vector shuffles)
7248 : cheaper than real permutations, in case the pass is able to remove
7249 : the latter. */
7250 32303 : return count == 0 ? 0 : 1;
7251 35564 : }
7252 :
7253 217557 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
7254 217557 : if (rep
7255 215640 : && STMT_VINFO_DATA_REF (rep)
7256 72998 : && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
7257 264348 : && SLP_TREE_LOAD_PERMUTATION (node).exists ())
7258 : {
7259 38354 : auto_load_permutation_t tmp_perm;
7260 38354 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7261 38354 : if (out_layout_i > 0)
7262 15037 : vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
7263 :
7264 38354 : poly_uint64 vf = 1;
7265 38354 : if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
7266 11360 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7267 38354 : unsigned int n_perms;
7268 38354 : if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
7269 : nullptr, vf, true, false, &n_perms))
7270 : {
7271 1998 : auto rep = SLP_TREE_REPRESENTATIVE (node);
7272 1998 : if (out_layout_i == 0)
7273 : {
7274 : /* Use the fallback cost if the load is an N-to-N permutation.
7275 : Otherwise assume that the node will be rejected later
7276 : and rebuilt from scalars. */
7277 1472 : if (STMT_VINFO_GROUPED_ACCESS (rep)
7278 2944 : && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
7279 1472 : == SLP_TREE_LANES (node)))
7280 668 : return fallback_cost;
7281 : return 0;
7282 : }
7283 : return -1;
7284 : }
7285 :
7286 : /* See the comment above the corresponding VEC_PERM_EXPR handling. */
7287 36356 : return n_perms == 0 ? 0 : 1;
7288 38354 : }
7289 :
7290 : return 0;
7291 : }
7292 :
7293 : /* Decide which element layouts we should consider using. Calculate the
7294 : weights associated with inserting layout changes on partition edges.
7295 : Also mark partitions that cannot change layout, by setting their
7296 : layout to zero. */
7297 :
7298 : void
7299 732178 : vect_optimize_slp_pass::start_choosing_layouts ()
7300 : {
7301 : /* Used to assign unique permutation indices. */
7302 732178 : using perm_hash = unbounded_hashmap_traits<
7303 : vec_free_hash_base<int_hash_base<unsigned>>,
7304 : int_hash<int, -1, -2>
7305 : >;
7306 732178 : hash_map<vec<unsigned>, int, perm_hash> layout_ids;
7307 :
7308 : /* Layout 0 is "no change". */
7309 732178 : m_perms.safe_push (vNULL);
7310 :
7311 : /* Create layouts from existing permutations. */
7312 732178 : auto_load_permutation_t tmp_perm;
7313 5969506 : for (unsigned int node_i : m_partitioned_nodes)
7314 : {
7315 : /* Leafs also double as entries to the reverse graph. Allow the
7316 : layout of those to be changed. */
7317 3772972 : auto &vertex = m_vertices[node_i];
7318 3772972 : auto &partition = m_partitions[vertex.partition];
7319 3772972 : if (!m_slpg->vertices[node_i].succ)
7320 966134 : partition.layout = 0;
7321 :
7322 : /* Loads and VEC_PERM_EXPRs are the only things generating permutes. */
7323 3772972 : slp_tree node = vertex.node;
7324 3772972 : stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
7325 3772972 : slp_tree child;
7326 3772972 : unsigned HOST_WIDE_INT imin, imax = 0;
7327 3772972 : bool any_permute = false;
7328 3772972 : tmp_perm.truncate (0);
7329 3772972 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
7330 : {
7331 : /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
7332 : unpermuted, record a layout that reverses this permutation.
7333 :
7334 : We would need more work to cope with loads that are internally
7335 : permuted and also have inputs (such as masks for
7336 : IFN_MASK_LOADs). */
7337 650230 : gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
7338 650230 : if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
7339 : {
7340 462971 : partition.layout = -1;
7341 3750645 : continue;
7342 : }
7343 187259 : dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
7344 187259 : imin = DR_GROUP_SIZE (dr_stmt) + 1;
7345 187259 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7346 : }
7347 6124936 : else if (SLP_TREE_PERMUTE_P (node)
7348 145724 : && SLP_TREE_CHILDREN (node).length () == 1
7349 120548 : && (child = SLP_TREE_CHILDREN (node)[0])
7350 3268466 : && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
7351 120548 : .is_constant (&imin)))
7352 : {
7353 : /* If the child has the same vector size as this node,
7354 : reversing the permutation can make the permutation a no-op.
7355 : In other cases it can change a true permutation into a
7356 : full-vector extract. */
7357 120548 : tmp_perm.reserve (SLP_TREE_LANES (node));
7358 449998 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7359 208902 : tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
7360 : }
7361 : else
7362 3002194 : continue;
7363 :
7364 830748 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7365 : {
7366 522941 : unsigned idx = tmp_perm[j];
7367 522941 : imin = MIN (imin, idx);
7368 522941 : imax = MAX (imax, idx);
7369 522941 : if (idx - tmp_perm[0] != j)
7370 170097 : any_permute = true;
7371 : }
7372 : /* If the span doesn't match we'd disrupt VF computation, avoid
7373 : that for now. */
7374 307807 : if (imax - imin + 1 != SLP_TREE_LANES (node))
7375 93770 : continue;
7376 : /* If there's no permute no need to split one out. In this case
7377 : we can consider turning a load into a permuted load, if that
7378 : turns out to be cheaper than alternatives. */
7379 214037 : if (!any_permute)
7380 : {
7381 191552 : partition.layout = -1;
7382 191552 : continue;
7383 : }
7384 :
7385 : /* For now only handle true permutes, like
7386 : vect_attempt_slp_rearrange_stmts did. This allows us to be lazy
7387 : when permuting constants and invariants keeping the permute
7388 : bijective. */
7389 22485 : auto_sbitmap load_index (SLP_TREE_LANES (node));
7390 22485 : bitmap_clear (load_index);
7391 112915 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7392 67945 : bitmap_set_bit (load_index, tmp_perm[j] - imin);
7393 : unsigned j;
7394 89574 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
7395 67247 : if (!bitmap_bit_p (load_index, j))
7396 : break;
7397 22485 : if (j != SLP_TREE_LANES (node))
7398 158 : continue;
7399 :
7400 22327 : vec<unsigned> perm = vNULL;
7401 22327 : perm.safe_grow (SLP_TREE_LANES (node), true);
7402 111616 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7403 66962 : perm[j] = tmp_perm[j] - imin;
7404 :
7405 44654 : if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
7406 : {
7407 : /* Continue to use existing layouts, but don't add any more. */
7408 0 : int *entry = layout_ids.get (perm);
7409 0 : partition.layout = entry ? *entry : 0;
7410 0 : perm.release ();
7411 : }
7412 : else
7413 : {
7414 22327 : bool existed;
7415 22327 : int &layout_i = layout_ids.get_or_insert (perm, &existed);
7416 22327 : if (existed)
7417 7882 : perm.release ();
7418 : else
7419 : {
7420 14445 : layout_i = m_perms.length ();
7421 14445 : m_perms.safe_push (perm);
7422 : }
7423 22327 : partition.layout = layout_i;
7424 : }
7425 22485 : }
7426 :
7427 : /* Initially assume that every layout is possible and has zero cost
7428 : in every partition. */
7429 732178 : m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
7430 1464356 : * m_perms.length ());
7431 :
7432 : /* We have to mark outgoing permutations facing non-associating-reduction
7433 : graph entries that are not represented as to be materialized.
7434 : slp_inst_kind_bb_reduc currently only covers associatable reductions. */
7435 3760918 : for (slp_instance instance : m_vinfo->slp_instances)
7436 1564384 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
7437 : {
7438 7515 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7439 7515 : m_partitions[m_vertices[node_i].partition].layout = 0;
7440 : }
7441 1556869 : else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
7442 : {
7443 582 : stmt_vec_info stmt_info
7444 582 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
7445 582 : vect_reduc_info reduc_info
7446 582 : = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
7447 : SLP_INSTANCE_TREE (instance));
7448 582 : if (needs_fold_left_reduction_p (TREE_TYPE
7449 : (gimple_get_lhs (stmt_info->stmt)),
7450 : VECT_REDUC_INFO_CODE (reduc_info)))
7451 : {
7452 124 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7453 124 : m_partitions[m_vertices[node_i].partition].layout = 0;
7454 : }
7455 : }
7456 :
7457 : /* Check which layouts each node and partition can handle. Calculate the
7458 : weights associated with inserting layout changes on edges. */
7459 5969506 : for (unsigned int node_i : m_partitioned_nodes)
7460 : {
7461 3772972 : auto &vertex = m_vertices[node_i];
7462 3772972 : auto &partition = m_partitions[vertex.partition];
7463 3772972 : slp_tree node = vertex.node;
7464 :
7465 3772972 : vertex.weight = vect_slp_node_weight (m_vinfo, node);
7466 :
7467 3772972 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
7468 : {
7469 : /* We do not handle stores with a permutation, so all
7470 : incoming permutations must have been materialized.
7471 :
7472 : We also don't handle masked grouped loads, which lack a
7473 : permutation vector. In this case the memory locations
7474 : form an implicit second input to the loads, on top of the
7475 : explicit mask input, and the memory input's layout cannot
7476 : be changed.
7477 :
7478 : On the other hand, we do support permuting gather loads and
7479 : masked gather loads, where each scalar load is independent
7480 : of the others. This can be useful if the address/index input
7481 : benefits from permutation. */
7482 3624013 : if (STMT_VINFO_DATA_REF (rep)
7483 1733113 : && STMT_VINFO_GROUPED_ACCESS (rep)
7484 4634454 : && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
7485 823182 : partition.layout = 0;
7486 :
7487 : /* We cannot change the layout of an operation that is
7488 : not independent on lanes. Note this is an explicit
7489 : negative list since that's much shorter than the respective
7490 : positive one but it's critical to keep maintaining it. */
7491 3624013 : if (is_gimple_call (STMT_VINFO_STMT (rep)))
7492 34462 : switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
7493 : {
7494 1098 : case CFN_COMPLEX_ADD_ROT90:
7495 1098 : case CFN_COMPLEX_ADD_ROT270:
7496 1098 : case CFN_COMPLEX_MUL:
7497 1098 : case CFN_COMPLEX_MUL_CONJ:
7498 1098 : case CFN_VEC_ADDSUB:
7499 1098 : case CFN_VEC_FMADDSUB:
7500 1098 : case CFN_VEC_FMSUBADD:
7501 1098 : partition.layout = 0;
7502 : default:;
7503 : }
7504 : }
7505 :
7506 8550846 : auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
7507 : {
7508 4777874 : auto &other_vertex = m_vertices[other_node_i];
7509 :
7510 : /* Count the number of edges from earlier partitions and the number
7511 : of edges to later partitions. */
7512 4777874 : if (other_vertex.partition < vertex.partition)
7513 2388937 : partition.in_degree += 1;
7514 : else
7515 2388937 : partition.out_degree += 1;
7516 :
7517 : /* If the current node uses the result of OTHER_NODE_I, accumulate
7518 : the effects of that. */
7519 4777874 : if (ud->src == int (node_i))
7520 : {
7521 2388937 : other_vertex.out_weight += vertex.weight;
7522 2388937 : other_vertex.out_degree += 1;
7523 : }
7524 8550846 : };
7525 3772972 : for_each_partition_edge (node_i, process_edge);
7526 : }
7527 732178 : }
7528 :
7529 : /* Return the incoming costs for node NODE_I, assuming that each input keeps
7530 : its current (provisional) choice of layout. The inputs do not necessarily
7531 : have the same layout as each other. */
7532 :
7533 : slpg_layout_cost
7534 4379 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
7535 : {
7536 4379 : auto &vertex = m_vertices[node_i];
7537 4379 : slpg_layout_cost cost;
7538 14497 : auto add_cost = [&](graph_edge *, unsigned int other_node_i)
7539 : {
7540 10118 : auto &other_vertex = m_vertices[other_node_i];
7541 10118 : if (other_vertex.partition < vertex.partition)
7542 : {
7543 6772 : auto &other_partition = m_partitions[other_vertex.partition];
7544 13544 : auto &other_costs = partition_layout_costs (other_vertex.partition,
7545 6772 : other_partition.layout);
7546 6772 : slpg_layout_cost this_cost = other_costs.in_cost;
7547 6772 : this_cost.add_serial_cost (other_costs.internal_cost);
7548 6772 : this_cost.split (other_partition.out_degree);
7549 6772 : cost.add_parallel_cost (this_cost);
7550 : }
7551 14497 : };
7552 4379 : for_each_partition_edge (node_i, add_cost);
7553 4379 : return cost;
7554 : }
7555 :
7556 : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
7557 : and layout LAYOUT2_I on cross-partition use-to-def edge UD. Return
7558 : slpg_layout_cost::impossible () if the change isn't possible. */
7559 :
7560 : slpg_layout_cost
7561 752488 : vect_optimize_slp_pass::
7562 : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
7563 : unsigned int layout2_i)
7564 : {
7565 752488 : auto &def_vertex = m_vertices[ud->dest];
7566 752488 : auto &use_vertex = m_vertices[ud->src];
7567 752488 : auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
7568 752488 : auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
7569 752488 : auto factor = change_layout_cost (def_vertex.node, def_layout_i,
7570 : use_layout_i);
7571 752488 : if (factor < 0)
7572 6026 : return slpg_layout_cost::impossible ();
7573 :
7574 : /* We have a choice of putting the layout change at the site of the
7575 : definition or at the site of the use. Prefer the former when
7576 : optimizing for size or when the execution frequency of the
7577 : definition is no greater than the combined execution frequencies of
7578 : the uses. When putting the layout change at the site of the definition,
7579 : divvy up the cost among all consumers. */
7580 746462 : if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
7581 : {
7582 704108 : slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
7583 704108 : cost.split (def_vertex.out_degree);
7584 704108 : return cost;
7585 : }
7586 42354 : return { use_vertex.weight * factor, m_optimize_size };
7587 : }
7588 :
7589 : /* UD represents a use-def link between FROM_NODE_I and a node in a later
7590 : partition; FROM_NODE_I could be the definition node or the use node.
7591 : The node at the other end of the link wants to use layout TO_LAYOUT_I.
7592 : Return the cost of any necessary fix-ups on edge UD, or return
7593 : slpg_layout_cost::impossible () if the change isn't possible.
7594 :
7595 : At this point, FROM_NODE_I's partition has chosen the cheapest
7596 : layout based on the information available so far, but this choice
7597 : is only provisional. */
7598 :
7599 : slpg_layout_cost
7600 202677 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
7601 : unsigned int to_layout_i)
7602 : {
7603 202677 : auto &from_vertex = m_vertices[from_node_i];
7604 202677 : unsigned int from_partition_i = from_vertex.partition;
7605 202677 : slpg_partition_info &from_partition = m_partitions[from_partition_i];
7606 202677 : gcc_assert (from_partition.layout >= 0);
7607 :
7608 : /* First calculate the cost on the assumption that FROM_PARTITION sticks
7609 : with its current layout preference. */
7610 202677 : slpg_layout_cost cost = slpg_layout_cost::impossible ();
7611 202677 : auto edge_cost = edge_layout_cost (ud, from_node_i,
7612 202677 : from_partition.layout, to_layout_i);
7613 202677 : if (edge_cost.is_possible ())
7614 : {
7615 398990 : auto &from_costs = partition_layout_costs (from_partition_i,
7616 199495 : from_partition.layout);
7617 199495 : cost = from_costs.in_cost;
7618 199495 : cost.add_serial_cost (from_costs.internal_cost);
7619 199495 : cost.split (from_partition.out_degree);
7620 199495 : cost.add_serial_cost (edge_cost);
7621 : }
7622 3182 : else if (from_partition.layout == 0)
7623 : /* We must allow the source partition to have layout 0 as a fallback,
7624 : in case all other options turn out to be impossible. */
7625 3182 : return cost;
7626 :
7627 : /* Take the minimum of that cost and the cost that applies if
7628 : FROM_PARTITION instead switches to TO_LAYOUT_I. */
7629 199495 : auto &direct_layout_costs = partition_layout_costs (from_partition_i,
7630 : to_layout_i);
7631 199495 : if (direct_layout_costs.is_possible ())
7632 : {
7633 171861 : slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
7634 171861 : direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
7635 171861 : direct_cost.split (from_partition.out_degree);
7636 171861 : if (!cost.is_possible ()
7637 171861 : || direct_cost.is_better_than (cost, m_optimize_size))
7638 35494 : cost = direct_cost;
7639 : }
7640 :
7641 199495 : return cost;
7642 : }
7643 :
7644 : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
7645 : partition; TO_NODE_I could be the definition node or the use node.
7646 : The node at the other end of the link wants to use layout FROM_LAYOUT_I;
7647 : return the cost of any necessary fix-ups on edge UD, or
7648 : slpg_layout_cost::impossible () if the choice cannot be made.
7649 :
7650 : At this point, TO_NODE_I's partition has a fixed choice of layout. */
7651 :
7652 : slpg_layout_cost
7653 178816 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
7654 : unsigned int from_layout_i)
7655 : {
7656 178816 : auto &to_vertex = m_vertices[to_node_i];
7657 178816 : unsigned int to_partition_i = to_vertex.partition;
7658 178816 : slpg_partition_info &to_partition = m_partitions[to_partition_i];
7659 178816 : gcc_assert (to_partition.layout >= 0);
7660 :
7661 : /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
7662 : adjusted for this input having layout FROM_LAYOUT_I. Assume that
7663 : any other inputs keep their current choice of layout. */
7664 178816 : auto &to_costs = partition_layout_costs (to_partition_i,
7665 : to_partition.layout);
7666 178816 : if (ud->src == int (to_node_i)
7667 178534 : && SLP_TREE_PERMUTE_P (to_vertex.node))
7668 : {
7669 11436 : auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
7670 11436 : auto old_layout = from_partition.layout;
7671 11436 : from_partition.layout = from_layout_i;
7672 22872 : int factor = internal_node_cost (to_vertex.node, -1,
7673 11436 : to_partition.layout);
7674 11436 : from_partition.layout = old_layout;
7675 11436 : if (factor >= 0)
7676 : {
7677 10529 : slpg_layout_cost cost = to_costs.out_cost;
7678 10529 : cost.add_serial_cost ({ to_vertex.weight * factor,
7679 : m_optimize_size });
7680 10529 : cost.split (to_partition.in_degree);
7681 10529 : return cost;
7682 : }
7683 : }
7684 :
7685 : /* Compute the cost if we insert any necessary layout change on edge UD. */
7686 168287 : auto edge_cost = edge_layout_cost (ud, to_node_i,
7687 168287 : to_partition.layout, from_layout_i);
7688 168287 : if (edge_cost.is_possible ())
7689 : {
7690 168287 : slpg_layout_cost cost = to_costs.out_cost;
7691 168287 : cost.add_serial_cost (to_costs.internal_cost);
7692 168287 : cost.split (to_partition.in_degree);
7693 168287 : cost.add_serial_cost (edge_cost);
7694 168287 : return cost;
7695 : }
7696 :
7697 0 : return slpg_layout_cost::impossible ();
7698 : }
7699 :
7700 : /* Make a forward pass through the partitions, accumulating input costs.
7701 : Make a tentative (provisional) choice of layout for each partition,
7702 : ensuring that this choice still allows later partitions to keep
7703 : their original layout. */
7704 :
7705 : void
7706 6616 : vect_optimize_slp_pass::forward_pass ()
7707 : {
7708 131184 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
7709 : ++partition_i)
7710 : {
7711 124568 : auto &partition = m_partitions[partition_i];
7712 :
7713 : /* If the partition consists of a single VEC_PERM_EXPR, precompute
7714 : the incoming cost that would apply if every predecessor partition
7715 : keeps its current layout. This is used within the loop below. */
7716 124568 : slpg_layout_cost in_cost;
7717 124568 : slp_tree single_node = nullptr;
7718 124568 : if (partition.node_end == partition.node_begin + 1)
7719 : {
7720 119919 : unsigned int node_i = m_partitioned_nodes[partition.node_begin];
7721 119919 : single_node = m_vertices[node_i].node;
7722 119919 : if (SLP_TREE_PERMUTE_P (single_node))
7723 4379 : in_cost = total_in_cost (node_i);
7724 : }
7725 :
7726 : /* Go through the possible layouts. Decide which ones are valid
7727 : for this partition and record which of the valid layouts has
7728 : the lowest cost. */
7729 124568 : unsigned int min_layout_i = 0;
7730 124568 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7731 384268 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7732 : {
7733 259700 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7734 259700 : if (!layout_costs.is_possible ())
7735 63679 : continue;
7736 :
7737 : /* If the recorded layout is already 0 then the layout cannot
7738 : change. */
7739 259700 : if (partition.layout == 0 && layout_i != 0)
7740 : {
7741 49678 : layout_costs.mark_impossible ();
7742 49678 : continue;
7743 : }
7744 :
7745 210022 : bool is_possible = true;
7746 442592 : for (unsigned int order_i = partition.node_begin;
7747 442592 : order_i < partition.node_end; ++order_i)
7748 : {
7749 243701 : unsigned int node_i = m_partitioned_nodes[order_i];
7750 243701 : auto &vertex = m_vertices[node_i];
7751 :
7752 : /* Reject the layout if it is individually incompatible
7753 : with any node in the partition. */
7754 243701 : if (!is_compatible_layout (vertex.node, layout_i))
7755 : {
7756 9621 : is_possible = false;
7757 11131 : break;
7758 : }
7759 :
7760 619273 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7761 : {
7762 385193 : auto &other_vertex = m_vertices[other_node_i];
7763 385193 : if (other_vertex.partition < vertex.partition)
7764 : {
7765 : /* Accumulate the incoming costs from earlier
7766 : partitions, plus the cost of any layout changes
7767 : on UD itself. */
7768 202677 : auto cost = forward_cost (ud, other_node_i, layout_i);
7769 202677 : if (!cost.is_possible ())
7770 3182 : is_possible = false;
7771 : else
7772 199495 : layout_costs.in_cost.add_parallel_cost (cost);
7773 : }
7774 : else
7775 : /* Reject the layout if it would make layout 0 impossible
7776 : for later partitions. This amounts to testing that the
7777 : target supports reversing the layout change on edges
7778 : to later partitions.
7779 :
7780 : In principle, it might be possible to push a layout
7781 : change all the way down a graph, so that it never
7782 : needs to be reversed and so that the target doesn't
7783 : need to support the reverse operation. But it would
7784 : be awkward to bail out if we hit a partition that
7785 : does not support the new layout, especially since
7786 : we are not dealing with a lattice. */
7787 182516 : is_possible &= edge_layout_cost (ud, other_node_i, 0,
7788 182516 : layout_i).is_possible ();
7789 619273 : };
7790 234080 : for_each_partition_edge (node_i, add_cost);
7791 :
7792 : /* Accumulate the cost of using LAYOUT_I within NODE,
7793 : both for the inputs and the outputs. */
7794 234080 : int factor = internal_node_cost (vertex.node, layout_i,
7795 : layout_i);
7796 234080 : if (factor < 0)
7797 : {
7798 1510 : is_possible = false;
7799 1510 : break;
7800 : }
7801 232570 : else if (factor)
7802 39009 : layout_costs.internal_cost.add_serial_cost
7803 39009 : ({ vertex.weight * factor, m_optimize_size });
7804 : }
7805 210022 : if (!is_possible)
7806 : {
7807 14001 : layout_costs.mark_impossible ();
7808 14001 : continue;
7809 : }
7810 :
7811 : /* Combine the incoming and partition-internal costs. */
7812 196021 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7813 196021 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7814 :
7815 : /* If this partition consists of a single VEC_PERM_EXPR, see
7816 : if the VEC_PERM_EXPR can be changed to support output layout
7817 : LAYOUT_I while keeping all the provisional choices of input
7818 : layout. */
7819 196021 : if (single_node && SLP_TREE_PERMUTE_P (single_node))
7820 : {
7821 7605 : int factor = internal_node_cost (single_node, -1, layout_i);
7822 7605 : if (factor >= 0)
7823 : {
7824 6879 : auto weight = m_vertices[single_node->vertex].weight;
7825 6879 : slpg_layout_cost internal_cost
7826 6879 : = { weight * factor, m_optimize_size };
7827 :
7828 6879 : slpg_layout_cost alt_cost = in_cost;
7829 6879 : alt_cost.add_serial_cost (internal_cost);
7830 6879 : if (alt_cost.is_better_than (combined_cost, m_optimize_size))
7831 : {
7832 1670 : combined_cost = alt_cost;
7833 1670 : layout_costs.in_cost = in_cost;
7834 1670 : layout_costs.internal_cost = internal_cost;
7835 : }
7836 : }
7837 : }
7838 :
7839 : /* Record the layout with the lowest cost. Prefer layout 0 in
7840 : the event of a tie between it and another layout. */
7841 196021 : if (!min_layout_cost.is_possible ()
7842 71453 : || combined_cost.is_better_than (min_layout_cost,
7843 : m_optimize_size))
7844 : {
7845 142025 : min_layout_i = layout_i;
7846 142025 : min_layout_cost = combined_cost;
7847 : }
7848 : }
7849 :
7850 : /* This loop's handling of earlier partitions should ensure that
7851 : choosing the original layout for the current partition is no
7852 : less valid than it was in the original graph, even with the
7853 : provisional layout choices for those earlier partitions. */
7854 124568 : gcc_assert (min_layout_cost.is_possible ());
7855 124568 : partition.layout = min_layout_i;
7856 : }
7857 6616 : }
7858 :
7859 : /* Make a backward pass through the partitions, accumulating output costs.
7860 : Make a final choice of layout for each partition. */
7861 :
7862 : void
7863 6616 : vect_optimize_slp_pass::backward_pass ()
7864 : {
7865 137800 : for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
7866 : {
7867 124568 : auto &partition = m_partitions[partition_i];
7868 :
7869 124568 : unsigned int min_layout_i = 0;
7870 124568 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7871 384268 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7872 : {
7873 259700 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7874 259700 : if (!layout_costs.is_possible ())
7875 63679 : continue;
7876 :
7877 : /* Accumulate the costs from successor partitions. */
7878 196021 : bool is_possible = true;
7879 425696 : for (unsigned int order_i = partition.node_begin;
7880 425696 : order_i < partition.node_end; ++order_i)
7881 : {
7882 229675 : unsigned int node_i = m_partitioned_nodes[order_i];
7883 229675 : auto &vertex = m_vertices[node_i];
7884 607499 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7885 : {
7886 377824 : auto &other_vertex = m_vertices[other_node_i];
7887 377824 : auto &other_partition = m_partitions[other_vertex.partition];
7888 377824 : if (other_vertex.partition > vertex.partition)
7889 : {
7890 : /* Accumulate the incoming costs from later
7891 : partitions, plus the cost of any layout changes
7892 : on UD itself. */
7893 178816 : auto cost = backward_cost (ud, other_node_i, layout_i);
7894 178816 : if (!cost.is_possible ())
7895 0 : is_possible = false;
7896 : else
7897 178816 : layout_costs.out_cost.add_parallel_cost (cost);
7898 : }
7899 : else
7900 : /* Make sure that earlier partitions can (if necessary
7901 : or beneficial) keep the layout that they chose in
7902 : the forward pass. This ensures that there is at
7903 : least one valid choice of layout. */
7904 199008 : is_possible &= edge_layout_cost (ud, other_node_i,
7905 199008 : other_partition.layout,
7906 199008 : layout_i).is_possible ();
7907 607499 : };
7908 229675 : for_each_partition_edge (node_i, add_cost);
7909 : }
7910 196021 : if (!is_possible)
7911 : {
7912 0 : layout_costs.mark_impossible ();
7913 0 : continue;
7914 : }
7915 :
7916 : /* Locally combine the costs from the forward and backward passes.
7917 : (This combined cost is not passed on, since that would lead
7918 : to double counting.) */
7919 196021 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7920 196021 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7921 196021 : combined_cost.add_serial_cost (layout_costs.out_cost);
7922 :
7923 : /* Record the layout with the lowest cost. Prefer layout 0 in
7924 : the event of a tie between it and another layout. */
7925 196021 : if (!min_layout_cost.is_possible ()
7926 71453 : || combined_cost.is_better_than (min_layout_cost,
7927 : m_optimize_size))
7928 : {
7929 135607 : min_layout_i = layout_i;
7930 135607 : min_layout_cost = combined_cost;
7931 : }
7932 : }
7933 :
7934 124568 : gcc_assert (min_layout_cost.is_possible ());
7935 124568 : partition.layout = min_layout_i;
7936 : }
7937 6616 : }
7938 :
7939 : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
7940 : NODE already has the layout that was selected for its partition. */
7941 :
7942 : slp_tree
7943 175417 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
7944 : unsigned int to_layout_i)
7945 : {
7946 175417 : unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
7947 175417 : slp_tree result = m_node_layouts[result_i];
7948 175417 : if (result)
7949 : return result;
7950 :
7951 174500 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
7952 174500 : || (SLP_TREE_DEF_TYPE (node) == vect_external_def
7953 : /* We can't permute vector defs in place. */
7954 22151 : && SLP_TREE_VEC_DEFS (node).is_empty ()))
7955 : {
7956 : /* If the vector is uniform or unchanged, there's nothing to do. */
7957 47100 : if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
7958 : result = node;
7959 : else
7960 : {
7961 3677 : auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
7962 3677 : result = vect_create_new_slp_node (scalar_ops);
7963 3677 : vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
7964 : }
7965 : }
7966 : else
7967 : {
7968 127400 : unsigned int partition_i = m_vertices[node->vertex].partition;
7969 127400 : unsigned int from_layout_i = m_partitions[partition_i].layout;
7970 127400 : if (from_layout_i == to_layout_i)
7971 126183 : return node;
7972 :
7973 : /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
7974 : permutation instead of a serial one. Leave the new permutation
7975 : in TMP_PERM on success. */
7976 1217 : auto_lane_permutation_t tmp_perm;
7977 1217 : unsigned int num_inputs = 1;
7978 1217 : if (SLP_TREE_PERMUTE_P (node))
7979 : {
7980 103 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7981 103 : if (from_layout_i != 0)
7982 43 : vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
7983 103 : if (to_layout_i != 0)
7984 64 : vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
7985 103 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7986 : tmp_perm,
7987 103 : SLP_TREE_CHILDREN (node),
7988 : false) >= 0)
7989 103 : num_inputs = SLP_TREE_CHILDREN (node).length ();
7990 : else
7991 0 : tmp_perm.truncate (0);
7992 : }
7993 :
7994 1217 : if (dump_enabled_p ())
7995 : {
7996 71 : if (tmp_perm.length () > 0)
7997 6 : dump_printf_loc (MSG_NOTE, vect_location,
7998 : "duplicating permutation node %p with"
7999 : " layout %d\n",
8000 : (void *) node, to_layout_i);
8001 : else
8002 65 : dump_printf_loc (MSG_NOTE, vect_location,
8003 : "inserting permutation node in place of %p\n",
8004 : (void *) node);
8005 : }
8006 :
8007 1217 : unsigned int num_lanes = SLP_TREE_LANES (node);
8008 1217 : result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
8009 1217 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
8010 : {
8011 1217 : auto &stmts = SLP_TREE_SCALAR_STMTS (result);
8012 1217 : stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
8013 1217 : if (from_layout_i != 0)
8014 500 : vect_slp_permute (m_perms[from_layout_i], stmts, false);
8015 1217 : if (to_layout_i != 0)
8016 721 : vect_slp_permute (m_perms[to_layout_i], stmts, true);
8017 : }
8018 1217 : SLP_TREE_LANES (result) = num_lanes;
8019 1217 : SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
8020 1217 : result->vertex = -1;
8021 :
8022 1217 : auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
8023 1217 : if (tmp_perm.length ())
8024 : {
8025 103 : lane_perm.safe_splice (tmp_perm);
8026 103 : SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
8027 : }
8028 : else
8029 : {
8030 1114 : lane_perm.create (num_lanes);
8031 4538 : for (unsigned j = 0; j < num_lanes; ++j)
8032 2310 : lane_perm.quick_push ({ 0, j });
8033 1114 : if (from_layout_i != 0)
8034 457 : vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
8035 1114 : if (to_layout_i != 0)
8036 657 : vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
8037 1114 : SLP_TREE_CHILDREN (result).safe_push (node);
8038 : }
8039 4940 : for (slp_tree child : SLP_TREE_CHILDREN (result))
8040 1289 : child->refcnt++;
8041 1217 : }
8042 48317 : m_node_layouts[result_i] = result;
8043 48317 : return result;
8044 : }
8045 :
8046 : /* Apply the chosen vector layouts to the SLP graph. */
8047 :
8048 : void
8049 13871 : vect_optimize_slp_pass::materialize ()
8050 : {
8051 : /* We no longer need the costs, so avoid having two O(N * P) arrays
8052 : live at the same time. */
8053 13871 : m_partition_layout_costs.release ();
8054 41613 : m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
8055 :
8056 27742 : auto_sbitmap fully_folded (m_vertices.length ());
8057 13871 : bitmap_clear (fully_folded);
8058 198586 : for (unsigned int node_i : m_partitioned_nodes)
8059 : {
8060 156973 : auto &vertex = m_vertices[node_i];
8061 156973 : slp_tree node = vertex.node;
8062 156973 : int layout_i = m_partitions[vertex.partition].layout;
8063 156973 : gcc_assert (layout_i >= 0);
8064 :
8065 : /* Rearrange the scalar statements to match the chosen layout. */
8066 156973 : if (layout_i > 0)
8067 26718 : vect_slp_permute (m_perms[layout_i],
8068 26718 : SLP_TREE_SCALAR_STMTS (node), true);
8069 :
8070 : /* Update load and lane permutations. */
8071 156973 : if (SLP_TREE_PERMUTE_P (node))
8072 : {
8073 : /* First try to absorb the input vector layouts. If that fails,
8074 : force the inputs to have layout LAYOUT_I too. We checked that
8075 : that was possible before deciding to use nonzero output layouts.
8076 : (Note that at this stage we don't really have any guarantee that
8077 : the target supports the original VEC_PERM_EXPR.) */
8078 8346 : auto &perm = SLP_TREE_LANE_PERMUTATION (node);
8079 8346 : auto_lane_permutation_t tmp_perm;
8080 8346 : tmp_perm.safe_splice (perm);
8081 8346 : change_vec_perm_layout (node, tmp_perm, -1, layout_i);
8082 8346 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
8083 : tmp_perm,
8084 8346 : SLP_TREE_CHILDREN (node),
8085 : false) >= 0)
8086 : {
8087 7702 : if (dump_enabled_p ()
8088 8630 : && !std::equal (tmp_perm.begin (), tmp_perm.end (),
8089 : perm.begin ()))
8090 58 : dump_printf_loc (MSG_NOTE, vect_location,
8091 : "absorbing input layouts into %p\n",
8092 : (void *) node);
8093 42423 : std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
8094 7702 : bitmap_set_bit (fully_folded, node_i);
8095 : }
8096 : else
8097 : {
8098 : /* Not MSG_MISSED because it would make no sense to users. */
8099 644 : if (dump_enabled_p ())
8100 46 : dump_printf_loc (MSG_NOTE, vect_location,
8101 : "failed to absorb input layouts into %p\n",
8102 : (void *) node);
8103 644 : change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
8104 : }
8105 8346 : }
8106 : else
8107 : {
8108 148627 : gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
8109 148627 : auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
8110 148627 : if (layout_i > 0)
8111 : /* ??? When we handle non-bijective permutes the idea
8112 : is that we can force the load-permutation to be
8113 : { min, min + 1, min + 2, ... max }. But then the
8114 : scalar defs might no longer match the lane content
8115 : which means wrong-code with live lane vectorization.
8116 : So we possibly have to have NULL entries for those. */
8117 25874 : vect_slp_permute (m_perms[layout_i], load_perm, true);
8118 : }
8119 : }
8120 :
8121 : /* Do this before any nodes disappear, since it involves a walk
8122 : over the leaves. */
8123 13871 : remove_redundant_permutations ();
8124 :
8125 : /* Replace each child with a correctly laid-out version. */
8126 198586 : for (unsigned int node_i : m_partitioned_nodes)
8127 : {
8128 : /* Skip nodes that have already been handled above. */
8129 156973 : if (bitmap_bit_p (fully_folded, node_i))
8130 7702 : continue;
8131 :
8132 149271 : auto &vertex = m_vertices[node_i];
8133 149271 : int in_layout_i = m_partitions[vertex.partition].layout;
8134 149271 : gcc_assert (in_layout_i >= 0);
8135 :
8136 : unsigned j;
8137 : slp_tree child;
8138 443965 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
8139 : {
8140 179699 : if (!child)
8141 4282 : continue;
8142 :
8143 175417 : slp_tree new_child = get_result_with_layout (child, in_layout_i);
8144 175417 : if (new_child != child)
8145 : {
8146 5478 : vect_free_slp_tree (child);
8147 5478 : SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
8148 5478 : new_child->refcnt += 1;
8149 : }
8150 : }
8151 : }
8152 13871 : }
8153 :
8154 : /* Elide load permutations that are not necessary. Such permutations might
8155 : be pre-existing, rather than created by the layout optimizations. */
8156 :
8157 : void
8158 732178 : vect_optimize_slp_pass::remove_redundant_permutations ()
8159 : {
8160 4849983 : for (unsigned int node_i : m_leafs)
8161 : {
8162 2653449 : slp_tree node = m_vertices[node_i].node;
8163 2653449 : if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
8164 2003219 : continue;
8165 :
8166 : /* In basic block vectorization we allow any subchain of an interleaving
8167 : chain.
8168 : FORNOW: not in loop SLP because of realignment complications. */
8169 650230 : if (is_a <bb_vec_info> (m_vinfo))
8170 : {
8171 193411 : bool subchain_p = true;
8172 : stmt_vec_info next_load_info = NULL;
8173 : stmt_vec_info load_info;
8174 : unsigned j;
8175 193411 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
8176 : {
8177 159766 : if (j != 0
8178 159766 : && (next_load_info != load_info
8179 70941 : || ! load_info
8180 70941 : || DR_GROUP_GAP (load_info) != 1))
8181 : {
8182 : subchain_p = false;
8183 : break;
8184 : }
8185 127966 : next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
8186 : }
8187 65445 : if (subchain_p)
8188 : {
8189 33645 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8190 33645 : continue;
8191 : }
8192 : }
8193 : else
8194 : {
8195 584785 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
8196 584785 : bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
8197 : /* When this isn't a grouped access we know it's single element
8198 : and contiguous. */
8199 584785 : if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
8200 : {
8201 462971 : if (!this_load_permuted
8202 462971 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8203 462183 : || SLP_TREE_LANES (node) == 1))
8204 462201 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8205 462971 : continue;
8206 : }
8207 121814 : stmt_vec_info first_stmt_info
8208 121814 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
8209 122347 : if (!this_load_permuted
8210 : /* The load requires permutation when unrolling exposes
8211 : a gap either because the group is larger than the SLP
8212 : group-size or because there is a gap between the groups. */
8213 121814 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8214 98842 : || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
8215 147 : && DR_GROUP_GAP (first_stmt_info) == 0)))
8216 : {
8217 533 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8218 533 : continue;
8219 : }
8220 : }
8221 : }
8222 732178 : }
8223 :
8224 : /* Print the partition graph and layout information to the dump file. */
8225 :
8226 : void
8227 691 : vect_optimize_slp_pass::dump ()
8228 : {
8229 691 : dump_printf_loc (MSG_NOTE, vect_location,
8230 : "SLP optimize permutations:\n");
8231 1395 : for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
8232 : {
8233 704 : dump_printf_loc (MSG_NOTE, vect_location, " %d: { ", layout_i);
8234 704 : const char *sep = "";
8235 5976 : for (unsigned int idx : m_perms[layout_i])
8236 : {
8237 3864 : dump_printf (MSG_NOTE, "%s%d", sep, idx);
8238 3864 : sep = ", ";
8239 : }
8240 704 : dump_printf (MSG_NOTE, " }\n");
8241 : }
8242 691 : dump_printf_loc (MSG_NOTE, vect_location,
8243 : "SLP optimize partitions:\n");
8244 5996 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
8245 : ++partition_i)
8246 : {
8247 5305 : auto &partition = m_partitions[partition_i];
8248 5305 : dump_printf_loc (MSG_NOTE, vect_location, " -------------\n");
8249 5305 : dump_printf_loc (MSG_NOTE, vect_location,
8250 : " partition %d (layout %d):\n",
8251 : partition_i, partition.layout);
8252 5305 : dump_printf_loc (MSG_NOTE, vect_location, " nodes:\n");
8253 10849 : for (unsigned int order_i = partition.node_begin;
8254 10849 : order_i < partition.node_end; ++order_i)
8255 : {
8256 5544 : auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
8257 11088 : dump_printf_loc (MSG_NOTE, vect_location, " - %p:\n",
8258 5544 : (void *) vertex.node);
8259 5544 : dump_printf_loc (MSG_NOTE, vect_location,
8260 : " weight: %f\n",
8261 : vertex.weight.to_double ());
8262 5544 : if (vertex.out_degree)
8263 4344 : dump_printf_loc (MSG_NOTE, vect_location,
8264 : " out weight: %f (degree %d)\n",
8265 : vertex.out_weight.to_double (),
8266 : vertex.out_degree);
8267 5544 : if (SLP_TREE_PERMUTE_P (vertex.node))
8268 510 : dump_printf_loc (MSG_NOTE, vect_location,
8269 : " op: VEC_PERM_EXPR\n");
8270 5034 : else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
8271 5016 : dump_printf_loc (MSG_NOTE, vect_location,
8272 : " op template: %G", rep->stmt);
8273 : }
8274 5305 : dump_printf_loc (MSG_NOTE, vect_location, " edges:\n");
8275 10849 : for (unsigned int order_i = partition.node_begin;
8276 10849 : order_i < partition.node_end; ++order_i)
8277 : {
8278 5544 : unsigned int node_i = m_partitioned_nodes[order_i];
8279 5544 : auto &vertex = m_vertices[node_i];
8280 16782 : auto print_edge = [&](graph_edge *, unsigned int other_node_i)
8281 : {
8282 11238 : auto &other_vertex = m_vertices[other_node_i];
8283 11238 : if (other_vertex.partition < vertex.partition)
8284 5619 : dump_printf_loc (MSG_NOTE, vect_location,
8285 : " - %p [%d] --> %p\n",
8286 5619 : (void *) other_vertex.node,
8287 : other_vertex.partition,
8288 5619 : (void *) vertex.node);
8289 : else
8290 5619 : dump_printf_loc (MSG_NOTE, vect_location,
8291 : " - %p --> [%d] %p\n",
8292 5619 : (void *) vertex.node,
8293 : other_vertex.partition,
8294 5619 : (void *) other_vertex.node);
8295 16782 : };
8296 5544 : for_each_partition_edge (node_i, print_edge);
8297 : }
8298 :
8299 16114 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
8300 : {
8301 10809 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
8302 10809 : if (layout_costs.is_possible ())
8303 : {
8304 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8305 : " layout %d:%s\n", layout_i,
8306 8952 : partition.layout == int (layout_i)
8307 : ? " (*)" : "");
8308 8952 : slpg_layout_cost combined_cost = layout_costs.in_cost;
8309 8952 : combined_cost.add_serial_cost (layout_costs.internal_cost);
8310 8952 : combined_cost.add_serial_cost (layout_costs.out_cost);
8311 : #define TEMPLATE "{depth: %f, total: %f}"
8312 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8313 : " " TEMPLATE "\n",
8314 : layout_costs.in_cost.depth.to_double (),
8315 : layout_costs.in_cost.total.to_double ());
8316 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8317 : " + " TEMPLATE "\n",
8318 : layout_costs.internal_cost.depth.to_double (),
8319 : layout_costs.internal_cost.total.to_double ());
8320 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8321 : " + " TEMPLATE "\n",
8322 : layout_costs.out_cost.depth.to_double (),
8323 : layout_costs.out_cost.total.to_double ());
8324 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8325 : " = " TEMPLATE "\n",
8326 : combined_cost.depth.to_double (),
8327 : combined_cost.total.to_double ());
8328 : #undef TEMPLATE
8329 : }
8330 : else
8331 1857 : dump_printf_loc (MSG_NOTE, vect_location,
8332 : " layout %d: rejected\n", layout_i);
8333 : }
8334 : }
8335 691 : }
8336 :
8337 : /* Masked load lanes discovery. */
8338 :
8339 : void
8340 732178 : vect_optimize_slp_pass::decide_masked_load_lanes ()
8341 : {
8342 7566247 : for (auto v : m_vertices)
8343 : {
8344 5369713 : slp_tree node = v.node;
8345 5369713 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8346 3770863 : || SLP_TREE_PERMUTE_P (node))
8347 1745700 : continue;
8348 3624013 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
8349 1733113 : if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
8350 : /* The mask has to be uniform. */
8351 1010441 : || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
8352 1010363 : || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
8353 3624098 : || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
8354 : IFN_MASK_LOAD))
8355 3623980 : continue;
8356 33 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8357 66 : if (STMT_VINFO_STRIDED_P (stmt_info)
8358 33 : || compare_step_with_zero (m_vinfo, stmt_info) <= 0
8359 63 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
8360 30 : DR_GROUP_SIZE (stmt_info),
8361 : true) == IFN_LAST)
8362 33 : continue;
8363 :
8364 : /* Uniform masks need to be suitably represented. */
8365 0 : slp_tree mask = SLP_TREE_CHILDREN (node)[0];
8366 0 : if (!SLP_TREE_PERMUTE_P (mask)
8367 0 : || SLP_TREE_CHILDREN (mask).length () != 1)
8368 0 : continue;
8369 0 : bool match = true;
8370 0 : for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
8371 0 : if (perm.first != 0 || perm.second != 0)
8372 : {
8373 : match = false;
8374 : break;
8375 : }
8376 0 : if (!match)
8377 0 : continue;
8378 :
8379 : /* Now see if the consumer side matches. */
8380 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8381 0 : pred; pred = pred->pred_next)
8382 : {
8383 0 : slp_tree pred_node = m_vertices[pred->src].node;
8384 : /* All consumers should be a permute with a single outgoing lane. */
8385 0 : if (!SLP_TREE_PERMUTE_P (pred_node)
8386 0 : || SLP_TREE_LANES (pred_node) != 1)
8387 : {
8388 : match = false;
8389 : break;
8390 : }
8391 0 : gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
8392 : }
8393 0 : if (!match)
8394 0 : continue;
8395 : /* Now we can mark the nodes as to use load lanes. */
8396 0 : node->ldst_lanes = true;
8397 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8398 0 : pred; pred = pred->pred_next)
8399 0 : m_vertices[pred->src].node->ldst_lanes = true;
8400 : /* The catch is we have to massage the mask. We have arranged
8401 : analyzed uniform masks to be represented by a splat VEC_PERM
8402 : which we can now simply elide as we cannot easily re-do SLP
8403 : discovery here. */
8404 0 : slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
8405 0 : SLP_TREE_REF_COUNT (new_mask)++;
8406 0 : SLP_TREE_CHILDREN (node)[0] = new_mask;
8407 0 : vect_free_slp_tree (mask);
8408 : }
8409 732178 : }
8410 :
8411 : /* Perform legitimizing attempts. This is intended to improve the
8412 : situation when layout 0 is not valid which is a situation the cost
8413 : based propagation does not handle well.
8414 : Return true if further layout optimization is possible, false if
8415 : the layout configuration should be considered final. */
8416 :
8417 : bool
8418 13871 : vect_optimize_slp_pass::legitimize ()
8419 : {
8420 : /* Perform a very simple legitimizing attempt by attempting to choose
8421 : a single layout for all partitions that will make all permutations
8422 : a noop. That should also be the optimal layout choice in case
8423 : layout zero is legitimate.
8424 : ??? Disconnected components of the SLP graph could have distinct
8425 : single layouts. */
8426 13871 : int single_layout_i = -1;
8427 13871 : unsigned deferred_up_to = -1U;
8428 42698 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8429 : ++partition_i)
8430 : {
8431 35426 : auto &partition = m_partitions[partition_i];
8432 35426 : if (single_layout_i == -1)
8433 : {
8434 17567 : single_layout_i = partition.layout;
8435 17567 : deferred_up_to = partition_i;
8436 : }
8437 17859 : else if (partition.layout == single_layout_i || partition.layout == -1)
8438 : ;
8439 : else
8440 : single_layout_i = 0;
8441 32313 : if (single_layout_i == 0)
8442 : return true;
8443 :
8444 28912 : if (single_layout_i != -1
8445 28912 : && !is_compatible_layout (partition, single_layout_i))
8446 : return true;
8447 : }
8448 :
8449 7272 : if (single_layout_i <= 0)
8450 : return true;
8451 :
8452 7861 : for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
8453 606 : if (!is_compatible_layout (m_partitions[partition_i],
8454 : single_layout_i))
8455 : return true;
8456 :
8457 22321 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8458 : ++partition_i)
8459 : {
8460 15066 : auto &partition = m_partitions[partition_i];
8461 15066 : partition.layout = single_layout_i;
8462 : }
8463 :
8464 : return false;
8465 : }
8466 :
8467 : /* Main entry point for the SLP graph optimization pass. */
8468 :
8469 : void
8470 732178 : vect_optimize_slp_pass::run ()
8471 : {
8472 732178 : build_graph ();
8473 732178 : create_partitions ();
8474 732178 : start_choosing_layouts ();
8475 732178 : if (m_perms.length () > 1)
8476 : {
8477 13871 : if (legitimize ())
8478 : {
8479 6616 : forward_pass ();
8480 6616 : backward_pass ();
8481 : }
8482 13871 : if (dump_enabled_p ())
8483 691 : dump ();
8484 13871 : materialize ();
8485 56058 : while (!m_perms.is_empty ())
8486 28316 : m_perms.pop ().release ();
8487 : }
8488 : else
8489 718307 : remove_redundant_permutations ();
8490 732178 : free_graph (m_slpg);
8491 732178 : build_graph ();
8492 732178 : decide_masked_load_lanes ();
8493 732178 : free_graph (m_slpg);
8494 732178 : }
8495 :
8496 : /* Apply CSE to NODE and its children using BST_MAP. */
8497 :
8498 : static void
8499 5796717 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
8500 : {
8501 5796717 : bool put_p = false;
8502 5796717 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
8503 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8504 : lack scalar stmts and thus CSE doesn't work via bst_map. Ideally
8505 : we'd have sth that works for all internal and external nodes. */
8506 5796717 : && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
8507 : {
8508 4162453 : slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
8509 4162453 : if (leader)
8510 : {
8511 : /* We've visited this node already. */
8512 430304 : if (!*leader || *leader == node)
8513 : return;
8514 :
8515 4362 : if (dump_enabled_p ())
8516 919 : dump_printf_loc (MSG_NOTE, vect_location,
8517 : "re-using SLP tree %p for %p\n",
8518 : (void *)*leader, (void *)node);
8519 4362 : vect_free_slp_tree (node);
8520 4362 : (*leader)->refcnt += 1;
8521 4362 : node = *leader;
8522 4362 : return;
8523 : }
8524 :
8525 : /* Avoid creating a cycle by populating the map only after recursion. */
8526 3732149 : bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
8527 3732149 : node->refcnt += 1;
8528 3732149 : put_p = true;
8529 : /* And recurse. */
8530 : }
8531 :
8532 16065552 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8533 4707781 : if (child)
8534 4232333 : vect_cse_slp_nodes (bst_map, child);
8535 :
8536 : /* Now record the node for CSE in other siblings. */
8537 5366413 : if (put_p)
8538 3732149 : *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
8539 : }
8540 :
8541 : /* Associate stmts with possible starts of a subset of lanes of NODE
8542 : in PART_STARTS. */
8543 :
8544 : static void
8545 2149175 : vect_cse_gather_part_starts (hash_set<slp_tree> &visited,
8546 : vec<vec<slp_tree>> part_starts, slp_tree node)
8547 : {
8548 : /* CSEing external nodes complicates scheduling since we materialize
8549 : those at the latest position, so avoid that. */
8550 2149175 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8551 2149175 : || visited.add (node))
8552 : return;
8553 :
8554 : /* Besides some VEC_PERM_EXPR, two-operator nodes also lack scalar stmts
8555 : and thus CSE doesn't work. For now gather two-lane aligned starts
8556 : of nodes with a multiple of two number of lanes. */
8557 1156500 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8558 1149092 : && SLP_TREE_LANES (node) > 2
8559 152211 : && (SLP_TREE_LANES (node) & 1) == 0)
8560 : {
8561 152211 : auto_vec<unsigned, 8> uids;
8562 631173 : for (unsigned i = 0; i < SLP_TREE_LANES (node); i += 2)
8563 : {
8564 478962 : stmt_vec_info s = SLP_TREE_SCALAR_STMTS (node)[i];
8565 478962 : if (!s)
8566 0 : continue;
8567 478962 : unsigned uid = gimple_uid (s->stmt);
8568 478962 : if (!uids.contains (uid))
8569 : {
8570 474944 : uids.safe_push (uid);
8571 474944 : part_starts[uid].safe_push (node);
8572 : }
8573 : }
8574 152211 : }
8575 :
8576 4492662 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8577 1314694 : if (child)
8578 1314694 : vect_cse_gather_part_starts (visited, part_starts, child);
8579 : }
8580 :
8581 : /* Apply CSE to NODE and its children using lowparts of nodes in BST_MAP. */
8582 :
8583 : static void
8584 2145483 : vect_cse_slp_node_parts (hash_set<slp_tree> &visited,
8585 : const vec<vec<slp_tree>> part_starts,
8586 : vec<slp_tree> &drops, slp_tree node)
8587 : {
8588 2145483 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8589 2145483 : || visited.add (node))
8590 : return;
8591 :
8592 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8593 : lack scalar stmts and thus CSE doesn't work. */
8594 1154803 : unsigned HOST_WIDE_INT c;
8595 1154803 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8596 1147399 : && SLP_TREE_SCALAR_STMTS (node)[0]
8597 : /* Avoid touching loads which need care with load permutations
8598 : and specialities like load-lane representations. */
8599 1147399 : && (SLP_TREE_PERMUTE_P (node)
8600 1139762 : || !STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node))))
8601 37927 : for (slp_tree cand
8602 431452 : : part_starts[gimple_uid (SLP_TREE_SCALAR_STMTS (node)[0]->stmt)])
8603 : /* ??? There is a possible ordering/optimality problem in that
8604 : the CSE then can keep a wider feeding live even though it itself
8605 : becomes dead by means of CSE. Which might be solvable by doing
8606 : the CSE in a wide-to-narrow order. */
8607 38394 : if (SLP_TREE_LANES (cand) > SLP_TREE_LANES (node)
8608 : /* We can do high/lo extracts and full vector copies. */
8609 : && constant_multiple_p
8610 4619 : (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (cand)),
8611 38886 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)), &c)
8612 43013 : && c <= 2)
8613 : {
8614 4127 : unsigned HOST_WIDE_INT s;
8615 4127 : bool const_p
8616 4127 : = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)).is_constant (&s);
8617 4127 : unsigned i;
8618 14819 : for (i = 0; i <= SLP_TREE_LANES (cand) - SLP_TREE_LANES (node);)
8619 : {
8620 : unsigned j;
8621 15786 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
8622 15319 : if (!SLP_TREE_SCALAR_STMTS (node)[j]
8623 15319 : || (SLP_TREE_SCALAR_STMTS (cand)[i+j]
8624 15319 : != SLP_TREE_SCALAR_STMTS (node)[j]))
8625 : break;
8626 11159 : if (j == SLP_TREE_LANES (node))
8627 : break;
8628 10692 : if (!const_p)
8629 : {
8630 : i = SLP_TREE_LANES (cand);
8631 : break;
8632 : }
8633 : /* We can extract only aligned on node vector type boundary. */
8634 10692 : i += s;
8635 : }
8636 4127 : if (i > SLP_TREE_LANES (cand) - SLP_TREE_LANES (node))
8637 3660 : continue;
8638 : /* Found node within cand at i. Put a permute in place
8639 : of it, selecting the subset from cand. */
8640 467 : if (dump_enabled_p ())
8641 41 : dump_printf (MSG_NOTE, "CSEd node %p as %spart of node %p\n",
8642 : (void *)node, i == 0 ? "low" : "high", (void *)cand);
8643 2302 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8644 : /* Delay SLP tree release since we might still reference a node
8645 : from the part_starts map. */
8646 901 : drops.safe_push (child);
8647 467 : SLP_TREE_CHILDREN (node).truncate (1);
8648 467 : SLP_TREE_REF_COUNT (cand)++;
8649 467 : SLP_TREE_CHILDREN (node)[0] = cand;
8650 467 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
8651 467 : SLP_TREE_REPRESENTATIVE (node) = NULL;
8652 467 : SLP_TREE_LANE_PERMUTATION (node).create (SLP_TREE_LANES (node));
8653 2000 : for (unsigned j = i; j < i + SLP_TREE_LANES (node); ++j)
8654 1066 : SLP_TREE_LANE_PERMUTATION (node).quick_push (std::make_pair (0, j));
8655 2145483 : return;
8656 : }
8657 :
8658 4483052 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8659 1311002 : if (child)
8660 1311002 : vect_cse_slp_node_parts (visited, part_starts, drops, child);
8661 : }
8662 :
8663 : /* Optimize the SLP graph of VINFO. */
8664 :
8665 : void
8666 1127020 : vect_optimize_slp (vec_info *vinfo)
8667 : {
8668 1127020 : if (vinfo->slp_instances.is_empty ())
8669 844495 : return;
8670 732178 : vect_optimize_slp_pass (vinfo).run ();
8671 :
8672 : /* Apply CSE again to nodes after permute optimization. */
8673 732178 : scalar_stmts_to_slp_tree_map_t *bst_map
8674 732178 : = new scalar_stmts_to_slp_tree_map_t ();
8675 :
8676 3760918 : for (auto inst : vinfo->slp_instances)
8677 1564384 : vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
8678 :
8679 732178 : release_scalar_stmts_to_slp_tree_map (bst_map);
8680 :
8681 732178 : if (!is_a <bb_vec_info> (vinfo))
8682 : return;
8683 :
8684 : /* Attempt to merge SLP sub-graphs that intersect in low or highparts of
8685 : each other. Build the reverse mapping from stmt to SLP node for
8686 : lanes starting at the low or high part.
8687 : ??? In the future we can extend this to do a two-step permute
8688 : and extract or extract and permute to put the high/low part in
8689 : place on the original vector or permute the hogh/low part to
8690 : match up the target lane order. */
8691 282525 : hash_set<slp_tree> visited;
8692 282525 : vec<vec<slp_tree>> start_for_part;
8693 565050 : start_for_part.create (vinfo->stmt_vec_infos.length () + 1);
8694 565050 : start_for_part.quick_grow_cleared (vinfo->stmt_vec_infos.length () + 1);
8695 1682056 : for (auto inst : vinfo->slp_instances)
8696 834481 : vect_cse_gather_part_starts (visited,
8697 : start_for_part, SLP_INSTANCE_TREE (inst));
8698 :
8699 : /* Now replace low/highpart copies with extracting permutes. */
8700 282525 : auto_vec<slp_tree> drops;
8701 282525 : visited.empty ();
8702 1682056 : for (auto inst : vinfo->slp_instances)
8703 834481 : vect_cse_slp_node_parts (visited, start_for_part, drops,
8704 : SLP_INSTANCE_TREE (inst));
8705 :
8706 : /* Now perform delayed releases of nodes. */
8707 283920 : for (slp_tree node : drops)
8708 901 : vect_free_slp_tree (node);
8709 :
8710 24239811 : for (auto v : start_for_part)
8711 23392236 : v.release ();
8712 282525 : start_for_part.release ();
8713 282525 : }
8714 :
8715 : /* Gather loads reachable from the individual SLP graph entries. */
8716 :
8717 : void
8718 1127020 : vect_gather_slp_loads (vec_info *vinfo)
8719 : {
8720 1127020 : unsigned i;
8721 1127020 : slp_instance instance;
8722 2691404 : FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
8723 : {
8724 1564384 : hash_set<slp_tree> visited;
8725 1564384 : vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
8726 : SLP_INSTANCE_TREE (instance), visited);
8727 1564384 : }
8728 1127020 : }
8729 :
8730 : /* For NODE update VF based on the number of lanes and the vector types
8731 : used. */
8732 :
8733 : static void
8734 4659079 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
8735 : hash_set<slp_tree> &visited)
8736 : {
8737 4659079 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
8738 1689362 : return;
8739 3355859 : if (visited.add (node))
8740 : return;
8741 :
8742 11322038 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8743 3833179 : vect_update_slp_vf_for_node (child, vf, visited);
8744 :
8745 : /* We do not visit SLP nodes for constants or externals - those neither
8746 : have a vector type set yet (vectorizable_* does this).
8747 : Note that when we stop using fixed size vectors externs and constants
8748 : shouldn't influence the (minimum) vectorization factor, instead
8749 : vectorizable_* should honor the vectorization factor when trying to
8750 : assign vector types to constants and externals and cause iteration
8751 : to a higher vectorization factor when required. */
8752 2969747 : tree vectype = SLP_TREE_VECTYPE (node);
8753 2969747 : if (!vectype)
8754 : /* OMP SIMD calls w/o LHS have no SLP_TREE_VECTYPE set. */
8755 : return;
8756 2969717 : poly_uint64 node_vf
8757 2969717 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8758 : SLP_TREE_LANES (node));
8759 2969717 : vf = force_common_multiple (vf, node_vf);
8760 :
8761 : /* For permute nodes that are fed from externs or constants we have to
8762 : consider their number of lanes as well. Likewise for store-lanes. */
8763 2969717 : if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
8764 745645 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8765 204460 : if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
8766 : {
8767 3684 : poly_uint64 child_vf
8768 3684 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8769 : SLP_TREE_LANES (child));
8770 3684 : vf = force_common_multiple (vf, child_vf);
8771 : }
8772 : }
8773 :
8774 : /* For each possible SLP instance decide whether to SLP it and calculate overall
8775 : unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at
8776 : least one instance. */
8777 :
8778 : bool
8779 517993 : vect_make_slp_decision (loop_vec_info loop_vinfo)
8780 : {
8781 517993 : unsigned int i;
8782 517993 : poly_uint64 unrolling_factor = 1;
8783 517993 : const vec<slp_instance> &slp_instances
8784 : = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
8785 517993 : slp_instance instance;
8786 517993 : int decided_to_slp = 0;
8787 :
8788 517993 : DUMP_VECT_SCOPE ("vect_make_slp_decision");
8789 :
8790 517993 : hash_set<slp_tree> visited;
8791 1861886 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
8792 : {
8793 825900 : slp_tree root = SLP_INSTANCE_TREE (instance);
8794 :
8795 : /* All unroll factors have the form:
8796 :
8797 : GET_MODE_SIZE (vinfo->vector_mode) * X
8798 :
8799 : for some rational X, so they must have a common multiple. */
8800 825900 : vect_update_slp_vf_for_node (root, unrolling_factor, visited);
8801 :
8802 : /* If all instances ended up with vector(1) T roots make sure to
8803 : not vectorize. RVV for example relies on loop vectorization
8804 : when some instances are essentially kept scalar. See PR121048. */
8805 825900 : if (SLP_TREE_VECTYPE (root)
8806 825900 : && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
8807 669498 : decided_to_slp++;
8808 : }
8809 :
8810 517993 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
8811 :
8812 517993 : if (decided_to_slp && dump_enabled_p ())
8813 : {
8814 20930 : dump_printf_loc (MSG_NOTE, vect_location,
8815 : "Decided to SLP %d instances. Unrolling factor ",
8816 : decided_to_slp);
8817 20930 : dump_dec (MSG_NOTE, unrolling_factor);
8818 20930 : dump_printf (MSG_NOTE, "\n");
8819 : }
8820 :
8821 517993 : return (decided_to_slp > 0);
8822 517993 : }
8823 :
8824 : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks. */
8825 :
8826 2312925 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
8827 : : vec_info (vec_info::bb, shared),
8828 2312925 : roots (vNULL)
8829 : {
8830 : /* The region we are operating on. bbs[0] is the entry, excluding
8831 : its PHI nodes. In the future we might want to track an explicit
8832 : entry edge to cover bbs[0] PHI nodes and have a region entry
8833 : insert location. */
8834 2312925 : bbs = _bbs.address ();
8835 2312925 : nbbs = _bbs.length ();
8836 :
8837 18469388 : for (unsigned i = 0; i < nbbs; ++i)
8838 : {
8839 16156463 : if (i != 0)
8840 21066647 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8841 7223109 : gsi_next (&si))
8842 : {
8843 7223109 : gphi *phi = si.phi ();
8844 7223109 : gimple_set_uid (phi, 0);
8845 7223109 : add_stmt (phi);
8846 : }
8847 32312926 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8848 147734531 : !gsi_end_p (gsi); gsi_next (&gsi))
8849 : {
8850 131578068 : gimple *stmt = gsi_stmt (gsi);
8851 131578068 : gimple_set_uid (stmt, 0);
8852 131578068 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
8853 84458367 : continue;
8854 47119701 : add_stmt (stmt);
8855 : }
8856 : }
8857 2312925 : }
8858 :
8859 :
8860 : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
8861 : stmts in the basic block. */
8862 :
8863 2312925 : _bb_vec_info::~_bb_vec_info ()
8864 : {
8865 : /* Reset region marker. */
8866 18469388 : for (unsigned i = 0; i < nbbs; ++i)
8867 : {
8868 16156463 : if (i != 0)
8869 21080644 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8870 7237106 : gsi_next (&si))
8871 : {
8872 7237106 : gphi *phi = si.phi ();
8873 7237106 : gimple_set_uid (phi, -1);
8874 : }
8875 32312926 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8876 147608310 : !gsi_end_p (gsi); gsi_next (&gsi))
8877 : {
8878 131451847 : gimple *stmt = gsi_stmt (gsi);
8879 131451847 : gimple_set_uid (stmt, -1);
8880 : }
8881 : }
8882 :
8883 3694508 : for (unsigned i = 0; i < roots.length (); ++i)
8884 : {
8885 1381583 : roots[i].stmts.release ();
8886 1381583 : roots[i].roots.release ();
8887 1381583 : roots[i].remain.release ();
8888 : }
8889 2312925 : roots.release ();
8890 2312925 : }
8891 :
8892 : /* Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE,
8893 : given then that child nodes have already been processed, and that
8894 : their def types currently match their SLP node's def type. */
8895 :
8896 : static bool
8897 2757127 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
8898 : slp_instance node_instance,
8899 : stmt_vector_for_cost *cost_vec)
8900 : {
8901 : /* Handle purely internal nodes. */
8902 2757127 : if (SLP_TREE_PERMUTE_P (node))
8903 : {
8904 83167 : gcc_checking_assert (!SLP_TREE_REPRESENTATIVE (node));
8905 83167 : if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
8906 : return false;
8907 :
8908 : stmt_vec_info slp_stmt_info;
8909 : unsigned int i;
8910 205178 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
8911 : {
8912 123632 : if (slp_stmt_info
8913 117554 : && STMT_VINFO_LIVE_P (slp_stmt_info)
8914 123632 : && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
8915 : node_instance, i,
8916 : false, cost_vec))
8917 : return false;
8918 : }
8919 81546 : SLP_TREE_TYPE (node) = permute_info_type;
8920 81546 : return true;
8921 : }
8922 :
8923 2673960 : return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
8924 : }
8925 :
8926 : static int
8927 913493 : sort_ints (const void *a_, const void *b_)
8928 : {
8929 913493 : int a = *(const int *)a_;
8930 913493 : int b = *(const int *)b_;
8931 913493 : return a - b;
8932 : }
8933 :
8934 : /* Verify if we can externalize a set of internal defs. */
8935 :
8936 : static bool
8937 412541 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
8938 : {
8939 : /* Constant generation uses get_later_stmt which can only handle
8940 : defs from the same BB or a set of defs that can be ordered
8941 : with a dominance query. */
8942 412541 : basic_block bb = NULL;
8943 412541 : bool all_same = true;
8944 412541 : auto_vec<int> bbs;
8945 825082 : bbs.reserve_exact (stmts.length ());
8946 2175175 : for (stmt_vec_info stmt : stmts)
8947 : {
8948 937552 : if (!stmt)
8949 : return false;
8950 937552 : else if (!bb)
8951 412541 : bb = gimple_bb (stmt->stmt);
8952 525011 : else if (gimple_bb (stmt->stmt) != bb)
8953 157408 : all_same = false;
8954 937552 : bbs.quick_push (gimple_bb (stmt->stmt)->index);
8955 : }
8956 412541 : if (all_same)
8957 : return true;
8958 :
8959 : /* Produce a vector of unique BB indexes for the defs. */
8960 140513 : bbs.qsort (sort_ints);
8961 140513 : unsigned i, j;
8962 304164 : for (i = 1, j = 1; i < bbs.length (); ++i)
8963 163651 : if (bbs[i] != bbs[j-1])
8964 144424 : bbs[j++] = bbs[i];
8965 140513 : gcc_assert (j >= 2);
8966 140513 : bbs.truncate (j);
8967 :
8968 281026 : if (bbs.length () == 2)
8969 138815 : return (dominated_by_p (CDI_DOMINATORS,
8970 138815 : BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
8971 138815 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
8972 271161 : || dominated_by_p (CDI_DOMINATORS,
8973 132346 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
8974 132346 : BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
8975 :
8976 : /* ??? For more than two BBs we can sort the vector and verify the
8977 : result is a total order. But we can't use vec::qsort with a
8978 : compare function using a dominance query since there's no way to
8979 : signal failure and any fallback for an unordered pair would
8980 : fail qsort_chk later.
8981 : For now simply hope that ordering after BB index provides the
8982 : best candidate total order. If required we can implement our
8983 : own mergesort or export an entry without checking. */
8984 419790 : for (unsigned i = 1; i < bbs.length (); ++i)
8985 5585 : if (!dominated_by_p (CDI_DOMINATORS,
8986 5585 : BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
8987 5585 : BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
8988 : return false;
8989 :
8990 : return true;
8991 412541 : }
8992 :
8993 : /* Try to build NODE from scalars, returning true on success.
8994 : NODE_INSTANCE is the SLP instance that contains NODE. */
8995 :
8996 : static bool
8997 675829 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
8998 : slp_instance node_instance)
8999 : {
9000 675829 : stmt_vec_info stmt_info;
9001 675829 : unsigned int i;
9002 :
9003 675829 : if (!is_a <bb_vec_info> (vinfo)
9004 83418 : || node == SLP_INSTANCE_TREE (node_instance)
9005 25051 : || !SLP_TREE_SCALAR_STMTS (node).exists ()
9006 25010 : || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
9007 : /* Force the mask use to be built from scalars instead. */
9008 22953 : || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
9009 698533 : || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
9010 : return false;
9011 :
9012 22704 : if (dump_enabled_p ())
9013 100 : dump_printf_loc (MSG_NOTE, vect_location,
9014 : "Building vector operands of %p from scalars instead\n",
9015 : (void *) node);
9016 :
9017 : /* Don't remove and free the child nodes here, since they could be
9018 : referenced by other structures. The analysis and scheduling phases
9019 : (need to) ignore child nodes of anything that isn't vect_internal_def. */
9020 22704 : unsigned int group_size = SLP_TREE_LANES (node);
9021 22704 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
9022 : /* Invariants get their vector type from the uses. */
9023 22704 : SLP_TREE_VECTYPE (node) = NULL_TREE;
9024 22704 : SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
9025 22704 : SLP_TREE_LOAD_PERMUTATION (node).release ();
9026 103142 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9027 : {
9028 57734 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
9029 57734 : SLP_TREE_SCALAR_OPS (node)[i] = lhs;
9030 : }
9031 : return true;
9032 : }
9033 :
9034 : /* Return true if all elements of the slice are the same. */
9035 : bool
9036 451335 : vect_scalar_ops_slice::all_same_p () const
9037 : {
9038 503208 : for (unsigned int i = 1; i < length; ++i)
9039 415704 : if (!operand_equal_p (op (0), op (i)))
9040 : return false;
9041 : return true;
9042 : }
9043 :
9044 : hashval_t
9045 426283 : vect_scalar_ops_slice_hash::hash (const value_type &s)
9046 : {
9047 426283 : hashval_t hash = 0;
9048 1630612 : for (unsigned i = 0; i < s.length; ++i)
9049 1204329 : hash = iterative_hash_expr (s.op (i), hash);
9050 426283 : return hash;
9051 : }
9052 :
9053 : bool
9054 234126 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
9055 : const compare_type &s2)
9056 : {
9057 234126 : if (s1.length != s2.length)
9058 : return false;
9059 401295 : for (unsigned i = 0; i < s1.length; ++i)
9060 350751 : if (!operand_equal_p (s1.op (i), s2.op (i)))
9061 : return false;
9062 : return true;
9063 : }
9064 :
9065 : /* Like vect_get_num_copies but N copies of the vector might have
9066 : excess elements in the last vector. Stores the number of excess
9067 : elements in the last vector in *EXCESS_ELTS.
9068 : Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed. */
9069 :
9070 : bool
9071 1608656 : vect_get_num_copies_for_invariant (vec_info *vinfo, slp_tree node,
9072 : unsigned *nvectors,
9073 : unsigned *excess_elts)
9074 : {
9075 1608656 : poly_uint64 vf;
9076 :
9077 1608656 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
9078 385506 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9079 : else
9080 1223150 : vf = 1;
9081 1608656 : vf *= SLP_TREE_LANES (node);
9082 :
9083 1608656 : tree vectype = SLP_TREE_VECTYPE (node);
9084 1608656 : uint64_t rem;
9085 1608656 : if (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
9086 : nvectors)
9087 1608656 : && (TYPE_VECTOR_SUBPARTS (vectype) * *nvectors - vf) .is_constant (&rem))
9088 : {
9089 1608656 : *excess_elts = rem;
9090 1608656 : return true;
9091 : }
9092 : return false;
9093 : }
9094 :
9095 : /* Compute the prologue cost for invariant or constant operands represented
9096 : by NODE. */
9097 :
9098 : static void
9099 1104857 : vect_prologue_cost_for_slp (slp_tree node, unsigned nvectors,
9100 : stmt_vector_for_cost *cost_vec)
9101 : {
9102 : /* There's a special case of an existing vector, that costs nothing. */
9103 1104857 : if (SLP_TREE_SCALAR_OPS (node).length () == 0
9104 1104857 : && !SLP_TREE_VEC_DEFS (node).is_empty ())
9105 2314 : return;
9106 : /* Without looking at the actual initializer a vector of
9107 : constants can be implemented as load from the constant pool.
9108 : When all elements are the same we can use a splat. */
9109 1102543 : tree vectype = SLP_TREE_VECTYPE (node);
9110 1102543 : unsigned group_size = SLP_TREE_LANES (node);
9111 1102543 : unsigned HOST_WIDE_INT const_nunits;
9112 1102543 : unsigned nelt_limit;
9113 1102543 : auto ops = &SLP_TREE_SCALAR_OPS (node);
9114 1102543 : auto_vec<unsigned int> starts (nvectors);
9115 1102543 : if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
9116 1102543 : && ! multiple_p (const_nunits, group_size))
9117 : {
9118 67685 : nelt_limit = const_nunits;
9119 67685 : hash_set<vect_scalar_ops_slice_hash> vector_ops;
9120 347006 : for (unsigned int i = 0; i < nvectors; ++i)
9121 211636 : if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
9122 161092 : starts.quick_push (i * nelt_limit);
9123 67685 : }
9124 : else
9125 : {
9126 : /* If either the vector has variable length or the vectors
9127 : are composed of repeated whole groups we only need to
9128 : cost construction once. All vectors will be the same. */
9129 1034858 : nelt_limit = group_size;
9130 1034858 : starts.quick_push (0);
9131 : }
9132 : /* ??? We're just tracking whether vectors in a single node are the same.
9133 : Ideally we'd do something more global. */
9134 4503579 : for (unsigned int start : starts)
9135 : {
9136 1195950 : vect_cost_for_stmt kind;
9137 1195950 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
9138 : kind = vector_load;
9139 451335 : else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
9140 : kind = scalar_to_vec;
9141 : else
9142 363831 : kind = vec_construct;
9143 1195950 : record_stmt_cost (cost_vec, 1, kind, nullptr, node,
9144 : vectype, 0, vect_prologue);
9145 : }
9146 1102543 : }
9147 :
9148 : /* Analyze statements contained in SLP tree NODE after recursively analyzing
9149 : the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
9150 :
9151 : Return true if the operations are supported. */
9152 :
9153 : static bool
9154 5138347 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
9155 : slp_instance node_instance,
9156 : hash_set<slp_tree> &visited_set,
9157 : vec<slp_tree> &visited_vec,
9158 : stmt_vector_for_cost *cost_vec)
9159 : {
9160 5138347 : int i, j;
9161 5138347 : slp_tree child;
9162 :
9163 : /* Costing and analysis of invariants is delayed. */
9164 5138347 : if (!node
9165 4729037 : || SLP_TREE_DEF_TYPE (node) == vect_constant_def
9166 3925623 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
9167 : return true;
9168 :
9169 3438647 : if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
9170 : {
9171 5 : if (dump_enabled_p ())
9172 0 : dump_printf_loc (MSG_NOTE, vect_location,
9173 : "Failed cyclic SLP reference in %p\n", (void *) node);
9174 : return false;
9175 : }
9176 3438642 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
9177 :
9178 : /* If we already analyzed the exact same set of scalar stmts we're done.
9179 : We share the generated vector stmts for those. */
9180 3438642 : if (visited_set.add (node))
9181 : return true;
9182 3145298 : visited_vec.safe_push (node);
9183 :
9184 3145298 : bool res = true;
9185 3145298 : unsigned visited_rec_start = visited_vec.length ();
9186 3145298 : unsigned cost_vec_rec_start = cost_vec->length ();
9187 3145298 : bool seen_non_constant_child = false;
9188 9077024 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9189 : {
9190 3749287 : res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
9191 : visited_set, visited_vec,
9192 : cost_vec);
9193 3749287 : if (!res)
9194 : break;
9195 3361334 : if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
9196 3361334 : seen_non_constant_child = true;
9197 : }
9198 : /* We're having difficulties scheduling nodes with just constant
9199 : operands and no scalar stmts since we then cannot compute a stmt
9200 : insertion place. */
9201 3145298 : if (res
9202 3145298 : && !seen_non_constant_child
9203 3145298 : && SLP_TREE_SCALAR_STMTS (node).is_empty ())
9204 : {
9205 218 : if (dump_enabled_p ())
9206 6 : dump_printf_loc (MSG_NOTE, vect_location,
9207 : "Cannot vectorize all-constant op node %p\n",
9208 : (void *) node);
9209 : res = false;
9210 : }
9211 :
9212 3145086 : if (res)
9213 2757127 : res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
9214 : cost_vec);
9215 : /* If analysis failed we have to pop all recursive visited nodes
9216 : plus ourselves. */
9217 3145298 : if (!res)
9218 : {
9219 3297876 : while (visited_vec.length () >= visited_rec_start)
9220 973109 : visited_set.remove (visited_vec.pop ());
9221 675829 : cost_vec->truncate (cost_vec_rec_start);
9222 : }
9223 :
9224 : /* When the node can be vectorized cost invariant nodes it references.
9225 : This is not done in DFS order to allow the referring node
9226 : vectorizable_* calls to nail down the invariant nodes vector type
9227 : and possibly unshare it if it needs a different vector type than
9228 : other referrers. */
9229 3145298 : if (res)
9230 9488049 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
9231 2996880 : if (child
9232 2671924 : && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
9233 2671924 : || SLP_TREE_DEF_TYPE (child) == vect_external_def)
9234 : /* Perform usual caching, note code-generation still
9235 : code-gens these nodes multiple times but we expect
9236 : to CSE them later. */
9237 4147323 : && !visited_set.add (child))
9238 : {
9239 1147394 : visited_vec.safe_push (child);
9240 : /* ??? After auditing more code paths make a "default"
9241 : and push the vector type from NODE to all children
9242 : if it is not already set. */
9243 : /* Compute the number of vectors to be generated. */
9244 1147394 : tree vector_type = SLP_TREE_VECTYPE (child);
9245 1147394 : if (!vector_type)
9246 : {
9247 : /* Masked loads can have an undefined (default SSA definition)
9248 : else operand. We do not need to cost it. */
9249 42537 : vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
9250 43978 : if (SLP_TREE_TYPE (node) == load_vec_info_type
9251 43978 : && ((ops.length ()
9252 1441 : && TREE_CODE (ops[0]) == SSA_NAME
9253 0 : && SSA_NAME_IS_DEFAULT_DEF (ops[0])
9254 0 : && VAR_P (SSA_NAME_VAR (ops[0])))
9255 1441 : || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
9256 1441 : continue;
9257 :
9258 : /* For shifts with a scalar argument we don't need
9259 : to cost or code-generate anything.
9260 : ??? Represent this more explicitly. */
9261 41096 : gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
9262 : && j == 1);
9263 41096 : continue;
9264 41096 : }
9265 :
9266 : /* Make sure we can generate them and then cost them. */
9267 1104857 : unsigned nvectors, excess_elts;
9268 1104857 : if (!vect_get_num_copies_for_invariant (vinfo, child, &nvectors,
9269 : &excess_elts))
9270 0 : return false;
9271 1104857 : vect_prologue_cost_for_slp (child, nvectors, cost_vec);
9272 : }
9273 :
9274 : /* If this node or any of its children can't be vectorized, try pruning
9275 : the tree here rather than felling the whole thing. */
9276 675829 : if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
9277 : {
9278 : /* We'll need to revisit this for invariant costing and number
9279 : of vectorized stmt setting. */
9280 : res = true;
9281 : }
9282 :
9283 : return res;
9284 : }
9285 :
9286 : /* Mark lanes of NODE that are live outside of the basic-block vectorized
9287 : region and that can be vectorized using vectorizable_live_operation
9288 : with STMT_VINFO_LIVE_P. Not handled live operations will cause the
9289 : scalar code computing it to be retained. */
9290 :
9291 : static void
9292 1011752 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
9293 : slp_instance instance,
9294 : stmt_vector_for_cost *cost_vec,
9295 : hash_set<stmt_vec_info> &svisited,
9296 : hash_set<slp_tree> &visited)
9297 : {
9298 1011752 : if (visited.add (node))
9299 58901 : return;
9300 :
9301 : unsigned i;
9302 : stmt_vec_info stmt_info;
9303 : gimple *last_stmt = NULL;
9304 3416365 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9305 : {
9306 2463514 : if (!stmt_info || svisited.contains (stmt_info))
9307 96614 : continue;
9308 2427254 : stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
9309 2427254 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
9310 41182 : && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
9311 : /* Only the pattern root stmt computes the original scalar value. */
9312 30457 : continue;
9313 2396797 : if (!PURE_SLP_STMT (orig_stmt_info))
9314 : /* Iff the stmt is not part of the vector coverage because it or
9315 : uses of it are used by SLP graph leafs as extern input there is
9316 : no point in trying to live code-generate from a vector stmt as
9317 : the scalar stmt will survive anyway. */
9318 29897 : continue;
9319 2366900 : bool mark_visited = true;
9320 2366900 : gimple *orig_stmt = orig_stmt_info->stmt;
9321 2366900 : ssa_op_iter op_iter;
9322 2366900 : def_operand_p def_p;
9323 5323530 : FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
9324 : {
9325 : /* We have to verify whether we can insert the lane extract
9326 : before all uses. The following is a conservative approximation.
9327 : We cannot put this into vectorizable_live_operation because
9328 : iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
9329 : doesn't work.
9330 : Note that while the fact that we emit code for loads at the
9331 : first load should make this a non-problem leafs we construct
9332 : from scalars are vectorized after the last scalar def.
9333 : ??? If we'd actually compute the insert location during
9334 : analysis we could use sth less conservative than the last
9335 : scalar stmt in the node for the dominance check. */
9336 : /* ??? What remains is "live" uses in vector CTORs in the same
9337 : SLP graph which is where those uses can end up code-generated
9338 : right after their definition instead of close to their original
9339 : use. But that would restrict us to code-generate lane-extracts
9340 : from the latest stmt in a node. So we compensate for this
9341 : during code-generation, simply not replacing uses for those
9342 : hopefully rare cases. */
9343 589730 : imm_use_iterator use_iter;
9344 :
9345 589730 : bool live_p = false;
9346 589730 : bool can_insert = true;
9347 589730 : use_operand_p use_p;
9348 1761626 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, DEF_FROM_PTR (def_p))
9349 : {
9350 1180005 : gimple *use_stmt = USE_STMT (use_p);
9351 1180005 : stmt_vec_info use_stmt_info;
9352 1180005 : if (!(!is_gimple_debug (use_stmt)
9353 912285 : && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
9354 900800 : || !PURE_SLP_STMT (use_stmt_info))))
9355 961835 : continue;
9356 218170 : live_p = true;
9357 218170 : if (!last_stmt)
9358 67758 : last_stmt
9359 68024 : = (node->si ? node->si
9360 266 : : vect_find_last_scalar_stmt_in_slp (node)->stmt);
9361 218170 : if (is_a <gphi *> (use_stmt))
9362 : {
9363 80416 : if (!dominated_by_p (CDI_DOMINATORS,
9364 40208 : phi_arg_edge_from_use (use_p)->src,
9365 40208 : gimple_bb (last_stmt)))
9366 : can_insert = false;
9367 : }
9368 : /* As we instert after last_stmt it may not be the use_stmt
9369 : itself. */
9370 177962 : else if (last_stmt == use_stmt
9371 177962 : || !vect_stmt_dominates_stmt_p (last_stmt, use_stmt))
9372 : can_insert = false;
9373 : if (!can_insert)
9374 : {
9375 8109 : if (dump_enabled_p ())
9376 82 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9377 : "Cannot determine insertion place for "
9378 : "lane extract of %T at node %p\n",
9379 : DEF_FROM_PTR (def_p), (void *)node);
9380 : can_insert = false;
9381 : break;
9382 : }
9383 589730 : }
9384 589730 : if (live_p && can_insert)
9385 : {
9386 : /* Only record a live stmt when we can replace all uses. We
9387 : record from which SLP tree we vectorize the uses, so we'll
9388 : cost once and can deal with the case that not all SLP nodes
9389 : may be suitable for code-generation of all live uses.
9390 : ??? But we never split up the work between multiple SLP
9391 : nodes. */
9392 109070 : STMT_VINFO_LIVE_P (stmt_info) = true;
9393 109070 : if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
9394 : instance, i, false, cost_vec))
9395 : {
9396 0 : STMT_VINFO_LIVE_P (stmt_info) = false;
9397 0 : mark_visited = false;
9398 : }
9399 : }
9400 : }
9401 2366900 : if (mark_visited)
9402 2366900 : svisited.add (stmt_info);
9403 : }
9404 :
9405 : slp_tree child;
9406 2782251 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9407 995275 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9408 289257 : vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
9409 : svisited, visited);
9410 : }
9411 :
9412 : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
9413 : are live outside of the basic-block vectorized region and that can be
9414 : vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. */
9415 :
9416 : static void
9417 248546 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
9418 : {
9419 248546 : if (bb_vinfo->slp_instances.is_empty ())
9420 0 : return;
9421 :
9422 248546 : hash_set<slp_tree> visited;
9423 248546 : hash_set<stmt_vec_info> svisited;
9424 1468133 : for (slp_instance instance : bb_vinfo->slp_instances)
9425 : {
9426 722495 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9427 55191 : STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
9428 722495 : vect_location = instance->location ();
9429 722495 : vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
9430 : instance, &instance->cost_vec,
9431 : svisited, visited);
9432 : }
9433 248546 : }
9434 :
9435 : /* Determine whether we can vectorize the reduction epilogue for INSTANCE. */
9436 :
9437 : static bool
9438 93278 : vectorizable_bb_reduc_epilogue (slp_instance instance,
9439 : stmt_vector_for_cost *cost_vec)
9440 : {
9441 93278 : gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
9442 93278 : enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
9443 93278 : if (reduc_code == MINUS_EXPR)
9444 0 : reduc_code = PLUS_EXPR;
9445 93278 : internal_fn reduc_fn;
9446 93278 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
9447 93278 : if (!vectype
9448 93266 : || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
9449 93266 : TREE_TYPE (vectype))
9450 56166 : || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
9451 1006 : && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
9452 1006 : || reduc_fn == IFN_LAST
9453 1006 : || !direct_internal_fn_supported_p (reduc_fn, vectype,
9454 : OPTIMIZE_FOR_BOTH)))
9455 : /* Two-element reductions do not need special-handling for fold-left,
9456 : other cases are not yet implemented. remain_defs also have to
9457 : be included here. */
9458 186592 : || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
9459 6020 : && (!instance->remain_defs.is_empty ()
9460 1886 : || SLP_TREE_LANES (SLP_INSTANCE_TREE (instance)) != 2)))
9461 : {
9462 39229 : if (dump_enabled_p ())
9463 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9464 : "not vectorized: basic block reduction epilogue "
9465 : "operation unsupported.\n");
9466 : return false;
9467 : }
9468 :
9469 : /* There's no way to cost a horizontal vector reduction via REDUC_FN so
9470 : cost log2 vector operations plus shuffles and one extraction. */
9471 54049 : unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
9472 54049 : record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
9473 : vectype, 0, vect_body);
9474 54049 : record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
9475 : vectype, 0, vect_body);
9476 54049 : record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
9477 : vectype, 0, vect_body);
9478 :
9479 : /* Since we replace all stmts of a possibly longer scalar reduction
9480 : chain account for the extra scalar stmts for that. */
9481 54049 : if (!instance->remain_defs.is_empty ())
9482 36818 : record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
9483 18409 : instance->root_stmts[0], 0, vect_body);
9484 : return true;
9485 : }
9486 :
9487 : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
9488 : and recurse to children. */
9489 :
9490 : static void
9491 387989 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
9492 : hash_set<slp_tree> &visited)
9493 : {
9494 387989 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
9495 387989 : || visited.add (node))
9496 : return;
9497 :
9498 1116833 : for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
9499 499694 : if (stmt)
9500 536530 : roots.remove (vect_orig_stmt (stmt));
9501 :
9502 824329 : for (auto child : SLP_TREE_CHILDREN (node))
9503 274000 : if (child)
9504 272394 : vect_slp_prune_covered_roots (child, roots, visited);
9505 : }
9506 :
9507 : /* Hand over COST_VEC to the target COSTS grouped by SLP node. */
9508 :
9509 : static void
9510 1041555 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
9511 : {
9512 3876960 : for (unsigned start = 0; start < cost_vec.length ();)
9513 : {
9514 2835405 : unsigned end = start + 1;
9515 3566457 : while (end < cost_vec.length ()
9516 6101101 : && cost_vec[start].node == cost_vec[end].node)
9517 731052 : end++;
9518 2835405 : if (cost_vec[start].node)
9519 2790380 : costs->add_slp_cost (cost_vec[start].node,
9520 2790380 : array_slice<stmt_info_for_cost>
9521 2790380 : (cost_vec.begin () + start, end - start));
9522 : else
9523 45025 : costs->vector_costs::add_slp_cost (cost_vec[start].node,
9524 45025 : array_slice<stmt_info_for_cost>
9525 45025 : (cost_vec.begin () + start, end - start));
9526 2835405 : start = end;
9527 : }
9528 1041555 : }
9529 :
9530 : /* Analyze statements in SLP instances of VINFO. Return true if the
9531 : operations are supported. */
9532 :
9533 : bool
9534 712375 : vect_slp_analyze_operations (vec_info *vinfo)
9535 : {
9536 712375 : slp_instance instance;
9537 712375 : int i;
9538 :
9539 712375 : DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
9540 :
9541 712375 : hash_set<slp_tree> visited;
9542 2542558 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9543 : {
9544 1389060 : auto_vec<slp_tree> visited_vec;
9545 1389060 : stmt_vector_for_cost cost_vec;
9546 1389060 : cost_vec.create (2);
9547 1389060 : if (is_a <bb_vec_info> (vinfo))
9548 825682 : vect_location = instance->location ();
9549 1389060 : if (!vect_slp_analyze_node_operations (vinfo,
9550 : SLP_INSTANCE_TREE (instance),
9551 : instance, visited, visited_vec,
9552 : &cost_vec)
9553 : /* CTOR instances require vectorized defs for the SLP tree root. */
9554 1123883 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
9555 6725 : && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
9556 : != vect_internal_def
9557 : /* Make sure we vectorized with the expected type. */
9558 6725 : || !useless_type_conversion_p
9559 6725 : (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
9560 : (instance->root_stmts[0]->stmt))),
9561 6725 : TREE_TYPE (SLP_TREE_VECTYPE
9562 : (SLP_INSTANCE_TREE (instance))))))
9563 : /* Check we can vectorize the reduction. */
9564 1123868 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
9565 93278 : && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
9566 : /* Check we can vectorize the gcond. */
9567 2473699 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
9568 67203 : && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
9569 67203 : SLP_INSTANCE_ROOT_STMTS (instance)[0],
9570 : NULL,
9571 : SLP_INSTANCE_TREE (instance),
9572 : &cost_vec)))
9573 : {
9574 368863 : cost_vec.release ();
9575 368863 : slp_tree node = SLP_INSTANCE_TREE (instance);
9576 368863 : stmt_vec_info stmt_info;
9577 368863 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9578 271889 : stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9579 96974 : else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
9580 96974 : && SLP_TREE_SCALAR_STMTS (node)[0])
9581 : stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
9582 : else
9583 3 : stmt_info = SLP_TREE_REPRESENTATIVE (node);
9584 368863 : if (is_a <loop_vec_info> (vinfo))
9585 : {
9586 271252 : if (dump_enabled_p ())
9587 7654 : dump_printf_loc (MSG_NOTE, vect_location,
9588 : "unsupported SLP instance starting from: %G",
9589 : stmt_info->stmt);
9590 271252 : return false;
9591 : }
9592 97611 : if (dump_enabled_p ())
9593 114 : dump_printf_loc (MSG_NOTE, vect_location,
9594 : "removing SLP instance operations starting from: %G",
9595 : stmt_info->stmt);
9596 234903 : while (!visited_vec.is_empty ())
9597 : {
9598 137292 : slp_tree node = visited_vec.pop ();
9599 137292 : SLP_TREE_TYPE (node) = undef_vec_info_type;
9600 137292 : if (node->data)
9601 : {
9602 8525 : delete node->data;
9603 8525 : node->data = nullptr;
9604 : }
9605 137292 : visited.remove (node);
9606 : }
9607 97611 : vect_free_slp_instance (instance);
9608 97611 : vinfo->slp_instances.ordered_remove (i);
9609 : }
9610 : else
9611 : {
9612 1020197 : i++;
9613 1020197 : if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
9614 : {
9615 292126 : add_slp_costs (loop_vinfo->vector_costs, cost_vec);
9616 292126 : cost_vec.release ();
9617 : }
9618 : else
9619 : /* For BB vectorization remember the SLP graph entry
9620 : cost for later. */
9621 728071 : instance->cost_vec = cost_vec;
9622 : }
9623 1389060 : }
9624 :
9625 : /* Now look for SLP instances with a root that are covered by other
9626 : instances and remove them. */
9627 441123 : hash_set<stmt_vec_info> roots;
9628 1830836 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9629 1012098 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9630 63508 : roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
9631 441123 : if (!roots.is_empty ())
9632 : {
9633 26712 : visited.empty ();
9634 142307 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9635 115595 : vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
9636 : visited);
9637 142307 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9638 115595 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
9639 63508 : && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
9640 : {
9641 5538 : stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9642 5538 : if (dump_enabled_p ())
9643 287 : dump_printf_loc (MSG_NOTE, vect_location,
9644 : "removing SLP instance operations starting "
9645 : "from: %G", root->stmt);
9646 5538 : vect_free_slp_instance (instance);
9647 5538 : vinfo->slp_instances.ordered_remove (i);
9648 : }
9649 : else
9650 110057 : ++i;
9651 : }
9652 :
9653 882246 : return !vinfo->slp_instances.is_empty ();
9654 1153498 : }
9655 :
9656 : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
9657 : closing the eventual chain. */
9658 :
9659 : static slp_instance
9660 814880 : get_ultimate_leader (slp_instance instance,
9661 : hash_map<slp_instance, slp_instance> &instance_leader)
9662 : {
9663 814880 : auto_vec<slp_instance *, 8> chain;
9664 814880 : slp_instance *tem;
9665 927661 : while (*(tem = instance_leader.get (instance)) != instance)
9666 : {
9667 112781 : chain.safe_push (tem);
9668 112781 : instance = *tem;
9669 : }
9670 927661 : while (!chain.is_empty ())
9671 112781 : *chain.pop () = instance;
9672 814880 : return instance;
9673 814880 : }
9674 :
9675 : namespace {
9676 : /* Subroutine of vect_bb_partition_graph_r. Map KEY to INSTANCE in
9677 : KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
9678 : for KEY. Return true if KEY was already in KEY_TO_INSTANCE.
9679 :
9680 : INSTANCE_LEADER is as for get_ultimate_leader. */
9681 :
9682 : template<typename T>
9683 : bool
9684 3614304 : vect_map_to_instance (slp_instance instance, T key,
9685 : hash_map<T, slp_instance> &key_to_instance,
9686 : hash_map<slp_instance, slp_instance> &instance_leader)
9687 : {
9688 : bool existed_p;
9689 3614304 : slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
9690 3614304 : if (!existed_p)
9691 : ;
9692 247407 : else if (key_instance != instance)
9693 : {
9694 : /* If we're running into a previously marked key make us the
9695 : leader of the current ultimate leader. This keeps the
9696 : leader chain acyclic and works even when the current instance
9697 : connects two previously independent graph parts. */
9698 92385 : slp_instance key_leader
9699 92385 : = get_ultimate_leader (key_instance, instance_leader);
9700 92385 : if (key_leader != instance)
9701 27619 : instance_leader.put (key_leader, instance);
9702 : }
9703 3614304 : key_instance = instance;
9704 3614304 : return existed_p;
9705 : }
9706 : }
9707 :
9708 : /* Worker of vect_bb_partition_graph, recurse on NODE. */
9709 :
9710 : static void
9711 1011752 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
9712 : slp_instance instance, slp_tree node,
9713 : hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
9714 : hash_map<slp_tree, slp_instance> &node_to_instance,
9715 : hash_map<slp_instance, slp_instance> &instance_leader)
9716 : {
9717 5626676 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
9718 2602552 : if (stmt_info)
9719 2602552 : vect_map_to_instance (instance, stmt_info, stmt_to_instance,
9720 : instance_leader);
9721 :
9722 1011752 : if (vect_map_to_instance (instance, node, node_to_instance,
9723 : instance_leader))
9724 : return;
9725 :
9726 3616376 : for (auto child : SLP_TREE_CHILDREN (node))
9727 995275 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9728 289257 : vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
9729 : node_to_instance, instance_leader);
9730 : }
9731 :
9732 : /* Partition the SLP graph into pieces that can be costed independently. */
9733 :
9734 : static void
9735 248546 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
9736 : {
9737 248546 : DUMP_VECT_SCOPE ("vect_bb_partition_graph");
9738 :
9739 : /* First walk the SLP graph assigning each involved scalar stmt a
9740 : corresponding SLP graph entry and upon visiting a previously
9741 : marked stmt, make the stmts leader the current SLP graph entry. */
9742 248546 : hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
9743 248546 : hash_map<slp_tree, slp_instance> node_to_instance;
9744 248546 : hash_map<slp_instance, slp_instance> instance_leader;
9745 248546 : slp_instance instance;
9746 1219587 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9747 : {
9748 722495 : instance_leader.put (instance, instance);
9749 722495 : vect_bb_partition_graph_r (bb_vinfo,
9750 : instance, SLP_INSTANCE_TREE (instance),
9751 : stmt_to_instance, node_to_instance,
9752 : instance_leader);
9753 : }
9754 :
9755 : /* Then collect entries to each independent subgraph. */
9756 1219587 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9757 : {
9758 722495 : slp_instance leader = get_ultimate_leader (instance, instance_leader);
9759 722495 : leader->subgraph_entries.safe_push (instance);
9760 722495 : if (dump_enabled_p ()
9761 722495 : && leader != instance)
9762 106 : dump_printf_loc (MSG_NOTE, vect_location,
9763 : "instance %p is leader of %p\n",
9764 : (void *) leader, (void *) instance);
9765 : }
9766 248546 : }
9767 :
9768 : /* Compute the scalar cost of the SLP node NODE and its children
9769 : and return it. Do not account defs that are marked in LIFE and
9770 : update LIFE according to uses of NODE. */
9771 :
9772 : static void
9773 718201 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
9774 : vec<stmt_vec_info> &worklist,
9775 : stmt_vector_for_cost *cost_vec,
9776 : hash_set<stmt_vec_info> &visited)
9777 : {
9778 3359350 : while (!worklist.is_empty ())
9779 : {
9780 2641149 : stmt_vec_info stmt = worklist.pop ();
9781 2967914 : if (!PURE_SLP_STMT (stmt))
9782 335405 : continue;
9783 :
9784 : /* When the stmt is live but not actually vectorized we have
9785 : to keep the feeding scalar defs. */
9786 2324732 : if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
9787 : {
9788 2207723 : bool live_p = false;
9789 2207723 : ssa_op_iter op_iter;
9790 2207723 : def_operand_p def_p;
9791 4869441 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
9792 : {
9793 453995 : imm_use_iterator use_iter;
9794 453995 : gimple *use_stmt;
9795 1157711 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9796 703716 : if (!is_gimple_debug (use_stmt))
9797 : {
9798 532747 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
9799 532747 : if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
9800 : {
9801 15859 : if (dump_enabled_p ())
9802 : {
9803 92 : dump_printf_loc (MSG_NOTE, vect_location,
9804 : "stmt considered live: %G",
9805 : stmt->stmt);
9806 92 : dump_printf_loc (MSG_NOTE, vect_location,
9807 : "because of use in: %G",
9808 : use_stmt);
9809 : }
9810 : live_p = true;
9811 : }
9812 453995 : }
9813 : }
9814 2207723 : if (live_p)
9815 8640 : continue;
9816 : }
9817 :
9818 : /* The following assert verifies that vect_bb_partition_graph
9819 : partitions the SLP graph in a way that each scalar stmt of
9820 : the coverage of the SLP graph belongs to exactly one subgraph.
9821 : ??? This is currently not guaranteed since the function
9822 : works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
9823 : tripping or scalar stmts costed multiple times, making vectorization
9824 : more profitable than it really is. */
9825 : /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
9826 :
9827 2305744 : if (vect_nop_conversion_p (stmt))
9828 : ;
9829 : /* For single-argument PHIs assume coalescing which means zero
9830 : cost for the scalar and the vector PHIs. This avoids
9831 : artificially favoring the vector path (but may pessimize it
9832 : in some cases). */
9833 2274260 : else if (is_a <gphi *> (stmt->stmt)
9834 2274260 : && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
9835 : ;
9836 : else
9837 : {
9838 2263762 : vect_cost_for_stmt kind;
9839 2263762 : if (STMT_VINFO_DATA_REF (stmt))
9840 : {
9841 2026686 : data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
9842 2026686 : tree base = get_base_address (DR_REF (dr));
9843 : /* When the scalar access is to a non-global not
9844 : address-taken decl that is not BLKmode assume we can
9845 : access it with a single non-load/store instruction. */
9846 2026686 : if (DECL_P (base)
9847 1550348 : && !is_global_var (base)
9848 1471897 : && !TREE_ADDRESSABLE (base)
9849 2586645 : && DECL_MODE (base) != BLKmode)
9850 : kind = scalar_stmt;
9851 1881680 : else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
9852 : kind = scalar_load;
9853 : else
9854 1625706 : kind = scalar_store;
9855 : }
9856 : else
9857 : kind = scalar_stmt;
9858 : /* Cost each scalar stmt only once. */
9859 2263762 : gimple_set_visited (stmt->stmt, true);
9860 2263762 : record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
9861 : }
9862 :
9863 : /* Now walk relevant parts of the SSA use-def graph. */
9864 2305744 : slp_oprnds child_ops (stmt);
9865 7185637 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
9866 : {
9867 2574149 : tree op = child_ops.get_op_for_slp_child (stmt, i);
9868 2574149 : stmt_vec_info def = vinfo->lookup_def (op);
9869 2574149 : if (def && !visited.add (def))
9870 779781 : worklist.safe_push (def);
9871 : }
9872 : }
9873 718201 : }
9874 :
9875 :
9876 : /* Comparator for the loop-index sorted cost vectors. */
9877 :
9878 : static int
9879 20718798 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
9880 : {
9881 20718798 : auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
9882 20718798 : auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
9883 20718798 : if (a->first < b->first)
9884 : return -1;
9885 19246867 : else if (a->first == b->first)
9886 18072385 : return 0;
9887 : return 1;
9888 : }
9889 :
9890 : /* Check if vectorization of the basic block is profitable for the
9891 : subgraph denoted by SLP_INSTANCES. */
9892 :
9893 : static bool
9894 690790 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
9895 : vec<slp_instance> slp_instances,
9896 : loop_p orig_loop)
9897 : {
9898 690790 : slp_instance instance;
9899 690790 : int i;
9900 690790 : unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
9901 690790 : unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
9902 :
9903 690790 : if (dump_enabled_p ())
9904 : {
9905 125 : dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph:\n");
9906 260 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9907 135 : dump_printf_loc (MSG_NOTE, vect_location, " entry instance %p -> "
9908 : "node %p\n", (void *)instance,
9909 135 : (void *)SLP_INSTANCE_TREE (instance));
9910 125 : hash_set<slp_tree> visited;
9911 510 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9912 135 : vect_print_slp_graph (MSG_NOTE, vect_location,
9913 : SLP_INSTANCE_TREE (instance), visited);
9914 125 : }
9915 :
9916 : /* Then DFS walk scalar stmts, performing costing and handling
9917 : still live scalar stmts via the previously computed vector coverage. */
9918 690790 : stmt_vector_for_cost scalar_costs = vNULL;
9919 690790 : stmt_vector_for_cost vector_costs = vNULL;
9920 690790 : hash_set<slp_tree> visited;
9921 690790 : hash_set<stmt_vec_info> svisited;
9922 2099781 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9923 : {
9924 718201 : auto_vec<stmt_vec_info> worklist;
9925 718201 : if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
9926 108700 : record_stmt_cost (&scalar_costs,
9927 54350 : SLP_INSTANCE_ROOT_STMTS (instance).length (),
9928 : scalar_stmt,
9929 54350 : SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
9930 4031489 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
9931 : {
9932 1876886 : stmt = vect_orig_stmt (stmt);
9933 1876886 : if (!svisited.add (stmt))
9934 1861368 : worklist.safe_push (stmt);
9935 : }
9936 718201 : vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
9937 718201 : vector_costs.safe_splice (instance->cost_vec);
9938 718201 : instance->cost_vec.release ();
9939 718201 : }
9940 :
9941 690790 : if (dump_enabled_p ())
9942 125 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
9943 :
9944 : /* When costing non-loop vectorization we need to consider each covered
9945 : loop independently and make sure vectorization is profitable. For
9946 : now we assume a loop may be not entered or executed an arbitrary
9947 : number of iterations (??? static information can provide more
9948 : precise info here) which means we can simply cost each containing
9949 : loops stmts separately. */
9950 :
9951 : /* First produce cost vectors sorted by loop index. */
9952 690790 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9953 1381580 : li_scalar_costs (scalar_costs.length ());
9954 690790 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9955 1381567 : li_vector_costs (vector_costs.length ());
9956 690790 : stmt_info_for_cost *cost;
9957 3699692 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9958 : {
9959 2318112 : unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9960 2318112 : li_scalar_costs.quick_push (std::make_pair (l, cost));
9961 : }
9962 : /* Use a random used loop as fallback in case the first vector_costs
9963 : entry does not have a location associated with it. */
9964 690790 : unsigned l = li_scalar_costs[0].first;
9965 2675746 : FOR_EACH_VEC_ELT (vector_costs, i, cost)
9966 : {
9967 : /* Use SLP node placement according to the computed schedule. */
9968 1984956 : if (cost->node && cost->node->si)
9969 1237331 : l = gimple_bb (cost->node->si)->loop_father->num;
9970 : /* For schedules at region boundary use the region entry loop. */
9971 747625 : else if (cost->node)
9972 588234 : l = bb_vinfo->bbs[0]->loop_father->num;
9973 : /* SLP instance root stmts do not have an associated SLP node. */
9974 159391 : else if (cost->stmt_info)
9975 159391 : l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9976 : /* And since vect_prologue_cost_for_slp can end up costing with
9977 : neither, inherit from the previous node. */
9978 1984956 : li_vector_costs.quick_push (std::make_pair (l, cost));
9979 : }
9980 690790 : li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
9981 690790 : li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
9982 :
9983 690790 : unsigned total_vec_outside_cost = 0;
9984 690790 : unsigned total_vec_inside_cost = 0;
9985 690790 : unsigned total_scalar_cost = 0;
9986 :
9987 : /* Now cost the portions individually. */
9988 690790 : unsigned vi = 0;
9989 690790 : unsigned si = 0;
9990 690790 : bool profitable = true;
9991 1382908 : while (si < li_scalar_costs.length ()
9992 2134181 : || vi < li_vector_costs.length ())
9993 : {
9994 751286 : unsigned sl
9995 1501244 : = si < li_scalar_costs.length () ? li_scalar_costs[si].first : -1U;
9996 751286 : unsigned vl
9997 1502488 : = vi < li_vector_costs.length () ? li_vector_costs[vi].first : -1U;
9998 :
9999 751286 : class vector_costs *scalar_target_cost_data = nullptr;
10000 751286 : scalar_cost = 0;
10001 751286 : if (sl <= vl)
10002 : {
10003 706183 : if (dump_enabled_p ())
10004 147 : dump_printf_loc (MSG_NOTE, vect_location,
10005 : "Scalar cost for part in loop %d\n", sl);
10006 706183 : scalar_target_cost_data = init_cost (bb_vinfo, true);
10007 2318112 : do
10008 : {
10009 2318112 : add_stmt_cost (scalar_target_cost_data,
10010 2318112 : li_scalar_costs[si].second);
10011 2318112 : si++;
10012 : }
10013 2318112 : while (si < li_scalar_costs.length ()
10014 4651617 : && li_scalar_costs[si].first == sl);
10015 706183 : scalar_target_cost_data->finish_cost (nullptr);
10016 706183 : scalar_cost = scalar_target_cost_data->body_cost ();
10017 706183 : total_scalar_cost += scalar_cost;
10018 706183 : if (sl < vl)
10019 : {
10020 1857 : if (dump_enabled_p ())
10021 2 : dump_printf_loc (MSG_NOTE, vect_location,
10022 : "Scalar %d loop part does not "
10023 : "have corresponding vector part\n", sl);
10024 1857 : delete scalar_target_cost_data;
10025 650714 : continue;
10026 : }
10027 : }
10028 :
10029 : /* Complete the target-specific vector cost calculation. */
10030 749429 : if (dump_enabled_p ())
10031 149 : dump_printf_loc (MSG_NOTE, vect_location,
10032 : "Vector cost for part in loop %d\n", vl);
10033 749429 : class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
10034 749429 : auto_vec<stmt_info_for_cost> tem;
10035 1984956 : do
10036 : {
10037 1984956 : tem.safe_push (*li_vector_costs[vi].second);
10038 1984956 : vi++;
10039 : }
10040 1984956 : while (vi < li_vector_costs.length ()
10041 4028564 : && li_vector_costs[vi].first == vl);
10042 749429 : add_slp_costs (vect_target_cost_data, tem);
10043 749429 : vect_target_cost_data->finish_cost (scalar_target_cost_data);
10044 749429 : vec_prologue_cost = vect_target_cost_data->prologue_cost ();
10045 749429 : vec_inside_cost = vect_target_cost_data->body_cost ();
10046 749429 : vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
10047 749429 : if (scalar_target_cost_data)
10048 704326 : delete scalar_target_cost_data;
10049 749429 : delete vect_target_cost_data;
10050 :
10051 749429 : vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
10052 :
10053 749429 : total_vec_outside_cost += vec_outside_cost;
10054 749429 : total_vec_inside_cost += vec_inside_cost;
10055 :
10056 749429 : if (sl > vl && dump_enabled_p ())
10057 4 : dump_printf_loc (MSG_NOTE, vect_location,
10058 : "Vector %d loop part does not "
10059 : "have corresponding scalar part\n", vl);
10060 :
10061 : /* When this is vector costs for the region entry delay costing
10062 : and instead only require the total costs to be profitable. */
10063 749429 : if (vl == (unsigned) bb_vinfo->bbs[0]->loop_father->num)
10064 648857 : continue;
10065 :
10066 100572 : if (dump_enabled_p ())
10067 : {
10068 52 : dump_printf_loc (MSG_NOTE, vect_location,
10069 : "Cost model analysis for part in loop %d:\n",
10070 52 : std::min (sl, vl));
10071 52 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
10072 : vec_inside_cost + vec_outside_cost);
10073 52 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", scalar_cost);
10074 : }
10075 :
10076 : /* Vectorization is profitable if its cost is more than the cost of scalar
10077 : version. Note that we err on the vector side for equal cost because
10078 : the cost estimate is otherwise quite pessimistic (constant uses are
10079 : free on the scalar side but cost a load on the vector side for
10080 : example). */
10081 100572 : if (vec_outside_cost + vec_inside_cost > scalar_cost)
10082 47225 : profitable = false;
10083 749429 : }
10084 :
10085 690790 : if (dump_enabled_p ())
10086 : {
10087 125 : dump_printf_loc (MSG_NOTE, vect_location,
10088 : "Cost model analysis for whole subgraph:\n");
10089 125 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
10090 : total_vec_inside_cost + total_vec_outside_cost);
10091 125 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", total_scalar_cost);
10092 : }
10093 :
10094 : /* For the case where the outermost region had no scalar cost require
10095 : overall profitability. */
10096 690790 : if (total_vec_outside_cost + total_vec_inside_cost > total_scalar_cost)
10097 212975 : profitable = false;
10098 :
10099 : /* Unset visited flag. This is delayed when the subgraph is profitable
10100 : and we process the loop for remaining unvectorized if-converted code. */
10101 690790 : if (!orig_loop || !profitable)
10102 3006850 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
10103 2316185 : gimple_set_visited (cost->stmt_info->stmt, false);
10104 :
10105 690790 : scalar_costs.release ();
10106 690790 : vector_costs.release ();
10107 :
10108 690790 : return profitable;
10109 690790 : }
10110 :
10111 : /* qsort comparator for lane defs. */
10112 :
10113 : static int
10114 128 : vld_cmp (const void *a_, const void *b_)
10115 : {
10116 128 : auto *a = (const std::pair<unsigned, tree> *)a_;
10117 128 : auto *b = (const std::pair<unsigned, tree> *)b_;
10118 128 : return a->first - b->first;
10119 : }
10120 :
10121 : /* Return true if USE_STMT is a vector lane insert into VEC and set
10122 : *THIS_LANE to the lane number that is set. */
10123 :
10124 : static bool
10125 295 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
10126 : {
10127 295 : gassign *use_ass = dyn_cast <gassign *> (use_stmt);
10128 110 : if (!use_ass
10129 110 : || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
10130 35 : || (vec
10131 35 : ? gimple_assign_rhs1 (use_ass) != vec
10132 26 : : ((vec = gimple_assign_rhs1 (use_ass)), false))
10133 61 : || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
10134 61 : TREE_TYPE (gimple_assign_rhs2 (use_ass)))
10135 61 : || !constant_multiple_p
10136 61 : (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
10137 356 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
10138 : this_lane))
10139 : return false;
10140 : return true;
10141 : }
10142 :
10143 : /* Find any vectorizable constructors and add them to the grouped_store
10144 : array. */
10145 :
10146 : static void
10147 2312925 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
10148 : {
10149 18469388 : for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
10150 32312926 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
10151 147734531 : !gsi_end_p (gsi); gsi_next (&gsi))
10152 : {
10153 131578068 : gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
10154 : /* This can be used to start SLP discovery for early breaks for BB early breaks
10155 : when we get that far. */
10156 131578068 : if (!assign)
10157 100348687 : continue;
10158 :
10159 33360505 : tree rhs = gimple_assign_rhs1 (assign);
10160 33360505 : enum tree_code code = gimple_assign_rhs_code (assign);
10161 33360505 : use_operand_p use_p;
10162 33360505 : gimple *use_stmt;
10163 33360505 : if (code == CONSTRUCTOR)
10164 : {
10165 1743699 : if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10166 1791284 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
10167 104692 : CONSTRUCTOR_NELTS (rhs))
10168 47585 : || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
10169 1791264 : || uniform_vector_p (rhs))
10170 1728137 : continue;
10171 :
10172 : unsigned j;
10173 : tree val;
10174 77229 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10175 61667 : if (TREE_CODE (val) != SSA_NAME
10176 61667 : || !bb_vinfo->lookup_def (val))
10177 : break;
10178 38248 : if (j != CONSTRUCTOR_NELTS (rhs))
10179 3562 : continue;
10180 :
10181 15562 : vec<stmt_vec_info> roots = vNULL;
10182 15562 : roots.safe_push (bb_vinfo->lookup_stmt (assign));
10183 15562 : vec<stmt_vec_info> stmts;
10184 15562 : stmts.create (CONSTRUCTOR_NELTS (rhs));
10185 87190 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10186 56066 : stmts.quick_push
10187 56066 : (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
10188 15562 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10189 15562 : stmts, roots));
10190 : }
10191 31616806 : else if (code == BIT_INSERT_EXPR
10192 1022 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
10193 721 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
10194 721 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
10195 718 : && integer_zerop (gimple_assign_rhs3 (assign))
10196 392 : && useless_type_conversion_p
10197 392 : (TREE_TYPE (TREE_TYPE (rhs)),
10198 392 : TREE_TYPE (gimple_assign_rhs2 (assign)))
10199 31617530 : && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
10200 : {
10201 : /* We start to match on insert to lane zero but since the
10202 : inserts need not be ordered we'd have to search both
10203 : the def and the use chains. */
10204 257 : tree vectype = TREE_TYPE (rhs);
10205 257 : unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10206 257 : auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
10207 257 : auto_sbitmap lanes (nlanes);
10208 257 : bitmap_clear (lanes);
10209 257 : bitmap_set_bit (lanes, 0);
10210 257 : tree def = gimple_assign_lhs (assign);
10211 257 : lane_defs.quick_push
10212 257 : (std::make_pair (0, gimple_assign_rhs2 (assign)));
10213 257 : unsigned lanes_found = 1;
10214 : /* Start with the use chains, the last stmt will be the root. */
10215 257 : stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
10216 257 : vec<stmt_vec_info> roots = vNULL;
10217 257 : roots.safe_push (last);
10218 268 : do
10219 : {
10220 268 : use_operand_p use_p;
10221 268 : gimple *use_stmt;
10222 268 : if (!single_imm_use (def, &use_p, &use_stmt))
10223 : break;
10224 256 : unsigned this_lane;
10225 256 : if (!bb_vinfo->lookup_stmt (use_stmt)
10226 256 : || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
10227 291 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
10228 : break;
10229 35 : if (bitmap_bit_p (lanes, this_lane))
10230 : break;
10231 15 : lanes_found++;
10232 15 : bitmap_set_bit (lanes, this_lane);
10233 15 : gassign *use_ass = as_a <gassign *> (use_stmt);
10234 15 : lane_defs.quick_push (std::make_pair
10235 15 : (this_lane, gimple_assign_rhs2 (use_ass)));
10236 15 : last = bb_vinfo->lookup_stmt (use_ass);
10237 15 : roots.safe_push (last);
10238 15 : def = gimple_assign_lhs (use_ass);
10239 : }
10240 15 : while (lanes_found < nlanes);
10241 257 : if (roots.length () > 1)
10242 7 : std::swap(roots[0], roots[roots.length () - 1]);
10243 257 : if (lanes_found < nlanes)
10244 : {
10245 : /* Now search the def chain. */
10246 253 : def = gimple_assign_rhs1 (assign);
10247 255 : do
10248 : {
10249 255 : if (TREE_CODE (def) != SSA_NAME
10250 255 : || !has_single_use (def))
10251 : break;
10252 59 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
10253 59 : unsigned this_lane;
10254 59 : if (!bb_vinfo->lookup_stmt (def_stmt)
10255 39 : || !vect_slp_is_lane_insert (def_stmt,
10256 : NULL_TREE, &this_lane)
10257 85 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
10258 : break;
10259 26 : if (bitmap_bit_p (lanes, this_lane))
10260 : break;
10261 6 : lanes_found++;
10262 6 : bitmap_set_bit (lanes, this_lane);
10263 12 : lane_defs.quick_push (std::make_pair
10264 6 : (this_lane,
10265 6 : gimple_assign_rhs2 (def_stmt)));
10266 6 : roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
10267 6 : def = gimple_assign_rhs1 (def_stmt);
10268 : }
10269 6 : while (lanes_found < nlanes);
10270 : }
10271 257 : if (lanes_found == nlanes)
10272 : {
10273 : /* Sort lane_defs after the lane index and register the root. */
10274 8 : lane_defs.qsort (vld_cmp);
10275 8 : vec<stmt_vec_info> stmts;
10276 8 : stmts.create (nlanes);
10277 44 : for (unsigned i = 0; i < nlanes; ++i)
10278 28 : stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
10279 8 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10280 8 : stmts, roots));
10281 : }
10282 : else
10283 249 : roots.release ();
10284 257 : }
10285 31616549 : else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10286 30542174 : && (associative_tree_code (code) || code == MINUS_EXPR)
10287 : /* Ops with constants at the tail can be stripped here. */
10288 6744313 : && TREE_CODE (rhs) == SSA_NAME
10289 6680135 : && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
10290 : /* Should be the chain end. */
10291 34522904 : && (!single_imm_use (gimple_assign_lhs (assign),
10292 : &use_p, &use_stmt)
10293 2340461 : || !is_gimple_assign (use_stmt)
10294 1736887 : || (gimple_assign_rhs_code (use_stmt) != code
10295 1165476 : && ((code != PLUS_EXPR && code != MINUS_EXPR)
10296 672777 : || (gimple_assign_rhs_code (use_stmt)
10297 672777 : != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
10298 : {
10299 : /* We start the match at the end of a possible association
10300 : chain. */
10301 2215827 : auto_vec<chain_op_t> chain;
10302 2215827 : auto_vec<std::pair<tree_code, gimple *> > worklist;
10303 2215827 : auto_vec<gimple *> chain_stmts;
10304 2215827 : gimple *code_stmt = NULL, *alt_code_stmt = NULL;
10305 2215827 : if (code == MINUS_EXPR)
10306 362105 : code = PLUS_EXPR;
10307 2215827 : internal_fn reduc_fn;
10308 2618814 : if (!reduction_fn_for_scalar_code (code, &reduc_fn)
10309 2215827 : || reduc_fn == IFN_LAST)
10310 402987 : continue;
10311 1812840 : vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
10312 : /* ??? */
10313 : code_stmt, alt_code_stmt, &chain_stmts,
10314 : false);
10315 3625680 : if (chain.length () > 1)
10316 : {
10317 : /* Sort the chain according to def_type and operation. */
10318 1812840 : chain.sort (dt_sort_cmp, bb_vinfo);
10319 : /* ??? Now we'd want to strip externals and constants
10320 : but record those to be handled in the epilogue. */
10321 : /* ??? For now do not allow mixing ops or externs/constants. */
10322 1812840 : bool invalid = false;
10323 1812840 : unsigned remain_cnt = 0;
10324 7483200 : for (unsigned i = 0; i < chain.length (); ++i)
10325 : {
10326 4219625 : if (chain[i].code != code)
10327 : {
10328 : invalid = true;
10329 : break;
10330 : }
10331 3857520 : if (chain[i].dt != vect_internal_def
10332 : /* Avoid stmts where the def is not the LHS, like
10333 : ASMs. */
10334 7342785 : || (gimple_get_lhs (bb_vinfo->lookup_def
10335 3485265 : (chain[i].op)->stmt)
10336 3485265 : != chain[i].op))
10337 375357 : remain_cnt++;
10338 : }
10339 1812840 : if (!invalid && chain.length () - remain_cnt > 1)
10340 : {
10341 1366013 : vec<stmt_vec_info> stmts;
10342 1366013 : vec<tree> remain = vNULL;
10343 1366013 : stmts.create (chain.length ());
10344 1366013 : if (remain_cnt > 0)
10345 52503 : remain.create (remain_cnt);
10346 4553027 : for (unsigned i = 0; i < chain.length (); ++i)
10347 : {
10348 3187014 : stmt_vec_info stmt_info;
10349 3187014 : if (chain[i].dt == vect_internal_def
10350 3187014 : && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
10351 3080150 : gimple_get_lhs (stmt_info->stmt) == chain[i].op))
10352 3079928 : stmts.quick_push (stmt_info);
10353 : else
10354 107086 : remain.quick_push (chain[i].op);
10355 : }
10356 1366013 : vec<stmt_vec_info> roots;
10357 1366013 : roots.create (chain_stmts.length ());
10358 4553027 : for (unsigned i = 0; i < chain_stmts.length (); ++i)
10359 1821001 : roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
10360 1366013 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
10361 1366013 : stmts, roots, remain));
10362 : }
10363 : }
10364 2215827 : }
10365 : }
10366 2312925 : }
10367 :
10368 : /* Walk the grouped store chains and replace entries with their
10369 : pattern variant if any. */
10370 :
10371 : static void
10372 677367 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
10373 : {
10374 677367 : stmt_vec_info first_element;
10375 677367 : unsigned i;
10376 :
10377 1593019 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
10378 : {
10379 : /* We also have CTORs in this array. */
10380 915652 : if (!STMT_VINFO_GROUPED_ACCESS (first_element))
10381 0 : continue;
10382 915652 : if (STMT_VINFO_IN_PATTERN_P (first_element))
10383 : {
10384 255 : stmt_vec_info orig = first_element;
10385 255 : first_element = STMT_VINFO_RELATED_STMT (first_element);
10386 255 : DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
10387 255 : DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
10388 255 : DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
10389 255 : DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
10390 255 : vinfo->grouped_stores[i] = first_element;
10391 : }
10392 915652 : stmt_vec_info prev = first_element;
10393 2573829 : while (DR_GROUP_NEXT_ELEMENT (prev))
10394 : {
10395 1658177 : stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
10396 1658177 : if (STMT_VINFO_IN_PATTERN_P (elt))
10397 : {
10398 870 : stmt_vec_info orig = elt;
10399 870 : elt = STMT_VINFO_RELATED_STMT (elt);
10400 870 : DR_GROUP_NEXT_ELEMENT (prev) = elt;
10401 870 : DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
10402 870 : DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
10403 : }
10404 1658177 : DR_GROUP_FIRST_ELEMENT (elt) = first_element;
10405 1658177 : prev = elt;
10406 : }
10407 : }
10408 677367 : }
10409 :
10410 : /* Check if the region described by BB_VINFO can be vectorized, returning
10411 : true if so. When returning false, set FATAL to true if the same failure
10412 : would prevent vectorization at other vector sizes, false if it is still
10413 : worth trying other sizes. N_STMTS is the number of statements in the
10414 : region. */
10415 :
10416 : static bool
10417 2312925 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
10418 : vec<int> *dataref_groups)
10419 : {
10420 2312925 : DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
10421 :
10422 2312925 : slp_instance instance;
10423 2312925 : int i;
10424 :
10425 : /* The first group of checks is independent of the vector size. */
10426 2312925 : fatal = true;
10427 :
10428 : /* Analyze the data references. */
10429 :
10430 2312925 : if (!vect_analyze_data_refs (bb_vinfo, NULL))
10431 : {
10432 0 : if (dump_enabled_p ())
10433 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10434 : "not vectorized: unhandled data-ref in basic "
10435 : "block.\n");
10436 : return false;
10437 : }
10438 :
10439 2312925 : if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
10440 : {
10441 0 : if (dump_enabled_p ())
10442 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10443 : "not vectorized: unhandled data access in "
10444 : "basic block.\n");
10445 : return false;
10446 : }
10447 :
10448 2312925 : vect_slp_check_for_roots (bb_vinfo);
10449 :
10450 : /* If there are no grouped stores and no constructors in the region
10451 : there is no need to continue with pattern recog as vect_analyze_slp
10452 : will fail anyway. */
10453 2312925 : if (bb_vinfo->grouped_stores.is_empty ()
10454 1960437 : && bb_vinfo->roots.is_empty ())
10455 : {
10456 1635558 : if (dump_enabled_p ())
10457 1114 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10458 : "not vectorized: no grouped stores in "
10459 : "basic block.\n");
10460 : return false;
10461 : }
10462 :
10463 : /* While the rest of the analysis below depends on it in some way. */
10464 677367 : fatal = false;
10465 :
10466 677367 : vect_pattern_recog (bb_vinfo);
10467 :
10468 : /* Update store groups from pattern processing. */
10469 677367 : vect_fixup_store_groups_with_patterns (bb_vinfo);
10470 :
10471 : /* Check the SLP opportunities in the basic block, analyze and build SLP
10472 : trees. */
10473 677367 : if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
10474 : {
10475 0 : if (dump_enabled_p ())
10476 : {
10477 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10478 : "Failed to SLP the basic block.\n");
10479 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10480 : "not vectorized: failed to find SLP opportunities "
10481 : "in basic block.\n");
10482 : }
10483 : return false;
10484 : }
10485 :
10486 : /* Optimize permutations. */
10487 677367 : vect_optimize_slp (bb_vinfo);
10488 :
10489 : /* Gather the loads reachable from the SLP graph entries. */
10490 677367 : vect_gather_slp_loads (bb_vinfo);
10491 :
10492 677367 : vect_record_base_alignments (bb_vinfo);
10493 :
10494 : /* Analyze and verify the alignment of data references and the
10495 : dependence in the SLP instances. */
10496 2189215 : for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
10497 : {
10498 834481 : vect_location = instance->location ();
10499 834481 : if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
10500 834481 : || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
10501 : {
10502 8799 : slp_tree node = SLP_INSTANCE_TREE (instance);
10503 8799 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10504 8799 : if (dump_enabled_p ())
10505 4 : dump_printf_loc (MSG_NOTE, vect_location,
10506 : "removing SLP instance operations starting from: %G",
10507 : stmt_info->stmt);
10508 8799 : vect_free_slp_instance (instance);
10509 8799 : BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
10510 8799 : continue;
10511 8799 : }
10512 :
10513 : /* Mark all the statements that we want to vectorize as relevant. */
10514 825682 : vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
10515 :
10516 825682 : i++;
10517 : }
10518 2593474 : if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
10519 : return false;
10520 :
10521 280549 : if (!vect_slp_analyze_operations (bb_vinfo))
10522 : {
10523 32003 : if (dump_enabled_p ())
10524 67 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10525 : "not vectorized: bad operation in basic block.\n");
10526 : return false;
10527 : }
10528 :
10529 : /* Compute vector stmt placement. */
10530 248546 : if (!vect_schedule_slp (bb_vinfo, BB_VINFO_SLP_INSTANCES (bb_vinfo), true))
10531 : {
10532 0 : if (dump_enabled_p ())
10533 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10534 : "not vectorized: cannot schedule SLP graph\n");
10535 : return false;
10536 : }
10537 :
10538 : /* Mark all the statements that we vectorize. */
10539 248546 : vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
10540 :
10541 : /* Compute vectorizable live stmts. */
10542 248546 : vect_bb_slp_mark_live_stmts (bb_vinfo);
10543 :
10544 248546 : vect_bb_partition_graph (bb_vinfo);
10545 :
10546 248546 : return true;
10547 : }
10548 :
10549 : /* Subroutine of vect_slp_bb. Try to vectorize the statements for all
10550 : basic blocks in BBS, returning true on success.
10551 : The region has N_STMTS statements and has the datarefs given by DATAREFS. */
10552 :
10553 : static bool
10554 1946392 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
10555 : vec<int> *dataref_groups, unsigned int n_stmts,
10556 : loop_p orig_loop)
10557 : {
10558 1946392 : bb_vec_info bb_vinfo;
10559 1946392 : auto_vector_modes vector_modes;
10560 :
10561 : /* Autodetect first vector size we try. */
10562 1946392 : machine_mode next_vector_mode = VOIDmode;
10563 1946392 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
10564 1946392 : unsigned int mode_i = 0;
10565 :
10566 1946392 : vec_info_shared shared;
10567 :
10568 1946392 : machine_mode autodetected_vector_mode = VOIDmode;
10569 2679458 : while (1)
10570 : {
10571 2312925 : bool vectorized = false;
10572 2312925 : bool fatal = false;
10573 2312925 : bb_vinfo = new _bb_vec_info (bbs, &shared);
10574 :
10575 2312925 : bool first_time_p = shared.datarefs.is_empty ();
10576 2312925 : BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
10577 2312925 : if (first_time_p)
10578 1971414 : bb_vinfo->shared->save_datarefs ();
10579 : else
10580 341511 : bb_vinfo->shared->check_datarefs ();
10581 2312925 : bb_vinfo->vector_mode = next_vector_mode;
10582 :
10583 2312925 : if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
10584 : {
10585 248546 : if (dump_enabled_p ())
10586 : {
10587 1734 : dump_printf_loc (MSG_NOTE, vect_location,
10588 : "***** Analysis succeeded with vector mode"
10589 867 : " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
10590 867 : dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
10591 : }
10592 :
10593 248546 : bb_vinfo->shared->check_datarefs ();
10594 :
10595 248546 : bool force_clear = false;
10596 248546 : auto_vec<slp_instance> profitable_subgraphs;
10597 1468133 : for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
10598 : {
10599 722495 : if (instance->subgraph_entries.is_empty ())
10600 240922 : continue;
10601 :
10602 694876 : dump_user_location_t saved_vect_location = vect_location;
10603 694876 : vect_location = instance->location ();
10604 694876 : if (!unlimited_cost_model (NULL)
10605 690799 : && !param_vect_allow_possibly_not_worthwhile_vectorizations
10606 1385666 : && !vect_bb_vectorization_profitable_p
10607 690790 : (bb_vinfo, instance->subgraph_entries, orig_loop))
10608 : {
10609 213303 : if (dump_enabled_p ())
10610 51 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10611 : "not vectorized: vectorization is not "
10612 : "profitable.\n");
10613 213303 : vect_location = saved_vect_location;
10614 213303 : continue;
10615 : }
10616 :
10617 481573 : vect_location = saved_vect_location;
10618 481573 : if (!dbg_cnt (vect_slp))
10619 : {
10620 0 : force_clear = true;
10621 0 : continue;
10622 : }
10623 :
10624 481573 : profitable_subgraphs.safe_push (instance);
10625 : }
10626 :
10627 : /* When we're vectorizing an if-converted loop body make sure
10628 : we vectorized all if-converted code. */
10629 406023 : if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
10630 : {
10631 130 : gcc_assert (bb_vinfo->nbbs == 1);
10632 260 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
10633 5837 : !gsi_end_p (gsi); gsi_next (&gsi))
10634 : {
10635 : /* The costing above left us with DCEable vectorized scalar
10636 : stmts having the visited flag set on profitable
10637 : subgraphs. Do the delayed clearing of the flag here. */
10638 5707 : if (gimple_visited_p (gsi_stmt (gsi)))
10639 : {
10640 1816 : gimple_set_visited (gsi_stmt (gsi), false);
10641 1816 : continue;
10642 : }
10643 3891 : if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
10644 893 : continue;
10645 :
10646 8450 : if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
10647 3456 : if (gimple_assign_rhs_code (ass) == COND_EXPR)
10648 : {
10649 151 : if (!profitable_subgraphs.is_empty ()
10650 66 : && dump_enabled_p ())
10651 0 : dump_printf_loc (MSG_NOTE, vect_location,
10652 : "not profitable because of "
10653 : "unprofitable if-converted scalar "
10654 : "code\n");
10655 85 : profitable_subgraphs.truncate (0);
10656 : }
10657 : }
10658 : }
10659 :
10660 : /* Finally schedule the profitable subgraphs. */
10661 1044990 : for (slp_instance instance : profitable_subgraphs)
10662 : {
10663 481490 : if (!vectorized && dump_enabled_p ())
10664 828 : dump_printf_loc (MSG_NOTE, vect_location,
10665 : "Basic block will be vectorized "
10666 : "using SLP\n");
10667 481490 : vectorized = true;
10668 :
10669 : /* Dump before scheduling as store vectorization will remove
10670 : the original stores and mess with the instance tree
10671 : so querying its location will eventually ICE. */
10672 481490 : if (flag_checking)
10673 1937166 : for (slp_instance sub : instance->subgraph_entries)
10674 492696 : gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
10675 481490 : unsigned HOST_WIDE_INT bytes;
10676 481490 : if (dump_enabled_p ())
10677 3999 : for (slp_instance sub : instance->subgraph_entries)
10678 : {
10679 1077 : tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
10680 2154 : if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
10681 1077 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10682 1077 : sub->location (),
10683 : "basic block part vectorized using %wu "
10684 : "byte vectors\n", bytes);
10685 : else
10686 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10687 : sub->location (),
10688 : "basic block part vectorized using "
10689 : "variable length vectors\n");
10690 : }
10691 :
10692 481490 : dump_user_location_t saved_vect_location = vect_location;
10693 481490 : vect_location = instance->location ();
10694 :
10695 481490 : vect_schedule_slp (bb_vinfo, instance->subgraph_entries, false);
10696 :
10697 481490 : vect_location = saved_vect_location;
10698 : }
10699 :
10700 : /* Generate the invariant statements. */
10701 248546 : if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
10702 : {
10703 23 : if (dump_enabled_p ())
10704 0 : dump_printf_loc (MSG_NOTE, vect_location,
10705 : "------>generating invariant statements\n");
10706 :
10707 23 : bb_vinfo->insert_seq_on_entry (NULL,
10708 : bb_vinfo->inv_pattern_def_seq);
10709 : }
10710 248546 : }
10711 : else
10712 : {
10713 2064379 : if (dump_enabled_p ())
10714 1448 : dump_printf_loc (MSG_NOTE, vect_location,
10715 : "***** Analysis failed with vector mode %s\n",
10716 1448 : GET_MODE_NAME (bb_vinfo->vector_mode));
10717 : }
10718 :
10719 2312925 : if (mode_i == 0)
10720 1946392 : autodetected_vector_mode = bb_vinfo->vector_mode;
10721 :
10722 2312925 : if (!fatal)
10723 3347316 : while (mode_i < vector_modes.length ()
10724 1915315 : && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
10725 : {
10726 357024 : if (dump_enabled_p ())
10727 1892 : dump_printf_loc (MSG_NOTE, vect_location,
10728 : "***** The result for vector mode %s would"
10729 : " be the same\n",
10730 946 : GET_MODE_NAME (vector_modes[mode_i]));
10731 357024 : mode_i += 1;
10732 : }
10733 :
10734 2312925 : delete bb_vinfo;
10735 :
10736 2312925 : if (mode_i < vector_modes.length ()
10737 2111858 : && VECTOR_MODE_P (autodetected_vector_mode)
10738 2089722 : && (related_vector_mode (vector_modes[mode_i],
10739 : GET_MODE_INNER (autodetected_vector_mode))
10740 1044861 : == autodetected_vector_mode)
10741 4424783 : && (related_vector_mode (autodetected_vector_mode,
10742 520817 : GET_MODE_INNER (vector_modes[mode_i]))
10743 1041634 : == vector_modes[mode_i]))
10744 : {
10745 520817 : if (dump_enabled_p ())
10746 230 : dump_printf_loc (MSG_NOTE, vect_location,
10747 : "***** Skipping vector mode %s, which would"
10748 : " repeat the analysis for %s\n",
10749 230 : GET_MODE_NAME (vector_modes[mode_i]),
10750 230 : GET_MODE_NAME (autodetected_vector_mode));
10751 520817 : mode_i += 1;
10752 : }
10753 :
10754 2312925 : if (vectorized
10755 2155514 : || mode_i == vector_modes.length ()
10756 1954491 : || autodetected_vector_mode == VOIDmode
10757 : /* If vect_slp_analyze_bb_1 signaled that analysis for all
10758 : vector sizes will fail do not bother iterating. */
10759 3200419 : || fatal)
10760 3892784 : return vectorized;
10761 :
10762 : /* Try the next biggest vector size. */
10763 366533 : next_vector_mode = vector_modes[mode_i++];
10764 366533 : if (dump_enabled_p ())
10765 257 : dump_printf_loc (MSG_NOTE, vect_location,
10766 : "***** Re-trying analysis with vector mode %s\n",
10767 257 : GET_MODE_NAME (next_vector_mode));
10768 366533 : }
10769 1946392 : }
10770 :
10771 :
10772 : /* Main entry for the BB vectorizer. Analyze and transform BBS, returns
10773 : true if anything in the basic-block was vectorized. */
10774 :
10775 : static bool
10776 1946392 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
10777 : {
10778 1946392 : vec<data_reference_p> datarefs = vNULL;
10779 1946392 : auto_vec<int> dataref_groups;
10780 1946392 : int insns = 0;
10781 1946392 : int current_group = 0;
10782 :
10783 12871403 : for (unsigned i = 0; i < bbs.length (); i++)
10784 : {
10785 10925011 : basic_block bb = bbs[i];
10786 94873300 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
10787 83948289 : gsi_next (&gsi))
10788 : {
10789 83948289 : gimple *stmt = gsi_stmt (gsi);
10790 83948289 : if (is_gimple_debug (stmt))
10791 53067155 : continue;
10792 :
10793 30881134 : insns++;
10794 :
10795 30881134 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
10796 27692030 : vect_location = stmt;
10797 :
10798 30881134 : if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
10799 : &dataref_groups, current_group))
10800 5315747 : ++current_group;
10801 : }
10802 : /* New BBs always start a new DR group. */
10803 10925011 : ++current_group;
10804 : }
10805 :
10806 1946392 : return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
10807 1946392 : }
10808 :
10809 : /* Special entry for the BB vectorizer. Analyze and transform a single
10810 : if-converted BB with ORIG_LOOPs body being the not if-converted
10811 : representation. Returns true if anything in the basic-block was
10812 : vectorized. */
10813 :
10814 : bool
10815 16943 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
10816 : {
10817 16943 : auto_vec<basic_block> bbs;
10818 16943 : bbs.safe_push (bb);
10819 16943 : return vect_slp_bbs (bbs, orig_loop);
10820 16943 : }
10821 :
10822 : /* Main entry for the BB vectorizer. Analyze and transform BB, returns
10823 : true if anything in the basic-block was vectorized. */
10824 :
10825 : bool
10826 925882 : vect_slp_function (function *fun)
10827 : {
10828 925882 : bool r = false;
10829 925882 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
10830 925882 : auto_bitmap exit_bbs;
10831 925882 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
10832 925882 : edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
10833 925882 : unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
10834 925882 : true, rpo, NULL);
10835 :
10836 : /* For the moment split the function into pieces to avoid making
10837 : the iteration on the vector mode moot. Split at points we know
10838 : to not handle well which is CFG merges (SLP discovery doesn't
10839 : handle non-loop-header PHIs) and loop exits. Since pattern
10840 : recog requires reverse iteration to visit uses before defs
10841 : simply chop RPO into pieces. */
10842 925882 : auto_vec<basic_block> bbs;
10843 11860734 : for (unsigned i = 0; i < n; i++)
10844 : {
10845 10934852 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
10846 10934852 : bool split = false;
10847 :
10848 : /* Split when a BB is not dominated by the first block. */
10849 20627440 : if (!bbs.is_empty ()
10850 9692588 : && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
10851 : {
10852 702626 : if (dump_enabled_p ())
10853 185 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10854 : "splitting region at dominance boundary bb%d\n",
10855 : bb->index);
10856 : split = true;
10857 : }
10858 : /* Split when the loop determined by the first block
10859 : is exited. This is because we eventually insert
10860 : invariants at region begin. */
10861 19222188 : else if (!bbs.is_empty ()
10862 8989962 : && bbs[0]->loop_father != bb->loop_father
10863 2356054 : && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
10864 : {
10865 3881 : if (dump_enabled_p ())
10866 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10867 : "splitting region at loop %d exit at bb%d\n",
10868 3 : bbs[0]->loop_father->num, bb->index);
10869 : split = true;
10870 : }
10871 10228345 : else if (!bbs.is_empty ()
10872 8986081 : && bb->loop_father->header == bb
10873 483423 : && bb->loop_father->dont_vectorize)
10874 : {
10875 6225 : if (dump_enabled_p ())
10876 72 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10877 : "splitting region at dont-vectorize loop %d "
10878 : "entry at bb%d\n",
10879 : bb->loop_father->num, bb->index);
10880 : split = true;
10881 : }
10882 10222120 : else if (!bbs.is_empty ()
10883 8979856 : && bb_has_abnormal_pred (bb))
10884 : {
10885 1237 : if (dump_enabled_p ())
10886 15 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10887 : "splitting region at incoming abnormal edge "
10888 : "at bb%d\n", bb->index);
10889 : split = true;
10890 : }
10891 :
10892 275 : if (split && !bbs.is_empty ())
10893 : {
10894 713969 : r |= vect_slp_bbs (bbs, NULL);
10895 713969 : bbs.truncate (0);
10896 : }
10897 :
10898 10934852 : if (bbs.is_empty ())
10899 : {
10900 : /* We need to be able to insert at the head of the region which
10901 : we cannot for region starting with a returns-twice call. */
10902 1956233 : if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
10903 415089 : if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
10904 : {
10905 612 : if (dump_enabled_p ())
10906 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10907 : "skipping bb%d as start of region as it "
10908 : "starts with returns-twice call\n",
10909 : bb->index);
10910 26784 : continue;
10911 : }
10912 : /* If the loop this BB belongs to is marked as not to be vectorized
10913 : honor that also for BB vectorization. */
10914 1955621 : if (bb->loop_father->dont_vectorize)
10915 26172 : continue;
10916 : }
10917 :
10918 10908068 : bbs.safe_push (bb);
10919 :
10920 : /* When we have a stmt ending this block and defining a
10921 : value we have to insert on edges when inserting after it for
10922 : a vector containing its definition. Avoid this for now. */
10923 21816136 : if (gimple *last = *gsi_last_bb (bb))
10924 8876804 : if (gimple_get_lhs (last)
10925 8876804 : && is_ctrl_altering_stmt (last))
10926 : {
10927 289608 : if (dump_enabled_p ())
10928 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10929 : "splitting region at control altering "
10930 : "definition %G", last);
10931 289608 : r |= vect_slp_bbs (bbs, NULL);
10932 289608 : bbs.truncate (0);
10933 : }
10934 : }
10935 :
10936 925882 : if (!bbs.is_empty ())
10937 925872 : r |= vect_slp_bbs (bbs, NULL);
10938 :
10939 925882 : free (rpo);
10940 :
10941 925882 : return r;
10942 925882 : }
10943 :
10944 : /* Build a variable-length vector in which the elements in ELTS are repeated
10945 : to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
10946 : RESULTS and add any new instructions to SEQ.
10947 :
10948 : The approach we use is:
10949 :
10950 : (1) Find a vector mode VM with integer elements of mode IM.
10951 :
10952 : (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10953 : ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
10954 : from small vectors to IM.
10955 :
10956 : (3) Duplicate each ELTS'[I] into a vector of mode VM.
10957 :
10958 : (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
10959 : correct byte contents.
10960 :
10961 : (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
10962 :
10963 : We try to find the largest IM for which this sequence works, in order
10964 : to cut down on the number of interleaves. */
10965 :
10966 : void
10967 0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
10968 : const vec<tree> &elts, unsigned int nresults,
10969 : vec<tree> &results)
10970 : {
10971 0 : unsigned int nelts = elts.length ();
10972 0 : tree element_type = TREE_TYPE (vector_type);
10973 :
10974 : /* (1) Find a vector mode VM with integer elements of mode IM. */
10975 0 : unsigned int nvectors = 1;
10976 0 : tree new_vector_type;
10977 0 : tree permutes[2];
10978 0 : if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
10979 : &nvectors, &new_vector_type,
10980 : permutes))
10981 0 : gcc_unreachable ();
10982 :
10983 : /* Get a vector type that holds ELTS[0:NELTS/NELTS']. */
10984 0 : unsigned int partial_nelts = nelts / nvectors;
10985 0 : tree partial_vector_type = build_vector_type (element_type, partial_nelts);
10986 :
10987 0 : tree_vector_builder partial_elts;
10988 0 : auto_vec<tree, 32> pieces (nvectors * 2);
10989 0 : pieces.quick_grow_cleared (nvectors * 2);
10990 0 : for (unsigned int i = 0; i < nvectors; ++i)
10991 : {
10992 : /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10993 : ELTS' has mode IM. */
10994 0 : partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
10995 0 : for (unsigned int j = 0; j < partial_nelts; ++j)
10996 0 : partial_elts.quick_push (elts[i * partial_nelts + j]);
10997 0 : tree t = gimple_build_vector (seq, &partial_elts);
10998 0 : t = gimple_build (seq, VIEW_CONVERT_EXPR,
10999 0 : TREE_TYPE (new_vector_type), t);
11000 :
11001 : /* (3) Duplicate each ELTS'[I] into a vector of mode VM. */
11002 0 : pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
11003 : }
11004 :
11005 : /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
11006 : correct byte contents.
11007 :
11008 : Conceptually, we need to repeat the following operation log2(nvectors)
11009 : times, where hi_start = nvectors / 2:
11010 :
11011 : out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
11012 : out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
11013 :
11014 : However, if each input repeats every N elements and the VF is
11015 : a multiple of N * 2, the HI result is the same as the LO result.
11016 : This will be true for the first N1 iterations of the outer loop,
11017 : followed by N2 iterations for which both the LO and HI results
11018 : are needed. I.e.:
11019 :
11020 : N1 + N2 = log2(nvectors)
11021 :
11022 : Each "N1 iteration" doubles the number of redundant vectors and the
11023 : effect of the process as a whole is to have a sequence of nvectors/2**N1
11024 : vectors that repeats 2**N1 times. Rather than generate these redundant
11025 : vectors, we halve the number of vectors for each N1 iteration. */
11026 : unsigned int in_start = 0;
11027 : unsigned int out_start = nvectors;
11028 : unsigned int new_nvectors = nvectors;
11029 0 : for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
11030 : {
11031 0 : unsigned int hi_start = new_nvectors / 2;
11032 0 : unsigned int out_i = 0;
11033 0 : for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
11034 : {
11035 0 : if ((in_i & 1) != 0
11036 0 : && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
11037 : 2 * in_repeat))
11038 0 : continue;
11039 :
11040 0 : tree output = make_ssa_name (new_vector_type);
11041 0 : tree input1 = pieces[in_start + (in_i / 2)];
11042 0 : tree input2 = pieces[in_start + (in_i / 2) + hi_start];
11043 0 : gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
11044 : input1, input2,
11045 : permutes[in_i & 1]);
11046 0 : gimple_seq_add_stmt (seq, stmt);
11047 0 : pieces[out_start + out_i] = output;
11048 0 : out_i += 1;
11049 : }
11050 0 : std::swap (in_start, out_start);
11051 0 : new_nvectors = out_i;
11052 : }
11053 :
11054 : /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type. */
11055 0 : results.reserve (nresults);
11056 0 : for (unsigned int i = 0; i < nresults; ++i)
11057 0 : if (i < new_nvectors)
11058 0 : results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
11059 0 : pieces[in_start + i]));
11060 : else
11061 0 : results.quick_push (results[i - new_nvectors]);
11062 0 : }
11063 :
11064 :
11065 : /* For constant and loop invariant defs in OP_NODE this function creates
11066 : vector defs that will be used in the vectorized stmts and stores them
11067 : to SLP_TREE_VEC_DEFS of OP_NODE. */
11068 :
11069 : static void
11070 503059 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
11071 : {
11072 503059 : unsigned HOST_WIDE_INT nunits;
11073 503059 : tree vec_cst;
11074 503059 : unsigned j, number_of_places_left_in_vector;
11075 503059 : tree vector_type;
11076 503059 : tree vop;
11077 503059 : int group_size = op_node->ops.length ();
11078 503059 : unsigned int vec_num, i;
11079 503059 : unsigned number_of_copies = 1;
11080 503059 : bool constant_p;
11081 503059 : gimple_seq ctor_seq = NULL;
11082 503059 : auto_vec<tree, 16> permute_results;
11083 :
11084 : /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
11085 503059 : vector_type = SLP_TREE_VECTYPE (op_node);
11086 :
11087 503059 : unsigned int number_of_vectors, excess_elts;
11088 503059 : bool res = vect_get_num_copies_for_invariant (vinfo, op_node,
11089 : &number_of_vectors,
11090 : &excess_elts);
11091 503059 : gcc_assert (res);
11092 503059 : SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
11093 503059 : auto_vec<tree> voprnds (number_of_vectors);
11094 :
11095 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
11096 : created vectors. It is greater than 1 if unrolling is performed.
11097 :
11098 : For example, we have two scalar operands, s1 and s2 (e.g., group of
11099 : strided accesses of size two), while NUNITS is four (i.e., four scalars
11100 : of this type can be packed in a vector). The output vector will contain
11101 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
11102 : will be 2).
11103 :
11104 : If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
11105 : containing the operands.
11106 :
11107 : For example, NUNITS is four as before, and the group size is 8
11108 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
11109 : {s5, s6, s7, s8}. */
11110 :
11111 : /* When using duplicate_and_interleave, we just need one element for
11112 : each scalar statement. */
11113 503059 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
11114 : nunits = group_size;
11115 :
11116 503059 : number_of_copies = (nunits * number_of_vectors - excess_elts) / group_size;
11117 :
11118 503059 : constant_p = true;
11119 503059 : tree uniform_elt = NULL_TREE;
11120 503059 : tree_vector_builder elts (vector_type, nunits, 1);
11121 503059 : elts.quick_grow (nunits);
11122 : /* Zero-pad the last vector if necessary. */
11123 503059 : number_of_places_left_in_vector = nunits - excess_elts;
11124 503317 : for (i = nunits; i > number_of_places_left_in_vector; --i)
11125 258 : elts[i-1] = build_zero_cst (TREE_TYPE (vector_type));
11126 : stmt_vec_info insert_after = NULL;
11127 1501902 : for (j = 0; j < number_of_copies; j++)
11128 : {
11129 998843 : tree op;
11130 3825581 : for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
11131 : {
11132 : /* Create 'vect_ = {op0,op1,...,opn}'. */
11133 1827895 : tree orig_op = op;
11134 1827895 : if (number_of_places_left_in_vector == nunits)
11135 : uniform_elt = op;
11136 1193072 : else if (uniform_elt && operand_equal_p (uniform_elt, op))
11137 757266 : op = elts[number_of_places_left_in_vector];
11138 : else
11139 : uniform_elt = NULL_TREE;
11140 1827895 : number_of_places_left_in_vector--;
11141 1827895 : if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
11142 : {
11143 279627 : if (CONSTANT_CLASS_P (op))
11144 : {
11145 101213 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
11146 : {
11147 : /* Can't use VIEW_CONVERT_EXPR for booleans because
11148 : of possibly different sizes of scalar value and
11149 : vector element. */
11150 66 : if (integer_zerop (op))
11151 66 : op = build_int_cst (TREE_TYPE (vector_type), 0);
11152 0 : else if (integer_onep (op))
11153 0 : op = build_all_ones_cst (TREE_TYPE (vector_type));
11154 : else
11155 0 : gcc_unreachable ();
11156 : }
11157 : else
11158 101147 : op = fold_unary (VIEW_CONVERT_EXPR,
11159 : TREE_TYPE (vector_type), op);
11160 101213 : gcc_assert (op && CONSTANT_CLASS_P (op));
11161 : }
11162 : else
11163 : {
11164 178414 : tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
11165 178414 : gimple *init_stmt;
11166 178414 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
11167 : {
11168 427 : tree true_val
11169 427 : = build_all_ones_cst (TREE_TYPE (vector_type));
11170 427 : tree false_val
11171 427 : = build_zero_cst (TREE_TYPE (vector_type));
11172 427 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
11173 427 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
11174 : op, true_val,
11175 : false_val);
11176 : }
11177 : else
11178 : {
11179 177987 : op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
11180 : op);
11181 177987 : init_stmt
11182 177987 : = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
11183 : op);
11184 : }
11185 178414 : gimple_seq_add_stmt (&ctor_seq, init_stmt);
11186 178414 : op = new_temp;
11187 : }
11188 : }
11189 1827895 : elts[number_of_places_left_in_vector] = op;
11190 1827895 : if (!CONSTANT_CLASS_P (op))
11191 322876 : constant_p = false;
11192 : /* For BB vectorization we have to compute an insert location
11193 : when a def is inside the analyzed region since we cannot
11194 : simply insert at the BB start in this case. */
11195 1827895 : stmt_vec_info opdef;
11196 1827895 : if (TREE_CODE (orig_op) == SSA_NAME
11197 185406 : && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
11198 164570 : && is_a <bb_vec_info> (vinfo)
11199 1934398 : && (opdef = vinfo->lookup_def (orig_op)))
11200 : {
11201 86346 : if (!insert_after)
11202 : insert_after = opdef;
11203 : else
11204 47617 : insert_after = get_later_stmt (insert_after, opdef);
11205 : }
11206 :
11207 1827895 : if (number_of_places_left_in_vector == 0)
11208 : {
11209 634852 : auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
11210 634852 : if (uniform_elt)
11211 657080 : vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
11212 328540 : elts[0]);
11213 612624 : else if (constant_p
11214 612624 : ? multiple_p (type_nunits, nunits)
11215 111864 : : known_eq (type_nunits, nunits))
11216 306312 : vec_cst = gimple_build_vector (&ctor_seq, &elts);
11217 : else
11218 : {
11219 0 : if (permute_results.is_empty ())
11220 0 : duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
11221 : elts, number_of_vectors,
11222 : permute_results);
11223 0 : vec_cst = permute_results[number_of_vectors - j - 1];
11224 : }
11225 634852 : if (!gimple_seq_empty_p (ctor_seq))
11226 : {
11227 139704 : if (insert_after)
11228 : {
11229 38729 : gimple_stmt_iterator gsi;
11230 38729 : if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
11231 : {
11232 612 : gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
11233 612 : gsi_insert_seq_before (&gsi, ctor_seq,
11234 : GSI_CONTINUE_LINKING);
11235 : }
11236 38117 : else if (!stmt_ends_bb_p (insert_after->stmt))
11237 : {
11238 38117 : gsi = gsi_for_stmt (insert_after->stmt);
11239 38117 : gsi_insert_seq_after (&gsi, ctor_seq,
11240 : GSI_CONTINUE_LINKING);
11241 : }
11242 : else
11243 : {
11244 : /* When we want to insert after a def where the
11245 : defining stmt throws then insert on the fallthru
11246 : edge. */
11247 0 : edge e = find_fallthru_edge
11248 0 : (gimple_bb (insert_after->stmt)->succs);
11249 0 : basic_block new_bb
11250 0 : = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
11251 0 : gcc_assert (!new_bb);
11252 : }
11253 : }
11254 : else
11255 100975 : vinfo->insert_seq_on_entry (NULL, ctor_seq);
11256 139704 : ctor_seq = NULL;
11257 : }
11258 634852 : voprnds.quick_push (vec_cst);
11259 634852 : insert_after = NULL;
11260 634852 : number_of_places_left_in_vector = nunits;
11261 634852 : constant_p = true;
11262 634852 : elts.new_vector (vector_type, nunits, 1);
11263 634852 : elts.quick_grow (nunits);
11264 : }
11265 : }
11266 : }
11267 :
11268 503059 : gcc_assert (number_of_places_left_in_vector == nunits);
11269 :
11270 : /* Since the vectors are created in the reverse order, we should invert
11271 : them. */
11272 503059 : vec_num = voprnds.length ();
11273 503059 : gcc_checking_assert (vec_num > 0);
11274 :
11275 1137911 : for (j = vec_num; j != 0; j--)
11276 : {
11277 634852 : vop = voprnds[j - 1];
11278 634852 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11279 : }
11280 :
11281 : /* In case that VF is greater than the unrolling factor needed for the SLP
11282 : group of stmts, NUMBER_OF_VECTORS to be created is greater than
11283 : NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
11284 : to replicate the vectors. */
11285 1006118 : while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
11286 0 : for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
11287 : i++)
11288 0 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11289 503059 : }
11290 :
11291 : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
11292 : if there is no definition for it in the scalar IL or it is not known. */
11293 :
11294 : tree
11295 78953 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
11296 : {
11297 78953 : if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
11298 : {
11299 2716 : if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
11300 : return NULL_TREE;
11301 2716 : stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
11302 2716 : if (!def)
11303 : return NULL_TREE;
11304 2716 : return gimple_get_lhs (STMT_VINFO_STMT (def));
11305 : }
11306 : else
11307 76237 : return SLP_TREE_SCALAR_OPS (slp_node)[n];
11308 : }
11309 :
11310 : /* Get the Ith vectorized definition from SLP_NODE. */
11311 :
11312 : tree
11313 141103 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
11314 : {
11315 141103 : return SLP_TREE_VEC_DEFS (slp_node)[i];
11316 : }
11317 :
11318 : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
11319 :
11320 : void
11321 942003 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
11322 : {
11323 1884006 : vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
11324 942003 : vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
11325 942003 : }
11326 :
11327 : /* Get N vectorized definitions for SLP_NODE. */
11328 :
11329 : void
11330 3378 : vect_get_slp_defs (vec_info *,
11331 : slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
11332 : {
11333 3378 : if (n == -1U)
11334 3378 : n = SLP_TREE_CHILDREN (slp_node).length ();
11335 :
11336 11803 : for (unsigned i = 0; i < n; ++i)
11337 : {
11338 8425 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
11339 8425 : vec<tree> vec_defs = vNULL;
11340 8425 : vect_get_slp_defs (child, &vec_defs);
11341 8425 : vec_oprnds->quick_push (vec_defs);
11342 : }
11343 3378 : }
11344 :
11345 : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
11346 : - PERM gives the permutation that the caller wants to use for NODE,
11347 : which might be different from SLP_LOAD_PERMUTATION.
11348 : - DUMP_P controls whether the function dumps information. */
11349 :
11350 : static bool
11351 134954 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
11352 : load_permutation_t &perm,
11353 : const vec<tree> &dr_chain,
11354 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11355 : bool analyze_only, bool dump_p,
11356 : unsigned *n_perms, unsigned int *n_loads,
11357 : bool dce_chain)
11358 : {
11359 134954 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
11360 134954 : int vec_index = 0;
11361 134954 : tree vectype = SLP_TREE_VECTYPE (node);
11362 134954 : unsigned int group_size = SLP_TREE_LANES (node);
11363 134954 : unsigned int mask_element;
11364 134954 : unsigned dr_group_size;
11365 134954 : machine_mode mode;
11366 :
11367 134954 : if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
11368 : {
11369 : /* We have both splats of the same non-grouped load and groups
11370 : of distinct invariant loads entering here. */
11371 1538 : unsigned max_idx = 0;
11372 8498 : for (auto idx : perm)
11373 3884 : max_idx = idx > max_idx ? idx : max_idx;
11374 1538 : dr_group_size = max_idx + 1;
11375 : }
11376 : else
11377 : {
11378 133416 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
11379 133416 : dr_group_size = DR_GROUP_SIZE (stmt_info);
11380 : }
11381 :
11382 134954 : mode = TYPE_MODE (vectype);
11383 134954 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11384 134954 : unsigned int nstmts = vect_get_num_copies (vinfo, node);
11385 :
11386 : /* Initialize the vect stmts of NODE to properly insert the generated
11387 : stmts later. */
11388 134954 : if (! analyze_only)
11389 57621 : for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
11390 22235 : SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
11391 :
11392 : /* Generate permutation masks for every NODE. Number of masks for each NODE
11393 : is equal to GROUP_SIZE.
11394 : E.g., we have a group of three nodes with three loads from the same
11395 : location in each node, and the vector size is 4. I.e., we have a
11396 : a0b0c0a1b1c1... sequence and we need to create the following vectors:
11397 : for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
11398 : for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
11399 : ...
11400 :
11401 : The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
11402 : The last mask is illegal since we assume two operands for permute
11403 : operation, and the mask element values can't be outside that range.
11404 : Hence, the last mask must be converted into {2,5,5,5}.
11405 : For the first two permutations we need the first and the second input
11406 : vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
11407 : we need the second and the third vectors: {b1,c1,a2,b2} and
11408 : {c2,a3,b3,c3}. */
11409 :
11410 134954 : int vect_stmts_counter = 0;
11411 134954 : unsigned int index = 0;
11412 134954 : int first_vec_index = -1;
11413 134954 : int second_vec_index = -1;
11414 134954 : bool noop_p = true;
11415 134954 : *n_perms = 0;
11416 :
11417 134954 : vec_perm_builder mask;
11418 134954 : unsigned int nelts_to_build;
11419 134954 : unsigned int nvectors_per_build;
11420 134954 : unsigned int in_nlanes;
11421 134954 : bool repeating_p = (group_size == dr_group_size
11422 170809 : && multiple_p (nunits, group_size));
11423 134954 : if (repeating_p)
11424 : {
11425 : /* A single vector contains a whole number of copies of the node, so:
11426 : (a) all permutes can use the same mask; and
11427 : (b) the permutes only need a single vector input. */
11428 33345 : mask.new_vector (nunits, group_size, 3);
11429 33345 : nelts_to_build = mask.encoded_nelts ();
11430 : /* It's possible to obtain zero nstmts during analyze_only, so make
11431 : it at least one to ensure the later computation for n_perms
11432 : proceed. */
11433 33345 : nvectors_per_build = nstmts > 0 ? nstmts : 1;
11434 33345 : in_nlanes = dr_group_size * 3;
11435 : }
11436 : else
11437 : {
11438 : /* We need to construct a separate mask for each vector statement. */
11439 101609 : unsigned HOST_WIDE_INT const_nunits, const_vf;
11440 101609 : if (!nunits.is_constant (&const_nunits)
11441 101609 : || !vf.is_constant (&const_vf))
11442 : return false;
11443 101609 : mask.new_vector (const_nunits, const_nunits, 1);
11444 101609 : nelts_to_build = const_vf * group_size;
11445 101609 : nvectors_per_build = 1;
11446 101609 : in_nlanes = const_vf * dr_group_size;
11447 : }
11448 134954 : auto_sbitmap used_in_lanes (in_nlanes);
11449 134954 : bitmap_clear (used_in_lanes);
11450 134954 : auto_bitmap used_defs;
11451 :
11452 134954 : unsigned int count = mask.encoded_nelts ();
11453 134954 : mask.quick_grow (count);
11454 134954 : vec_perm_indices indices;
11455 :
11456 709405 : for (unsigned int j = 0; j < nelts_to_build; j++)
11457 : {
11458 585515 : unsigned int iter_num = j / group_size;
11459 585515 : unsigned int stmt_num = j % group_size;
11460 585515 : unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
11461 585515 : bitmap_set_bit (used_in_lanes, i);
11462 585515 : if (repeating_p)
11463 : {
11464 : first_vec_index = 0;
11465 : mask_element = i;
11466 : }
11467 : else
11468 : {
11469 : /* Enforced before the loop when !repeating_p. */
11470 368135 : unsigned int const_nunits = nunits.to_constant ();
11471 368135 : vec_index = i / const_nunits;
11472 368135 : mask_element = i % const_nunits;
11473 368135 : if (vec_index == first_vec_index
11474 368135 : || first_vec_index == -1)
11475 : {
11476 : first_vec_index = vec_index;
11477 : }
11478 146081 : else if (vec_index == second_vec_index
11479 146081 : || second_vec_index == -1)
11480 : {
11481 139545 : second_vec_index = vec_index;
11482 139545 : mask_element += const_nunits;
11483 : }
11484 : else
11485 : {
11486 6536 : if (dump_p)
11487 280 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11488 : "permutation requires at "
11489 : "least three vectors %G",
11490 : stmt_info->stmt);
11491 6536 : gcc_assert (analyze_only);
11492 : return false;
11493 : }
11494 :
11495 361599 : gcc_assert (mask_element < 2 * const_nunits);
11496 : }
11497 :
11498 578979 : if (mask_element != index)
11499 365456 : noop_p = false;
11500 578979 : mask[index++] = mask_element;
11501 :
11502 578979 : if (index == count)
11503 : {
11504 159586 : if (!noop_p)
11505 : {
11506 129297 : indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
11507 129297 : if (!can_vec_perm_const_p (mode, mode, indices))
11508 : {
11509 4528 : if (dump_p)
11510 : {
11511 90 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11512 : "unsupported vect permute { ");
11513 876 : for (i = 0; i < count; ++i)
11514 : {
11515 786 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11516 786 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11517 : }
11518 90 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11519 : }
11520 4528 : gcc_assert (analyze_only);
11521 : return false;
11522 : }
11523 :
11524 124769 : tree mask_vec = NULL_TREE;
11525 124769 : if (!analyze_only)
11526 20513 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11527 :
11528 124769 : if (second_vec_index == -1)
11529 36809 : second_vec_index = first_vec_index;
11530 :
11531 250363 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11532 : {
11533 125594 : ++*n_perms;
11534 125594 : if (analyze_only)
11535 104798 : continue;
11536 : /* Generate the permute statement if necessary. */
11537 20796 : tree first_vec = dr_chain[first_vec_index + ri];
11538 20796 : tree second_vec = dr_chain[second_vec_index + ri];
11539 20796 : gassign *stmt = as_a<gassign *> (stmt_info->stmt);
11540 20796 : tree perm_dest
11541 20796 : = vect_create_destination_var (gimple_assign_lhs (stmt),
11542 : vectype);
11543 20796 : perm_dest = make_ssa_name (perm_dest);
11544 20796 : gimple *perm_stmt
11545 20796 : = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
11546 : second_vec, mask_vec);
11547 20796 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
11548 : gsi);
11549 20796 : if (dce_chain)
11550 : {
11551 19873 : bitmap_set_bit (used_defs, first_vec_index + ri);
11552 19873 : bitmap_set_bit (used_defs, second_vec_index + ri);
11553 : }
11554 :
11555 : /* Store the vector statement in NODE. */
11556 20796 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
11557 : }
11558 : }
11559 30289 : else if (!analyze_only)
11560 : {
11561 2878 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11562 : {
11563 1439 : tree first_vec = dr_chain[first_vec_index + ri];
11564 : /* If mask was NULL_TREE generate the requested
11565 : identity transform. */
11566 1439 : if (dce_chain)
11567 1432 : bitmap_set_bit (used_defs, first_vec_index + ri);
11568 :
11569 : /* Store the vector statement in NODE. */
11570 1439 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
11571 : }
11572 : }
11573 :
11574 : index = 0;
11575 : first_vec_index = -1;
11576 : second_vec_index = -1;
11577 : noop_p = true;
11578 : }
11579 : }
11580 :
11581 123890 : if (n_loads)
11582 : {
11583 86741 : if (repeating_p)
11584 9334 : *n_loads = nstmts;
11585 : else
11586 : {
11587 : /* Enforced above when !repeating_p. */
11588 77407 : unsigned int const_nunits = nunits.to_constant ();
11589 77407 : *n_loads = 0;
11590 77407 : bool load_seen = false;
11591 1051979 : for (unsigned i = 0; i < in_nlanes; ++i)
11592 : {
11593 974572 : if (i % const_nunits == 0)
11594 : {
11595 420474 : if (load_seen)
11596 123491 : *n_loads += 1;
11597 : load_seen = false;
11598 : }
11599 974572 : if (bitmap_bit_p (used_in_lanes, i))
11600 265428 : load_seen = true;
11601 : }
11602 77407 : if (load_seen)
11603 50902 : *n_loads += 1;
11604 : }
11605 : }
11606 :
11607 123890 : if (dce_chain)
11608 208689 : for (unsigned i = 0; i < dr_chain.length (); ++i)
11609 73735 : if (!bitmap_bit_p (used_defs, i))
11610 : {
11611 40470 : tree def = dr_chain[i];
11612 40844 : do
11613 : {
11614 40844 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11615 40844 : if (is_gimple_assign (stmt)
11616 40844 : && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
11617 40844 : || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
11618 4987 : def = single_ssa_tree_operand (stmt, SSA_OP_USE);
11619 : else
11620 : def = NULL;
11621 40844 : gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
11622 40844 : gsi_remove (&rgsi, true);
11623 40844 : release_defs (stmt);
11624 : }
11625 40844 : while (def);
11626 : }
11627 :
11628 : return true;
11629 134954 : }
11630 :
11631 : /* Generate vector permute statements from a list of loads in DR_CHAIN.
11632 : If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
11633 : permute statements for the SLP node NODE. Store the number of vector
11634 : permute instructions in *N_PERMS and the number of vector load
11635 : instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions
11636 : that were not needed. */
11637 :
11638 : bool
11639 96600 : vect_transform_slp_perm_load (vec_info *vinfo,
11640 : slp_tree node, const vec<tree> &dr_chain,
11641 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11642 : bool analyze_only, unsigned *n_perms,
11643 : unsigned int *n_loads, bool dce_chain)
11644 : {
11645 96600 : return vect_transform_slp_perm_load_1 (vinfo, node,
11646 96600 : SLP_TREE_LOAD_PERMUTATION (node),
11647 : dr_chain, gsi, vf, analyze_only,
11648 : dump_enabled_p (), n_perms, n_loads,
11649 96600 : dce_chain);
11650 : }
11651 :
11652 : /* Produce the next vector result for SLP permutation NODE by adding a vector
11653 : statement at GSI. If MASK_VEC is nonnull, add:
11654 :
11655 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
11656 :
11657 : otherwise add:
11658 :
11659 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
11660 : { N, N+1, N+2, ... }>
11661 :
11662 : where N == IDENTITY_OFFSET which is either zero or equal to the
11663 : number of elements of the result. */
11664 :
11665 : static void
11666 32257 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11667 : slp_tree node, tree first_def, tree second_def,
11668 : tree mask_vec, poly_uint64 identity_offset)
11669 : {
11670 32257 : tree vectype = SLP_TREE_VECTYPE (node);
11671 :
11672 : /* ??? We SLP match existing vector element extracts but
11673 : allow punning which we need to re-instantiate at uses
11674 : but have no good way of explicitly representing. */
11675 32257 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
11676 32257 : && !types_compatible_p (TREE_TYPE (first_def), vectype))
11677 : {
11678 15 : gassign *conv_stmt
11679 15 : = gimple_build_assign (make_ssa_name (vectype),
11680 : build1 (VIEW_CONVERT_EXPR, vectype, first_def));
11681 15 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11682 15 : first_def = gimple_assign_lhs (conv_stmt);
11683 : }
11684 32257 : gassign *perm_stmt;
11685 32257 : if (mask_vec)
11686 : {
11687 28339 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
11688 28339 : TYPE_SIZE (vectype))
11689 28339 : && !types_compatible_p (TREE_TYPE (second_def), vectype))
11690 : {
11691 8 : gassign *conv_stmt
11692 8 : = gimple_build_assign (make_ssa_name (vectype),
11693 : build1 (VIEW_CONVERT_EXPR,
11694 : vectype, second_def));
11695 8 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11696 8 : second_def = gimple_assign_lhs (conv_stmt);
11697 : }
11698 28339 : tree perm_dest = make_ssa_name (vectype);
11699 28339 : perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
11700 : first_def, second_def,
11701 : mask_vec);
11702 : }
11703 : else
11704 : {
11705 3918 : auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
11706 3918 : unsigned HOST_WIDE_INT vecno;
11707 3918 : poly_uint64 eltno;
11708 3918 : if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
11709 : &vecno, &eltno))
11710 : gcc_unreachable ();
11711 3918 : tree def = vecno & 1 ? second_def : first_def;
11712 3918 : if (!types_compatible_p (TREE_TYPE (def), vectype))
11713 : {
11714 : /* For identity permutes we still need to handle the case
11715 : of offsetted extracts or concats. */
11716 370 : tree perm_dest = make_ssa_name (vectype);
11717 370 : unsigned HOST_WIDE_INT c;
11718 370 : if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
11719 : {
11720 366 : unsigned HOST_WIDE_INT elsz
11721 366 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
11722 732 : tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
11723 366 : TYPE_SIZE (vectype),
11724 366 : bitsize_int (eltno * elsz));
11725 366 : perm_stmt = gimple_build_assign (perm_dest, lowpart);
11726 : }
11727 4 : else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
11728 4 : def_nunits, &c) && c == 2)
11729 : {
11730 4 : gcc_assert (known_eq (identity_offset, 0U));
11731 4 : tree ctor = build_constructor_va (vectype, 2,
11732 : NULL_TREE, first_def,
11733 : NULL_TREE, second_def);
11734 4 : perm_stmt = gimple_build_assign (perm_dest, ctor);
11735 : }
11736 : else
11737 0 : gcc_unreachable ();
11738 : }
11739 : else
11740 : {
11741 3548 : gcc_assert (known_eq (eltno, 0U));
11742 3548 : node->push_vec_def (def);
11743 3548 : return;
11744 : }
11745 : }
11746 28709 : vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
11747 : /* Store the vector statement in NODE. */
11748 28709 : node->push_vec_def (perm_stmt);
11749 : }
11750 :
11751 : /* Subroutine of vectorizable_slp_permutation. Check whether the target
11752 : can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
11753 : If GSI is nonnull, emit the permutation there.
11754 :
11755 : When GSI is null, the only purpose of NODE is to give properties
11756 : of the result, such as the vector type and number of SLP lanes.
11757 : The node does not need to be a VEC_PERM_EXPR.
11758 :
11759 : If the target supports the operation, return the number of individual
11760 : VEC_PERM_EXPRs needed, otherwise return -1. Print information to the
11761 : dump file if DUMP_P is true. */
11762 :
11763 : static int
11764 473095 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
11765 : slp_tree node, lane_permutation_t &perm,
11766 : vec<slp_tree> &children, bool dump_p)
11767 : {
11768 473095 : tree vectype = SLP_TREE_VECTYPE (node);
11769 :
11770 : /* ??? We currently only support all same vector input types
11771 : while the SLP IL should really do a concat + select and thus accept
11772 : arbitrary mismatches. */
11773 473095 : slp_tree child;
11774 473095 : unsigned i;
11775 473095 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11776 473095 : bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
11777 : /* True if we're permuting a single input of 2N vectors down
11778 : to N vectors. This case doesn't generalize beyond 2 since
11779 : VEC_PERM_EXPR only takes 2 inputs. */
11780 473095 : bool pack_p = false;
11781 : /* If we're permuting inputs of N vectors each into X*N outputs,
11782 : this is the value of X, otherwise it is 1. */
11783 473095 : unsigned int unpack_factor = 1;
11784 473095 : tree op_vectype = NULL_TREE;
11785 474699 : FOR_EACH_VEC_ELT (children, i, child)
11786 474599 : if (SLP_TREE_VECTYPE (child))
11787 : {
11788 : op_vectype = SLP_TREE_VECTYPE (child);
11789 : break;
11790 : }
11791 473095 : if (!op_vectype)
11792 100 : op_vectype = vectype;
11793 996757 : FOR_EACH_VEC_ELT (children, i, child)
11794 : {
11795 523662 : if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
11796 18783 : && !vect_maybe_update_slp_op_vectype (child, op_vectype))
11797 523662 : || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
11798 1047324 : || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
11799 : {
11800 0 : if (dump_p)
11801 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11802 : "Unsupported vector types in lane permutation\n");
11803 473095 : return -1;
11804 : }
11805 523662 : auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
11806 523662 : unsigned int this_unpack_factor;
11807 : /* Detect permutations of external, pre-existing vectors. The external
11808 : node's SLP_TREE_LANES stores the total number of units in the vector,
11809 : or zero if the vector has variable length.
11810 :
11811 : We are expected to keep the original VEC_PERM_EXPR for such cases.
11812 : There is no repetition to model. */
11813 523662 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def
11814 523662 : && SLP_TREE_SCALAR_OPS (child).is_empty ())
11815 : repeating_p = false;
11816 : /* Check whether the input has twice as many lanes per vector. */
11817 507623 : else if (children.length () == 1
11818 507623 : && known_eq (SLP_TREE_LANES (child) * nunits,
11819 : SLP_TREE_LANES (node) * op_nunits * 2))
11820 : pack_p = true;
11821 : /* Check whether the output has N times as many lanes per vector. */
11822 523662 : else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
11823 463351 : SLP_TREE_LANES (child) * nunits,
11824 : &this_unpack_factor)
11825 428428 : && (i == 0 || unpack_factor == this_unpack_factor))
11826 : unpack_factor = this_unpack_factor;
11827 : else
11828 : repeating_p = false;
11829 : }
11830 :
11831 946190 : gcc_assert (perm.length () == SLP_TREE_LANES (node));
11832 :
11833 : /* Load-lanes permute. This permute only acts as a forwarder to
11834 : select the correct vector def of the load-lanes load which
11835 : has the permuted vectors in its vector defs like
11836 : { v0, w0, r0, v1, w1, r1 ... } for a ld3. All costs are
11837 : accounted for in the costing for the actual load so we
11838 : return zero here. */
11839 473095 : if (node->ldst_lanes)
11840 : {
11841 0 : gcc_assert (children.length () == 1);
11842 0 : if (!gsi)
11843 : /* This is a trivial op always supported. */
11844 : return 0;
11845 0 : slp_tree child = children[0];
11846 0 : unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
11847 0 : / SLP_TREE_LANES (node));
11848 0 : unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
11849 0 : unsigned nvectors = vect_get_num_copies (vinfo, node);
11850 0 : for (unsigned i = 0; i < nvectors; ++i)
11851 : {
11852 0 : tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num + vec_idx];
11853 0 : node->push_vec_def (def);
11854 : }
11855 : return 0;
11856 : }
11857 :
11858 : /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
11859 : and if we can generate the vectors in a vector-length agnostic way.
11860 : This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
11861 : compile time.
11862 :
11863 : The significance of UNPACK_STEP is that, when PACK_P is false,
11864 : output vector I operates on a window of UNPACK_STEP elements from each
11865 : input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR). For example,
11866 : when UNPACK_FACTOR is 2, the first output vector operates on lanes
11867 : [0, NUNITS / 2 - 1] of each input vector and the second output vector
11868 : operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
11869 :
11870 : When REPEATING_P is true, NOUTPUTS holds the total number of outputs
11871 : that we actually need to generate. */
11872 473095 : uint64_t noutputs = 0;
11873 473095 : poly_uint64 unpack_step = 0;
11874 473095 : loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
11875 210147 : if (!linfo
11876 512206 : || !multiple_p (nunits, unpack_factor, &unpack_step)
11877 209207 : || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
11878 209207 : * SLP_TREE_LANES (node), nunits, &noutputs))
11879 : repeating_p = false;
11880 :
11881 : /* We can handle the conditions described for REPEATING_P above for
11882 : both variable- and constant-length vectors. The fallback requires
11883 : us to generate every element of every permute vector explicitly,
11884 : which is only possible for constant-length permute vectors.
11885 :
11886 : Set:
11887 :
11888 : - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
11889 : mask vectors that we want to build.
11890 :
11891 : - NCOPIES to the number of copies of PERM that we need in order
11892 : to build the necessary permute mask vectors. */
11893 209207 : uint64_t npatterns;
11894 209207 : unsigned nelts_per_pattern;
11895 209207 : uint64_t ncopies;
11896 209207 : if (repeating_p)
11897 : {
11898 : /* We need permute mask vectors that have the form:
11899 :
11900 : { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
11901 :
11902 : In other words, the original n-element permute in PERM is
11903 : "unrolled" to fill a full vector. The stepped vector encoding
11904 : that we use for permutes requires 3n elements. */
11905 170096 : npatterns = SLP_TREE_LANES (node);
11906 170096 : nelts_per_pattern = ncopies = 3;
11907 : }
11908 : else
11909 : {
11910 : /* Calculate every element of every permute mask vector explicitly,
11911 : instead of relying on the pattern described above. */
11912 302999 : if (!nunits.is_constant (&npatterns)
11913 302999 : || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
11914 : {
11915 : if (dump_p)
11916 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11917 : "unsupported permutation %p on variable-length"
11918 : " vectors\n", (void *) node);
11919 : return -1;
11920 : }
11921 302999 : nelts_per_pattern = ncopies = 1;
11922 302999 : if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
11923 : {
11924 : if (dump_p)
11925 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11926 : "unsupported permutation %p for variable VF\n",
11927 : (void *) node);
11928 : return -1;
11929 : }
11930 : pack_p = false;
11931 : unpack_factor = 1;
11932 : }
11933 473095 : unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
11934 473095 : gcc_assert (repeating_p || multiple_p (olanes, nunits));
11935 :
11936 : /* Compute the { { SLP operand, vector index}, lane } permutation sequence
11937 : from the { SLP operand, scalar lane } permutation as recorded in the
11938 : SLP node as intermediate step. This part should already work
11939 : with SLP children with arbitrary number of lanes. */
11940 473095 : auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
11941 473095 : auto_vec<poly_uint64> active_lane;
11942 473095 : vperm.create (olanes);
11943 473095 : active_lane.safe_grow_cleared (children.length (), true);
11944 1427829 : for (unsigned int ui = 0; ui < unpack_factor; ++ui)
11945 : {
11946 2044730 : for (unsigned j = 0; j < children.length (); ++j)
11947 540726 : active_lane[j] = ui * unpack_step;
11948 1418310 : for (unsigned i = 0; i < ncopies; ++i)
11949 : {
11950 5887130 : for (unsigned pi = 0; pi < perm.length (); ++pi)
11951 : {
11952 2006894 : std::pair<unsigned, unsigned> p = perm[pi];
11953 2006894 : tree vtype = SLP_TREE_VECTYPE (children[p.first]);
11954 2006894 : if (repeating_p)
11955 999150 : vperm.quick_push ({{p.first, 0},
11956 999150 : p.second + active_lane[p.first]});
11957 : else
11958 : {
11959 : /* We checked above that the vectors are constant-length. */
11960 1007744 : unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
11961 1007744 : .to_constant ();
11962 1007744 : unsigned lane = active_lane[p.first].to_constant ();
11963 1007744 : unsigned vi = (lane + p.second) / vnunits;
11964 1007744 : unsigned vl = (lane + p.second) % vnunits;
11965 1007744 : vperm.quick_push ({{p.first, vi}, vl});
11966 : }
11967 : }
11968 : /* Advance to the next group. */
11969 2008896 : for (unsigned j = 0; j < children.length (); ++j)
11970 1072225 : active_lane[j] += SLP_TREE_LANES (children[j]);
11971 : }
11972 : }
11973 :
11974 473095 : if (dump_p)
11975 : {
11976 9035 : dump_printf_loc (MSG_NOTE, vect_location,
11977 : "vectorizing permutation %p", (void *)node);
11978 32694 : for (unsigned i = 0; i < perm.length (); ++i)
11979 23659 : dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
11980 9035 : if (repeating_p)
11981 7574 : dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
11982 9035 : dump_printf (MSG_NOTE, "\n");
11983 9035 : dump_printf_loc (MSG_NOTE, vect_location, "as");
11984 90774 : for (unsigned i = 0; i < vperm.length (); ++i)
11985 : {
11986 81739 : if (i != 0
11987 81739 : && (repeating_p
11988 55351 : ? multiple_p (i, npatterns)
11989 81739 : : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
11990 24385 : dump_printf (MSG_NOTE, ",");
11991 81739 : dump_printf (MSG_NOTE, " vops%u[%u][",
11992 81739 : vperm[i].first.first, vperm[i].first.second);
11993 81739 : dump_dec (MSG_NOTE, vperm[i].second);
11994 81739 : dump_printf (MSG_NOTE, "]");
11995 : }
11996 9035 : dump_printf (MSG_NOTE, "\n");
11997 : }
11998 :
11999 : /* We can only handle two-vector permutes, everything else should
12000 : be lowered on the SLP level. The following is closely inspired
12001 : by vect_transform_slp_perm_load and is supposed to eventually
12002 : replace it.
12003 : ??? As intermediate step do code-gen in the SLP tree representation
12004 : somehow? */
12005 473095 : std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
12006 473095 : std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
12007 473095 : unsigned int index = 0;
12008 473095 : poly_uint64 mask_element;
12009 473095 : vec_perm_builder mask;
12010 473095 : mask.new_vector (nunits, npatterns, nelts_per_pattern);
12011 473095 : unsigned int count = mask.encoded_nelts ();
12012 473095 : mask.quick_grow (count);
12013 473095 : vec_perm_indices indices;
12014 473095 : unsigned nperms = 0;
12015 : /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
12016 : vectors to check during analysis, but we need to generate NOUTPUTS
12017 : vectors during transformation. */
12018 473095 : unsigned total_nelts = olanes;
12019 473095 : unsigned process_nelts = olanes;
12020 473095 : if (repeating_p)
12021 : {
12022 170096 : total_nelts = (total_nelts / unpack_factor) * noutputs;
12023 170096 : if (gsi)
12024 9986 : process_nelts = total_nelts;
12025 : }
12026 473095 : unsigned last_ei = (total_nelts - 1) % process_nelts;
12027 2484307 : for (unsigned i = 0; i < process_nelts; ++i)
12028 : {
12029 : /* VI is the input vector index when generating code for REPEATING_P. */
12030 2021407 : unsigned vi = i / olanes * (pack_p ? 2 : 1);
12031 2021407 : unsigned ei = i % olanes;
12032 2021407 : mask_element = vperm[ei].second;
12033 2021407 : if (pack_p)
12034 : {
12035 : /* In this case, we have N outputs and the single child provides 2N
12036 : inputs. Output X permutes inputs 2X and 2X+1.
12037 :
12038 : The mask indices are taken directly from the SLP permutation node.
12039 : Index X selects from the first vector if (X / NUNITS) % 2 == 0;
12040 : X selects from the second vector otherwise. These conditions
12041 : are only known at compile time for constant-length vectors. */
12042 : first_vec = std::make_pair (0, 0);
12043 : second_vec = std::make_pair (0, 1);
12044 : }
12045 1850560 : else if (first_vec.first == -1U
12046 1850560 : || first_vec == vperm[ei].first)
12047 1636454 : first_vec = vperm[ei].first;
12048 214106 : else if (second_vec.first == -1U
12049 214106 : || second_vec == vperm[ei].first)
12050 : {
12051 213698 : second_vec = vperm[ei].first;
12052 213698 : mask_element += nunits;
12053 : }
12054 : else
12055 : {
12056 408 : if (dump_p)
12057 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12058 : "permutation requires at "
12059 : "least three vectors\n");
12060 408 : gcc_assert (!gsi);
12061 : return -1;
12062 : }
12063 :
12064 2020999 : mask[index++] = mask_element;
12065 :
12066 2020999 : if (index == count)
12067 : {
12068 617779 : indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
12069 : TYPE_VECTOR_SUBPARTS (op_vectype));
12070 617779 : bool identity_p = (indices.series_p (0, 1, mask[0], 1)
12071 956411 : && constant_multiple_p (mask[0], nunits));
12072 617779 : machine_mode vmode = TYPE_MODE (vectype);
12073 617779 : machine_mode op_vmode = TYPE_MODE (op_vectype);
12074 617779 : unsigned HOST_WIDE_INT c;
12075 617779 : if ((!identity_p
12076 568386 : && !can_vec_perm_const_p (vmode, op_vmode, indices))
12077 617779 : || (identity_p
12078 49393 : && !known_le (nunits,
12079 : TYPE_VECTOR_SUBPARTS (op_vectype))
12080 8 : && (!constant_multiple_p (nunits,
12081 9795 : TYPE_VECTOR_SUBPARTS (op_vectype),
12082 8 : &c) || c != 2)))
12083 : {
12084 9787 : if (dump_p)
12085 : {
12086 154 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
12087 : vect_location,
12088 : "unsupported vect permute { ");
12089 1596 : for (i = 0; i < count; ++i)
12090 : {
12091 1442 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
12092 1442 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
12093 : }
12094 154 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
12095 : }
12096 9787 : gcc_assert (!gsi);
12097 473095 : return -1;
12098 : }
12099 :
12100 607992 : if (!identity_p)
12101 558599 : nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
12102 607992 : if (gsi)
12103 : {
12104 32257 : if (second_vec.first == -1U)
12105 7760 : second_vec = first_vec;
12106 :
12107 32257 : slp_tree
12108 32257 : first_node = children[first_vec.first],
12109 32257 : second_node = children[second_vec.first];
12110 :
12111 32257 : tree mask_vec = NULL_TREE;
12112 32257 : if (!identity_p)
12113 28339 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
12114 :
12115 32257 : tree first_def
12116 32257 : = vect_get_slp_vect_def (first_node, first_vec.second + vi);
12117 32257 : tree second_def
12118 32257 : = vect_get_slp_vect_def (second_node, second_vec.second + vi);
12119 32257 : vect_add_slp_permutation (vinfo, gsi, node, first_def,
12120 32257 : second_def, mask_vec, mask[0]);
12121 : }
12122 :
12123 : index = 0;
12124 : first_vec = std::make_pair (-1U, -1U);
12125 : second_vec = std::make_pair (-1U, -1U);
12126 : }
12127 : }
12128 :
12129 462900 : return nperms;
12130 473095 : }
12131 :
12132 : /* Vectorize the SLP permutations in NODE as specified
12133 : in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
12134 : child number and lane number.
12135 : Interleaving of two two-lane two-child SLP subtrees (not supported):
12136 : [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
12137 : A blend of two four-lane two-child SLP subtrees:
12138 : [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
12139 : Highpart of a four-lane one-child SLP subtree (not supported):
12140 : [ { 0, 2 }, { 0, 3 } ]
12141 : Where currently only a subset is supported by code generating below. */
12142 :
12143 : bool
12144 100075 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
12145 : slp_tree node, stmt_vector_for_cost *cost_vec)
12146 : {
12147 100075 : tree vectype = SLP_TREE_VECTYPE (node);
12148 100075 : lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
12149 100075 : int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
12150 100075 : SLP_TREE_CHILDREN (node),
12151 : dump_enabled_p ());
12152 100075 : if (nperms < 0)
12153 : return false;
12154 :
12155 98454 : if (!gsi && nperms != 0)
12156 74610 : record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
12157 :
12158 : return true;
12159 : }
12160 :
12161 : /* Vectorize SLP NODE. Only compute the vector insertion places when
12162 : PLACE_ONLY is true. When placing, return false if there is no possible
12163 : schedule. */
12164 :
12165 : static bool
12166 3144981 : vect_schedule_slp_node (vec_info *vinfo,
12167 : slp_tree node, slp_instance instance, bool place_only)
12168 : {
12169 3144981 : int i;
12170 3144981 : slp_tree child;
12171 :
12172 : /* Vectorize externals and constants. */
12173 3144981 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
12174 3144981 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
12175 : {
12176 1215091 : if (place_only)
12177 : {
12178 703498 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
12179 : return true;
12180 320421 : gimple *last_stmt = NULL;
12181 320421 : vec<tree> &defs = (!SLP_TREE_SCALAR_OPS (node).is_empty ()
12182 : ? SLP_TREE_SCALAR_OPS (node)
12183 320421 : : SLP_TREE_VEC_DEFS (node));
12184 1715528 : for (tree def : defs)
12185 : /* If the stmt is not inside the region do not
12186 : use it as possible insertion point. */
12187 754265 : if (auto stmt_info = vinfo->lookup_def (def))
12188 : {
12189 393034 : gimple *stmt = stmt_info->stmt;
12190 393034 : if (!last_stmt)
12191 : last_stmt = stmt;
12192 221441 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12193 : last_stmt = stmt;
12194 67639 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12195 : ;
12196 : else
12197 0 : gcc_unreachable ();
12198 : }
12199 320421 : node->si = last_stmt;
12200 320421 : return true;
12201 : }
12202 :
12203 : /* ??? vectorizable_shift can end up using a scalar operand which is
12204 : currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
12205 : node in this case. */
12206 511593 : if (!SLP_TREE_VECTYPE (node))
12207 : return true;
12208 :
12209 : /* There are two reasons vector defs might already exist. The first
12210 : is that we are vectorizing an existing vector def. The second is
12211 : when performing BB vectorization shared constant/external nodes
12212 : are not split apart during partitioning so during the code-gen
12213 : DFS walk we can end up visiting them twice. */
12214 503916 : if (! SLP_TREE_VEC_DEFS (node).exists ())
12215 503059 : vect_create_constant_vectors (vinfo, node);
12216 : return true;
12217 : }
12218 :
12219 1929890 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
12220 :
12221 1929890 : gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
12222 1929890 : if (!place_only && SLP_TREE_VECTYPE (node))
12223 976988 : SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
12224 :
12225 1929890 : gimple *last_stmt;
12226 1929890 : gimple_stmt_iterator si;
12227 : /* ??? When !place_only we'd like to re-use place_only computed info,
12228 : but this is a bit awkward due to using gsi_insert_before and the
12229 : requirement to insert after vector defs. So we compute last_stmt
12230 : during pre-scheduling and si during scheduling. */
12231 1929890 : if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
12232 : {
12233 : /* Vectorized loads go before the first scalar load to make it
12234 : ready early, vectorized stores go before the last scalar
12235 : stmt which is where all uses are ready.
12236 : In theory, if we delay dependence checking until after
12237 : placing, we can schedule at other points, but then
12238 : dependence checking would need to honor that. On the
12239 : other hand dependence checking could request a different
12240 : scheduling point as well, if dependences require that. */
12241 1494206 : stmt_vec_info last_stmt_info = NULL;
12242 1494206 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
12243 276382 : last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
12244 : else /* DR_IS_WRITE */
12245 : {
12246 1217824 : last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
12247 1217824 : if (place_only)
12248 1334620 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12249 : {
12250 667312 : if (child->si
12251 667312 : && !vect_stmt_dominates_stmt_p (child->si,
12252 : last_stmt_info->stmt))
12253 : return false;
12254 : }
12255 : }
12256 1494202 : last_stmt = last_stmt_info->stmt;
12257 1494202 : si = gsi_for_stmt (last_stmt);
12258 1494202 : }
12259 435684 : else if (!SLP_TREE_PERMUTE_P (node)
12260 413950 : && (SLP_TREE_TYPE (node) == cycle_phi_info_type
12261 : || SLP_TREE_TYPE (node) == induc_vec_info_type
12262 : || SLP_TREE_TYPE (node) == phi_info_type))
12263 : {
12264 : /* For PHI node vectorization we do not use the insertion iterator. */
12265 107322 : last_stmt = SLP_TREE_SCALAR_STMTS (node)[0]->stmt;
12266 107322 : if (place_only)
12267 184225 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12268 : {
12269 130306 : if (child->si
12270 227279 : && !dominated_by_p (CDI_DOMINATORS,
12271 : gimple_phi_arg_edge
12272 96973 : (as_a <gphi *> (last_stmt), i)->src,
12273 96973 : gimple_bb (child->si)))
12274 : return false;
12275 : }
12276 107320 : si = gsi_none ();
12277 : }
12278 : else
12279 : {
12280 : /* Emit other stmts after the children vectorized defs which is
12281 : earliest possible. */
12282 : last_stmt = NULL;
12283 910051 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12284 581691 : if (place_only)
12285 : {
12286 197724 : gimple *vstmt = child->si;
12287 197724 : if (!vstmt)
12288 : {
12289 : /* vect_constant_def and defs at region boundary do not
12290 : constrain placement. */
12291 : gcc_assert (SLP_TREE_DEF_TYPE (child) == vect_constant_def
12292 : /* ??? Region boundary is not representated
12293 : by a NULL stmt. */
12294 : || true);
12295 : }
12296 154577 : else if (!last_stmt)
12297 : last_stmt = vstmt;
12298 44667 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12299 : last_stmt = vstmt;
12300 11194 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12301 : ;
12302 : else
12303 : /* Non-trapping stmts from different BBs might be combined,
12304 : and if we later CSE a low/high part we can run into this. */
12305 : return false;
12306 : }
12307 383967 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
12308 : {
12309 : /* For fold-left reductions we are retaining the scalar
12310 : reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
12311 : set so the representation isn't perfect. Resort to the
12312 : last scalar def here. */
12313 306147 : if (SLP_TREE_VEC_DEFS (child).is_empty ())
12314 : {
12315 1006 : gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
12316 1006 : gphi *phi = as_a <gphi *>
12317 1006 : (vect_find_last_scalar_stmt_in_slp (child)->stmt);
12318 1006 : if (!last_stmt)
12319 : last_stmt = phi;
12320 756 : else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
12321 : last_stmt = phi;
12322 745 : else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
12323 : ;
12324 : else
12325 0 : gcc_unreachable ();
12326 : }
12327 : /* We are emitting all vectorized stmts in the same place and
12328 : the last one is the last.
12329 : ??? Unless we have a load permutation applied and that
12330 : figures to re-use an earlier generated load. */
12331 306147 : unsigned j;
12332 306147 : tree vdef;
12333 720754 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12334 414607 : if (TREE_CODE (vdef) == SSA_NAME
12335 414607 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12336 : {
12337 414555 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12338 : /* Ignore existing external vector defs so we insert
12339 : at region boundary. Those can leak to an
12340 : internal identity permute node. */
12341 414555 : if (is_a <bb_vec_info> (vinfo)
12342 30568 : && vinfo->bbs[0] != gimple_bb (vstmt)
12343 435145 : && dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
12344 20590 : gimple_bb (vstmt)))
12345 : ;
12346 414541 : else if (!last_stmt)
12347 : last_stmt = vstmt;
12348 210624 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12349 : last_stmt = vstmt;
12350 47964 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12351 : ;
12352 : else
12353 0 : gcc_unreachable ();
12354 : }
12355 : }
12356 77820 : else if (!SLP_TREE_VECTYPE (child))
12357 : {
12358 : /* For externals we use unvectorized at all scalar defs. */
12359 : unsigned j;
12360 : tree def;
12361 16524 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
12362 : /* If the stmt is not inside the region do not
12363 : use it as possible insertion point. */
12364 9477 : if (auto stmt_info = vinfo->lookup_def (def))
12365 : {
12366 138 : gimple *stmt = stmt_info->stmt;
12367 138 : if (!last_stmt)
12368 : last_stmt = stmt;
12369 118 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12370 : last_stmt = stmt;
12371 26 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12372 : ;
12373 : else
12374 0 : gcc_unreachable ();
12375 : }
12376 : }
12377 : else
12378 : {
12379 : /* For externals we have to look at all defs since their
12380 : insertion place is decided per vector. But beware
12381 : of pre-existing vectors where we need to make sure
12382 : we do not insert before the region boundary. */
12383 141403 : if (SLP_TREE_SCALAR_OPS (child).is_empty ()
12384 620 : && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
12385 : ;
12386 : else
12387 : {
12388 : unsigned j;
12389 : tree vdef;
12390 750811 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12391 98492 : if (TREE_CODE (vdef) == SSA_NAME
12392 98492 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12393 : {
12394 20811 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12395 20811 : if (!last_stmt)
12396 : last_stmt = vstmt;
12397 11356 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12398 : last_stmt = vstmt;
12399 9014 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12400 : ;
12401 : else
12402 0 : gcc_unreachable ();
12403 : }
12404 : }
12405 : }
12406 :
12407 : /* We split regions to vectorize at control altering stmts
12408 : with a definition so this can only be an external. */
12409 328360 : gcc_checking_assert (!last_stmt
12410 : || !is_ctrl_altering_stmt (last_stmt));
12411 :
12412 328360 : if (is_a <bb_vec_info> (vinfo)
12413 133212 : && !SLP_TREE_PERMUTE_P (node)
12414 126711 : && (!last_stmt
12415 123811 : || !dominated_by_p (CDI_DOMINATORS, gimple_bb (last_stmt),
12416 123811 : gimple_bb (stmt_info->stmt)))
12417 353944 : && gimple_could_trap_p (stmt_info->stmt))
12418 : {
12419 : /* We've constrained possibly trapping operations to all come
12420 : from the same basic-block, if vectorized defs would allow earlier
12421 : scheduling still force vectorized stmts to the original block.
12422 : This is only necessary for BB vectorization since for loop vect
12423 : all operations are in a single BB and scalar stmt based
12424 : placement doesn't play well with epilogue vectorization. */
12425 686 : if (last_stmt
12426 1272 : && !dominated_by_p (CDI_DOMINATORS,
12427 586 : gimple_bb (stmt_info->stmt),
12428 586 : gimple_bb (last_stmt)))
12429 : {
12430 4 : gcc_assert (place_only);
12431 : return false;
12432 : }
12433 682 : si = gsi_after_labels (gimple_bb (stmt_info->stmt));
12434 682 : last_stmt = gsi_stmt (si);
12435 : }
12436 : /* When there is no in-region child def to guide placement, insert
12437 : at region boundary. */
12438 327674 : else if (!last_stmt)
12439 : {
12440 4710 : si = gsi_after_labels (vinfo->bbs[0]);
12441 : /* last_stmt NULL marks the region start. */
12442 : }
12443 322964 : else if (is_a <gphi *> (last_stmt))
12444 31939 : si = gsi_after_labels (gimple_bb (last_stmt));
12445 : else
12446 : {
12447 291025 : si = gsi_for_stmt (last_stmt);
12448 : /* We use gsi_insert_before, so when last_stmt is a vector
12449 : def we have to advance (or use gsi_insert_after). */
12450 291025 : gsi_next (&si);
12451 :
12452 291025 : if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
12453 : {
12454 : /* We'll have to fix this up for loop vect. */
12455 178761 : gcc_assert (!place_only);
12456 : /* Avoid scheduling stmts to random places in the CFG, any
12457 : stmt dominance check we performed is possibly wrong as UIDs
12458 : are not initialized for all of the function for loop
12459 : vectorization. Instead append to the loop preheader. */
12460 178761 : if ((LOOP_VINFO_LOOP (loop_vinfo)->header
12461 178761 : != gimple_bb (last_stmt))
12462 181664 : && dominated_by_p (CDI_DOMINATORS,
12463 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12464 2903 : gimple_bb (last_stmt)))
12465 1120 : si = gsi_end_bb (loop_preheader_edge
12466 560 : (LOOP_VINFO_LOOP (loop_vinfo))->src);
12467 : /* Avoid scheduling internal defs outside of the loop when
12468 : we might have only implicitly tracked loop mask/len defs. */
12469 84 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
12470 178761 : || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12471 : {
12472 84 : gimple_stmt_iterator si2
12473 84 : = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
12474 84 : if ((gsi_end_p (si2)
12475 0 : && (LOOP_VINFO_LOOP (loop_vinfo)->header
12476 0 : != gimple_bb (last_stmt))
12477 0 : && dominated_by_p (CDI_DOMINATORS,
12478 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12479 0 : gimple_bb (last_stmt)))
12480 84 : || (!gsi_end_p (si2)
12481 84 : && last_stmt != *si2
12482 79 : && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
12483 3 : si = si2;
12484 : }
12485 : }
12486 : }
12487 : }
12488 :
12489 1929878 : if (place_only)
12490 : {
12491 952884 : if (dump_enabled_p () && last_stmt)
12492 4016 : dump_printf_loc (MSG_NOTE, vect_location,
12493 : "placing node %p at %G:", (void *)node, last_stmt);
12494 : /* Verify we either get a stmt anchor or region start. */
12495 952884 : gcc_assert ((last_stmt && gimple_bb (last_stmt))
12496 : || (!last_stmt && gsi_bb (si)));
12497 952884 : node->si = last_stmt;
12498 952884 : return true;
12499 : }
12500 :
12501 : /* ??? Asserting vect_stmt_dominates_stmt_p (gsi_stmt (si), node->si)
12502 : does not work because in some cases we advance si from last_stmt (as
12503 : we want to insert after vector stmts) and because vector stmts of
12504 : children have been inserted possibly at the same location constraint,
12505 : moving si even further. */
12506 976994 : if (flag_checking && node->si && gimple_bb (node->si) && !gsi_end_p (si))
12507 : {
12508 602534 : auto gsi2 = si;
12509 602534 : while (1)
12510 : {
12511 602534 : if (vect_stmt_dominates_stmt_p (gsi_stmt (gsi2), node->si))
12512 : break;
12513 : /* As we have possibly advanced si it might now point to the
12514 : scalar stmt immediately following node->si. That's OK. */
12515 23830 : if (gsi_stmt (gsi2) != gsi_stmt (si)
12516 23830 : && gimple_uid (gsi_stmt (gsi2)) != 0)
12517 0 : gcc_unreachable ();
12518 23830 : gsi_prev (&gsi2);
12519 23830 : if (gsi_end_p (gsi2))
12520 : {
12521 545 : if (is_a <gphi *> (node->si)
12522 545 : && gimple_bb (node->si) == gsi_bb (si))
12523 : break;
12524 0 : gcc_unreachable ();
12525 : }
12526 : }
12527 : }
12528 :
12529 976994 : if (dump_enabled_p ())
12530 : {
12531 75104 : if (stmt_info)
12532 71682 : dump_printf_loc (MSG_NOTE, vect_location,
12533 : "------>vectorizing SLP node starting from: %G",
12534 : stmt_info->stmt);
12535 : else
12536 : {
12537 3422 : dump_printf_loc (MSG_NOTE, vect_location,
12538 : "------>vectorizing SLP node:\n");
12539 3422 : vect_print_slp_tree (MSG_NOTE, vect_location, node);
12540 : }
12541 : }
12542 976994 : vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
12543 976994 : return true;
12544 : }
12545 :
12546 : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
12547 : For loop vectorization this is done in vectorizable_call, but for SLP
12548 : it needs to be deferred until end of vect_schedule_slp, because multiple
12549 : SLP instances may refer to the same scalar stmt. */
12550 :
12551 : static void
12552 619861 : vect_remove_slp_scalar_calls (vec_info *vinfo,
12553 : slp_tree node, hash_set<slp_tree> &visited)
12554 : {
12555 619861 : gimple *new_stmt;
12556 619861 : gimple_stmt_iterator gsi;
12557 619861 : tree lhs;
12558 :
12559 619861 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12560 194780 : return;
12561 :
12562 469439 : if (visited.add (node))
12563 : return;
12564 :
12565 1586645 : for (auto child : SLP_TREE_CHILDREN (node))
12566 526990 : vect_remove_slp_scalar_calls (vinfo, child, visited);
12567 :
12568 1765540 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
12569 : {
12570 498941 : if (!stmt_info)
12571 3763 : continue;
12572 495178 : stmt_info = vect_orig_stmt (stmt_info);
12573 495178 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
12574 5442 : if (!stmt || gimple_bb (stmt) == NULL)
12575 489778 : continue;
12576 5400 : lhs = gimple_call_lhs (stmt);
12577 5400 : if (lhs)
12578 4816 : new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
12579 : else
12580 584 : new_stmt = gimple_build_nop ();
12581 5400 : unlink_stmt_vdef (stmt_info->stmt);
12582 5400 : gsi = gsi_for_stmt (stmt);
12583 5400 : vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
12584 5400 : if (lhs)
12585 4816 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12586 : }
12587 : }
12588 :
12589 : static void
12590 92871 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
12591 : {
12592 92871 : hash_set<slp_tree> visited;
12593 92871 : vect_remove_slp_scalar_calls (vinfo, node, visited);
12594 92871 : }
12595 :
12596 : /* Vectorize the instance root. */
12597 :
12598 : void
12599 9614 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
12600 : {
12601 9614 : if (instance->kind == slp_inst_kind_ctor)
12602 : {
12603 5724 : tree new_def;
12604 5724 : if (SLP_TREE_VEC_DEFS (node).length () == 1)
12605 : {
12606 5678 : new_def = SLP_TREE_VEC_DEFS (node)[0];
12607 5678 : tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12608 5678 : if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
12609 5678 : TREE_TYPE (new_def)))
12610 0 : new_def = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
12611 : new_def);
12612 : }
12613 : else
12614 : {
12615 46 : gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
12616 46 : tree child_def;
12617 46 : int j;
12618 46 : vec<constructor_elt, va_gc> *v;
12619 46 : vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
12620 :
12621 : /* A CTOR can handle V16HI composition from VNx8HI so we
12622 : do not need to convert vector elements if the types
12623 : do not match. */
12624 186 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
12625 94 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
12626 46 : tree rtype
12627 46 : = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
12628 46 : new_def = build_constructor (rtype, v);
12629 : }
12630 :
12631 5724 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12632 5724 : gimple_assign_set_rhs_from_tree (&rgsi, new_def);
12633 5724 : update_stmt (gsi_stmt (rgsi));
12634 5724 : return;
12635 : }
12636 3890 : else if (instance->kind == slp_inst_kind_bb_reduc)
12637 : {
12638 : /* Largely inspired by reduction chain epilogue handling in
12639 : vect_create_epilog_for_reduction. */
12640 2233 : vec<tree> vec_defs = vNULL;
12641 2233 : vect_get_slp_defs (node, &vec_defs);
12642 2233 : enum tree_code reduc_code
12643 2233 : = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
12644 : /* ??? We actually have to reflect signs somewhere. */
12645 2233 : if (reduc_code == MINUS_EXPR)
12646 0 : reduc_code = PLUS_EXPR;
12647 2233 : gimple_seq epilogue = NULL;
12648 : /* We may end up with more than one vector result, reduce them
12649 : to one vector. */
12650 2233 : tree vec_def = vec_defs[0];
12651 2233 : tree vectype = TREE_TYPE (vec_def);
12652 2233 : tree compute_vectype = vectype;
12653 2233 : bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
12654 1071 : && TYPE_OVERFLOW_UNDEFINED (vectype)
12655 2917 : && operation_can_overflow (reduc_code));
12656 495 : if (pun_for_overflow_p)
12657 : {
12658 495 : compute_vectype = unsigned_type_for (vectype);
12659 495 : vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12660 : compute_vectype, vec_def);
12661 : }
12662 2500 : for (unsigned i = 1; i < vec_defs.length (); ++i)
12663 : {
12664 267 : tree def = vec_defs[i];
12665 267 : if (pun_for_overflow_p)
12666 41 : def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12667 : compute_vectype, def);
12668 267 : vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
12669 : vec_def, def);
12670 : }
12671 2233 : vec_defs.release ();
12672 : /* ??? Support other schemes than direct internal fn or two
12673 : element vectors. */
12674 2233 : tree scalar_def;
12675 2233 : internal_fn reduc_fn;
12676 3225 : if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
12677 2233 : || reduc_fn == IFN_LAST
12678 4466 : || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
12679 : OPTIMIZE_FOR_BOTH))
12680 : {
12681 992 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
12682 992 : tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
12683 992 : TREE_TYPE (compute_vectype), vec_def,
12684 992 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12685 992 : bitsize_zero_node);
12686 992 : tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
12687 992 : TREE_TYPE (compute_vectype), vec_def,
12688 992 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12689 992 : TYPE_SIZE (TREE_TYPE (compute_vectype)));
12690 992 : scalar_def = gimple_build (&epilogue, reduc_code,
12691 992 : TREE_TYPE (compute_vectype), tem0, tem1);
12692 : }
12693 : else
12694 1241 : scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
12695 1241 : TREE_TYPE (compute_vectype), vec_def);
12696 2233 : if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
12697 : {
12698 917 : tree rem_def = NULL_TREE;
12699 2121 : for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
12700 : {
12701 1204 : def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
12702 1204 : if (!rem_def)
12703 : rem_def = def;
12704 : else
12705 287 : rem_def = gimple_build (&epilogue, reduc_code,
12706 287 : TREE_TYPE (scalar_def),
12707 : rem_def, def);
12708 : }
12709 917 : scalar_def = gimple_build (&epilogue, reduc_code,
12710 917 : TREE_TYPE (scalar_def),
12711 : scalar_def, rem_def);
12712 : }
12713 2233 : scalar_def = gimple_convert (&epilogue,
12714 2233 : TREE_TYPE (vectype), scalar_def);
12715 2233 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12716 2233 : gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
12717 2233 : gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
12718 2233 : update_stmt (gsi_stmt (rgsi));
12719 2233 : return;
12720 : }
12721 1657 : else if (instance->kind == slp_inst_kind_gcond)
12722 : {
12723 : /* Only support a single root for now as we can't codegen CFG yet and so we
12724 : can't support lane > 1 at this time. */
12725 1657 : gcc_assert (instance->root_stmts.length () == 1);
12726 1657 : auto root_stmt_info = instance->root_stmts[0];
12727 1657 : auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
12728 1657 : gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
12729 1657 : gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
12730 1657 : bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
12731 : root_stmt_info, &rgsi, node, NULL);
12732 1657 : gcc_assert (res);
12733 1657 : return;
12734 : }
12735 : else
12736 0 : gcc_unreachable ();
12737 : }
12738 :
12739 : struct slp_scc_info
12740 : {
12741 : bool on_stack;
12742 : bool res;
12743 : int dfs;
12744 : int lowlink;
12745 : };
12746 :
12747 : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs.
12748 : When PLACE_ONLY, return false if there is no possible schedule. */
12749 :
12750 : static bool
12751 3144981 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
12752 : hash_map<slp_tree, slp_scc_info> &scc_info,
12753 : int &maxdfs, vec<slp_tree> &stack, bool place_only)
12754 : {
12755 3144981 : bool existed_p;
12756 3144981 : slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
12757 3144981 : gcc_assert (!existed_p);
12758 3144981 : info->dfs = maxdfs;
12759 3144981 : info->lowlink = maxdfs;
12760 3144981 : maxdfs++;
12761 :
12762 : /* Leaf. */
12763 3144981 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12764 : {
12765 1215091 : info->on_stack = false;
12766 1215091 : bool res = vect_schedule_slp_node (vinfo, node, instance, place_only);
12767 1215091 : gcc_assert (res);
12768 1215091 : info->res = res;
12769 1215091 : return res;
12770 : }
12771 :
12772 1929890 : info->on_stack = true;
12773 1929890 : info->res = true;
12774 1929890 : stack.safe_push (node);
12775 :
12776 1929890 : bool res = true;
12777 1929890 : unsigned i;
12778 1929890 : slp_tree child;
12779 : /* DFS recurse. */
12780 7559390 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12781 : {
12782 2046412 : if (!child)
12783 56824 : continue;
12784 1989588 : slp_scc_info *child_info = scc_info.get (child);
12785 1989588 : if (!child_info)
12786 : {
12787 1840786 : res &= vect_schedule_scc (vinfo, child, instance, scc_info,
12788 : maxdfs, stack, place_only);
12789 : /* Recursion might have re-allocated the node. */
12790 1840786 : info = scc_info.get (node);
12791 1840786 : child_info = scc_info.get (child);
12792 1840786 : info->lowlink = MIN (info->lowlink, child_info->lowlink);
12793 : }
12794 148802 : else if (child_info->on_stack)
12795 35573 : info->lowlink = MIN (info->lowlink, child_info->dfs);
12796 : else
12797 113229 : res &= child_info->res;
12798 : }
12799 1929890 : if (info->lowlink != info->dfs)
12800 : return res;
12801 :
12802 1890455 : auto_vec<slp_tree, 4> phis_to_fixup;
12803 :
12804 : /* Singleton. */
12805 1890455 : if (stack.last () == node)
12806 : {
12807 1861494 : stack.pop ();
12808 1861494 : info->on_stack = false;
12809 1861494 : res &= vect_schedule_slp_node (vinfo, node, instance, place_only);
12810 1861494 : info->res = res;
12811 1861494 : if (!SLP_TREE_PERMUTE_P (node)
12812 1861494 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
12813 72429 : phis_to_fixup.quick_push (node);
12814 : }
12815 : else
12816 : {
12817 : /* SCC. */
12818 28961 : int last_idx = stack.length () - 1;
12819 68396 : while (stack[last_idx] != node)
12820 39435 : last_idx--;
12821 : /* We can break the cycle at PHIs who have at least one child
12822 : code generated. Then we could re-start the DFS walk until
12823 : all nodes in the SCC are covered (we might have new entries
12824 : for only back-reachable nodes). But it's simpler to just
12825 : iterate and schedule those that are ready. */
12826 28961 : unsigned todo = stack.length () - last_idx;
12827 28961 : auto_vec<slp_tree, 4> saved_scc (todo);
12828 28961 : saved_scc.quick_grow (todo);
12829 86883 : memcpy (saved_scc.address (), &stack.address ()[last_idx],
12830 : sizeof (slp_tree) * todo);
12831 29757 : do
12832 : {
12833 132073 : for (int idx = stack.length () - 1; idx >= last_idx; --idx)
12834 : {
12835 72559 : slp_tree entry = stack[idx];
12836 72559 : if (!entry)
12837 2531 : continue;
12838 70028 : bool phi = (!SLP_TREE_PERMUTE_P (entry)
12839 70028 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
12840 70028 : bool ready = !phi;
12841 213019 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
12842 132853 : if (!child)
12843 : {
12844 23009 : gcc_assert (phi);
12845 : ready = true;
12846 : break;
12847 : }
12848 109844 : else if (scc_info.get (child)->on_stack)
12849 : {
12850 30801 : if (!phi)
12851 : {
12852 : ready = false;
12853 : break;
12854 : }
12855 : }
12856 : else
12857 : {
12858 79043 : if (phi)
12859 : {
12860 : ready = true;
12861 : break;
12862 : }
12863 : }
12864 47019 : if (ready)
12865 : {
12866 68396 : res &= vect_schedule_slp_node (vinfo, entry, instance,
12867 : place_only);
12868 68396 : scc_info.get (entry)->on_stack = false;
12869 68396 : stack[idx] = NULL;
12870 68396 : todo--;
12871 68396 : if (phi)
12872 35425 : phis_to_fixup.safe_push (entry);
12873 : }
12874 : }
12875 : }
12876 29757 : while (todo != 0);
12877 :
12878 : /* Push the scheduling result to all of the SCC. */
12879 155279 : for (slp_tree entry : saved_scc)
12880 68396 : scc_info.get (entry)->res = res;
12881 :
12882 : /* Pop the SCC. */
12883 28961 : stack.truncate (last_idx);
12884 28961 : }
12885 :
12886 1890455 : if (place_only)
12887 : return res;
12888 :
12889 : /* Now fixup the backedge def of the vectorized PHIs in this SCC. */
12890 : slp_tree phi_node;
12891 999149 : FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
12892 : {
12893 53933 : gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
12894 53933 : edge_iterator ei;
12895 53933 : edge e;
12896 168045 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
12897 : {
12898 114112 : unsigned dest_idx = e->dest_idx;
12899 114112 : child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
12900 114112 : if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
12901 66800 : continue;
12902 47312 : unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
12903 : /* Simply fill all args. */
12904 47312 : if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
12905 : != vect_first_order_recurrence)
12906 99769 : for (unsigned i = 0; i < n; ++i)
12907 : {
12908 52500 : tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
12909 52500 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
12910 52500 : add_phi_arg (phi, vect_get_slp_vect_def (child, i),
12911 : e, gimple_phi_arg_location (phi, dest_idx));
12912 : }
12913 : else
12914 : {
12915 : /* Unless it is a first order recurrence which needs
12916 : args filled in for both the PHI node and the permutes. */
12917 43 : gimple *perm
12918 43 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
12919 43 : gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
12920 43 : add_phi_arg (as_a <gphi *> (rphi),
12921 : vect_get_slp_vect_def (child, n - 1),
12922 : e, gimple_phi_arg_location (phi, dest_idx));
12923 166 : for (unsigned i = 0; i < n; ++i)
12924 : {
12925 80 : gimple *perm
12926 80 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
12927 80 : if (i > 0)
12928 37 : gimple_assign_set_rhs1 (perm,
12929 : vect_get_slp_vect_def (child, i - 1));
12930 80 : gimple_assign_set_rhs2 (perm,
12931 : vect_get_slp_vect_def (child, i));
12932 80 : update_stmt (perm);
12933 : }
12934 : }
12935 : }
12936 : }
12937 :
12938 945216 : gcc_assert (res);
12939 : return true;
12940 1890455 : }
12941 :
12942 : /* Generate vector code for SLP_INSTANCES in the loop/basic block. Perform
12943 : vector stmt placement only when PLACE_ONLY is true, removing SLP graph
12944 : entries that cannot be scheduled. If placing, return false if a schedule
12945 : cannot be computed for any entry. */
12946 :
12947 : bool
12948 794428 : vect_schedule_slp (vec_info *vinfo, vec<slp_instance> &slp_instances,
12949 : bool place_only)
12950 : {
12951 794428 : slp_instance instance;
12952 794428 : unsigned int i;
12953 :
12954 794428 : hash_map<slp_tree, slp_scc_info> scc_info;
12955 794428 : int maxdfs = 0;
12956 2102528 : for (i = 0; slp_instances.iterate (i, &instance); )
12957 : {
12958 1308100 : slp_tree node = SLP_INSTANCE_TREE (instance);
12959 1308100 : if (!place_only && dump_enabled_p ())
12960 : {
12961 16816 : dump_printf_loc (MSG_NOTE, vect_location,
12962 : "Vectorizing SLP tree:\n");
12963 : /* ??? Dump all? */
12964 16816 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12965 592 : dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
12966 592 : SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
12967 16816 : vect_print_slp_graph (MSG_NOTE, vect_location,
12968 : SLP_INSTANCE_TREE (instance));
12969 : }
12970 : /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
12971 : have a PHI be the node breaking the cycle. */
12972 1308100 : bool res;
12973 1308100 : if (slp_scc_info *info = scc_info.get (node))
12974 3905 : res = info->res;
12975 : else
12976 : {
12977 1304195 : auto_vec<slp_tree> stack;
12978 1304195 : res = vect_schedule_scc (vinfo, node, instance, scc_info,
12979 : maxdfs, stack, place_only);
12980 1304195 : }
12981 :
12982 1308100 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12983 : {
12984 64835 : if (place_only)
12985 : {
12986 55221 : gimple *root_stmt = instance->root_stmts[0]->stmt;
12987 55221 : res &= (!node->si
12988 : /* As we instert after node->si it may not be the
12989 : root_stmt itself. */
12990 55221 : || (node->si != root_stmt
12991 54300 : && vect_stmt_dominates_stmt_p (node->si, root_stmt)));
12992 : }
12993 : else
12994 9614 : vectorize_slp_instance_root_stmt (vinfo, node, instance);
12995 : }
12996 :
12997 1308100 : if (!place_only && dump_enabled_p ())
12998 16816 : dump_printf_loc (MSG_NOTE, vect_location,
12999 : "vectorizing stmts using SLP.\n");
13000 :
13001 1308100 : if (!res)
13002 : {
13003 38 : gcc_assert (place_only);
13004 38 : if (dump_enabled_p ())
13005 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
13006 : "not vectorized: cannot schedule SLP graph "
13007 : "entry %p\n", (void *)instance);
13008 38 : vect_free_slp_instance (instance);
13009 38 : slp_instances.ordered_remove (i);
13010 38 : continue;
13011 : }
13012 1308062 : ++i;
13013 : }
13014 :
13015 794428 : if (place_only)
13016 497092 : return !slp_instances.is_empty ();
13017 :
13018 1925877 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
13019 : {
13020 585567 : slp_tree root = SLP_INSTANCE_TREE (instance);
13021 585567 : stmt_vec_info store_info;
13022 585567 : unsigned int j;
13023 :
13024 : /* Remove scalar call stmts. Do not do this for basic-block
13025 : vectorization as not all uses may be vectorized.
13026 : ??? Why should this be necessary? DCE should be able to
13027 : remove the stmts itself.
13028 : ??? For BB vectorization we can as well remove scalar
13029 : stmts starting from the SLP tree root if they have no
13030 : uses. */
13031 585567 : if (is_a <loop_vec_info> (vinfo))
13032 92871 : vect_remove_slp_scalar_calls (vinfo, root);
13033 :
13034 : /* Remove vectorized stores original scalar stmts. */
13035 2617854 : for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
13036 : {
13037 1481775 : if (!store_info
13038 1481758 : || !STMT_VINFO_DATA_REF (store_info)
13039 1452698 : || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
13040 : break;
13041 :
13042 1446720 : store_info = vect_orig_stmt (store_info);
13043 : /* Free the attached stmt_vec_info and remove the stmt. */
13044 1446720 : vinfo->remove_stmt (store_info);
13045 :
13046 : /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
13047 : to not crash in vect_free_slp_tree later. */
13048 1446720 : if (SLP_TREE_REPRESENTATIVE (root) == store_info)
13049 550179 : SLP_TREE_REPRESENTATIVE (root) = NULL;
13050 : }
13051 : }
13052 :
13053 : return true;
13054 794428 : }
|