Line data Source code
1 : /* SLP - Basic Block Vectorization
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #define INCLUDE_ALGORITHM
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "optabs-tree.h"
34 : #include "insn-config.h"
35 : #include "recog.h" /* FIXME: for insn_data */
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "gimple-iterator.h"
39 : #include "cfgloop.h"
40 : #include "tree-vectorizer.h"
41 : #include "langhooks.h"
42 : #include "gimple-walk.h"
43 : #include "dbgcnt.h"
44 : #include "tree-vector-builder.h"
45 : #include "vec-perm-indices.h"
46 : #include "gimple-fold.h"
47 : #include "internal-fn.h"
48 : #include "dump-context.h"
49 : #include "cfganal.h"
50 : #include "tree-eh.h"
51 : #include "tree-cfg.h"
52 : #include "alloc-pool.h"
53 : #include "sreal.h"
54 : #include "predict.h"
55 :
56 : #define REDUC_GROUP_FIRST_ELEMENT(S) \
57 : (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
58 :
59 : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
60 : load_permutation_t &,
61 : const vec<tree> &,
62 : gimple_stmt_iterator *,
63 : poly_uint64, bool, bool,
64 : unsigned *,
65 : unsigned * = nullptr,
66 : bool = false);
67 : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
68 : slp_tree, lane_permutation_t &,
69 : vec<slp_tree> &, bool);
70 : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
71 : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
72 :
73 : static object_allocator<_slp_tree> *slp_tree_pool;
74 : static slp_tree slp_first_node;
75 :
76 : void
77 1131492 : vect_slp_init (void)
78 : {
79 1131492 : slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
80 1131492 : }
81 :
82 : void
83 1131492 : vect_slp_fini (void)
84 : {
85 1793008 : while (slp_first_node)
86 661516 : delete slp_first_node;
87 2262984 : delete slp_tree_pool;
88 1131492 : slp_tree_pool = NULL;
89 1131492 : }
90 :
91 : void *
92 7885793 : _slp_tree::operator new (size_t n)
93 : {
94 7885793 : gcc_assert (n == sizeof (_slp_tree));
95 7885793 : return slp_tree_pool->allocate_raw ();
96 : }
97 :
98 : void
99 7885793 : _slp_tree::operator delete (void *node, size_t n)
100 : {
101 7885793 : gcc_assert (n == sizeof (_slp_tree));
102 7885793 : slp_tree_pool->remove_raw (node);
103 7885793 : }
104 :
105 :
106 : /* Initialize a SLP node. */
107 :
108 7885793 : _slp_tree::_slp_tree ()
109 : {
110 7885793 : this->prev_node = NULL;
111 7885793 : if (slp_first_node)
112 6895983 : slp_first_node->prev_node = this;
113 7885793 : this->next_node = slp_first_node;
114 7885793 : slp_first_node = this;
115 7885793 : SLP_TREE_SCALAR_STMTS (this) = vNULL;
116 7885793 : SLP_TREE_SCALAR_OPS (this) = vNULL;
117 7885793 : SLP_TREE_LIVE_LANES (this) = vNULL;
118 7885793 : SLP_TREE_VEC_DEFS (this) = vNULL;
119 7885793 : SLP_TREE_CHILDREN (this) = vNULL;
120 7885793 : SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
121 7885793 : SLP_TREE_LANE_PERMUTATION (this) = vNULL;
122 7885793 : SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
123 7885793 : SLP_TREE_CODE (this) = ERROR_MARK;
124 7885793 : SLP_TREE_GS_SCALE (this) = 0;
125 7885793 : SLP_TREE_GS_BASE (this) = NULL_TREE;
126 7885793 : this->ldst_lanes = false;
127 7885793 : this->avoid_stlf_fail = false;
128 7885793 : SLP_TREE_VECTYPE (this) = NULL_TREE;
129 7885793 : SLP_TREE_REPRESENTATIVE (this) = NULL;
130 7885793 : this->cycle_info.id = -1;
131 7885793 : this->cycle_info.reduc_idx = -1;
132 7885793 : SLP_TREE_REF_COUNT (this) = 1;
133 7885793 : this->failed = NULL;
134 7885793 : this->max_nunits = 1;
135 7885793 : this->lanes = 0;
136 7885793 : SLP_TREE_TYPE (this) = undef_vec_info_type;
137 7885793 : this->data = NULL;
138 7885793 : }
139 :
140 : /* Tear down a SLP node. */
141 :
142 7885793 : _slp_tree::~_slp_tree ()
143 : {
144 7885793 : if (this->prev_node)
145 4781649 : this->prev_node->next_node = this->next_node;
146 : else
147 3104144 : slp_first_node = this->next_node;
148 7885793 : if (this->next_node)
149 5938701 : this->next_node->prev_node = this->prev_node;
150 7885793 : SLP_TREE_CHILDREN (this).release ();
151 7885793 : SLP_TREE_SCALAR_STMTS (this).release ();
152 7885793 : SLP_TREE_SCALAR_OPS (this).release ();
153 7885793 : SLP_TREE_LIVE_LANES (this).release ();
154 7885793 : SLP_TREE_VEC_DEFS (this).release ();
155 7885793 : SLP_TREE_LOAD_PERMUTATION (this).release ();
156 7885793 : SLP_TREE_LANE_PERMUTATION (this).release ();
157 7885793 : if (this->failed)
158 2098446 : free (failed);
159 7885793 : if (this->data)
160 1259151 : delete this->data;
161 7885793 : }
162 :
163 : /* Push the single SSA definition in DEF to the vector of vector defs. */
164 :
165 : void
166 530864 : _slp_tree::push_vec_def (gimple *def)
167 : {
168 530864 : if (gphi *phi = dyn_cast <gphi *> (def))
169 59176 : vec_defs.quick_push (gimple_phi_result (phi));
170 : else
171 : {
172 471688 : def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
173 471688 : vec_defs.quick_push (get_def_from_ptr (defop));
174 : }
175 530864 : }
176 :
177 : /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
178 :
179 : void
180 14920559 : vect_free_slp_tree (slp_tree node)
181 : {
182 14920559 : int i;
183 14920559 : slp_tree child;
184 :
185 14920559 : if (--SLP_TREE_REF_COUNT (node) != 0)
186 14920559 : return;
187 :
188 11182598 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
189 3958321 : if (child)
190 3594517 : vect_free_slp_tree (child);
191 :
192 7224277 : delete node;
193 : }
194 :
195 : /* Return a location suitable for dumpings related to the SLP instance. */
196 :
197 : dump_user_location_t
198 3461550 : _slp_instance::location () const
199 : {
200 3461550 : if (!root_stmts.is_empty ())
201 338386 : return root_stmts[0]->stmt;
202 : else
203 3123164 : return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
204 : }
205 :
206 :
207 : /* Free the memory allocated for the SLP instance. */
208 :
209 : void
210 1587959 : vect_free_slp_instance (slp_instance instance)
211 : {
212 1587959 : vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
213 1587959 : SLP_INSTANCE_LOADS (instance).release ();
214 1587959 : SLP_INSTANCE_ROOT_STMTS (instance).release ();
215 1587959 : SLP_INSTANCE_REMAIN_DEFS (instance).release ();
216 1587959 : instance->subgraph_entries.release ();
217 1587959 : instance->cost_vec.release ();
218 1587959 : free (instance);
219 1587959 : }
220 :
221 :
222 : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
223 : for a permute node or else ERROR_MARK. */
224 :
225 : slp_tree
226 96137 : vect_create_new_slp_node (unsigned nops, tree_code code)
227 : {
228 96137 : gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
229 96137 : slp_tree node = new _slp_tree;
230 96137 : SLP_TREE_SCALAR_STMTS (node) = vNULL;
231 96137 : SLP_TREE_CHILDREN (node).create (nops);
232 96137 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
233 96137 : SLP_TREE_CODE (node) = code;
234 96137 : return node;
235 : }
236 :
237 : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children. */
238 :
239 : static slp_tree
240 3811784 : vect_create_new_slp_node (slp_tree node,
241 : vec<stmt_vec_info> scalar_stmts, unsigned nops)
242 : {
243 3811784 : SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
244 3811784 : SLP_TREE_CHILDREN (node).create (nops);
245 3811784 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
246 3811784 : SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
247 3811784 : SLP_TREE_LANES (node) = scalar_stmts.length ();
248 3811784 : return node;
249 : }
250 :
251 : /* Create an SLP node for SCALAR_STMTS and NOPS children. */
252 :
253 : static slp_tree
254 8179 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
255 : {
256 8179 : return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
257 : }
258 :
259 : /* Create a vect_external_def SLP node inplace at NODE for scalar
260 : operands OPS. */
261 :
262 : static slp_tree
263 1869034 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
264 : {
265 1869034 : SLP_TREE_SCALAR_OPS (node) = ops;
266 1869034 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
267 0 : SLP_TREE_LANES (node) = ops.length ();
268 1869034 : return node;
269 : }
270 :
271 : /* Create a vect_external_def SLP node for scalar operands OPS. */
272 :
273 : static slp_tree
274 1869034 : vect_create_new_slp_node (vec<tree> ops)
275 : {
276 1869034 : return vect_create_new_slp_node (new _slp_tree, ops);
277 : }
278 :
279 :
280 : /* This structure is used in creation of an SLP tree. Each instance
281 : corresponds to the same operand in a group of scalar stmts in an SLP
282 : node. */
283 : typedef struct _slp_oprnd_info
284 : {
285 : /* Def-stmts for the operands. */
286 : vec<stmt_vec_info> def_stmts;
287 : /* Operands. */
288 : vec<tree> ops;
289 : /* Information about the first statement, its vector def-type, type, the
290 : operand itself in case it's constant, and an indication if it's a pattern
291 : stmt and gather/scatter info. */
292 : tree first_op_type;
293 : enum vect_def_type first_dt;
294 : bool any_pattern;
295 : bool first_gs_p;
296 : gather_scatter_info first_gs_info;
297 : } *slp_oprnd_info;
298 :
299 :
300 : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
301 : operand. */
302 : static vec<slp_oprnd_info>
303 3393756 : vect_create_oprnd_info (int nops, int group_size)
304 : {
305 3393756 : int i;
306 3393756 : slp_oprnd_info oprnd_info;
307 3393756 : vec<slp_oprnd_info> oprnds_info;
308 :
309 3393756 : oprnds_info.create (nops);
310 12184628 : for (i = 0; i < nops; i++)
311 : {
312 5397116 : oprnd_info = XNEW (struct _slp_oprnd_info);
313 5397116 : oprnd_info->def_stmts.create (group_size);
314 5397116 : oprnd_info->ops.create (group_size);
315 5397116 : oprnd_info->first_dt = vect_uninitialized_def;
316 5397116 : oprnd_info->first_op_type = NULL_TREE;
317 5397116 : oprnd_info->any_pattern = false;
318 5397116 : oprnd_info->first_gs_p = false;
319 5397116 : oprnds_info.quick_push (oprnd_info);
320 : }
321 :
322 3393756 : return oprnds_info;
323 : }
324 :
325 :
326 : /* Free operands info. */
327 :
328 : static void
329 3393756 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
330 : {
331 3393756 : int i;
332 3393756 : slp_oprnd_info oprnd_info;
333 :
334 8790872 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
335 : {
336 5397116 : oprnd_info->def_stmts.release ();
337 5397116 : oprnd_info->ops.release ();
338 5397116 : XDELETE (oprnd_info);
339 : }
340 :
341 3393756 : oprnds_info.release ();
342 3393756 : }
343 :
344 : /* Return the execution frequency of NODE (so that a higher value indicates
345 : a "more important" node when optimizing for speed). */
346 :
347 : static sreal
348 3531577 : vect_slp_node_weight (slp_tree node)
349 : {
350 3531577 : stmt_vec_info stmt_info = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (node));
351 3531577 : basic_block bb = gimple_bb (stmt_info->stmt);
352 3531577 : return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
353 : }
354 :
355 : /* Return true if STMTS contains a pattern statement. */
356 :
357 : static bool
358 22406 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
359 : {
360 22406 : stmt_vec_info stmt_info;
361 22406 : unsigned int i;
362 73396 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
363 53041 : if (stmt_info && is_pattern_stmt_p (stmt_info))
364 : return true;
365 : return false;
366 : }
367 :
368 : /* Return true when all lanes in the external or constant NODE have
369 : the same value. */
370 :
371 : static bool
372 603436 : vect_slp_tree_uniform_p (slp_tree node)
373 : {
374 603436 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
375 : || SLP_TREE_DEF_TYPE (node) == vect_external_def);
376 :
377 : /* Pre-existing vectors. */
378 1059477 : if (SLP_TREE_SCALAR_OPS (node).is_empty ())
379 : return false;
380 :
381 : unsigned i;
382 : tree op, first = NULL_TREE;
383 1384221 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
384 1236826 : if (!first)
385 : first = op;
386 633390 : else if (!operand_equal_p (first, op, 0))
387 : return false;
388 :
389 : return true;
390 : }
391 :
392 : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
393 : that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
394 : of the chain. */
395 :
396 : int
397 712331 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
398 : stmt_vec_info first_stmt_info)
399 : {
400 712331 : stmt_vec_info next_stmt_info = first_stmt_info;
401 712331 : int result = 0;
402 :
403 712331 : if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
404 : return -1;
405 :
406 1782083 : do
407 : {
408 1782083 : if (next_stmt_info == stmt_info)
409 : return result;
410 1069752 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
411 1069752 : if (next_stmt_info)
412 1069752 : result += DR_GROUP_GAP (next_stmt_info);
413 : }
414 1069752 : while (next_stmt_info);
415 :
416 : return -1;
417 : }
418 :
419 : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
420 : using the method implemented by duplicate_and_interleave. Return true
421 : if so, returning the number of intermediate vectors in *NVECTORS_OUT
422 : (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
423 : (if nonnull). */
424 :
425 : bool
426 0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
427 : tree elt_type, unsigned int *nvectors_out,
428 : tree *vector_type_out,
429 : tree *permutes)
430 : {
431 0 : tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
432 0 : if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
433 0 : return false;
434 :
435 0 : machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
436 0 : poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
437 0 : unsigned int nvectors = 1;
438 0 : for (;;)
439 : {
440 0 : scalar_int_mode int_mode;
441 0 : poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
442 0 : if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
443 : {
444 : /* Get the natural vector type for this SLP group size. */
445 0 : tree int_type = build_nonstandard_integer_type
446 0 : (GET_MODE_BITSIZE (int_mode), 1);
447 0 : tree vector_type
448 0 : = get_vectype_for_scalar_type (vinfo, int_type, count);
449 0 : poly_int64 half_nelts;
450 0 : if (vector_type
451 0 : && VECTOR_MODE_P (TYPE_MODE (vector_type))
452 0 : && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
453 : GET_MODE_SIZE (base_vector_mode))
454 0 : && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
455 : 2, &half_nelts))
456 : {
457 : /* Try fusing consecutive sequences of COUNT / NVECTORS elements
458 : together into elements of type INT_TYPE and using the result
459 : to build NVECTORS vectors. */
460 0 : poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
461 0 : vec_perm_builder sel1 (nelts, 2, 3);
462 0 : vec_perm_builder sel2 (nelts, 2, 3);
463 :
464 0 : for (unsigned int i = 0; i < 3; ++i)
465 : {
466 0 : sel1.quick_push (i);
467 0 : sel1.quick_push (i + nelts);
468 0 : sel2.quick_push (half_nelts + i);
469 0 : sel2.quick_push (half_nelts + i + nelts);
470 : }
471 0 : vec_perm_indices indices1 (sel1, 2, nelts);
472 0 : vec_perm_indices indices2 (sel2, 2, nelts);
473 0 : machine_mode vmode = TYPE_MODE (vector_type);
474 0 : if (can_vec_perm_const_p (vmode, vmode, indices1)
475 0 : && can_vec_perm_const_p (vmode, vmode, indices2))
476 : {
477 0 : if (nvectors_out)
478 0 : *nvectors_out = nvectors;
479 0 : if (vector_type_out)
480 0 : *vector_type_out = vector_type;
481 0 : if (permutes)
482 : {
483 0 : permutes[0] = vect_gen_perm_mask_checked (vector_type,
484 : indices1);
485 0 : permutes[1] = vect_gen_perm_mask_checked (vector_type,
486 : indices2);
487 : }
488 0 : return true;
489 : }
490 0 : }
491 : }
492 0 : if (!multiple_p (elt_bytes, 2, &elt_bytes))
493 : return false;
494 0 : nvectors *= 2;
495 : /* We need to be able to fuse COUNT / NVECTORS elements together. */
496 0 : if (!multiple_p (count, nvectors))
497 : return false;
498 : }
499 : }
500 :
501 : /* Return true if DTA and DTB match. */
502 :
503 : static bool
504 17052206 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
505 : {
506 17052206 : return (dta == dtb
507 357982 : || ((dta == vect_external_def || dta == vect_constant_def)
508 222165 : && (dtb == vect_external_def || dtb == vect_constant_def)));
509 : }
510 :
511 : #define GATHER_SCATTER_OFFSET (-3)
512 :
513 : /* For most SLP statements, there is a one-to-one mapping between
514 : gimple arguments and child nodes. If that is not true for STMT,
515 : return an array that contains:
516 :
517 : - the number of child nodes, followed by
518 : - for each child node, the index of the argument associated with that node.
519 : The special index -1 is the first operand of an embedded comparison and
520 : the special index -2 is the second operand of an embedded comparison.
521 : The special index -3 is the offset of a gather as analyzed by
522 : vect_check_gather_scatter.
523 :
524 : SWAP is as for vect_get_and_check_slp_defs. */
525 :
526 : static const int *
527 24586313 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
528 : unsigned char swap)
529 : {
530 24586313 : static const int no_arg_map[] = { 0 };
531 24586313 : static const int arg0_map[] = { 1, 0 };
532 24586313 : static const int arg2_map[] = { 1, 2 };
533 24586313 : static const int arg2_arg3_map[] = { 2, 2, 3 };
534 24586313 : static const int arg2_arg4_map[] = { 2, 2, 4 };
535 24586313 : static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
536 24586313 : static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
537 24586313 : static const int arg3_arg2_map[] = { 2, 3, 2 };
538 24586313 : static const int op00_map[] = { 1, -1 };
539 24586313 : static const int op1_op0_map[] = { 2, 1, 0 };
540 24586313 : static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
541 24586313 : static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
542 24586313 : static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
543 24586313 : static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
544 24586313 : static const int mask_call_maps[6][7] = {
545 : { 1, 1, },
546 : { 2, 1, 2, },
547 : { 3, 1, 2, 3, },
548 : { 4, 1, 2, 3, 4, },
549 : { 5, 1, 2, 3, 4, 5, },
550 : { 6, 1, 2, 3, 4, 5, 6 },
551 : };
552 :
553 24586313 : gcc_checking_assert (!swap
554 : || !is_gimple_assign (stmt)
555 : || TREE_CODE_CLASS
556 : (gimple_assign_rhs_code (stmt)) == tcc_comparison
557 : || commutative_tree_code
558 : (gimple_assign_rhs_code (stmt)));
559 :
560 24586313 : if (auto assign = dyn_cast<const gassign *> (stmt))
561 : {
562 23121222 : tree_code code = gimple_assign_rhs_code (assign);
563 23121222 : if (code == COND_EXPR
564 23121222 : && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
565 0 : gcc_unreachable ();
566 23121222 : else if ((TREE_CODE_CLASS (code) == tcc_comparison
567 21669934 : || commutative_tree_code (code))
568 32079947 : && swap)
569 : return op1_op0_map;
570 23078902 : else if (code == VIEW_CONVERT_EXPR)
571 : return op00_map;
572 23070297 : else if (gather_scatter_p)
573 43601 : return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
574 43601 : ? off_op0_map : off_map);
575 : }
576 1465091 : else if (auto call = dyn_cast<const gcall *> (stmt))
577 : {
578 162153 : if (gimple_call_internal_p (call))
579 92078 : switch (gimple_call_internal_fn (call))
580 : {
581 15905 : case IFN_MASK_LOAD:
582 27090 : return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
583 :
584 : case IFN_GATHER_LOAD:
585 : return arg2_map;
586 :
587 0 : case IFN_MASK_GATHER_LOAD:
588 0 : case IFN_MASK_LEN_GATHER_LOAD:
589 0 : return arg2_arg5_arg6_map;
590 :
591 0 : case IFN_SCATTER_STORE:
592 0 : return arg2_arg4_map;
593 :
594 0 : case IFN_MASK_SCATTER_STORE:
595 0 : case IFN_MASK_LEN_SCATTER_STORE:
596 0 : return arg2_arg4_arg5_map;
597 :
598 9311 : case IFN_MASK_STORE:
599 17200 : return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
600 :
601 988 : case IFN_MASK_CALL:
602 988 : {
603 988 : unsigned nargs = gimple_call_num_args (call);
604 988 : if (nargs >= 2 && nargs <= 7)
605 988 : return mask_call_maps[nargs-2];
606 : else
607 : return nullptr;
608 : }
609 :
610 278 : case IFN_CLZ:
611 278 : case IFN_CTZ:
612 278 : return arg0_map;
613 :
614 6294 : case IFN_GOMP_SIMD_LANE:
615 6294 : return no_arg_map;
616 :
617 : default:
618 : break;
619 : }
620 : }
621 : return nullptr;
622 : }
623 :
624 : static const int *
625 24569584 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
626 : {
627 0 : return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
628 0 : swap);
629 : }
630 :
631 : /* Return the SLP node child index for operand OP of STMT. */
632 :
633 : int
634 1390022 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
635 : {
636 1390022 : const int *opmap = vect_get_operand_map (stmt);
637 1390022 : if (!opmap)
638 : return op;
639 21870 : for (int i = 1; i < 1 + opmap[0]; ++i)
640 21870 : if (opmap[i] == op)
641 12222 : return i - 1;
642 0 : gcc_unreachable ();
643 : }
644 :
645 : /* Helper class for mapping of GIMPLE operands to SLP children. */
646 : /* ??? Add vect_slp_child_index_for_operand here and amend opmaps
647 : with the full reverse mapping and indicating the position of the
648 : first commutative operand index, eliding the swap_p argument from
649 : vect_get_operand_map. Adjust all consumers. */
650 :
651 : struct slp_oprnds {
652 : slp_oprnds (stmt_vec_info);
653 : tree get_op_for_slp_child (stmt_vec_info, unsigned);
654 : const int *opmap;
655 : const unsigned int num_slp_children;
656 : };
657 :
658 4460111 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
659 4460111 : : opmap (vect_get_operand_map (stmt_info)),
660 4460111 : num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
661 : {
662 4460111 : }
663 :
664 : /* For SLP child number N get the corresponding tree operand from GIMPLE
665 : statement described by STMT_INFO. */
666 :
667 : tree
668 4907576 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
669 : {
670 4907576 : gcc_assert (n < num_slp_children);
671 4907576 : int opno = opmap ? opmap[n + 1] : (int) n;
672 4907576 : if (opno == GATHER_SCATTER_OFFSET)
673 0 : gcc_unreachable (); // TODO
674 4907576 : else if (opno < 0)
675 2106 : return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
676 : else
677 4905470 : return gimple_arg (stmt_info->stmt, opno);
678 : }
679 :
680 : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
681 : they are of a valid type and that they match the defs of the first stmt of
682 : the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
683 : by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP
684 : indicates swap is required for cond_expr stmts. Specifically, SWAP
685 : is 1 if STMT is cond and operands of comparison need to be swapped;
686 : SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
687 :
688 : If there was a fatal error return -1; if the error could be corrected by
689 : swapping operands of father node of this one, return 1; if everything is
690 : ok return 0. */
691 : static int
692 12820856 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
693 : bool *skip_args,
694 : vec<stmt_vec_info> stmts, unsigned stmt_num,
695 : vec<slp_oprnd_info> *oprnds_info)
696 : {
697 12820856 : stmt_vec_info stmt_info = stmts[stmt_num];
698 12820856 : tree oprnd;
699 12820856 : unsigned int i, number_of_oprnds;
700 12820856 : enum vect_def_type dt = vect_uninitialized_def;
701 12820856 : slp_oprnd_info oprnd_info;
702 12820856 : gather_scatter_info gs_info;
703 12820856 : unsigned int gs_op = -1u;
704 12820856 : unsigned int commutative_op = -1U;
705 12820856 : bool first = stmt_num == 0;
706 :
707 12820856 : if (!stmt_info)
708 : {
709 0 : for (auto oi : *oprnds_info)
710 : {
711 0 : oi->def_stmts.quick_push (NULL);
712 0 : oi->ops.quick_push (NULL_TREE);
713 : }
714 : return 0;
715 : }
716 :
717 12820856 : if (!is_a<gcall *> (stmt_info->stmt)
718 : && !is_a<gassign *> (stmt_info->stmt)
719 : && !is_a<gphi *> (stmt_info->stmt))
720 : return -1;
721 :
722 12820856 : number_of_oprnds = gimple_num_args (stmt_info->stmt);
723 12820856 : const int *map = vect_get_operand_map (stmt_info, swap);
724 12820856 : if (map)
725 77738 : number_of_oprnds = *map++;
726 12820856 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
727 : {
728 50765 : if (gimple_call_internal_p (stmt))
729 : {
730 32563 : internal_fn ifn = gimple_call_internal_fn (stmt);
731 32563 : commutative_op = first_commutative_argument (ifn);
732 32563 : if (internal_gather_scatter_fn_p (ifn))
733 : {
734 0 : vect_describe_gather_scatter_call
735 0 : (stmt_info,
736 0 : first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
737 0 : if (first)
738 0 : (*oprnds_info)[0]->first_gs_p = true;
739 : gs_op = 0;
740 : }
741 : }
742 : }
743 12770091 : else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
744 : {
745 14919398 : if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
746 8432730 : commutative_op = 0;
747 : }
748 :
749 12820856 : bool swapped = (swap != 0);
750 12820856 : bool backedge = false;
751 12820856 : enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
752 35455394 : for (i = 0; i < number_of_oprnds; i++)
753 : {
754 22635744 : oprnd_info = (*oprnds_info)[i];
755 22635744 : int opno = map ? map[i] : int (i);
756 22635744 : if (opno == GATHER_SCATTER_OFFSET)
757 : {
758 22866 : gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
759 22866 : if (!is_a <loop_vec_info> (vinfo)
760 22866 : || !vect_check_gather_scatter (stmt_info, vectype,
761 : as_a <loop_vec_info> (vinfo),
762 : first ? &oprnd_info->first_gs_info
763 : : &gs_info))
764 1206 : return -1;
765 :
766 22866 : if (first)
767 : {
768 22609 : oprnd_info->first_gs_p = true;
769 22609 : oprnd = oprnd_info->first_gs_info.offset;
770 : }
771 : else
772 : {
773 257 : gs_op = i;
774 257 : oprnd = gs_info.offset;
775 : }
776 : }
777 22612878 : else if (opno < 0)
778 3050 : oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
779 : else
780 : {
781 22609828 : oprnd = gimple_arg (stmt_info->stmt, opno);
782 22609828 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
783 : {
784 1224446 : edge e = gimple_phi_arg_edge (stmt, opno);
785 2448892 : backedge = (is_a <bb_vec_info> (vinfo)
786 1893123 : ? e->flags & EDGE_DFS_BACK
787 668677 : : dominated_by_p (CDI_DOMINATORS, e->src,
788 668677 : gimple_bb (stmt_info->stmt)));
789 : }
790 : }
791 :
792 22635744 : stmt_vec_info def_stmt_info;
793 22635744 : if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
794 : {
795 982 : if (dump_enabled_p ())
796 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
797 : "Build SLP failed: can't analyze def for %T\n",
798 : oprnd);
799 :
800 982 : return -1;
801 : }
802 :
803 22634762 : if (skip_args[i])
804 : {
805 534170 : oprnd_info->def_stmts.quick_push (NULL);
806 534170 : oprnd_info->ops.quick_push (NULL_TREE);
807 534170 : oprnd_info->first_dt = vect_uninitialized_def;
808 534170 : continue;
809 : }
810 :
811 22100592 : oprnd_info->def_stmts.quick_push (def_stmt_info);
812 22100592 : oprnd_info->ops.quick_push (oprnd);
813 :
814 22100592 : if (def_stmt_info
815 22100592 : && is_pattern_stmt_p (def_stmt_info))
816 : {
817 393387 : if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
818 : != def_stmt_info)
819 276541 : oprnd_info->any_pattern = true;
820 : else
821 : /* If we promote this to external use the original stmt def. */
822 116846 : oprnd_info->ops.last ()
823 233692 : = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
824 : }
825 :
826 : /* If there's a extern def on a backedge make sure we can
827 : code-generate at the region start.
828 : ??? This is another case that could be fixed by adjusting
829 : how we split the function but at the moment we'd have conflicting
830 : goals there. */
831 22100592 : if (backedge
832 168566 : && dts[i] == vect_external_def
833 245 : && is_a <bb_vec_info> (vinfo)
834 245 : && TREE_CODE (oprnd) == SSA_NAME
835 224 : && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
836 22100816 : && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
837 224 : gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
838 : {
839 224 : if (dump_enabled_p ())
840 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
841 : "Build SLP failed: extern def %T only defined "
842 : "on backedge\n", oprnd);
843 224 : return -1;
844 : }
845 :
846 22100368 : if (first)
847 : {
848 4901672 : tree type = TREE_TYPE (oprnd);
849 4901672 : dt = dts[i];
850 :
851 : /* For the swapping logic below force vect_reduction_def
852 : for the reduction op in a SLP reduction group. */
853 4901672 : if (!STMT_VINFO_DATA_REF (stmt_info)
854 3716891 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
855 5224 : && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
856 4904244 : && def_stmt_info)
857 2572 : dts[i] = dt = vect_reduction_def;
858 :
859 : /* Check the types of the definition. */
860 4901672 : switch (dt)
861 : {
862 4901672 : case vect_external_def:
863 4901672 : case vect_constant_def:
864 4901672 : case vect_internal_def:
865 4901672 : case vect_reduction_def:
866 4901672 : case vect_double_reduction_def:
867 4901672 : case vect_induction_def:
868 4901672 : case vect_nested_cycle:
869 4901672 : case vect_first_order_recurrence:
870 4901672 : break;
871 :
872 0 : default:
873 : /* FORNOW: Not supported. */
874 0 : if (dump_enabled_p ())
875 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
876 : "Build SLP failed: illegal type of def %T\n",
877 : oprnd);
878 0 : return -1;
879 : }
880 :
881 4901672 : oprnd_info->first_dt = dt;
882 4901672 : oprnd_info->first_op_type = type;
883 : }
884 : }
885 12819650 : if (first)
886 : return 0;
887 :
888 : /* Now match the operand definition types to that of the first stmt. */
889 26334122 : for (i = 0; i < number_of_oprnds;)
890 : {
891 17195073 : if (skip_args[i])
892 : {
893 44124 : ++i;
894 44124 : continue;
895 : }
896 :
897 17150949 : oprnd_info = (*oprnds_info)[i];
898 17150949 : dt = dts[i];
899 17150949 : stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
900 17150949 : oprnd = oprnd_info->ops[stmt_num];
901 17150949 : tree type = TREE_TYPE (oprnd);
902 :
903 17150949 : if (!types_compatible_p (oprnd_info->first_op_type, type))
904 : {
905 104866 : if (dump_enabled_p ())
906 91 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
907 : "Build SLP failed: different operand types\n");
908 104866 : return 1;
909 : }
910 :
911 17046083 : if ((gs_op == i) != oprnd_info->first_gs_p)
912 : {
913 0 : if (dump_enabled_p ())
914 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
915 : "Build SLP failed: mixed gather and non-gather\n");
916 0 : return 1;
917 : }
918 17046083 : else if (gs_op == i)
919 : {
920 227 : if (!operand_equal_p (oprnd_info->first_gs_info.base,
921 227 : gs_info.base))
922 : {
923 16 : if (dump_enabled_p ())
924 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
925 : "Build SLP failed: different gather base\n");
926 16 : return 1;
927 : }
928 211 : if (oprnd_info->first_gs_info.scale != gs_info.scale)
929 : {
930 8 : if (dump_enabled_p ())
931 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
932 : "Build SLP failed: different gather scale\n");
933 8 : return 1;
934 : }
935 : }
936 :
937 : /* Not first stmt of the group, check that the def-stmt/s match
938 : the def-stmt/s of the first stmt. Allow different definition
939 : types for reduction chains: the first stmt must be a
940 : vect_reduction_def (a phi node), and the rest
941 : end in the reduction chain. */
942 17046059 : if ((!vect_def_types_match (oprnd_info->first_dt, dt)
943 299269 : && !(oprnd_info->first_dt == vect_reduction_def
944 4797 : && !STMT_VINFO_DATA_REF (stmt_info)
945 4797 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
946 4771 : && def_stmt_info
947 4769 : && !STMT_VINFO_DATA_REF (def_stmt_info)
948 4769 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
949 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
950 16751559 : || (!STMT_VINFO_DATA_REF (stmt_info)
951 15424698 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
952 9943 : && ((!def_stmt_info
953 9747 : || STMT_VINFO_DATA_REF (def_stmt_info)
954 17960 : || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
955 : != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
956 9943 : != (oprnd_info->first_dt != vect_reduction_def))))
957 : {
958 : /* Try swapping operands if we got a mismatch. For BB
959 : vectorization only in case it will clearly improve things. */
960 296958 : if (i == commutative_op && !swapped
961 294500 : && (!is_a <bb_vec_info> (vinfo)
962 4838 : || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
963 4838 : dts[i+1])
964 1153 : && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
965 : || vect_def_types_match
966 156 : ((*oprnds_info)[i+1]->first_dt, dts[i])))))
967 : {
968 2458 : if (dump_enabled_p ())
969 153 : dump_printf_loc (MSG_NOTE, vect_location,
970 : "trying swapped operands\n");
971 2458 : std::swap (dts[i], dts[i+1]);
972 2458 : std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
973 2458 : (*oprnds_info)[i+1]->def_stmts[stmt_num]);
974 2458 : std::swap ((*oprnds_info)[i]->ops[stmt_num],
975 2458 : (*oprnds_info)[i+1]->ops[stmt_num]);
976 : /* After swapping some operands we lost track whether an
977 : operand has any pattern defs so be conservative here. */
978 2458 : if ((*oprnds_info)[i]->any_pattern
979 2458 : || (*oprnds_info)[i+1]->any_pattern)
980 36 : (*oprnds_info)[i]->any_pattern
981 18 : = (*oprnds_info)[i+1]->any_pattern = true;
982 2458 : swapped = true;
983 2458 : continue;
984 : }
985 :
986 292042 : if (is_a <bb_vec_info> (vinfo)
987 276512 : && !oprnd_info->any_pattern
988 568318 : && number_of_oprnds > 1)
989 : {
990 : /* Now for commutative ops we should see whether we can
991 : make the other operand matching. */
992 105070 : if (dump_enabled_p ())
993 203 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
994 : "treating operand as external\n");
995 105070 : oprnd_info->first_dt = dt = vect_external_def;
996 : }
997 : else
998 : {
999 186972 : if (dump_enabled_p ())
1000 411 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1001 : "Build SLP failed: different types\n");
1002 186972 : return 1;
1003 : }
1004 : }
1005 :
1006 : /* Make sure to demote the overall operand to external. */
1007 16856629 : if (dt == vect_external_def)
1008 341166 : oprnd_info->first_dt = vect_external_def;
1009 : /* For a SLP reduction chain we want to duplicate the reduction to
1010 : each of the chain members. That gets us a sane SLP graph (still
1011 : the stmts are not 100% correct wrt the initial values). */
1012 16515463 : else if ((dt == vect_internal_def
1013 16515463 : || dt == vect_reduction_def)
1014 15573795 : && oprnd_info->first_dt == vect_reduction_def
1015 101224 : && !STMT_VINFO_DATA_REF (stmt_info)
1016 101224 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
1017 4769 : && !STMT_VINFO_DATA_REF (def_stmt_info)
1018 16520232 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
1019 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
1020 : {
1021 4769 : oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
1022 4769 : oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
1023 : }
1024 :
1025 16856629 : ++i;
1026 : }
1027 :
1028 : /* Swap operands. */
1029 9139049 : if (swapped)
1030 : {
1031 42312 : if (dump_enabled_p ())
1032 457 : dump_printf_loc (MSG_NOTE, vect_location,
1033 : "swapped operands to match def types in %G",
1034 : stmt_info->stmt);
1035 : }
1036 :
1037 : return 0;
1038 : }
1039 :
1040 : /* Return true if call statements CALL1 and CALL2 are similar enough
1041 : to be combined into the same SLP group. */
1042 :
1043 : bool
1044 21985 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
1045 : {
1046 21985 : unsigned int nargs = gimple_call_num_args (call1);
1047 21985 : if (nargs != gimple_call_num_args (call2))
1048 : return false;
1049 :
1050 20034 : auto cfn1 = gimple_call_combined_fn (call1);
1051 20034 : auto cfn2 = gimple_call_combined_fn (call2);
1052 20034 : if (cfn1 != cfn2
1053 2 : && (!allow_two_operators
1054 2 : || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
1055 2 : && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
1056 : return false;
1057 :
1058 20034 : if (gimple_call_internal_p (call1))
1059 : {
1060 7033 : if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
1061 7033 : TREE_TYPE (gimple_call_lhs (call2))))
1062 : return false;
1063 14480 : for (unsigned int i = 0; i < nargs; ++i)
1064 7447 : if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
1065 7447 : TREE_TYPE (gimple_call_arg (call2, i))))
1066 : return false;
1067 : }
1068 : else
1069 : {
1070 13001 : if (!operand_equal_p (gimple_call_fn (call1),
1071 13001 : gimple_call_fn (call2), 0))
1072 : return false;
1073 :
1074 29088 : if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
1075 : return false;
1076 : }
1077 :
1078 : /* Check that any unvectorized arguments are equal. */
1079 16729 : if (const int *map = vect_get_operand_map (call1, false, false))
1080 : {
1081 15 : unsigned int nkept = *map++;
1082 15 : unsigned int mapi = 0;
1083 57 : for (unsigned int i = 0; i < nargs; ++i)
1084 42 : if (mapi < nkept && map[mapi] == int (i))
1085 27 : mapi += 1;
1086 15 : else if (!operand_equal_p (gimple_call_arg (call1, i),
1087 15 : gimple_call_arg (call2, i)))
1088 : return false;
1089 : }
1090 :
1091 : return true;
1092 : }
1093 :
1094 : /* A subroutine of vect_build_slp_tree for checking VECTYPE, which is the
1095 : caller's attempt to find the vector type in STMT_INFO with the narrowest
1096 : element type. Return true if VECTYPE is nonnull and if it is valid
1097 : for STMT_INFO. When returning true, update MAX_NUNITS to reflect the
1098 : number of units in VECTYPE. GROUP_SIZE and MAX_NUNITS are as for
1099 : vect_build_slp_tree. */
1100 :
1101 : static bool
1102 5620401 : vect_record_max_nunits (vec_info *vinfo, stmt_vec_info stmt_info,
1103 : unsigned int group_size,
1104 : tree vectype, poly_uint64 *max_nunits)
1105 : {
1106 5620401 : if (!vectype)
1107 : {
1108 4057 : if (dump_enabled_p ())
1109 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1110 : "Build SLP failed: unsupported data-type in %G\n",
1111 : stmt_info->stmt);
1112 : /* Fatal mismatch. */
1113 4057 : return false;
1114 : }
1115 :
1116 : /* If populating the vector type requires unrolling then fail
1117 : before adjusting *max_nunits for basic-block vectorization. */
1118 5616344 : if (is_a <bb_vec_info> (vinfo)
1119 5616344 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
1120 : {
1121 146140 : if (dump_enabled_p ())
1122 36 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1123 : "Build SLP failed: unrolling required "
1124 : "in basic block SLP\n");
1125 : /* Fatal mismatch. */
1126 146140 : return false;
1127 : }
1128 :
1129 : /* In case of multiple types we need to detect the smallest type. */
1130 5470204 : vect_update_max_nunits (max_nunits, vectype);
1131 5470204 : return true;
1132 : }
1133 :
1134 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1135 : permutation or are of unsupported types of operation. Return
1136 : true if they are, otherwise return false and indicate in *MATCHES
1137 : which stmts are not isomorphic to the first one. If MATCHES[0]
1138 : is false then this indicates the comparison could not be
1139 : carried out or the stmts will never be vectorized by SLP.
1140 :
1141 : Note COND_EXPR is possibly isomorphic to another one after swapping its
1142 : operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
1143 : the first stmt by swapping the two operands of comparison; set SWAP[i]
1144 : to 2 if stmt I is isormorphic to the first stmt by inverting the code
1145 : of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
1146 : to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
1147 :
1148 : static bool
1149 5894532 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
1150 : vec<stmt_vec_info> stmts,
1151 : poly_uint64 *max_nunits, bool *matches,
1152 : bool *two_operators, tree *node_vectype)
1153 : {
1154 5894532 : unsigned int group_size = stmts.length ();
1155 5894532 : unsigned int i;
1156 5894532 : stmt_vec_info first_stmt_info = stmts[0];
1157 5894532 : code_helper first_stmt_code = ERROR_MARK;
1158 5894532 : code_helper alt_stmt_code = ERROR_MARK;
1159 5894532 : code_helper first_cond_code = ERROR_MARK;
1160 5894532 : bool need_same_oprnds = false;
1161 5894532 : tree first_lhs = NULL_TREE;
1162 5894532 : tree first_op1 = NULL_TREE;
1163 5894532 : stmt_vec_info first_load = NULL, prev_first_load = NULL;
1164 5894532 : bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
1165 5894532 : bool first_stmt_phi_p = false;
1166 5894532 : int first_reduc_idx = -1;
1167 5894532 : bool maybe_soft_fail = false;
1168 5894532 : tree soft_fail_nunits_vectype = NULL_TREE;
1169 :
1170 5894532 : tree vectype, nunits_vectype;
1171 5894532 : if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
1172 : &nunits_vectype, group_size))
1173 : {
1174 : /* Fatal mismatch. */
1175 210989 : matches[0] = false;
1176 210989 : return false;
1177 : }
1178 5683543 : if (is_a <bb_vec_info> (vinfo)
1179 5683543 : && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
1180 : {
1181 366826 : if (dump_enabled_p ())
1182 298 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1183 : "Build SLP failed: not using single lane "
1184 : "vector type %T\n", vectype);
1185 366826 : matches[0] = false;
1186 366826 : return false;
1187 : }
1188 : /* Record nunits required but continue analysis, producing matches[]
1189 : as if nunits was not an issue. This allows splitting of groups
1190 : to happen. */
1191 5316717 : if (nunits_vectype
1192 5316717 : && !vect_record_max_nunits (vinfo, first_stmt_info, group_size,
1193 : nunits_vectype, max_nunits))
1194 : {
1195 146140 : gcc_assert (is_a <bb_vec_info> (vinfo));
1196 146140 : maybe_soft_fail = true;
1197 146140 : soft_fail_nunits_vectype = nunits_vectype;
1198 : }
1199 :
1200 5316717 : gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
1201 5316717 : *node_vectype = vectype;
1202 :
1203 : /* For every stmt in NODE find its def stmt/s. */
1204 5316717 : stmt_vec_info stmt_info;
1205 22524050 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
1206 : {
1207 17374183 : bool ldst_p = false;
1208 17374183 : bool ldst_masklen_p = false;
1209 17374183 : bool phi_p = false;
1210 17374183 : code_helper rhs_code = ERROR_MARK;
1211 :
1212 17374183 : swap[i] = 0;
1213 17374183 : matches[i] = false;
1214 17374183 : if (!stmt_info)
1215 : {
1216 40709 : matches[i] = true;
1217 17248042 : continue;
1218 : }
1219 :
1220 17333474 : gimple *stmt = stmt_info->stmt;
1221 17333474 : if (dump_enabled_p ())
1222 220090 : dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
1223 :
1224 : /* Fail to vectorize statements marked as unvectorizable, throw
1225 : or are volatile. */
1226 17333474 : if (!STMT_VINFO_VECTORIZABLE (stmt_info)
1227 17139799 : || stmt_can_throw_internal (cfun, stmt)
1228 33676091 : || gimple_has_volatile_ops (stmt))
1229 : {
1230 199182 : if (dump_enabled_p ())
1231 203 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1232 : "Build SLP failed: unvectorizable statement %G",
1233 : stmt);
1234 : /* ??? For BB vectorization we want to commutate operands in a way
1235 : to shuffle all unvectorizable defs into one operand and have
1236 : the other still vectorized. The following doesn't reliably
1237 : work for this though but it's the easiest we can do here. */
1238 199182 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1239 65923 : continue;
1240 : /* Fatal mismatch. */
1241 133259 : matches[0] = false;
1242 133259 : return false;
1243 : }
1244 :
1245 17134292 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
1246 17134292 : tree lhs = gimple_get_lhs (stmt);
1247 17134292 : if (lhs == NULL_TREE && !call_stmt)
1248 : {
1249 36 : if (dump_enabled_p ())
1250 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1251 : "Build SLP failed: not GIMPLE_ASSIGN nor "
1252 : "GIMPLE_CALL %G", stmt);
1253 36 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1254 36 : continue;
1255 : /* Fatal mismatch. */
1256 0 : matches[0] = false;
1257 0 : return false;
1258 : }
1259 :
1260 17134256 : if (call_stmt)
1261 : {
1262 102453 : combined_fn cfn = gimple_call_combined_fn (call_stmt);
1263 102453 : if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
1264 56996 : rhs_code = cfn;
1265 : else
1266 : rhs_code = CALL_EXPR;
1267 :
1268 102453 : if (cfn == CFN_GATHER_LOAD
1269 102453 : || cfn == CFN_SCATTER_STORE)
1270 : ldst_p = true;
1271 : else if (cfn == CFN_MASK_LOAD
1272 : || cfn == CFN_MASK_GATHER_LOAD
1273 : || cfn == CFN_MASK_LEN_GATHER_LOAD
1274 : || cfn == CFN_MASK_SCATTER_STORE
1275 : || cfn == CFN_MASK_LEN_SCATTER_STORE)
1276 : {
1277 : ldst_p = true;
1278 : ldst_masklen_p = true;
1279 : }
1280 : else if (cfn == CFN_MASK_STORE)
1281 : {
1282 : ldst_p = true;
1283 : ldst_masklen_p = true;
1284 : rhs_code = CFN_MASK_STORE;
1285 : }
1286 : else if (cfn == CFN_GOMP_SIMD_LANE)
1287 : ;
1288 91125 : else if ((cfn != CFN_LAST
1289 : && cfn != CFN_MASK_CALL
1290 45668 : && internal_fn_p (cfn)
1291 37092 : && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
1292 91050 : || gimple_call_tail_p (call_stmt)
1293 91050 : || gimple_call_noreturn_p (call_stmt)
1294 182175 : || gimple_call_chain (call_stmt))
1295 : {
1296 424 : if (dump_enabled_p ())
1297 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1298 : "Build SLP failed: unsupported call type %G",
1299 : (gimple *) call_stmt);
1300 424 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1301 64 : continue;
1302 : /* Fatal mismatch. */
1303 360 : matches[0] = false;
1304 360 : return false;
1305 : }
1306 : }
1307 17031803 : else if (gimple_code (stmt) == GIMPLE_PHI)
1308 : {
1309 : rhs_code = ERROR_MARK;
1310 : phi_p = true;
1311 : }
1312 : else
1313 : {
1314 16234621 : rhs_code = gimple_assign_rhs_code (stmt);
1315 16234621 : ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
1316 : }
1317 :
1318 : /* Check the operation. */
1319 17133832 : if (i == 0)
1320 : {
1321 5183098 : first_lhs = lhs;
1322 5183098 : first_stmt_code = rhs_code;
1323 5183098 : first_stmt_ldst_p = ldst_p;
1324 5183098 : first_stmt_ldst_masklen_p = ldst_masklen_p;
1325 5183098 : first_stmt_phi_p = phi_p;
1326 5183098 : first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
1327 :
1328 : /* Shift arguments should be equal in all the packed stmts for a
1329 : vector shift with scalar shift operand. */
1330 5183098 : if (rhs_code == LSHIFT_EXPR || rhs_code == RSHIFT_EXPR
1331 5048242 : || rhs_code == LROTATE_EXPR
1332 10231268 : || rhs_code == RROTATE_EXPR)
1333 : {
1334 : /* First see if we have a vector/vector shift. */
1335 135311 : if (!directly_supported_p (rhs_code, vectype, optab_vector))
1336 : {
1337 : /* No vector/vector shift, try for a vector/scalar shift. */
1338 123296 : if (!directly_supported_p (rhs_code, vectype, optab_scalar))
1339 : {
1340 11915 : if (dump_enabled_p ())
1341 386 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1342 : "Build SLP failed: "
1343 : "op not supported by target.\n");
1344 11915 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1345 : continue;
1346 : /* Fatal mismatch. */
1347 11915 : matches[0] = false;
1348 11915 : return false;
1349 : }
1350 111381 : need_same_oprnds = true;
1351 111381 : first_op1 = gimple_assign_rhs2 (stmt);
1352 : }
1353 : }
1354 5047787 : else if (rhs_code == WIDEN_LSHIFT_EXPR)
1355 : {
1356 0 : need_same_oprnds = true;
1357 0 : first_op1 = gimple_assign_rhs2 (stmt);
1358 : }
1359 5047787 : else if (!ldst_p
1360 5047787 : && rhs_code == BIT_FIELD_REF)
1361 : {
1362 5590 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
1363 5590 : if (!is_a <bb_vec_info> (vinfo)
1364 5464 : || TREE_CODE (vec) != SSA_NAME
1365 : /* When the element types are not compatible we pun the
1366 : source to the target vectype which requires equal size. */
1367 11042 : || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
1368 4741 : || !types_compatible_p (TREE_TYPE (vectype),
1369 4741 : TREE_TYPE (TREE_TYPE (vec))))
1370 1006 : && !operand_equal_p (TYPE_SIZE (vectype),
1371 1006 : TYPE_SIZE (TREE_TYPE (vec)))))
1372 : {
1373 757 : if (dump_enabled_p ())
1374 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1375 : "Build SLP failed: "
1376 : "BIT_FIELD_REF not supported\n");
1377 : /* Fatal mismatch. */
1378 757 : matches[0] = false;
1379 757 : return false;
1380 : }
1381 : }
1382 5042197 : else if (rhs_code == CFN_DIV_POW2)
1383 : {
1384 0 : need_same_oprnds = true;
1385 0 : first_op1 = gimple_call_arg (call_stmt, 1);
1386 : }
1387 5042197 : else if (rhs_code == CFN_GOMP_SIMD_LANE)
1388 : {
1389 3147 : need_same_oprnds = true;
1390 3147 : first_op1 = gimple_call_arg (call_stmt, 1);
1391 : }
1392 : }
1393 : else
1394 : {
1395 11950734 : int comm_arg;
1396 11951112 : if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1397 : /* For SLP reduction groups the index isn't necessarily
1398 : uniform but only that of the first stmt matters. */
1399 2334 : && !(first_reduc_idx != -1
1400 2334 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1401 2334 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
1402 11950734 : && !(first_reduc_idx != -1
1403 1049 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1404 1049 : && (comm_arg = first_commutative_argument
1405 1049 : (rhs_code, TREE_TYPE (lhs))) >= 0
1406 : && (first_reduc_idx
1407 815 : == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
1408 : {
1409 378 : if (dump_enabled_p ())
1410 : {
1411 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1412 : "Build SLP failed: different reduc_idx "
1413 : "%d instead of %d in %G",
1414 : STMT_VINFO_REDUC_IDX (stmt_info),
1415 : first_reduc_idx, stmt);
1416 : }
1417 : /* Mismatch. */
1418 378 : continue;
1419 : }
1420 11950356 : if (!ldst_p
1421 9358902 : && first_stmt_code != rhs_code
1422 13388747 : && alt_stmt_code == ERROR_MARK)
1423 : alt_stmt_code = rhs_code;
1424 13363848 : if ((!ldst_p
1425 9358902 : && first_stmt_code != rhs_code
1426 1438391 : && (first_stmt_code != IMAGPART_EXPR
1427 128 : || rhs_code != REALPART_EXPR)
1428 1438371 : && (first_stmt_code != REALPART_EXPR
1429 532 : || rhs_code != IMAGPART_EXPR)
1430 : /* Handle mismatches in plus/minus by computing both
1431 : and merging the results. */
1432 1438360 : && !((((first_stmt_code == PLUS_EXPR
1433 1324830 : || first_stmt_code == MINUS_EXPR)
1434 140667 : && (alt_stmt_code == PLUS_EXPR
1435 132090 : || alt_stmt_code == MINUS_EXPR))
1436 1409341 : || ((first_stmt_code == CFN_FMA
1437 1409339 : || first_stmt_code == CFN_FMS)
1438 2 : && (alt_stmt_code == CFN_FMA
1439 2 : || alt_stmt_code == CFN_FMS)))
1440 29021 : && rhs_code == alt_stmt_code)
1441 1451014 : && !(first_stmt_code.is_tree_code ()
1442 1334654 : && rhs_code.is_tree_code ()
1443 1239131 : && (TREE_CODE_CLASS (tree_code (first_stmt_code))
1444 : == tcc_comparison)
1445 135764 : && (swap_tree_comparison (tree_code (first_stmt_code))
1446 135764 : == tree_code (rhs_code))
1447 : && (first_reduc_idx == -1
1448 0 : || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
1449 : || (ldst_p
1450 5182908 : && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1451 2591454 : != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
1452 : || (ldst_p
1453 2546142 : && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1454 2546142 : != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
1455 10537015 : || first_stmt_ldst_p != ldst_p
1456 10536872 : || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
1457 22487220 : || first_stmt_phi_p != phi_p)
1458 : {
1459 1413492 : if (dump_enabled_p ())
1460 : {
1461 3133 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1462 : "Build SLP failed: different operation "
1463 : "in stmt %G", stmt);
1464 3133 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1465 : "original stmt %G", first_stmt_info->stmt);
1466 : }
1467 : /* Mismatch. */
1468 1413492 : continue;
1469 : }
1470 :
1471 10538899 : if (!ldst_p
1472 7990857 : && first_stmt_code == BIT_FIELD_REF
1473 10542081 : && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
1474 5217 : != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
1475 : {
1476 2035 : if (dump_enabled_p ())
1477 40 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1478 : "Build SLP failed: different BIT_FIELD_REF "
1479 : "arguments in %G", stmt);
1480 : /* Mismatch. */
1481 2035 : continue;
1482 : }
1483 :
1484 10534829 : if (call_stmt
1485 22698 : && first_stmt_code != CFN_MASK_LOAD
1486 10557125 : && first_stmt_code != CFN_MASK_STORE)
1487 : {
1488 21985 : if (!is_a <gcall *> (stmts[0]->stmt)
1489 21985 : || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
1490 : call_stmt, true))
1491 : {
1492 5256 : if (dump_enabled_p ())
1493 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1494 : "Build SLP failed: different calls in %G",
1495 : stmt);
1496 : /* Mismatch. */
1497 5256 : continue;
1498 : }
1499 : }
1500 :
1501 10343083 : if ((phi_p || gimple_could_trap_p (stmt_info->stmt))
1502 11338176 : && (gimple_bb (first_stmt_info->stmt)
1503 995093 : != gimple_bb (stmt_info->stmt)))
1504 : {
1505 28425 : if (dump_enabled_p ())
1506 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1507 : "Build SLP failed: different BB for PHI "
1508 : "or possibly trapping operation in %G", stmt);
1509 : /* Mismatch. */
1510 28425 : continue;
1511 : }
1512 :
1513 10501148 : if (need_same_oprnds)
1514 : {
1515 55114 : tree other_op1 = gimple_arg (stmt, 1);
1516 55114 : if (!operand_equal_p (first_op1, other_op1, 0))
1517 : {
1518 6329 : if (dump_enabled_p ())
1519 123 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1520 : "Build SLP failed: different shift "
1521 : "arguments in %G", stmt);
1522 : /* Mismatch. */
1523 6329 : continue;
1524 : }
1525 : }
1526 :
1527 10495556 : if (first_lhs
1528 10494819 : && lhs
1529 10494819 : && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
1530 : {
1531 737 : if (dump_enabled_p ())
1532 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1533 : "Build SLP failed: different vector type "
1534 : "in %G", stmt);
1535 : /* Mismatch. */
1536 737 : continue;
1537 : }
1538 : }
1539 :
1540 : /* Grouped store or load. */
1541 15664508 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1542 : {
1543 3932609 : gcc_assert (ldst_p);
1544 3932609 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
1545 : {
1546 : /* Store. */
1547 3083134 : gcc_assert (rhs_code == CFN_MASK_STORE
1548 : || REFERENCE_CLASS_P (lhs)
1549 : || DECL_P (lhs));
1550 : }
1551 : else
1552 : {
1553 : /* Load. */
1554 849475 : first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
1555 849475 : if (prev_first_load)
1556 : {
1557 : /* Check that there are no loads from different interleaving
1558 : chains in the same node. */
1559 387529 : if (prev_first_load != first_load)
1560 : {
1561 55091 : if (dump_enabled_p ())
1562 1998 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
1563 : vect_location,
1564 : "Build SLP failed: different "
1565 : "interleaving chains in one node %G",
1566 : stmt);
1567 : /* Mismatch. */
1568 55091 : continue;
1569 : }
1570 : }
1571 : else
1572 : prev_first_load = first_load;
1573 : }
1574 : }
1575 : /* Non-grouped store or load. */
1576 11731899 : else if (ldst_p)
1577 : {
1578 904521 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1579 631472 : && rhs_code != CFN_GATHER_LOAD
1580 : && rhs_code != CFN_MASK_GATHER_LOAD
1581 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1582 : && rhs_code != CFN_SCATTER_STORE
1583 : && rhs_code != CFN_MASK_SCATTER_STORE
1584 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1585 631472 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1586 : /* Not grouped loads are handled as externals for BB
1587 : vectorization. For loop vectorization we can handle
1588 : splats the same we handle single element interleaving.
1589 : Likewise we can handle a collection of invariant refs. */
1590 1517036 : && (is_a <bb_vec_info> (vinfo)
1591 612515 : || (stmt_info != first_stmt_info
1592 68088 : && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
1593 241 : && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
1594 : (first_stmt_info)))))))
1595 : {
1596 : /* Not grouped load. */
1597 67606 : if (dump_enabled_p ())
1598 145 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1599 : "Build SLP failed: not grouped load %G", stmt);
1600 :
1601 67606 : if (i != 0)
1602 67606 : continue;
1603 : /* Fatal mismatch. */
1604 0 : matches[0] = false;
1605 0 : return false;
1606 : }
1607 : }
1608 : /* Not memory operation. */
1609 : else
1610 : {
1611 10827378 : if (!phi_p
1612 10155438 : && rhs_code.is_tree_code ()
1613 10108289 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
1614 1608886 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
1615 1022017 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
1616 960145 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
1617 68300 : && rhs_code != VIEW_CONVERT_EXPR
1618 : && rhs_code != CALL_EXPR
1619 : && rhs_code != BIT_FIELD_REF
1620 10827378 : && rhs_code != SSA_NAME)
1621 : {
1622 20559 : if (dump_enabled_p ())
1623 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1624 : "Build SLP failed: operation unsupported %G",
1625 : stmt);
1626 20559 : if (is_a <bb_vec_info> (vinfo) && i != 0)
1627 0 : continue;
1628 : /* Fatal mismatch. */
1629 20559 : matches[0] = false;
1630 20559 : return false;
1631 : }
1632 :
1633 10806819 : if (rhs_code == COND_EXPR)
1634 : {
1635 59022 : tree cond_expr = gimple_assign_rhs1 (stmt);
1636 59022 : enum tree_code cond_code = TREE_CODE (cond_expr);
1637 59022 : enum tree_code swap_code = ERROR_MARK;
1638 59022 : enum tree_code invert_code = ERROR_MARK;
1639 :
1640 59022 : if (i == 0)
1641 49834 : first_cond_code = TREE_CODE (cond_expr);
1642 9188 : else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
1643 : {
1644 0 : bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
1645 0 : swap_code = swap_tree_comparison (cond_code);
1646 0 : invert_code = invert_tree_comparison (cond_code, honor_nans);
1647 : }
1648 :
1649 59022 : if (first_cond_code == cond_code)
1650 : ;
1651 : /* Isomorphic can be achieved by swapping. */
1652 0 : else if (first_cond_code == swap_code)
1653 0 : swap[i] = 1;
1654 : /* Isomorphic can be achieved by inverting. */
1655 0 : else if (first_cond_code == invert_code)
1656 0 : swap[i] = 2;
1657 : else
1658 : {
1659 0 : if (dump_enabled_p ())
1660 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1661 : "Build SLP failed: different"
1662 : " operation %G", stmt);
1663 : /* Mismatch. */
1664 0 : continue;
1665 : }
1666 : }
1667 :
1668 10806819 : if (i != 0
1669 7950125 : && first_stmt_code != rhs_code
1670 70350 : && first_stmt_code.is_tree_code ()
1671 70348 : && rhs_code.is_tree_code ()
1672 70348 : && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
1673 10848321 : && (swap_tree_comparison ((tree_code)first_stmt_code)
1674 41502 : == (tree_code)rhs_code))
1675 41502 : swap[i] = 1;
1676 :
1677 10806819 : if (i != 0
1678 7950125 : && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1679 1686 : && first_reduc_idx != -1
1680 1686 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1681 1686 : && rhs_code.is_tree_code ()
1682 1678 : && commutative_tree_code (tree_code (rhs_code))
1683 10808495 : && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
1684 1676 : swap[i] = 1;
1685 : }
1686 :
1687 15521252 : matches[i] = true;
1688 : }
1689 :
1690 20678919 : for (i = 0; i < group_size; ++i)
1691 16242297 : if (!matches[i])
1692 : return false;
1693 :
1694 : /* If we allowed a two-operation SLP node verify the target can cope
1695 : with the permute we are going to use. */
1696 4436622 : if (alt_stmt_code != ERROR_MARK
1697 4436622 : && (!alt_stmt_code.is_tree_code ()
1698 54430 : || (TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_reference
1699 54430 : && TREE_CODE_CLASS (tree_code (alt_stmt_code)) != tcc_comparison)))
1700 : {
1701 14501 : *two_operators = true;
1702 : }
1703 :
1704 4436622 : if (maybe_soft_fail)
1705 : {
1706 145703 : unsigned HOST_WIDE_INT const_nunits;
1707 145703 : if (!TYPE_VECTOR_SUBPARTS
1708 145703 : (soft_fail_nunits_vectype).is_constant (&const_nunits)
1709 145703 : || const_nunits > group_size)
1710 0 : matches[0] = false;
1711 : else
1712 : {
1713 : /* With constant vector elements simulate a mismatch at the
1714 : point we need to split. */
1715 145703 : unsigned tail = group_size & (const_nunits - 1);
1716 145703 : memset (&matches[group_size - tail], 0, sizeof (bool) * tail);
1717 : }
1718 145703 : return false;
1719 : }
1720 :
1721 : return true;
1722 : }
1723 :
1724 : /* Traits for the hash_set to record failed SLP builds for a stmt set.
1725 : Note we never remove apart from at destruction time so we do not
1726 : need a special value for deleted that differs from empty. */
1727 : struct bst_traits
1728 : {
1729 : typedef vec <stmt_vec_info> value_type;
1730 : typedef vec <stmt_vec_info> compare_type;
1731 : static inline hashval_t hash (value_type);
1732 : static inline bool equal (value_type existing, value_type candidate);
1733 488140062 : static inline bool is_empty (value_type x) { return !x.exists (); }
1734 109127264 : static inline bool is_deleted (value_type x) { return !x.exists (); }
1735 : static const bool empty_zero_p = true;
1736 0 : static inline void mark_empty (value_type &x) { x.release (); }
1737 : static inline void mark_deleted (value_type &x) { x.release (); }
1738 9405119 : static inline void remove (value_type &x) { x.release (); }
1739 : };
1740 : inline hashval_t
1741 95022915 : bst_traits::hash (value_type x)
1742 : {
1743 95022915 : inchash::hash h;
1744 428240820 : for (unsigned i = 0; i < x.length (); ++i)
1745 333217905 : h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
1746 95022915 : return h.end ();
1747 : }
1748 : inline bool
1749 83113409 : bst_traits::equal (value_type existing, value_type candidate)
1750 : {
1751 249340227 : if (existing.length () != candidate.length ())
1752 : return false;
1753 84415806 : for (unsigned i = 0; i < existing.length (); ++i)
1754 80024426 : if (existing[i] != candidate[i])
1755 : return false;
1756 : return true;
1757 : }
1758 :
1759 : typedef hash_map <vec <stmt_vec_info>, slp_tree,
1760 : simple_hashmap_traits <bst_traits, slp_tree> >
1761 : scalar_stmts_to_slp_tree_map_t;
1762 :
1763 : /* Release BST_MAP. */
1764 :
1765 : static void
1766 1838178 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
1767 : {
1768 : /* The map keeps a reference on SLP nodes built, release that. */
1769 11243297 : for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
1770 20648416 : it != bst_map->end (); ++it)
1771 9405119 : if ((*it).second)
1772 9405119 : vect_free_slp_tree ((*it).second);
1773 1838178 : delete bst_map;
1774 1838178 : }
1775 :
1776 : /* ??? This was std::pair<std::pair<tree_code, vect_def_type>, tree>
1777 : but then vec::insert does memmove and that's not compatible with
1778 : std::pair. */
1779 : struct chain_op_t
1780 : {
1781 3832239 : chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
1782 3832239 : : code (code_), dt (dt_), op (op_) {}
1783 : tree_code code;
1784 : vect_def_type dt;
1785 : tree op;
1786 : };
1787 :
1788 : /* Comparator for sorting associatable chains. */
1789 :
1790 : static int
1791 8512455 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
1792 : {
1793 8512455 : auto *op1 = (const chain_op_t *) op1_;
1794 8512455 : auto *op2 = (const chain_op_t *) op2_;
1795 8512455 : if (op1->dt != op2->dt)
1796 955424 : return (int)op1->dt - (int)op2->dt;
1797 7557031 : return (int)op1->code - (int)op2->code;
1798 : }
1799 :
1800 : /* Linearize the associatable expression chain at START with the
1801 : associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
1802 : filling CHAIN with the result and using WORKLIST as intermediate storage.
1803 : CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
1804 : or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation
1805 : stmts, starting with START. When ALLOW_ALT_CODE is false, do not
1806 : follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf). */
1807 :
1808 : static void
1809 1736450 : vect_slp_linearize_chain (vec_info *vinfo,
1810 : vec<std::pair<tree_code, gimple *> > &worklist,
1811 : vec<chain_op_t> &chain,
1812 : enum tree_code code, gimple *start,
1813 : gimple *&code_stmt, gimple *&alt_code_stmt,
1814 : vec<gimple *> *chain_stmts,
1815 : bool allow_alt_code = true)
1816 : {
1817 : /* For each lane linearize the addition/subtraction (or other
1818 : uniform associatable operation) expression tree. */
1819 1736450 : worklist.safe_push (std::make_pair (code, start));
1820 3832239 : while (!worklist.is_empty ())
1821 : {
1822 2095789 : auto entry = worklist.pop ();
1823 2095789 : gassign *stmt = as_a <gassign *> (entry.second);
1824 2095789 : enum tree_code in_code = entry.first;
1825 4191578 : enum tree_code this_code = gimple_assign_rhs_code (stmt);
1826 : /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE. */
1827 2095789 : if (!code_stmt
1828 2095789 : && gimple_assign_rhs_code (stmt) == code)
1829 1477654 : code_stmt = stmt;
1830 618135 : else if (!alt_code_stmt
1831 618135 : && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
1832 311579 : alt_code_stmt = stmt;
1833 2095789 : if (chain_stmts)
1834 2015008 : chain_stmts->safe_push (stmt);
1835 6287367 : for (unsigned opnum = 1; opnum <= 2; ++opnum)
1836 : {
1837 4191578 : tree op = gimple_op (stmt, opnum);
1838 4191578 : vect_def_type dt;
1839 4191578 : stmt_vec_info def_stmt_info;
1840 4191578 : bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
1841 4191578 : gcc_assert (res);
1842 4191578 : if (dt == vect_internal_def
1843 4191578 : && is_pattern_stmt_p (def_stmt_info))
1844 9019 : op = gimple_get_lhs (def_stmt_info->stmt);
1845 4191578 : gimple *use_stmt;
1846 4191578 : use_operand_p use_p;
1847 4191578 : if (dt == vect_internal_def
1848 3896056 : && single_imm_use (op, &use_p, &use_stmt)
1849 2449886 : && is_gimple_assign (def_stmt_info->stmt)
1850 6460382 : && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
1851 1909780 : || (allow_alt_code
1852 57047 : && code == PLUS_EXPR
1853 36181 : && (gimple_assign_rhs_code (def_stmt_info->stmt)
1854 : == MINUS_EXPR))))
1855 : {
1856 359339 : tree_code op_def_code = this_code;
1857 359339 : if (op_def_code == MINUS_EXPR && opnum == 1)
1858 51070 : op_def_code = PLUS_EXPR;
1859 359339 : if (in_code == MINUS_EXPR)
1860 135 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1861 359339 : worklist.safe_push (std::make_pair (op_def_code,
1862 359339 : def_stmt_info->stmt));
1863 : }
1864 : else
1865 : {
1866 3832239 : tree_code op_def_code = this_code;
1867 3832239 : if (op_def_code == MINUS_EXPR && opnum == 1)
1868 260626 : op_def_code = PLUS_EXPR;
1869 3832239 : if (in_code == MINUS_EXPR)
1870 4017 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1871 3832239 : chain.safe_push (chain_op_t (op_def_code, dt, op));
1872 : }
1873 : }
1874 : }
1875 1736450 : }
1876 :
1877 : /* Distance from the node currently being discovered to the closest upthread
1878 : commutative operation whose operand-zero discovery may still be fixed by
1879 : retrying with swapped operands, or -1U if there is none. */
1880 :
1881 : static unsigned least_upthread_swappable_op_distance = -1U;
1882 :
1883 : static slp_tree
1884 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
1885 : vec<stmt_vec_info> stmts,
1886 : poly_uint64 *max_nunits,
1887 : bool *matches, unsigned *limit, unsigned *tree_size,
1888 : scalar_stmts_to_slp_tree_map_t *bst_map);
1889 :
1890 : static slp_tree
1891 6380307 : vect_build_slp_tree (vec_info *vinfo,
1892 : vec<stmt_vec_info> stmts,
1893 : poly_uint64 *max_nunits,
1894 : bool *matches, unsigned *limit, unsigned *tree_size,
1895 : scalar_stmts_to_slp_tree_map_t *bst_map)
1896 : {
1897 6380307 : unsigned int group_size = stmts.length ();
1898 6380307 : if (slp_tree *leader = bst_map->get (stmts))
1899 : {
1900 480388 : if (dump_enabled_p ())
1901 17111 : dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
1902 17111 : !(*leader)->failed ? "" : "failed ",
1903 : (void *) *leader);
1904 480388 : if (!(*leader)->failed)
1905 : {
1906 432649 : SLP_TREE_REF_COUNT (*leader)++;
1907 432649 : vect_update_max_nunits (max_nunits, (*leader)->max_nunits);
1908 432649 : stmts.release ();
1909 432649 : return *leader;
1910 : }
1911 47739 : memcpy (matches, (*leader)->failed, sizeof (bool) * group_size);
1912 47739 : return NULL;
1913 : }
1914 :
1915 : /* Single-lane SLP doesn't have the chance of run-away, do not account
1916 : it to the limit. */
1917 5899919 : if (stmts.length () > 1)
1918 : {
1919 3275877 : if (*limit == 0)
1920 : {
1921 1238 : if (dump_enabled_p ())
1922 15 : dump_printf_loc (MSG_NOTE, vect_location,
1923 : "SLP discovery limit exceeded\n");
1924 1238 : memset (matches, 0, sizeof (bool) * group_size);
1925 1238 : return NULL;
1926 : }
1927 3274639 : --*limit;
1928 : }
1929 :
1930 : /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
1931 : so we can pick up backedge destinations during discovery. */
1932 5898681 : slp_tree res = new _slp_tree;
1933 5898681 : SLP_TREE_DEF_TYPE (res) = vect_internal_def;
1934 5898681 : SLP_TREE_SCALAR_STMTS (res) = stmts;
1935 5898681 : bst_map->put (stmts.copy (), res);
1936 :
1937 5898681 : if (dump_enabled_p ())
1938 146613 : dump_printf_loc (MSG_NOTE, vect_location,
1939 : "starting SLP discovery for node %p\n", (void *) res);
1940 :
1941 5898681 : poly_uint64 this_max_nunits = 1;
1942 5898681 : slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
1943 : &this_max_nunits,
1944 : matches, limit, tree_size, bst_map);
1945 5898681 : if (!res_)
1946 : {
1947 2098446 : if (dump_enabled_p ())
1948 8494 : dump_printf_loc (MSG_NOTE, vect_location,
1949 : "SLP discovery for node %p failed\n", (void *) res);
1950 : /* Mark the node invalid so we can detect those when still in use
1951 : as backedge destinations. */
1952 2098446 : SLP_TREE_SCALAR_STMTS (res) = vNULL;
1953 2098446 : SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
1954 2098446 : res->failed = XNEWVEC (bool, group_size);
1955 2098446 : if (flag_checking)
1956 : {
1957 : unsigned i;
1958 3737422 : for (i = 0; i < group_size; ++i)
1959 3737422 : if (!matches[i])
1960 : break;
1961 2098446 : gcc_assert (i < group_size);
1962 : }
1963 2098446 : memcpy (res->failed, matches, sizeof (bool) * group_size);
1964 : }
1965 : else
1966 : {
1967 3800235 : if (dump_enabled_p ())
1968 138119 : dump_printf_loc (MSG_NOTE, vect_location,
1969 : "SLP discovery for node %p succeeded\n",
1970 : (void *) res);
1971 3800235 : gcc_assert (res_ == res);
1972 3800235 : res->max_nunits = this_max_nunits;
1973 3800235 : vect_update_max_nunits (max_nunits, this_max_nunits);
1974 : /* Keep a reference for the bst_map use. */
1975 3800235 : SLP_TREE_REF_COUNT (res)++;
1976 : }
1977 : return res_;
1978 : }
1979 :
1980 : /* Helper for building an associated SLP node chain. */
1981 :
1982 : static void
1983 158 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
1984 : slp_tree op0, slp_tree op1,
1985 : stmt_vec_info oper1, stmt_vec_info oper2,
1986 : vec<std::pair<unsigned, unsigned> > lperm)
1987 : {
1988 158 : unsigned group_size = SLP_TREE_LANES (op1);
1989 :
1990 158 : slp_tree child1 = new _slp_tree;
1991 158 : SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
1992 158 : SLP_TREE_VECTYPE (child1) = vectype;
1993 158 : SLP_TREE_LANES (child1) = group_size;
1994 158 : SLP_TREE_CHILDREN (child1).create (2);
1995 158 : SLP_TREE_CHILDREN (child1).quick_push (op0);
1996 158 : SLP_TREE_CHILDREN (child1).quick_push (op1);
1997 158 : SLP_TREE_REPRESENTATIVE (child1) = oper1;
1998 :
1999 158 : slp_tree child2 = new _slp_tree;
2000 158 : SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
2001 158 : SLP_TREE_VECTYPE (child2) = vectype;
2002 158 : SLP_TREE_LANES (child2) = group_size;
2003 158 : SLP_TREE_CHILDREN (child2).create (2);
2004 158 : SLP_TREE_CHILDREN (child2).quick_push (op0);
2005 158 : SLP_TREE_REF_COUNT (op0)++;
2006 158 : SLP_TREE_CHILDREN (child2).quick_push (op1);
2007 158 : SLP_TREE_REF_COUNT (op1)++;
2008 158 : SLP_TREE_REPRESENTATIVE (child2) = oper2;
2009 :
2010 158 : SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
2011 158 : SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
2012 158 : SLP_TREE_VECTYPE (perm) = vectype;
2013 158 : SLP_TREE_LANES (perm) = group_size;
2014 : /* ??? We should set this NULL but that's not expected. */
2015 158 : SLP_TREE_REPRESENTATIVE (perm) = oper1;
2016 158 : SLP_TREE_LANE_PERMUTATION (perm) = lperm;
2017 158 : SLP_TREE_CHILDREN (perm).quick_push (child1);
2018 158 : SLP_TREE_CHILDREN (perm).quick_push (child2);
2019 158 : }
2020 :
2021 : /* Recursively build an SLP tree starting from NODE.
2022 : Fail (and return a value not equal to zero) if def-stmts are not
2023 : isomorphic, require data permutation or are of unsupported types of
2024 : operation. Otherwise, return 0.
2025 : The value returned is the depth in the SLP tree where a mismatch
2026 : was found. */
2027 :
2028 : static slp_tree
2029 5898681 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
2030 : vec<stmt_vec_info> stmts,
2031 : poly_uint64 *max_nunits,
2032 : bool *matches, unsigned *limit, unsigned *tree_size,
2033 : scalar_stmts_to_slp_tree_map_t *bst_map)
2034 : {
2035 5898681 : unsigned int group_size = stmts.length ();
2036 5898681 : unsigned nops, i, this_tree_size = 0;
2037 5898681 : poly_uint64 this_max_nunits = *max_nunits;
2038 :
2039 5898681 : matches[0] = false;
2040 :
2041 5898681 : stmt_vec_info stmt_info = stmts[0];
2042 5898681 : if (!is_a<gcall *> (stmt_info->stmt)
2043 : && !is_a<gassign *> (stmt_info->stmt)
2044 : && !is_a<gphi *> (stmt_info->stmt))
2045 : return NULL;
2046 :
2047 5898595 : nops = gimple_num_args (stmt_info->stmt);
2048 5898595 : if (const int *map = vect_get_operand_map (stmt_info))
2049 35205 : nops = map[0];
2050 :
2051 : /* If the SLP node is a PHI (induction or reduction), terminate
2052 : the recursion. */
2053 5898595 : bool *skip_args = XALLOCAVEC (bool, nops);
2054 5898595 : memset (skip_args, 0, sizeof (bool) * nops);
2055 5898595 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2056 2821181 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
2057 : {
2058 303704 : tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
2059 303704 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
2060 : group_size);
2061 303704 : if (!vect_record_max_nunits (vinfo, stmt_info, group_size, vectype,
2062 : max_nunits))
2063 : return NULL;
2064 :
2065 299647 : vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
2066 299647 : if (def_type == vect_induction_def)
2067 : {
2068 : /* Induction PHIs are not cycles but walk the initial
2069 : value. Only for inner loops through, for outer loops
2070 : we need to pick up the value from the actual PHIs
2071 : to more easily support peeling and epilogue vectorization. */
2072 193348 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2073 193348 : if (!nested_in_vect_loop_p (loop, stmt_info))
2074 192524 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2075 : else
2076 : loop = loop->inner;
2077 193348 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2078 : }
2079 106299 : else if (def_type == vect_reduction_def
2080 : || def_type == vect_double_reduction_def
2081 : || def_type == vect_nested_cycle
2082 106299 : || def_type == vect_first_order_recurrence)
2083 : {
2084 : /* Else def types have to match. */
2085 : stmt_vec_info other_info;
2086 : bool all_same = true;
2087 240806 : FOR_EACH_VEC_ELT (stmts, i, other_info)
2088 : {
2089 135821 : if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
2090 1840077 : return NULL;
2091 135815 : if (other_info != stmt_info)
2092 26231 : all_same = false;
2093 : }
2094 104985 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2095 : /* Reduction initial values are not explicitly represented. */
2096 104985 : if (def_type != vect_first_order_recurrence
2097 104985 : && gimple_bb (stmt_info->stmt) == loop->header)
2098 101820 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2099 : /* Reduction chain backedge defs are filled manually.
2100 : ??? Need a better way to identify a SLP reduction chain PHI.
2101 : Or a better overall way to SLP match those. */
2102 104985 : if (stmts.length () > 1
2103 104985 : && all_same && def_type == vect_reduction_def)
2104 2354 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2105 : }
2106 1308 : else if (def_type != vect_internal_def)
2107 : return NULL;
2108 : }
2109 :
2110 :
2111 5894532 : bool two_operators = false;
2112 5894532 : unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
2113 5894532 : tree vectype = NULL_TREE;
2114 5894532 : if (!vect_build_slp_tree_1 (vinfo, swap, stmts,
2115 : &this_max_nunits, matches, &two_operators,
2116 : &vectype))
2117 : return NULL;
2118 :
2119 : /* If the SLP node is a load, terminate the recursion unless masked. */
2120 4290919 : if (STMT_VINFO_DATA_REF (stmt_info)
2121 2070563 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
2122 : {
2123 920360 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2124 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
2125 : else
2126 : {
2127 901660 : *max_nunits = this_max_nunits;
2128 901660 : (*tree_size)++;
2129 901660 : node = vect_create_new_slp_node (node, stmts, 0);
2130 901660 : SLP_TREE_VECTYPE (node) = vectype;
2131 : /* And compute the load permutation. Whether it is actually
2132 : a permutation depends on the unrolling factor which is
2133 : decided later. */
2134 901660 : vec<unsigned> load_permutation;
2135 901660 : int j;
2136 901660 : stmt_vec_info load_info;
2137 901660 : load_permutation.create (group_size);
2138 901660 : stmt_vec_info first_stmt_info
2139 901660 : = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2140 901660 : ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
2141 901660 : bool any_permute = false;
2142 2171241 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
2143 : {
2144 1269581 : int load_place;
2145 1269581 : if (! load_info)
2146 : {
2147 40469 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2148 : load_place = j;
2149 : else
2150 : load_place = 0;
2151 : }
2152 1229112 : else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2153 712331 : load_place = vect_get_place_in_interleaving_chain
2154 712331 : (load_info, first_stmt_info);
2155 : else
2156 : /* Recognize the splat case as { 0, 0, ... } but make
2157 : sure to use the appropriate refs for collections
2158 : of invariant refs. */
2159 516781 : load_place = (load_info == stmt_info) ? 0 : j;
2160 753041 : gcc_assert (load_place != -1);
2161 1269581 : any_permute |= load_place != j;
2162 1269581 : load_permutation.quick_push (load_place);
2163 : }
2164 :
2165 901660 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
2166 : {
2167 3386 : gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
2168 3386 : bool has_gaps = false;
2169 3386 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2170 189 : for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
2171 846 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2172 657 : if (DR_GROUP_GAP (si) != 1)
2173 80 : has_gaps = true;
2174 : /* We cannot handle permuted masked loads directly, see
2175 : PR114375. We cannot handle strided masked loads or masked
2176 : loads with gaps unless the mask is uniform. */
2177 3386 : if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
2178 189 : && (DR_GROUP_GAP (first_stmt_info) != 0
2179 129 : || (has_gaps
2180 35 : && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
2181 6692 : || STMT_VINFO_STRIDED_P (stmt_info))
2182 : {
2183 93 : load_permutation.release ();
2184 93 : matches[0] = false;
2185 898406 : return NULL;
2186 : }
2187 :
2188 : /* For permuted masked loads do an unpermuted masked load of
2189 : the whole group followed by a SLP permute node. */
2190 3293 : if (any_permute
2191 3293 : || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
2192 83 : && DR_GROUP_SIZE (first_stmt_info) != group_size))
2193 : {
2194 : /* Discover the whole unpermuted load. */
2195 39 : vec<stmt_vec_info> stmts2;
2196 39 : unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2197 68 : ? DR_GROUP_SIZE (first_stmt_info) : 1;
2198 39 : stmts2.create (dr_group_size);
2199 39 : stmts2.quick_grow_cleared (dr_group_size);
2200 39 : unsigned i = 0;
2201 39 : for (stmt_vec_info si = first_stmt_info;
2202 464 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2203 : {
2204 425 : if (si != first_stmt_info)
2205 1586 : for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
2206 1200 : stmts2[i++] = NULL;
2207 425 : stmts2[i++] = si;
2208 : }
2209 39 : bool *matches2 = XALLOCAVEC (bool, dr_group_size);
2210 39 : slp_tree unperm_load
2211 39 : = vect_build_slp_tree (vinfo, stmts2,
2212 : &this_max_nunits, matches2, limit,
2213 39 : &this_tree_size, bst_map);
2214 : /* When we are able to do the full masked load emit that
2215 : followed by 'node' being the desired final permutation. */
2216 39 : if (unperm_load)
2217 : {
2218 16 : gcc_assert
2219 : (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
2220 16 : lane_permutation_t lperm;
2221 16 : lperm.create (group_size);
2222 56 : for (unsigned j = 0; j < load_permutation.length (); ++j)
2223 40 : lperm.quick_push
2224 40 : (std::make_pair (0, load_permutation[j]));
2225 16 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2226 16 : SLP_TREE_CHILDREN (node).safe_push (unperm_load);
2227 16 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2228 16 : load_permutation.release ();
2229 16 : return node;
2230 : }
2231 23 : stmts2.release ();
2232 23 : load_permutation.release ();
2233 23 : matches[0] = false;
2234 23 : return NULL;
2235 : }
2236 3254 : load_permutation.release ();
2237 : }
2238 : else
2239 : {
2240 898274 : if (!any_permute
2241 783256 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2242 1190673 : && group_size == DR_GROUP_SIZE (first_stmt_info))
2243 128135 : load_permutation.release ();
2244 898274 : SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
2245 898274 : return node;
2246 : }
2247 : }
2248 : }
2249 3370559 : else if (gimple_assign_single_p (stmt_info->stmt)
2250 2304792 : && !gimple_vuse (stmt_info->stmt)
2251 3378265 : && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
2252 : {
2253 : /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
2254 : the same SSA name vector of a compatible type to vectype. */
2255 2245 : vec<std::pair<unsigned, unsigned> > lperm = vNULL;
2256 2245 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
2257 2245 : stmt_vec_info estmt_info;
2258 7077 : FOR_EACH_VEC_ELT (stmts, i, estmt_info)
2259 : {
2260 4979 : gassign *estmt = as_a <gassign *> (estmt_info->stmt);
2261 4979 : tree bfref = gimple_assign_rhs1 (estmt);
2262 4979 : HOST_WIDE_INT lane;
2263 4979 : if (!known_eq (bit_field_size (bfref),
2264 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
2265 9811 : || !constant_multiple_p (bit_field_offset (bfref),
2266 4832 : bit_field_size (bfref), &lane))
2267 : {
2268 147 : lperm.release ();
2269 147 : matches[0] = false;
2270 147 : return NULL;
2271 : }
2272 4832 : lperm.safe_push (std::make_pair (0, (unsigned)lane));
2273 : }
2274 2098 : slp_tree vnode = vect_create_new_slp_node (vNULL);
2275 2098 : if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
2276 : /* ??? We record vectype here but we hide eventually necessary
2277 : punning and instead rely on code generation to materialize
2278 : VIEW_CONVERT_EXPRs as necessary. We instead should make
2279 : this explicit somehow. */
2280 628 : SLP_TREE_VECTYPE (vnode) = vectype;
2281 : else
2282 : {
2283 : /* For different size but compatible elements we can still
2284 : use VEC_PERM_EXPR without punning. */
2285 1470 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
2286 : && types_compatible_p (TREE_TYPE (vectype),
2287 : TREE_TYPE (TREE_TYPE (vec))));
2288 1470 : SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
2289 : }
2290 2098 : auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
2291 2098 : unsigned HOST_WIDE_INT const_nunits;
2292 2098 : if (nunits.is_constant (&const_nunits))
2293 2098 : SLP_TREE_LANES (vnode) = const_nunits;
2294 2098 : SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
2295 : /* We are always building a permutation node even if it is an identity
2296 : permute to shield the rest of the vectorizer from the odd node
2297 : representing an actual vector without any scalar ops.
2298 : ??? We could hide it completely with making the permute node
2299 : external? */
2300 2098 : node = vect_create_new_slp_node (node, stmts, 1);
2301 2098 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2302 2098 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2303 2098 : SLP_TREE_VECTYPE (node) = vectype;
2304 2098 : SLP_TREE_CHILDREN (node).quick_push (vnode);
2305 2098 : return node;
2306 : }
2307 : /* When discovery reaches an associatable operation see whether we can
2308 : improve that to match up lanes in a way superior to the operand
2309 : swapping code which at most looks at two defs.
2310 : ??? For BB vectorization we cannot do the brute-force search
2311 : for matching as we can succeed by means of builds from scalars
2312 : and have no good way to "cost" one build against another. */
2313 3368314 : else if (is_a <loop_vec_info> (vinfo)
2314 : /* Do not bother for single-lane SLP. */
2315 1988383 : && group_size > 1
2316 : /* ??? We don't handle !vect_internal_def defs below. */
2317 112640 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
2318 : /* ??? Do not associate a reduction, this will wreck REDUC_IDX
2319 : mapping as long as that exists on the stmt_info level. */
2320 87124 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1
2321 78584 : && is_gimple_assign (stmt_info->stmt)
2322 78268 : && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
2323 51625 : || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
2324 3396842 : && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
2325 16273 : || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
2326 13742 : && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
2327 : {
2328 : /* See if we have a chain of (mixed) adds or subtracts or other
2329 : associatable ops. */
2330 21555 : enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
2331 21555 : if (code == MINUS_EXPR)
2332 799 : code = PLUS_EXPR;
2333 21555 : stmt_vec_info other_op_stmt_info = NULL;
2334 21555 : stmt_vec_info op_stmt_info = NULL;
2335 21555 : unsigned chain_len = 0;
2336 21555 : auto_vec<chain_op_t> chain;
2337 21555 : auto_vec<std::pair<tree_code, gimple *> > worklist;
2338 21555 : auto_vec<vec<chain_op_t> > chains (group_size);
2339 21555 : auto_vec<slp_tree, 4> children;
2340 21555 : bool hard_fail = true;
2341 22582 : for (unsigned lane = 0; lane < group_size; ++lane)
2342 : {
2343 22246 : if (!stmts[lane])
2344 : {
2345 : /* ??? Below we require lane zero is present. */
2346 0 : if (lane == 0)
2347 : {
2348 : hard_fail = false;
2349 21219 : break;
2350 : }
2351 0 : chains.quick_push (vNULL);
2352 0 : continue;
2353 : }
2354 : /* For each lane linearize the addition/subtraction (or other
2355 : uniform associatable operation) expression tree. */
2356 22246 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
2357 22246 : vect_slp_linearize_chain (vinfo, worklist, chain, code,
2358 22246 : stmts[lane]->stmt, op_stmt, other_op_stmt,
2359 : NULL);
2360 22246 : if (!op_stmt_info && op_stmt)
2361 20953 : op_stmt_info = vinfo->lookup_stmt (op_stmt);
2362 22246 : if (!other_op_stmt_info && other_op_stmt)
2363 835 : other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
2364 22246 : if (chain.length () == 2)
2365 : {
2366 : /* In a chain of just two elements resort to the regular
2367 : operand swapping scheme. Likewise if we run into a
2368 : length mismatch process regularly as well as we did not
2369 : process the other lanes we cannot report a good hint what
2370 : lanes to try swapping in the parent. */
2371 : hard_fail = false;
2372 : break;
2373 : }
2374 1030 : else if (chain_len == 0)
2375 376 : chain_len = chain.length ();
2376 1308 : else if (chain.length () != chain_len)
2377 : {
2378 : /* ??? Here we could slip in magic to compensate with
2379 : neutral operands. */
2380 3 : matches[lane] = false;
2381 3 : if (lane != group_size - 1)
2382 3 : matches[0] = false;
2383 : break;
2384 : }
2385 1027 : chains.quick_push (chain.copy ());
2386 1027 : chain.truncate (0);
2387 : }
2388 43110 : if (chains.length () == group_size)
2389 : {
2390 : /* We cannot yet use SLP_TREE_CODE to communicate the operation. */
2391 336 : if (!op_stmt_info)
2392 : {
2393 3 : hard_fail = false;
2394 3 : goto out;
2395 : }
2396 : /* Now we have a set of chains with the same length. */
2397 : /* 1. pre-sort according to def_type and operation. */
2398 1248 : for (unsigned lane = 0; lane < group_size; ++lane)
2399 1830 : chains[lane].stablesort (dt_sort_cmp, vinfo);
2400 333 : if (dump_enabled_p ())
2401 : {
2402 157 : dump_printf_loc (MSG_NOTE, vect_location,
2403 : "pre-sorted chains of %s\n",
2404 : get_tree_code_name (code));
2405 685 : for (unsigned lane = 0; lane < group_size; ++lane)
2406 : {
2407 528 : if (!stmts[lane])
2408 0 : dump_printf (MSG_NOTE, "--");
2409 : else
2410 2422 : for (unsigned opnum = 0; opnum < chain_len; ++opnum)
2411 3788 : dump_printf (MSG_NOTE, "%s %T ",
2412 1894 : get_tree_code_name (chains[lane][opnum].code),
2413 1894 : chains[lane][opnum].op);
2414 528 : dump_printf (MSG_NOTE, "\n");
2415 : }
2416 : }
2417 : /* 2. try to build children nodes, associating as necessary. */
2418 : /* 2a. prepare and perform early checks to avoid eating into
2419 : discovery limit unnecessarily. */
2420 333 : vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
2421 1407 : for (unsigned n = 0; n < chain_len; ++n)
2422 : {
2423 1074 : vect_def_type dt = chains[0][n].dt;
2424 1074 : unsigned lane;
2425 4177 : for (lane = 0; lane < group_size; ++lane)
2426 6206 : if (stmts[lane] && chains[lane][n].dt != dt)
2427 : {
2428 0 : if (dt == vect_constant_def
2429 0 : && chains[lane][n].dt == vect_external_def)
2430 : dt = vect_external_def;
2431 0 : else if (dt == vect_external_def
2432 0 : && chains[lane][n].dt == vect_constant_def)
2433 : ;
2434 : else
2435 : break;
2436 : }
2437 1074 : if (lane != group_size)
2438 : {
2439 0 : if (dump_enabled_p ())
2440 0 : dump_printf_loc (MSG_NOTE, vect_location,
2441 : "giving up on chain due to mismatched "
2442 : "def types\n");
2443 0 : matches[lane] = false;
2444 0 : if (lane != group_size - 1)
2445 0 : matches[0] = false;
2446 0 : goto out;
2447 : }
2448 1074 : dts[n] = dt;
2449 1074 : if (dt == vect_constant_def
2450 1074 : || dt == vect_external_def)
2451 : {
2452 : /* Check whether we can build the invariant. If we can't
2453 : we never will be able to. */
2454 93 : tree type = TREE_TYPE (chains[0][n].op);
2455 1074 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
2456 : && (TREE_CODE (type) == BOOLEAN_TYPE
2457 : || !can_duplicate_and_interleave_p (vinfo, group_size,
2458 : type)))
2459 : {
2460 : matches[0] = false;
2461 : goto out;
2462 : }
2463 : }
2464 981 : else if (dt != vect_internal_def)
2465 : {
2466 : /* Not sure, we might need sth special.
2467 : gcc.dg/vect/pr96854.c,
2468 : gfortran.dg/vect/fast-math-pr37021.f90
2469 : and gfortran.dg/vect/pr61171.f trigger. */
2470 : /* Soft-fail for now. */
2471 0 : hard_fail = false;
2472 0 : goto out;
2473 : }
2474 : }
2475 : /* 2b. do the actual build. */
2476 1349 : for (unsigned n = 0; n < chain_len; ++n)
2477 : {
2478 1036 : vect_def_type dt = dts[n];
2479 1036 : unsigned lane;
2480 1036 : if (dt == vect_constant_def
2481 1036 : || dt == vect_external_def)
2482 : {
2483 93 : vec<tree> ops;
2484 93 : ops.create (group_size);
2485 461 : for (lane = 0; lane < group_size; ++lane)
2486 275 : if (stmts[lane])
2487 275 : ops.quick_push (chains[lane][n].op);
2488 : else
2489 0 : ops.quick_push (NULL_TREE);
2490 93 : slp_tree child = vect_create_new_slp_node (ops);
2491 93 : SLP_TREE_DEF_TYPE (child) = dt;
2492 93 : children.safe_push (child);
2493 : }
2494 : else
2495 : {
2496 943 : vec<stmt_vec_info> op_stmts;
2497 943 : op_stmts.create (group_size);
2498 943 : slp_tree child = NULL;
2499 : /* Brute-force our way. We have to consider a lane
2500 : failing after fixing an earlier fail up in the
2501 : SLP discovery recursion. So track the current
2502 : permute per lane. */
2503 943 : unsigned *perms = XALLOCAVEC (unsigned, group_size);
2504 943 : memset (perms, 0, sizeof (unsigned) * group_size);
2505 1037 : do
2506 : {
2507 1037 : op_stmts.truncate (0);
2508 5080 : for (lane = 0; lane < group_size; ++lane)
2509 3006 : if (stmts[lane])
2510 3006 : op_stmts.quick_push
2511 3006 : (vinfo->lookup_def (chains[lane][n].op));
2512 : else
2513 0 : op_stmts.quick_push (NULL);
2514 1037 : child = vect_build_slp_tree (vinfo, op_stmts,
2515 : &this_max_nunits,
2516 : matches, limit,
2517 : &this_tree_size, bst_map);
2518 : /* ??? We're likely getting too many fatal mismatches
2519 : here so maybe we want to ignore them (but then we
2520 : have no idea which lanes fatally mismatched). */
2521 1037 : if (child || !matches[0])
2522 : break;
2523 : /* Swap another lane we have not yet matched up into
2524 : lanes that did not match. If we run out of
2525 : permute possibilities for a lane terminate the
2526 : search. */
2527 287 : bool term = false;
2528 287 : for (lane = 1; lane < group_size; ++lane)
2529 193 : if (!matches[lane])
2530 : {
2531 165 : if (n + perms[lane] + 1 == chain_len)
2532 : {
2533 : term = true;
2534 : break;
2535 : }
2536 146 : if (dump_enabled_p ())
2537 113 : dump_printf_loc (MSG_NOTE, vect_location,
2538 : "swapping operand %d and %d "
2539 : "of lane %d\n",
2540 : n, n + perms[lane] + 1, lane);
2541 292 : std::swap (chains[lane][n],
2542 146 : chains[lane][n + perms[lane] + 1]);
2543 146 : perms[lane]++;
2544 : }
2545 113 : if (term)
2546 : break;
2547 : }
2548 : while (1);
2549 943 : if (!child)
2550 : {
2551 20 : if (dump_enabled_p ())
2552 18 : dump_printf_loc (MSG_NOTE, vect_location,
2553 : "failed to match up op %d\n", n);
2554 20 : op_stmts.release ();
2555 20 : if (lane != group_size - 1)
2556 10 : matches[0] = false;
2557 : else
2558 10 : matches[lane] = false;
2559 20 : goto out;
2560 : }
2561 923 : if (dump_enabled_p ())
2562 : {
2563 421 : dump_printf_loc (MSG_NOTE, vect_location,
2564 : "matched up op %d to\n", n);
2565 421 : vect_print_slp_tree (MSG_NOTE, vect_location, child);
2566 : }
2567 923 : children.safe_push (child);
2568 : }
2569 : }
2570 : /* 3. build SLP nodes to combine the chain. */
2571 1153 : for (unsigned lane = 0; lane < group_size; ++lane)
2572 1692 : if (stmts[lane] && chains[lane][0].code != code)
2573 : {
2574 : /* See if there's any alternate all-PLUS entry. */
2575 : unsigned n;
2576 6 : for (n = 1; n < chain_len; ++n)
2577 : {
2578 30 : for (lane = 0; lane < group_size; ++lane)
2579 48 : if (stmts[lane] && chains[lane][n].code != code)
2580 : break;
2581 6 : if (lane == group_size)
2582 : break;
2583 : }
2584 6 : if (n != chain_len)
2585 : {
2586 : /* Swap that in at first position. */
2587 6 : std::swap (children[0], children[n]);
2588 30 : for (lane = 0; lane < group_size; ++lane)
2589 24 : if (stmts[lane])
2590 24 : std::swap (chains[lane][0], chains[lane][n]);
2591 : }
2592 : else
2593 : {
2594 : /* ??? When this triggers and we end up with two
2595 : vect_constant/external_def up-front things break (ICE)
2596 : spectacularly finding an insertion place for the
2597 : all-constant op. We should have a fully
2598 : vect_internal_def operand though(?) so we can swap
2599 : that into first place and then prepend the all-zero
2600 : constant. */
2601 0 : if (dump_enabled_p ())
2602 0 : dump_printf_loc (MSG_NOTE, vect_location,
2603 : "inserting constant zero to compensate "
2604 : "for (partially) negated first "
2605 : "operand\n");
2606 0 : chain_len++;
2607 0 : for (lane = 0; lane < group_size; ++lane)
2608 0 : if (stmts[lane])
2609 0 : chains[lane].safe_insert
2610 0 : (0, chain_op_t (code, vect_constant_def, NULL_TREE));
2611 0 : vec<tree> zero_ops;
2612 0 : zero_ops.create (group_size);
2613 0 : zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
2614 0 : for (lane = 1; lane < group_size; ++lane)
2615 0 : if (stmts[lane])
2616 0 : zero_ops.quick_push (zero_ops[0]);
2617 : else
2618 0 : zero_ops.quick_push (NULL_TREE);
2619 0 : slp_tree zero = vect_create_new_slp_node (zero_ops);
2620 0 : SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
2621 0 : children.safe_insert (0, zero);
2622 : }
2623 : break;
2624 : }
2625 1011 : for (unsigned i = 1; i < children.length (); ++i)
2626 : {
2627 698 : slp_tree op0 = children[i - 1];
2628 698 : slp_tree op1 = children[i];
2629 698 : bool this_two_op = false;
2630 2560 : for (unsigned lane = 0; lane < group_size; ++lane)
2631 4040 : if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
2632 : {
2633 : this_two_op = true;
2634 : break;
2635 : }
2636 698 : slp_tree child;
2637 698 : if (i == children.length () - 1)
2638 313 : child = vect_create_new_slp_node (node, stmts, 2);
2639 : else
2640 385 : child = vect_create_new_slp_node (2, ERROR_MARK);
2641 698 : if (this_two_op)
2642 : {
2643 158 : vec<std::pair<unsigned, unsigned> > lperm;
2644 158 : lperm.create (group_size);
2645 570 : for (unsigned lane = 0; lane < group_size; ++lane)
2646 824 : lperm.quick_push (std::make_pair
2647 412 : (chains[lane][i].code != chains[0][i].code, lane));
2648 316 : vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
2649 158 : (chains[0][i].code == code
2650 : ? op_stmt_info
2651 : : other_op_stmt_info),
2652 158 : (chains[0][i].code == code
2653 : ? other_op_stmt_info
2654 : : op_stmt_info),
2655 : lperm);
2656 : }
2657 : else
2658 : {
2659 540 : SLP_TREE_DEF_TYPE (child) = vect_internal_def;
2660 540 : SLP_TREE_VECTYPE (child) = vectype;
2661 540 : SLP_TREE_LANES (child) = group_size;
2662 540 : SLP_TREE_CHILDREN (child).quick_push (op0);
2663 540 : SLP_TREE_CHILDREN (child).quick_push (op1);
2664 540 : SLP_TREE_REPRESENTATIVE (child)
2665 1080 : = (chains[0][i].code == code
2666 540 : ? op_stmt_info : other_op_stmt_info);
2667 : }
2668 698 : children[i] = child;
2669 : }
2670 313 : *tree_size += this_tree_size + 1;
2671 313 : *max_nunits = this_max_nunits;
2672 1513 : while (!chains.is_empty ())
2673 864 : chains.pop ().release ();
2674 : return node;
2675 : }
2676 21219 : out:
2677 21242 : if (dump_enabled_p ())
2678 2817 : dump_printf_loc (MSG_NOTE, vect_location,
2679 : "failed to line up SLP graph by re-associating "
2680 : "operations in lanes%s\n",
2681 : !hard_fail ? " trying regular discovery" : "");
2682 21247 : while (!children.is_empty ())
2683 5 : vect_free_slp_tree (children.pop ());
2684 21405 : while (!chains.is_empty ())
2685 163 : chains.pop ().release ();
2686 : /* Hard-fail, otherwise we might run into quadratic processing of the
2687 : chains starting one stmt into the chain again. */
2688 21242 : if (hard_fail)
2689 : return NULL;
2690 : /* Fall thru to normal processing. */
2691 21555 : }
2692 :
2693 : /* Get at the operands, verifying they are compatible. */
2694 3389932 : vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
2695 3389932 : slp_oprnd_info oprnd_info;
2696 16209582 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
2697 : {
2698 25641712 : int res = vect_get_and_check_slp_defs (vinfo, vectype,
2699 12820856 : swap[i], skip_args,
2700 : stmts, i, &oprnds_info);
2701 12820856 : if (res != 0)
2702 584930 : matches[(res == -1) ? 0 : i] = false;
2703 12820856 : if (!matches[0])
2704 : break;
2705 : }
2706 15877181 : for (i = 0; i < group_size; ++i)
2707 12719564 : if (!matches[i])
2708 : {
2709 232315 : vect_free_oprnd_info (oprnds_info);
2710 232315 : return NULL;
2711 : }
2712 9472851 : swap = NULL;
2713 :
2714 9472851 : bool has_two_operators_perm = false;
2715 18945702 : auto_vec<unsigned> two_op_perm_indices[2];
2716 3157617 : vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
2717 :
2718 3171910 : if (two_operators && oprnds_info.length () == 2 && group_size > 2)
2719 : {
2720 3824 : unsigned idx = 0;
2721 3824 : hash_map<gimple *, unsigned> seen;
2722 3824 : vec<slp_oprnd_info> new_oprnds_info
2723 3824 : = vect_create_oprnd_info (1, group_size);
2724 3824 : bool success = true;
2725 :
2726 3824 : enum tree_code code = ERROR_MARK;
2727 3824 : if (oprnds_info[0]->def_stmts[0]
2728 3824 : && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
2729 3766 : code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
2730 3824 : basic_block bb = nullptr;
2731 :
2732 7426 : for (unsigned j = 0; j < group_size; ++j)
2733 : {
2734 17402 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2735 : {
2736 13800 : stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
2737 13800 : if (!stmt_info
2738 13639 : || !is_a<gassign *> (stmt_info->stmt)
2739 13636 : || gimple_assign_rhs_code (stmt_info->stmt) != code
2740 24239 : || skip_args[i])
2741 : {
2742 : success = false;
2743 3365 : break;
2744 : }
2745 : /* Avoid mixing lanes with defs in different basic-blocks. */
2746 10439 : if (!bb)
2747 3944 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
2748 8261 : else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
2749 : {
2750 : success = false;
2751 : break;
2752 : }
2753 :
2754 10435 : bool exists;
2755 10435 : unsigned &stmt_idx
2756 10435 : = seen.get_or_insert (stmt_info->stmt, &exists);
2757 :
2758 10435 : if (!exists)
2759 : {
2760 9094 : new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
2761 9094 : new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
2762 9094 : stmt_idx = idx;
2763 9094 : idx++;
2764 : }
2765 :
2766 10435 : two_op_perm_indices[i].safe_push (stmt_idx);
2767 : }
2768 :
2769 6967 : if (!success)
2770 : break;
2771 : }
2772 :
2773 3824 : if (success && idx == group_size)
2774 : {
2775 94 : if (dump_enabled_p ())
2776 : {
2777 0 : dump_printf_loc (MSG_NOTE, vect_location,
2778 : "Replace two_operators operands:\n");
2779 :
2780 0 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2781 : {
2782 0 : dump_printf_loc (MSG_NOTE, vect_location,
2783 : "Operand %u:\n", i);
2784 0 : for (unsigned j = 0; j < group_size; j++)
2785 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2786 0 : j, oprnd_info->def_stmts[j]->stmt);
2787 : }
2788 :
2789 0 : dump_printf_loc (MSG_NOTE, vect_location,
2790 : "With a single operand:\n");
2791 0 : for (unsigned j = 0; j < group_size; j++)
2792 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2793 0 : j, new_oprnds_info[0]->def_stmts[j]->stmt);
2794 : }
2795 :
2796 94 : two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
2797 94 : two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
2798 :
2799 94 : new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
2800 94 : new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
2801 94 : new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
2802 94 : new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
2803 94 : new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
2804 :
2805 94 : vect_free_oprnd_info (oprnds_info);
2806 94 : oprnds_info = new_oprnds_info;
2807 94 : nops = 1;
2808 94 : has_two_operators_perm = true;
2809 : }
2810 : else
2811 3730 : vect_free_oprnd_info (new_oprnds_info);
2812 3824 : }
2813 :
2814 6315234 : auto_vec<slp_tree, 4> children;
2815 :
2816 3157617 : stmt_info = stmts[0];
2817 :
2818 3157617 : int reduc_idx = -1;
2819 3157617 : int gs_scale = 0;
2820 3157617 : tree gs_base = NULL_TREE;
2821 :
2822 : /* Create SLP_TREE nodes for the definition node/s. */
2823 8065527 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2824 : {
2825 5036041 : slp_tree child = nullptr;
2826 5036041 : unsigned int j;
2827 5036041 : unsigned old_swap_distance;
2828 5036041 : bool can_swap;
2829 5036041 : bool can_swap_nonmatching;
2830 5036041 : bool *stmt_can_swap;
2831 :
2832 : /* We're skipping certain operands from processing, for example
2833 : outer loop reduction initial defs. */
2834 5036041 : if (skip_args[i])
2835 : {
2836 490046 : children.safe_push (NULL);
2837 5397956 : continue;
2838 : }
2839 :
2840 4545995 : if (oprnd_info->first_dt == vect_uninitialized_def)
2841 : {
2842 : /* COND_EXPR have one too many eventually if the condition
2843 : is a SSA name. */
2844 0 : gcc_assert (i == 3 && nops == 4);
2845 0 : continue;
2846 : }
2847 :
2848 4545995 : if (oprnd_info->first_gs_p)
2849 : {
2850 22561 : gs_scale = oprnd_info->first_gs_info.scale;
2851 22561 : gs_base = oprnd_info->first_gs_info.base;
2852 : }
2853 :
2854 4545995 : if (is_a <bb_vec_info> (vinfo)
2855 1596775 : && oprnd_info->first_dt == vect_internal_def
2856 5372365 : && !oprnd_info->any_pattern)
2857 : {
2858 : /* For BB vectorization, if all defs are the same do not
2859 : bother to continue the build along the single-lane
2860 : graph but use a splat of the scalar value. */
2861 782979 : stmt_vec_info first_def = oprnd_info->def_stmts[0];
2862 842092 : for (j = 1; j < group_size; ++j)
2863 798633 : if (oprnd_info->def_stmts[j] != first_def)
2864 : break;
2865 782979 : if (j == group_size
2866 : /* But avoid doing this for loads where we may be
2867 : able to CSE things, unless the stmt is not
2868 : vectorizable. */
2869 782979 : && (!STMT_VINFO_VECTORIZABLE (first_def)
2870 50792 : || !gimple_vuse (first_def->stmt)))
2871 : {
2872 34488 : if (dump_enabled_p ())
2873 107 : dump_printf_loc (MSG_NOTE, vect_location,
2874 : "Using a splat of the uniform operand %G",
2875 : first_def->stmt);
2876 34488 : oprnd_info->first_dt = vect_external_def;
2877 : }
2878 : }
2879 :
2880 4545995 : if (oprnd_info->first_dt == vect_external_def
2881 4545995 : || oprnd_info->first_dt == vect_constant_def)
2882 : {
2883 1500090 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
2884 : {
2885 : tree op0;
2886 : tree uniform_val = op0 = oprnd_info->ops[0];
2887 : for (j = 1; j < oprnd_info->ops.length (); ++j)
2888 : if (oprnd_info->ops[j]
2889 : && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
2890 : {
2891 : uniform_val = NULL_TREE;
2892 : break;
2893 : }
2894 : if (!uniform_val
2895 : && !can_duplicate_and_interleave_p (vinfo,
2896 : oprnd_info->ops.length (),
2897 : TREE_TYPE (op0)))
2898 : {
2899 : matches[j] = false;
2900 : if (dump_enabled_p ())
2901 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2902 : "Build SLP failed: invalid type of def "
2903 : "for variable-length SLP %T\n", op0);
2904 : goto fail;
2905 : }
2906 : }
2907 1500090 : slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
2908 1500090 : SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
2909 1500090 : oprnd_info->ops = vNULL;
2910 1500090 : children.safe_push (invnode);
2911 1500090 : continue;
2912 1500090 : }
2913 :
2914 : /* See which SLP operand a reduction chain continues on. We want
2915 : to chain even PHIs but not backedges. */
2916 3045905 : if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
2917 3045905 : || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
2918 : {
2919 233784 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
2920 : {
2921 776 : if (oprnd_info->first_dt == vect_double_reduction_def)
2922 388 : reduc_idx = i;
2923 : }
2924 233008 : else if (is_a <gphi *> (stmt_info->stmt)
2925 233008 : && gimple_phi_num_args
2926 99859 : (as_a <gphi *> (stmt_info->stmt)) != 1)
2927 : ;
2928 133542 : else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
2929 393 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
2930 : ;
2931 133542 : else if (reduc_idx == -1)
2932 125133 : reduc_idx = i;
2933 : else
2934 : /* For .COND_* reduction operations the else value can be the
2935 : same as one of the operation operands. The other def
2936 : stmts have been moved, so we can't check easily. Check
2937 : it's a call at least. */
2938 8409 : gcc_assert (is_a <gcall *> (stmt_info->stmt));
2939 : }
2940 :
2941 : /* When we have a masked load with uniform mask discover this
2942 : as a single-lane mask with a splat permute. This way we can
2943 : recognize this as a masked load-lane by stripping the splat. */
2944 3045905 : if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
2945 57492 : && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
2946 : IFN_MASK_LOAD)
2947 6071 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2948 3045982 : && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
2949 : {
2950 35 : vec<stmt_vec_info> def_stmts2;
2951 35 : def_stmts2.create (1);
2952 35 : def_stmts2.quick_push (oprnd_info->def_stmts[0]);
2953 35 : child = vect_build_slp_tree (vinfo, def_stmts2,
2954 : &this_max_nunits,
2955 : matches, limit,
2956 : &this_tree_size, bst_map);
2957 35 : if (child)
2958 : {
2959 35 : slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
2960 35 : SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
2961 35 : SLP_TREE_LANES (pnode) = group_size;
2962 35 : SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
2963 35 : SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
2964 210 : for (unsigned k = 0; k < group_size; ++k)
2965 : {
2966 175 : SLP_TREE_SCALAR_STMTS (pnode)
2967 175 : .quick_push (oprnd_info->def_stmts[0]);
2968 175 : SLP_TREE_LANE_PERMUTATION (pnode)
2969 175 : .quick_push (std::make_pair (0u, 0u));
2970 : }
2971 35 : SLP_TREE_CHILDREN (pnode).quick_push (child);
2972 35 : pnode->max_nunits = child->max_nunits;
2973 35 : children.safe_push (pnode);
2974 35 : oprnd_info->def_stmts = vNULL;
2975 35 : continue;
2976 35 : }
2977 : else
2978 0 : def_stmts2.release ();
2979 : }
2980 :
2981 6091740 : can_swap = (i == 0
2982 2260169 : && (nops == 2 || nops == 3)
2983 1455776 : && oprnds_info.length () > 1
2984 1455776 : && oprnds_info[1]->first_dt == vect_internal_def
2985 593208 : && (is_gimple_assign (stmt_info->stmt)
2986 49489 : || is_gimple_call (stmt_info->stmt))
2987 : /* Swapping operands for reductions breaks assumptions
2988 : later on. */
2989 3594778 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
2990 3045870 : can_swap_nonmatching = can_swap;
2991 3045870 : stmt_can_swap = NULL;
2992 3045870 : if (can_swap)
2993 : {
2994 493398 : stmt_can_swap = XALLOCAVEC (bool, group_size);
2995 8125958 : for (j = 0; j < group_size; ++j)
2996 : {
2997 7632560 : stmt_can_swap[j] = false;
2998 7632560 : if (!stmts[j])
2999 : /* NULL lanes are gaps and have no stmt to swap. */
3000 0 : stmt_can_swap[j] = true;
3001 7632560 : else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
3002 : {
3003 7626950 : tree_code code = gimple_assign_rhs_code (stmt);
3004 15253900 : stmt_can_swap[j] = (commutative_tree_code (code)
3005 7626950 : || commutative_ternary_tree_code (code));
3006 : }
3007 5610 : else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
3008 : {
3009 5610 : internal_fn fn = (gimple_call_internal_p (call)
3010 5610 : ? gimple_call_internal_fn (call) : IFN_LAST);
3011 11220 : stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
3012 5294 : || commutative_ternary_fn_p (fn))
3013 5646 : && first_commutative_argument (fn) == 0);
3014 : }
3015 :
3016 7632560 : if (j != 0 && !stmt_can_swap[j])
3017 7632560 : can_swap_nonmatching = false;
3018 : }
3019 : }
3020 :
3021 3045870 : old_swap_distance = least_upthread_swappable_op_distance;
3022 3045870 : if (can_swap_nonmatching)
3023 459418 : least_upthread_swappable_op_distance = 1;
3024 2586452 : else if (least_upthread_swappable_op_distance != -1U)
3025 304495 : least_upthread_swappable_op_distance++;
3026 3045870 : child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3027 : &this_max_nunits,
3028 : matches, limit,
3029 : &this_tree_size, bst_map);
3030 3045870 : least_upthread_swappable_op_distance = old_swap_distance;
3031 3045870 : if (child != NULL)
3032 : {
3033 2546719 : oprnd_info->def_stmts = vNULL;
3034 2546719 : children.safe_push (child);
3035 2546719 : continue;
3036 : }
3037 :
3038 : /* If the SLP build for operand zero failed and operand zero
3039 : and one can be commuted try that for the scalar stmts
3040 : that failed the match. */
3041 499151 : if (/* A first scalar stmt mismatch signals a fatal mismatch. */
3042 499151 : matches[0]
3043 263553 : && can_swap)
3044 : {
3045 : /* See whether we can swap the matching or the non-matching
3046 : stmt operands. */
3047 : bool swap_not_matching = true;
3048 66799 : do
3049 : {
3050 7108693 : for (j = 0; j < group_size; ++j)
3051 : {
3052 7056967 : if (matches[j] != !swap_not_matching)
3053 88237 : continue;
3054 : /* Verify if we can swap operands of this stmt. */
3055 6968730 : if (!stmt_can_swap[j])
3056 : {
3057 15073 : if (!swap_not_matching)
3058 7027 : goto fail;
3059 : swap_not_matching = false;
3060 : break;
3061 : }
3062 : }
3063 : }
3064 59772 : while (j != group_size);
3065 :
3066 : /* Swap mismatched definition stmts. */
3067 51726 : if (dump_enabled_p ())
3068 389 : dump_printf_loc (MSG_NOTE, vect_location,
3069 : "Re-trying with swapped operands of stmts ");
3070 7084649 : for (j = 0; j < group_size; ++j)
3071 7032923 : if (matches[j] == !swap_not_matching)
3072 : {
3073 13906914 : std::swap (oprnds_info[0]->def_stmts[j],
3074 6953457 : oprnds_info[1]->def_stmts[j]);
3075 13906914 : std::swap (oprnds_info[0]->ops[j],
3076 6953457 : oprnds_info[1]->ops[j]);
3077 6953457 : if (dump_enabled_p ())
3078 1076 : dump_printf (MSG_NOTE, "%d ", j);
3079 : }
3080 51726 : if (dump_enabled_p ())
3081 389 : dump_printf (MSG_NOTE, "\n");
3082 : /* After swapping some operands we lost track whether an
3083 : operand has any pattern defs so be conservative here. */
3084 100863 : if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
3085 3049 : oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
3086 : /* And try again with scratch 'matches' ... */
3087 51726 : bool *tem = XALLOCAVEC (bool, group_size);
3088 51726 : if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3089 : &this_max_nunits,
3090 : tem, limit,
3091 : &this_tree_size, bst_map)) != NULL)
3092 : {
3093 6533 : oprnd_info->def_stmts = vNULL;
3094 6533 : children.safe_push (child);
3095 6533 : continue;
3096 : }
3097 : }
3098 440398 : fail:
3099 :
3100 : /* If the SLP build failed and we analyze a basic-block
3101 : simply treat nodes we fail to build as externally defined
3102 : (and thus build vectors from the scalar defs).
3103 : The cost model will reject outright expensive cases.
3104 : ??? This doesn't treat cases where permutation ultimatively
3105 : fails (or we don't try permutation below). Ideally we'd
3106 : even compute a permutation that will end up with the maximum
3107 : SLP tree size... */
3108 492618 : if (is_a <bb_vec_info> (vinfo)
3109 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3110 : do extra work to cancel the pattern so the uses see the
3111 : scalar version. */
3112 : /* Skip building vector operands from scalars while operand
3113 : discovery may still be fixed by retrying with swapped operands. */
3114 411665 : && (least_upthread_swappable_op_distance != 1
3115 : /* A first scalar stmt mismatch signals a fatal mismatch
3116 : that the parent commutative retry cannot recover. */
3117 26480 : || !matches[0])
3118 393648 : && !is_pattern_stmt_p (stmt_info)
3119 863611 : && !oprnd_info->any_pattern)
3120 : {
3121 : /* But if there's a leading vector sized set of matching stmts
3122 : fail here so we can split the group. This matches the condition
3123 : vect_analyze_slp_instance uses. */
3124 : /* ??? We might want to split here and combine the results to support
3125 : multiple vector sizes better. */
3126 574175 : for (j = 0; j < group_size; ++j)
3127 574175 : if (!matches[j])
3128 : break;
3129 370755 : if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
3130 370724 : && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
3131 : {
3132 364487 : if (dump_enabled_p ())
3133 622 : dump_printf_loc (MSG_NOTE, vect_location,
3134 : "Building vector operands from scalars\n");
3135 364487 : this_tree_size++;
3136 364487 : child = vect_create_new_slp_node (oprnd_info->ops);
3137 364487 : children.safe_push (child);
3138 364487 : oprnd_info->ops = vNULL;
3139 364487 : continue;
3140 : }
3141 : }
3142 :
3143 128131 : gcc_assert (child == NULL);
3144 146850 : FOR_EACH_VEC_ELT (children, j, child)
3145 18719 : if (child)
3146 18719 : vect_free_slp_tree (child);
3147 128131 : vect_free_oprnd_info (oprnds_info);
3148 128131 : return NULL;
3149 : }
3150 :
3151 3029486 : vect_free_oprnd_info (oprnds_info);
3152 :
3153 : /* If we have all children of a child built up from uniform scalars
3154 : or does more than one possibly expensive vector construction then
3155 : just throw that away, causing it built up from scalars.
3156 : The exception is the SLP node for the vector store. */
3157 3029486 : if (is_a <bb_vec_info> (vinfo)
3158 1106246 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
3159 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3160 : do extra work to cancel the pattern so the uses see the
3161 : scalar version. */
3162 3464077 : && !is_pattern_stmt_p (stmt_info))
3163 : {
3164 : slp_tree child;
3165 : unsigned j;
3166 : bool all_uniform_p = true;
3167 : unsigned n_vector_builds = 0;
3168 1233733 : FOR_EACH_VEC_ELT (children, j, child)
3169 : {
3170 824979 : if (!child)
3171 : ;
3172 824979 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
3173 : all_uniform_p = false;
3174 600220 : else if (!vect_slp_tree_uniform_p (child))
3175 : {
3176 453775 : all_uniform_p = false;
3177 453775 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
3178 416324 : n_vector_builds++;
3179 : }
3180 : }
3181 408754 : if (all_uniform_p
3182 408754 : || n_vector_builds > 1
3183 692466 : || (n_vector_builds == children.length ()
3184 30873 : && is_a <gphi *> (stmt_info->stmt)))
3185 : {
3186 : /* Roll back. */
3187 129952 : matches[0] = false;
3188 412428 : FOR_EACH_VEC_ELT (children, j, child)
3189 282476 : if (child)
3190 282476 : vect_free_slp_tree (child);
3191 :
3192 129952 : if (dump_enabled_p ())
3193 205 : dump_printf_loc (MSG_NOTE, vect_location,
3194 : "Building parent vector operands from "
3195 : "scalars instead\n");
3196 129952 : return NULL;
3197 : }
3198 : }
3199 :
3200 2899534 : *tree_size += this_tree_size + 1;
3201 2899534 : *max_nunits = this_max_nunits;
3202 :
3203 2899534 : if (two_operators)
3204 : {
3205 : /* ??? We'd likely want to either cache in bst_map sth like
3206 : { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
3207 : the true { a+b, a+b, a+b, a+b } ... but there we don't have
3208 : explicit stmts to put in so the keying on 'stmts' doesn't
3209 : work (but we have the same issue with nodes that use 'ops'). */
3210 :
3211 6723 : if (has_two_operators_perm)
3212 : {
3213 40 : slp_tree child = children[0];
3214 40 : children.truncate (0);
3215 120 : for (i = 0; i < 2; i++)
3216 : {
3217 80 : slp_tree pnode
3218 80 : = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
3219 80 : SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
3220 80 : SLP_TREE_VECTYPE (pnode) = vectype;
3221 80 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3222 80 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3223 80 : lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
3224 80 : children.safe_push (pnode);
3225 :
3226 656 : for (unsigned j = 0; j < stmts.length (); j++)
3227 576 : perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
3228 : }
3229 :
3230 40 : SLP_TREE_REF_COUNT (child) += 4;
3231 : }
3232 :
3233 6723 : slp_tree one = new _slp_tree;
3234 6723 : slp_tree two = new _slp_tree;
3235 6723 : SLP_TREE_DEF_TYPE (one) = vect_internal_def;
3236 6723 : SLP_TREE_DEF_TYPE (two) = vect_internal_def;
3237 6723 : SLP_TREE_VECTYPE (one) = vectype;
3238 6723 : SLP_TREE_VECTYPE (two) = vectype;
3239 6723 : SLP_TREE_CHILDREN (one).safe_splice (children);
3240 6723 : SLP_TREE_CHILDREN (two).safe_splice (children);
3241 6723 : slp_tree child;
3242 26894 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
3243 13448 : SLP_TREE_REF_COUNT (child)++;
3244 :
3245 : /* Here we record the original defs since this
3246 : node represents the final lane configuration. */
3247 6723 : node = vect_create_new_slp_node (node, stmts, 2);
3248 6723 : SLP_TREE_VECTYPE (node) = vectype;
3249 6723 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
3250 6723 : SLP_TREE_CHILDREN (node).quick_push (one);
3251 6723 : SLP_TREE_CHILDREN (node).quick_push (two);
3252 6723 : enum tree_code code0 = ERROR_MARK;
3253 6723 : enum tree_code ocode = ERROR_MARK;
3254 6723 : if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
3255 6721 : code0 = gimple_assign_rhs_code (stmt);
3256 6723 : stmt_vec_info ostmt_info;
3257 6723 : unsigned j = 0;
3258 24542 : FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
3259 : {
3260 17819 : int op = 0;
3261 17819 : if (gassign *ostmt = dyn_cast <gassign *> (ostmt_info->stmt))
3262 : {
3263 17815 : if (gimple_assign_rhs_code (ostmt) != code0)
3264 : {
3265 8928 : ocode = gimple_assign_rhs_code (ostmt);
3266 : op = 1;
3267 : j = i;
3268 : }
3269 : }
3270 : else
3271 : {
3272 8 : if (gimple_call_combined_fn (stmts[0]->stmt)
3273 4 : != gimple_call_combined_fn (ostmt_info->stmt))
3274 : {
3275 2 : op = 1;
3276 2 : j = i;
3277 : }
3278 : }
3279 17819 : SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
3280 : }
3281 6723 : SLP_TREE_CODE (one) = code0;
3282 6723 : SLP_TREE_CODE (two) = ocode;
3283 6723 : SLP_TREE_LANES (one) = stmts.length ();
3284 6723 : SLP_TREE_LANES (two) = stmts.length ();
3285 6723 : SLP_TREE_REPRESENTATIVE (one) = stmts[0];
3286 6723 : SLP_TREE_REPRESENTATIVE (two) = stmts[j];
3287 :
3288 6723 : return node;
3289 : }
3290 :
3291 2892811 : node = vect_create_new_slp_node (node, stmts, nops);
3292 2892811 : SLP_TREE_VECTYPE (node) = vectype;
3293 2892811 : SLP_TREE_CHILDREN (node).splice (children);
3294 2892811 : SLP_TREE_GS_SCALE (node) = gs_scale;
3295 2892811 : SLP_TREE_GS_BASE (node) = gs_base;
3296 2892811 : if (reduc_idx != -1)
3297 : {
3298 116738 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
3299 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3300 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
3301 116738 : SLP_TREE_REDUC_IDX (node) = reduc_idx;
3302 116738 : node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
3303 : }
3304 : /* When reaching the reduction PHI, create a vect_reduc_info. */
3305 2776073 : else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
3306 2776073 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3307 2776073 : && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
3308 : {
3309 101820 : loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
3310 101820 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3311 101820 : node->cycle_info.id = loop_vinfo->reduc_infos.length ();
3312 101820 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
3313 101820 : loop_vinfo->reduc_infos.safe_push (reduc_info);
3314 101820 : stmt_vec_info reduc_phi = stmt_info;
3315 : /* ??? For double reductions vect_is_simple_reduction stores the
3316 : reduction type and code on the inner loop header PHI. */
3317 101820 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3318 : {
3319 388 : use_operand_p use_p;
3320 388 : gimple *use_stmt;
3321 388 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
3322 : &use_p, &use_stmt);
3323 388 : gcc_assert (res);
3324 388 : reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
3325 : }
3326 101820 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
3327 101820 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
3328 101820 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
3329 101820 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
3330 : }
3331 : return node;
3332 9472851 : }
3333 :
3334 : /* Dump a single SLP tree NODE. */
3335 :
3336 : static void
3337 445934 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
3338 : slp_tree node)
3339 : {
3340 445934 : unsigned i, j;
3341 445934 : slp_tree child;
3342 445934 : stmt_vec_info stmt_info;
3343 445934 : tree op;
3344 :
3345 445934 : dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
3346 445934 : dump_user_location_t user_loc = loc.get_user_location ();
3347 445934 : dump_printf_loc (metadata, user_loc,
3348 : "node%s %p (max_nunits=" HOST_WIDE_INT_PRINT_UNSIGNED
3349 : ", refcnt=%u)",
3350 445934 : SLP_TREE_DEF_TYPE (node) == vect_external_def
3351 : ? " (external)"
3352 : : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
3353 430157 : ? " (constant)"
3354 : : ""), (void *) node,
3355 445934 : estimated_poly_value (node->max_nunits),
3356 : SLP_TREE_REF_COUNT (node));
3357 445934 : if (SLP_TREE_VECTYPE (node))
3358 378102 : dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
3359 445934 : dump_printf (metadata, "%s",
3360 445934 : node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
3361 445934 : if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
3362 23994 : dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
3363 : node->cycle_info.reduc_idx);
3364 445934 : dump_printf (metadata, "\n");
3365 445934 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3366 : {
3367 363031 : if (SLP_TREE_PERMUTE_P (node))
3368 13825 : dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
3369 : else
3370 349206 : dump_printf_loc (metadata, user_loc, "op template: %G",
3371 349206 : SLP_TREE_REPRESENTATIVE (node)->stmt);
3372 : }
3373 445934 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
3374 869146 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
3375 514300 : if (stmt_info)
3376 508935 : dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
3377 508935 : SLP_TREE_LIVE_LANES (node).contains (i)
3378 505262 : ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
3379 505262 : ? "[l] " : ""),
3380 : i, stmt_info->stmt);
3381 : else
3382 5365 : dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
3383 : else
3384 : {
3385 91088 : dump_printf_loc (metadata, user_loc, "\t{ ");
3386 200131 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
3387 109043 : dump_printf (metadata, "%T%s ", op,
3388 109043 : i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
3389 91088 : dump_printf (metadata, "}\n");
3390 : }
3391 445934 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
3392 : {
3393 65022 : dump_printf_loc (metadata, user_loc, "\tload permutation {");
3394 148519 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
3395 83497 : dump_printf (dump_kind, " %u", j);
3396 65022 : dump_printf (dump_kind, " }\n");
3397 : }
3398 445934 : if (SLP_TREE_LANE_PERMUTATION (node).exists ())
3399 : {
3400 13833 : dump_printf_loc (metadata, user_loc, "\tlane permutation {");
3401 51967 : for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
3402 38134 : dump_printf (dump_kind, " %u[%u]",
3403 38134 : SLP_TREE_LANE_PERMUTATION (node)[i].first,
3404 38134 : SLP_TREE_LANE_PERMUTATION (node)[i].second);
3405 13833 : dump_printf (dump_kind, " }%s\n",
3406 13833 : node->ldst_lanes ? " (load-lanes)" : "");
3407 : }
3408 445934 : if (SLP_TREE_CHILDREN (node).is_empty ())
3409 170192 : return;
3410 275742 : dump_printf_loc (metadata, user_loc, "\tchildren");
3411 727722 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3412 451980 : dump_printf (dump_kind, " %p", (void *)child);
3413 275742 : dump_printf (dump_kind, "%s\n",
3414 275742 : node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
3415 : ? " (store-lanes)" : "");
3416 : }
3417 :
3418 : DEBUG_FUNCTION void
3419 0 : debug (slp_tree node)
3420 : {
3421 0 : debug_dump_context ctx;
3422 0 : vect_print_slp_tree (MSG_NOTE,
3423 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3424 : node);
3425 0 : }
3426 :
3427 : /* Recursive helper for the dot producer below. */
3428 :
3429 : static void
3430 0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
3431 : {
3432 0 : if (visited.add (node))
3433 : return;
3434 :
3435 0 : fprintf (f, "\"%p\" [label=\"", (void *)node);
3436 0 : vect_print_slp_tree (MSG_NOTE,
3437 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3438 : node);
3439 0 : fprintf (f, "\"];\n");
3440 :
3441 :
3442 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3443 0 : fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
3444 :
3445 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3446 0 : if (child)
3447 0 : dot_slp_tree (f, child, visited);
3448 : }
3449 :
3450 : DEBUG_FUNCTION void
3451 0 : dot_slp_tree (const char *fname, slp_tree node)
3452 : {
3453 0 : FILE *f = fopen (fname, "w");
3454 0 : fprintf (f, "digraph {\n");
3455 0 : fflush (f);
3456 0 : {
3457 0 : debug_dump_context ctx (f);
3458 0 : hash_set<slp_tree> visited;
3459 0 : dot_slp_tree (f, node, visited);
3460 0 : }
3461 0 : fflush (f);
3462 0 : fprintf (f, "}\n");
3463 0 : fclose (f);
3464 0 : }
3465 :
3466 : DEBUG_FUNCTION void
3467 0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
3468 : {
3469 0 : FILE *f = fopen (fname, "w");
3470 0 : fprintf (f, "digraph {\n");
3471 0 : fflush (f);
3472 0 : {
3473 0 : debug_dump_context ctx (f);
3474 0 : hash_set<slp_tree> visited;
3475 0 : for (auto inst : slp_instances)
3476 0 : dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
3477 0 : }
3478 0 : fflush (f);
3479 0 : fprintf (f, "}\n");
3480 0 : fclose (f);
3481 0 : }
3482 :
3483 : /* Dump a slp tree NODE using flags specified in DUMP_KIND. */
3484 :
3485 : static void
3486 484820 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3487 : slp_tree node, hash_set<slp_tree> &visited)
3488 : {
3489 484820 : unsigned i;
3490 484820 : slp_tree child;
3491 :
3492 484820 : if (visited.add (node))
3493 484820 : return;
3494 :
3495 445461 : vect_print_slp_tree (dump_kind, loc, node);
3496 :
3497 1342389 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3498 451467 : if (child)
3499 408789 : vect_print_slp_graph (dump_kind, loc, child, visited);
3500 : }
3501 :
3502 : static void
3503 46701 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3504 : slp_tree entry)
3505 : {
3506 46701 : hash_set<slp_tree> visited;
3507 46701 : vect_print_slp_graph (dump_kind, loc, entry, visited);
3508 46701 : }
3509 :
3510 : DEBUG_FUNCTION void
3511 0 : debug (slp_instance instance)
3512 : {
3513 0 : debug_dump_context ctx;
3514 0 : vect_print_slp_graph (MSG_NOTE,
3515 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3516 : SLP_INSTANCE_TREE (instance));
3517 0 : }
3518 :
3519 :
3520 : /* Compute the set of scalar stmts participating in external nodes. */
3521 :
3522 : static void
3523 1581434 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
3524 : hash_set<slp_tree> &visited,
3525 : hash_set<stmt_vec_info> &estmts)
3526 : {
3527 1581434 : if (visited.add (node))
3528 : return;
3529 :
3530 1536888 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3531 : {
3532 : slp_tree child;
3533 : int i;
3534 1770409 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3535 888517 : if (child)
3536 888517 : vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
3537 : }
3538 : else
3539 3688210 : for (tree def : SLP_TREE_SCALAR_OPS (node))
3540 : {
3541 1724546 : stmt_vec_info def_stmt = vinfo->lookup_def (def);
3542 1724546 : if (def_stmt)
3543 340906 : estmts.add (def_stmt);
3544 : }
3545 : }
3546 :
3547 : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
3548 : with STMT_SLP_TYPE == pure_slp. */
3549 :
3550 : static void
3551 239088 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
3552 : {
3553 : /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
3554 : walk on. */
3555 239088 : hash_set<stmt_vec_info> scalar_stmts_in_externs;
3556 239088 : hash_set<slp_tree> visited;
3557 1410181 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3558 692917 : vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
3559 : visited, scalar_stmts_in_externs);
3560 :
3561 : /* DFS walk scalar stmts to compute the vectorized coverage indicated
3562 : by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
3563 : stmts. */
3564 1410181 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3565 : {
3566 808059 : for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
3567 53956 : if (!scalar_stmts_in_externs.contains (stmt))
3568 53299 : STMT_SLP_TYPE (stmt) = pure_slp;
3569 692917 : auto_vec<stmt_vec_info> worklist;
3570 3915183 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
3571 : {
3572 1836432 : stmt = vect_orig_stmt (stmt);
3573 1836432 : if (!scalar_stmts_in_externs.contains (stmt)
3574 1836432 : && STMT_SLP_TYPE (stmt) != pure_slp)
3575 : {
3576 1825813 : STMT_SLP_TYPE (stmt) = pure_slp;
3577 1825813 : worklist.safe_push (stmt);
3578 : }
3579 : }
3580 3650040 : while (!worklist.is_empty ())
3581 : {
3582 2267160 : stmt_vec_info stmt = worklist.pop ();
3583 :
3584 : /* Now walk relevant parts of the SSA use-def graph. */
3585 2267160 : slp_oprnds child_ops (stmt);
3586 4771849 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
3587 : {
3588 2504689 : tree op = child_ops.get_op_for_slp_child (stmt, i);
3589 2504689 : stmt_vec_info def = vinfo->lookup_def (op);
3590 2504689 : if (def
3591 866013 : && !scalar_stmts_in_externs.contains (def)
3592 3031609 : && STMT_SLP_TYPE (def) != pure_slp)
3593 : {
3594 441347 : STMT_SLP_TYPE (def) = pure_slp;
3595 441347 : worklist.safe_push (def);
3596 : }
3597 : }
3598 : }
3599 692917 : }
3600 239088 : }
3601 :
3602 : /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
3603 :
3604 : static void
3605 2463991 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
3606 : {
3607 2463991 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3608 : return;
3609 :
3610 1464877 : if (visited.add (node))
3611 : return;
3612 :
3613 6887159 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
3614 3080442 : if (stmt_info)
3615 : {
3616 3080442 : gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
3617 : || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
3618 3080442 : STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
3619 : }
3620 :
3621 5331375 : for (auto child: SLP_TREE_CHILDREN (node))
3622 1663770 : if (child)
3623 1663770 : vect_mark_slp_stmts_relevant (child, visited);
3624 : }
3625 :
3626 : static void
3627 800221 : vect_mark_slp_stmts_relevant (slp_tree node)
3628 : {
3629 800221 : hash_set<slp_tree> visited;
3630 800221 : vect_mark_slp_stmts_relevant (node, visited);
3631 800221 : }
3632 :
3633 :
3634 : /* Gather loads in the SLP graph NODE and populate the INST loads array. */
3635 :
3636 : static void
3637 10678868 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
3638 : hash_set<slp_tree> &visited)
3639 : {
3640 10678868 : if (!node || visited.add (node))
3641 1750275 : return;
3642 :
3643 8928593 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3644 : return;
3645 :
3646 6615767 : if (!SLP_TREE_PERMUTE_P (node))
3647 : {
3648 6409433 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
3649 6409433 : if (STMT_VINFO_DATA_REF (stmt_info)
3650 2803755 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
3651 1587553 : loads.safe_push (node);
3652 : }
3653 :
3654 : unsigned i;
3655 : slp_tree child;
3656 15021582 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3657 8405815 : vect_gather_slp_loads (loads, child, visited);
3658 : }
3659 :
3660 :
3661 : /* Find the last store in SLP INSTANCE. */
3662 :
3663 : stmt_vec_info
3664 2770282 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
3665 : {
3666 2770282 : stmt_vec_info last = NULL;
3667 15632767 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3668 7329683 : if (stmt_vinfo)
3669 : {
3670 7329683 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3671 7329683 : last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
3672 : }
3673 :
3674 2770282 : return last;
3675 : }
3676 :
3677 : /* Find the first stmt in NODE. */
3678 :
3679 : stmt_vec_info
3680 540014 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
3681 : {
3682 540014 : stmt_vec_info first = NULL;
3683 :
3684 2911515 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3685 1291473 : if (stmt_vinfo)
3686 : {
3687 1288777 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3688 1288777 : if (!first
3689 1288777 : || get_later_stmt (stmt_vinfo, first) == first)
3690 : first = stmt_vinfo;
3691 : }
3692 :
3693 540014 : return first;
3694 : }
3695 :
3696 : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
3697 : two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
3698 : (also containing the first GROUP1_SIZE stmts, since stores are
3699 : consecutive), the second containing the remainder.
3700 : Return the first stmt in the second group. */
3701 :
3702 : static stmt_vec_info
3703 161969 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
3704 : {
3705 161969 : gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
3706 161969 : gcc_assert (group1_size > 0);
3707 161969 : int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
3708 161969 : gcc_assert (group2_size > 0);
3709 161969 : DR_GROUP_SIZE (first_vinfo) = group1_size;
3710 :
3711 161969 : stmt_vec_info stmt_info = first_vinfo;
3712 541148 : for (unsigned i = group1_size; i > 1; i--)
3713 : {
3714 379179 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
3715 379179 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3716 : }
3717 : /* STMT is now the last element of the first group. */
3718 161969 : stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
3719 161969 : DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
3720 :
3721 161969 : DR_GROUP_SIZE (group2) = group2_size;
3722 451369 : for (stmt_info = group2; stmt_info;
3723 289400 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
3724 : {
3725 289400 : DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
3726 289400 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3727 : }
3728 :
3729 : /* For the second group, the DR_GROUP_GAP is that before the original group,
3730 : plus skipping over the first vector. */
3731 161969 : DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
3732 :
3733 : /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
3734 161969 : DR_GROUP_GAP (first_vinfo) += group2_size;
3735 :
3736 161969 : if (dump_enabled_p ())
3737 70 : dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
3738 : group1_size, group2_size);
3739 :
3740 161969 : return group2;
3741 : }
3742 :
3743 : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
3744 : statements and a vector of NUNITS elements. */
3745 :
3746 : static poly_uint64
3747 4230687 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
3748 : {
3749 4230687 : return exact_div (common_multiple (nunits, group_size), group_size);
3750 : }
3751 :
3752 : /* Helper that checks to see if a node is a load node. */
3753 :
3754 : static inline bool
3755 102 : vect_is_slp_load_node (slp_tree root)
3756 : {
3757 102 : return (!SLP_TREE_PERMUTE_P (root)
3758 102 : && SLP_TREE_DEF_TYPE (root) == vect_internal_def
3759 96 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
3760 166 : && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
3761 : }
3762 :
3763 :
3764 : /* Helper function of optimize_load_redistribution that performs the operation
3765 : recursively. */
3766 :
3767 : static slp_tree
3768 19448 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
3769 : vec_info *vinfo, unsigned int group_size,
3770 : hash_map<slp_tree, slp_tree> *load_map,
3771 : slp_tree root)
3772 : {
3773 19448 : if (slp_tree *leader = load_map->get (root))
3774 3535 : return *leader;
3775 :
3776 15913 : slp_tree node;
3777 15913 : unsigned i;
3778 :
3779 : /* For now, we don't know anything about externals so do not do anything. */
3780 15913 : if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
3781 : return NULL;
3782 11416 : else if (SLP_TREE_PERMUTE_P (root))
3783 : {
3784 : /* First convert this node into a load node and add it to the leaves
3785 : list and flatten the permute from a lane to a load one. If it's
3786 : unneeded it will be elided later. */
3787 70 : vec<stmt_vec_info> stmts;
3788 70 : stmts.create (SLP_TREE_LANES (root));
3789 70 : lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
3790 134 : for (unsigned j = 0; j < lane_perm.length (); j++)
3791 : {
3792 102 : std::pair<unsigned, unsigned> perm = lane_perm[j];
3793 102 : node = SLP_TREE_CHILDREN (root)[perm.first];
3794 :
3795 102 : if (!vect_is_slp_load_node (node)
3796 102 : || SLP_TREE_CHILDREN (node).exists ())
3797 : {
3798 38 : stmts.release ();
3799 38 : goto next;
3800 : }
3801 :
3802 64 : stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
3803 : }
3804 :
3805 32 : if (dump_enabled_p ())
3806 0 : dump_printf_loc (MSG_NOTE, vect_location,
3807 : "converting stmts on permute node %p\n",
3808 : (void *) root);
3809 :
3810 32 : bool *matches = XALLOCAVEC (bool, group_size);
3811 32 : poly_uint64 max_nunits = 1;
3812 32 : unsigned tree_size = 0, limit = 1;
3813 32 : node = vect_build_slp_tree (vinfo, stmts, &max_nunits,
3814 : matches, &limit, &tree_size, bst_map);
3815 32 : if (!node)
3816 0 : stmts.release ();
3817 :
3818 32 : load_map->put (root, node);
3819 32 : return node;
3820 : }
3821 :
3822 11346 : next:
3823 11384 : load_map->put (root, NULL);
3824 :
3825 26565 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3826 : {
3827 15181 : slp_tree value
3828 15181 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3829 : node);
3830 15181 : if (value)
3831 : {
3832 32 : SLP_TREE_REF_COUNT (value)++;
3833 32 : SLP_TREE_CHILDREN (root)[i] = value;
3834 : /* ??? We know the original leafs of the replaced nodes will
3835 : be referenced by bst_map, only the permutes created by
3836 : pattern matching are not. */
3837 32 : if (SLP_TREE_REF_COUNT (node) == 1)
3838 32 : load_map->remove (node);
3839 32 : vect_free_slp_tree (node);
3840 : }
3841 : }
3842 :
3843 : return NULL;
3844 : }
3845 :
3846 : /* Temporary workaround for loads not being CSEd during SLP build. This
3847 : function will traverse the SLP tree rooted in ROOT for INSTANCE and find
3848 : VEC_PERM nodes that blend vectors from multiple nodes that all read from the
3849 : same DR such that the final operation is equal to a permuted load. Such
3850 : NODES are then directly converted into LOADS themselves. The nodes are
3851 : CSEd using BST_MAP. */
3852 :
3853 : static void
3854 3121 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
3855 : vec_info *vinfo, unsigned int group_size,
3856 : hash_map<slp_tree, slp_tree> *load_map,
3857 : slp_tree root)
3858 : {
3859 3121 : slp_tree node;
3860 3121 : unsigned i;
3861 :
3862 7388 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3863 : {
3864 4267 : slp_tree value
3865 4267 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3866 : node);
3867 4267 : if (value)
3868 : {
3869 0 : SLP_TREE_REF_COUNT (value)++;
3870 0 : SLP_TREE_CHILDREN (root)[i] = value;
3871 : /* ??? We know the original leafs of the replaced nodes will
3872 : be referenced by bst_map, only the permutes created by
3873 : pattern matching are not. */
3874 0 : if (SLP_TREE_REF_COUNT (node) == 1)
3875 0 : load_map->remove (node);
3876 0 : vect_free_slp_tree (node);
3877 : }
3878 : }
3879 3121 : }
3880 :
3881 : /* Helper function of vect_match_slp_patterns.
3882 :
3883 : Attempts to match patterns against the slp tree rooted in REF_NODE using
3884 : VINFO. Patterns are matched in post-order traversal.
3885 :
3886 : If matching is successful the value in REF_NODE is updated and returned, if
3887 : not then it is returned unchanged. */
3888 :
3889 : static bool
3890 6193994 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
3891 : slp_tree_to_load_perm_map_t *perm_cache,
3892 : slp_compat_nodes_map_t *compat_cache,
3893 : hash_set<slp_tree> *visited)
3894 : {
3895 6193994 : unsigned i;
3896 6193994 : slp_tree node = *ref_node;
3897 6193994 : bool found_p = false;
3898 6193994 : if (!node || visited->add (node))
3899 879078 : return false;
3900 :
3901 : slp_tree child;
3902 9923945 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3903 4609029 : found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
3904 : vinfo, perm_cache, compat_cache,
3905 : visited);
3906 :
3907 15944748 : for (unsigned x = 0; x < num__slp_patterns; x++)
3908 : {
3909 10629832 : vect_pattern *pattern
3910 10629832 : = slp_patterns[x] (perm_cache, compat_cache, ref_node);
3911 10629832 : if (pattern)
3912 : {
3913 1107 : pattern->build (vinfo);
3914 1107 : delete pattern;
3915 1107 : found_p = true;
3916 : }
3917 : }
3918 :
3919 : return found_p;
3920 : }
3921 :
3922 : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
3923 : vec_info VINFO.
3924 :
3925 : The modified tree is returned. Patterns are tried in order and multiple
3926 : patterns may match. */
3927 :
3928 : static bool
3929 1584965 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
3930 : hash_set<slp_tree> *visited,
3931 : slp_tree_to_load_perm_map_t *perm_cache,
3932 : slp_compat_nodes_map_t *compat_cache)
3933 : {
3934 1584965 : DUMP_VECT_SCOPE ("vect_match_slp_patterns");
3935 1584965 : slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
3936 :
3937 1584965 : if (dump_enabled_p ())
3938 30533 : dump_printf_loc (MSG_NOTE, vect_location,
3939 : "Analyzing SLP tree %p for patterns\n",
3940 30533 : (void *) SLP_INSTANCE_TREE (instance));
3941 :
3942 1584965 : return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
3943 1584965 : visited);
3944 : }
3945 :
3946 : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
3947 : vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether
3948 : the stores are masked.
3949 : Return true if we could use IFN_STORE_LANES instead and if that appears
3950 : to be the better approach. */
3951 :
3952 : static bool
3953 6126 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
3954 : tree vectype, bool masked_p,
3955 : unsigned int group_size,
3956 : unsigned int new_group_size)
3957 : {
3958 6126 : if (!vectype)
3959 : {
3960 6126 : tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
3961 6126 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
3962 : }
3963 6126 : if (!vectype)
3964 : return false;
3965 : /* Allow the split if one of the two new groups would operate on full
3966 : vectors *within* rather than across one scalar loop iteration.
3967 : This is purely a heuristic, but it should work well for group
3968 : sizes of 3 and 4, where the possible splits are:
3969 :
3970 : 3->2+1: OK if the vector has exactly two elements
3971 : 4->2+2: Likewise
3972 : 4->3+1: Less clear-cut. */
3973 6126 : if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
3974 3456 : || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
3975 2693 : return false;
3976 3433 : return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
3977 : }
3978 :
3979 : /* Analyze an SLP instance starting from a group of grouped stores. Call
3980 : vect_build_slp_tree to build a tree of packed stmts if possible.
3981 : Return FALSE if it's impossible to SLP any stmt in the loop. */
3982 :
3983 : static bool
3984 : vect_analyze_slp_instance (vec_info *vinfo,
3985 : scalar_stmts_to_slp_tree_map_t *bst_map,
3986 : stmt_vec_info stmt_info, slp_instance_kind kind,
3987 : unsigned max_tree_size, unsigned *limit,
3988 : bool force_single_lane);
3989 :
3990 : /* Build an interleaving scheme for the store sources RHS_NODES from
3991 : SCALAR_STMTS. */
3992 :
3993 : static slp_tree
3994 8056 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
3995 : vec<stmt_vec_info> &scalar_stmts,
3996 : poly_uint64 max_nunits)
3997 : {
3998 8056 : unsigned int group_size = scalar_stmts.length ();
3999 16112 : slp_tree node = vect_create_new_slp_node (scalar_stmts,
4000 8056 : SLP_TREE_CHILDREN
4001 : (rhs_nodes[0]).length ());
4002 8056 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
4003 8056 : node->max_nunits = max_nunits;
4004 8056 : for (unsigned l = 0;
4005 16139 : l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
4006 : {
4007 : /* And a permute merging all RHS SLP trees. */
4008 8083 : slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
4009 8083 : VEC_PERM_EXPR);
4010 8083 : SLP_TREE_CHILDREN (node).quick_push (perm);
4011 8083 : SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
4012 8083 : SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
4013 8083 : perm->max_nunits = max_nunits;
4014 8083 : SLP_TREE_LANES (perm) = group_size;
4015 : /* ??? We should set this NULL but that's not expected. */
4016 8083 : SLP_TREE_REPRESENTATIVE (perm)
4017 8083 : = SLP_TREE_REPRESENTATIVE (SLP_TREE_CHILDREN (rhs_nodes[0])[l]);
4018 31449 : for (unsigned j = 0; j < rhs_nodes.length (); ++j)
4019 : {
4020 23366 : SLP_TREE_CHILDREN (perm)
4021 23366 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
4022 23366 : SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
4023 23366 : for (unsigned k = 0;
4024 49048 : k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
4025 : {
4026 : /* ??? We should populate SLP_TREE_SCALAR_STMTS
4027 : or SLP_TREE_SCALAR_OPS but then we might have
4028 : a mix of both in our children. */
4029 25682 : SLP_TREE_LANE_PERMUTATION (perm)
4030 25682 : .quick_push (std::make_pair (j, k));
4031 : }
4032 : }
4033 :
4034 : /* Now we have a single permute node but we cannot code-generate
4035 : the case with more than two inputs.
4036 : Perform pairwise reduction, reducing the two inputs
4037 : with the least number of lanes to one and then repeat until
4038 : we end up with two inputs. That scheme makes sure we end
4039 : up with permutes satisfying the restriction of requiring at
4040 : most two vector inputs to produce a single vector output
4041 : when the number of lanes is even. */
4042 15283 : while (SLP_TREE_CHILDREN (perm).length () > 2)
4043 : {
4044 : /* When we have three equal sized groups left the pairwise
4045 : reduction does not result in a scheme that avoids using
4046 : three vectors. Instead merge the first two groups
4047 : to the final size with do-not-care elements (chosen
4048 : from the first group) and then merge with the third.
4049 : { A0, B0, x, A1, B1, x, ... }
4050 : -> { A0, B0, C0, A1, B1, C1, ... }
4051 : This handles group size of three (and at least
4052 : power-of-two multiples of that). */
4053 7200 : if (SLP_TREE_CHILDREN (perm).length () == 3
4054 3290 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4055 3290 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
4056 7200 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4057 2458 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
4058 : {
4059 2152 : int ai = 0;
4060 2152 : int bi = 1;
4061 2152 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4062 2152 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4063 2152 : unsigned n = SLP_TREE_LANES (perm);
4064 :
4065 2152 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4066 2152 : SLP_TREE_LANES (permab) = n;
4067 2152 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4068 2152 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4069 2152 : permab->max_nunits = max_nunits;
4070 : /* ??? Should be NULL but that's not expected. */
4071 2152 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4072 2152 : SLP_TREE_CHILDREN (permab).quick_push (a);
4073 4318 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4074 2166 : SLP_TREE_LANE_PERMUTATION (permab)
4075 2166 : .quick_push (std::make_pair (0, k));
4076 2152 : SLP_TREE_CHILDREN (permab).quick_push (b);
4077 4318 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4078 2166 : SLP_TREE_LANE_PERMUTATION (permab)
4079 2166 : .quick_push (std::make_pair (1, k));
4080 : /* Push the do-not-care lanes. */
4081 4318 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4082 2166 : SLP_TREE_LANE_PERMUTATION (permab)
4083 2166 : .quick_push (std::make_pair (0, k));
4084 :
4085 : /* Put the merged node into 'perm', in place of a. */
4086 2152 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4087 : /* Adjust the references to b in the permutation
4088 : of perm and to the later children which we'll
4089 : remove. */
4090 8650 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4091 : {
4092 6498 : std::pair<unsigned, unsigned> &p
4093 6498 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4094 6498 : if (p.first == (unsigned) bi)
4095 : {
4096 2166 : p.first = ai;
4097 2166 : p.second += SLP_TREE_LANES (a);
4098 : }
4099 4332 : else if (p.first > (unsigned) bi)
4100 2166 : p.first--;
4101 : }
4102 2152 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4103 2152 : break;
4104 : }
4105 :
4106 : /* Pick the two nodes with the least number of lanes,
4107 : prefer the earliest candidate and maintain ai < bi. */
4108 : int ai = -1;
4109 : int bi = -1;
4110 45840 : for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
4111 : {
4112 40792 : if (ai == -1)
4113 5048 : ai = ci;
4114 35744 : else if (bi == -1)
4115 5048 : bi = ci;
4116 30696 : else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4117 30696 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
4118 30696 : || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4119 25256 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
4120 : {
4121 11768 : if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
4122 5884 : <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
4123 2727 : bi = ci;
4124 : else
4125 : {
4126 3157 : ai = bi;
4127 3157 : bi = ci;
4128 : }
4129 : }
4130 : }
4131 :
4132 : /* Produce a merge of nodes ai and bi. */
4133 5048 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4134 5048 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4135 5048 : unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
4136 5048 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4137 5048 : SLP_TREE_LANES (permab) = n;
4138 5048 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4139 5048 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4140 5048 : permab->max_nunits = max_nunits;
4141 : /* ??? Should be NULL but that's not expected. */
4142 5048 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4143 5048 : SLP_TREE_CHILDREN (permab).quick_push (a);
4144 13340 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4145 8292 : SLP_TREE_LANE_PERMUTATION (permab)
4146 8292 : .quick_push (std::make_pair (0, k));
4147 5048 : SLP_TREE_CHILDREN (permab).quick_push (b);
4148 12664 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4149 7616 : SLP_TREE_LANE_PERMUTATION (permab)
4150 7616 : .quick_push (std::make_pair (1, k));
4151 :
4152 : /* Put the merged node into 'perm', in place of a. */
4153 5048 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4154 : /* Adjust the references to b in the permutation
4155 : of perm and to the later children which we'll
4156 : remove. */
4157 73221 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4158 : {
4159 68173 : std::pair<unsigned, unsigned> &p
4160 68173 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4161 68173 : if (p.first == (unsigned) bi)
4162 : {
4163 7616 : p.first = ai;
4164 7616 : p.second += SLP_TREE_LANES (a);
4165 : }
4166 60557 : else if (p.first > (unsigned) bi)
4167 25450 : p.first--;
4168 : }
4169 5048 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4170 : }
4171 : }
4172 :
4173 8056 : return node;
4174 : }
4175 :
4176 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4177 : of KIND. Return true if successful. SCALAR_STMTS is owned by this
4178 : function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
4179 : the caller upon failure. */
4180 :
4181 : static bool
4182 1965274 : vect_build_slp_instance (vec_info *vinfo,
4183 : slp_instance_kind kind,
4184 : vec<stmt_vec_info> &scalar_stmts,
4185 : vec<stmt_vec_info> &root_stmt_infos,
4186 : vec<tree> &remain,
4187 : unsigned max_tree_size, unsigned *limit,
4188 : scalar_stmts_to_slp_tree_map_t *bst_map,
4189 : bool force_single_lane)
4190 : {
4191 : /* If there's no budget left bail out early. */
4192 1965274 : if (*limit == 0)
4193 : {
4194 22248 : scalar_stmts.release ();
4195 22248 : return false;
4196 : }
4197 :
4198 1943026 : if (kind == slp_inst_kind_ctor)
4199 : {
4200 14399 : if (dump_enabled_p ())
4201 86 : dump_printf_loc (MSG_NOTE, vect_location,
4202 : "Analyzing vectorizable constructor: %G\n",
4203 43 : root_stmt_infos[0]->stmt);
4204 : }
4205 1928627 : else if (kind == slp_inst_kind_gcond)
4206 : {
4207 287986 : if (dump_enabled_p ())
4208 5730 : dump_printf_loc (MSG_NOTE, vect_location,
4209 : "Analyzing vectorizable control flow: %G",
4210 2865 : root_stmt_infos[0]->stmt);
4211 : }
4212 1640641 : else if (kind == slp_inst_kind_bb_reduc)
4213 : {
4214 1257681 : if (dump_enabled_p ())
4215 6142 : dump_printf_loc (MSG_NOTE, vect_location,
4216 : "Analyzing vectorizable BB reduction: %G",
4217 3071 : root_stmt_infos[0]->stmt);
4218 : }
4219 :
4220 1943026 : if (dump_enabled_p ())
4221 : {
4222 25565 : dump_printf_loc (MSG_NOTE, vect_location,
4223 : "Starting SLP discovery for\n");
4224 54578 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4225 58026 : dump_printf_loc (MSG_NOTE, vect_location,
4226 29013 : " %G", scalar_stmts[i]->stmt);
4227 : }
4228 :
4229 : /* Build the tree for the SLP instance. */
4230 1943026 : unsigned int group_size = scalar_stmts.length ();
4231 1943026 : bool *matches = XALLOCAVEC (bool, group_size);
4232 1943026 : poly_uint64 max_nunits = 1;
4233 1943026 : unsigned tree_size = 0;
4234 :
4235 1943026 : slp_tree node = NULL;
4236 1943026 : if (group_size > 1 && force_single_lane)
4237 : {
4238 0 : matches[0] = true;
4239 0 : matches[1] = false;
4240 : }
4241 : else
4242 1943026 : node = vect_build_slp_tree (vinfo, scalar_stmts,
4243 : &max_nunits, matches, limit,
4244 : &tree_size, bst_map);
4245 1943026 : if (node != NULL)
4246 : {
4247 : /* Calculate the unrolling factor based on the smallest type. */
4248 787608 : poly_uint64 unrolling_factor
4249 787608 : = calculate_unrolling_factor (max_nunits, group_size);
4250 :
4251 787608 : if (maybe_ne (unrolling_factor, 1U)
4252 787608 : && is_a <bb_vec_info> (vinfo))
4253 : {
4254 0 : unsigned HOST_WIDE_INT const_max_nunits;
4255 0 : if (!max_nunits.is_constant (&const_max_nunits)
4256 0 : || const_max_nunits > group_size)
4257 : {
4258 0 : if (dump_enabled_p ())
4259 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
4260 : "Build SLP failed: store group "
4261 : "size not a multiple of the vector size "
4262 : "in basic block SLP\n");
4263 0 : vect_free_slp_tree (node);
4264 0 : return false;
4265 : }
4266 : /* Fatal mismatch. */
4267 0 : if (dump_enabled_p ())
4268 0 : dump_printf_loc (MSG_NOTE, vect_location,
4269 : "SLP discovery succeeded but node needs "
4270 : "splitting\n");
4271 0 : memset (matches, true, group_size);
4272 0 : matches[group_size / const_max_nunits * const_max_nunits] = false;
4273 0 : vect_free_slp_tree (node);
4274 : }
4275 : else
4276 : {
4277 : /* Create a new SLP instance. */
4278 787608 : slp_instance new_instance = XNEW (class _slp_instance);
4279 787608 : SLP_INSTANCE_TREE (new_instance) = node;
4280 787608 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4281 787608 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4282 787608 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4283 787608 : SLP_INSTANCE_KIND (new_instance) = kind;
4284 787608 : new_instance->reduc_phis = NULL;
4285 787608 : new_instance->cost_vec = vNULL;
4286 787608 : new_instance->subgraph_entries = vNULL;
4287 :
4288 787608 : if (dump_enabled_p ())
4289 22527 : dump_printf_loc (MSG_NOTE, vect_location,
4290 : "SLP size %u vs. limit %u.\n",
4291 : tree_size, max_tree_size);
4292 :
4293 787608 : vinfo->slp_instances.safe_push (new_instance);
4294 :
4295 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4296 : the number of scalar stmts in the root in a few places.
4297 : Verify that assumption holds. */
4298 1575216 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
4299 : .length () == group_size);
4300 :
4301 787608 : if (dump_enabled_p ())
4302 : {
4303 22527 : if (kind == slp_inst_kind_reduc_group)
4304 1460 : dump_printf_loc (MSG_NOTE, vect_location,
4305 : "SLP discovery of size %d reduction group "
4306 : "succeeded\n", group_size);
4307 22527 : dump_printf_loc (MSG_NOTE, vect_location,
4308 : "Final SLP tree for instance %p:\n",
4309 : (void *) new_instance);
4310 22527 : vect_print_slp_graph (MSG_NOTE, vect_location,
4311 : SLP_INSTANCE_TREE (new_instance));
4312 : }
4313 :
4314 787608 : return true;
4315 : }
4316 : }
4317 : /* Failed to SLP. */
4318 :
4319 : /* While we arrive here even with slp_inst_kind_store we should only
4320 : for group_size == 1. The code to split store groups is only in
4321 : vect_analyze_slp_instance now. */
4322 1155418 : gcc_assert (kind != slp_inst_kind_store || group_size == 1);
4323 :
4324 : /* Free the allocated memory. */
4325 1155418 : scalar_stmts.release ();
4326 :
4327 : /* Failed to SLP. */
4328 1155418 : if (dump_enabled_p ())
4329 3038 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4330 : return false;
4331 : }
4332 :
4333 : /* Analyze an SLP instance starting from a the start of a reduction chain.
4334 : Call vect_build_slp_tree to build a tree of packed stmts if possible.
4335 : Return FALSE if SLP build fails. */
4336 :
4337 : static bool
4338 73102 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
4339 : scalar_stmts_to_slp_tree_map_t *bst_map,
4340 : stmt_vec_info scalar_stmt,
4341 : unsigned max_tree_size, unsigned *limit)
4342 : {
4343 73102 : vec<stmt_vec_info> scalar_stmts = vNULL;
4344 :
4345 73102 : bool fail = false;
4346 : /* ??? We could leave operation code checking to SLP discovery. */
4347 73102 : code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
4348 : (vect_orig_stmt (scalar_stmt)));
4349 73102 : bool first = true;
4350 73102 : stmt_vec_info next_stmt = scalar_stmt;
4351 83067 : do
4352 : {
4353 83067 : stmt_vec_info stmt = next_stmt;
4354 83067 : gimple_match_op op;
4355 83067 : if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
4356 0 : gcc_unreachable ();
4357 166134 : tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
4358 83067 : STMT_VINFO_REDUC_IDX (stmt));
4359 83067 : next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
4360 83067 : gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
4361 : || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
4362 88976 : if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)), &op))
4363 0 : gcc_unreachable ();
4364 83067 : if (CONVERT_EXPR_CODE_P (op.code)
4365 4973 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
4366 88028 : && (first
4367 2496 : || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
4368 : ;
4369 78166 : else if (code != op.code)
4370 : {
4371 2666 : fail = true;
4372 2666 : break;
4373 : }
4374 : else
4375 75500 : scalar_stmts.safe_push (stmt);
4376 80401 : first = false;
4377 : }
4378 80401 : while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
4379 73102 : if (fail)
4380 2666 : return false;
4381 :
4382 : /* Remember a stmt with the actual reduction operation. */
4383 70436 : stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
4384 :
4385 : /* When the SSA def chain through reduc-idx does not form a natural
4386 : reduction chain try to linearize an associative operation manually. */
4387 70436 : if (scalar_stmts.length () == 1
4388 67822 : && code.is_tree_code ()
4389 61750 : && associative_tree_code ((tree_code)code)
4390 : /* We may not associate if a fold-left reduction is required. */
4391 130768 : && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
4392 : (reduc_scalar_stmt->stmt)),
4393 : code))
4394 : {
4395 57079 : auto_vec<chain_op_t> chain;
4396 57079 : auto_vec<std::pair<tree_code, gimple *> > worklist;
4397 57079 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
4398 57079 : if (is_a <gassign *> (scalar_stmts[0]->stmt)
4399 : /* We cannot linearize an operation that vect_slp_linearize_chain
4400 : would not put on its worklist. */
4401 57079 : && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
4402 : {
4403 56426 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4404 56426 : scalar_stmts[0]->stmt, op_stmt,
4405 : other_op_stmt,
4406 : NULL);
4407 :
4408 56426 : scalar_stmts.truncate (0);
4409 56426 : stmt_vec_info tail = NULL;
4410 282315 : for (auto el : chain)
4411 : {
4412 113400 : if (el.dt == vect_external_def
4413 113400 : || el.dt == vect_constant_def
4414 113400 : || el.code != (tree_code) code)
4415 : {
4416 363 : scalar_stmts.release ();
4417 363 : return false;
4418 : }
4419 113037 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4420 113037 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4421 110753 : || STMT_VINFO_REDUC_DEF (stmt))
4422 : {
4423 56296 : gcc_assert (tail == NULL);
4424 56296 : tail = stmt;
4425 56296 : continue;
4426 : }
4427 56741 : scalar_stmts.safe_push (stmt);
4428 : }
4429 56063 : gcc_assert (tail);
4430 : }
4431 :
4432 : /* When this linearization didn't produce a chain see if stripping
4433 : a wrapping sign conversion produces one. */
4434 56716 : if (scalar_stmts.length () == 1
4435 56716 : && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
4436 : || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
4437 : {
4438 54929 : gimple *stmt = scalar_stmts[0]->stmt;
4439 54929 : if (!is_gimple_assign (stmt)
4440 53763 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
4441 4578 : || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
4442 59507 : || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
4443 4578 : TREE_TYPE (gimple_assign_rhs1 (stmt))))
4444 : {
4445 53174 : scalar_stmts.release ();
4446 53174 : return false;
4447 : }
4448 1755 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
4449 1755 : if (!is_gimple_assign (stmt)
4450 1755 : || gimple_assign_rhs_code (stmt) != (tree_code)code)
4451 : {
4452 1736 : scalar_stmts.release ();
4453 1736 : return false;
4454 : }
4455 19 : chain.truncate (0);
4456 19 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4457 : stmt, op_stmt, other_op_stmt, NULL);
4458 :
4459 19 : scalar_stmts.truncate (0);
4460 19 : stmt_vec_info tail = NULL;
4461 93 : for (auto el : chain)
4462 : {
4463 44 : if (el.dt == vect_external_def
4464 44 : || el.dt == vect_constant_def
4465 44 : || el.code != (tree_code) code)
4466 : {
4467 8 : scalar_stmts.release ();
4468 8 : return false;
4469 : }
4470 36 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4471 36 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4472 36 : || STMT_VINFO_REDUC_DEF (stmt))
4473 : {
4474 0 : gcc_assert (tail == NULL);
4475 0 : tail = stmt;
4476 0 : continue;
4477 : }
4478 36 : scalar_stmts.safe_push (stmt);
4479 : }
4480 : /* Unlike the above this does not include the reduction SSA
4481 : cycle. */
4482 11 : gcc_assert (!tail);
4483 : }
4484 :
4485 1798 : if (scalar_stmts.length () < 2)
4486 : {
4487 1673 : scalar_stmts.release ();
4488 1673 : return false;
4489 : }
4490 :
4491 125 : if (dump_enabled_p ())
4492 : {
4493 34 : dump_printf_loc (MSG_NOTE, vect_location,
4494 : "Starting SLP discovery of reduction chain for\n");
4495 140 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4496 212 : dump_printf_loc (MSG_NOTE, vect_location,
4497 106 : " %G", scalar_stmts[i]->stmt);
4498 : }
4499 :
4500 125 : unsigned int group_size = scalar_stmts.length ();
4501 125 : bool *matches = XALLOCAVEC (bool, group_size);
4502 125 : poly_uint64 max_nunits = 1;
4503 125 : unsigned tree_size = 0;
4504 125 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4505 : &max_nunits, matches, limit,
4506 125 : &tree_size, bst_map);
4507 125 : if (!node)
4508 : {
4509 47 : scalar_stmts.release ();
4510 47 : return false;
4511 : }
4512 :
4513 78 : unsigned cycle_id = vinfo->reduc_infos.length ();
4514 78 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
4515 78 : vinfo->reduc_infos.safe_push (reduc_info);
4516 78 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
4517 78 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
4518 78 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
4519 78 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
4520 78 : reduc_info->is_reduc_chain = true;
4521 :
4522 : /* Build the node for the PHI and possibly the conversions. */
4523 78 : slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
4524 78 : SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
4525 78 : phis->cycle_info.id = cycle_id;
4526 78 : SLP_TREE_LANES (phis) = group_size;
4527 78 : if (reduc_scalar_stmt == scalar_stmt)
4528 74 : SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
4529 : else
4530 4 : SLP_TREE_VECTYPE (phis)
4531 4 : = signed_or_unsigned_type_for (TYPE_UNSIGNED
4532 : (TREE_TYPE (gimple_get_lhs
4533 : (scalar_stmt->stmt))),
4534 : SLP_TREE_VECTYPE (node));
4535 : /* ??? vect_cse_slp_nodes cannot cope with cycles without any
4536 : SLP_TREE_SCALAR_STMTS. */
4537 78 : SLP_TREE_SCALAR_STMTS (phis).create (group_size);
4538 393 : for (unsigned i = 0; i < group_size; ++i)
4539 315 : SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
4540 :
4541 78 : slp_tree op_input = phis;
4542 78 : if (reduc_scalar_stmt != scalar_stmt)
4543 : {
4544 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4545 4 : SLP_TREE_REPRESENTATIVE (conv)
4546 4 : = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
4547 4 : STMT_VINFO_REDUC_IDX
4548 : (reduc_scalar_stmt)));
4549 4 : SLP_TREE_CHILDREN (conv).quick_push (phis);
4550 4 : conv->cycle_info.id = cycle_id;
4551 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4552 4 : SLP_TREE_LANES (conv) = group_size;
4553 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
4554 4 : SLP_TREE_SCALAR_STMTS (conv) = vNULL;
4555 4 : op_input = conv;
4556 : }
4557 :
4558 78 : slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
4559 78 : SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
4560 78 : SLP_TREE_CHILDREN (reduc).quick_push (op_input);
4561 78 : SLP_TREE_CHILDREN (reduc).quick_push (node);
4562 78 : reduc->cycle_info.id = cycle_id;
4563 78 : SLP_TREE_REDUC_IDX (reduc) = 0;
4564 78 : SLP_TREE_LANES (reduc) = group_size;
4565 78 : SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
4566 : /* ??? For the reduction epilogue we need a live lane. */
4567 78 : SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
4568 78 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
4569 315 : for (unsigned i = 1; i < group_size; ++i)
4570 237 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
4571 :
4572 78 : if (reduc_scalar_stmt != scalar_stmt)
4573 : {
4574 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4575 4 : SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
4576 4 : SLP_TREE_CHILDREN (conv).quick_push (reduc);
4577 4 : conv->cycle_info.id = cycle_id;
4578 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4579 4 : SLP_TREE_LANES (conv) = group_size;
4580 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
4581 : /* ??? For the reduction epilogue we need a live lane. */
4582 4 : SLP_TREE_SCALAR_STMTS (conv).create (group_size);
4583 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
4584 8 : for (unsigned i = 1; i < group_size; ++i)
4585 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
4586 4 : reduc = conv;
4587 : }
4588 :
4589 78 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
4590 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4591 78 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4592 78 : SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
4593 78 : SLP_TREE_REF_COUNT (reduc)++;
4594 :
4595 : /* Create a new SLP instance. */
4596 78 : slp_instance new_instance = XNEW (class _slp_instance);
4597 78 : SLP_INSTANCE_TREE (new_instance) = reduc;
4598 78 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4599 78 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4600 78 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4601 78 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4602 78 : new_instance->reduc_phis = NULL;
4603 78 : new_instance->cost_vec = vNULL;
4604 78 : new_instance->subgraph_entries = vNULL;
4605 :
4606 78 : vinfo->slp_instances.safe_push (new_instance);
4607 :
4608 78 : if (dump_enabled_p ())
4609 : {
4610 24 : dump_printf_loc (MSG_NOTE, vect_location,
4611 : "Final SLP tree for instance %p:\n",
4612 : (void *) new_instance);
4613 24 : vect_print_slp_graph (MSG_NOTE, vect_location,
4614 : SLP_INSTANCE_TREE (new_instance));
4615 : }
4616 :
4617 78 : return true;
4618 57079 : }
4619 :
4620 13357 : if (scalar_stmts.length () <= 1)
4621 : {
4622 10743 : scalar_stmts.release ();
4623 10743 : return false;
4624 : }
4625 :
4626 2614 : scalar_stmts.reverse ();
4627 2614 : stmt_vec_info reduc_phi_info = next_stmt;
4628 :
4629 : /* Build the tree for the SLP instance. */
4630 2614 : vec<stmt_vec_info> root_stmt_infos = vNULL;
4631 2614 : vec<tree> remain = vNULL;
4632 :
4633 2614 : if (dump_enabled_p ())
4634 : {
4635 193 : dump_printf_loc (MSG_NOTE, vect_location,
4636 : "Starting SLP discovery of reduction chain for\n");
4637 1029 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4638 1672 : dump_printf_loc (MSG_NOTE, vect_location,
4639 836 : " %G", scalar_stmts[i]->stmt);
4640 : }
4641 :
4642 : /* Build the tree for the SLP instance. */
4643 2614 : unsigned int group_size = scalar_stmts.length ();
4644 2614 : bool *matches = XALLOCAVEC (bool, group_size);
4645 2614 : poly_uint64 max_nunits = 1;
4646 2614 : unsigned tree_size = 0;
4647 :
4648 : /* ??? We need this only for SLP discovery. */
4649 10210 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4650 7596 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
4651 :
4652 2614 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4653 : &max_nunits, matches, limit,
4654 2614 : &tree_size, bst_map);
4655 :
4656 10210 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4657 7596 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
4658 :
4659 2614 : if (node != NULL)
4660 : {
4661 : /* Create a new SLP instance. */
4662 2329 : slp_instance new_instance = XNEW (class _slp_instance);
4663 2329 : SLP_INSTANCE_TREE (new_instance) = node;
4664 2329 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4665 2329 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4666 2329 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4667 2329 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4668 2329 : new_instance->reduc_phis = NULL;
4669 2329 : new_instance->cost_vec = vNULL;
4670 2329 : new_instance->subgraph_entries = vNULL;
4671 :
4672 2329 : vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
4673 2329 : reduc_info->is_reduc_chain = true;
4674 :
4675 2329 : if (dump_enabled_p ())
4676 144 : dump_printf_loc (MSG_NOTE, vect_location,
4677 : "SLP size %u vs. limit %u.\n",
4678 : tree_size, max_tree_size);
4679 :
4680 : /* Fixup SLP reduction chains. If this is a reduction chain with
4681 : a conversion in front amend the SLP tree with a node for that. */
4682 2329 : gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
4683 2329 : if (is_gimple_assign (scalar_def)
4684 2329 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
4685 : {
4686 43 : stmt_vec_info conv_info = vect_stmt_to_vectorize
4687 43 : (STMT_VINFO_REDUC_DEF (reduc_phi_info));
4688 43 : scalar_stmts = vNULL;
4689 43 : scalar_stmts.create (group_size);
4690 135 : for (unsigned i = 0; i < group_size; ++i)
4691 92 : scalar_stmts.quick_push (conv_info);
4692 43 : slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
4693 43 : SLP_TREE_VECTYPE (conv)
4694 43 : = get_vectype_for_scalar_type (vinfo,
4695 43 : TREE_TYPE
4696 : (gimple_assign_lhs (scalar_def)),
4697 : group_size);
4698 43 : SLP_TREE_REDUC_IDX (conv) = 0;
4699 43 : conv->cycle_info.id = node->cycle_info.id;
4700 43 : SLP_TREE_CHILDREN (conv).quick_push (node);
4701 43 : SLP_INSTANCE_TREE (new_instance) = conv;
4702 : }
4703 : /* Fill the backedge child of the PHI SLP node. The
4704 : general matching code cannot find it because the
4705 : scalar code does not reflect how we vectorize the
4706 : reduction. */
4707 2329 : use_operand_p use_p;
4708 2329 : imm_use_iterator imm_iter;
4709 2329 : class loop *loop = LOOP_VINFO_LOOP (vinfo);
4710 11195 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
4711 : gimple_get_lhs (scalar_def))
4712 : /* There are exactly two non-debug uses, the reduction
4713 : PHI and the loop-closed PHI node. */
4714 6537 : if (!is_gimple_debug (USE_STMT (use_p))
4715 6537 : && gimple_bb (USE_STMT (use_p)) == loop->header)
4716 : {
4717 2329 : auto_vec<stmt_vec_info, 64> phis (group_size);
4718 2329 : stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
4719 9180 : for (unsigned i = 0; i < group_size; ++i)
4720 6851 : phis.quick_push (phi_info);
4721 2329 : slp_tree *phi_node = bst_map->get (phis);
4722 2329 : unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
4723 4658 : SLP_TREE_CHILDREN (*phi_node)[dest_idx]
4724 2329 : = SLP_INSTANCE_TREE (new_instance);
4725 2329 : SLP_INSTANCE_TREE (new_instance)->refcnt++;
4726 2329 : }
4727 :
4728 2329 : vinfo->slp_instances.safe_push (new_instance);
4729 :
4730 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4731 : the number of scalar stmts in the root in a few places.
4732 : Verify that assumption holds. */
4733 4658 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
4734 : .length () == group_size);
4735 :
4736 2329 : if (dump_enabled_p ())
4737 : {
4738 144 : dump_printf_loc (MSG_NOTE, vect_location,
4739 : "Final SLP tree for instance %p:\n",
4740 : (void *) new_instance);
4741 144 : vect_print_slp_graph (MSG_NOTE, vect_location,
4742 : SLP_INSTANCE_TREE (new_instance));
4743 : }
4744 :
4745 2329 : return true;
4746 : }
4747 :
4748 : /* Failed to SLP. */
4749 285 : scalar_stmts.release ();
4750 285 : if (dump_enabled_p ())
4751 49 : dump_printf_loc (MSG_NOTE, vect_location,
4752 : "SLP discovery of reduction chain failed\n");
4753 : return false;
4754 : }
4755 :
4756 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4757 : of KIND. Return true if successful. */
4758 :
4759 : static bool
4760 99183 : vect_analyze_slp_reduction (loop_vec_info vinfo,
4761 : stmt_vec_info scalar_stmt,
4762 : unsigned max_tree_size, unsigned *limit,
4763 : scalar_stmts_to_slp_tree_map_t *bst_map,
4764 : bool force_single_lane)
4765 : {
4766 99183 : slp_instance_kind kind = slp_inst_kind_reduc_group;
4767 :
4768 : /* Try to gather a reduction chain. Only attempt if there's budget left
4769 : since chain analysis may build multi-lane trees that consume limit. */
4770 99183 : if (! force_single_lane
4771 73387 : && *limit != 0
4772 73387 : && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
4773 172285 : && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
4774 : max_tree_size, limit))
4775 : return true;
4776 :
4777 96776 : vec<stmt_vec_info> scalar_stmts;
4778 96776 : scalar_stmts.create (1);
4779 96776 : scalar_stmts.quick_push (scalar_stmt);
4780 :
4781 96776 : if (dump_enabled_p ())
4782 : {
4783 3864 : dump_printf_loc (MSG_NOTE, vect_location,
4784 : "Starting SLP discovery for\n");
4785 7728 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4786 7728 : dump_printf_loc (MSG_NOTE, vect_location,
4787 3864 : " %G", scalar_stmts[i]->stmt);
4788 : }
4789 :
4790 : /* Build the tree for the SLP instance. */
4791 96776 : unsigned int group_size = scalar_stmts.length ();
4792 96776 : bool *matches = XALLOCAVEC (bool, group_size);
4793 96776 : poly_uint64 max_nunits = 1;
4794 96776 : unsigned tree_size = 0;
4795 :
4796 96776 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts,
4797 : &max_nunits, matches, limit,
4798 : &tree_size, bst_map);
4799 96776 : if (node != NULL)
4800 : {
4801 : /* Create a new SLP instance. */
4802 93769 : slp_instance new_instance = XNEW (class _slp_instance);
4803 93769 : SLP_INSTANCE_TREE (new_instance) = node;
4804 93769 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4805 93769 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4806 93769 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4807 93769 : SLP_INSTANCE_KIND (new_instance) = kind;
4808 93769 : new_instance->reduc_phis = NULL;
4809 93769 : new_instance->cost_vec = vNULL;
4810 93769 : new_instance->subgraph_entries = vNULL;
4811 :
4812 93769 : if (dump_enabled_p ())
4813 3744 : dump_printf_loc (MSG_NOTE, vect_location,
4814 : "SLP size %u vs. limit %u.\n",
4815 : tree_size, max_tree_size);
4816 :
4817 93769 : vinfo->slp_instances.safe_push (new_instance);
4818 :
4819 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4820 : the number of scalar stmts in the root in a few places.
4821 : Verify that assumption holds. */
4822 187538 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
4823 : .length () == group_size);
4824 :
4825 93769 : if (dump_enabled_p ())
4826 : {
4827 3744 : dump_printf_loc (MSG_NOTE, vect_location,
4828 : "Final SLP tree for instance %p:\n",
4829 : (void *) new_instance);
4830 3744 : vect_print_slp_graph (MSG_NOTE, vect_location,
4831 : SLP_INSTANCE_TREE (new_instance));
4832 : }
4833 :
4834 93769 : return true;
4835 : }
4836 : /* Failed to SLP. */
4837 :
4838 : /* Free the allocated memory. */
4839 3007 : scalar_stmts.release ();
4840 :
4841 : /* Failed to SLP. */
4842 3007 : if (dump_enabled_p ())
4843 120 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4844 : return false;
4845 : }
4846 :
4847 : /* Analyze a single SLP reduction group. If successful add a SLP instance
4848 : for it and return true, otherwise return false and have *MATCHES
4849 : populated. */
4850 :
4851 : static bool
4852 24148 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
4853 : vec<stmt_vec_info> scalar_stmts,
4854 : scalar_stmts_to_slp_tree_map_t *bst_map,
4855 : unsigned max_tree_size, unsigned *limit,
4856 : bool *matches)
4857 : {
4858 : /* Try to form a reduction group. Size-1 groups are not suitable
4859 : for SLP reduction and should fall back to single-lane reduction. */
4860 45533 : unsigned int group_size = scalar_stmts.length ();
4861 24148 : if (group_size <= 1)
4862 : return false;
4863 17471 : if (!matches)
4864 4550 : matches = XALLOCAVEC (bool, group_size);
4865 17471 : poly_uint64 max_nunits = 1;
4866 17471 : unsigned tree_size = 0;
4867 17471 : slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts,
4868 : &max_nunits, matches, limit,
4869 : &tree_size, bst_map);
4870 17471 : if (!node)
4871 : return false;
4872 :
4873 : /* Create a new SLP instance. */
4874 2763 : slp_instance new_instance = XNEW (class _slp_instance);
4875 2763 : SLP_INSTANCE_TREE (new_instance) = node;
4876 2763 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4877 2763 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4878 2763 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4879 2763 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
4880 2763 : new_instance->reduc_phis = NULL;
4881 2763 : new_instance->cost_vec = vNULL;
4882 2763 : new_instance->subgraph_entries = vNULL;
4883 :
4884 2763 : if (dump_enabled_p ())
4885 213 : dump_printf_loc (MSG_NOTE, vect_location,
4886 : "SLP size %u vs. limit %u.\n",
4887 : tree_size, max_tree_size);
4888 :
4889 2763 : loop_vinfo->slp_instances.safe_push (new_instance);
4890 :
4891 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4892 : the number of scalar stmts in the root in a few places.
4893 : Verify that assumption holds. */
4894 5526 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
4895 : .length () == group_size);
4896 :
4897 2763 : if (dump_enabled_p ())
4898 : {
4899 213 : dump_printf_loc (MSG_NOTE, vect_location,
4900 : "SLP discovery of size %d reduction group "
4901 : "succeeded\n", group_size);
4902 213 : dump_printf_loc (MSG_NOTE, vect_location,
4903 : "Final SLP tree for instance %p:\n",
4904 : (void *) new_instance);
4905 213 : vect_print_slp_graph (MSG_NOTE, vect_location,
4906 : SLP_INSTANCE_TREE (new_instance));
4907 : }
4908 :
4909 : return true;
4910 : }
4911 :
4912 : /* Analyze reductions in LOOP_VINFO and populate SLP instances
4913 : accordingly. Returns false if something fails. */
4914 :
4915 : static bool
4916 503996 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
4917 : unsigned max_tree_size, unsigned *limit,
4918 : scalar_stmts_to_slp_tree_map_t *bst_map,
4919 : bool force_single_lane)
4920 : {
4921 572840 : if (loop_vinfo->reductions.is_empty ())
4922 : return true;
4923 :
4924 : /* Collect reduction statements we can combine into
4925 : a SLP reduction. */
4926 73407 : vec<stmt_vec_info> scalar_stmts;
4927 73407 : scalar_stmts.create (loop_vinfo->reductions.length ());
4928 325802 : for (auto next_info : loop_vinfo->reductions)
4929 : {
4930 105581 : next_info = vect_stmt_to_vectorize (next_info);
4931 105581 : if ((STMT_VINFO_RELEVANT_P (next_info)
4932 14 : || STMT_VINFO_LIVE_P (next_info))
4933 : /* ??? Make sure we didn't skip a conversion around a
4934 : reduction path. In that case we'd have to reverse
4935 : engineer that conversion stmt following the chain using
4936 : reduc_idx and from the PHI using reduc_def. */
4937 105567 : && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
4938 105567 : || (STMT_VINFO_DEF_TYPE (next_info)
4939 : == vect_double_reduction_def)))
4940 : {
4941 : /* Do not discover SLP reductions combining lane-reducing
4942 : ops, that will fail later. */
4943 105567 : if (!force_single_lane
4944 105567 : && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
4945 79074 : scalar_stmts.quick_push (next_info);
4946 : /* Do SLP discovery for single-lane reductions. */
4947 26493 : else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
4948 : max_tree_size, limit,
4949 : bst_map,
4950 : force_single_lane))
4951 : {
4952 0 : scalar_stmts.release ();
4953 0 : return false;
4954 : }
4955 : }
4956 : }
4957 :
4958 73407 : if (scalar_stmts.length () > 1)
4959 : {
4960 : /* Try to form a reduction group. */
4961 4644 : unsigned int group_size = scalar_stmts.length ();
4962 4644 : bool *matches = XALLOCAVEC (bool, group_size);
4963 4644 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
4964 : max_tree_size, limit, matches))
4965 1581 : return true;
4966 :
4967 : /* When analysis as a single SLP reduction group failed try to
4968 : form sub-groups by collecting matching lanes. Do not recurse
4969 : that on failure (to limit compile-time costs), but recurse
4970 : for the initial non-matching parts. Everything not covered
4971 : by a sub-group gets single-reduction treatment. */
4972 3518 : vec<stmt_vec_info> cands = vNULL;
4973 11365 : while (matches[0])
4974 : {
4975 11227 : cands.truncate (0);
4976 11227 : cands.reserve (group_size, true);
4977 88333 : for (unsigned i = 0; i < group_size; ++i)
4978 77106 : if (matches[i])
4979 19560 : cands.quick_push (scalar_stmts[i]);
4980 :
4981 : /* Try to form a reduction group. */
4982 11227 : if (vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
4983 : max_tree_size, limit, NULL))
4984 1207 : cands = vNULL;
4985 : else
4986 : {
4987 : /* Do SLP discovery for single-lane reductions. */
4988 47153 : for (auto stmt_info : cands)
4989 17118 : if (! vect_analyze_slp_reduction (loop_vinfo,
4990 : vect_stmt_to_vectorize
4991 : (stmt_info),
4992 : max_tree_size, limit,
4993 : bst_map, force_single_lane))
4994 : {
4995 25 : scalar_stmts.release ();
4996 25 : cands.release ();
4997 25 : return false;
4998 : }
4999 : }
5000 : /* Remove the handled stmts from scalar_stmts and try again,
5001 : possibly repeating the above with updated matches[]. */
5002 : unsigned j = 0;
5003 88238 : for (unsigned i = 0; i < group_size; ++i)
5004 77036 : if (!matches[i])
5005 : {
5006 57516 : scalar_stmts[j] = scalar_stmts[i];
5007 57516 : ++j;
5008 : }
5009 11202 : scalar_stmts.truncate (j);
5010 11202 : group_size = scalar_stmts.length ();
5011 11202 : if (group_size <= 1)
5012 : break;
5013 8277 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts,
5014 : bst_map, max_tree_size, limit,
5015 : matches))
5016 : return true;
5017 : }
5018 : }
5019 : /* Do SLP discovery for single-lane reductions. */
5020 268068 : for (auto stmt_info : scalar_stmts)
5021 55572 : if (! vect_analyze_slp_reduction (loop_vinfo,
5022 : vect_stmt_to_vectorize (stmt_info),
5023 : max_tree_size, limit,
5024 : bst_map, force_single_lane))
5025 : {
5026 2982 : scalar_stmts.release ();
5027 2982 : return false;
5028 : }
5029 :
5030 68844 : scalar_stmts.release ();
5031 68844 : return true;
5032 : }
5033 :
5034 : /* Analyze an SLP instance starting from a group of grouped stores. Call
5035 : vect_build_slp_tree to build a tree of packed stmts if possible.
5036 : Return FALSE if it's impossible to SLP any stmt in the group. */
5037 :
5038 : static bool
5039 1118724 : vect_analyze_slp_instance (vec_info *vinfo,
5040 : scalar_stmts_to_slp_tree_map_t *bst_map,
5041 : stmt_vec_info stmt_info,
5042 : slp_instance_kind kind,
5043 : unsigned max_tree_size, unsigned *limit,
5044 : bool force_single_lane)
5045 : {
5046 1118724 : vec<stmt_vec_info> scalar_stmts;
5047 :
5048 1118724 : if (is_a <bb_vec_info> (vinfo))
5049 1088693 : vect_location = stmt_info->stmt;
5050 :
5051 1118724 : gcc_assert (kind == slp_inst_kind_store);
5052 :
5053 : /* Collect the stores and store them in scalar_stmts. */
5054 1118724 : scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
5055 1118724 : stmt_vec_info next_info = stmt_info;
5056 5560791 : while (next_info)
5057 : {
5058 3323343 : scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
5059 3323343 : next_info = DR_GROUP_NEXT_ELEMENT (next_info);
5060 : }
5061 :
5062 1118724 : vec<stmt_vec_info> root_stmt_infos = vNULL;
5063 1118724 : vec<tree> remain = vNULL;
5064 :
5065 : /* Build the tree for the SLP instance. */
5066 :
5067 : /* If there's no budget left bail out early. */
5068 1118724 : if (*limit == 0)
5069 : return false;
5070 :
5071 1118704 : if (dump_enabled_p ())
5072 : {
5073 4127 : dump_printf_loc (MSG_NOTE, vect_location,
5074 : "Starting SLP discovery for\n");
5075 24101 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
5076 39948 : dump_printf_loc (MSG_NOTE, vect_location,
5077 19974 : " %G", scalar_stmts[i]->stmt);
5078 : }
5079 :
5080 : /* Build the tree for the SLP instance. */
5081 1118704 : unsigned int group_size = scalar_stmts.length ();
5082 1118704 : bool *matches = XALLOCAVEC (bool, group_size);
5083 1118704 : poly_uint64 max_nunits = 1;
5084 1118704 : unsigned tree_size = 0;
5085 1118704 : unsigned i;
5086 :
5087 1118704 : slp_tree node = NULL;
5088 1118704 : if (group_size > 1 && force_single_lane)
5089 : {
5090 1733 : matches[0] = true;
5091 1733 : matches[1] = false;
5092 : }
5093 : else
5094 1116971 : node = vect_build_slp_tree (vinfo, scalar_stmts,
5095 : &max_nunits, matches, limit,
5096 : &tree_size, bst_map);
5097 1118704 : if (node != NULL)
5098 : {
5099 : /* Calculate the unrolling factor based on the smallest type. */
5100 693356 : poly_uint64 unrolling_factor
5101 693356 : = calculate_unrolling_factor (max_nunits, group_size);
5102 :
5103 693356 : if (maybe_ne (unrolling_factor, 1U)
5104 693356 : && is_a <bb_vec_info> (vinfo))
5105 : {
5106 0 : unsigned HOST_WIDE_INT const_max_nunits;
5107 0 : if (!max_nunits.is_constant (&const_max_nunits)
5108 0 : || const_max_nunits > group_size)
5109 : {
5110 0 : if (dump_enabled_p ())
5111 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5112 : "Build SLP failed: store group "
5113 : "size not a multiple of the vector size "
5114 : "in basic block SLP\n");
5115 0 : vect_free_slp_tree (node);
5116 0 : return false;
5117 : }
5118 : /* Fatal mismatch. */
5119 0 : if (dump_enabled_p ())
5120 0 : dump_printf_loc (MSG_NOTE, vect_location,
5121 : "SLP discovery succeeded but node needs "
5122 : "splitting\n");
5123 0 : memset (matches, true, group_size);
5124 0 : matches[group_size / const_max_nunits * const_max_nunits] = false;
5125 0 : vect_free_slp_tree (node);
5126 : }
5127 : else
5128 : {
5129 : /* Create a new SLP instance. */
5130 693356 : slp_instance new_instance = XNEW (class _slp_instance);
5131 693356 : SLP_INSTANCE_TREE (new_instance) = node;
5132 693356 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5133 693356 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5134 693356 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5135 693356 : SLP_INSTANCE_KIND (new_instance) = kind;
5136 693356 : new_instance->reduc_phis = NULL;
5137 693356 : new_instance->cost_vec = vNULL;
5138 693356 : new_instance->subgraph_entries = vNULL;
5139 :
5140 693356 : if (dump_enabled_p ())
5141 3122 : dump_printf_loc (MSG_NOTE, vect_location,
5142 : "SLP size %u vs. limit %u.\n",
5143 : tree_size, max_tree_size);
5144 :
5145 693356 : vinfo->slp_instances.safe_push (new_instance);
5146 :
5147 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5148 : the number of scalar stmts in the root in a few places.
5149 : Verify that assumption holds. */
5150 1386712 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
5151 : .length () == group_size);
5152 :
5153 693356 : if (dump_enabled_p ())
5154 : {
5155 3122 : dump_printf_loc (MSG_NOTE, vect_location,
5156 : "Final SLP tree for instance %p:\n",
5157 : (void *) new_instance);
5158 3122 : vect_print_slp_graph (MSG_NOTE, vect_location,
5159 : SLP_INSTANCE_TREE (new_instance));
5160 : }
5161 :
5162 693356 : return true;
5163 : }
5164 : }
5165 : /* Failed to SLP. */
5166 :
5167 : /* Try to break the group up into pieces. */
5168 425348 : if (*limit > 0 && kind == slp_inst_kind_store)
5169 : {
5170 : /* ??? We could delay all the actual splitting of store-groups
5171 : until after SLP discovery of the original group completed.
5172 : Then we can recurse to vect_build_slp_instance directly. */
5173 1112158 : for (i = 0; i < group_size; i++)
5174 1112158 : if (!matches[i])
5175 : break;
5176 :
5177 : /* For basic block SLP, try to break the group up into multiples of
5178 : a vector size. */
5179 425347 : if (is_a <bb_vec_info> (vinfo)
5180 425347 : && (i > 1 && i < group_size))
5181 : {
5182 : /* Free the allocated memory. */
5183 159338 : scalar_stmts.release ();
5184 :
5185 159338 : tree scalar_type
5186 159338 : = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
5187 318676 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
5188 159338 : 1 << floor_log2 (i));
5189 159338 : unsigned HOST_WIDE_INT const_nunits;
5190 159338 : if (vectype
5191 159338 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
5192 : {
5193 : /* Split into two groups at the first vector boundary. */
5194 159338 : gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
5195 159338 : unsigned group1_size = i & ~(const_nunits - 1);
5196 :
5197 159338 : if (dump_enabled_p ())
5198 66 : dump_printf_loc (MSG_NOTE, vect_location,
5199 : "Splitting SLP group at stmt %u\n", i);
5200 159338 : stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
5201 : group1_size);
5202 159338 : bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
5203 : kind, max_tree_size,
5204 : limit, false);
5205 : /* Split the rest at the failure point and possibly
5206 : re-analyze the remaining matching part if it has
5207 : at least two lanes. */
5208 159338 : if (group1_size < i
5209 5667 : && (i + 1 < group_size
5210 3064 : || i - group1_size > 1))
5211 : {
5212 2631 : stmt_vec_info rest2 = rest;
5213 2631 : rest = vect_split_slp_store_group (rest, i - group1_size);
5214 2631 : if (i - group1_size > 1)
5215 57 : res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
5216 : kind, max_tree_size,
5217 : limit, false);
5218 : }
5219 : /* Re-analyze the non-matching tail if it has at least
5220 : two lanes. */
5221 159338 : if (i + 1 < group_size)
5222 22519 : res |= vect_analyze_slp_instance (vinfo, bst_map,
5223 : rest, kind, max_tree_size,
5224 : limit, false);
5225 159338 : return res;
5226 : }
5227 : }
5228 :
5229 : /* For loop vectorization split the RHS into arbitrary pieces of
5230 : size >= 1. */
5231 266009 : else if (is_a <loop_vec_info> (vinfo)
5232 266009 : && (group_size != 1 && i < group_size))
5233 : {
5234 8319 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
5235 28 : bool masked_p = call
5236 28 : && gimple_call_internal_p (call)
5237 28 : && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
5238 : /* There are targets that cannot do even/odd interleaving schemes
5239 : so they absolutely need to use load/store-lanes. For now
5240 : force single-lane SLP for them - they would be happy with
5241 : uniform power-of-two lanes (but depending on element size),
5242 : but even if we can use 'i' as indicator we would need to
5243 : backtrack when later lanes fail to discover with the same
5244 : granularity. We cannot turn any of strided or scatter store
5245 : into store-lanes. */
5246 : /* ??? If this is not in sync with what get_load_store_type
5247 : later decides the SLP representation is not good for other
5248 : store vectorization methods. */
5249 8319 : bool want_store_lanes
5250 8319 : = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5251 8319 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5252 6219 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5253 6215 : && compare_step_with_zero (vinfo, stmt_info) > 0
5254 14445 : && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
5255 16638 : masked_p, group_size, i));
5256 8319 : if (want_store_lanes || force_single_lane)
5257 : i = 1;
5258 :
5259 : /* A fatal discovery fail doesn't always mean single-lane SLP
5260 : isn't a possibility, so try. */
5261 6586 : if (i == 0)
5262 : i = 1;
5263 :
5264 8319 : if (dump_enabled_p ())
5265 897 : dump_printf_loc (MSG_NOTE, vect_location,
5266 : "Splitting SLP group at stmt %u\n", i);
5267 :
5268 : /* Analyze the stored values and pinch them together with
5269 : a permute node so we can preserve the whole store group. */
5270 8319 : auto_vec<slp_tree> rhs_nodes;
5271 8319 : poly_uint64 max_nunits = 1;
5272 :
5273 8319 : unsigned int rhs_common_nlanes = 0;
5274 8319 : unsigned int start = 0, end = i;
5275 37229 : while (start < group_size)
5276 : {
5277 29173 : gcc_assert (end - start >= 1);
5278 29173 : vec<stmt_vec_info> substmts;
5279 29173 : substmts.create (end - start);
5280 90715 : for (unsigned j = start; j < end; ++j)
5281 61542 : substmts.quick_push (scalar_stmts[j]);
5282 29173 : max_nunits = 1;
5283 29173 : node = vect_build_slp_tree (vinfo, substmts,
5284 : &max_nunits,
5285 : matches, limit, &tree_size, bst_map);
5286 29173 : if (node)
5287 : {
5288 23311 : rhs_nodes.safe_push (node);
5289 23311 : vect_update_max_nunits (&max_nunits, node->max_nunits);
5290 23311 : if (start == 0)
5291 8062 : rhs_common_nlanes = SLP_TREE_LANES (node);
5292 15249 : else if (rhs_common_nlanes != SLP_TREE_LANES (node))
5293 1375 : rhs_common_nlanes = 0;
5294 23311 : start = end;
5295 23311 : if (want_store_lanes || force_single_lane)
5296 5207 : end = start + 1;
5297 : else
5298 : end = group_size;
5299 : }
5300 : else
5301 : {
5302 5862 : substmts.release ();
5303 5862 : if (end - start == 1)
5304 : {
5305 : /* Single-lane discovery failed. Free resources. */
5306 283 : for (auto node : rhs_nodes)
5307 8 : vect_free_slp_tree (node);
5308 263 : scalar_stmts.release ();
5309 263 : if (dump_enabled_p ())
5310 39 : dump_printf_loc (MSG_NOTE, vect_location,
5311 : "SLP discovery failed\n");
5312 263 : return false;
5313 : }
5314 :
5315 : /* ??? It really happens that we soft-fail SLP
5316 : build at a mismatch but the matching part hard-fails
5317 : later. As we know we arrived here with a group
5318 : larger than one try a group of size one! */
5319 5599 : if (!matches[0])
5320 44 : end = start + 1;
5321 : else
5322 12209 : for (unsigned j = start; j < end; j++)
5323 12209 : if (!matches[j - start])
5324 : {
5325 : end = j;
5326 : break;
5327 : }
5328 : }
5329 : }
5330 :
5331 : /* Now re-assess whether we want store lanes in case the
5332 : discovery ended up producing all single-lane RHSs. */
5333 8056 : if (! want_store_lanes
5334 8056 : && rhs_common_nlanes == 1
5335 7007 : && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5336 7007 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5337 5272 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5338 5269 : && compare_step_with_zero (vinfo, stmt_info) > 0
5339 13264 : && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
5340 : group_size, masked_p)
5341 : != IFN_LAST))
5342 : want_store_lanes = true;
5343 :
5344 : /* Now we assume we can build the root SLP node from all stores. */
5345 8056 : if (want_store_lanes)
5346 : {
5347 : /* For store-lanes feed the store node with all RHS nodes
5348 : in order. */
5349 0 : node = vect_create_new_slp_node (scalar_stmts,
5350 0 : SLP_TREE_CHILDREN
5351 : (rhs_nodes[0]).length ());
5352 0 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
5353 0 : node->max_nunits = max_nunits;
5354 0 : node->ldst_lanes = true;
5355 0 : SLP_TREE_CHILDREN (node)
5356 0 : .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
5357 0 : + rhs_nodes.length () - 1);
5358 : /* First store value and possibly mask. */
5359 0 : SLP_TREE_CHILDREN (node)
5360 0 : .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
5361 : /* Rest of the store values. All mask nodes are the same,
5362 : this should be guaranteed by dataref group discovery. */
5363 0 : for (unsigned j = 1; j < rhs_nodes.length (); ++j)
5364 0 : SLP_TREE_CHILDREN (node)
5365 0 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
5366 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
5367 0 : child->refcnt++;
5368 : }
5369 : else
5370 8056 : node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts,
5371 : max_nunits);
5372 :
5373 31359 : while (!rhs_nodes.is_empty ())
5374 23303 : vect_free_slp_tree (rhs_nodes.pop ());
5375 :
5376 : /* Create a new SLP instance. */
5377 8056 : slp_instance new_instance = XNEW (class _slp_instance);
5378 8056 : SLP_INSTANCE_TREE (new_instance) = node;
5379 8056 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5380 8056 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5381 8056 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5382 8056 : SLP_INSTANCE_KIND (new_instance) = kind;
5383 8056 : new_instance->reduc_phis = NULL;
5384 8056 : new_instance->cost_vec = vNULL;
5385 8056 : new_instance->subgraph_entries = vNULL;
5386 :
5387 8056 : if (dump_enabled_p ())
5388 858 : dump_printf_loc (MSG_NOTE, vect_location,
5389 : "SLP size %u vs. limit %u.\n",
5390 : tree_size, max_tree_size);
5391 :
5392 8056 : vinfo->slp_instances.safe_push (new_instance);
5393 :
5394 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5395 : the number of scalar stmts in the root in a few places.
5396 : Verify that assumption holds. */
5397 16112 : gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
5398 : .length () == group_size);
5399 :
5400 8056 : if (dump_enabled_p ())
5401 : {
5402 858 : dump_printf_loc (MSG_NOTE, vect_location,
5403 : "Final SLP tree for instance %p:\n",
5404 : (void *) new_instance);
5405 858 : vect_print_slp_graph (MSG_NOTE, vect_location,
5406 : SLP_INSTANCE_TREE (new_instance));
5407 : }
5408 8056 : return true;
5409 8319 : }
5410 : else
5411 : /* Free the allocated memory. */
5412 257690 : scalar_stmts.release ();
5413 :
5414 : /* Even though the first vector did not all match, we might be able to SLP
5415 : (some) of the remainder. FORNOW ignore this possibility. */
5416 : }
5417 : else
5418 : /* Free the allocated memory. */
5419 1 : scalar_stmts.release ();
5420 :
5421 : /* Failed to SLP. */
5422 257691 : if (dump_enabled_p ())
5423 42 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
5424 : return false;
5425 : }
5426 :
5427 : /* qsort comparator ordering SLP load nodes. */
5428 :
5429 : static int
5430 2664531 : vllp_cmp (const void *a_, const void *b_)
5431 : {
5432 2664531 : const slp_tree a = *(const slp_tree *)a_;
5433 2664531 : const slp_tree b = *(const slp_tree *)b_;
5434 2664531 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
5435 2664531 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
5436 2664531 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5437 1542763 : && STMT_VINFO_GROUPED_ACCESS (b0)
5438 4145954 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5439 : {
5440 : /* Same group, order after lanes used. */
5441 345656 : if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
5442 : return 1;
5443 336865 : else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
5444 : return -1;
5445 : else
5446 : {
5447 : /* Try to order loads using the same lanes together, breaking
5448 : the tie with the lane number that first differs. */
5449 327325 : if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5450 327325 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5451 : return 0;
5452 327325 : else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
5453 327325 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5454 : return 1;
5455 323270 : else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5456 323270 : && SLP_TREE_LOAD_PERMUTATION (b).exists ())
5457 : return -1;
5458 : else
5459 : {
5460 315792 : for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
5461 315792 : if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5462 315792 : != SLP_TREE_LOAD_PERMUTATION (b)[i])
5463 : {
5464 : /* In-order lane first, that's what the above case for
5465 : no permutation does. */
5466 314480 : if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
5467 : return -1;
5468 192586 : else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
5469 : return 1;
5470 101058 : else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5471 101058 : < SLP_TREE_LOAD_PERMUTATION (b)[i])
5472 : return -1;
5473 : else
5474 : return 1;
5475 : }
5476 : return 0;
5477 : }
5478 : }
5479 : }
5480 : else /* Different groups or non-groups. */
5481 : {
5482 : /* Order groups as their first element to keep them together. */
5483 2318875 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5484 2318875 : a0 = DR_GROUP_FIRST_ELEMENT (a0);
5485 2318875 : if (STMT_VINFO_GROUPED_ACCESS (b0))
5486 2318875 : b0 = DR_GROUP_FIRST_ELEMENT (b0);
5487 2318875 : if (a0 == b0)
5488 : return 0;
5489 : /* Tie using UID. */
5490 2318755 : else if (gimple_uid (STMT_VINFO_STMT (a0))
5491 2318755 : < gimple_uid (STMT_VINFO_STMT (b0)))
5492 : return -1;
5493 : else
5494 : {
5495 1030512 : gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
5496 : != gimple_uid (STMT_VINFO_STMT (b0)));
5497 : return 1;
5498 : }
5499 : }
5500 : }
5501 :
5502 : /* Return whether if the load permutation of NODE is consecutive starting
5503 : with value START_VAL in the first element. If START_VAL is not given
5504 : the first element's value is used. */
5505 :
5506 : bool
5507 637002 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
5508 : {
5509 637002 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
5510 :
5511 637002 : if (!perm.exists () || !perm.length ())
5512 : return false;
5513 :
5514 637002 : if (start_val == UINT_MAX)
5515 79425 : start_val = perm[0];
5516 :
5517 1257884 : for (unsigned int i = 0; i < perm.length (); i++)
5518 644293 : if (perm[i] != start_val + (unsigned int) i)
5519 : return false;
5520 :
5521 : return true;
5522 : }
5523 :
5524 : /* Process the set of LOADS that are all from the same dataref group. */
5525 :
5526 : static void
5527 161498 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5528 : scalar_stmts_to_slp_tree_map_t *bst_map,
5529 : const array_slice<slp_tree> &loads,
5530 : bool force_single_lane)
5531 : {
5532 : /* We at this point want to lower without a fixed VF or vector
5533 : size in mind which means we cannot actually compute whether we
5534 : need three or more vectors for a load permutation yet. So always
5535 : lower. */
5536 161498 : stmt_vec_info first
5537 161498 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
5538 161498 : unsigned group_lanes = DR_GROUP_SIZE (first);
5539 :
5540 : /* Verify if all load permutations can be implemented with a suitably
5541 : large element load-lanes operation. */
5542 161498 : unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
5543 161498 : if (STMT_VINFO_STRIDED_P (first)
5544 159051 : || compare_step_with_zero (loop_vinfo, first) <= 0
5545 156387 : || exact_log2 (ld_lanes_lanes) == -1
5546 : /* ??? For now only support the single-lane case as there is
5547 : missing support on the store-lane side and code generation
5548 : isn't up to the task yet. */
5549 153600 : || ld_lanes_lanes != 1
5550 304128 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
5551 : group_lanes / ld_lanes_lanes,
5552 : false) == IFN_LAST)
5553 : ld_lanes_lanes = 0;
5554 : else
5555 : /* Verify the loads access the same number of lanes aligned to
5556 : ld_lanes_lanes. */
5557 0 : for (slp_tree load : loads)
5558 : {
5559 0 : if (SLP_TREE_LANES (load) != ld_lanes_lanes)
5560 : {
5561 : ld_lanes_lanes = 0;
5562 : break;
5563 : }
5564 0 : unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
5565 0 : if (first % ld_lanes_lanes != 0)
5566 : {
5567 : ld_lanes_lanes = 0;
5568 : break;
5569 : }
5570 0 : if (!vect_load_perm_consecutive_p (load))
5571 : {
5572 : ld_lanes_lanes = 0;
5573 : break;
5574 : }
5575 : }
5576 :
5577 : /* Only a power-of-two number of lanes matches interleaving with N levels.
5578 : ??? An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
5579 : at each step. */
5580 262025 : if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
5581 : return;
5582 :
5583 265495 : for (slp_tree load : loads)
5584 : {
5585 : /* Leave masked or gather loads alone for now. */
5586 187391 : if (!SLP_TREE_CHILDREN (load).is_empty ())
5587 60685 : continue;
5588 :
5589 : /* For single-element interleaving spanning multiple vectors avoid
5590 : lowering, we want to use VMAT_ELEMENTWISE later. */
5591 187385 : if (ld_lanes_lanes == 0
5592 187385 : && SLP_TREE_LANES (load) == 1
5593 168019 : && !DR_GROUP_NEXT_ELEMENT (first)
5594 267033 : && maybe_gt (group_lanes,
5595 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
5596 51294 : return;
5597 :
5598 : /* We want to pattern-match special cases here and keep those
5599 : alone. Candidates are splats and load-lane. */
5600 :
5601 : /* We need to lower only loads of less than half of the groups
5602 : lanes, including duplicate lanes. Note this leaves nodes
5603 : with a non-1:1 load permutation around instead of canonicalizing
5604 : those into a load and a permute node. Removing this early
5605 : check would do such canonicalization. */
5606 136091 : if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
5607 57099 : && ld_lanes_lanes == 0)
5608 57099 : continue;
5609 :
5610 : /* Build the permute to get the original load permutation order. */
5611 78992 : bool contiguous = vect_load_perm_consecutive_p (load);
5612 78992 : lane_permutation_t final_perm;
5613 78992 : final_perm.create (SLP_TREE_LANES (load));
5614 158898 : for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
5615 159812 : final_perm.quick_push (
5616 79906 : std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
5617 :
5618 : /* When the load permutation accesses a contiguous unpermuted,
5619 : power-of-two aligned and sized chunk leave the load alone.
5620 : We can likely (re-)load it more efficiently rather than
5621 : extracting it from the larger load.
5622 : ??? Long-term some of the lowering should move to where
5623 : the vector types involved are fixed. */
5624 82572 : if (!force_single_lane
5625 78992 : && ld_lanes_lanes == 0
5626 53168 : && contiguous
5627 52925 : && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
5628 6576 : && pow2p_hwi (SLP_TREE_LANES (load))
5629 6540 : && pow2p_hwi (group_lanes)
5630 3580 : && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
5631 82572 : && group_lanes % SLP_TREE_LANES (load) == 0)
5632 : {
5633 3580 : final_perm.release ();
5634 3580 : continue;
5635 : }
5636 :
5637 : /* First build (and possibly re-use) a load node for the
5638 : unpermuted group. Gaps in the middle and on the end are
5639 : represented with NULL stmts. */
5640 75412 : vec<stmt_vec_info> stmts;
5641 75412 : stmts.create (group_lanes);
5642 268639 : for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
5643 : {
5644 193227 : if (s != first)
5645 122668 : for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
5646 4853 : stmts.quick_push (NULL);
5647 193227 : stmts.quick_push (s);
5648 : }
5649 138987 : for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
5650 63575 : stmts.quick_push (NULL);
5651 75412 : poly_uint64 max_nunits = 1;
5652 75412 : bool *matches = XALLOCAVEC (bool, group_lanes);
5653 75412 : unsigned limit = 1;
5654 75412 : unsigned tree_size = 0;
5655 75412 : slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts,
5656 : &max_nunits, matches, &limit,
5657 75412 : &tree_size, bst_map);
5658 75412 : gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
5659 :
5660 75412 : if (ld_lanes_lanes != 0)
5661 : {
5662 : /* ??? If this is not in sync with what get_load_store_type
5663 : later decides the SLP representation is not good for other
5664 : store vectorization methods. */
5665 0 : l0->ldst_lanes = true;
5666 0 : load->ldst_lanes = true;
5667 : }
5668 :
5669 234746 : while (1)
5670 : {
5671 155079 : unsigned group_lanes = SLP_TREE_LANES (l0);
5672 155079 : if (ld_lanes_lanes != 0
5673 155079 : || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
5674 : break;
5675 :
5676 : /* Try to lower by reducing the group to half its size using an
5677 : interleaving scheme. For this try to compute whether all
5678 : elements needed for this load are in even or odd elements of
5679 : an even/odd decomposition with N consecutive elements.
5680 : Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
5681 : with N == 2. */
5682 : /* ??? Only an even number of lanes can be handed this way, but the
5683 : fallback below could work for any number. We have to make sure
5684 : to round up in that case. */
5685 79667 : gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
5686 11858 : unsigned even = 0, odd = 0;
5687 11858 : if ((group_lanes & 1) == 0)
5688 : {
5689 11858 : even = (1 << ceil_log2 (group_lanes)) - 1;
5690 11858 : odd = even;
5691 48109 : for (auto l : final_perm)
5692 : {
5693 12535 : even &= ~l.second;
5694 12535 : odd &= l.second;
5695 : }
5696 : }
5697 :
5698 : /* Now build an even or odd extraction from the unpermuted load. */
5699 79667 : lane_permutation_t perm;
5700 79667 : perm.create ((group_lanes + 1) / 2);
5701 79667 : unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
5702 79667 : unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
5703 79667 : if (even_level
5704 10935 : && group_lanes % (2 * even_level) == 0
5705 : /* ??? When code generating permutes we do not try to pun
5706 : to larger component modes so level != 1 isn't a natural
5707 : even/odd extract. Prefer one if possible. */
5708 10935 : && (even_level == 1 || !odd_level || odd_level != 1))
5709 : {
5710 : /* { 0, 1, ... 4, 5 ..., } */
5711 38623 : for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
5712 60858 : for (unsigned j = 0; j < even_level; ++j)
5713 30602 : perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
5714 : }
5715 68732 : else if (odd_level)
5716 : {
5717 : /* { ..., 2, 3, ... 6, 7 } */
5718 3461 : gcc_assert (group_lanes % (2 * odd_level) == 0);
5719 15029 : for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
5720 23190 : for (unsigned j = 0; j < odd_level; ++j)
5721 11622 : perm.quick_push
5722 11622 : (std::make_pair (0, (2 * i + 1) * odd_level + j));
5723 : }
5724 : else
5725 : {
5726 : /* As fallback extract all used lanes and fill to half the
5727 : group size by repeating the last element.
5728 : ??? This is quite a bad strathegy for re-use - we could
5729 : brute force our way to find more optimal filling lanes to
5730 : maximize re-use when looking at all loads from the group. */
5731 67839 : auto_bitmap l;
5732 271412 : for (auto p : final_perm)
5733 67895 : bitmap_set_bit (l, p.second);
5734 67839 : unsigned i = 0;
5735 67839 : bitmap_iterator bi;
5736 135734 : EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
5737 67895 : perm.quick_push (std::make_pair (0, i));
5738 271508 : while (perm.length () < (group_lanes + 1) / 2)
5739 67915 : perm.quick_push (perm.last ());
5740 67839 : }
5741 :
5742 : /* Update final_perm with the intermediate permute. */
5743 160011 : for (unsigned i = 0; i < final_perm.length (); ++i)
5744 : {
5745 80344 : unsigned l = final_perm[i].second;
5746 80344 : unsigned j;
5747 89074 : for (j = 0; j < perm.length (); ++j)
5748 89074 : if (perm[j].second == l)
5749 : {
5750 80344 : final_perm[i].second = j;
5751 80344 : break;
5752 : }
5753 80344 : gcc_assert (j < perm.length ());
5754 : }
5755 :
5756 : /* And create scalar stmts. */
5757 79667 : vec<stmt_vec_info> perm_stmts;
5758 79667 : perm_stmts.create (perm.length ());
5759 257701 : for (unsigned i = 0; i < perm.length (); ++i)
5760 178034 : perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
5761 :
5762 79667 : slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
5763 79667 : SLP_TREE_CHILDREN (p).quick_push (l0);
5764 79667 : SLP_TREE_LANE_PERMUTATION (p) = perm;
5765 79667 : SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
5766 79667 : SLP_TREE_LANES (p) = perm.length ();
5767 79667 : SLP_TREE_REPRESENTATIVE (p) = SLP_TREE_REPRESENTATIVE (load);
5768 : /* ??? As we have scalar stmts for this intermediate permute we
5769 : could CSE it via bst_map but we do not want to pick up
5770 : another SLP node with a load permutation. We instead should
5771 : have a "local" CSE map here. */
5772 79667 : SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
5773 :
5774 : /* We now have a node for (group_lanes + 1) / 2 lanes. */
5775 79667 : l0 = p;
5776 79667 : }
5777 :
5778 : /* And finally from the ordered reduction node create the
5779 : permute to shuffle the lanes into the original load-permutation
5780 : order. We replace the original load node with this. */
5781 75412 : SLP_TREE_CODE (load) = VEC_PERM_EXPR;
5782 75412 : SLP_TREE_LOAD_PERMUTATION (load).release ();
5783 75412 : SLP_TREE_LANE_PERMUTATION (load) = final_perm;
5784 75412 : SLP_TREE_CHILDREN (load).create (1);
5785 75412 : SLP_TREE_CHILDREN (load).quick_push (l0);
5786 : }
5787 : }
5788 :
5789 : /* Transform SLP loads in the SLP graph created by SLP discovery to
5790 : group loads from the same group and lower load permutations that
5791 : are unlikely to be supported into a series of permutes.
5792 : In the degenerate case of having only single-lane SLP instances
5793 : this should result in a series of permute nodes emulating an
5794 : interleaving scheme. */
5795 :
5796 : static void
5797 485653 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5798 : scalar_stmts_to_slp_tree_map_t *bst_map,
5799 : bool force_single_lane)
5800 : {
5801 : /* Gather and sort loads across all instances. */
5802 485653 : hash_set<slp_tree> visited;
5803 485653 : auto_vec<slp_tree> loads;
5804 2231028 : for (auto inst : loop_vinfo->slp_instances)
5805 776011 : vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
5806 485653 : if (loads.is_empty ())
5807 91163 : return;
5808 394490 : loads.qsort (vllp_cmp);
5809 :
5810 : /* Now process each dataref group separately. */
5811 394490 : unsigned firsti = 0;
5812 734289 : for (unsigned i = 1; i < loads.length (); ++i)
5813 : {
5814 339799 : slp_tree first = loads[firsti];
5815 339799 : slp_tree next = loads[i];
5816 339799 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
5817 339799 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
5818 339799 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5819 158243 : && STMT_VINFO_GROUPED_ACCESS (b0)
5820 484975 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5821 63064 : continue;
5822 : /* Now we have one or multiple SLP loads of the same group from
5823 : firsti to i - 1. */
5824 276735 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5825 95179 : vect_lower_load_permutations (loop_vinfo, bst_map,
5826 95179 : make_array_slice (&loads[firsti],
5827 : i - firsti),
5828 : force_single_lane);
5829 : firsti = i;
5830 : }
5831 788980 : if (firsti < loads.length ()
5832 788980 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
5833 66319 : vect_lower_load_permutations (loop_vinfo, bst_map,
5834 66319 : make_array_slice (&loads[firsti],
5835 66319 : loads.length () - firsti),
5836 : force_single_lane);
5837 485653 : }
5838 :
5839 : /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
5840 : trees of packed scalar stmts if SLP is possible. */
5841 :
5842 : opt_result
5843 1140723 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
5844 : bool force_single_lane)
5845 : {
5846 1140723 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5847 1140723 : unsigned int i;
5848 1140723 : stmt_vec_info first_element;
5849 1140723 : slp_instance instance;
5850 :
5851 1140723 : DUMP_VECT_SCOPE ("vect_analyze_slp");
5852 :
5853 1140723 : unsigned limit = max_tree_size;
5854 :
5855 1140723 : scalar_stmts_to_slp_tree_map_t *bst_map
5856 1140723 : = new scalar_stmts_to_slp_tree_map_t ();
5857 :
5858 : /* Find SLP sequences starting from groups of grouped stores. */
5859 3217985 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
5860 936810 : if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
5861 : slp_inst_kind_store, max_tree_size, &limit,
5862 : force_single_lane)
5863 936810 : && loop_vinfo)
5864 : {
5865 271 : release_scalar_stmts_to_slp_tree_map (bst_map);
5866 271 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5867 : }
5868 :
5869 : /* For loops also start SLP discovery from non-grouped stores. */
5870 1140452 : if (loop_vinfo)
5871 : {
5872 : data_reference_p dr;
5873 1662348 : FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
5874 1158352 : if (DR_IS_WRITE (dr))
5875 : {
5876 374664 : stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
5877 : /* Grouped stores are already handled above. */
5878 374664 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5879 101615 : continue;
5880 273049 : vec<stmt_vec_info> stmts;
5881 273049 : vec<stmt_vec_info> roots = vNULL;
5882 273049 : vec<tree> remain = vNULL;
5883 273049 : stmts.create (1);
5884 273049 : stmts.quick_push (stmt_info);
5885 273049 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5886 : stmts, roots, remain, max_tree_size,
5887 : &limit, bst_map, force_single_lane))
5888 : {
5889 7006 : release_scalar_stmts_to_slp_tree_map (bst_map);
5890 7006 : return opt_result::failure_at (vect_location,
5891 : "SLP build failed.\n");
5892 : }
5893 : }
5894 :
5895 : stmt_vec_info stmt_info;
5896 504036 : FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
5897 : {
5898 20 : vec<stmt_vec_info> stmts;
5899 20 : vec<stmt_vec_info> roots = vNULL;
5900 20 : vec<tree> remain = vNULL;
5901 20 : stmts.create (1);
5902 20 : stmts.quick_push (stmt_info);
5903 20 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5904 : stmts, roots, remain, max_tree_size,
5905 : &limit, bst_map, force_single_lane))
5906 : {
5907 0 : release_scalar_stmts_to_slp_tree_map (bst_map);
5908 0 : return opt_result::failure_at (vect_location,
5909 : "SLP build failed.\n");
5910 : }
5911 : }
5912 : }
5913 :
5914 1133446 : if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
5915 : {
5916 1923778 : for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
5917 : {
5918 1294328 : vect_location = bb_vinfo->roots[i].roots[0]->stmt;
5919 : /* Apply patterns. */
5920 4042258 : for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
5921 5495860 : bb_vinfo->roots[i].stmts[j]
5922 2835865 : = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
5923 1294328 : if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
5924 1294328 : bb_vinfo->roots[i].stmts,
5925 1294328 : bb_vinfo->roots[i].roots,
5926 1294328 : bb_vinfo->roots[i].remain,
5927 : max_tree_size, &limit, bst_map, false))
5928 : {
5929 137302 : bb_vinfo->roots[i].roots = vNULL;
5930 137302 : bb_vinfo->roots[i].remain = vNULL;
5931 : }
5932 1294328 : bb_vinfo->roots[i].stmts = vNULL;
5933 : }
5934 : }
5935 :
5936 1133446 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
5937 : {
5938 : /* Find SLP sequences starting from groups of reductions. */
5939 503996 : if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
5940 : bst_map, force_single_lane))
5941 : {
5942 3007 : release_scalar_stmts_to_slp_tree_map (bst_map);
5943 3007 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5944 : }
5945 :
5946 : /* Make sure to vectorize only-live stmts, usually inductions. */
5947 2259753 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
5948 1461787 : for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
5949 695149 : gsi_next (&gsi))
5950 : {
5951 705001 : gphi *lc_phi = *gsi;
5952 705001 : tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
5953 705001 : stmt_vec_info stmt_info;
5954 705001 : if (TREE_CODE (def) == SSA_NAME
5955 590267 : && !virtual_operand_p (def)
5956 303559 : && (stmt_info = loop_vinfo->lookup_def (def))
5957 272046 : && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
5958 272046 : && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
5959 211883 : && STMT_VINFO_LIVE_P (stmt_info)
5960 211883 : && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
5961 814979 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
5962 : {
5963 109891 : vec<stmt_vec_info> stmts;
5964 109891 : vec<stmt_vec_info> roots = vNULL;
5965 109891 : vec<tree> remain = vNULL;
5966 109891 : stmts.create (1);
5967 109891 : stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
5968 109891 : if (! vect_build_slp_instance (vinfo,
5969 : slp_inst_kind_reduc_group,
5970 : stmts, roots, remain,
5971 : max_tree_size, &limit,
5972 : bst_map, force_single_lane))
5973 : {
5974 9852 : release_scalar_stmts_to_slp_tree_map (bst_map);
5975 9852 : return opt_result::failure_at (vect_location,
5976 : "SLP build failed.\n");
5977 : }
5978 : }
5979 9852 : }
5980 :
5981 : /* Find SLP sequences starting from gconds. */
5982 1232771 : for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
5983 : {
5984 289688 : auto cond_info = loop_vinfo->lookup_stmt (cond);
5985 :
5986 289688 : cond_info = vect_stmt_to_vectorize (cond_info);
5987 289688 : vec<stmt_vec_info> roots = vNULL;
5988 289688 : roots.safe_push (cond_info);
5989 289688 : gimple *stmt = STMT_VINFO_STMT (cond_info);
5990 289688 : tree args0 = gimple_cond_lhs (stmt);
5991 289688 : tree args1 = gimple_cond_rhs (stmt);
5992 :
5993 : /* These should be enforced by cond lowering, but if it failed
5994 : bail. */
5995 289688 : if (gimple_cond_code (stmt) != NE_EXPR
5996 288561 : || TREE_TYPE (args0) != boolean_type_node
5997 577674 : || !integer_zerop (args1))
5998 : {
5999 1702 : roots.release ();
6000 1702 : release_scalar_stmts_to_slp_tree_map (bst_map);
6001 1702 : return opt_result::failure_at (vect_location,
6002 : "SLP build failed.\n");
6003 : }
6004 :
6005 : /* An argument without a loop def will be codegened from vectorizing the
6006 : root gcond itself. As such we don't need to try to build an SLP tree
6007 : from them. It's highly likely that the resulting SLP tree here if both
6008 : arguments have a def will be incompatible, but we rely on it being split
6009 : later on. */
6010 287986 : auto varg = loop_vinfo->lookup_def (args0);
6011 287986 : vec<stmt_vec_info> stmts;
6012 287986 : vec<tree> remain = vNULL;
6013 287986 : stmts.create (1);
6014 287986 : stmts.quick_push (vect_stmt_to_vectorize (varg));
6015 :
6016 287986 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
6017 : stmts, roots, remain,
6018 : max_tree_size, &limit,
6019 : bst_map, force_single_lane))
6020 : {
6021 3782 : roots.release ();
6022 3782 : release_scalar_stmts_to_slp_tree_map (bst_map);
6023 3782 : return opt_result::failure_at (vect_location,
6024 : "SLP build failed.\n");
6025 : }
6026 : }
6027 : }
6028 :
6029 1115103 : hash_set<slp_tree> visited_patterns;
6030 1115103 : slp_tree_to_load_perm_map_t perm_cache;
6031 1115103 : slp_compat_nodes_map_t compat_cache;
6032 :
6033 : /* See if any patterns can be found in the SLP tree. */
6034 1115103 : bool pattern_found = false;
6035 3815171 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6036 1584965 : pattern_found |= vect_match_slp_patterns (instance, vinfo,
6037 : &visited_patterns, &perm_cache,
6038 : &compat_cache);
6039 :
6040 : /* If any were found optimize permutations of loads. */
6041 1115103 : if (pattern_found)
6042 : {
6043 264 : hash_map<slp_tree, slp_tree> load_map;
6044 3649 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6045 : {
6046 3121 : slp_tree root = SLP_INSTANCE_TREE (instance);
6047 3121 : optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
6048 : &load_map, root);
6049 : }
6050 264 : }
6051 :
6052 : /* Check whether we should force some SLP instances to use load/store-lanes
6053 : and do so by forcing SLP re-discovery with single lanes. We used
6054 : to cancel SLP when this applied to all instances in a loop but now
6055 : we decide this per SLP instance. It's important to do this only
6056 : after SLP pattern recognition. */
6057 1115103 : if (is_a <loop_vec_info> (vinfo))
6058 1261664 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6059 776011 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
6060 293935 : && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
6061 : {
6062 293935 : slp_tree slp_root = SLP_INSTANCE_TREE (instance);
6063 293935 : unsigned int group_size = SLP_TREE_LANES (slp_root);
6064 293935 : tree vectype = SLP_TREE_VECTYPE (slp_root);
6065 :
6066 293935 : stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
6067 293935 : gimple *rep = STMT_VINFO_STMT (rep_info);
6068 293935 : bool masked = (is_gimple_call (rep)
6069 2508 : && gimple_call_internal_p (rep)
6070 296423 : && internal_fn_mask_index
6071 2488 : (gimple_call_internal_fn (rep)) != -1);
6072 293915 : if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
6073 29725 : || slp_root->ldst_lanes
6074 323660 : || (vect_store_lanes_supported (vectype, group_size, masked)
6075 : == IFN_LAST))
6076 293935 : continue;
6077 :
6078 0 : auto_vec<slp_tree> loads;
6079 0 : hash_set<slp_tree> visited;
6080 0 : vect_gather_slp_loads (loads, slp_root, visited);
6081 :
6082 : /* Check whether any load in the SLP instance is possibly
6083 : permuted. */
6084 0 : bool loads_permuted = false;
6085 0 : slp_tree load_node;
6086 0 : unsigned j;
6087 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6088 : {
6089 0 : if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
6090 0 : continue;
6091 0 : for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
6092 0 : if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
6093 : {
6094 : loads_permuted = true;
6095 : break;
6096 : }
6097 : }
6098 :
6099 : /* If the loads and stores can use load/store-lanes force re-discovery
6100 : with single lanes. */
6101 0 : if (loads_permuted)
6102 : {
6103 0 : bool can_use_lanes = true;
6104 : bool prefer_load_lanes = false;
6105 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6106 0 : if (STMT_VINFO_GROUPED_ACCESS
6107 : (SLP_TREE_REPRESENTATIVE (load_node)))
6108 : {
6109 0 : stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
6110 : (SLP_TREE_REPRESENTATIVE (load_node));
6111 0 : rep = STMT_VINFO_STMT (stmt_vinfo);
6112 0 : masked = (is_gimple_call (rep)
6113 0 : && gimple_call_internal_p (rep)
6114 0 : && internal_fn_mask_index
6115 0 : (gimple_call_internal_fn (rep)));
6116 : /* Use SLP for strided accesses (or if we can't
6117 : load-lanes). */
6118 0 : if (STMT_VINFO_STRIDED_P (stmt_vinfo)
6119 0 : || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
6120 0 : || vect_load_lanes_supported
6121 0 : (SLP_TREE_VECTYPE (load_node),
6122 0 : DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
6123 : /* ??? During SLP re-discovery with a single lane
6124 : a masked grouped load will appear permuted and
6125 : discovery will fail. We have to rework this
6126 : on the discovery side - for now avoid ICEing. */
6127 0 : || masked)
6128 : {
6129 : can_use_lanes = false;
6130 : break;
6131 : }
6132 : /* Make sure that the target would prefer store-lanes
6133 : for at least one of the loads.
6134 :
6135 : ??? Perhaps we should instead require this for
6136 : all loads? */
6137 0 : prefer_load_lanes
6138 : = (prefer_load_lanes
6139 0 : || SLP_TREE_LANES (load_node) == group_size
6140 0 : || (vect_slp_prefer_store_lanes_p
6141 0 : (vinfo, stmt_vinfo,
6142 : SLP_TREE_VECTYPE (load_node), masked,
6143 : group_size, SLP_TREE_LANES (load_node))));
6144 : }
6145 :
6146 0 : if (can_use_lanes && prefer_load_lanes)
6147 : {
6148 0 : if (dump_enabled_p ())
6149 0 : dump_printf_loc (MSG_NOTE, vect_location,
6150 : "SLP instance %p can use load/store-lanes,"
6151 : " re-discovering with single-lanes\n",
6152 : (void *) instance);
6153 :
6154 0 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
6155 :
6156 0 : vect_free_slp_instance (instance);
6157 0 : limit = max_tree_size;
6158 0 : bool res = vect_analyze_slp_instance (vinfo, bst_map,
6159 : stmt_info,
6160 : slp_inst_kind_store,
6161 : max_tree_size, &limit,
6162 : true);
6163 0 : gcc_assert (res);
6164 0 : auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
6165 0 : LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
6166 : }
6167 : }
6168 0 : }
6169 :
6170 : /* When we end up with load permutations that we cannot possibly handle,
6171 : like those requiring three vector inputs, lower them using interleaving
6172 : like schemes. */
6173 1115103 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6174 : {
6175 485653 : vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
6176 485653 : if (dump_enabled_p ())
6177 : {
6178 20040 : dump_printf_loc (MSG_NOTE, vect_location,
6179 : "SLP graph after lowering permutations:\n");
6180 20040 : hash_set<slp_tree> visited;
6181 89280 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6182 29185 : vect_print_slp_graph (MSG_NOTE, vect_location,
6183 : SLP_INSTANCE_TREE (instance), visited);
6184 20040 : }
6185 : }
6186 :
6187 1115103 : release_scalar_stmts_to_slp_tree_map (bst_map);
6188 :
6189 1115103 : if (pattern_found && dump_enabled_p ())
6190 : {
6191 18 : dump_printf_loc (MSG_NOTE, vect_location,
6192 : "Pattern matched SLP tree\n");
6193 18 : hash_set<slp_tree> visited;
6194 90 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6195 36 : vect_print_slp_graph (MSG_NOTE, vect_location,
6196 : SLP_INSTANCE_TREE (instance), visited);
6197 18 : }
6198 :
6199 1115103 : return opt_result::success ();
6200 1115103 : }
6201 :
6202 : /* Estimates the cost of inserting layout changes into the SLP graph.
6203 : It can also say that the insertion is impossible. */
6204 :
6205 : struct slpg_layout_cost
6206 : {
6207 10733369 : slpg_layout_cost () = default;
6208 : slpg_layout_cost (sreal, bool);
6209 :
6210 471016 : static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
6211 5050125 : bool is_possible () const { return depth != sreal::max (); }
6212 :
6213 : bool operator== (const slpg_layout_cost &) const;
6214 : bool operator!= (const slpg_layout_cost &) const;
6215 :
6216 : bool is_better_than (const slpg_layout_cost &, bool) const;
6217 :
6218 : void add_parallel_cost (const slpg_layout_cost &);
6219 : void add_serial_cost (const slpg_layout_cost &);
6220 : void split (unsigned int);
6221 :
6222 : /* The longest sequence of layout changes needed during any traversal
6223 : of the partition dag, weighted by execution frequency.
6224 :
6225 : This is the most important metric when optimizing for speed, since
6226 : it helps to ensure that we keep the number of operations on
6227 : critical paths to a minimum. */
6228 : sreal depth = 0;
6229 :
6230 : /* An estimate of the total number of operations needed. It is weighted by
6231 : execution frequency when optimizing for speed but not when optimizing for
6232 : size. In order to avoid double-counting, a node with a fanout of N will
6233 : distribute 1/N of its total cost to each successor.
6234 :
6235 : This is the most important metric when optimizing for size, since
6236 : it helps to keep the total number of operations to a minimum, */
6237 : sreal total = 0;
6238 : };
6239 :
6240 : /* Construct costs for a node with weight WEIGHT. A higher weight
6241 : indicates more frequent execution. IS_FOR_SIZE is true if we are
6242 : optimizing for size rather than speed. */
6243 :
6244 1191260 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
6245 1192222 : : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
6246 : {
6247 1191260 : }
6248 :
6249 : bool
6250 0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
6251 : {
6252 0 : return depth == other.depth && total == other.total;
6253 : }
6254 :
6255 : bool
6256 0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
6257 : {
6258 0 : return !operator== (other);
6259 : }
6260 :
6261 : /* Return true if these costs are better than OTHER. IS_FOR_SIZE is
6262 : true if we are optimizing for size rather than speed. */
6263 :
6264 : bool
6265 287988 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
6266 : bool is_for_size) const
6267 : {
6268 287988 : if (is_for_size)
6269 : {
6270 429 : if (total != other.total)
6271 181 : return total < other.total;
6272 248 : return depth < other.depth;
6273 : }
6274 : else
6275 : {
6276 287559 : if (depth != other.depth)
6277 119630 : return depth < other.depth;
6278 167929 : return total < other.total;
6279 : }
6280 : }
6281 :
6282 : /* Increase the costs to account for something with cost INPUT_COST
6283 : happening in parallel with the current costs. */
6284 :
6285 : void
6286 346543 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
6287 : {
6288 346543 : depth = std::max (depth, input_cost.depth);
6289 346543 : total += input_cost.total;
6290 346543 : }
6291 :
6292 : /* Increase the costs to account for something with cost INPUT_COST
6293 : happening in series with the current costs. */
6294 :
6295 : void
6296 1420411 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
6297 : {
6298 1420411 : depth += other.depth;
6299 1420411 : total += other.total;
6300 1420411 : }
6301 :
6302 : /* Split the total cost among TIMES successors or predecessors. */
6303 :
6304 : void
6305 1156242 : slpg_layout_cost::split (unsigned int times)
6306 : {
6307 1156242 : if (times > 1)
6308 529889 : total /= times;
6309 1156242 : }
6310 :
6311 : /* Information about one node in the SLP graph, for use during
6312 : vect_optimize_slp_pass. */
6313 :
6314 : struct slpg_vertex
6315 : {
6316 10080725 : slpg_vertex (slp_tree node_) : node (node_) {}
6317 :
6318 : /* The node itself. */
6319 : slp_tree node;
6320 :
6321 : /* Which partition the node belongs to, or -1 if none. Nodes outside of
6322 : partitions are flexible; they can have whichever layout consumers
6323 : want them to have. */
6324 : int partition = -1;
6325 :
6326 : /* The number of nodes that directly use the result of this one
6327 : (i.e. the number of nodes that count this one as a child). */
6328 : unsigned int out_degree = 0;
6329 :
6330 : /* The execution frequency of the node. */
6331 : sreal weight = 0;
6332 :
6333 : /* The total execution frequency of all nodes that directly use the
6334 : result of this one. */
6335 : sreal out_weight = 0;
6336 : };
6337 :
6338 : /* Information about one partition of the SLP graph, for use during
6339 : vect_optimize_slp_pass. */
6340 :
6341 : struct slpg_partition_info
6342 : {
6343 : /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
6344 : of m_partitioned_nodes. */
6345 : unsigned int node_begin = 0;
6346 : unsigned int node_end = 0;
6347 :
6348 : /* Which layout we've chosen to use for this partition, or -1 if
6349 : we haven't picked one yet. */
6350 : int layout = -1;
6351 :
6352 : /* The number of predecessors and successors in the partition dag.
6353 : The predecessors always have lower partition numbers and the
6354 : successors always have higher partition numbers.
6355 :
6356 : Note that the directions of these edges are not necessarily the
6357 : same as in the data flow graph. For example, if an SCC has separate
6358 : partitions for an inner loop and an outer loop, the inner loop's
6359 : partition will have at least two incoming edges from the outer loop's
6360 : partition: one for a live-in value and one for a live-out value.
6361 : In data flow terms, one of these edges would also be from the outer loop
6362 : to the inner loop, but the other would be in the opposite direction. */
6363 : unsigned int in_degree = 0;
6364 : unsigned int out_degree = 0;
6365 : };
6366 :
6367 : /* Information about the costs of using a particular layout for a
6368 : particular partition. It can also say that the combination is
6369 : impossible. */
6370 :
6371 : struct slpg_partition_layout_costs
6372 : {
6373 1469578 : bool is_possible () const { return internal_cost.is_possible (); }
6374 57514 : void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
6375 :
6376 : /* The costs inherited from predecessor partitions. */
6377 : slpg_layout_cost in_cost;
6378 :
6379 : /* The inherent cost of the layout within the node itself. For example,
6380 : this is nonzero for a load if choosing a particular layout would require
6381 : the load to permute the loaded elements. It is nonzero for a
6382 : VEC_PERM_EXPR if the permutation cannot be eliminated or converted
6383 : to full-vector moves. */
6384 : slpg_layout_cost internal_cost;
6385 :
6386 : /* The costs inherited from successor partitions. */
6387 : slpg_layout_cost out_cost;
6388 : };
6389 :
6390 : /* This class tries to optimize the layout of vectors in order to avoid
6391 : unnecessary shuffling. At the moment, the set of possible layouts are
6392 : restricted to bijective permutations.
6393 :
6394 : The goal of the pass depends on whether we're optimizing for size or
6395 : for speed. When optimizing for size, the goal is to reduce the overall
6396 : number of layout changes (including layout changes implied by things
6397 : like load permutations). When optimizing for speed, the goal is to
6398 : reduce the maximum latency attributable to layout changes on any
6399 : non-cyclical path through the data flow graph.
6400 :
6401 : For example, when optimizing a loop nest for speed, we will prefer
6402 : to make layout changes outside of a loop rather than inside of a loop,
6403 : and will prefer to make layout changes in parallel rather than serially,
6404 : even if that increases the overall number of layout changes.
6405 :
6406 : The high-level procedure is:
6407 :
6408 : (1) Build a graph in which edges go from uses (parents) to definitions
6409 : (children).
6410 :
6411 : (2) Divide the graph into a dag of strongly-connected components (SCCs).
6412 :
6413 : (3) When optimizing for speed, partition the nodes in each SCC based
6414 : on their containing cfg loop. When optimizing for size, treat
6415 : each SCC as a single partition.
6416 :
6417 : This gives us a dag of partitions. The goal is now to assign a
6418 : layout to each partition.
6419 :
6420 : (4) Construct a set of vector layouts that are worth considering.
6421 : Record which nodes must keep their current layout.
6422 :
6423 : (5) Perform a forward walk over the partition dag (from loads to stores)
6424 : accumulating the "forward" cost of using each layout. When visiting
6425 : each partition, assign a tentative choice of layout to the partition
6426 : and use that choice when calculating the cost of using a different
6427 : layout in successor partitions.
6428 :
6429 : (6) Perform a backward walk over the partition dag (from stores to loads),
6430 : accumulating the "backward" cost of using each layout. When visiting
6431 : each partition, make a final choice of layout for that partition based
6432 : on the accumulated forward costs (from (5)) and backward costs
6433 : (from (6)).
6434 :
6435 : (7) Apply the chosen layouts to the SLP graph.
6436 :
6437 : For example, consider the SLP statements:
6438 :
6439 : S1: a_1 = load
6440 : loop:
6441 : S2: a_2 = PHI<a_1, a_3>
6442 : S3: b_1 = load
6443 : S4: a_3 = a_2 + b_1
6444 : exit:
6445 : S5: a_4 = PHI<a_3>
6446 : S6: store a_4
6447 :
6448 : S2 and S4 form an SCC and are part of the same loop. Every other
6449 : statement is in a singleton SCC. In this example there is a one-to-one
6450 : mapping between SCCs and partitions and the partition dag looks like this;
6451 :
6452 : S1 S3
6453 : \ /
6454 : S2+S4
6455 : |
6456 : S5
6457 : |
6458 : S6
6459 :
6460 : S2, S3 and S4 will have a higher execution frequency than the other
6461 : statements, so when optimizing for speed, the goal is to avoid any
6462 : layout changes:
6463 :
6464 : - within S3
6465 : - within S2+S4
6466 : - on the S3->S2+S4 edge
6467 :
6468 : For example, if S3 was originally a reversing load, the goal of the
6469 : pass is to make it an unreversed load and change the layout on the
6470 : S1->S2+S4 and S2+S4->S5 edges to compensate. (Changing the layout
6471 : on S1->S2+S4 and S5->S6 would also be acceptable.)
6472 :
6473 : The difference between SCCs and partitions becomes important if we
6474 : add an outer loop:
6475 :
6476 : S1: a_1 = ...
6477 : loop1:
6478 : S2: a_2 = PHI<a_1, a_6>
6479 : S3: b_1 = load
6480 : S4: a_3 = a_2 + b_1
6481 : loop2:
6482 : S5: a_4 = PHI<a_3, a_5>
6483 : S6: c_1 = load
6484 : S7: a_5 = a_4 + c_1
6485 : exit2:
6486 : S8: a_6 = PHI<a_5>
6487 : S9: store a_6
6488 : exit1:
6489 :
6490 : Here, S2, S4, S5, S7 and S8 form a single SCC. However, when optimizing
6491 : for speed, we usually do not want restrictions in the outer loop to "infect"
6492 : the decision for the inner loop. For example, if an outer-loop node
6493 : in the SCC contains a statement with a fixed layout, that should not
6494 : prevent the inner loop from using a different layout. Conversely,
6495 : the inner loop should not dictate a layout to the outer loop: if the
6496 : outer loop does a lot of computation, then it may not be efficient to
6497 : do all of that computation in the inner loop's preferred layout.
6498 :
6499 : So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
6500 : and S5+S7 (inner). We also try to arrange partitions so that:
6501 :
6502 : - the partition for an outer loop comes before the partition for
6503 : an inner loop
6504 :
6505 : - if a sibling loop A dominates a sibling loop B, A's partition
6506 : comes before B's
6507 :
6508 : This gives the following partition dag for the example above:
6509 :
6510 : S1 S3
6511 : \ /
6512 : S2+S4+S8 S6
6513 : | \\ /
6514 : | S5+S7
6515 : |
6516 : S9
6517 :
6518 : There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
6519 : one for a reversal of the edge S7->S8.
6520 :
6521 : The backward walk picks a layout for S5+S7 before S2+S4+S8. The choice
6522 : for S2+S4+S8 therefore has to balance the cost of using the outer loop's
6523 : preferred layout against the cost of changing the layout on entry to the
6524 : inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
6525 :
6526 : Although this works well when optimizing for speed, it has the downside
6527 : when optimizing for size that the choice of layout for S5+S7 is completely
6528 : independent of S9, which lessens the chance of reducing the overall number
6529 : of permutations. We therefore do not partition SCCs when optimizing
6530 : for size.
6531 :
6532 : To give a concrete example of the difference between optimizing
6533 : for size and speed, consider:
6534 :
6535 : a[0] = (b[1] << c[3]) - d[1];
6536 : a[1] = (b[0] << c[2]) - d[0];
6537 : a[2] = (b[3] << c[1]) - d[3];
6538 : a[3] = (b[2] << c[0]) - d[2];
6539 :
6540 : There are three different layouts here: one for a, one for b and d,
6541 : and one for c. When optimizing for speed it is better to permute each
6542 : of b, c and d into the order required by a, since those permutations
6543 : happen in parallel. But when optimizing for size, it is better to:
6544 :
6545 : - permute c into the same order as b
6546 : - do the arithmetic
6547 : - permute the result into the order required by a
6548 :
6549 : This gives 2 permutations rather than 3. */
6550 :
6551 : class vect_optimize_slp_pass
6552 : {
6553 : public:
6554 697455 : vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
6555 : void run ();
6556 :
6557 : private:
6558 : /* Graph building. */
6559 : struct loop *containing_loop (slp_tree);
6560 : bool is_cfg_latch_edge (graph_edge *);
6561 : void build_vertices (hash_set<slp_tree> &, slp_tree);
6562 : void build_vertices ();
6563 : void build_graph ();
6564 :
6565 : /* Partitioning. */
6566 : void create_partitions ();
6567 : template<typename T> void for_each_partition_edge (unsigned int, T);
6568 :
6569 : /* Layout selection. */
6570 : bool is_compatible_layout (slp_tree, unsigned int);
6571 : bool is_compatible_layout (const slpg_partition_info &, unsigned int);
6572 : int change_layout_cost (slp_tree, unsigned int, unsigned int);
6573 : slpg_partition_layout_costs &partition_layout_costs (unsigned int,
6574 : unsigned int);
6575 : void change_vec_perm_layout (slp_tree, lane_permutation_t &,
6576 : int, unsigned int);
6577 : int internal_node_cost (slp_tree, int, unsigned int);
6578 : void start_choosing_layouts ();
6579 : bool legitimize ();
6580 :
6581 : /* Cost propagation. */
6582 : slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
6583 : unsigned int, unsigned int);
6584 : slpg_layout_cost total_in_cost (unsigned int);
6585 : slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
6586 : slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
6587 : void forward_pass ();
6588 : void backward_pass ();
6589 :
6590 : /* Rematerialization. */
6591 : slp_tree get_result_with_layout (slp_tree, unsigned int);
6592 : void materialize ();
6593 :
6594 : /* Clean-up. */
6595 : void remove_redundant_permutations ();
6596 :
6597 : /* Masked load lanes discovery. */
6598 : void decide_masked_load_lanes ();
6599 :
6600 : void dump ();
6601 :
6602 : vec_info *m_vinfo;
6603 :
6604 : /* True if we should optimize the graph for size, false if we should
6605 : optimize it for speed. (It wouldn't be easy to make this decision
6606 : more locally.) */
6607 : bool m_optimize_size;
6608 :
6609 : /* A graph of all SLP nodes, with edges leading from uses to definitions.
6610 : In other words, a node's predecessors are its slp_tree parents and
6611 : a node's successors are its slp_tree children. */
6612 : graph *m_slpg = nullptr;
6613 :
6614 : /* The vertices of M_SLPG, indexed by slp_tree::vertex. */
6615 : auto_vec<slpg_vertex> m_vertices;
6616 :
6617 : /* The list of all leaves of M_SLPG. such as external definitions, constants,
6618 : and loads. */
6619 : auto_vec<int> m_leafs;
6620 :
6621 : /* This array has one entry for every vector layout that we're considering.
6622 : Element 0 is null and indicates "no change". Other entries describe
6623 : permutations that are inherent in the current graph and that we would
6624 : like to reverse if possible.
6625 :
6626 : For example, a permutation { 1, 2, 3, 0 } means that something has
6627 : effectively been permuted in that way, such as a load group
6628 : { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
6629 : We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
6630 : in order to put things "back" in order. */
6631 : auto_vec<vec<unsigned> > m_perms;
6632 :
6633 : /* A partitioning of the nodes for which a layout must be chosen.
6634 : Each partition represents an <SCC, cfg loop> pair; that is,
6635 : nodes in different SCCs belong to different partitions, and nodes
6636 : within an SCC can be further partitioned according to a containing
6637 : cfg loop. Partition <SCC1, L1> comes before <SCC2, L2> if:
6638 :
6639 : - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
6640 : from leaves (such as loads) to roots (such as stores).
6641 :
6642 : - SCC1 == SCC2 and L1's header strictly dominates L2's header. */
6643 : auto_vec<slpg_partition_info> m_partitions;
6644 :
6645 : /* The list of all nodes for which a layout must be chosen. Nodes for
6646 : partition P come before the nodes for partition P+1. Nodes within a
6647 : partition are in reverse postorder. */
6648 : auto_vec<unsigned int> m_partitioned_nodes;
6649 :
6650 : /* Index P * num-layouts + L contains the cost of using layout L
6651 : for partition P. */
6652 : auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
6653 :
6654 : /* Index N * num-layouts + L, if nonnull, is a node that provides the
6655 : original output of node N adjusted to have layout L. */
6656 : auto_vec<slp_tree> m_node_layouts;
6657 : };
6658 :
6659 : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
6660 : Also record whether we should optimize anything for speed rather
6661 : than size. */
6662 :
6663 : void
6664 10887145 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
6665 : slp_tree node)
6666 : {
6667 10887145 : unsigned i;
6668 10887145 : slp_tree child;
6669 :
6670 10887145 : if (visited.add (node))
6671 10887145 : return;
6672 :
6673 10080725 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
6674 : {
6675 7937112 : basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
6676 7063722 : if (optimize_bb_for_speed_p (bb))
6677 6936070 : m_optimize_size = false;
6678 : }
6679 :
6680 10080725 : node->vertex = m_vertices.length ();
6681 10080725 : m_vertices.safe_push (slpg_vertex (node));
6682 :
6683 10080725 : bool leaf = true;
6684 10080725 : bool force_leaf = false;
6685 18849872 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
6686 8769147 : if (child)
6687 : {
6688 7893061 : leaf = false;
6689 7893061 : build_vertices (visited, child);
6690 : }
6691 : else
6692 : force_leaf = true;
6693 : /* Since SLP discovery works along use-def edges all cycles have an
6694 : entry - but there's the exception of cycles where we do not handle
6695 : the entry explicitly (but with a NULL SLP node), like some reductions
6696 : and inductions. Force those SLP PHIs to act as leafs to make them
6697 : backwards reachable. */
6698 10080725 : if (leaf || force_leaf)
6699 5000060 : m_leafs.safe_push (node->vertex);
6700 : }
6701 :
6702 : /* Fill the vertices and leafs vector with all nodes in the SLP graph. */
6703 :
6704 : void
6705 1394910 : vect_optimize_slp_pass::build_vertices ()
6706 : {
6707 1394910 : hash_set<slp_tree> visited;
6708 1394910 : unsigned i;
6709 1394910 : slp_instance instance;
6710 1394910 : m_vertices.truncate (0);
6711 1394910 : m_leafs.truncate (0);
6712 7178814 : FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
6713 2994084 : build_vertices (visited, SLP_INSTANCE_TREE (instance));
6714 1394910 : }
6715 :
6716 : /* Apply (reverse) bijectite PERM to VEC. */
6717 :
6718 : template <class T>
6719 : static void
6720 187716 : vect_slp_permute (vec<unsigned> perm,
6721 : vec<T> &vec, bool reverse)
6722 : {
6723 187716 : auto_vec<T, 64> saved;
6724 187716 : saved.create (vec.length ());
6725 615304 : for (unsigned i = 0; i < vec.length (); ++i)
6726 427588 : saved.quick_push (vec[i]);
6727 :
6728 187716 : if (reverse)
6729 : {
6730 1219937 : for (unsigned i = 0; i < vec.length (); ++i)
6731 426256 : vec[perm[i]] = saved[i];
6732 613374 : for (unsigned i = 0; i < vec.length (); ++i)
6733 743883 : gcc_assert (vec[perm[i]] == saved[i]);
6734 : }
6735 : else
6736 : {
6737 3860 : for (unsigned i = 0; i < vec.length (); ++i)
6738 1332 : vec[i] = saved[perm[i]];
6739 189048 : for (unsigned i = 0; i < vec.length (); ++i)
6740 1998 : gcc_assert (vec[i] == saved[perm[i]]);
6741 : }
6742 187716 : }
6743 :
6744 : /* Return the cfg loop that contains NODE. */
6745 :
6746 : struct loop *
6747 3933860 : vect_optimize_slp_pass::containing_loop (slp_tree node)
6748 : {
6749 3933860 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
6750 3933860 : if (!rep)
6751 5168 : return ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father;
6752 4378572 : return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
6753 : }
6754 :
6755 : /* Return true if UD (an edge from a use to a definition) is associated
6756 : with a loop latch edge in the cfg. */
6757 :
6758 : bool
6759 7893061 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
6760 : {
6761 7893061 : slp_tree use = m_vertices[ud->src].node;
6762 7893061 : slp_tree def = m_vertices[ud->dest].node;
6763 7893061 : if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
6764 7893061 : || SLP_TREE_PERMUTE_P (use))
6765 7580582 : || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
6766 : return false;
6767 :
6768 4576774 : stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
6769 4576774 : return (is_a<gphi *> (use_rep->stmt)
6770 376552 : && bb_loop_header_p (gimple_bb (use_rep->stmt))
6771 4788702 : && containing_loop (def) == containing_loop (use));
6772 : }
6773 :
6774 : /* Build the graph. Mark edges that correspond to cfg loop latch edges with
6775 : a nonnull data field. */
6776 :
6777 : void
6778 1394910 : vect_optimize_slp_pass::build_graph ()
6779 : {
6780 1394910 : m_optimize_size = true;
6781 1394910 : build_vertices ();
6782 :
6783 2789820 : m_slpg = new_graph (m_vertices.length ());
6784 14265455 : for (slpg_vertex &v : m_vertices)
6785 30068594 : for (slp_tree child : SLP_TREE_CHILDREN (v.node))
6786 8769147 : if (child)
6787 : {
6788 7893061 : graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
6789 7893061 : if (is_cfg_latch_edge (ud))
6790 203056 : ud->data = this;
6791 : }
6792 1394910 : }
6793 :
6794 : /* Return true if E corresponds to a loop latch edge in the cfg. */
6795 :
6796 : static bool
6797 4047772 : skip_cfg_latch_edges (graph_edge *e)
6798 : {
6799 4047772 : return e->data;
6800 : }
6801 :
6802 : /* Create the node partitions. */
6803 :
6804 : void
6805 697455 : vect_optimize_slp_pass::create_partitions ()
6806 : {
6807 : /* Calculate a postorder of the graph, ignoring edges that correspond
6808 : to natural latch edges in the cfg. Reading the vector from the end
6809 : to the beginning gives the reverse postorder. */
6810 697455 : auto_vec<int> initial_rpo;
6811 1394910 : graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
6812 : false, NULL, skip_cfg_latch_edges);
6813 2092365 : gcc_assert (initial_rpo.length () == m_vertices.length ());
6814 :
6815 : /* Calculate the strongly connected components of the graph. */
6816 697455 : auto_vec<int> scc_grouping;
6817 697455 : unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
6818 :
6819 : /* Create a new index order in which all nodes from the same SCC are
6820 : consecutive. Use scc_pos to record the index of the first node in
6821 : each SCC. */
6822 697455 : auto_vec<unsigned int> scc_pos (num_sccs);
6823 697455 : int last_component = -1;
6824 697455 : unsigned int node_count = 0;
6825 7132443 : for (unsigned int node_i : scc_grouping)
6826 : {
6827 5040078 : if (last_component != m_slpg->vertices[node_i].component)
6828 : {
6829 4912712 : last_component = m_slpg->vertices[node_i].component;
6830 9825424 : gcc_assert (last_component == int (scc_pos.length ()));
6831 4912712 : scc_pos.quick_push (node_count);
6832 : }
6833 5040078 : node_count += 1;
6834 : }
6835 1394910 : gcc_assert (node_count == initial_rpo.length ()
6836 : && last_component + 1 == int (num_sccs));
6837 :
6838 : /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
6839 : inside each SCC following the RPO we calculated above. The fact that
6840 : we ignored natural latch edges when calculating the RPO should ensure
6841 : that, for natural loop nests:
6842 :
6843 : - the first node that we encounter in a cfg loop is the loop header phi
6844 : - the loop header phis are in dominance order
6845 :
6846 : Arranging for this is an optimization (see below) rather than a
6847 : correctness issue. Unnatural loops with a tangled mess of backedges
6848 : will still work correctly, but might give poorer results.
6849 :
6850 : Also update scc_pos so that it gives 1 + the index of the last node
6851 : in the SCC. */
6852 697455 : m_partitioned_nodes.safe_grow (node_count);
6853 6434988 : for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
6854 : {
6855 5040078 : unsigned int node_i = initial_rpo[old_i];
6856 5040078 : unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
6857 5040078 : m_partitioned_nodes[new_i] = node_i;
6858 : }
6859 :
6860 : /* When optimizing for speed, partition each SCC based on the containing
6861 : cfg loop. The order we constructed above should ensure that, for natural
6862 : cfg loops, we'll create sub-SCC partitions for outer loops before
6863 : the corresponding sub-SCC partitions for inner loops. Similarly,
6864 : when one sibling loop A dominates another sibling loop B, we should
6865 : create a sub-SCC partition for A before a sub-SCC partition for B.
6866 :
6867 : As above, nothing depends for correctness on whether this achieves
6868 : a natural nesting, but we should get better results when it does. */
6869 1394910 : m_partitions.reserve (m_vertices.length ());
6870 697455 : unsigned int next_partition_i = 0;
6871 697455 : hash_map<struct loop *, int> loop_partitions;
6872 697455 : unsigned int rpo_begin = 0;
6873 697455 : unsigned int num_partitioned_nodes = 0;
6874 7005077 : for (unsigned int rpo_end : scc_pos)
6875 : {
6876 4912712 : loop_partitions.empty ();
6877 : unsigned int partition_i = next_partition_i;
6878 9952790 : for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
6879 : {
6880 : /* Handle externals and constants optimistically throughout.
6881 : But treat existing vectors as fixed since we do not handle
6882 : permuting them. */
6883 5040078 : unsigned int node_i = m_partitioned_nodes[rpo_i];
6884 5040078 : auto &vertex = m_vertices[node_i];
6885 5040078 : if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
6886 511379 : && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
6887 5042148 : || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
6888 1503333 : vertex.partition = -1;
6889 : else
6890 : {
6891 3536745 : bool existed;
6892 3536745 : if (m_optimize_size)
6893 26741 : existed = next_partition_i > partition_i;
6894 : else
6895 : {
6896 3510004 : struct loop *loop = containing_loop (vertex.node);
6897 3510004 : auto &entry = loop_partitions.get_or_insert (loop, &existed);
6898 3510004 : if (!existed)
6899 3383629 : entry = next_partition_i;
6900 3510004 : partition_i = entry;
6901 : }
6902 3536745 : if (!existed)
6903 : {
6904 3410280 : m_partitions.quick_push (slpg_partition_info ());
6905 3410280 : next_partition_i += 1;
6906 : }
6907 3536745 : vertex.partition = partition_i;
6908 3536745 : num_partitioned_nodes += 1;
6909 3536745 : m_partitions[partition_i].node_end += 1;
6910 : }
6911 : }
6912 4912712 : rpo_begin = rpo_end;
6913 : }
6914 :
6915 : /* Assign ranges of consecutive node indices to each partition,
6916 : in partition order. Start with node_end being the same as
6917 : node_begin so that the next loop can use it as a counter. */
6918 697455 : unsigned int node_begin = 0;
6919 5502645 : for (auto &partition : m_partitions)
6920 : {
6921 3410280 : partition.node_begin = node_begin;
6922 3410280 : node_begin += partition.node_end;
6923 3410280 : partition.node_end = partition.node_begin;
6924 : }
6925 697455 : gcc_assert (node_begin == num_partitioned_nodes);
6926 :
6927 : /* Finally build the list of nodes in partition order. */
6928 697455 : m_partitioned_nodes.truncate (num_partitioned_nodes);
6929 5737533 : for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
6930 : {
6931 5040078 : int partition_i = m_vertices[node_i].partition;
6932 5040078 : if (partition_i >= 0)
6933 : {
6934 3536745 : unsigned int order_i = m_partitions[partition_i].node_end++;
6935 3536745 : m_partitioned_nodes[order_i] = node_i;
6936 : }
6937 : }
6938 697455 : }
6939 :
6940 : /* Look for edges from earlier partitions into node NODE_I and edges from
6941 : node NODE_I into later partitions. Call:
6942 :
6943 : FN (ud, other_node_i)
6944 :
6945 : for each such use-to-def edge ud, where other_node_i is the node at the
6946 : other end of the edge. */
6947 :
6948 : template<typename T>
6949 : void
6950 3955318 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
6951 : {
6952 3955318 : int partition_i = m_vertices[node_i].partition;
6953 3955318 : for (graph_edge *pred = m_slpg->vertices[node_i].pred;
6954 6820312 : pred; pred = pred->pred_next)
6955 : {
6956 2864994 : int src_partition_i = m_vertices[pred->src].partition;
6957 2864994 : if (src_partition_i >= 0 && src_partition_i != partition_i)
6958 2540767 : fn (pred, pred->src);
6959 : }
6960 3955318 : for (graph_edge *succ = m_slpg->vertices[node_i].succ;
6961 8474447 : succ; succ = succ->succ_next)
6962 : {
6963 4519129 : int dest_partition_i = m_vertices[succ->dest].partition;
6964 4519129 : if (dest_partition_i >= 0 && dest_partition_i != partition_i)
6965 2569663 : fn (succ, succ->dest);
6966 : }
6967 3955318 : }
6968 :
6969 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
6970 : that NODE would operate on. This test is independent of NODE's actual
6971 : operation. */
6972 :
6973 : bool
6974 1605258 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
6975 : unsigned int layout_i)
6976 : {
6977 1605258 : if (layout_i == 0)
6978 : return true;
6979 :
6980 918558 : if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
6981 15826 : return false;
6982 :
6983 : return true;
6984 : }
6985 :
6986 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
6987 : that NODE would operate on for each NODE in PARTITION.
6988 : This test is independent of NODE's actual operations. */
6989 :
6990 : bool
6991 18979 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
6992 : &partition,
6993 : unsigned int layout_i)
6994 : {
6995 38389 : for (unsigned int order_i = partition.node_begin;
6996 38389 : order_i < partition.node_end; ++order_i)
6997 : {
6998 19502 : unsigned int node_i = m_partitioned_nodes[order_i];
6999 19502 : auto &vertex = m_vertices[node_i];
7000 :
7001 : /* The layout is incompatible if it is individually incompatible
7002 : with any node in the partition. */
7003 19502 : if (!is_compatible_layout (vertex.node, layout_i))
7004 : return false;
7005 : }
7006 : return true;
7007 : }
7008 :
7009 : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
7010 : to layout TO_LAYOUT_I for a node like NODE. Return -1 if either of the
7011 : layouts is incompatible with NODE or if the change is not possible for
7012 : some other reason.
7013 :
7014 : The properties taken from NODE include the number of lanes and the
7015 : vector type. The actual operation doesn't matter. */
7016 :
7017 : int
7018 679541 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
7019 : unsigned int from_layout_i,
7020 : unsigned int to_layout_i)
7021 : {
7022 679541 : if (!is_compatible_layout (node, from_layout_i)
7023 679541 : || !is_compatible_layout (node, to_layout_i))
7024 577 : return -1;
7025 :
7026 678964 : if (from_layout_i == to_layout_i)
7027 : return 0;
7028 :
7029 279310 : auto_vec<slp_tree, 1> children (1);
7030 279310 : children.quick_push (node);
7031 279310 : auto_lane_permutation_t perm (SLP_TREE_LANES (node));
7032 279310 : if (from_layout_i > 0)
7033 797160 : for (unsigned int i : m_perms[from_layout_i])
7034 353199 : perm.quick_push ({ 0, i });
7035 : else
7036 427332 : for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
7037 296009 : perm.quick_push ({ 0, i });
7038 279310 : if (to_layout_i > 0)
7039 131864 : vect_slp_permute (m_perms[to_layout_i], perm, true);
7040 279310 : auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
7041 : children, false);
7042 279310 : if (count >= 0)
7043 274381 : return MAX (count, 1);
7044 :
7045 : /* ??? In principle we could try changing via layout 0, giving two
7046 : layout changes rather than 1. Doing that would require
7047 : corresponding support in get_result_with_layout. */
7048 : return -1;
7049 279310 : }
7050 :
7051 : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I. */
7052 :
7053 : inline slpg_partition_layout_costs &
7054 1002809 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
7055 : unsigned int layout_i)
7056 : {
7057 2005618 : return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
7058 : }
7059 :
7060 : /* Change PERM in one of two ways:
7061 :
7062 : - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
7063 : chosen for child I of NODE.
7064 :
7065 : - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
7066 :
7067 : In both cases, arrange for the output to have layout OUT_LAYOUT_I */
7068 :
7069 : void
7070 30651 : vect_optimize_slp_pass::
7071 : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
7072 : int in_layout_i, unsigned int out_layout_i)
7073 : {
7074 178071 : for (auto &entry : perm)
7075 : {
7076 86118 : int this_in_layout_i = in_layout_i;
7077 86118 : if (this_in_layout_i < 0)
7078 : {
7079 59823 : slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
7080 59823 : unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
7081 59823 : if (in_partition_i == -1u)
7082 329 : continue;
7083 59494 : this_in_layout_i = m_partitions[in_partition_i].layout;
7084 : }
7085 85789 : if (this_in_layout_i > 0)
7086 19335 : entry.second = m_perms[this_in_layout_i][entry.second];
7087 : }
7088 30651 : if (out_layout_i > 0)
7089 7099 : vect_slp_permute (m_perms[out_layout_i], perm, true);
7090 30651 : }
7091 :
7092 : /* Check whether the target allows NODE to be rearranged so that the node's
7093 : output has layout OUT_LAYOUT_I. Return the cost of the change if so,
7094 : in the same arbitrary units as for change_layout_cost. Return -1 otherwise.
7095 :
7096 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
7097 : NODE can adapt to the layout changes that have (perhaps provisionally)
7098 : been chosen for NODE's children, so that no extra permutations are
7099 : needed on either the input or the output of NODE.
7100 :
7101 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
7102 : that all inputs will be forced into layout IN_LAYOUT_I beforehand.
7103 :
7104 : IN_LAYOUT_I has no meaning for other types of node.
7105 :
7106 : Keeping the node as-is is always valid. If the target doesn't appear
7107 : to support the node as-is, but might realistically support other layouts,
7108 : then layout 0 instead has the cost of a worst-case permutation. On the
7109 : one hand, this ensures that every node has at least one valid layout,
7110 : avoiding what would otherwise be an awkward special case. On the other,
7111 : it still encourages the pass to change an invalid pre-existing layout
7112 : choice into a valid one. */
7113 :
7114 : int
7115 221955 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
7116 : unsigned int out_layout_i)
7117 : {
7118 221955 : const int fallback_cost = 1;
7119 :
7120 221955 : if (SLP_TREE_PERMUTE_P (node))
7121 : {
7122 25494 : auto_lane_permutation_t tmp_perm;
7123 25494 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7124 :
7125 : /* Check that the child nodes support the chosen layout. Checking
7126 : the first child is enough, since any second child would have the
7127 : same shape. */
7128 25494 : auto first_child = SLP_TREE_CHILDREN (node)[0];
7129 25494 : if (in_layout_i > 0
7130 25494 : && !is_compatible_layout (first_child, in_layout_i))
7131 : return -1;
7132 :
7133 24922 : change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
7134 49844 : int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
7135 : node, tmp_perm,
7136 24922 : SLP_TREE_CHILDREN (node),
7137 : false);
7138 24922 : if (count < 0)
7139 : {
7140 1546 : if (in_layout_i == 0 && out_layout_i == 0)
7141 : {
7142 : /* Use the fallback cost if the node could in principle support
7143 : some nonzero layout for both the inputs and the outputs.
7144 : Otherwise assume that the node will be rejected later
7145 : and rebuilt from scalars. */
7146 379 : if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
7147 : return fallback_cost;
7148 309 : return 0;
7149 : }
7150 : return -1;
7151 : }
7152 :
7153 : /* We currently have no way of telling whether the new layout is cheaper
7154 : or more expensive than the old one. But at least in principle,
7155 : it should be worth making zero permutations (whole-vector shuffles)
7156 : cheaper than real permutations, in case the pass is able to remove
7157 : the latter. */
7158 23376 : return count == 0 ? 0 : 1;
7159 25494 : }
7160 :
7161 196461 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
7162 196461 : if (rep
7163 195600 : && STMT_VINFO_DATA_REF (rep)
7164 64321 : && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
7165 239857 : && SLP_TREE_LOAD_PERMUTATION (node).exists ())
7166 : {
7167 35664 : auto_load_permutation_t tmp_perm;
7168 35664 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7169 35664 : if (out_layout_i > 0)
7170 13543 : vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
7171 :
7172 35664 : poly_uint64 vf = 1;
7173 35664 : if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
7174 12152 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7175 35664 : unsigned int n_perms;
7176 35664 : if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
7177 : nullptr, vf, true, false, &n_perms))
7178 : {
7179 1698 : auto rep = SLP_TREE_REPRESENTATIVE (node);
7180 1698 : if (out_layout_i == 0)
7181 : {
7182 : /* Use the fallback cost if the load is an N-to-N permutation.
7183 : Otherwise assume that the node will be rejected later
7184 : and rebuilt from scalars. */
7185 1259 : if (STMT_VINFO_GROUPED_ACCESS (rep)
7186 2518 : && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
7187 1259 : == SLP_TREE_LANES (node)))
7188 654 : return fallback_cost;
7189 : return 0;
7190 : }
7191 : return -1;
7192 : }
7193 :
7194 : /* See the comment above the corresponding VEC_PERM_EXPR handling. */
7195 33966 : return n_perms == 0 ? 0 : 1;
7196 35664 : }
7197 :
7198 : return 0;
7199 : }
7200 :
7201 : /* Decide which element layouts we should consider using. Calculate the
7202 : weights associated with inserting layout changes on partition edges.
7203 : Also mark partitions that cannot change layout, by setting their
7204 : layout to zero. */
7205 :
7206 : void
7207 697455 : vect_optimize_slp_pass::start_choosing_layouts ()
7208 : {
7209 : /* Used to assign unique permutation indices. */
7210 697455 : using perm_hash = unbounded_hashmap_traits<
7211 : vec_free_hash_base<int_hash_base<unsigned>>,
7212 : int_hash<int, -1, -2>
7213 : >;
7214 697455 : hash_map<vec<unsigned>, int, perm_hash> layout_ids;
7215 :
7216 : /* Layout 0 is "no change". */
7217 697455 : m_perms.safe_push (vNULL);
7218 :
7219 : /* Create layouts from existing permutations. */
7220 697455 : auto_load_permutation_t tmp_perm;
7221 5629110 : for (unsigned int node_i : m_partitioned_nodes)
7222 : {
7223 : /* Leafs also double as entries to the reverse graph. Allow the
7224 : layout of those to be changed. */
7225 3536745 : auto &vertex = m_vertices[node_i];
7226 3536745 : auto &partition = m_partitions[vertex.partition];
7227 3536745 : if (!m_slpg->vertices[node_i].succ)
7228 906044 : partition.layout = 0;
7229 :
7230 : /* Loads and VEC_PERM_EXPRs are the only things generating permutes. */
7231 3536745 : slp_tree node = vertex.node;
7232 3536745 : stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
7233 3536745 : slp_tree child;
7234 3536745 : unsigned HOST_WIDE_INT imin, imax = 0;
7235 3536745 : bool any_permute = false;
7236 3536745 : tmp_perm.truncate (0);
7237 3536745 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
7238 : {
7239 : /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
7240 : unpermuted, record a layout that reverses this permutation.
7241 :
7242 : We would need more work to cope with loads that are internally
7243 : permuted and also have inputs (such as masks for
7244 : IFN_MASK_LOADs). */
7245 611338 : gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
7246 611338 : if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
7247 : {
7248 436382 : partition.layout = -1;
7249 3519388 : continue;
7250 : }
7251 174956 : dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
7252 174956 : imin = DR_GROUP_SIZE (dr_stmt) + 1;
7253 174956 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7254 : }
7255 5731987 : else if (SLP_TREE_PERMUTE_P (node)
7256 137390 : && SLP_TREE_CHILDREN (node).length () == 1
7257 118827 : && (child = SLP_TREE_CHILDREN (node)[0])
7258 3044234 : && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
7259 118827 : .is_constant (&imin)))
7260 : {
7261 : /* If the child has the same vector size as this node,
7262 : reversing the permutation can make the permutation a no-op.
7263 : In other cases it can change a true permutation into a
7264 : full-vector extract. */
7265 118827 : tmp_perm.reserve (SLP_TREE_LANES (node));
7266 319562 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7267 200735 : tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
7268 : }
7269 : else
7270 2806580 : continue;
7271 :
7272 777016 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7273 : {
7274 483233 : unsigned idx = tmp_perm[j];
7275 483233 : imin = MIN (imin, idx);
7276 483233 : imax = MAX (imax, idx);
7277 483233 : if (idx - tmp_perm[0] != j)
7278 142334 : any_permute = true;
7279 : }
7280 : /* If the span doesn't match we'd disrupt VF computation, avoid
7281 : that for now. */
7282 293783 : if (imax - imin + 1 != SLP_TREE_LANES (node))
7283 84202 : continue;
7284 : /* If there's no permute no need to split one out. In this case
7285 : we can consider turning a load into a permuted load, if that
7286 : turns out to be cheaper than alternatives. */
7287 209581 : if (!any_permute)
7288 : {
7289 192073 : partition.layout = -1;
7290 192073 : continue;
7291 : }
7292 :
7293 : /* For now only handle true permutes, like
7294 : vect_attempt_slp_rearrange_stmts did. This allows us to be lazy
7295 : when permuting constants and invariants keeping the permute
7296 : bijective. */
7297 17508 : auto_sbitmap load_index (SLP_TREE_LANES (node));
7298 17508 : bitmap_clear (load_index);
7299 67192 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7300 49684 : bitmap_set_bit (load_index, tmp_perm[j] - imin);
7301 : unsigned j;
7302 66357 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
7303 49000 : if (!bitmap_bit_p (load_index, j))
7304 : break;
7305 17508 : if (j != SLP_TREE_LANES (node))
7306 151 : continue;
7307 :
7308 17357 : vec<unsigned> perm = vNULL;
7309 17357 : perm.safe_grow (SLP_TREE_LANES (node), true);
7310 66082 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7311 48725 : perm[j] = tmp_perm[j] - imin;
7312 :
7313 34714 : if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
7314 : {
7315 : /* Continue to use existing layouts, but don't add any more. */
7316 0 : int *entry = layout_ids.get (perm);
7317 0 : partition.layout = entry ? *entry : 0;
7318 0 : perm.release ();
7319 : }
7320 : else
7321 : {
7322 17357 : bool existed;
7323 17357 : int &layout_i = layout_ids.get_or_insert (perm, &existed);
7324 17357 : if (existed)
7325 6280 : perm.release ();
7326 : else
7327 : {
7328 11077 : layout_i = m_perms.length ();
7329 11077 : m_perms.safe_push (perm);
7330 : }
7331 17357 : partition.layout = layout_i;
7332 : }
7333 17508 : }
7334 :
7335 : /* Initially assume that every layout is possible and has zero cost
7336 : in every partition. */
7337 697455 : m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
7338 1394910 : * m_perms.length ());
7339 :
7340 : /* We have to mark outgoing permutations facing non-associating-reduction
7341 : graph entries that are not represented as to be materialized.
7342 : slp_inst_kind_bb_reduc currently only covers associatable reductions. */
7343 3589407 : for (slp_instance instance : m_vinfo->slp_instances)
7344 1497042 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
7345 : {
7346 6918 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7347 6918 : m_partitions[m_vertices[node_i].partition].layout = 0;
7348 : }
7349 1490124 : else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
7350 : {
7351 2300 : stmt_vec_info stmt_info
7352 2300 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
7353 2300 : vect_reduc_info reduc_info
7354 2300 : = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
7355 : SLP_INSTANCE_TREE (instance));
7356 2300 : if (needs_fold_left_reduction_p (TREE_TYPE
7357 : (gimple_get_lhs (stmt_info->stmt)),
7358 : VECT_REDUC_INFO_CODE (reduc_info)))
7359 : {
7360 97 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7361 97 : m_partitions[m_vertices[node_i].partition].layout = 0;
7362 : }
7363 : }
7364 :
7365 : /* Check which layouts each node and partition can handle. Calculate the
7366 : weights associated with inserting layout changes on edges. */
7367 5629110 : for (unsigned int node_i : m_partitioned_nodes)
7368 : {
7369 3536745 : auto &vertex = m_vertices[node_i];
7370 3536745 : auto &partition = m_partitions[vertex.partition];
7371 3536745 : slp_tree node = vertex.node;
7372 :
7373 3536745 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
7374 : {
7375 3531577 : vertex.weight = vect_slp_node_weight (node);
7376 :
7377 : /* We do not handle stores with a permutation, so all
7378 : incoming permutations must have been materialized.
7379 :
7380 : We also don't handle masked grouped loads, which lack a
7381 : permutation vector. In this case the memory locations
7382 : form an implicit second input to the loads, on top of the
7383 : explicit mask input, and the memory input's layout cannot
7384 : be changed.
7385 :
7386 : On the other hand, we do support permuting gather loads and
7387 : masked gather loads, where each scalar load is independent
7388 : of the others. This can be useful if the address/index input
7389 : benefits from permutation. */
7390 3531577 : if (STMT_VINFO_DATA_REF (rep)
7391 1789408 : && STMT_VINFO_GROUPED_ACCESS (rep)
7392 4638712 : && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
7393 932179 : partition.layout = 0;
7394 :
7395 : /* We cannot change the layout of an operation that is
7396 : not independent on lanes. Note this is an explicit
7397 : negative list since that's much shorter than the respective
7398 : positive one but it's critical to keep maintaining it. */
7399 3531577 : if (is_gimple_call (STMT_VINFO_STMT (rep)))
7400 32204 : switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
7401 : {
7402 1091 : case CFN_COMPLEX_ADD_ROT90:
7403 1091 : case CFN_COMPLEX_ADD_ROT270:
7404 1091 : case CFN_COMPLEX_MUL:
7405 1091 : case CFN_COMPLEX_MUL_CONJ:
7406 1091 : case CFN_VEC_ADDSUB:
7407 1091 : case CFN_VEC_FMADDSUB:
7408 1091 : case CFN_VEC_FMSUBADD:
7409 1091 : partition.layout = 0;
7410 : default:;
7411 : }
7412 : }
7413 :
7414 7939857 : auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
7415 : {
7416 4403112 : auto &other_vertex = m_vertices[other_node_i];
7417 :
7418 : /* Count the number of edges from earlier partitions and the number
7419 : of edges to later partitions. */
7420 4403112 : if (other_vertex.partition < vertex.partition)
7421 2201556 : partition.in_degree += 1;
7422 : else
7423 2201556 : partition.out_degree += 1;
7424 :
7425 : /* If the current node uses the result of OTHER_NODE_I, accumulate
7426 : the effects of that. */
7427 4403112 : if (ud->src == int (node_i))
7428 : {
7429 2201556 : other_vertex.out_weight += vertex.weight;
7430 2201556 : other_vertex.out_degree += 1;
7431 : }
7432 7939857 : };
7433 3536745 : for_each_partition_edge (node_i, process_edge);
7434 : }
7435 697455 : }
7436 :
7437 : /* Return the incoming costs for node NODE_I, assuming that each input keeps
7438 : its current (provisional) choice of layout. The inputs do not necessarily
7439 : have the same layout as each other. */
7440 :
7441 : slpg_layout_cost
7442 3143 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
7443 : {
7444 3143 : auto &vertex = m_vertices[node_i];
7445 3143 : slpg_layout_cost cost;
7446 11488 : auto add_cost = [&](graph_edge *, unsigned int other_node_i)
7447 : {
7448 8345 : auto &other_vertex = m_vertices[other_node_i];
7449 8345 : if (other_vertex.partition < vertex.partition)
7450 : {
7451 5348 : auto &other_partition = m_partitions[other_vertex.partition];
7452 10696 : auto &other_costs = partition_layout_costs (other_vertex.partition,
7453 5348 : other_partition.layout);
7454 5348 : slpg_layout_cost this_cost = other_costs.in_cost;
7455 5348 : this_cost.add_serial_cost (other_costs.internal_cost);
7456 5348 : this_cost.split (other_partition.out_degree);
7457 5348 : cost.add_parallel_cost (this_cost);
7458 : }
7459 11488 : };
7460 3143 : for_each_partition_edge (node_i, add_cost);
7461 3143 : return cost;
7462 : }
7463 :
7464 : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
7465 : and layout LAYOUT2_I on cross-partition use-to-def edge UD. Return
7466 : slpg_layout_cost::impossible () if the change isn't possible. */
7467 :
7468 : slpg_layout_cost
7469 679541 : vect_optimize_slp_pass::
7470 : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
7471 : unsigned int layout2_i)
7472 : {
7473 679541 : auto &def_vertex = m_vertices[ud->dest];
7474 679541 : auto &use_vertex = m_vertices[ud->src];
7475 679541 : auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
7476 679541 : auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
7477 679541 : auto factor = change_layout_cost (def_vertex.node, def_layout_i,
7478 : use_layout_i);
7479 679541 : if (factor < 0)
7480 5506 : return slpg_layout_cost::impossible ();
7481 :
7482 : /* We have a choice of putting the layout change at the site of the
7483 : definition or at the site of the use. Prefer the former when
7484 : optimizing for size or when the execution frequency of the
7485 : definition is no greater than the combined execution frequencies of
7486 : the uses. When putting the layout change at the site of the definition,
7487 : divvy up the cost among all consumers. */
7488 674035 : if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
7489 : {
7490 652653 : slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
7491 652653 : cost.split (def_vertex.out_degree);
7492 652653 : return cost;
7493 : }
7494 21382 : return { use_vertex.weight * factor, m_optimize_size };
7495 : }
7496 :
7497 : /* UD represents a use-def link between FROM_NODE_I and a node in a later
7498 : partition; FROM_NODE_I could be the definition node or the use node.
7499 : The node at the other end of the link wants to use layout TO_LAYOUT_I.
7500 : Return the cost of any necessary fix-ups on edge UD, or return
7501 : slpg_layout_cost::impossible () if the change isn't possible.
7502 :
7503 : At this point, FROM_NODE_I's partition has chosen the cheapest
7504 : layout based on the information available so far, but this choice
7505 : is only provisional. */
7506 :
7507 : slpg_layout_cost
7508 180284 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
7509 : unsigned int to_layout_i)
7510 : {
7511 180284 : auto &from_vertex = m_vertices[from_node_i];
7512 180284 : unsigned int from_partition_i = from_vertex.partition;
7513 180284 : slpg_partition_info &from_partition = m_partitions[from_partition_i];
7514 180284 : gcc_assert (from_partition.layout >= 0);
7515 :
7516 : /* First calculate the cost on the assumption that FROM_PARTITION sticks
7517 : with its current layout preference. */
7518 180284 : slpg_layout_cost cost = slpg_layout_cost::impossible ();
7519 180284 : auto edge_cost = edge_layout_cost (ud, from_node_i,
7520 180284 : from_partition.layout, to_layout_i);
7521 180284 : if (edge_cost.is_possible ())
7522 : {
7523 354822 : auto &from_costs = partition_layout_costs (from_partition_i,
7524 177411 : from_partition.layout);
7525 177411 : cost = from_costs.in_cost;
7526 177411 : cost.add_serial_cost (from_costs.internal_cost);
7527 177411 : cost.split (from_partition.out_degree);
7528 177411 : cost.add_serial_cost (edge_cost);
7529 : }
7530 2873 : else if (from_partition.layout == 0)
7531 : /* We must allow the source partition to have layout 0 as a fallback,
7532 : in case all other options turn out to be impossible. */
7533 2873 : return cost;
7534 :
7535 : /* Take the minimum of that cost and the cost that applies if
7536 : FROM_PARTITION instead switches to TO_LAYOUT_I. */
7537 177411 : auto &direct_layout_costs = partition_layout_costs (from_partition_i,
7538 : to_layout_i);
7539 177411 : if (direct_layout_costs.is_possible ())
7540 : {
7541 157046 : slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
7542 157046 : direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
7543 157046 : direct_cost.split (from_partition.out_degree);
7544 157046 : if (!cost.is_possible ()
7545 157046 : || direct_cost.is_better_than (cost, m_optimize_size))
7546 34807 : cost = direct_cost;
7547 : }
7548 :
7549 177411 : return cost;
7550 : }
7551 :
7552 : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
7553 : partition; TO_NODE_I could be the definition node or the use node.
7554 : The node at the other end of the link wants to use layout FROM_LAYOUT_I;
7555 : return the cost of any necessary fix-ups on edge UD, or
7556 : slpg_layout_cost::impossible () if the choice cannot be made.
7557 :
7558 : At this point, TO_NODE_I's partition has a fixed choice of layout. */
7559 :
7560 : slpg_layout_cost
7561 163784 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
7562 : unsigned int from_layout_i)
7563 : {
7564 163784 : auto &to_vertex = m_vertices[to_node_i];
7565 163784 : unsigned int to_partition_i = to_vertex.partition;
7566 163784 : slpg_partition_info &to_partition = m_partitions[to_partition_i];
7567 163784 : gcc_assert (to_partition.layout >= 0);
7568 :
7569 : /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
7570 : adjusted for this input having layout FROM_LAYOUT_I. Assume that
7571 : any other inputs keep their current choice of layout. */
7572 163784 : auto &to_costs = partition_layout_costs (to_partition_i,
7573 : to_partition.layout);
7574 163784 : if (ud->src == int (to_node_i)
7575 163582 : && SLP_TREE_PERMUTE_P (to_vertex.node))
7576 : {
7577 9564 : auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
7578 9564 : auto old_layout = from_partition.layout;
7579 9564 : from_partition.layout = from_layout_i;
7580 19128 : int factor = internal_node_cost (to_vertex.node, -1,
7581 9564 : to_partition.layout);
7582 9564 : from_partition.layout = old_layout;
7583 9564 : if (factor >= 0)
7584 : {
7585 8922 : slpg_layout_cost cost = to_costs.out_cost;
7586 17844 : cost.add_serial_cost ({ to_vertex.weight * factor,
7587 8922 : m_optimize_size });
7588 8922 : cost.split (to_partition.in_degree);
7589 8922 : return cost;
7590 : }
7591 : }
7592 :
7593 : /* Compute the cost if we insert any necessary layout change on edge UD. */
7594 154862 : auto edge_cost = edge_layout_cost (ud, to_node_i,
7595 154862 : to_partition.layout, from_layout_i);
7596 154862 : if (edge_cost.is_possible ())
7597 : {
7598 154862 : slpg_layout_cost cost = to_costs.out_cost;
7599 154862 : cost.add_serial_cost (to_costs.internal_cost);
7600 154862 : cost.split (to_partition.in_degree);
7601 154862 : cost.add_serial_cost (edge_cost);
7602 154862 : return cost;
7603 : }
7604 :
7605 0 : return slpg_layout_cost::impossible ();
7606 : }
7607 :
7608 : /* Make a forward pass through the partitions, accumulating input costs.
7609 : Make a tentative (provisional) choice of layout for each partition,
7610 : ensuring that this choice still allows later partitions to keep
7611 : their original layout. */
7612 :
7613 : void
7614 5631 : vect_optimize_slp_pass::forward_pass ()
7615 : {
7616 119487 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
7617 : ++partition_i)
7618 : {
7619 113856 : auto &partition = m_partitions[partition_i];
7620 :
7621 : /* If the partition consists of a single VEC_PERM_EXPR, precompute
7622 : the incoming cost that would apply if every predecessor partition
7623 : keeps its current layout. This is used within the loop below. */
7624 113856 : slpg_layout_cost in_cost;
7625 113856 : slp_tree single_node = nullptr;
7626 113856 : if (partition.node_end == partition.node_begin + 1)
7627 : {
7628 107516 : unsigned int node_i = m_partitioned_nodes[partition.node_begin];
7629 107516 : single_node = m_vertices[node_i].node;
7630 107516 : if (SLP_TREE_PERMUTE_P (single_node))
7631 3143 : in_cost = total_in_cost (node_i);
7632 : }
7633 :
7634 : /* Go through the possible layouts. Decide which ones are valid
7635 : for this partition and record which of the valid layouts has
7636 : the lowest cost. */
7637 113856 : unsigned int min_layout_i = 0;
7638 113856 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7639 348204 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7640 : {
7641 234348 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7642 234348 : if (!layout_costs.is_possible ())
7643 57514 : continue;
7644 :
7645 : /* If the recorded layout is already 0 then the layout cannot
7646 : change. */
7647 234348 : if (partition.layout == 0 && layout_i != 0)
7648 : {
7649 39280 : layout_costs.mark_impossible ();
7650 39280 : continue;
7651 : }
7652 :
7653 195068 : bool is_possible = true;
7654 400937 : for (unsigned int order_i = partition.node_begin;
7655 400937 : order_i < partition.node_end; ++order_i)
7656 : {
7657 221535 : unsigned int node_i = m_partitioned_nodes[order_i];
7658 221535 : auto &vertex = m_vertices[node_i];
7659 :
7660 : /* Reject the layout if it is individually incompatible
7661 : with any node in the partition. */
7662 221535 : if (!is_compatible_layout (vertex.node, layout_i))
7663 : {
7664 14585 : is_possible = false;
7665 15666 : break;
7666 : }
7667 :
7668 554409 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7669 : {
7670 347459 : auto &other_vertex = m_vertices[other_node_i];
7671 347459 : if (other_vertex.partition < vertex.partition)
7672 : {
7673 : /* Accumulate the incoming costs from earlier
7674 : partitions, plus the cost of any layout changes
7675 : on UD itself. */
7676 180284 : auto cost = forward_cost (ud, other_node_i, layout_i);
7677 180284 : if (!cost.is_possible ())
7678 2873 : is_possible = false;
7679 : else
7680 177411 : layout_costs.in_cost.add_parallel_cost (cost);
7681 : }
7682 : else
7683 : /* Reject the layout if it would make layout 0 impossible
7684 : for later partitions. This amounts to testing that the
7685 : target supports reversing the layout change on edges
7686 : to later partitions.
7687 :
7688 : In principle, it might be possible to push a layout
7689 : change all the way down a graph, so that it never
7690 : needs to be reversed and so that the target doesn't
7691 : need to support the reverse operation. But it would
7692 : be awkward to bail out if we hit a partition that
7693 : does not support the new layout, especially since
7694 : we are not dealing with a lattice. */
7695 167175 : is_possible &= edge_layout_cost (ud, other_node_i, 0,
7696 167175 : layout_i).is_possible ();
7697 554409 : };
7698 206950 : for_each_partition_edge (node_i, add_cost);
7699 :
7700 : /* Accumulate the cost of using LAYOUT_I within NODE,
7701 : both for the inputs and the outputs. */
7702 206950 : int factor = internal_node_cost (vertex.node, layout_i,
7703 : layout_i);
7704 206950 : if (factor < 0)
7705 : {
7706 1081 : is_possible = false;
7707 1081 : break;
7708 : }
7709 205869 : else if (factor)
7710 32301 : layout_costs.internal_cost.add_serial_cost
7711 32301 : ({ vertex.weight * factor, m_optimize_size });
7712 : }
7713 195068 : if (!is_possible)
7714 : {
7715 18234 : layout_costs.mark_impossible ();
7716 18234 : continue;
7717 : }
7718 :
7719 : /* Combine the incoming and partition-internal costs. */
7720 176834 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7721 176834 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7722 :
7723 : /* If this partition consists of a single VEC_PERM_EXPR, see
7724 : if the VEC_PERM_EXPR can be changed to support output layout
7725 : LAYOUT_I while keeping all the provisional choices of input
7726 : layout. */
7727 176834 : if (single_node && SLP_TREE_PERMUTE_P (single_node))
7728 : {
7729 5441 : int factor = internal_node_cost (single_node, -1, layout_i);
7730 5441 : if (factor >= 0)
7731 : {
7732 4986 : auto weight = m_vertices[single_node->vertex].weight;
7733 4986 : slpg_layout_cost internal_cost
7734 4986 : = { weight * factor, m_optimize_size };
7735 :
7736 4986 : slpg_layout_cost alt_cost = in_cost;
7737 4986 : alt_cost.add_serial_cost (internal_cost);
7738 4986 : if (alt_cost.is_better_than (combined_cost, m_optimize_size))
7739 : {
7740 1563 : combined_cost = alt_cost;
7741 1563 : layout_costs.in_cost = in_cost;
7742 1563 : layout_costs.internal_cost = internal_cost;
7743 : }
7744 : }
7745 : }
7746 :
7747 : /* Record the layout with the lowest cost. Prefer layout 0 in
7748 : the event of a tie between it and another layout. */
7749 176834 : if (!min_layout_cost.is_possible ()
7750 62978 : || combined_cost.is_better_than (min_layout_cost,
7751 62978 : m_optimize_size))
7752 : {
7753 128124 : min_layout_i = layout_i;
7754 128124 : min_layout_cost = combined_cost;
7755 : }
7756 : }
7757 :
7758 : /* This loop's handling of earlier partitions should ensure that
7759 : choosing the original layout for the current partition is no
7760 : less valid than it was in the original graph, even with the
7761 : provisional layout choices for those earlier partitions. */
7762 113856 : gcc_assert (min_layout_cost.is_possible ());
7763 113856 : partition.layout = min_layout_i;
7764 : }
7765 5631 : }
7766 :
7767 : /* Make a backward pass through the partitions, accumulating output costs.
7768 : Make a final choice of layout for each partition. */
7769 :
7770 : void
7771 5631 : vect_optimize_slp_pass::backward_pass ()
7772 : {
7773 125118 : for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
7774 : {
7775 113856 : auto &partition = m_partitions[partition_i];
7776 :
7777 113856 : unsigned int min_layout_i = 0;
7778 113856 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7779 348204 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7780 : {
7781 234348 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7782 234348 : if (!layout_costs.is_possible ())
7783 57514 : continue;
7784 :
7785 : /* Accumulate the costs from successor partitions. */
7786 176834 : bool is_possible = true;
7787 380098 : for (unsigned int order_i = partition.node_begin;
7788 380098 : order_i < partition.node_end; ++order_i)
7789 : {
7790 203264 : unsigned int node_i = m_partitioned_nodes[order_i];
7791 203264 : auto &vertex = m_vertices[node_i];
7792 544268 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7793 : {
7794 341004 : auto &other_vertex = m_vertices[other_node_i];
7795 341004 : auto &other_partition = m_partitions[other_vertex.partition];
7796 341004 : if (other_vertex.partition > vertex.partition)
7797 : {
7798 : /* Accumulate the incoming costs from later
7799 : partitions, plus the cost of any layout changes
7800 : on UD itself. */
7801 163784 : auto cost = backward_cost (ud, other_node_i, layout_i);
7802 163784 : if (!cost.is_possible ())
7803 0 : is_possible = false;
7804 : else
7805 163784 : layout_costs.out_cost.add_parallel_cost (cost);
7806 : }
7807 : else
7808 : /* Make sure that earlier partitions can (if necessary
7809 : or beneficial) keep the layout that they chose in
7810 : the forward pass. This ensures that there is at
7811 : least one valid choice of layout. */
7812 177220 : is_possible &= edge_layout_cost (ud, other_node_i,
7813 177220 : other_partition.layout,
7814 177220 : layout_i).is_possible ();
7815 544268 : };
7816 203264 : for_each_partition_edge (node_i, add_cost);
7817 : }
7818 176834 : if (!is_possible)
7819 : {
7820 0 : layout_costs.mark_impossible ();
7821 0 : continue;
7822 : }
7823 :
7824 : /* Locally combine the costs from the forward and backward passes.
7825 : (This combined cost is not passed on, since that would lead
7826 : to double counting.) */
7827 176834 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7828 176834 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7829 176834 : combined_cost.add_serial_cost (layout_costs.out_cost);
7830 :
7831 : /* Record the layout with the lowest cost. Prefer layout 0 in
7832 : the event of a tie between it and another layout. */
7833 176834 : if (!min_layout_cost.is_possible ()
7834 62978 : || combined_cost.is_better_than (min_layout_cost,
7835 62978 : m_optimize_size))
7836 : {
7837 121258 : min_layout_i = layout_i;
7838 121258 : min_layout_cost = combined_cost;
7839 : }
7840 : }
7841 :
7842 113856 : gcc_assert (min_layout_cost.is_possible ());
7843 113856 : partition.layout = min_layout_i;
7844 : }
7845 5631 : }
7846 :
7847 : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
7848 : NODE already has the layout that was selected for its partition. */
7849 :
7850 : slp_tree
7851 158912 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
7852 : unsigned int to_layout_i)
7853 : {
7854 158912 : unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
7855 158912 : slp_tree result = m_node_layouts[result_i];
7856 158912 : if (result)
7857 : return result;
7858 :
7859 158369 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
7860 158369 : || (SLP_TREE_DEF_TYPE (node) == vect_external_def
7861 : /* We can't permute vector defs in place. */
7862 18546 : && SLP_TREE_VEC_DEFS (node).is_empty ()))
7863 : {
7864 : /* If the vector is uniform or unchanged, there's nothing to do. */
7865 38585 : if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
7866 : result = node;
7867 : else
7868 : {
7869 2266 : auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
7870 2266 : result = vect_create_new_slp_node (scalar_ops);
7871 2266 : vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
7872 : }
7873 : }
7874 : else
7875 : {
7876 119784 : unsigned int partition_i = m_vertices[node->vertex].partition;
7877 119784 : unsigned int from_layout_i = m_partitions[partition_i].layout;
7878 119784 : if (from_layout_i == to_layout_i)
7879 119213 : return node;
7880 :
7881 : /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
7882 : permutation instead of a serial one. Leave the new permutation
7883 : in TMP_PERM on success. */
7884 571 : auto_lane_permutation_t tmp_perm;
7885 571 : unsigned int num_inputs = 1;
7886 571 : if (SLP_TREE_PERMUTE_P (node))
7887 : {
7888 7 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7889 7 : if (from_layout_i != 0)
7890 7 : vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
7891 7 : if (to_layout_i != 0)
7892 4 : vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
7893 7 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7894 : tmp_perm,
7895 7 : SLP_TREE_CHILDREN (node),
7896 : false) >= 0)
7897 7 : num_inputs = SLP_TREE_CHILDREN (node).length ();
7898 : else
7899 0 : tmp_perm.truncate (0);
7900 : }
7901 :
7902 571 : if (dump_enabled_p ())
7903 : {
7904 70 : if (tmp_perm.length () > 0)
7905 6 : dump_printf_loc (MSG_NOTE, vect_location,
7906 : "duplicating permutation node %p with"
7907 : " layout %d\n",
7908 : (void *) node, to_layout_i);
7909 : else
7910 64 : dump_printf_loc (MSG_NOTE, vect_location,
7911 : "inserting permutation node in place of %p\n",
7912 : (void *) node);
7913 : }
7914 :
7915 571 : unsigned int num_lanes = SLP_TREE_LANES (node);
7916 571 : result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
7917 571 : if (SLP_TREE_SCALAR_STMTS (node).length ())
7918 : {
7919 570 : auto &stmts = SLP_TREE_SCALAR_STMTS (result);
7920 570 : stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
7921 570 : if (from_layout_i != 0)
7922 299 : vect_slp_permute (m_perms[from_layout_i], stmts, false);
7923 570 : if (to_layout_i != 0)
7924 275 : vect_slp_permute (m_perms[to_layout_i], stmts, true);
7925 : }
7926 571 : SLP_TREE_REPRESENTATIVE (result) = SLP_TREE_REPRESENTATIVE (node);
7927 571 : SLP_TREE_LANES (result) = num_lanes;
7928 571 : SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
7929 571 : result->vertex = -1;
7930 :
7931 571 : auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
7932 571 : if (tmp_perm.length ())
7933 : {
7934 7 : lane_perm.safe_splice (tmp_perm);
7935 7 : SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
7936 : }
7937 : else
7938 : {
7939 564 : lane_perm.create (num_lanes);
7940 1756 : for (unsigned j = 0; j < num_lanes; ++j)
7941 1192 : lane_perm.quick_push ({ 0, j });
7942 564 : if (from_layout_i != 0)
7943 292 : vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
7944 564 : if (to_layout_i != 0)
7945 272 : vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
7946 564 : SLP_TREE_CHILDREN (result).safe_push (node);
7947 : }
7948 2288 : for (slp_tree child : SLP_TREE_CHILDREN (result))
7949 575 : child->refcnt++;
7950 571 : }
7951 39156 : m_node_layouts[result_i] = result;
7952 39156 : return result;
7953 : }
7954 :
7955 : /* Apply the chosen vector layouts to the SLP graph. */
7956 :
7957 : void
7958 10655 : vect_optimize_slp_pass::materialize ()
7959 : {
7960 : /* We no longer need the costs, so avoid having two O(N * P) arrays
7961 : live at the same time. */
7962 10655 : m_partition_layout_costs.release ();
7963 31965 : m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
7964 :
7965 21310 : auto_sbitmap fully_folded (m_vertices.length ());
7966 10655 : bitmap_clear (fully_folded);
7967 169276 : for (unsigned int node_i : m_partitioned_nodes)
7968 : {
7969 137311 : auto &vertex = m_vertices[node_i];
7970 137311 : slp_tree node = vertex.node;
7971 137311 : int layout_i = m_partitions[vertex.partition].layout;
7972 137311 : gcc_assert (layout_i >= 0);
7973 :
7974 : /* Rearrange the scalar statements to match the chosen layout. */
7975 137311 : if (layout_i > 0)
7976 15947 : vect_slp_permute (m_perms[layout_i],
7977 15947 : SLP_TREE_SCALAR_STMTS (node), true);
7978 :
7979 : /* Update load and lane permutations. */
7980 137311 : if (SLP_TREE_PERMUTE_P (node))
7981 : {
7982 : /* First try to absorb the input vector layouts. If that fails,
7983 : force the inputs to have layout LAYOUT_I too. We checked that
7984 : that was possible before deciding to use nonzero output layouts.
7985 : (Note that at this stage we don't really have any guarantee that
7986 : the target supports the original VEC_PERM_EXPR.) */
7987 5350 : auto &perm = SLP_TREE_LANE_PERMUTATION (node);
7988 5350 : auto_lane_permutation_t tmp_perm;
7989 5350 : tmp_perm.safe_splice (perm);
7990 5350 : change_vec_perm_layout (node, tmp_perm, -1, layout_i);
7991 5350 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7992 : tmp_perm,
7993 5350 : SLP_TREE_CHILDREN (node),
7994 : false) >= 0)
7995 : {
7996 4971 : if (dump_enabled_p ()
7997 5891 : && !std::equal (tmp_perm.begin (), tmp_perm.end (),
7998 : perm.begin ()))
7999 58 : dump_printf_loc (MSG_NOTE, vect_location,
8000 : "absorbing input layouts into %p\n",
8001 : (void *) node);
8002 27994 : std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
8003 4971 : bitmap_set_bit (fully_folded, node_i);
8004 : }
8005 : else
8006 : {
8007 : /* Not MSG_MISSED because it would make no sense to users. */
8008 379 : if (dump_enabled_p ())
8009 46 : dump_printf_loc (MSG_NOTE, vect_location,
8010 : "failed to absorb input layouts into %p\n",
8011 : (void *) node);
8012 379 : change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
8013 : }
8014 5350 : }
8015 : else
8016 : {
8017 131961 : gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
8018 131961 : auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
8019 131961 : if (layout_i > 0)
8020 : /* ??? When we handle non-bijective permutes the idea
8021 : is that we can force the load-permutation to be
8022 : { min, min + 1, min + 2, ... max }. But then the
8023 : scalar defs might no longer match the lane content
8024 : which means wrong-code with live lane vectorization.
8025 : So we possibly have to have NULL entries for those. */
8026 15848 : vect_slp_permute (m_perms[layout_i], load_perm, true);
8027 : }
8028 : }
8029 :
8030 : /* Do this before any nodes disappear, since it involves a walk
8031 : over the leaves. */
8032 10655 : remove_redundant_permutations ();
8033 :
8034 : /* Replace each child with a correctly laid-out version. */
8035 169276 : for (unsigned int node_i : m_partitioned_nodes)
8036 : {
8037 : /* Skip nodes that have already been handled above. */
8038 137311 : if (bitmap_bit_p (fully_folded, node_i))
8039 4971 : continue;
8040 :
8041 132340 : auto &vertex = m_vertices[node_i];
8042 132340 : int in_layout_i = m_partitions[vertex.partition].layout;
8043 132340 : gcc_assert (in_layout_i >= 0);
8044 :
8045 : unsigned j;
8046 : slp_tree child;
8047 398356 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
8048 : {
8049 164928 : if (!child)
8050 6016 : continue;
8051 :
8052 158912 : slp_tree new_child = get_result_with_layout (child, in_layout_i);
8053 158912 : if (new_child != child)
8054 : {
8055 3080 : vect_free_slp_tree (child);
8056 3080 : SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
8057 3080 : new_child->refcnt += 1;
8058 : }
8059 : }
8060 : }
8061 10655 : }
8062 :
8063 : /* Elide load permutations that are not necessary. Such permutations might
8064 : be pre-existing, rather than created by the layout optimizations. */
8065 :
8066 : void
8067 697455 : vect_optimize_slp_pass::remove_redundant_permutations ()
8068 : {
8069 4592395 : for (unsigned int node_i : m_leafs)
8070 : {
8071 2500030 : slp_tree node = m_vertices[node_i].node;
8072 2500030 : if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
8073 1888692 : continue;
8074 :
8075 : /* In basic block vectorization we allow any subchain of an interleaving
8076 : chain.
8077 : FORNOW: not in loop SLP because of realignment complications. */
8078 611338 : if (is_a <bb_vec_info> (m_vinfo))
8079 : {
8080 162607 : bool subchain_p = true;
8081 : stmt_vec_info next_load_info = NULL;
8082 : stmt_vec_info load_info;
8083 : unsigned j;
8084 162607 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
8085 : {
8086 132439 : if (j != 0
8087 132439 : && (next_load_info != load_info
8088 62467 : || ! load_info
8089 62467 : || DR_GROUP_GAP (load_info) != 1))
8090 : {
8091 : subchain_p = false;
8092 : break;
8093 : }
8094 108846 : next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
8095 : }
8096 53761 : if (subchain_p)
8097 : {
8098 30168 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8099 30168 : continue;
8100 : }
8101 : }
8102 : else
8103 : {
8104 557577 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
8105 557577 : bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
8106 : /* When this isn't a grouped access we know it's single element
8107 : and contiguous. */
8108 557577 : if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
8109 : {
8110 436382 : if (!this_load_permuted
8111 436382 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8112 435627 : || SLP_TREE_LANES (node) == 1))
8113 435629 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8114 436382 : continue;
8115 : }
8116 121195 : stmt_vec_info first_stmt_info
8117 121195 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
8118 121700 : if (!this_load_permuted
8119 : /* The load requires permutation when unrolling exposes
8120 : a gap either because the group is larger than the SLP
8121 : group-size or because there is a gap between the groups. */
8122 121195 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8123 98753 : || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
8124 140 : && DR_GROUP_GAP (first_stmt_info) == 0)))
8125 : {
8126 505 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8127 505 : continue;
8128 : }
8129 : }
8130 : }
8131 697455 : }
8132 :
8133 : /* Print the partition graph and layout information to the dump file. */
8134 :
8135 : void
8136 679 : vect_optimize_slp_pass::dump ()
8137 : {
8138 679 : dump_printf_loc (MSG_NOTE, vect_location,
8139 : "SLP optimize permutations:\n");
8140 1371 : for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
8141 : {
8142 692 : dump_printf_loc (MSG_NOTE, vect_location, " %d: { ", layout_i);
8143 692 : const char *sep = "";
8144 5909 : for (unsigned int idx : m_perms[layout_i])
8145 : {
8146 3833 : dump_printf (MSG_NOTE, "%s%d", sep, idx);
8147 3833 : sep = ", ";
8148 : }
8149 692 : dump_printf (MSG_NOTE, " }\n");
8150 : }
8151 679 : dump_printf_loc (MSG_NOTE, vect_location,
8152 : "SLP optimize partitions:\n");
8153 5659 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
8154 : ++partition_i)
8155 : {
8156 4980 : auto &partition = m_partitions[partition_i];
8157 4980 : dump_printf_loc (MSG_NOTE, vect_location, " -------------\n");
8158 4980 : dump_printf_loc (MSG_NOTE, vect_location,
8159 : " partition %d (layout %d):\n",
8160 : partition_i, partition.layout);
8161 4980 : dump_printf_loc (MSG_NOTE, vect_location, " nodes:\n");
8162 10196 : for (unsigned int order_i = partition.node_begin;
8163 10196 : order_i < partition.node_end; ++order_i)
8164 : {
8165 5216 : auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
8166 10432 : dump_printf_loc (MSG_NOTE, vect_location, " - %p:\n",
8167 5216 : (void *) vertex.node);
8168 5216 : dump_printf_loc (MSG_NOTE, vect_location,
8169 : " weight: %f\n",
8170 : vertex.weight.to_double ());
8171 5216 : if (vertex.out_degree)
8172 4083 : dump_printf_loc (MSG_NOTE, vect_location,
8173 : " out weight: %f (degree %d)\n",
8174 : vertex.out_weight.to_double (),
8175 : vertex.out_degree);
8176 5216 : if (SLP_TREE_PERMUTE_P (vertex.node))
8177 506 : dump_printf_loc (MSG_NOTE, vect_location,
8178 : " op: VEC_PERM_EXPR\n");
8179 4710 : else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
8180 4692 : dump_printf_loc (MSG_NOTE, vect_location,
8181 : " op template: %G", rep->stmt);
8182 : }
8183 4980 : dump_printf_loc (MSG_NOTE, vect_location, " edges:\n");
8184 10196 : for (unsigned int order_i = partition.node_begin;
8185 10196 : order_i < partition.node_end; ++order_i)
8186 : {
8187 5216 : unsigned int node_i = m_partitioned_nodes[order_i];
8188 5216 : auto &vertex = m_vertices[node_i];
8189 15726 : auto print_edge = [&](graph_edge *, unsigned int other_node_i)
8190 : {
8191 10510 : auto &other_vertex = m_vertices[other_node_i];
8192 10510 : if (other_vertex.partition < vertex.partition)
8193 5255 : dump_printf_loc (MSG_NOTE, vect_location,
8194 : " - %p [%d] --> %p\n",
8195 5255 : (void *) other_vertex.node,
8196 : other_vertex.partition,
8197 5255 : (void *) vertex.node);
8198 : else
8199 5255 : dump_printf_loc (MSG_NOTE, vect_location,
8200 : " - %p --> [%d] %p\n",
8201 5255 : (void *) vertex.node,
8202 : other_vertex.partition,
8203 5255 : (void *) other_vertex.node);
8204 15726 : };
8205 5216 : for_each_partition_edge (node_i, print_edge);
8206 : }
8207 :
8208 15139 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
8209 : {
8210 10159 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
8211 10159 : if (layout_costs.is_possible ())
8212 : {
8213 8380 : dump_printf_loc (MSG_NOTE, vect_location,
8214 : " layout %d:%s\n", layout_i,
8215 8380 : partition.layout == int (layout_i)
8216 : ? " (*)" : "");
8217 8380 : slpg_layout_cost combined_cost = layout_costs.in_cost;
8218 8380 : combined_cost.add_serial_cost (layout_costs.internal_cost);
8219 8380 : combined_cost.add_serial_cost (layout_costs.out_cost);
8220 : #define TEMPLATE "{depth: %f, total: %f}"
8221 8380 : dump_printf_loc (MSG_NOTE, vect_location,
8222 : " " TEMPLATE "\n",
8223 : layout_costs.in_cost.depth.to_double (),
8224 : layout_costs.in_cost.total.to_double ());
8225 8380 : dump_printf_loc (MSG_NOTE, vect_location,
8226 : " + " TEMPLATE "\n",
8227 : layout_costs.internal_cost.depth.to_double (),
8228 : layout_costs.internal_cost.total.to_double ());
8229 8380 : dump_printf_loc (MSG_NOTE, vect_location,
8230 : " + " TEMPLATE "\n",
8231 : layout_costs.out_cost.depth.to_double (),
8232 : layout_costs.out_cost.total.to_double ());
8233 8380 : dump_printf_loc (MSG_NOTE, vect_location,
8234 : " = " TEMPLATE "\n",
8235 : combined_cost.depth.to_double (),
8236 : combined_cost.total.to_double ());
8237 : #undef TEMPLATE
8238 : }
8239 : else
8240 1779 : dump_printf_loc (MSG_NOTE, vect_location,
8241 : " layout %d: rejected\n", layout_i);
8242 : }
8243 : }
8244 679 : }
8245 :
8246 : /* Masked load lanes discovery. */
8247 :
8248 : void
8249 697455 : vect_optimize_slp_pass::decide_masked_load_lanes ()
8250 : {
8251 7133012 : for (auto v : m_vertices)
8252 : {
8253 5040647 : slp_tree node = v.node;
8254 5040647 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8255 3535239 : || SLP_TREE_PERMUTE_P (node))
8256 1643367 : continue;
8257 3397280 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
8258 1669362 : if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
8259 : /* The mask has to be uniform. */
8260 987686 : || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
8261 987608 : || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
8262 3397365 : || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
8263 : IFN_MASK_LOAD))
8264 3397247 : continue;
8265 33 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8266 66 : if (STMT_VINFO_STRIDED_P (stmt_info)
8267 33 : || compare_step_with_zero (m_vinfo, stmt_info) <= 0
8268 63 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
8269 30 : DR_GROUP_SIZE (stmt_info),
8270 : true) == IFN_LAST)
8271 33 : continue;
8272 :
8273 : /* Uniform masks need to be suitably represented. */
8274 0 : slp_tree mask = SLP_TREE_CHILDREN (node)[0];
8275 0 : if (!SLP_TREE_PERMUTE_P (mask)
8276 0 : || SLP_TREE_CHILDREN (mask).length () != 1)
8277 0 : continue;
8278 0 : bool match = true;
8279 0 : for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
8280 0 : if (perm.first != 0 || perm.second != 0)
8281 : {
8282 : match = false;
8283 : break;
8284 : }
8285 0 : if (!match)
8286 0 : continue;
8287 :
8288 : /* Now see if the consumer side matches. */
8289 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8290 0 : pred; pred = pred->pred_next)
8291 : {
8292 0 : slp_tree pred_node = m_vertices[pred->src].node;
8293 : /* All consumers should be a permute with a single outgoing lane. */
8294 0 : if (!SLP_TREE_PERMUTE_P (pred_node)
8295 0 : || SLP_TREE_LANES (pred_node) != 1)
8296 : {
8297 : match = false;
8298 : break;
8299 : }
8300 0 : gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
8301 : }
8302 0 : if (!match)
8303 0 : continue;
8304 : /* Now we can mark the nodes as to use load lanes. */
8305 0 : node->ldst_lanes = true;
8306 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8307 0 : pred; pred = pred->pred_next)
8308 0 : m_vertices[pred->src].node->ldst_lanes = true;
8309 : /* The catch is we have to massage the mask. We have arranged
8310 : analyzed uniform masks to be represented by a splat VEC_PERM
8311 : which we can now simply elide as we cannot easily re-do SLP
8312 : discovery here. */
8313 0 : slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
8314 0 : SLP_TREE_REF_COUNT (new_mask)++;
8315 0 : SLP_TREE_CHILDREN (node)[0] = new_mask;
8316 0 : vect_free_slp_tree (mask);
8317 : }
8318 697455 : }
8319 :
8320 : /* Perform legitimizing attempts. This is intended to improve the
8321 : situation when layout 0 is not valid which is a situation the cost
8322 : based propagation does not handle well.
8323 : Return true if further layout optimization is possible, false if
8324 : the layout configuration should be considered final. */
8325 :
8326 : bool
8327 10655 : vect_optimize_slp_pass::legitimize ()
8328 : {
8329 : /* Perform a very simple legitimizing attempt by attempting to choose
8330 : a single layout for all partitions that will make all permutations
8331 : a noop. That should also be the optimal layout choice in case
8332 : layout zero is legitimate.
8333 : ??? Disconnected components of the SLP graph could have distinct
8334 : single layouts. */
8335 10655 : int single_layout_i = -1;
8336 10655 : unsigned deferred_up_to = -1U;
8337 33380 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8338 : ++partition_i)
8339 : {
8340 28347 : auto &partition = m_partitions[partition_i];
8341 28347 : if (single_layout_i == -1)
8342 : {
8343 14571 : single_layout_i = partition.layout;
8344 14571 : deferred_up_to = partition_i;
8345 : }
8346 13776 : else if (partition.layout == single_layout_i || partition.layout == -1)
8347 : ;
8348 : else
8349 : single_layout_i = 0;
8350 25244 : if (single_layout_i == 0)
8351 : return true;
8352 :
8353 22808 : if (single_layout_i != -1
8354 22808 : && !is_compatible_layout (partition, single_layout_i))
8355 : return true;
8356 : }
8357 :
8358 5033 : if (single_layout_i <= 0)
8359 : return true;
8360 :
8361 5111 : for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
8362 87 : if (!is_compatible_layout (m_partitions[partition_i],
8363 : single_layout_i))
8364 : return true;
8365 :
8366 13042 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8367 : ++partition_i)
8368 : {
8369 8018 : auto &partition = m_partitions[partition_i];
8370 8018 : partition.layout = single_layout_i;
8371 : }
8372 :
8373 : return false;
8374 : }
8375 :
8376 : /* Main entry point for the SLP graph optimization pass. */
8377 :
8378 : void
8379 697455 : vect_optimize_slp_pass::run ()
8380 : {
8381 697455 : build_graph ();
8382 697455 : create_partitions ();
8383 697455 : start_choosing_layouts ();
8384 697455 : if (m_perms.length () > 1)
8385 : {
8386 10655 : if (legitimize ())
8387 : {
8388 5631 : forward_pass ();
8389 5631 : backward_pass ();
8390 : }
8391 10655 : if (dump_enabled_p ())
8392 679 : dump ();
8393 10655 : materialize ();
8394 43042 : while (!m_perms.is_empty ())
8395 21732 : m_perms.pop ().release ();
8396 : }
8397 : else
8398 686800 : remove_redundant_permutations ();
8399 697455 : free_graph (m_slpg);
8400 697455 : build_graph ();
8401 697455 : decide_masked_load_lanes ();
8402 697455 : free_graph (m_slpg);
8403 697455 : }
8404 :
8405 : /* Apply CSE to NODE and its children using BST_MAP. */
8406 :
8407 : static void
8408 5439948 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
8409 : {
8410 5439948 : bool put_p = false;
8411 5439948 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
8412 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8413 : lack scalar stmts and thus CSE doesn't work via bst_map. Ideally
8414 : we'd have sth that works for all internal and external nodes. */
8415 5439948 : && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
8416 : {
8417 3908663 : slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
8418 3908663 : if (leader)
8419 : {
8420 : /* We've visited this node already. */
8421 402225 : if (!*leader || *leader == node)
8422 : return;
8423 :
8424 3127 : if (dump_enabled_p ())
8425 912 : dump_printf_loc (MSG_NOTE, vect_location,
8426 : "re-using SLP tree %p for %p\n",
8427 : (void *)*leader, (void *)node);
8428 3127 : vect_free_slp_tree (node);
8429 3127 : (*leader)->refcnt += 1;
8430 3127 : node = *leader;
8431 3127 : return;
8432 : }
8433 :
8434 : /* Avoid creating a cycle by populating the map only after recursion. */
8435 3506438 : bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
8436 3506438 : node->refcnt += 1;
8437 3506438 : put_p = true;
8438 : /* And recurse. */
8439 : }
8440 :
8441 15020996 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8442 4380943 : if (child)
8443 3942906 : vect_cse_slp_nodes (bst_map, child);
8444 :
8445 : /* Now record the node for CSE in other siblings. */
8446 5037723 : if (put_p)
8447 3506438 : *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
8448 : }
8449 :
8450 : /* Optimize the SLP graph of VINFO. */
8451 :
8452 : void
8453 1053074 : vect_optimize_slp (vec_info *vinfo)
8454 : {
8455 1053074 : if (vinfo->slp_instances.is_empty ())
8456 : return;
8457 697455 : vect_optimize_slp_pass (vinfo).run ();
8458 :
8459 : /* Apply CSE again to nodes after permute optimization. */
8460 697455 : scalar_stmts_to_slp_tree_map_t *bst_map
8461 697455 : = new scalar_stmts_to_slp_tree_map_t ();
8462 :
8463 3589407 : for (auto inst : vinfo->slp_instances)
8464 1497042 : vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
8465 :
8466 697455 : release_scalar_stmts_to_slp_tree_map (bst_map);
8467 : }
8468 :
8469 : /* Gather loads reachable from the individual SLP graph entries. */
8470 :
8471 : void
8472 1053074 : vect_gather_slp_loads (vec_info *vinfo)
8473 : {
8474 1053074 : unsigned i;
8475 1053074 : slp_instance instance;
8476 2550116 : FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
8477 : {
8478 1497042 : hash_set<slp_tree> visited;
8479 1497042 : vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
8480 : SLP_INSTANCE_TREE (instance), visited);
8481 1497042 : }
8482 1053074 : }
8483 :
8484 : /* For NODE update VF based on the number of lanes and the vector types
8485 : used. */
8486 :
8487 : static void
8488 4285081 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
8489 : hash_set<slp_tree> &visited)
8490 : {
8491 4285081 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
8492 1538943 : return;
8493 3110796 : if (visited.add (node))
8494 : return;
8495 :
8496 10395618 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8497 3509070 : vect_update_slp_vf_for_node (child, vf, visited);
8498 :
8499 : /* We do not visit SLP nodes for constants or externals - those neither
8500 : have a vector type set yet (vectorizable_* does this) nor do they
8501 : have max_nunits set. Instead we rely on internal nodes max_nunit
8502 : to cover constant/external operands.
8503 : Note that when we stop using fixed size vectors externs and constants
8504 : shouldn't influence the (minimum) vectorization factor, instead
8505 : vectorizable_* should honor the vectorization factor when trying to
8506 : assign vector types to constants and externals and cause iteration
8507 : to a higher vectorization factor when required. */
8508 2746138 : poly_uint64 node_vf
8509 2746138 : = calculate_unrolling_factor (node->max_nunits, SLP_TREE_LANES (node));
8510 2746138 : vf = force_common_multiple (vf, node_vf);
8511 :
8512 : /* For permute nodes that are fed from externs or constants we have to
8513 : consider their number of lanes as well. Likewise for store-lanes. */
8514 2746138 : if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
8515 711200 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8516 191402 : if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
8517 : {
8518 3585 : poly_uint64 child_vf
8519 3585 : = calculate_unrolling_factor (node->max_nunits,
8520 : SLP_TREE_LANES (child));
8521 3585 : vf = force_common_multiple (vf, child_vf);
8522 : }
8523 : }
8524 :
8525 : /* For each possible SLP instance decide whether to SLP it and calculate overall
8526 : unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at
8527 : least one instance. */
8528 :
8529 : bool
8530 485653 : vect_make_slp_decision (loop_vec_info loop_vinfo)
8531 : {
8532 485653 : unsigned int i;
8533 485653 : poly_uint64 unrolling_factor = 1;
8534 485653 : const vec<slp_instance> &slp_instances
8535 : = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
8536 485653 : slp_instance instance;
8537 485653 : int decided_to_slp = 0;
8538 :
8539 485653 : DUMP_VECT_SCOPE ("vect_make_slp_decision");
8540 :
8541 485653 : hash_set<slp_tree> visited;
8542 1261664 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
8543 : {
8544 776011 : slp_tree root = SLP_INSTANCE_TREE (instance);
8545 :
8546 : /* All unroll factors have the form:
8547 :
8548 : GET_MODE_SIZE (vinfo->vector_mode) * X
8549 :
8550 : for some rational X, so they must have a common multiple. */
8551 776011 : vect_update_slp_vf_for_node (root, unrolling_factor, visited);
8552 :
8553 : /* If all instances ended up with vector(1) T roots make sure to
8554 : not vectorize. RVV for example relies on loop vectorization
8555 : when some instances are essentially kept scalar. See PR121048. */
8556 776011 : if (SLP_TREE_VECTYPE (root)
8557 776011 : && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
8558 635153 : decided_to_slp++;
8559 : }
8560 :
8561 485653 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
8562 :
8563 485653 : if (decided_to_slp && dump_enabled_p ())
8564 : {
8565 19135 : dump_printf_loc (MSG_NOTE, vect_location,
8566 : "Decided to SLP %d instances. Unrolling factor ",
8567 : decided_to_slp);
8568 19135 : dump_dec (MSG_NOTE, unrolling_factor);
8569 19135 : dump_printf (MSG_NOTE, "\n");
8570 : }
8571 :
8572 485653 : return (decided_to_slp > 0);
8573 485653 : }
8574 :
8575 : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks. */
8576 :
8577 2254092 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
8578 : : vec_info (vec_info::bb, shared),
8579 2254092 : roots (vNULL)
8580 : {
8581 : /* The region we are operating on. bbs[0] is the entry, excluding
8582 : its PHI nodes. In the future we might want to track an explicit
8583 : entry edge to cover bbs[0] PHI nodes and have a region entry
8584 : insert location. */
8585 2254092 : bbs = _bbs.address ();
8586 2254092 : nbbs = _bbs.length ();
8587 :
8588 18136958 : for (unsigned i = 0; i < nbbs; ++i)
8589 : {
8590 15882866 : if (i != 0)
8591 20638364 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8592 7009590 : gsi_next (&si))
8593 : {
8594 7009590 : gphi *phi = si.phi ();
8595 7009590 : gimple_set_uid (phi, 0);
8596 7009590 : add_stmt (phi);
8597 : }
8598 31765732 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8599 144474276 : !gsi_end_p (gsi); gsi_next (&gsi))
8600 : {
8601 128591410 : gimple *stmt = gsi_stmt (gsi);
8602 128591410 : gimple_set_uid (stmt, 0);
8603 128591410 : if (is_gimple_debug (stmt))
8604 81761302 : continue;
8605 46830108 : add_stmt (stmt);
8606 : }
8607 : }
8608 2254092 : }
8609 :
8610 :
8611 : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
8612 : stmts in the basic block. */
8613 :
8614 2254092 : _bb_vec_info::~_bb_vec_info ()
8615 : {
8616 : /* Reset region marker. */
8617 18136958 : for (unsigned i = 0; i < nbbs; ++i)
8618 : {
8619 15882866 : if (i != 0)
8620 20654316 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8621 7025542 : gsi_next (&si))
8622 : {
8623 7025542 : gphi *phi = si.phi ();
8624 7025542 : gimple_set_uid (phi, -1);
8625 : }
8626 31765732 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8627 144417903 : !gsi_end_p (gsi); gsi_next (&gsi))
8628 : {
8629 128535037 : gimple *stmt = gsi_stmt (gsi);
8630 128535037 : gimple_set_uid (stmt, -1);
8631 : }
8632 : }
8633 :
8634 3548420 : for (unsigned i = 0; i < roots.length (); ++i)
8635 : {
8636 1294328 : roots[i].stmts.release ();
8637 1294328 : roots[i].roots.release ();
8638 1294328 : roots[i].remain.release ();
8639 : }
8640 2254092 : roots.release ();
8641 2254092 : }
8642 :
8643 : /* Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE,
8644 : given then that child nodes have already been processed, and that
8645 : their def types currently match their SLP node's def type. */
8646 :
8647 : static bool
8648 2835684 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
8649 : slp_instance node_instance,
8650 : stmt_vector_for_cost *cost_vec)
8651 : {
8652 : /* Handle purely internal nodes. */
8653 2835684 : if (SLP_TREE_PERMUTE_P (node))
8654 : {
8655 121122 : if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
8656 : return false;
8657 :
8658 : stmt_vec_info slp_stmt_info;
8659 : unsigned int i;
8660 319091 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
8661 : {
8662 199268 : if (slp_stmt_info
8663 193597 : && STMT_VINFO_LIVE_P (slp_stmt_info)
8664 199268 : && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
8665 : node_instance, i,
8666 : false, cost_vec))
8667 : return false;
8668 : }
8669 119823 : SLP_TREE_TYPE (node) = permute_info_type;
8670 119823 : return true;
8671 : }
8672 :
8673 2714562 : return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
8674 : }
8675 :
8676 : static int
8677 1746214 : sort_ints (const void *a_, const void *b_)
8678 : {
8679 1746214 : int a = *(const int *)a_;
8680 1746214 : int b = *(const int *)b_;
8681 1746214 : return a - b;
8682 : }
8683 :
8684 : /* Verify if we can externalize a set of internal defs. */
8685 :
8686 : static bool
8687 384733 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
8688 : {
8689 : /* Constant generation uses get_later_stmt which can only handle
8690 : defs from the same BB or a set of defs that can be ordered
8691 : with a dominance query. */
8692 384733 : basic_block bb = NULL;
8693 384733 : bool all_same = true;
8694 384733 : auto_vec<int> bbs;
8695 769466 : bbs.reserve_exact (stmts.length ());
8696 2066869 : for (stmt_vec_info stmt : stmts)
8697 : {
8698 912670 : if (!stmt)
8699 : return false;
8700 912670 : else if (!bb)
8701 384733 : bb = gimple_bb (stmt->stmt);
8702 527937 : else if (gimple_bb (stmt->stmt) != bb)
8703 174145 : all_same = false;
8704 912670 : bbs.quick_push (gimple_bb (stmt->stmt)->index);
8705 : }
8706 384733 : if (all_same)
8707 : return true;
8708 :
8709 : /* Produce a vector of unique BB indexes for the defs. */
8710 133244 : bbs.qsort (sort_ints);
8711 : unsigned i, j;
8712 318704 : for (i = 1, j = 1; i < bbs.length (); ++i)
8713 185460 : if (bbs[i] != bbs[j-1])
8714 141406 : bbs[j++] = bbs[i];
8715 133244 : gcc_assert (j >= 2);
8716 133244 : bbs.truncate (j);
8717 :
8718 266488 : if (bbs.length () == 2)
8719 130167 : return (dominated_by_p (CDI_DOMINATORS,
8720 130167 : BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
8721 130167 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
8722 254348 : || dominated_by_p (CDI_DOMINATORS,
8723 124181 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
8724 124181 : BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
8725 :
8726 : /* ??? For more than two BBs we can sort the vector and verify the
8727 : result is a total order. But we can't use vec::qsort with a
8728 : compare function using a dominance query since there's no way to
8729 : signal failure and any fallback for an unordered pair would
8730 : fail qsort_chk later.
8731 : For now simply hope that ordering after BB index provides the
8732 : best candidate total order. If required we can implement our
8733 : own mergesort or export an entry without checking. */
8734 398994 : for (unsigned i = 1; i < bbs.length (); ++i)
8735 11215 : if (!dominated_by_p (CDI_DOMINATORS,
8736 11215 : BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
8737 11215 : BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
8738 : return false;
8739 :
8740 : return true;
8741 384733 : }
8742 :
8743 : /* Try to build NODE from scalars, returning true on success.
8744 : NODE_INSTANCE is the SLP instance that contains NODE. */
8745 :
8746 : static bool
8747 577937 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
8748 : slp_instance node_instance)
8749 : {
8750 577937 : stmt_vec_info stmt_info;
8751 577937 : unsigned int i;
8752 :
8753 577937 : if (!is_a <bb_vec_info> (vinfo)
8754 74657 : || node == SLP_INSTANCE_TREE (node_instance)
8755 22447 : || !SLP_TREE_SCALAR_STMTS (node).exists ()
8756 22406 : || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
8757 : /* Force the mask use to be built from scalars instead. */
8758 20355 : || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
8759 598152 : || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
8760 557722 : return false;
8761 :
8762 20215 : if (dump_enabled_p ())
8763 74 : dump_printf_loc (MSG_NOTE, vect_location,
8764 : "Building vector operands of %p from scalars instead\n",
8765 : (void *) node);
8766 :
8767 : /* Don't remove and free the child nodes here, since they could be
8768 : referenced by other structures. The analysis and scheduling phases
8769 : (need to) ignore child nodes of anything that isn't vect_internal_def. */
8770 20215 : unsigned int group_size = SLP_TREE_LANES (node);
8771 20215 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
8772 : /* Invariants get their vector type from the uses. */
8773 20215 : SLP_TREE_VECTYPE (node) = NULL_TREE;
8774 20215 : SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
8775 20215 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8776 70683 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
8777 : {
8778 50468 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
8779 50468 : SLP_TREE_SCALAR_OPS (node)[i] = lhs;
8780 : }
8781 : return true;
8782 : }
8783 :
8784 : /* Return true if all elements of the slice are the same. */
8785 : bool
8786 485016 : vect_scalar_ops_slice::all_same_p () const
8787 : {
8788 534108 : for (unsigned int i = 1; i < length; ++i)
8789 450126 : if (!operand_equal_p (op (0), op (i)))
8790 : return false;
8791 : return true;
8792 : }
8793 :
8794 : hashval_t
8795 408341 : vect_scalar_ops_slice_hash::hash (const value_type &s)
8796 : {
8797 408341 : hashval_t hash = 0;
8798 1567935 : for (unsigned i = 0; i < s.length; ++i)
8799 1159594 : hash = iterative_hash_expr (s.op (i), hash);
8800 408341 : return hash;
8801 : }
8802 :
8803 : bool
8804 220245 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
8805 : const compare_type &s2)
8806 : {
8807 220245 : if (s1.length != s2.length)
8808 : return false;
8809 385891 : for (unsigned i = 0; i < s1.length; ++i)
8810 335571 : if (!operand_equal_p (s1.op (i), s2.op (i)))
8811 : return false;
8812 : return true;
8813 : }
8814 :
8815 : /* Compute the prologue cost for invariant or constant operands represented
8816 : by NODE. */
8817 :
8818 : static void
8819 1123347 : vect_prologue_cost_for_slp (vec_info *vinfo, slp_tree node,
8820 : stmt_vector_for_cost *cost_vec)
8821 : {
8822 : /* There's a special case of an existing vector, that costs nothing. */
8823 1123347 : if (SLP_TREE_SCALAR_OPS (node).length () == 0
8824 1123347 : && !SLP_TREE_VEC_DEFS (node).is_empty ())
8825 1448 : return;
8826 : /* Without looking at the actual initializer a vector of
8827 : constants can be implemented as load from the constant pool.
8828 : When all elements are the same we can use a splat. */
8829 1121899 : tree vectype = SLP_TREE_VECTYPE (node);
8830 1121899 : unsigned group_size = SLP_TREE_SCALAR_OPS (node).length ();
8831 1121899 : unsigned HOST_WIDE_INT const_nunits;
8832 1121899 : unsigned nelt_limit;
8833 1121899 : unsigned nvectors = vect_get_num_copies (vinfo, node);
8834 1121899 : auto ops = &SLP_TREE_SCALAR_OPS (node);
8835 1121899 : auto_vec<unsigned int> starts (nvectors);
8836 1121899 : if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
8837 1121899 : && ! multiple_p (const_nunits, group_size))
8838 : {
8839 65460 : nelt_limit = const_nunits;
8840 65460 : hash_set<vect_scalar_ops_slice_hash> vector_ops;
8841 271321 : for (unsigned int i = 0; i < nvectors; ++i)
8842 205861 : if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
8843 155541 : starts.quick_push (i * nelt_limit);
8844 65460 : }
8845 : else
8846 : {
8847 : /* If either the vector has variable length or the vectors
8848 : are composed of repeated whole groups we only need to
8849 : cost construction once. All vectors will be the same. */
8850 1056439 : nelt_limit = group_size;
8851 1056439 : starts.quick_push (0);
8852 : }
8853 : /* ??? We're just tracking whether vectors in a single node are the same.
8854 : Ideally we'd do something more global. */
8855 1121899 : bool passed = false;
8856 4577677 : for (unsigned int start : starts)
8857 : {
8858 1211980 : vect_cost_for_stmt kind;
8859 1211980 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
8860 : kind = vector_load;
8861 485016 : else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
8862 : kind = scalar_to_vec;
8863 : else
8864 401034 : kind = vec_construct;
8865 : /* The target cost hook has no idea which part of the SLP node
8866 : we are costing so avoid passing it down more than once. Pass
8867 : it to the first vec_construct or scalar_to_vec part since for those
8868 : the x86 backend tries to account for GPR to XMM register moves. */
8869 1211980 : record_stmt_cost (cost_vec, 1, kind, nullptr,
8870 1211980 : (kind != vector_load && !passed) ? node : nullptr,
8871 : vectype, 0, vect_prologue);
8872 1211980 : if (kind != vector_load)
8873 485016 : passed = true;
8874 : }
8875 1121899 : }
8876 :
8877 : /* Analyze statements contained in SLP tree NODE after recursively analyzing
8878 : the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
8879 :
8880 : Return true if the operations are supported. */
8881 :
8882 : static bool
8883 5246578 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
8884 : slp_instance node_instance,
8885 : hash_set<slp_tree> &visited_set,
8886 : vec<slp_tree> &visited_vec,
8887 : stmt_vector_for_cost *cost_vec)
8888 : {
8889 5246578 : int i, j;
8890 5246578 : slp_tree child;
8891 :
8892 : /* Assume we can code-generate all invariants. */
8893 5246578 : if (!node
8894 4866177 : || SLP_TREE_DEF_TYPE (node) == vect_constant_def
8895 4082964 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
8896 : return true;
8897 :
8898 3524994 : if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
8899 : {
8900 5 : if (dump_enabled_p ())
8901 0 : dump_printf_loc (MSG_NOTE, vect_location,
8902 : "Failed cyclic SLP reference in %p\n", (void *) node);
8903 5 : return false;
8904 : }
8905 3524989 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
8906 :
8907 : /* If we already analyzed the exact same set of scalar stmts we're done.
8908 : We share the generated vector stmts for those. */
8909 3524989 : if (visited_set.add (node))
8910 : return true;
8911 3154056 : visited_vec.safe_push (node);
8912 :
8913 3154056 : bool res = true;
8914 3154056 : unsigned visited_rec_start = visited_vec.length ();
8915 3154056 : unsigned cost_vec_rec_start = cost_vec->length ();
8916 3154056 : bool seen_non_constant_child = false;
8917 6748288 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
8918 : {
8919 3912390 : res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
8920 : visited_set, visited_vec,
8921 : cost_vec);
8922 3912390 : if (!res)
8923 : break;
8924 3594232 : if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
8925 3594232 : seen_non_constant_child = true;
8926 : }
8927 : /* We're having difficulties scheduling nodes with just constant
8928 : operands and no scalar stmts since we then cannot compute a stmt
8929 : insertion place. */
8930 3154056 : if (res
8931 3154056 : && !seen_non_constant_child
8932 3154056 : && SLP_TREE_SCALAR_STMTS (node).is_empty ())
8933 : {
8934 214 : if (dump_enabled_p ())
8935 6 : dump_printf_loc (MSG_NOTE, vect_location,
8936 : "Cannot vectorize all-constant op node %p\n",
8937 : (void *) node);
8938 : res = false;
8939 : }
8940 :
8941 3153842 : if (res)
8942 2835684 : res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
8943 : cost_vec);
8944 : /* If analysis failed we have to pop all recursive visited nodes
8945 : plus ourselves. */
8946 3154056 : if (!res)
8947 : {
8948 2879468 : while (visited_vec.length () >= visited_rec_start)
8949 861797 : visited_set.remove (visited_vec.pop ());
8950 577937 : cost_vec->truncate (cost_vec_rec_start);
8951 : }
8952 :
8953 : /* When the node can be vectorized cost invariant nodes it references.
8954 : This is not done in DFS order to allow the referring node
8955 : vectorizable_* calls to nail down the invariant nodes vector type
8956 : and possibly unshare it if it needs a different vector type than
8957 : other referrers. */
8958 3154056 : if (res)
8959 5843519 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
8960 3267400 : if (child
8961 2956002 : && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
8962 2956002 : || SLP_TREE_DEF_TYPE (child) == vect_external_def)
8963 : /* Perform usual caching, note code-generation still
8964 : code-gens these nodes multiple times but we expect
8965 : to CSE them later. */
8966 4477684 : && !visited_set.add (child))
8967 : {
8968 1166808 : visited_vec.safe_push (child);
8969 : /* ??? After auditing more code paths make a "default"
8970 : and push the vector type from NODE to all children
8971 : if it is not already set. */
8972 : /* Compute the number of vectors to be generated. */
8973 1166808 : tree vector_type = SLP_TREE_VECTYPE (child);
8974 1166808 : if (!vector_type)
8975 : {
8976 : /* Masked loads can have an undefined (default SSA definition)
8977 : else operand. We do not need to cost it. */
8978 43461 : vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
8979 44892 : if (SLP_TREE_TYPE (node) == load_vec_info_type
8980 44892 : && ((ops.length ()
8981 1431 : && TREE_CODE (ops[0]) == SSA_NAME
8982 0 : && SSA_NAME_IS_DEFAULT_DEF (ops[0])
8983 0 : && VAR_P (SSA_NAME_VAR (ops[0])))
8984 1431 : || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
8985 1431 : continue;
8986 :
8987 : /* For shifts with a scalar argument we don't need
8988 : to cost or code-generate anything.
8989 : ??? Represent this more explicitly. */
8990 42030 : gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
8991 : && j == 1);
8992 42030 : continue;
8993 42030 : }
8994 :
8995 : /* And cost them. */
8996 1123347 : vect_prologue_cost_for_slp (vinfo, child, cost_vec);
8997 : }
8998 :
8999 : /* If this node or any of its children can't be vectorized, try pruning
9000 : the tree here rather than felling the whole thing. */
9001 577937 : if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
9002 : {
9003 : /* We'll need to revisit this for invariant costing and number
9004 : of vectorized stmt setting. */
9005 : res = true;
9006 : }
9007 :
9008 : return res;
9009 : }
9010 :
9011 : /* Mark lanes of NODE that are live outside of the basic-block vectorized
9012 : region and that can be vectorized using vectorizable_live_operation
9013 : with STMT_VINFO_LIVE_P. Not handled live operations will cause the
9014 : scalar code computing it to be retained. */
9015 :
9016 : static void
9017 924982 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
9018 : slp_instance instance,
9019 : stmt_vector_for_cost *cost_vec,
9020 : hash_set<stmt_vec_info> &svisited,
9021 : hash_set<slp_tree> &visited)
9022 : {
9023 924982 : if (visited.add (node))
9024 43073 : return;
9025 :
9026 881909 : unsigned i;
9027 881909 : stmt_vec_info stmt_info;
9028 881909 : stmt_vec_info last_stmt = vect_find_last_scalar_stmt_in_slp (node);
9029 3197221 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9030 : {
9031 2315312 : if (!stmt_info || svisited.contains (stmt_info))
9032 50988 : continue;
9033 2289741 : stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
9034 2289741 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
9035 12983 : && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
9036 : /* Only the pattern root stmt computes the original scalar value. */
9037 9541 : continue;
9038 2280200 : if (!PURE_SLP_STMT (orig_stmt_info))
9039 : /* Iff the stmt is not part of the vector coverage because it or
9040 : uses of it are used by SLP graph leafs as extern input there is
9041 : no point in trying to live code-generate from a vector stmt as
9042 : the scalar stmt will survive anyway. */
9043 15876 : continue;
9044 2264324 : bool mark_visited = true;
9045 2264324 : gimple *orig_stmt = orig_stmt_info->stmt;
9046 2264324 : ssa_op_iter op_iter;
9047 2264324 : def_operand_p def_p;
9048 5028604 : FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
9049 : {
9050 : /* We have to verify whether we can insert the lane extract
9051 : before all uses. The following is a conservative approximation.
9052 : We cannot put this into vectorizable_live_operation because
9053 : iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
9054 : doesn't work.
9055 : Note that while the fact that we emit code for loads at the
9056 : first load should make this a non-problem leafs we construct
9057 : from scalars are vectorized after the last scalar def.
9058 : ??? If we'd actually compute the insert location during
9059 : analysis we could use sth less conservative than the last
9060 : scalar stmt in the node for the dominance check. */
9061 : /* ??? What remains is "live" uses in vector CTORs in the same
9062 : SLP graph which is where those uses can end up code-generated
9063 : right after their definition instead of close to their original
9064 : use. But that would restrict us to code-generate lane-extracts
9065 : from the latest stmt in a node. So we compensate for this
9066 : during code-generation, simply not replacing uses for those
9067 : hopefully rare cases. */
9068 499956 : imm_use_iterator use_iter;
9069 499956 : gimple *use_stmt;
9070 499956 : stmt_vec_info use_stmt_info;
9071 :
9072 499956 : bool live_p = false;
9073 499956 : bool can_insert = true;
9074 1936292 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9075 952588 : if (!is_gimple_debug (use_stmt)
9076 952588 : && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
9077 708344 : || !PURE_SLP_STMT (use_stmt_info)))
9078 : {
9079 150442 : live_p = true;
9080 150442 : if (!vect_stmt_dominates_stmt_p (last_stmt->stmt, use_stmt))
9081 : {
9082 16208 : if (dump_enabled_p ())
9083 34 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9084 : "Cannot determine insertion place for "
9085 : "lane extract\n");
9086 : can_insert = false;
9087 : break;
9088 : }
9089 499956 : }
9090 499956 : if (live_p && can_insert)
9091 : {
9092 : /* Only record a live stmt when we can replace all uses. We
9093 : record from which SLP tree we vectorize the uses, so we'll
9094 : cost once and can deal with the case that not all SLP nodes
9095 : may be suitable for code-generation of all live uses.
9096 : ??? But we never split up the work between multiple SLP
9097 : nodes. */
9098 67146 : STMT_VINFO_LIVE_P (stmt_info) = true;
9099 67146 : if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
9100 : instance, i, false, cost_vec))
9101 : {
9102 0 : STMT_VINFO_LIVE_P (stmt_info) = false;
9103 0 : mark_visited = false;
9104 : }
9105 : }
9106 : }
9107 2264324 : if (mark_visited)
9108 2264324 : svisited.add (stmt_info);
9109 : }
9110 :
9111 : slp_tree child;
9112 2544607 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9113 888541 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9114 232065 : vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
9115 : svisited, visited);
9116 : }
9117 :
9118 : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
9119 : are live outside of the basic-block vectorized region and that can be
9120 : vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. */
9121 :
9122 : static void
9123 239088 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
9124 : {
9125 239088 : if (bb_vinfo->slp_instances.is_empty ())
9126 0 : return;
9127 :
9128 239088 : hash_set<slp_tree> visited;
9129 239088 : hash_set<stmt_vec_info> svisited;
9130 1410181 : for (slp_instance instance : bb_vinfo->slp_instances)
9131 : {
9132 692917 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9133 30593 : STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
9134 692917 : vect_location = instance->location ();
9135 692917 : vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
9136 : instance, &instance->cost_vec,
9137 : svisited, visited);
9138 : }
9139 239088 : }
9140 :
9141 : /* Determine whether we can vectorize the reduction epilogue for INSTANCE. */
9142 :
9143 : static bool
9144 79546 : vectorizable_bb_reduc_epilogue (slp_instance instance,
9145 : stmt_vector_for_cost *cost_vec)
9146 : {
9147 79546 : gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
9148 79546 : enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
9149 79546 : if (reduc_code == MINUS_EXPR)
9150 0 : reduc_code = PLUS_EXPR;
9151 79546 : internal_fn reduc_fn;
9152 79546 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
9153 79546 : if (!vectype
9154 79534 : || !reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
9155 79534 : || reduc_fn == IFN_LAST
9156 79534 : || !direct_internal_fn_supported_p (reduc_fn, vectype, OPTIMIZE_FOR_BOTH)
9157 117290 : || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
9158 37744 : TREE_TYPE (vectype)))
9159 : {
9160 54223 : if (dump_enabled_p ())
9161 313 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9162 : "not vectorized: basic block reduction epilogue "
9163 : "operation unsupported.\n");
9164 54223 : return false;
9165 : }
9166 :
9167 : /* There's no way to cost a horizontal vector reduction via REDUC_FN so
9168 : cost log2 vector operations plus shuffles and one extraction. */
9169 25323 : unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
9170 25323 : record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
9171 : vectype, 0, vect_body);
9172 25323 : record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
9173 : vectype, 0, vect_body);
9174 25323 : record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
9175 : vectype, 0, vect_body);
9176 :
9177 : /* Since we replace all stmts of a possibly longer scalar reduction
9178 : chain account for the extra scalar stmts for that. */
9179 25323 : if (!instance->remain_defs.is_empty ())
9180 19462 : record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
9181 9731 : instance->root_stmts[0], 0, vect_body);
9182 : return true;
9183 : }
9184 :
9185 : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
9186 : and recurse to children. */
9187 :
9188 : static void
9189 174157 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
9190 : hash_set<slp_tree> &visited)
9191 : {
9192 174157 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
9193 174157 : || visited.add (node))
9194 76654 : return;
9195 :
9196 530531 : for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
9197 240388 : if (stmt)
9198 246782 : roots.remove (vect_orig_stmt (stmt));
9199 :
9200 358009 : for (auto child : SLP_TREE_CHILDREN (node))
9201 111424 : if (child)
9202 110012 : vect_slp_prune_covered_roots (child, roots, visited);
9203 : }
9204 :
9205 : /* Hand over COST_VEC to the target COSTS grouped by SLP node. */
9206 :
9207 : static void
9208 959197 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
9209 : {
9210 3629789 : for (unsigned start = 0; start < cost_vec.length ();)
9211 : {
9212 2670592 : unsigned end = start + 1;
9213 3262571 : while (end < cost_vec.length ()
9214 5574541 : && cost_vec[start].node == cost_vec[end].node)
9215 591979 : end++;
9216 2670592 : costs->add_slp_cost (cost_vec[start].node,
9217 2670592 : array_slice<stmt_info_for_cost>
9218 2670592 : (cost_vec.begin () + start, end - start));
9219 2670592 : start = end;
9220 : }
9221 959197 : }
9222 :
9223 : /* Analyze statements in SLP instances of VINFO. Return true if the
9224 : operations are supported. */
9225 :
9226 : bool
9227 677800 : vect_slp_analyze_operations (vec_info *vinfo)
9228 : {
9229 677800 : slp_instance instance;
9230 677800 : int i;
9231 :
9232 677800 : DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
9233 :
9234 677800 : hash_set<slp_tree> visited;
9235 1761928 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9236 : {
9237 1334188 : auto_vec<slp_tree> visited_vec;
9238 1334188 : stmt_vector_for_cost cost_vec;
9239 1334188 : cost_vec.create (2);
9240 1334188 : if (is_a <bb_vec_info> (vinfo))
9241 800221 : vect_location = instance->location ();
9242 1334188 : if (!vect_slp_analyze_node_operations (vinfo,
9243 : SLP_INSTANCE_TREE (instance),
9244 : instance, visited, visited_vec,
9245 : &cost_vec)
9246 : /* CTOR instances require vectorized defs for the SLP tree root. */
9247 1094619 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
9248 6141 : && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
9249 : != vect_internal_def
9250 : /* Make sure we vectorized with the expected type. */
9251 6141 : || !useless_type_conversion_p
9252 6141 : (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
9253 : (instance->root_stmts[0]->stmt))),
9254 6141 : TREE_TYPE (SLP_TREE_VECTYPE
9255 : (SLP_INSTANCE_TREE (instance))))))
9256 : /* Check we can vectorize the reduction. */
9257 1094604 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
9258 79546 : && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
9259 : /* Check we can vectorize the gcond. */
9260 2374569 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
9261 65330 : && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
9262 65330 : SLP_INSTANCE_ROOT_STMTS (instance)[0],
9263 : NULL,
9264 : SLP_INSTANCE_TREE (instance),
9265 : &cost_vec)))
9266 : {
9267 356508 : cost_vec.release ();
9268 356508 : slp_tree node = SLP_INSTANCE_TREE (instance);
9269 356508 : stmt_vec_info stmt_info;
9270 356508 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9271 272328 : stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9272 84180 : else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
9273 84180 : && SLP_TREE_SCALAR_STMTS (node)[0])
9274 : stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
9275 : else
9276 0 : stmt_info = SLP_TREE_REPRESENTATIVE (node);
9277 356508 : if (is_a <loop_vec_info> (vinfo))
9278 : {
9279 250060 : if (dump_enabled_p ())
9280 6492 : dump_printf_loc (MSG_NOTE, vect_location,
9281 : "unsupported SLP instance starting from: %G",
9282 : stmt_info->stmt);
9283 250060 : return false;
9284 : }
9285 106448 : if (dump_enabled_p ())
9286 369 : dump_printf_loc (MSG_NOTE, vect_location,
9287 : "removing SLP instance operations starting from: %G",
9288 : stmt_info->stmt);
9289 545117 : while (!visited_vec.is_empty ())
9290 : {
9291 438669 : slp_tree node = visited_vec.pop ();
9292 438669 : SLP_TREE_TYPE (node) = undef_vec_info_type;
9293 438669 : if (node->data)
9294 : {
9295 12065 : delete node->data;
9296 12065 : node->data = nullptr;
9297 : }
9298 438669 : visited.remove (node);
9299 : }
9300 106448 : vect_free_slp_instance (instance);
9301 106448 : vinfo->slp_instances.ordered_remove (i);
9302 : }
9303 : else
9304 : {
9305 977680 : i++;
9306 977680 : if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
9307 : {
9308 283907 : add_slp_costs (loop_vinfo->vector_costs, cost_vec);
9309 283907 : cost_vec.release ();
9310 : }
9311 : else
9312 : /* For BB vectorization remember the SLP graph entry
9313 : cost for later. */
9314 693773 : instance->cost_vec = cost_vec;
9315 : }
9316 1334188 : }
9317 :
9318 : /* Now look for SLP instances with a root that are covered by other
9319 : instances and remove them. */
9320 427740 : hash_set<stmt_vec_info> roots;
9321 1763172 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9322 941758 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9323 34066 : roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
9324 427740 : if (!roots.is_empty ())
9325 : {
9326 13301 : visited.empty ();
9327 77446 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9328 64145 : vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
9329 : visited);
9330 77446 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9331 64145 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
9332 34066 : && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
9333 : {
9334 856 : stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9335 856 : if (dump_enabled_p ())
9336 17 : dump_printf_loc (MSG_NOTE, vect_location,
9337 : "removing SLP instance operations starting "
9338 : "from: %G", root->stmt);
9339 856 : vect_free_slp_instance (instance);
9340 856 : vinfo->slp_instances.ordered_remove (i);
9341 : }
9342 : else
9343 63289 : ++i;
9344 : }
9345 :
9346 855480 : return !vinfo->slp_instances.is_empty ();
9347 1105540 : }
9348 :
9349 : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
9350 : closing the eventual chain. */
9351 :
9352 : static slp_instance
9353 765428 : get_ultimate_leader (slp_instance instance,
9354 : hash_map<slp_instance, slp_instance> &instance_leader)
9355 : {
9356 765428 : auto_vec<slp_instance *, 8> chain;
9357 765428 : slp_instance *tem;
9358 853113 : while (*(tem = instance_leader.get (instance)) != instance)
9359 : {
9360 87685 : chain.safe_push (tem);
9361 87685 : instance = *tem;
9362 : }
9363 853113 : while (!chain.is_empty ())
9364 87685 : *chain.pop () = instance;
9365 765428 : return instance;
9366 765428 : }
9367 :
9368 : namespace {
9369 : /* Subroutine of vect_bb_partition_graph_r. Map KEY to INSTANCE in
9370 : KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
9371 : for KEY. Return true if KEY was already in KEY_TO_INSTANCE.
9372 :
9373 : INSTANCE_LEADER is as for get_ultimate_leader. */
9374 :
9375 : template<typename T>
9376 : bool
9377 3345868 : vect_map_to_instance (slp_instance instance, T key,
9378 : hash_map<T, slp_instance> &key_to_instance,
9379 : hash_map<slp_instance, slp_instance> &instance_leader)
9380 : {
9381 : bool existed_p;
9382 3345868 : slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
9383 3345868 : if (!existed_p)
9384 : ;
9385 177547 : else if (key_instance != instance)
9386 : {
9387 : /* If we're running into a previously marked key make us the
9388 : leader of the current ultimate leader. This keeps the
9389 : leader chain acyclic and works even when the current instance
9390 : connects two previously independent graph parts. */
9391 72511 : slp_instance key_leader
9392 72511 : = get_ultimate_leader (key_instance, instance_leader);
9393 72511 : if (key_leader != instance)
9394 22112 : instance_leader.put (key_leader, instance);
9395 : }
9396 3345868 : key_instance = instance;
9397 3345868 : return existed_p;
9398 : }
9399 : }
9400 :
9401 : /* Worker of vect_bb_partition_graph, recurse on NODE. */
9402 :
9403 : static void
9404 924982 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
9405 : slp_instance instance, slp_tree node,
9406 : hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
9407 : hash_map<slp_tree, slp_instance> &node_to_instance,
9408 : hash_map<slp_instance, slp_instance> &instance_leader)
9409 : {
9410 5188054 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
9411 2420886 : if (stmt_info)
9412 2420886 : vect_map_to_instance (instance, stmt_info, stmt_to_instance,
9413 : instance_leader);
9414 :
9415 924982 : if (vect_map_to_instance (instance, node, node_to_instance,
9416 : instance_leader))
9417 : return;
9418 :
9419 3318764 : for (auto child : SLP_TREE_CHILDREN (node))
9420 888541 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9421 232065 : vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
9422 : node_to_instance, instance_leader);
9423 : }
9424 :
9425 : /* Partition the SLP graph into pieces that can be costed independently. */
9426 :
9427 : static void
9428 239088 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
9429 : {
9430 239088 : DUMP_VECT_SCOPE ("vect_bb_partition_graph");
9431 :
9432 : /* First walk the SLP graph assigning each involved scalar stmt a
9433 : corresponding SLP graph entry and upon visiting a previously
9434 : marked stmt, make the stmts leader the current SLP graph entry. */
9435 239088 : hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
9436 239088 : hash_map<slp_tree, slp_instance> node_to_instance;
9437 239088 : hash_map<slp_instance, slp_instance> instance_leader;
9438 239088 : slp_instance instance;
9439 932005 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9440 : {
9441 692917 : instance_leader.put (instance, instance);
9442 692917 : vect_bb_partition_graph_r (bb_vinfo,
9443 : instance, SLP_INSTANCE_TREE (instance),
9444 : stmt_to_instance, node_to_instance,
9445 : instance_leader);
9446 : }
9447 :
9448 : /* Then collect entries to each independent subgraph. */
9449 1171093 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9450 : {
9451 692917 : slp_instance leader = get_ultimate_leader (instance, instance_leader);
9452 692917 : leader->subgraph_entries.safe_push (instance);
9453 692917 : if (dump_enabled_p ()
9454 692917 : && leader != instance)
9455 71 : dump_printf_loc (MSG_NOTE, vect_location,
9456 : "instance %p is leader of %p\n",
9457 : (void *) leader, (void *) instance);
9458 : }
9459 239088 : }
9460 :
9461 : /* Compute the scalar cost of the SLP node NODE and its children
9462 : and return it. Do not account defs that are marked in LIFE and
9463 : update LIFE according to uses of NODE. */
9464 :
9465 : static void
9466 689411 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
9467 : vec<stmt_vec_info> &worklist,
9468 : stmt_vector_for_cost *cost_vec,
9469 : hash_set<stmt_vec_info> &visited)
9470 : {
9471 3189014 : while (!worklist.is_empty ())
9472 : {
9473 2499603 : stmt_vec_info stmt = worklist.pop ();
9474 2790339 : if (!PURE_SLP_STMT (stmt))
9475 306652 : continue;
9476 :
9477 : /* When the stmt is live but not actually vectorized we have
9478 : to keep the feeding scalar defs. */
9479 2212270 : if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
9480 : {
9481 2143509 : bool live_p = false;
9482 2143509 : ssa_op_iter op_iter;
9483 2143509 : def_operand_p def_p;
9484 4688977 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
9485 : {
9486 401959 : imm_use_iterator use_iter;
9487 401959 : gimple *use_stmt;
9488 1453074 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9489 649156 : if (!is_gimple_debug (use_stmt))
9490 : {
9491 480242 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
9492 480242 : if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
9493 : {
9494 24629 : if (dump_enabled_p ())
9495 : {
9496 67 : dump_printf_loc (MSG_NOTE, vect_location,
9497 : "stmt considered live: %G",
9498 : stmt->stmt);
9499 67 : dump_printf_loc (MSG_NOTE, vect_location,
9500 : "because of use in: %G",
9501 : use_stmt);
9502 : }
9503 : live_p = true;
9504 : }
9505 401959 : }
9506 : }
9507 2143509 : if (live_p)
9508 15916 : continue;
9509 : }
9510 :
9511 : /* The following assert verifies that vect_bb_partition_graph
9512 : partitions the SLP graph in a way that each scalar stmt of
9513 : the coverage of the SLP graph belongs to exactly one subgraph.
9514 : ??? This is currently not guaranteed since the function
9515 : works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
9516 : tripping or scalar stmts costed multiple times, making vectorization
9517 : more profitable than it really is. */
9518 : /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
9519 :
9520 2192951 : if (vect_nop_conversion_p (stmt))
9521 : ;
9522 : /* For single-argument PHIs assume coalescing which means zero
9523 : cost for the scalar and the vector PHIs. This avoids
9524 : artificially favoring the vector path (but may pessimize it
9525 : in some cases). */
9526 2170227 : else if (is_a <gphi *> (stmt->stmt)
9527 2170227 : && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
9528 : ;
9529 : else
9530 : {
9531 2161408 : vect_cost_for_stmt kind;
9532 2161408 : if (STMT_VINFO_DATA_REF (stmt))
9533 : {
9534 1986913 : data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
9535 1986913 : tree base = get_base_address (DR_REF (dr));
9536 : /* When the scalar access is to a non-global not
9537 : address-taken decl that is not BLKmode assume we can
9538 : access it with a single non-load/store instruction. */
9539 1986913 : if (DECL_P (base)
9540 1532840 : && !is_global_var (base)
9541 1456781 : && !TREE_ADDRESSABLE (base)
9542 2538192 : && DECL_MODE (base) != BLKmode)
9543 : kind = scalar_stmt;
9544 1842968 : else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
9545 : kind = scalar_load;
9546 : else
9547 1614014 : kind = scalar_store;
9548 : }
9549 : else
9550 : kind = scalar_stmt;
9551 : /* Cost each scalar stmt only once. */
9552 2161408 : gimple_set_visited (stmt->stmt, true);
9553 2161408 : record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
9554 : }
9555 :
9556 : /* Now walk relevant parts of the SSA use-def graph. */
9557 2192951 : slp_oprnds child_ops (stmt);
9558 4595838 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
9559 : {
9560 2402887 : tree op = child_ops.get_op_for_slp_child (stmt, i);
9561 2402887 : stmt_vec_info def = vinfo->lookup_def (op);
9562 2402887 : if (def && !visited.add (def))
9563 696709 : worklist.safe_push (def);
9564 : }
9565 : }
9566 689411 : }
9567 :
9568 :
9569 : /* Comparator for the loop-index sorted cost vectors. */
9570 :
9571 : static int
9572 17165490 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
9573 : {
9574 17165490 : auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
9575 17165490 : auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
9576 17165490 : if (a->first < b->first)
9577 : return -1;
9578 16346256 : else if (a->first == b->first)
9579 15662879 : return 0;
9580 : return 1;
9581 : }
9582 :
9583 : /* Check if vectorization of the basic block is profitable for the
9584 : subgraph denoted by SLP_INSTANCES. */
9585 :
9586 : static bool
9587 667451 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
9588 : vec<slp_instance> slp_instances,
9589 : loop_p orig_loop)
9590 : {
9591 667451 : slp_instance instance;
9592 667451 : int i;
9593 667451 : unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
9594 667451 : unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
9595 :
9596 667451 : if (dump_enabled_p ())
9597 : {
9598 104 : dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph: \n");
9599 104 : hash_set<slp_tree> visited;
9600 421 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9601 109 : vect_print_slp_graph (MSG_NOTE, vect_location,
9602 : SLP_INSTANCE_TREE (instance), visited);
9603 104 : }
9604 :
9605 : /* Then DFS walk scalar stmts, performing costing and handling
9606 : still live scalar stmts via the previously computed vector coverage. */
9607 667451 : stmt_vector_for_cost scalar_costs = vNULL;
9608 667451 : stmt_vector_for_cost vector_costs = vNULL;
9609 667451 : hash_set<slp_tree> visited;
9610 667451 : hash_set<stmt_vec_info> svisited;
9611 1356862 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9612 : {
9613 689411 : auto_vec<stmt_vec_info> worklist;
9614 689411 : if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
9615 60616 : record_stmt_cost (&scalar_costs,
9616 30308 : SLP_INSTANCE_ROOT_STMTS (instance).length (),
9617 : scalar_stmt,
9618 30308 : SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
9619 3880771 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
9620 : {
9621 1812538 : stmt = vect_orig_stmt (stmt);
9622 1812538 : if (!svisited.add (stmt))
9623 1802894 : worklist.safe_push (stmt);
9624 : }
9625 689411 : vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
9626 689411 : vector_costs.safe_splice (instance->cost_vec);
9627 689411 : instance->cost_vec.release ();
9628 689411 : }
9629 :
9630 667451 : if (dump_enabled_p ())
9631 104 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
9632 :
9633 : /* When costing non-loop vectorization we need to consider each covered
9634 : loop independently and make sure vectorization is profitable. For
9635 : now we assume a loop may be not entered or executed an arbitrary
9636 : number of iterations (??? static information can provide more
9637 : precise info here) which means we can simply cost each containing
9638 : loops stmts separately. */
9639 :
9640 : /* First produce cost vectors sorted by loop index. */
9641 667451 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9642 667451 : li_scalar_costs (scalar_costs.length ());
9643 667451 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9644 667451 : li_vector_costs (vector_costs.length ());
9645 667451 : stmt_info_for_cost *cost;
9646 2859167 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9647 : {
9648 2191716 : unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9649 2191716 : li_scalar_costs.quick_push (std::make_pair (l, cost));
9650 : }
9651 : /* Use a random used loop as fallback in case the first vector_costs
9652 : entry does not have a stmt_info associated with it. */
9653 667451 : unsigned l = li_scalar_costs[0].first;
9654 2435866 : FOR_EACH_VEC_ELT (vector_costs, i, cost)
9655 : {
9656 : /* We inherit from the previous COST, invariants, externals and
9657 : extracts immediately follow the cost for the related stmt. */
9658 1768415 : if (cost->stmt_info)
9659 1038567 : l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9660 1768415 : li_vector_costs.quick_push (std::make_pair (l, cost));
9661 : }
9662 667451 : li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
9663 667451 : li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
9664 :
9665 : /* Now cost the portions individually. */
9666 : unsigned vi = 0;
9667 : unsigned si = 0;
9668 : bool profitable = true;
9669 1343856 : while (si < li_scalar_costs.length ()
9670 2020303 : && vi < li_vector_costs.length ())
9671 : {
9672 676405 : unsigned sl = li_scalar_costs[si].first;
9673 676405 : unsigned vl = li_vector_costs[vi].first;
9674 676405 : if (sl != vl)
9675 : {
9676 1115 : if (dump_enabled_p ())
9677 0 : dump_printf_loc (MSG_NOTE, vect_location,
9678 : "Scalar %d and vector %d loop part do not "
9679 : "match up, skipping scalar part\n", sl, vl);
9680 : /* Skip the scalar part, assuming zero cost on the vector side. */
9681 1821 : do
9682 : {
9683 1821 : si++;
9684 : }
9685 1821 : while (si < li_scalar_costs.length ()
9686 3807 : && li_scalar_costs[si].first == sl);
9687 1115 : continue;
9688 : }
9689 :
9690 675290 : if (dump_enabled_p ())
9691 111 : dump_printf_loc (MSG_NOTE, vect_location,
9692 : "Scalar cost for part in loop %d\n", sl);
9693 675290 : class vector_costs *scalar_target_cost_data = init_cost (bb_vinfo, true);
9694 2189762 : do
9695 : {
9696 2189762 : add_stmt_cost (scalar_target_cost_data, li_scalar_costs[si].second);
9697 2189762 : si++;
9698 : }
9699 2189762 : while (si < li_scalar_costs.length ()
9700 4387662 : && li_scalar_costs[si].first == sl);
9701 675290 : scalar_target_cost_data->finish_cost (nullptr);
9702 675290 : scalar_cost = scalar_target_cost_data->body_cost ();
9703 :
9704 : /* Complete the target-specific vector cost calculation. */
9705 675290 : if (dump_enabled_p ())
9706 111 : dump_printf_loc (MSG_NOTE, vect_location,
9707 : "Vector cost for part in loop %d\n", vl);
9708 675290 : class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
9709 675290 : auto_vec<stmt_info_for_cost> tem;
9710 1762008 : do
9711 : {
9712 1762008 : tem.safe_push (*li_vector_costs[vi].second);
9713 1762008 : vi++;
9714 : }
9715 1762008 : while (vi < li_vector_costs.length ()
9716 3533285 : && li_vector_costs[vi].first == vl);
9717 675290 : add_slp_costs (vect_target_cost_data, tem);
9718 675290 : vect_target_cost_data->finish_cost (scalar_target_cost_data);
9719 675290 : vec_prologue_cost = vect_target_cost_data->prologue_cost ();
9720 675290 : vec_inside_cost = vect_target_cost_data->body_cost ();
9721 675290 : vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
9722 675290 : delete scalar_target_cost_data;
9723 675290 : delete vect_target_cost_data;
9724 :
9725 675290 : vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
9726 :
9727 675290 : if (dump_enabled_p ())
9728 : {
9729 111 : dump_printf_loc (MSG_NOTE, vect_location,
9730 : "Cost model analysis for part in loop %d:\n", sl);
9731 111 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
9732 : vec_inside_cost + vec_outside_cost);
9733 111 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", scalar_cost);
9734 : }
9735 :
9736 : /* Vectorization is profitable if its cost is more than the cost of scalar
9737 : version. Note that we err on the vector side for equal cost because
9738 : the cost estimate is otherwise quite pessimistic (constant uses are
9739 : free on the scalar side but cost a load on the vector side for
9740 : example). */
9741 675290 : if (vec_outside_cost + vec_inside_cost > scalar_cost)
9742 184523 : profitable = false;
9743 675290 : }
9744 1152923 : if (profitable && vi < li_vector_costs.length ())
9745 : {
9746 1064 : if (dump_enabled_p ())
9747 0 : dump_printf_loc (MSG_NOTE, vect_location,
9748 : "Excess vector cost for part in loop %d:\n",
9749 0 : li_vector_costs[vi].first);
9750 : profitable = false;
9751 : }
9752 :
9753 : /* Unset visited flag. This is delayed when the subgraph is profitable
9754 : and we process the loop for remaining unvectorized if-converted code. */
9755 667451 : if (!orig_loop || !profitable)
9756 2857748 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9757 2190400 : gimple_set_visited (cost->stmt_info->stmt, false);
9758 :
9759 667451 : scalar_costs.release ();
9760 667451 : vector_costs.release ();
9761 :
9762 667451 : return profitable;
9763 667451 : }
9764 :
9765 : /* qsort comparator for lane defs. */
9766 :
9767 : static int
9768 120 : vld_cmp (const void *a_, const void *b_)
9769 : {
9770 120 : auto *a = (const std::pair<unsigned, tree> *)a_;
9771 120 : auto *b = (const std::pair<unsigned, tree> *)b_;
9772 120 : return a->first - b->first;
9773 : }
9774 :
9775 : /* Return true if USE_STMT is a vector lane insert into VEC and set
9776 : *THIS_LANE to the lane number that is set. */
9777 :
9778 : static bool
9779 262 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
9780 : {
9781 262 : gassign *use_ass = dyn_cast <gassign *> (use_stmt);
9782 104 : if (!use_ass
9783 104 : || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
9784 35 : || (vec
9785 35 : ? gimple_assign_rhs1 (use_ass) != vec
9786 24 : : ((vec = gimple_assign_rhs1 (use_ass)), false))
9787 59 : || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
9788 59 : TREE_TYPE (gimple_assign_rhs2 (use_ass)))
9789 59 : || !constant_multiple_p
9790 59 : (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
9791 118 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
9792 : this_lane))
9793 203 : return false;
9794 : return true;
9795 : }
9796 :
9797 : /* Find any vectorizable constructors and add them to the grouped_store
9798 : array. */
9799 :
9800 : static void
9801 2254092 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
9802 : {
9803 18136958 : for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
9804 31765732 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
9805 144474276 : !gsi_end_p (gsi); gsi_next (&gsi))
9806 : {
9807 128591410 : gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
9808 : /* This can be used to start SLP discovery for early breaks for BB early breaks
9809 : when we get that far. */
9810 128591410 : if (!assign)
9811 195067499 : continue;
9812 :
9813 32057645 : tree rhs = gimple_assign_rhs1 (assign);
9814 32057645 : enum tree_code code = gimple_assign_rhs_code (assign);
9815 32057645 : use_operand_p use_p;
9816 32057645 : gimple *use_stmt;
9817 32057645 : if (code == CONSTRUCTOR)
9818 : {
9819 1709165 : if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
9820 68028 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
9821 98111 : CONSTRUCTOR_NELTS (rhs))
9822 45061 : || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
9823 1754206 : || uniform_vector_p (rhs))
9824 1694772 : continue;
9825 :
9826 : unsigned j;
9827 : tree val;
9828 70896 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
9829 56503 : if (TREE_CODE (val) != SSA_NAME
9830 56503 : || !bb_vinfo->lookup_def (val))
9831 : break;
9832 35140 : if (j != CONSTRUCTOR_NELTS (rhs))
9833 3177 : continue;
9834 :
9835 14393 : vec<stmt_vec_info> roots = vNULL;
9836 14393 : roots.safe_push (bb_vinfo->lookup_stmt (assign));
9837 14393 : vec<stmt_vec_info> stmts;
9838 14393 : stmts.create (CONSTRUCTOR_NELTS (rhs));
9839 80322 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
9840 51536 : stmts.quick_push
9841 51536 : (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
9842 14393 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
9843 14393 : stmts, roots));
9844 : }
9845 30348480 : else if (code == BIT_INSERT_EXPR
9846 958 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
9847 632 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
9848 632 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
9849 629 : && integer_zerop (gimple_assign_rhs3 (assign))
9850 341 : && useless_type_conversion_p
9851 341 : (TREE_TYPE (TREE_TYPE (rhs)),
9852 341 : TREE_TYPE (gimple_assign_rhs2 (assign)))
9853 30349102 : && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
9854 : {
9855 : /* We start to match on insert to lane zero but since the
9856 : inserts need not be ordered we'd have to search both
9857 : the def and the use chains. */
9858 220 : tree vectype = TREE_TYPE (rhs);
9859 220 : unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
9860 220 : auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
9861 220 : auto_sbitmap lanes (nlanes);
9862 220 : bitmap_clear (lanes);
9863 220 : bitmap_set_bit (lanes, 0);
9864 220 : tree def = gimple_assign_lhs (assign);
9865 220 : lane_defs.quick_push
9866 220 : (std::make_pair (0, gimple_assign_rhs2 (assign)));
9867 220 : unsigned lanes_found = 1;
9868 : /* Start with the use chains, the last stmt will be the root. */
9869 220 : stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
9870 220 : vec<stmt_vec_info> roots = vNULL;
9871 220 : roots.safe_push (last);
9872 231 : do
9873 : {
9874 231 : use_operand_p use_p;
9875 231 : gimple *use_stmt;
9876 231 : if (!single_imm_use (def, &use_p, &use_stmt))
9877 : break;
9878 225 : unsigned this_lane;
9879 225 : if (!bb_vinfo->lookup_stmt (use_stmt)
9880 225 : || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
9881 260 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
9882 : break;
9883 35 : if (bitmap_bit_p (lanes, this_lane))
9884 : break;
9885 15 : lanes_found++;
9886 15 : bitmap_set_bit (lanes, this_lane);
9887 15 : gassign *use_ass = as_a <gassign *> (use_stmt);
9888 15 : lane_defs.quick_push (std::make_pair
9889 15 : (this_lane, gimple_assign_rhs2 (use_ass)));
9890 15 : last = bb_vinfo->lookup_stmt (use_ass);
9891 15 : roots.safe_push (last);
9892 15 : def = gimple_assign_lhs (use_ass);
9893 : }
9894 15 : while (lanes_found < nlanes);
9895 220 : if (roots.length () > 1)
9896 7 : std::swap(roots[0], roots[roots.length () - 1]);
9897 220 : if (lanes_found < nlanes)
9898 : {
9899 : /* Now search the def chain. */
9900 216 : def = gimple_assign_rhs1 (assign);
9901 218 : do
9902 : {
9903 218 : if (TREE_CODE (def) != SSA_NAME
9904 218 : || !has_single_use (def))
9905 : break;
9906 57 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
9907 57 : unsigned this_lane;
9908 57 : if (!bb_vinfo->lookup_stmt (def_stmt)
9909 37 : || !vect_slp_is_lane_insert (def_stmt,
9910 : NULL_TREE, &this_lane)
9911 81 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
9912 : break;
9913 24 : if (bitmap_bit_p (lanes, this_lane))
9914 : break;
9915 4 : lanes_found++;
9916 4 : bitmap_set_bit (lanes, this_lane);
9917 8 : lane_defs.quick_push (std::make_pair
9918 4 : (this_lane,
9919 4 : gimple_assign_rhs2 (def_stmt)));
9920 4 : roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
9921 4 : def = gimple_assign_rhs1 (def_stmt);
9922 : }
9923 4 : while (lanes_found < nlanes);
9924 : }
9925 220 : if (lanes_found == nlanes)
9926 : {
9927 : /* Sort lane_defs after the lane index and register the root. */
9928 6 : lane_defs.qsort (vld_cmp);
9929 6 : vec<stmt_vec_info> stmts;
9930 6 : stmts.create (nlanes);
9931 30 : for (unsigned i = 0; i < nlanes; ++i)
9932 24 : stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
9933 6 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
9934 6 : stmts, roots));
9935 : }
9936 : else
9937 214 : roots.release ();
9938 220 : }
9939 30348260 : else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
9940 29333190 : && (associative_tree_code (code) || code == MINUS_EXPR)
9941 : /* ??? This pessimizes a two-element reduction. PR54400.
9942 : ??? In-order reduction could be handled if we only
9943 : traverse one operand chain in vect_slp_linearize_chain. */
9944 34402375 : && !needs_fold_left_reduction_p (TREE_TYPE (rhs), code)
9945 : /* Ops with constants at the tail can be stripped here. */
9946 6017071 : && TREE_CODE (rhs) == SSA_NAME
9947 5948706 : && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
9948 : /* Should be the chain end. */
9949 32739293 : && (!single_imm_use (gimple_assign_lhs (assign),
9950 : &use_p, &use_stmt)
9951 1839517 : || !is_gimple_assign (use_stmt)
9952 1249835 : || (gimple_assign_rhs_code (use_stmt) != code
9953 933804 : && ((code != PLUS_EXPR && code != MINUS_EXPR)
9954 518325 : || (gimple_assign_rhs_code (use_stmt)
9955 518325 : != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
9956 : {
9957 : /* We start the match at the end of a possible association
9958 : chain. */
9959 1962956 : auto_vec<chain_op_t> chain;
9960 1962956 : auto_vec<std::pair<tree_code, gimple *> > worklist;
9961 1962956 : auto_vec<gimple *> chain_stmts;
9962 1962956 : gimple *code_stmt = NULL, *alt_code_stmt = NULL;
9963 1962956 : if (code == MINUS_EXPR)
9964 310445 : code = PLUS_EXPR;
9965 1962956 : internal_fn reduc_fn;
9966 2268153 : if (!reduction_fn_for_scalar_code (code, &reduc_fn)
9967 1962956 : || reduc_fn == IFN_LAST)
9968 305197 : continue;
9969 1657759 : vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
9970 : /* ??? */
9971 : code_stmt, alt_code_stmt, &chain_stmts,
9972 : false);
9973 3315518 : if (chain.length () > 1)
9974 : {
9975 : /* Sort the chain according to def_type and operation. */
9976 1657759 : chain.sort (dt_sort_cmp, bb_vinfo);
9977 : /* ??? Now we'd want to strip externals and constants
9978 : but record those to be handled in the epilogue. */
9979 : /* ??? For now do not allow mixing ops or externs/constants. */
9980 1657759 : bool invalid = false;
9981 1657759 : unsigned remain_cnt = 0;
9982 1657759 : unsigned last_idx = 0;
9983 5007931 : for (unsigned i = 0; i < chain.length (); ++i)
9984 : {
9985 3660617 : if (chain[i].code != code)
9986 : {
9987 : invalid = true;
9988 : break;
9989 : }
9990 3350172 : if (chain[i].dt != vect_internal_def
9991 : /* Avoid stmts where the def is not the LHS, like
9992 : ASMs. */
9993 6483522 : || (gimple_get_lhs (bb_vinfo->lookup_def
9994 3133350 : (chain[i].op)->stmt)
9995 3133350 : != chain[i].op))
9996 219766 : remain_cnt++;
9997 : else
9998 : last_idx = i;
9999 : }
10000 : /* Make sure to have an even number of lanes as we later do
10001 : all-or-nothing discovery, not trying to split further. */
10002 1657759 : if ((chain.length () - remain_cnt) & 1)
10003 178489 : remain_cnt++;
10004 1657759 : if (!invalid && chain.length () - remain_cnt > 1)
10005 : {
10006 1279929 : vec<stmt_vec_info> stmts;
10007 1279929 : vec<tree> remain = vNULL;
10008 1279929 : stmts.create (chain.length ());
10009 1279929 : if (remain_cnt > 0)
10010 122244 : remain.create (remain_cnt);
10011 4109933 : for (unsigned i = 0; i < chain.length (); ++i)
10012 : {
10013 2830004 : stmt_vec_info stmt_info;
10014 2830004 : if (chain[i].dt == vect_internal_def
10015 2789715 : && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
10016 2789715 : gimple_get_lhs (stmt_info->stmt) == chain[i].op)
10017 5619635 : && (i != last_idx
10018 1279929 : || (stmts.length () & 1)))
10019 2696370 : stmts.quick_push (stmt_info);
10020 : else
10021 133634 : remain.quick_push (chain[i].op);
10022 : }
10023 1279929 : vec<stmt_vec_info> roots;
10024 1279929 : roots.create (chain_stmts.length ());
10025 2830004 : for (unsigned i = 0; i < chain_stmts.length (); ++i)
10026 1550075 : roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
10027 1279929 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
10028 1279929 : stmts, roots, remain));
10029 : }
10030 : }
10031 1962956 : }
10032 : }
10033 2254092 : }
10034 :
10035 : /* Walk the grouped store chains and replace entries with their
10036 : pattern variant if any. */
10037 :
10038 : static void
10039 629450 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
10040 : {
10041 629450 : stmt_vec_info first_element;
10042 629450 : unsigned i;
10043 :
10044 1536229 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
10045 : {
10046 : /* We also have CTORs in this array. */
10047 906779 : if (!STMT_VINFO_GROUPED_ACCESS (first_element))
10048 0 : continue;
10049 906779 : if (STMT_VINFO_IN_PATTERN_P (first_element))
10050 : {
10051 252 : stmt_vec_info orig = first_element;
10052 252 : first_element = STMT_VINFO_RELATED_STMT (first_element);
10053 252 : DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
10054 252 : DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
10055 252 : DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
10056 252 : DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
10057 252 : vinfo->grouped_stores[i] = first_element;
10058 : }
10059 906779 : stmt_vec_info prev = first_element;
10060 2549177 : while (DR_GROUP_NEXT_ELEMENT (prev))
10061 : {
10062 1642398 : stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
10063 1642398 : if (STMT_VINFO_IN_PATTERN_P (elt))
10064 : {
10065 849 : stmt_vec_info orig = elt;
10066 849 : elt = STMT_VINFO_RELATED_STMT (elt);
10067 849 : DR_GROUP_NEXT_ELEMENT (prev) = elt;
10068 849 : DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
10069 849 : DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
10070 : }
10071 1642398 : DR_GROUP_FIRST_ELEMENT (elt) = first_element;
10072 1642398 : prev = elt;
10073 : }
10074 : }
10075 629450 : }
10076 :
10077 : /* Check if the region described by BB_VINFO can be vectorized, returning
10078 : true if so. When returning false, set FATAL to true if the same failure
10079 : would prevent vectorization at other vector sizes, false if it is still
10080 : worth trying other sizes. N_STMTS is the number of statements in the
10081 : region. */
10082 :
10083 : static bool
10084 2254092 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
10085 : vec<int> *dataref_groups)
10086 : {
10087 2254092 : DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
10088 :
10089 2254092 : slp_instance instance;
10090 2254092 : int i;
10091 :
10092 : /* The first group of checks is independent of the vector size. */
10093 2254092 : fatal = true;
10094 :
10095 : /* Analyze the data references. */
10096 :
10097 2254092 : if (!vect_analyze_data_refs (bb_vinfo, NULL))
10098 : {
10099 0 : if (dump_enabled_p ())
10100 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10101 : "not vectorized: unhandled data-ref in basic "
10102 : "block.\n");
10103 0 : return false;
10104 : }
10105 :
10106 2254092 : if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
10107 : {
10108 0 : if (dump_enabled_p ())
10109 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10110 : "not vectorized: unhandled data access in "
10111 : "basic block.\n");
10112 0 : return false;
10113 : }
10114 :
10115 2254092 : vect_slp_check_for_roots (bb_vinfo);
10116 :
10117 : /* If there are no grouped stores and no constructors in the region
10118 : there is no need to continue with pattern recog as vect_analyze_slp
10119 : will fail anyway. */
10120 2254092 : if (bb_vinfo->grouped_stores.is_empty ()
10121 1904817 : && bb_vinfo->roots.is_empty ())
10122 : {
10123 1624642 : if (dump_enabled_p ())
10124 1026 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10125 : "not vectorized: no grouped stores in "
10126 : "basic block.\n");
10127 1624642 : return false;
10128 : }
10129 :
10130 : /* While the rest of the analysis below depends on it in some way. */
10131 629450 : fatal = false;
10132 :
10133 629450 : vect_pattern_recog (bb_vinfo);
10134 :
10135 : /* Update store groups from pattern processing. */
10136 629450 : vect_fixup_store_groups_with_patterns (bb_vinfo);
10137 :
10138 : /* Check the SLP opportunities in the basic block, analyze and build SLP
10139 : trees. */
10140 629450 : if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
10141 : {
10142 0 : if (dump_enabled_p ())
10143 : {
10144 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10145 : "Failed to SLP the basic block.\n");
10146 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10147 : "not vectorized: failed to find SLP opportunities "
10148 : "in basic block.\n");
10149 : }
10150 0 : return false;
10151 : }
10152 :
10153 : /* Optimize permutations. */
10154 629450 : vect_optimize_slp (bb_vinfo);
10155 :
10156 : /* Gather the loads reachable from the SLP graph entries. */
10157 629450 : vect_gather_slp_loads (bb_vinfo);
10158 :
10159 629450 : vect_record_base_alignments (bb_vinfo);
10160 :
10161 : /* Analyze and verify the alignment of data references and the
10162 : dependence in the SLP instances. */
10163 1438404 : for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
10164 : {
10165 808954 : vect_location = instance->location ();
10166 808954 : if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
10167 808954 : || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
10168 : {
10169 8733 : slp_tree node = SLP_INSTANCE_TREE (instance);
10170 8733 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10171 8733 : if (dump_enabled_p ())
10172 4 : dump_printf_loc (MSG_NOTE, vect_location,
10173 : "removing SLP instance operations starting from: %G",
10174 : stmt_info->stmt);
10175 8733 : vect_free_slp_instance (instance);
10176 8733 : BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
10177 8733 : continue;
10178 8733 : }
10179 :
10180 : /* Mark all the statements that we want to vectorize as relevant. */
10181 800221 : vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
10182 :
10183 800221 : i++;
10184 : }
10185 2286857 : if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
10186 : return false;
10187 :
10188 271853 : if (!vect_slp_analyze_operations (bb_vinfo))
10189 : {
10190 32765 : if (dump_enabled_p ())
10191 93 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10192 : "not vectorized: bad operation in basic block.\n");
10193 32765 : return false;
10194 : }
10195 :
10196 : /* Mark all the statements that we vectorize. */
10197 239088 : vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
10198 :
10199 : /* Compute vectorizable live stmts. */
10200 239088 : vect_bb_slp_mark_live_stmts (bb_vinfo);
10201 :
10202 239088 : vect_bb_partition_graph (bb_vinfo);
10203 :
10204 239088 : return true;
10205 : }
10206 :
10207 : /* Subroutine of vect_slp_bb. Try to vectorize the statements for all
10208 : basic blocks in BBS, returning true on success.
10209 : The region has N_STMTS statements and has the datarefs given by DATAREFS. */
10210 :
10211 : static bool
10212 1922664 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
10213 : vec<int> *dataref_groups, unsigned int n_stmts,
10214 : loop_p orig_loop)
10215 : {
10216 1922664 : bb_vec_info bb_vinfo;
10217 1922664 : auto_vector_modes vector_modes;
10218 :
10219 : /* Autodetect first vector size we try. */
10220 1922664 : machine_mode next_vector_mode = VOIDmode;
10221 1922664 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
10222 1922664 : unsigned int mode_i = 0;
10223 :
10224 1922664 : vec_info_shared shared;
10225 :
10226 1922664 : machine_mode autodetected_vector_mode = VOIDmode;
10227 2585520 : while (1)
10228 : {
10229 2254092 : bool vectorized = false;
10230 2254092 : bool fatal = false;
10231 2254092 : bb_vinfo = new _bb_vec_info (bbs, &shared);
10232 :
10233 2254092 : bool first_time_p = shared.datarefs.is_empty ();
10234 2254092 : BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
10235 2254092 : if (first_time_p)
10236 1946284 : bb_vinfo->shared->save_datarefs ();
10237 : else
10238 307808 : bb_vinfo->shared->check_datarefs ();
10239 2254092 : bb_vinfo->vector_mode = next_vector_mode;
10240 :
10241 2254092 : if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
10242 : {
10243 239088 : if (dump_enabled_p ())
10244 : {
10245 1520 : dump_printf_loc (MSG_NOTE, vect_location,
10246 : "***** Analysis succeeded with vector mode"
10247 760 : " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
10248 760 : dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
10249 : }
10250 :
10251 239088 : bb_vinfo->shared->check_datarefs ();
10252 :
10253 239088 : bool force_clear = false;
10254 239088 : auto_vec<slp_instance> profitable_subgraphs;
10255 1410181 : for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
10256 : {
10257 692917 : if (instance->subgraph_entries.is_empty ())
10258 227254 : continue;
10259 :
10260 670805 : dump_user_location_t saved_vect_location = vect_location;
10261 670805 : vect_location = instance->location ();
10262 670805 : if (!unlimited_cost_model (NULL)
10263 667456 : && !param_vect_allow_possibly_not_worthwhile_vectorizations
10264 1338256 : && !vect_bb_vectorization_profitable_p
10265 667451 : (bb_vinfo, instance->subgraph_entries, orig_loop))
10266 : {
10267 183030 : if (dump_enabled_p ())
10268 32 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10269 : "not vectorized: vectorization is not "
10270 : "profitable.\n");
10271 183030 : vect_location = saved_vect_location;
10272 183030 : continue;
10273 : }
10274 :
10275 487775 : vect_location = saved_vect_location;
10276 487775 : if (!dbg_cnt (vect_slp))
10277 : {
10278 0 : force_clear = true;
10279 0 : continue;
10280 : }
10281 :
10282 487775 : profitable_subgraphs.safe_push (instance);
10283 : }
10284 :
10285 : /* When we're vectorizing an if-converted loop body make sure
10286 : we vectorized all if-converted code. */
10287 400151 : if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
10288 : {
10289 106 : gcc_assert (bb_vinfo->nbbs == 1);
10290 212 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
10291 4390 : !gsi_end_p (gsi); gsi_next (&gsi))
10292 : {
10293 : /* The costing above left us with DCEable vectorized scalar
10294 : stmts having the visited flag set on profitable
10295 : subgraphs. Do the delayed clearing of the flag here. */
10296 4284 : if (gimple_visited_p (gsi_stmt (gsi)))
10297 : {
10298 1260 : gimple_set_visited (gsi_stmt (gsi), false);
10299 1260 : continue;
10300 : }
10301 3024 : if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
10302 813 : continue;
10303 :
10304 6338 : if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
10305 2670 : if (gimple_assign_rhs_code (ass) == COND_EXPR)
10306 : {
10307 69 : if (!profitable_subgraphs.is_empty ()
10308 31 : && dump_enabled_p ())
10309 0 : dump_printf_loc (MSG_NOTE, vect_location,
10310 : "not profitable because of "
10311 : "unprofitable if-converted scalar "
10312 : "code\n");
10313 38 : profitable_subgraphs.truncate (0);
10314 : }
10315 : }
10316 : }
10317 :
10318 : /* Finally schedule the profitable subgraphs. */
10319 1048943 : for (slp_instance instance : profitable_subgraphs)
10320 : {
10321 487729 : if (!vectorized && dump_enabled_p ())
10322 734 : dump_printf_loc (MSG_NOTE, vect_location,
10323 : "Basic block will be vectorized "
10324 : "using SLP\n");
10325 487729 : vectorized = true;
10326 :
10327 : /* Dump before scheduling as store vectorization will remove
10328 : the original stores and mess with the instance tree
10329 : so querying its location will eventually ICE. */
10330 487729 : if (flag_checking)
10331 1962413 : for (slp_instance sub : instance->subgraph_entries)
10332 499226 : gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
10333 487729 : unsigned HOST_WIDE_INT bytes;
10334 487729 : if (dump_enabled_p ())
10335 3489 : for (slp_instance sub : instance->subgraph_entries)
10336 : {
10337 924 : tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
10338 1848 : if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
10339 924 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10340 924 : sub->location (),
10341 : "basic block part vectorized using %wu "
10342 : "byte vectors\n", bytes);
10343 : else
10344 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10345 : sub->location (),
10346 : "basic block part vectorized using "
10347 : "variable length vectors\n");
10348 : }
10349 :
10350 487729 : dump_user_location_t saved_vect_location = vect_location;
10351 487729 : vect_location = instance->location ();
10352 :
10353 487729 : vect_schedule_slp (bb_vinfo, instance->subgraph_entries);
10354 :
10355 487729 : vect_location = saved_vect_location;
10356 : }
10357 :
10358 :
10359 : /* Generate the invariant statements. */
10360 239088 : if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
10361 : {
10362 23 : if (dump_enabled_p ())
10363 0 : dump_printf_loc (MSG_NOTE, vect_location,
10364 : "------>generating invariant statements\n");
10365 :
10366 23 : bb_vinfo->insert_seq_on_entry (NULL,
10367 : bb_vinfo->inv_pattern_def_seq);
10368 : }
10369 239088 : }
10370 : else
10371 : {
10372 2015004 : if (dump_enabled_p ())
10373 1312 : dump_printf_loc (MSG_NOTE, vect_location,
10374 : "***** Analysis failed with vector mode %s\n",
10375 1312 : GET_MODE_NAME (bb_vinfo->vector_mode));
10376 : }
10377 :
10378 2254092 : if (mode_i == 0)
10379 1922664 : autodetected_vector_mode = bb_vinfo->vector_mode;
10380 :
10381 2254092 : if (!fatal)
10382 3225291 : while (mode_i < vector_modes.length ()
10383 1805363 : && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
10384 : {
10385 341749 : if (dump_enabled_p ())
10386 1666 : dump_printf_loc (MSG_NOTE, vect_location,
10387 : "***** The result for vector mode %s would"
10388 : " be the same\n",
10389 833 : GET_MODE_NAME (vector_modes[mode_i]));
10390 341749 : mode_i += 1;
10391 : }
10392 :
10393 2254092 : delete bb_vinfo;
10394 :
10395 2254092 : if (mode_i < vector_modes.length ()
10396 2071276 : && VECTOR_MODE_P (autodetected_vector_mode)
10397 2052786 : && (related_vector_mode (vector_modes[mode_i],
10398 : GET_MODE_INNER (autodetected_vector_mode))
10399 1026393 : == autodetected_vector_mode)
10400 4325368 : && (related_vector_mode (autodetected_vector_mode,
10401 533503 : GET_MODE_INNER (vector_modes[mode_i]))
10402 1067006 : == vector_modes[mode_i]))
10403 : {
10404 533503 : if (dump_enabled_p ())
10405 207 : dump_printf_loc (MSG_NOTE, vect_location,
10406 : "***** Skipping vector mode %s, which would"
10407 : " repeat the analysis for %s\n",
10408 207 : GET_MODE_NAME (vector_modes[mode_i]),
10409 207 : GET_MODE_NAME (autodetected_vector_mode));
10410 533503 : mode_i += 1;
10411 : }
10412 :
10413 2254092 : if (vectorized
10414 2093060 : || mode_i == vector_modes.length ()
10415 1910289 : || autodetected_vector_mode == VOIDmode
10416 : /* If vect_slp_analyze_bb_1 signaled that analysis for all
10417 : vector sizes will fail do not bother iterating. */
10418 3119498 : || fatal)
10419 3845328 : return vectorized;
10420 :
10421 : /* Try the next biggest vector size. */
10422 331428 : next_vector_mode = vector_modes[mode_i++];
10423 331428 : if (dump_enabled_p ())
10424 215 : dump_printf_loc (MSG_NOTE, vect_location,
10425 : "***** Re-trying analysis with vector mode %s\n",
10426 215 : GET_MODE_NAME (next_vector_mode));
10427 331428 : }
10428 1922664 : }
10429 :
10430 :
10431 : /* Main entry for the BB vectorizer. Analyze and transform BBS, returns
10432 : true if anything in the basic-block was vectorized. */
10433 :
10434 : static bool
10435 1922664 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
10436 : {
10437 1922664 : vec<data_reference_p> datarefs = vNULL;
10438 1922664 : auto_vec<int> dataref_groups;
10439 1922664 : int insns = 0;
10440 1922664 : int current_group = 0;
10441 :
10442 12747958 : for (unsigned i = 0; i < bbs.length (); i++)
10443 : {
10444 10825294 : basic_block bb = bbs[i];
10445 93582285 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
10446 82756991 : gsi_next (&gsi))
10447 : {
10448 82756991 : gimple *stmt = gsi_stmt (gsi);
10449 82756991 : if (is_gimple_debug (stmt))
10450 52202939 : continue;
10451 :
10452 30554052 : insns++;
10453 :
10454 30554052 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
10455 27425903 : vect_location = stmt;
10456 :
10457 30554052 : if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
10458 : &dataref_groups, current_group))
10459 5222996 : ++current_group;
10460 : }
10461 : /* New BBs always start a new DR group. */
10462 10825294 : ++current_group;
10463 : }
10464 :
10465 1922664 : return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
10466 1922664 : }
10467 :
10468 : /* Special entry for the BB vectorizer. Analyze and transform a single
10469 : if-converted BB with ORIG_LOOPs body being the not if-converted
10470 : representation. Returns true if anything in the basic-block was
10471 : vectorized. */
10472 :
10473 : bool
10474 19425 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
10475 : {
10476 19425 : auto_vec<basic_block> bbs;
10477 19425 : bbs.safe_push (bb);
10478 19425 : return vect_slp_bbs (bbs, orig_loop);
10479 19425 : }
10480 :
10481 : /* Main entry for the BB vectorizer. Analyze and transform BB, returns
10482 : true if anything in the basic-block was vectorized. */
10483 :
10484 : bool
10485 921021 : vect_slp_function (function *fun)
10486 : {
10487 921021 : bool r = false;
10488 921021 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
10489 921021 : auto_bitmap exit_bbs;
10490 921021 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
10491 921021 : edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
10492 921021 : unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
10493 921021 : true, rpo, NULL);
10494 :
10495 : /* For the moment split the function into pieces to avoid making
10496 : the iteration on the vector mode moot. Split at points we know
10497 : to not handle well which is CFG merges (SLP discovery doesn't
10498 : handle non-loop-header PHIs) and loop exits. Since pattern
10499 : recog requires reverse iteration to visit uses before defs
10500 : simply chop RPO into pieces. */
10501 921021 : auto_vec<basic_block> bbs;
10502 11757804 : for (unsigned i = 0; i < n; i++)
10503 : {
10504 10836783 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
10505 10836783 : bool split = false;
10506 :
10507 : /* Split when a BB is not dominated by the first block. */
10508 20440427 : if (!bbs.is_empty ()
10509 9603644 : && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
10510 : {
10511 689889 : if (dump_enabled_p ())
10512 146 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10513 : "splitting region at dominance boundary bb%d\n",
10514 : bb->index);
10515 : split = true;
10516 : }
10517 : /* Split when the loop determined by the first block
10518 : is exited. This is because we eventually insert
10519 : invariants at region begin. */
10520 19060649 : else if (!bbs.is_empty ()
10521 8913755 : && bbs[0]->loop_father != bb->loop_father
10522 2313890 : && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
10523 : {
10524 3834 : if (dump_enabled_p ())
10525 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10526 : "splitting region at loop %d exit at bb%d\n",
10527 3 : bbs[0]->loop_father->num, bb->index);
10528 : split = true;
10529 : }
10530 10143060 : else if (!bbs.is_empty ()
10531 8909921 : && bb->loop_father->header == bb
10532 474781 : && bb->loop_father->dont_vectorize)
10533 : {
10534 7291 : if (dump_enabled_p ())
10535 72 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10536 : "splitting region at dont-vectorize loop %d "
10537 : "entry at bb%d\n",
10538 : bb->loop_father->num, bb->index);
10539 : split = true;
10540 : }
10541 :
10542 11537797 : if (split && !bbs.is_empty ())
10543 : {
10544 701014 : r |= vect_slp_bbs (bbs, NULL);
10545 701014 : bbs.truncate (0);
10546 : }
10547 :
10548 10836783 : if (bbs.is_empty ())
10549 : {
10550 : /* We need to be able to insert at the head of the region which
10551 : we cannot for region starting with a returns-twice call. */
10552 1934153 : if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
10553 409129 : if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
10554 : {
10555 306 : if (dump_enabled_p ())
10556 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10557 : "skipping bb%d as start of region as it "
10558 : "starts with returns-twice call\n",
10559 : bb->index);
10560 30914 : continue;
10561 : }
10562 : /* If the loop this BB belongs to is marked as not to be vectorized
10563 : honor that also for BB vectorization. */
10564 1933847 : if (bb->loop_father->dont_vectorize)
10565 30608 : continue;
10566 : }
10567 :
10568 10805869 : bbs.safe_push (bb);
10569 :
10570 : /* When we have a stmt ending this block and defining a
10571 : value we have to insert on edges when inserting after it for
10572 : a vector containing its definition. Avoid this for now. */
10573 21611738 : if (gimple *last = *gsi_last_bb (bb))
10574 8787110 : if (gimple_get_lhs (last)
10575 8787110 : && is_ctrl_altering_stmt (last))
10576 : {
10577 281211 : if (dump_enabled_p ())
10578 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10579 : "splitting region at control altering "
10580 : "definition %G", last);
10581 281211 : r |= vect_slp_bbs (bbs, NULL);
10582 281211 : bbs.truncate (0);
10583 : }
10584 : }
10585 :
10586 921021 : if (!bbs.is_empty ())
10587 921014 : r |= vect_slp_bbs (bbs, NULL);
10588 :
10589 921021 : free (rpo);
10590 :
10591 921021 : return r;
10592 921021 : }
10593 :
10594 : /* Build a variable-length vector in which the elements in ELTS are repeated
10595 : to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
10596 : RESULTS and add any new instructions to SEQ.
10597 :
10598 : The approach we use is:
10599 :
10600 : (1) Find a vector mode VM with integer elements of mode IM.
10601 :
10602 : (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10603 : ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
10604 : from small vectors to IM.
10605 :
10606 : (3) Duplicate each ELTS'[I] into a vector of mode VM.
10607 :
10608 : (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
10609 : correct byte contents.
10610 :
10611 : (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
10612 :
10613 : We try to find the largest IM for which this sequence works, in order
10614 : to cut down on the number of interleaves. */
10615 :
10616 : void
10617 0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
10618 : const vec<tree> &elts, unsigned int nresults,
10619 : vec<tree> &results)
10620 : {
10621 0 : unsigned int nelts = elts.length ();
10622 0 : tree element_type = TREE_TYPE (vector_type);
10623 :
10624 : /* (1) Find a vector mode VM with integer elements of mode IM. */
10625 0 : unsigned int nvectors = 1;
10626 0 : tree new_vector_type;
10627 0 : tree permutes[2];
10628 0 : if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
10629 : &nvectors, &new_vector_type,
10630 : permutes))
10631 0 : gcc_unreachable ();
10632 :
10633 : /* Get a vector type that holds ELTS[0:NELTS/NELTS']. */
10634 0 : unsigned int partial_nelts = nelts / nvectors;
10635 0 : tree partial_vector_type = build_vector_type (element_type, partial_nelts);
10636 :
10637 0 : tree_vector_builder partial_elts;
10638 0 : auto_vec<tree, 32> pieces (nvectors * 2);
10639 0 : pieces.quick_grow_cleared (nvectors * 2);
10640 0 : for (unsigned int i = 0; i < nvectors; ++i)
10641 : {
10642 : /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10643 : ELTS' has mode IM. */
10644 0 : partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
10645 0 : for (unsigned int j = 0; j < partial_nelts; ++j)
10646 0 : partial_elts.quick_push (elts[i * partial_nelts + j]);
10647 0 : tree t = gimple_build_vector (seq, &partial_elts);
10648 0 : t = gimple_build (seq, VIEW_CONVERT_EXPR,
10649 0 : TREE_TYPE (new_vector_type), t);
10650 :
10651 : /* (3) Duplicate each ELTS'[I] into a vector of mode VM. */
10652 0 : pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
10653 : }
10654 :
10655 : /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
10656 : correct byte contents.
10657 :
10658 : Conceptually, we need to repeat the following operation log2(nvectors)
10659 : times, where hi_start = nvectors / 2:
10660 :
10661 : out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
10662 : out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
10663 :
10664 : However, if each input repeats every N elements and the VF is
10665 : a multiple of N * 2, the HI result is the same as the LO result.
10666 : This will be true for the first N1 iterations of the outer loop,
10667 : followed by N2 iterations for which both the LO and HI results
10668 : are needed. I.e.:
10669 :
10670 : N1 + N2 = log2(nvectors)
10671 :
10672 : Each "N1 iteration" doubles the number of redundant vectors and the
10673 : effect of the process as a whole is to have a sequence of nvectors/2**N1
10674 : vectors that repeats 2**N1 times. Rather than generate these redundant
10675 : vectors, we halve the number of vectors for each N1 iteration. */
10676 : unsigned int in_start = 0;
10677 : unsigned int out_start = nvectors;
10678 : unsigned int new_nvectors = nvectors;
10679 0 : for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
10680 : {
10681 0 : unsigned int hi_start = new_nvectors / 2;
10682 0 : unsigned int out_i = 0;
10683 0 : for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
10684 : {
10685 0 : if ((in_i & 1) != 0
10686 0 : && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
10687 : 2 * in_repeat))
10688 0 : continue;
10689 :
10690 0 : tree output = make_ssa_name (new_vector_type);
10691 0 : tree input1 = pieces[in_start + (in_i / 2)];
10692 0 : tree input2 = pieces[in_start + (in_i / 2) + hi_start];
10693 0 : gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
10694 : input1, input2,
10695 : permutes[in_i & 1]);
10696 0 : gimple_seq_add_stmt (seq, stmt);
10697 0 : pieces[out_start + out_i] = output;
10698 0 : out_i += 1;
10699 : }
10700 0 : std::swap (in_start, out_start);
10701 0 : new_nvectors = out_i;
10702 : }
10703 :
10704 : /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type. */
10705 0 : results.reserve (nresults);
10706 0 : for (unsigned int i = 0; i < nresults; ++i)
10707 0 : if (i < new_nvectors)
10708 0 : results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
10709 0 : pieces[in_start + i]));
10710 : else
10711 0 : results.quick_push (results[i - new_nvectors]);
10712 0 : }
10713 :
10714 :
10715 : /* For constant and loop invariant defs in OP_NODE this function creates
10716 : vector defs that will be used in the vectorized stmts and stores them
10717 : to SLP_TREE_VEC_DEFS of OP_NODE. */
10718 :
10719 : static void
10720 496541 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
10721 : {
10722 496541 : unsigned HOST_WIDE_INT nunits;
10723 496541 : tree vec_cst;
10724 496541 : unsigned j, number_of_places_left_in_vector;
10725 496541 : tree vector_type;
10726 496541 : tree vop;
10727 496541 : int group_size = op_node->ops.length ();
10728 496541 : unsigned int vec_num, i;
10729 496541 : unsigned number_of_copies = 1;
10730 496541 : bool constant_p;
10731 496541 : gimple_seq ctor_seq = NULL;
10732 496541 : auto_vec<tree, 16> permute_results;
10733 :
10734 : /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
10735 496541 : vector_type = SLP_TREE_VECTYPE (op_node);
10736 :
10737 496541 : unsigned int number_of_vectors = vect_get_num_copies (vinfo, op_node);
10738 496541 : SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
10739 496541 : auto_vec<tree> voprnds (number_of_vectors);
10740 :
10741 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
10742 : created vectors. It is greater than 1 if unrolling is performed.
10743 :
10744 : For example, we have two scalar operands, s1 and s2 (e.g., group of
10745 : strided accesses of size two), while NUNITS is four (i.e., four scalars
10746 : of this type can be packed in a vector). The output vector will contain
10747 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
10748 : will be 2).
10749 :
10750 : If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
10751 : containing the operands.
10752 :
10753 : For example, NUNITS is four as before, and the group size is 8
10754 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
10755 : {s5, s6, s7, s8}. */
10756 :
10757 : /* When using duplicate_and_interleave, we just need one element for
10758 : each scalar statement. */
10759 496541 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
10760 : nunits = group_size;
10761 :
10762 496541 : number_of_copies = nunits * number_of_vectors / group_size;
10763 :
10764 496541 : number_of_places_left_in_vector = nunits;
10765 496541 : constant_p = true;
10766 496541 : tree uniform_elt = NULL_TREE;
10767 496541 : tree_vector_builder elts (vector_type, nunits, 1);
10768 496541 : elts.quick_grow (nunits);
10769 496541 : stmt_vec_info insert_after = NULL;
10770 1477121 : for (j = 0; j < number_of_copies; j++)
10771 : {
10772 980580 : tree op;
10773 3764106 : for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
10774 : {
10775 : /* Create 'vect_ = {op0,op1,...,opn}'. */
10776 1802946 : tree orig_op = op;
10777 1802946 : if (number_of_places_left_in_vector == nunits)
10778 : uniform_elt = op;
10779 1175237 : else if (uniform_elt && operand_equal_p (uniform_elt, op))
10780 745598 : op = elts[number_of_places_left_in_vector];
10781 : else
10782 : uniform_elt = NULL_TREE;
10783 1802946 : number_of_places_left_in_vector--;
10784 1802946 : if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
10785 : {
10786 281149 : if (CONSTANT_CLASS_P (op))
10787 : {
10788 103183 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10789 : {
10790 : /* Can't use VIEW_CONVERT_EXPR for booleans because
10791 : of possibly different sizes of scalar value and
10792 : vector element. */
10793 65 : if (integer_zerop (op))
10794 65 : op = build_int_cst (TREE_TYPE (vector_type), 0);
10795 0 : else if (integer_onep (op))
10796 0 : op = build_all_ones_cst (TREE_TYPE (vector_type));
10797 : else
10798 0 : gcc_unreachable ();
10799 : }
10800 : else
10801 103118 : op = fold_unary (VIEW_CONVERT_EXPR,
10802 : TREE_TYPE (vector_type), op);
10803 103183 : gcc_assert (op && CONSTANT_CLASS_P (op));
10804 : }
10805 : else
10806 : {
10807 177966 : tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
10808 177966 : gimple *init_stmt;
10809 177966 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
10810 : {
10811 427 : tree true_val
10812 427 : = build_all_ones_cst (TREE_TYPE (vector_type));
10813 427 : tree false_val
10814 427 : = build_zero_cst (TREE_TYPE (vector_type));
10815 427 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
10816 427 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
10817 : op, true_val,
10818 : false_val);
10819 : }
10820 : else
10821 : {
10822 177539 : op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
10823 : op);
10824 177539 : init_stmt
10825 177539 : = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
10826 : op);
10827 : }
10828 177966 : gimple_seq_add_stmt (&ctor_seq, init_stmt);
10829 177966 : op = new_temp;
10830 : }
10831 : }
10832 1802946 : elts[number_of_places_left_in_vector] = op;
10833 1802946 : if (!CONSTANT_CLASS_P (op))
10834 319095 : constant_p = false;
10835 : /* For BB vectorization we have to compute an insert location
10836 : when a def is inside the analyzed region since we cannot
10837 : simply insert at the BB start in this case. */
10838 1802946 : stmt_vec_info opdef;
10839 1802946 : if (TREE_CODE (orig_op) == SSA_NAME
10840 182058 : && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
10841 162012 : && is_a <bb_vec_info> (vinfo)
10842 1907281 : && (opdef = vinfo->lookup_def (orig_op)))
10843 : {
10844 84581 : if (!insert_after)
10845 : insert_after = opdef;
10846 : else
10847 46570 : insert_after = get_later_stmt (insert_after, opdef);
10848 : }
10849 :
10850 1802946 : if (number_of_places_left_in_vector == 0)
10851 : {
10852 627709 : auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
10853 627709 : if (uniform_elt)
10854 653070 : vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
10855 326535 : elts[0]);
10856 602348 : else if (constant_p
10857 602348 : ? multiple_p (type_nunits, nunits)
10858 110618 : : known_eq (type_nunits, nunits))
10859 301174 : vec_cst = gimple_build_vector (&ctor_seq, &elts);
10860 : else
10861 : {
10862 0 : if (permute_results.is_empty ())
10863 0 : duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
10864 : elts, number_of_vectors,
10865 : permute_results);
10866 0 : vec_cst = permute_results[number_of_vectors - j - 1];
10867 : }
10868 627709 : if (!gimple_seq_empty_p (ctor_seq))
10869 : {
10870 138097 : if (insert_after)
10871 : {
10872 38011 : gimple_stmt_iterator gsi;
10873 38011 : if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
10874 : {
10875 679 : gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
10876 679 : gsi_insert_seq_before (&gsi, ctor_seq,
10877 : GSI_CONTINUE_LINKING);
10878 : }
10879 37332 : else if (!stmt_ends_bb_p (insert_after->stmt))
10880 : {
10881 37332 : gsi = gsi_for_stmt (insert_after->stmt);
10882 37332 : gsi_insert_seq_after (&gsi, ctor_seq,
10883 : GSI_CONTINUE_LINKING);
10884 : }
10885 : else
10886 : {
10887 : /* When we want to insert after a def where the
10888 : defining stmt throws then insert on the fallthru
10889 : edge. */
10890 0 : edge e = find_fallthru_edge
10891 0 : (gimple_bb (insert_after->stmt)->succs);
10892 0 : basic_block new_bb
10893 0 : = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
10894 0 : gcc_assert (!new_bb);
10895 : }
10896 : }
10897 : else
10898 100086 : vinfo->insert_seq_on_entry (NULL, ctor_seq);
10899 138097 : ctor_seq = NULL;
10900 : }
10901 627709 : voprnds.quick_push (vec_cst);
10902 627709 : insert_after = NULL;
10903 627709 : number_of_places_left_in_vector = nunits;
10904 627709 : constant_p = true;
10905 627709 : elts.new_vector (vector_type, nunits, 1);
10906 627709 : elts.quick_grow (nunits);
10907 : }
10908 : }
10909 : }
10910 :
10911 : /* Since the vectors are created in the reverse order, we should invert
10912 : them. */
10913 496541 : vec_num = voprnds.length ();
10914 1124250 : for (j = vec_num; j != 0; j--)
10915 : {
10916 627709 : vop = voprnds[j - 1];
10917 627709 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
10918 : }
10919 :
10920 : /* In case that VF is greater than the unrolling factor needed for the SLP
10921 : group of stmts, NUMBER_OF_VECTORS to be created is greater than
10922 : NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
10923 : to replicate the vectors. */
10924 496541 : while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
10925 496541 : for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
10926 : i++)
10927 0 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
10928 496541 : }
10929 :
10930 : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
10931 : if there is no definition for it in the scalar IL or it is not known. */
10932 :
10933 : tree
10934 2629 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
10935 : {
10936 2629 : if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
10937 : {
10938 2617 : if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
10939 : return NULL_TREE;
10940 2617 : stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
10941 2617 : if (!def)
10942 : return NULL_TREE;
10943 2617 : return gimple_get_lhs (STMT_VINFO_STMT (def));
10944 : }
10945 : else
10946 12 : return SLP_TREE_SCALAR_OPS (slp_node)[n];
10947 : }
10948 :
10949 : /* Get the Ith vectorized definition from SLP_NODE. */
10950 :
10951 : tree
10952 146411 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
10953 : {
10954 146411 : return SLP_TREE_VEC_DEFS (slp_node)[i];
10955 : }
10956 :
10957 : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
10958 :
10959 : void
10960 940121 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
10961 : {
10962 1880242 : vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
10963 940121 : vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
10964 940121 : }
10965 :
10966 : /* Get N vectorized definitions for SLP_NODE. */
10967 :
10968 : void
10969 2939 : vect_get_slp_defs (vec_info *,
10970 : slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
10971 : {
10972 2939 : if (n == -1U)
10973 2939 : n = SLP_TREE_CHILDREN (slp_node).length ();
10974 :
10975 10611 : for (unsigned i = 0; i < n; ++i)
10976 : {
10977 7672 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
10978 7672 : vec<tree> vec_defs = vNULL;
10979 7672 : vect_get_slp_defs (child, &vec_defs);
10980 7672 : vec_oprnds->quick_push (vec_defs);
10981 : }
10982 2939 : }
10983 :
10984 : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
10985 : - PERM gives the permutation that the caller wants to use for NODE,
10986 : which might be different from SLP_LOAD_PERMUTATION.
10987 : - DUMP_P controls whether the function dumps information. */
10988 :
10989 : static bool
10990 126673 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
10991 : load_permutation_t &perm,
10992 : const vec<tree> &dr_chain,
10993 : gimple_stmt_iterator *gsi, poly_uint64 vf,
10994 : bool analyze_only, bool dump_p,
10995 : unsigned *n_perms, unsigned int *n_loads,
10996 : bool dce_chain)
10997 : {
10998 126673 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10999 126673 : int vec_index = 0;
11000 126673 : tree vectype = SLP_TREE_VECTYPE (node);
11001 126673 : unsigned int group_size = SLP_TREE_SCALAR_STMTS (node).length ();
11002 126673 : unsigned int mask_element;
11003 126673 : unsigned dr_group_size;
11004 126673 : machine_mode mode;
11005 :
11006 126673 : if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
11007 : {
11008 : /* We have both splats of the same non-grouped load and groups
11009 : of distinct invariant loads entering here. */
11010 1485 : unsigned max_idx = 0;
11011 8231 : for (auto idx : perm)
11012 3776 : max_idx = idx > max_idx ? idx : max_idx;
11013 1485 : dr_group_size = max_idx + 1;
11014 : }
11015 : else
11016 : {
11017 125188 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
11018 125188 : dr_group_size = DR_GROUP_SIZE (stmt_info);
11019 : }
11020 :
11021 126673 : mode = TYPE_MODE (vectype);
11022 126673 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11023 126673 : unsigned int nstmts = vect_get_num_copies (vinfo, node);
11024 :
11025 : /* Initialize the vect stmts of NODE to properly insert the generated
11026 : stmts later. */
11027 126673 : if (! analyze_only)
11028 58430 : for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
11029 22488 : SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
11030 :
11031 : /* Generate permutation masks for every NODE. Number of masks for each NODE
11032 : is equal to GROUP_SIZE.
11033 : E.g., we have a group of three nodes with three loads from the same
11034 : location in each node, and the vector size is 4. I.e., we have a
11035 : a0b0c0a1b1c1... sequence and we need to create the following vectors:
11036 : for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
11037 : for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
11038 : ...
11039 :
11040 : The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
11041 : The last mask is illegal since we assume two operands for permute
11042 : operation, and the mask element values can't be outside that range.
11043 : Hence, the last mask must be converted into {2,5,5,5}.
11044 : For the first two permutations we need the first and the second input
11045 : vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
11046 : we need the second and the third vectors: {b1,c1,a2,b2} and
11047 : {c2,a3,b3,c3}. */
11048 :
11049 126673 : int vect_stmts_counter = 0;
11050 126673 : unsigned int index = 0;
11051 126673 : int first_vec_index = -1;
11052 126673 : int second_vec_index = -1;
11053 126673 : bool noop_p = true;
11054 126673 : *n_perms = 0;
11055 :
11056 126673 : vec_perm_builder mask;
11057 126673 : unsigned int nelts_to_build;
11058 126673 : unsigned int nvectors_per_build;
11059 126673 : unsigned int in_nlanes;
11060 126673 : bool repeating_p = (group_size == dr_group_size
11061 160602 : && multiple_p (nunits, group_size));
11062 126673 : if (repeating_p)
11063 : {
11064 : /* A single vector contains a whole number of copies of the node, so:
11065 : (a) all permutes can use the same mask; and
11066 : (b) the permutes only need a single vector input. */
11067 31627 : mask.new_vector (nunits, group_size, 3);
11068 31627 : nelts_to_build = mask.encoded_nelts ();
11069 : /* It's possible to obtain zero nstmts during analyze_only, so make
11070 : it at least one to ensure the later computation for n_perms
11071 : proceed. */
11072 31627 : nvectors_per_build = nstmts > 0 ? nstmts : 1;
11073 31627 : in_nlanes = dr_group_size * 3;
11074 : }
11075 : else
11076 : {
11077 : /* We need to construct a separate mask for each vector statement. */
11078 95046 : unsigned HOST_WIDE_INT const_nunits, const_vf;
11079 95046 : if (!nunits.is_constant (&const_nunits)
11080 95046 : || !vf.is_constant (&const_vf))
11081 : return false;
11082 95046 : mask.new_vector (const_nunits, const_nunits, 1);
11083 95046 : nelts_to_build = const_vf * group_size;
11084 95046 : nvectors_per_build = 1;
11085 95046 : in_nlanes = const_vf * dr_group_size;
11086 : }
11087 126673 : auto_sbitmap used_in_lanes (in_nlanes);
11088 126673 : bitmap_clear (used_in_lanes);
11089 126673 : auto_bitmap used_defs;
11090 :
11091 126673 : unsigned int count = mask.encoded_nelts ();
11092 126673 : mask.quick_grow (count);
11093 126673 : vec_perm_indices indices;
11094 :
11095 680567 : for (unsigned int j = 0; j < nelts_to_build; j++)
11096 : {
11097 563901 : unsigned int iter_num = j / group_size;
11098 563901 : unsigned int stmt_num = j % group_size;
11099 563901 : unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
11100 563901 : bitmap_set_bit (used_in_lanes, i);
11101 563901 : if (repeating_p)
11102 : {
11103 : first_vec_index = 0;
11104 : mask_element = i;
11105 : }
11106 : else
11107 : {
11108 : /* Enforced before the loop when !repeating_p. */
11109 357711 : unsigned int const_nunits = nunits.to_constant ();
11110 357711 : vec_index = i / const_nunits;
11111 357711 : mask_element = i % const_nunits;
11112 357711 : if (vec_index == first_vec_index
11113 357711 : || first_vec_index == -1)
11114 : {
11115 : first_vec_index = vec_index;
11116 : }
11117 142522 : else if (vec_index == second_vec_index
11118 142522 : || second_vec_index == -1)
11119 : {
11120 136423 : second_vec_index = vec_index;
11121 136423 : mask_element += const_nunits;
11122 : }
11123 : else
11124 : {
11125 6099 : if (dump_p)
11126 280 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11127 : "permutation requires at "
11128 : "least three vectors %G",
11129 : stmt_info->stmt);
11130 6099 : gcc_assert (analyze_only);
11131 : return false;
11132 : }
11133 :
11134 351612 : gcc_assert (mask_element < 2 * const_nunits);
11135 : }
11136 :
11137 557802 : if (mask_element != index)
11138 360733 : noop_p = false;
11139 557802 : mask[index++] = mask_element;
11140 :
11141 557802 : if (index == count)
11142 : {
11143 150698 : if (!noop_p)
11144 : {
11145 121938 : indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
11146 121938 : if (!can_vec_perm_const_p (mode, mode, indices))
11147 : {
11148 3908 : if (dump_p)
11149 : {
11150 79 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11151 : "unsupported vect permute { ");
11152 669 : for (i = 0; i < count; ++i)
11153 : {
11154 590 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11155 590 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11156 : }
11157 79 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11158 : }
11159 3908 : gcc_assert (analyze_only);
11160 : return false;
11161 : }
11162 :
11163 118030 : tree mask_vec = NULL_TREE;
11164 118030 : if (!analyze_only)
11165 20782 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11166 :
11167 118030 : if (second_vec_index == -1)
11168 35591 : second_vec_index = first_vec_index;
11169 :
11170 238934 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11171 : {
11172 120904 : ++*n_perms;
11173 120904 : if (analyze_only)
11174 99839 : continue;
11175 : /* Generate the permute statement if necessary. */
11176 21065 : tree first_vec = dr_chain[first_vec_index + ri];
11177 21065 : tree second_vec = dr_chain[second_vec_index + ri];
11178 21065 : gassign *stmt = as_a<gassign *> (stmt_info->stmt);
11179 21065 : tree perm_dest
11180 21065 : = vect_create_destination_var (gimple_assign_lhs (stmt),
11181 : vectype);
11182 21065 : perm_dest = make_ssa_name (perm_dest);
11183 21065 : gimple *perm_stmt
11184 21065 : = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
11185 : second_vec, mask_vec);
11186 21065 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
11187 : gsi);
11188 21065 : if (dce_chain)
11189 : {
11190 20142 : bitmap_set_bit (used_defs, first_vec_index + ri);
11191 20142 : bitmap_set_bit (used_defs, second_vec_index + ri);
11192 : }
11193 :
11194 : /* Store the vector statement in NODE. */
11195 21065 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
11196 : }
11197 : }
11198 28760 : else if (!analyze_only)
11199 : {
11200 2846 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11201 : {
11202 1423 : tree first_vec = dr_chain[first_vec_index + ri];
11203 : /* If mask was NULL_TREE generate the requested
11204 : identity transform. */
11205 1423 : if (dce_chain)
11206 1416 : bitmap_set_bit (used_defs, first_vec_index + ri);
11207 :
11208 : /* Store the vector statement in NODE. */
11209 1423 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
11210 : }
11211 : }
11212 :
11213 : index = 0;
11214 : first_vec_index = -1;
11215 : second_vec_index = -1;
11216 : noop_p = true;
11217 : }
11218 : }
11219 :
11220 116666 : if (n_loads)
11221 : {
11222 81907 : if (repeating_p)
11223 10562 : *n_loads = nstmts;
11224 : else
11225 : {
11226 : /* Enforced above when !repeating_p. */
11227 71345 : unsigned int const_nunits = nunits.to_constant ();
11228 71345 : *n_loads = 0;
11229 71345 : bool load_seen = false;
11230 990931 : for (unsigned i = 0; i < in_nlanes; ++i)
11231 : {
11232 919586 : if (i % const_nunits == 0)
11233 : {
11234 389146 : if (load_seen)
11235 112369 : *n_loads += 1;
11236 : load_seen = false;
11237 : }
11238 919586 : if (bitmap_bit_p (used_in_lanes, i))
11239 256096 : load_seen = true;
11240 : }
11241 71345 : if (load_seen)
11242 49311 : *n_loads += 1;
11243 : }
11244 : }
11245 :
11246 116666 : if (dce_chain)
11247 217523 : for (unsigned i = 0; i < dr_chain.length (); ++i)
11248 73672 : if (!bitmap_bit_p (used_defs, i))
11249 : {
11250 40647 : tree def = dr_chain[i];
11251 41038 : do
11252 : {
11253 41038 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11254 41038 : if (is_gimple_assign (stmt)
11255 41038 : && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
11256 41038 : || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
11257 4960 : def = single_ssa_tree_operand (stmt, SSA_OP_USE);
11258 : else
11259 : def = NULL;
11260 41038 : gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
11261 41038 : gsi_remove (&rgsi, true);
11262 41038 : release_defs (stmt);
11263 : }
11264 41038 : while (def);
11265 : }
11266 :
11267 : return true;
11268 126673 : }
11269 :
11270 : /* Generate vector permute statements from a list of loads in DR_CHAIN.
11271 : If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
11272 : permute statements for the SLP node NODE. Store the number of vector
11273 : permute instructions in *N_PERMS and the number of vector load
11274 : instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions
11275 : that were not needed. */
11276 :
11277 : bool
11278 91009 : vect_transform_slp_perm_load (vec_info *vinfo,
11279 : slp_tree node, const vec<tree> &dr_chain,
11280 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11281 : bool analyze_only, unsigned *n_perms,
11282 : unsigned int *n_loads, bool dce_chain)
11283 : {
11284 91009 : return vect_transform_slp_perm_load_1 (vinfo, node,
11285 91009 : SLP_TREE_LOAD_PERMUTATION (node),
11286 : dr_chain, gsi, vf, analyze_only,
11287 : dump_enabled_p (), n_perms, n_loads,
11288 91009 : dce_chain);
11289 : }
11290 :
11291 : /* Produce the next vector result for SLP permutation NODE by adding a vector
11292 : statement at GSI. If MASK_VEC is nonnull, add:
11293 :
11294 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
11295 :
11296 : otherwise add:
11297 :
11298 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
11299 : { N, N+1, N+2, ... }>
11300 :
11301 : where N == IDENTITY_OFFSET which is either zero or equal to the
11302 : number of elements of the result. */
11303 :
11304 : static void
11305 31246 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11306 : slp_tree node, tree first_def, tree second_def,
11307 : tree mask_vec, poly_uint64 identity_offset)
11308 : {
11309 31246 : tree vectype = SLP_TREE_VECTYPE (node);
11310 :
11311 : /* ??? We SLP match existing vector element extracts but
11312 : allow punning which we need to re-instantiate at uses
11313 : but have no good way of explicitly representing. */
11314 31246 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
11315 31246 : && !types_compatible_p (TREE_TYPE (first_def), vectype))
11316 : {
11317 18 : gassign *conv_stmt
11318 18 : = gimple_build_assign (make_ssa_name (vectype),
11319 : build1 (VIEW_CONVERT_EXPR, vectype, first_def));
11320 18 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11321 18 : first_def = gimple_assign_lhs (conv_stmt);
11322 : }
11323 31246 : gassign *perm_stmt;
11324 31246 : tree perm_dest = make_ssa_name (vectype);
11325 31246 : if (mask_vec)
11326 : {
11327 28003 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
11328 28003 : TYPE_SIZE (vectype))
11329 28003 : && !types_compatible_p (TREE_TYPE (second_def), vectype))
11330 : {
11331 8 : gassign *conv_stmt
11332 8 : = gimple_build_assign (make_ssa_name (vectype),
11333 : build1 (VIEW_CONVERT_EXPR,
11334 : vectype, second_def));
11335 8 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11336 8 : second_def = gimple_assign_lhs (conv_stmt);
11337 : }
11338 28003 : perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
11339 : first_def, second_def,
11340 : mask_vec);
11341 : }
11342 : else
11343 : {
11344 3243 : auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
11345 3243 : unsigned HOST_WIDE_INT vecno;
11346 3243 : poly_uint64 eltno;
11347 3243 : if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
11348 : &vecno, &eltno))
11349 : gcc_unreachable ();
11350 3243 : tree def = vecno & 1 ? second_def : first_def;
11351 3243 : if (!types_compatible_p (TREE_TYPE (def), vectype))
11352 : {
11353 : /* For identity permutes we still need to handle the case
11354 : of offsetted extracts or concats. */
11355 239 : unsigned HOST_WIDE_INT c;
11356 239 : if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
11357 : {
11358 235 : unsigned HOST_WIDE_INT elsz
11359 235 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
11360 470 : tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
11361 235 : TYPE_SIZE (vectype),
11362 235 : bitsize_int (eltno * elsz));
11363 235 : perm_stmt = gimple_build_assign (perm_dest, lowpart);
11364 : }
11365 4 : else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
11366 4 : def_nunits, &c) && c == 2)
11367 : {
11368 4 : gcc_assert (known_eq (identity_offset, 0U));
11369 4 : tree ctor = build_constructor_va (vectype, 2,
11370 : NULL_TREE, first_def,
11371 : NULL_TREE, second_def);
11372 4 : perm_stmt = gimple_build_assign (perm_dest, ctor);
11373 : }
11374 : else
11375 0 : gcc_unreachable ();
11376 : }
11377 : else
11378 : {
11379 : /* We need a copy here in case the def was external. */
11380 3004 : gcc_assert (known_eq (eltno, 0U));
11381 3004 : perm_stmt = gimple_build_assign (perm_dest, def);
11382 : }
11383 : }
11384 31246 : vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
11385 : /* Store the vector statement in NODE. */
11386 31246 : node->push_vec_def (perm_stmt);
11387 31246 : }
11388 :
11389 : /* Subroutine of vectorizable_slp_permutation. Check whether the target
11390 : can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
11391 : If GSI is nonnull, emit the permutation there.
11392 :
11393 : When GSI is null, the only purpose of NODE is to give properties
11394 : of the result, such as the vector type and number of SLP lanes.
11395 : The node does not need to be a VEC_PERM_EXPR.
11396 :
11397 : If the target supports the operation, return the number of individual
11398 : VEC_PERM_EXPRs needed, otherwise return -1. Print information to the
11399 : dump file if DUMP_P is true. */
11400 :
11401 : static int
11402 447096 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
11403 : slp_tree node, lane_permutation_t &perm,
11404 : vec<slp_tree> &children, bool dump_p)
11405 : {
11406 447096 : tree vectype = SLP_TREE_VECTYPE (node);
11407 :
11408 : /* ??? We currently only support all same vector input types
11409 : while the SLP IL should really do a concat + select and thus accept
11410 : arbitrary mismatches. */
11411 447096 : slp_tree child;
11412 447096 : unsigned i;
11413 447096 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11414 447096 : bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
11415 : /* True if we're permuting a single input of 2N vectors down
11416 : to N vectors. This case doesn't generalize beyond 2 since
11417 : VEC_PERM_EXPR only takes 2 inputs. */
11418 447096 : bool pack_p = false;
11419 : /* If we're permuting inputs of N vectors each into X*N outputs,
11420 : this is the value of X, otherwise it is 1. */
11421 447096 : unsigned int unpack_factor = 1;
11422 447096 : tree op_vectype = NULL_TREE;
11423 448654 : FOR_EACH_VEC_ELT (children, i, child)
11424 448582 : if (SLP_TREE_VECTYPE (child))
11425 : {
11426 : op_vectype = SLP_TREE_VECTYPE (child);
11427 : break;
11428 : }
11429 447096 : if (!op_vectype)
11430 72 : op_vectype = vectype;
11431 977418 : FOR_EACH_VEC_ELT (children, i, child)
11432 : {
11433 530322 : if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
11434 9580 : && !vect_maybe_update_slp_op_vectype (child, op_vectype))
11435 530322 : || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
11436 1060644 : || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
11437 : {
11438 0 : if (dump_p)
11439 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11440 : "Unsupported vector types in lane permutation\n");
11441 0 : return -1;
11442 : }
11443 530322 : auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
11444 530322 : unsigned int this_unpack_factor;
11445 : /* Detect permutations of external, pre-existing vectors. The external
11446 : node's SLP_TREE_LANES stores the total number of units in the vector,
11447 : or zero if the vector has variable length.
11448 :
11449 : We are expected to keep the original VEC_PERM_EXPR for such cases.
11450 : There is no repetition to model. */
11451 530322 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def
11452 530322 : && SLP_TREE_SCALAR_OPS (child).is_empty ())
11453 : repeating_p = false;
11454 : /* Check whether the input has twice as many lanes per vector. */
11455 523290 : else if (children.length () == 1
11456 523290 : && known_eq (SLP_TREE_LANES (child) * nunits,
11457 : SLP_TREE_LANES (node) * op_nunits * 2))
11458 : pack_p = true;
11459 : /* Check whether the output has N times as many lanes per vector. */
11460 530322 : else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
11461 479459 : SLP_TREE_LANES (child) * nunits,
11462 : &this_unpack_factor)
11463 444621 : && (i == 0 || unpack_factor == this_unpack_factor))
11464 : unpack_factor = this_unpack_factor;
11465 : else
11466 : repeating_p = false;
11467 : }
11468 :
11469 894192 : gcc_assert (perm.length () == SLP_TREE_LANES (node));
11470 :
11471 : /* Load-lanes permute. This permute only acts as a forwarder to
11472 : select the correct vector def of the load-lanes load which
11473 : has the permuted vectors in its vector defs like
11474 : { v0, w0, r0, v1, w1, r1 ... } for a ld3. All costs are
11475 : accounted for in the costing for the actual load so we
11476 : return zero here. */
11477 447096 : if (node->ldst_lanes)
11478 : {
11479 0 : gcc_assert (children.length () == 1);
11480 0 : if (!gsi)
11481 : /* This is a trivial op always supported. */
11482 : return 0;
11483 0 : slp_tree child = children[0];
11484 0 : unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
11485 0 : / SLP_TREE_LANES (node));
11486 0 : unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
11487 0 : unsigned nvectors = vect_get_num_copies (vinfo, node);
11488 0 : for (unsigned i = 0; i < nvectors; ++i)
11489 : {
11490 0 : tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num + vec_idx];
11491 0 : node->push_vec_def (def);
11492 : }
11493 : return 0;
11494 : }
11495 :
11496 : /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
11497 : and if we can generate the vectors in a vector-length agnostic way.
11498 : This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
11499 : compile time.
11500 :
11501 : The significance of UNPACK_STEP is that, when PACK_P is false,
11502 : output vector I operates on a window of UNPACK_STEP elements from each
11503 : input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR). For example,
11504 : when UNPACK_FACTOR is 2, the first output vector operates on lanes
11505 : [0, NUNITS / 2 - 1] of each input vector and the second output vector
11506 : operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
11507 :
11508 : When REPEATING_P is true, NOUTPUTS holds the total number of outputs
11509 : that we actually need to generate. */
11510 447096 : uint64_t noutputs = 0;
11511 447096 : poly_uint64 unpack_step = 0;
11512 447096 : loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
11513 182933 : if (!linfo
11514 486127 : || !multiple_p (nunits, unpack_factor, &unpack_step)
11515 181993 : || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
11516 181993 : * SLP_TREE_LANES (node), nunits, &noutputs))
11517 : repeating_p = false;
11518 :
11519 : /* We can handle the conditions described for REPEATING_P above for
11520 : both variable- and constant-length vectors. The fallback requires
11521 : us to generate every element of every permute vector explicitly,
11522 : which is only possible for constant-length permute vectors.
11523 :
11524 : Set:
11525 :
11526 : - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
11527 : mask vectors that we want to build.
11528 :
11529 : - NCOPIES to the number of copies of PERM that we need in order
11530 : to build the necessary permute mask vectors. */
11531 181993 : uint64_t npatterns;
11532 181993 : unsigned nelts_per_pattern;
11533 181993 : uint64_t ncopies;
11534 181993 : if (repeating_p)
11535 : {
11536 : /* We need permute mask vectors that have the form:
11537 :
11538 : { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
11539 :
11540 : In other words, the original n-element permute in PERM is
11541 : "unrolled" to fill a full vector. The stepped vector encoding
11542 : that we use for permutes requires 3n elements. */
11543 142962 : npatterns = SLP_TREE_LANES (node);
11544 142962 : nelts_per_pattern = ncopies = 3;
11545 : }
11546 : else
11547 : {
11548 : /* Calculate every element of every permute mask vector explicitly,
11549 : instead of relying on the pattern described above. */
11550 304134 : if (!nunits.is_constant (&npatterns)
11551 304134 : || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
11552 : {
11553 : if (dump_p)
11554 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11555 : "unsupported permutation %p on variable-length"
11556 : " vectors\n", (void *) node);
11557 : return -1;
11558 : }
11559 304134 : nelts_per_pattern = ncopies = 1;
11560 304134 : if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
11561 : {
11562 : if (dump_p)
11563 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11564 : "unsupported permutation %p for variable VF\n",
11565 : (void *) node);
11566 : return -1;
11567 : }
11568 : pack_p = false;
11569 : unpack_factor = 1;
11570 : }
11571 447096 : unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
11572 447096 : gcc_assert (repeating_p || multiple_p (olanes, nunits));
11573 :
11574 : /* Compute the { { SLP operand, vector index}, lane } permutation sequence
11575 : from the { SLP operand, scalar lane } permutation as recorded in the
11576 : SLP node as intermediate step. This part should already work
11577 : with SLP children with arbitrary number of lanes. */
11578 447096 : auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
11579 447096 : auto_vec<poly_uint64> active_lane;
11580 447096 : vperm.create (olanes);
11581 447096 : active_lane.safe_grow_cleared (children.length (), true);
11582 902462 : for (unsigned int ui = 0; ui < unpack_factor; ++ui)
11583 : {
11584 2004408 : for (unsigned j = 0; j < children.length (); ++j)
11585 546838 : active_lane[j] = ui * unpack_step;
11586 1311102 : for (unsigned i = 0; i < ncopies; ++i)
11587 : {
11588 5343730 : for (unsigned pi = 0; pi < perm.length (); ++pi)
11589 : {
11590 1816129 : std::pair<unsigned, unsigned> p = perm[pi];
11591 1816129 : tree vtype = SLP_TREE_VECTYPE (children[p.first]);
11592 1816129 : if (repeating_p)
11593 834009 : vperm.quick_push ({{p.first, 0},
11594 834009 : p.second + active_lane[p.first]});
11595 : else
11596 : {
11597 : /* We checked above that the vectors are constant-length. */
11598 982120 : unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
11599 982120 : .to_constant ();
11600 982120 : unsigned lane = active_lane[p.first].to_constant ();
11601 982120 : unsigned vi = (lane + p.second) / vnunits;
11602 982120 : unsigned vl = (lane + p.second) % vnunits;
11603 982120 : vperm.quick_push ({{p.first, vi}, vl});
11604 : }
11605 : }
11606 : /* Advance to the next group. */
11607 1867336 : for (unsigned j = 0; j < children.length (); ++j)
11608 1011600 : active_lane[j] += SLP_TREE_LANES (children[j]);
11609 : }
11610 : }
11611 :
11612 447096 : if (dump_p)
11613 : {
11614 8985 : dump_printf_loc (MSG_NOTE, vect_location,
11615 : "vectorizing permutation %p", (void *)node);
11616 32518 : for (unsigned i = 0; i < perm.length (); ++i)
11617 23533 : dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
11618 8985 : if (repeating_p)
11619 7586 : dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
11620 8985 : dump_printf (MSG_NOTE, "\n");
11621 8985 : dump_printf_loc (MSG_NOTE, vect_location, "as");
11622 90546 : for (unsigned i = 0; i < vperm.length (); ++i)
11623 : {
11624 81561 : if (i != 0
11625 81561 : && (repeating_p
11626 55369 : ? multiple_p (i, npatterns)
11627 60659 : : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
11628 24399 : dump_printf (MSG_NOTE, ",");
11629 81561 : dump_printf (MSG_NOTE, " vops%u[%u][",
11630 81561 : vperm[i].first.first, vperm[i].first.second);
11631 81561 : dump_dec (MSG_NOTE, vperm[i].second);
11632 81561 : dump_printf (MSG_NOTE, "]");
11633 : }
11634 8985 : dump_printf (MSG_NOTE, "\n");
11635 : }
11636 :
11637 : /* We can only handle two-vector permutes, everything else should
11638 : be lowered on the SLP level. The following is closely inspired
11639 : by vect_transform_slp_perm_load and is supposed to eventually
11640 : replace it.
11641 : ??? As intermediate step do code-gen in the SLP tree representation
11642 : somehow? */
11643 447096 : std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
11644 447096 : std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
11645 447096 : unsigned int index = 0;
11646 447096 : poly_uint64 mask_element;
11647 447096 : vec_perm_builder mask;
11648 447096 : mask.new_vector (nunits, npatterns, nelts_per_pattern);
11649 447096 : unsigned int count = mask.encoded_nelts ();
11650 447096 : mask.quick_grow (count);
11651 447096 : vec_perm_indices indices;
11652 447096 : unsigned nperms = 0;
11653 : /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
11654 : vectors to check during analysis, but we need to generate NOUTPUTS
11655 : vectors during transformation. */
11656 447096 : unsigned total_nelts = olanes;
11657 447096 : unsigned process_nelts = olanes;
11658 447096 : if (repeating_p)
11659 : {
11660 142962 : total_nelts = (total_nelts / unpack_factor) * noutputs;
11661 142962 : if (gsi)
11662 9815 : process_nelts = total_nelts;
11663 : }
11664 447096 : unsigned last_ei = (total_nelts - 1) % process_nelts;
11665 2271786 : for (unsigned i = 0; i < process_nelts; ++i)
11666 : {
11667 : /* VI is the input vector index when generating code for REPEATING_P. */
11668 1832843 : unsigned vi = i / olanes * (pack_p ? 2 : 1);
11669 1832843 : unsigned ei = i % olanes;
11670 1832843 : mask_element = vperm[ei].second;
11671 1832843 : if (pack_p)
11672 : {
11673 : /* In this case, we have N outputs and the single child provides 2N
11674 : inputs. Output X permutes inputs 2X and 2X+1.
11675 :
11676 : The mask indices are taken directly from the SLP permutation node.
11677 : Index X selects from the first vector if (X / NUNITS) % 2 == 0;
11678 : X selects from the second vector otherwise. These conditions
11679 : are only known at compile time for constant-length vectors. */
11680 : first_vec = std::make_pair (0, 0);
11681 : second_vec = std::make_pair (0, 1);
11682 : }
11683 1663415 : else if (first_vec.first == -1U
11684 1663415 : || first_vec == vperm[ei].first)
11685 1431265 : first_vec = vperm[ei].first;
11686 232150 : else if (second_vec.first == -1U
11687 232150 : || second_vec == vperm[ei].first)
11688 : {
11689 231763 : second_vec = vperm[ei].first;
11690 231763 : mask_element += nunits;
11691 : }
11692 : else
11693 : {
11694 387 : if (dump_p)
11695 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11696 : "permutation requires at "
11697 : "least three vectors\n");
11698 387 : gcc_assert (!gsi);
11699 : return -1;
11700 : }
11701 :
11702 1832456 : mask[index++] = mask_element;
11703 :
11704 1832456 : if (index == count)
11705 : {
11706 588238 : indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
11707 : TYPE_VECTOR_SUBPARTS (op_vectype));
11708 588238 : bool identity_p = (indices.series_p (0, 1, mask[0], 1)
11709 882709 : && constant_multiple_p (mask[0], nunits));
11710 588238 : machine_mode vmode = TYPE_MODE (vectype);
11711 588238 : machine_mode op_vmode = TYPE_MODE (op_vectype);
11712 588238 : unsigned HOST_WIDE_INT c;
11713 588238 : if ((!identity_p
11714 544416 : && !can_vec_perm_const_p (vmode, op_vmode, indices))
11715 588238 : || (identity_p
11716 43822 : && !known_le (nunits,
11717 : TYPE_VECTOR_SUBPARTS (op_vectype))
11718 7774 : && (!constant_multiple_p (nunits,
11719 8 : TYPE_VECTOR_SUBPARTS (op_vectype),
11720 8 : &c) || c != 2)))
11721 : {
11722 7766 : if (dump_p)
11723 : {
11724 152 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
11725 : vect_location,
11726 : "unsupported vect permute { ");
11727 1586 : for (i = 0; i < count; ++i)
11728 : {
11729 1434 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11730 1434 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11731 : }
11732 152 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11733 : }
11734 7766 : gcc_assert (!gsi);
11735 8153 : return -1;
11736 : }
11737 :
11738 580472 : if (!identity_p)
11739 536650 : nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
11740 580472 : if (gsi)
11741 : {
11742 31246 : if (second_vec.first == -1U)
11743 6998 : second_vec = first_vec;
11744 :
11745 31246 : slp_tree
11746 31246 : first_node = children[first_vec.first],
11747 31246 : second_node = children[second_vec.first];
11748 :
11749 31246 : tree mask_vec = NULL_TREE;
11750 31246 : if (!identity_p)
11751 28003 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11752 :
11753 31246 : tree first_def
11754 31246 : = vect_get_slp_vect_def (first_node, first_vec.second + vi);
11755 31246 : tree second_def
11756 31246 : = vect_get_slp_vect_def (second_node, second_vec.second + vi);
11757 31246 : vect_add_slp_permutation (vinfo, gsi, node, first_def,
11758 31246 : second_def, mask_vec, mask[0]);
11759 : }
11760 :
11761 : index = 0;
11762 : first_vec = std::make_pair (-1U, -1U);
11763 : second_vec = std::make_pair (-1U, -1U);
11764 : }
11765 : }
11766 :
11767 438943 : return nperms;
11768 447096 : }
11769 :
11770 : /* Vectorize the SLP permutations in NODE as specified
11771 : in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
11772 : child number and lane number.
11773 : Interleaving of two two-lane two-child SLP subtrees (not supported):
11774 : [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
11775 : A blend of two four-lane two-child SLP subtrees:
11776 : [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
11777 : Highpart of a four-lane one-child SLP subtree (not supported):
11778 : [ { 0, 2 }, { 0, 3 } ]
11779 : Where currently only a subset is supported by code generating below. */
11780 :
11781 : bool
11782 137507 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11783 : slp_tree node, stmt_vector_for_cost *cost_vec)
11784 : {
11785 137507 : tree vectype = SLP_TREE_VECTYPE (node);
11786 137507 : lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
11787 137507 : int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
11788 137507 : SLP_TREE_CHILDREN (node),
11789 : dump_enabled_p ());
11790 137507 : if (nperms < 0)
11791 : return false;
11792 :
11793 136208 : if (!gsi && nperms != 0)
11794 114317 : record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
11795 :
11796 : return true;
11797 : }
11798 :
11799 : /* Vectorize SLP NODE. */
11800 :
11801 : static void
11802 1487738 : vect_schedule_slp_node (vec_info *vinfo,
11803 : slp_tree node, slp_instance instance)
11804 : {
11805 1487738 : gimple_stmt_iterator si;
11806 1487738 : int i;
11807 1487738 : slp_tree child;
11808 :
11809 : /* Vectorize externals and constants. */
11810 1487738 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
11811 1487738 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
11812 : {
11813 : /* ??? vectorizable_shift can end up using a scalar operand which is
11814 : currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
11815 : node in this case. */
11816 504425 : if (!SLP_TREE_VECTYPE (node))
11817 504425 : return;
11818 :
11819 : /* There are two reasons vector defs might already exist. The first
11820 : is that we are vectorizing an existing vector def. The second is
11821 : when performing BB vectorization shared constant/external nodes
11822 : are not split apart during partitioning so during the code-gen
11823 : DFS walk we can end up visiting them twice. */
11824 497282 : if (! SLP_TREE_VEC_DEFS (node).exists ())
11825 496541 : vect_create_constant_vectors (vinfo, node);
11826 497282 : return;
11827 : }
11828 :
11829 983313 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
11830 :
11831 983313 : gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
11832 983313 : if (SLP_TREE_VECTYPE (node))
11833 983307 : SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
11834 :
11835 983313 : if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
11836 : {
11837 : /* Vectorized loads go before the first scalar load to make it
11838 : ready early, vectorized stores go before the last scalar
11839 : stmt which is where all uses are ready. */
11840 720848 : stmt_vec_info last_stmt_info = NULL;
11841 720848 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
11842 168028 : last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
11843 : else /* DR_IS_WRITE */
11844 552820 : last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
11845 720848 : si = gsi_for_stmt (last_stmt_info->stmt);
11846 720848 : }
11847 262465 : else if (!SLP_TREE_PERMUTE_P (node)
11848 246080 : && (SLP_TREE_TYPE (node) == cycle_phi_info_type
11849 : || SLP_TREE_TYPE (node) == induc_vec_info_type
11850 : || SLP_TREE_TYPE (node) == phi_info_type))
11851 : {
11852 : /* For PHI node vectorization we do not use the insertion iterator. */
11853 54396 : si = gsi_none ();
11854 : }
11855 : else
11856 : {
11857 : /* Emit other stmts after the children vectorized defs which is
11858 : earliest possible. */
11859 : gimple *last_stmt = NULL;
11860 : bool seen_vector_def = false;
11861 578758 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
11862 370689 : if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
11863 : {
11864 : /* For fold-left reductions we are retaining the scalar
11865 : reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
11866 : set so the representation isn't perfect. Resort to the
11867 : last scalar def here. */
11868 297217 : if (SLP_TREE_VEC_DEFS (child).is_empty ())
11869 : {
11870 946 : gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
11871 946 : gphi *phi = as_a <gphi *>
11872 946 : (vect_find_last_scalar_stmt_in_slp (child)->stmt);
11873 946 : if (!last_stmt)
11874 : last_stmt = phi;
11875 727 : else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
11876 : last_stmt = phi;
11877 716 : else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
11878 : ;
11879 : else
11880 0 : gcc_unreachable ();
11881 : }
11882 : /* We are emitting all vectorized stmts in the same place and
11883 : the last one is the last.
11884 : ??? Unless we have a load permutation applied and that
11885 : figures to re-use an earlier generated load. */
11886 : unsigned j;
11887 : tree vdef;
11888 702613 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
11889 : {
11890 405396 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
11891 405396 : if (!last_stmt)
11892 : last_stmt = vstmt;
11893 208007 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
11894 : last_stmt = vstmt;
11895 45754 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
11896 : ;
11897 : else
11898 0 : gcc_unreachable ();
11899 : }
11900 : }
11901 73472 : else if (!SLP_TREE_VECTYPE (child))
11902 : {
11903 : /* For externals we use unvectorized at all scalar defs. */
11904 : unsigned j;
11905 : tree def;
11906 15553 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
11907 9039 : if (TREE_CODE (def) == SSA_NAME
11908 9039 : && !SSA_NAME_IS_DEFAULT_DEF (def))
11909 : {
11910 407 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11911 407 : if (gimple_uid (stmt) == -1u)
11912 : /* If the stmt is not inside the region do not
11913 : use it as possible insertion point. */
11914 : ;
11915 397 : else if (!last_stmt)
11916 : last_stmt = stmt;
11917 363 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
11918 : last_stmt = stmt;
11919 159 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
11920 : ;
11921 : else
11922 0 : gcc_unreachable ();
11923 : }
11924 : }
11925 : else
11926 : {
11927 : /* For externals we have to look at all defs since their
11928 : insertion place is decided per vector. But beware
11929 : of pre-existing vectors where we need to make sure
11930 : we do not insert before the region boundary. */
11931 66958 : if (SLP_TREE_SCALAR_OPS (child).is_empty ()
11932 539 : && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
11933 : seen_vector_def = true;
11934 : else
11935 : {
11936 : unsigned j;
11937 : tree vdef;
11938 532364 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
11939 94834 : if (TREE_CODE (vdef) == SSA_NAME
11940 94834 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
11941 : {
11942 19614 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
11943 19614 : if (!last_stmt)
11944 : last_stmt = vstmt;
11945 10906 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
11946 : last_stmt = vstmt;
11947 8765 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
11948 : ;
11949 : else
11950 0 : gcc_unreachable ();
11951 : }
11952 : }
11953 : }
11954 : /* This can happen when all children are pre-existing vectors or
11955 : constants. */
11956 208069 : if (!last_stmt)
11957 1719 : last_stmt = vect_find_first_scalar_stmt_in_slp (node)->stmt;
11958 1719 : if (!last_stmt)
11959 : {
11960 0 : gcc_assert (seen_vector_def);
11961 0 : si = gsi_after_labels (vinfo->bbs[0]);
11962 : }
11963 208069 : else if (is_ctrl_altering_stmt (last_stmt))
11964 : {
11965 : /* We split regions to vectorize at control altering stmts
11966 : with a definition so this must be an external which
11967 : we can insert at the start of the region. */
11968 0 : si = gsi_after_labels (vinfo->bbs[0]);
11969 : }
11970 208069 : else if (is_a <bb_vec_info> (vinfo)
11971 18183 : && !SLP_TREE_PERMUTE_P (node)
11972 16799 : && gimple_bb (last_stmt) != gimple_bb (stmt_info->stmt)
11973 209598 : && gimple_could_trap_p (stmt_info->stmt))
11974 : {
11975 : /* We've constrained possibly trapping operations to all come
11976 : from the same basic-block, if vectorized defs would allow earlier
11977 : scheduling still force vectorized stmts to the original block.
11978 : This is only necessary for BB vectorization since for loop vect
11979 : all operations are in a single BB and scalar stmt based
11980 : placement doesn't play well with epilogue vectorization. */
11981 86 : gcc_assert (dominated_by_p (CDI_DOMINATORS,
11982 : gimple_bb (stmt_info->stmt),
11983 : gimple_bb (last_stmt)));
11984 86 : si = gsi_after_labels (gimple_bb (stmt_info->stmt));
11985 : }
11986 207983 : else if (is_a <gphi *> (last_stmt))
11987 14509 : si = gsi_after_labels (gimple_bb (last_stmt));
11988 : else
11989 : {
11990 193474 : si = gsi_for_stmt (last_stmt);
11991 193474 : gsi_next (&si);
11992 :
11993 193474 : if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
11994 : {
11995 : /* Avoid scheduling stmts to random places in the CFG, any
11996 : stmt dominance check we performed is possibly wrong as UIDs
11997 : are not initialized for all of the function for loop
11998 : vectorization. Instead append to the loop preheader. */
11999 175611 : if ((LOOP_VINFO_LOOP (loop_vinfo)->header
12000 175611 : != gimple_bb (last_stmt))
12001 178806 : && dominated_by_p (CDI_DOMINATORS,
12002 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12003 3195 : gimple_bb (last_stmt)))
12004 1362 : si = gsi_end_bb (loop_preheader_edge
12005 681 : (LOOP_VINFO_LOOP (loop_vinfo))->src);
12006 : /* Avoid scheduling internal defs outside of the loop when
12007 : we might have only implicitly tracked loop mask/len defs. */
12008 74 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
12009 175611 : || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12010 : {
12011 74 : gimple_stmt_iterator si2
12012 74 : = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
12013 74 : if ((gsi_end_p (si2)
12014 0 : && (LOOP_VINFO_LOOP (loop_vinfo)->header
12015 0 : != gimple_bb (last_stmt))
12016 0 : && dominated_by_p (CDI_DOMINATORS,
12017 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12018 0 : gimple_bb (last_stmt)))
12019 74 : || (!gsi_end_p (si2)
12020 74 : && last_stmt != *si2
12021 72 : && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
12022 3 : si = si2;
12023 : }
12024 : }
12025 : }
12026 : }
12027 :
12028 983313 : if (dump_enabled_p ())
12029 : {
12030 71754 : if (stmt_info)
12031 71702 : dump_printf_loc (MSG_NOTE, vect_location,
12032 : "------>vectorizing SLP node starting from: %G",
12033 : stmt_info->stmt);
12034 : else
12035 : {
12036 52 : dump_printf_loc (MSG_NOTE, vect_location,
12037 : "------>vectorizing SLP node:\n");
12038 52 : vect_print_slp_tree (MSG_NOTE, vect_location, node);
12039 : }
12040 : }
12041 983313 : vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
12042 : }
12043 :
12044 : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
12045 : For loop vectorization this is done in vectorizable_call, but for SLP
12046 : it needs to be deferred until end of vect_schedule_slp, because multiple
12047 : SLP instances may refer to the same scalar stmt. */
12048 :
12049 : static void
12050 603369 : vect_remove_slp_scalar_calls (vec_info *vinfo,
12051 : slp_tree node, hash_set<slp_tree> &visited)
12052 : {
12053 603369 : gimple *new_stmt;
12054 603369 : gimple_stmt_iterator gsi;
12055 603369 : tree lhs;
12056 :
12057 603369 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12058 188881 : return;
12059 :
12060 458155 : if (visited.add (node))
12061 : return;
12062 :
12063 1544912 : for (auto child : SLP_TREE_CHILDREN (node))
12064 513106 : vect_remove_slp_scalar_calls (vinfo, child, visited);
12065 :
12066 1722211 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
12067 : {
12068 487025 : if (!stmt_info)
12069 3978 : continue;
12070 483047 : stmt_info = vect_orig_stmt (stmt_info);
12071 483047 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
12072 5157 : if (!stmt || gimple_bb (stmt) == NULL)
12073 477934 : continue;
12074 5113 : lhs = gimple_call_lhs (stmt);
12075 5113 : if (lhs)
12076 4530 : new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
12077 : else
12078 583 : new_stmt = gimple_build_nop ();
12079 5113 : unlink_stmt_vdef (stmt_info->stmt);
12080 5113 : gsi = gsi_for_stmt (stmt);
12081 5113 : vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
12082 5113 : if (lhs)
12083 4530 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12084 : }
12085 : }
12086 :
12087 : static void
12088 90263 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
12089 : {
12090 90263 : hash_set<slp_tree> visited;
12091 90263 : vect_remove_slp_scalar_calls (vinfo, node, visited);
12092 90263 : }
12093 :
12094 : /* Vectorize the instance root. */
12095 :
12096 : void
12097 11215 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
12098 : {
12099 11215 : gassign *rstmt = NULL;
12100 :
12101 11215 : if (instance->kind == slp_inst_kind_ctor)
12102 : {
12103 5467 : if (SLP_TREE_VEC_DEFS (node).length () == 1)
12104 : {
12105 5422 : tree vect_lhs = SLP_TREE_VEC_DEFS (node)[0];
12106 5422 : tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12107 5422 : if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
12108 5422 : TREE_TYPE (vect_lhs)))
12109 0 : vect_lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
12110 : vect_lhs);
12111 5422 : rstmt = gimple_build_assign (root_lhs, vect_lhs);
12112 : }
12113 : else
12114 : {
12115 45 : gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
12116 45 : tree child_def;
12117 45 : int j;
12118 45 : vec<constructor_elt, va_gc> *v;
12119 45 : vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
12120 :
12121 : /* A CTOR can handle V16HI composition from VNx8HI so we
12122 : do not need to convert vector elements if the types
12123 : do not match. */
12124 135 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
12125 90 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
12126 45 : tree lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12127 45 : tree rtype
12128 45 : = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
12129 45 : tree r_constructor = build_constructor (rtype, v);
12130 45 : rstmt = gimple_build_assign (lhs, r_constructor);
12131 : }
12132 : }
12133 5748 : else if (instance->kind == slp_inst_kind_bb_reduc)
12134 : {
12135 : /* Largely inspired by reduction chain epilogue handling in
12136 : vect_create_epilog_for_reduction. */
12137 4159 : vec<tree> vec_defs = vNULL;
12138 4159 : vect_get_slp_defs (node, &vec_defs);
12139 4159 : enum tree_code reduc_code
12140 4159 : = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
12141 : /* ??? We actually have to reflect signs somewhere. */
12142 4159 : if (reduc_code == MINUS_EXPR)
12143 0 : reduc_code = PLUS_EXPR;
12144 4159 : gimple_seq epilogue = NULL;
12145 : /* We may end up with more than one vector result, reduce them
12146 : to one vector. */
12147 4159 : tree vec_def = vec_defs[0];
12148 4159 : tree vectype = TREE_TYPE (vec_def);
12149 4159 : tree compute_vectype = vectype;
12150 4159 : bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
12151 3964 : && TYPE_OVERFLOW_UNDEFINED (vectype)
12152 6936 : && operation_can_overflow (reduc_code));
12153 2630 : if (pun_for_overflow_p)
12154 : {
12155 2630 : compute_vectype = unsigned_type_for (vectype);
12156 2630 : vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12157 : compute_vectype, vec_def);
12158 : }
12159 6537 : for (unsigned i = 1; i < vec_defs.length (); ++i)
12160 : {
12161 2378 : tree def = vec_defs[i];
12162 2378 : if (pun_for_overflow_p)
12163 2275 : def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12164 : compute_vectype, def);
12165 2378 : vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
12166 : vec_def, def);
12167 : }
12168 4159 : vec_defs.release ();
12169 : /* ??? Support other schemes than direct internal fn. */
12170 4159 : internal_fn reduc_fn;
12171 4159 : if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
12172 4159 : || reduc_fn == IFN_LAST)
12173 0 : gcc_unreachable ();
12174 4159 : tree scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
12175 4159 : TREE_TYPE (compute_vectype), vec_def);
12176 4159 : if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
12177 : {
12178 2557 : tree rem_def = NULL_TREE;
12179 11891 : for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
12180 : {
12181 9334 : def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
12182 9334 : if (!rem_def)
12183 : rem_def = def;
12184 : else
12185 6777 : rem_def = gimple_build (&epilogue, reduc_code,
12186 6777 : TREE_TYPE (scalar_def),
12187 : rem_def, def);
12188 : }
12189 2557 : scalar_def = gimple_build (&epilogue, reduc_code,
12190 2557 : TREE_TYPE (scalar_def),
12191 : scalar_def, rem_def);
12192 : }
12193 4159 : scalar_def = gimple_convert (&epilogue,
12194 4159 : TREE_TYPE (vectype), scalar_def);
12195 4159 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12196 4159 : gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
12197 4159 : gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
12198 4159 : update_stmt (gsi_stmt (rgsi));
12199 4159 : return;
12200 : }
12201 1589 : else if (instance->kind == slp_inst_kind_gcond)
12202 : {
12203 : /* Only support a single root for now as we can't codegen CFG yet and so we
12204 : can't support lane > 1 at this time. */
12205 1589 : gcc_assert (instance->root_stmts.length () == 1);
12206 1589 : auto root_stmt_info = instance->root_stmts[0];
12207 1589 : auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
12208 1589 : gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
12209 1589 : gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
12210 1589 : bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
12211 : root_stmt_info, &rgsi, node, NULL);
12212 1589 : gcc_assert (res);
12213 1589 : return;
12214 : }
12215 : else
12216 0 : gcc_unreachable ();
12217 :
12218 5467 : gcc_assert (rstmt);
12219 :
12220 5467 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12221 5467 : gsi_replace (&rgsi, rstmt, true);
12222 : }
12223 :
12224 : struct slp_scc_info
12225 : {
12226 : bool on_stack;
12227 : int dfs;
12228 : int lowlink;
12229 : };
12230 :
12231 : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs. */
12232 :
12233 : static void
12234 1487738 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
12235 : hash_map<slp_tree, slp_scc_info> &scc_info,
12236 : int &maxdfs, vec<slp_tree> &stack)
12237 : {
12238 1487738 : bool existed_p;
12239 1487738 : slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
12240 1487738 : gcc_assert (!existed_p);
12241 1487738 : info->dfs = maxdfs;
12242 1487738 : info->lowlink = maxdfs;
12243 1487738 : maxdfs++;
12244 :
12245 : /* Leaf. */
12246 1487738 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12247 : {
12248 504425 : info->on_stack = false;
12249 504425 : vect_schedule_slp_node (vinfo, node, instance);
12250 1040677 : return;
12251 : }
12252 :
12253 983313 : info->on_stack = true;
12254 983313 : stack.safe_push (node);
12255 :
12256 983313 : unsigned i;
12257 983313 : slp_tree child;
12258 : /* DFS recurse. */
12259 2027848 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12260 : {
12261 1044535 : if (!child)
12262 55439 : continue;
12263 989096 : slp_scc_info *child_info = scc_info.get (child);
12264 989096 : if (!child_info)
12265 : {
12266 898366 : vect_schedule_scc (vinfo, child, instance, scc_info, maxdfs, stack);
12267 : /* Recursion might have re-allocated the node. */
12268 898366 : info = scc_info.get (node);
12269 898366 : child_info = scc_info.get (child);
12270 898366 : info->lowlink = MIN (info->lowlink, child_info->lowlink);
12271 : }
12272 90730 : else if (child_info->on_stack)
12273 25569 : info->lowlink = MIN (info->lowlink, child_info->dfs);
12274 : }
12275 983313 : if (info->lowlink != info->dfs)
12276 : return;
12277 :
12278 951486 : auto_vec<slp_tree, 4> phis_to_fixup;
12279 :
12280 : /* Singleton. */
12281 951486 : if (stack.last () == node)
12282 : {
12283 927591 : stack.pop ();
12284 927591 : info->on_stack = false;
12285 927591 : vect_schedule_slp_node (vinfo, node, instance);
12286 927591 : if (!SLP_TREE_PERMUTE_P (node)
12287 927591 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
12288 30630 : phis_to_fixup.quick_push (node);
12289 : }
12290 : else
12291 : {
12292 : /* SCC. */
12293 23895 : int last_idx = stack.length () - 1;
12294 55722 : while (stack[last_idx] != node)
12295 31827 : last_idx--;
12296 : /* We can break the cycle at PHIs who have at least one child
12297 : code generated. Then we could re-start the DFS walk until
12298 : all nodes in the SCC are covered (we might have new entries
12299 : for only back-reachable nodes). But it's simpler to just
12300 : iterate and schedule those that are ready. */
12301 23895 : unsigned todo = stack.length () - last_idx;
12302 24234 : do
12303 : {
12304 105974 : for (int idx = stack.length () - 1; idx >= last_idx; --idx)
12305 : {
12306 57506 : slp_tree entry = stack[idx];
12307 57506 : if (!entry)
12308 956 : continue;
12309 56550 : bool phi = (!SLP_TREE_PERMUTE_P (entry)
12310 56550 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
12311 56550 : bool ready = !phi;
12312 143117 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
12313 111706 : if (!child)
12314 : {
12315 23019 : gcc_assert (phi);
12316 : ready = true;
12317 : break;
12318 : }
12319 88687 : else if (scc_info.get (child)->on_stack)
12320 : {
12321 24089 : if (!phi)
12322 : {
12323 : ready = false;
12324 : break;
12325 : }
12326 : }
12327 : else
12328 : {
12329 64598 : if (phi)
12330 : {
12331 : ready = true;
12332 : break;
12333 : }
12334 : }
12335 33531 : if (ready)
12336 : {
12337 55722 : vect_schedule_slp_node (vinfo, entry, instance);
12338 55722 : scc_info.get (entry)->on_stack = false;
12339 55722 : stack[idx] = NULL;
12340 55722 : todo--;
12341 55722 : if (phi)
12342 24341 : phis_to_fixup.safe_push (entry);
12343 : }
12344 : }
12345 : }
12346 24234 : while (todo != 0);
12347 :
12348 : /* Pop the SCC. */
12349 23895 : stack.truncate (last_idx);
12350 : }
12351 :
12352 : /* Now fixup the backedge def of the vectorized PHIs in this SCC. */
12353 : slp_tree phi_node;
12354 1957943 : FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
12355 : {
12356 54971 : gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
12357 54971 : edge_iterator ei;
12358 54971 : edge e;
12359 173585 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
12360 : {
12361 118614 : unsigned dest_idx = e->dest_idx;
12362 118614 : child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
12363 118614 : if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
12364 66658 : continue;
12365 51956 : unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
12366 : /* Simply fill all args. */
12367 51956 : if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
12368 : != vect_first_order_recurrence)
12369 111660 : for (unsigned i = 0; i < n; ++i)
12370 : {
12371 59749 : tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
12372 59749 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
12373 59749 : add_phi_arg (phi, vect_get_slp_vect_def (child, i),
12374 : e, gimple_phi_arg_location (phi, dest_idx));
12375 : }
12376 : else
12377 : {
12378 : /* Unless it is a first order recurrence which needs
12379 : args filled in for both the PHI node and the permutes. */
12380 45 : gimple *perm
12381 45 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
12382 45 : gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
12383 45 : add_phi_arg (as_a <gphi *> (rphi),
12384 : vect_get_slp_vect_def (child, n - 1),
12385 : e, gimple_phi_arg_location (phi, dest_idx));
12386 127 : for (unsigned i = 0; i < n; ++i)
12387 : {
12388 82 : gimple *perm
12389 82 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
12390 82 : if (i > 0)
12391 37 : gimple_assign_set_rhs1 (perm,
12392 : vect_get_slp_vect_def (child, i - 1));
12393 82 : gimple_assign_set_rhs2 (perm,
12394 : vect_get_slp_vect_def (child, i));
12395 82 : update_stmt (perm);
12396 : }
12397 : }
12398 : }
12399 : }
12400 951486 : }
12401 :
12402 : /* Generate vector code for SLP_INSTANCES in the loop/basic block. */
12403 :
12404 : void
12405 549708 : vect_schedule_slp (vec_info *vinfo, const vec<slp_instance> &slp_instances)
12406 : {
12407 549708 : slp_instance instance;
12408 549708 : unsigned int i;
12409 :
12410 549708 : hash_map<slp_tree, slp_scc_info> scc_info;
12411 549708 : int maxdfs = 0;
12412 1139197 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12413 : {
12414 589489 : slp_tree node = SLP_INSTANCE_TREE (instance);
12415 589489 : if (dump_enabled_p ())
12416 : {
12417 16069 : dump_printf_loc (MSG_NOTE, vect_location,
12418 : "Vectorizing SLP tree:\n");
12419 : /* ??? Dump all? */
12420 16069 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12421 468 : dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
12422 468 : SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
12423 16069 : vect_print_slp_graph (MSG_NOTE, vect_location,
12424 : SLP_INSTANCE_TREE (instance));
12425 : }
12426 : /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
12427 : have a PHI be the node breaking the cycle. */
12428 589489 : auto_vec<slp_tree> stack;
12429 589489 : if (!scc_info.get (node))
12430 589372 : vect_schedule_scc (vinfo, node, instance, scc_info, maxdfs, stack);
12431 :
12432 589489 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12433 11215 : vectorize_slp_instance_root_stmt (vinfo, node, instance);
12434 :
12435 589489 : if (dump_enabled_p ())
12436 16069 : dump_printf_loc (MSG_NOTE, vect_location,
12437 : "vectorizing stmts using SLP.\n");
12438 589489 : }
12439 :
12440 1688905 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12441 : {
12442 589489 : slp_tree root = SLP_INSTANCE_TREE (instance);
12443 589489 : stmt_vec_info store_info;
12444 589489 : unsigned int j;
12445 :
12446 : /* Remove scalar call stmts. Do not do this for basic-block
12447 : vectorization as not all uses may be vectorized.
12448 : ??? Why should this be necessary? DCE should be able to
12449 : remove the stmts itself.
12450 : ??? For BB vectorization we can as well remove scalar
12451 : stmts starting from the SLP tree root if they have no
12452 : uses. */
12453 589489 : if (is_a <loop_vec_info> (vinfo))
12454 90263 : vect_remove_slp_scalar_calls (vinfo, root);
12455 :
12456 : /* Remove vectorized stores original scalar stmts. */
12457 2631974 : for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
12458 : {
12459 1489665 : if (!store_info
12460 1489651 : || !STMT_VINFO_DATA_REF (store_info)
12461 1461707 : || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
12462 : break;
12463 :
12464 1452996 : store_info = vect_orig_stmt (store_info);
12465 : /* Free the attached stmt_vec_info and remove the stmt. */
12466 1452996 : vinfo->remove_stmt (store_info);
12467 :
12468 : /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
12469 : to not crash in vect_free_slp_tree later. */
12470 1452996 : if (SLP_TREE_REPRESENTATIVE (root) == store_info)
12471 552489 : SLP_TREE_REPRESENTATIVE (root) = NULL;
12472 : }
12473 : }
12474 549708 : }
|
