Line data Source code
1 : /* SLP - Basic Block Vectorization
2 : Copyright (C) 2007-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #define INCLUDE_ALGORITHM
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "optabs-tree.h"
34 : #include "insn-config.h"
35 : #include "recog.h" /* FIXME: for insn_data */
36 : #include "fold-const.h"
37 : #include "stor-layout.h"
38 : #include "gimple-iterator.h"
39 : #include "cfgloop.h"
40 : #include "tree-vectorizer.h"
41 : #include "langhooks.h"
42 : #include "gimple-walk.h"
43 : #include "dbgcnt.h"
44 : #include "tree-vector-builder.h"
45 : #include "vec-perm-indices.h"
46 : #include "gimple-fold.h"
47 : #include "internal-fn.h"
48 : #include "dump-context.h"
49 : #include "cfganal.h"
50 : #include "tree-eh.h"
51 : #include "tree-cfg.h"
52 : #include "alloc-pool.h"
53 : #include "sreal.h"
54 : #include "predict.h"
55 :
56 : #define REDUC_GROUP_FIRST_ELEMENT(S) \
57 : (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
58 :
59 : static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree,
60 : load_permutation_t &,
61 : const vec<tree> &,
62 : gimple_stmt_iterator *,
63 : poly_uint64, bool, bool,
64 : unsigned *,
65 : unsigned * = nullptr,
66 : bool = false);
67 : static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *,
68 : slp_tree, lane_permutation_t &,
69 : vec<slp_tree> &, bool);
70 : static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree);
71 : static bool vect_slp_can_convert_to_external (const vec<stmt_vec_info> &);
72 :
73 : static object_allocator<_slp_tree> *slp_tree_pool;
74 : static slp_tree slp_first_node;
75 :
76 : void
77 1146012 : vect_slp_init (void)
78 : {
79 1146012 : slp_tree_pool = new object_allocator<_slp_tree> ("SLP nodes");
80 1146012 : }
81 :
82 : void
83 1146012 : vect_slp_fini (void)
84 : {
85 1839626 : while (slp_first_node)
86 693614 : delete slp_first_node;
87 2292024 : delete slp_tree_pool;
88 1146012 : slp_tree_pool = NULL;
89 1146012 : }
90 :
91 : void *
92 8297432 : _slp_tree::operator new (size_t n)
93 : {
94 8297432 : gcc_assert (n == sizeof (_slp_tree));
95 8297432 : return slp_tree_pool->allocate_raw ();
96 : }
97 :
98 : void
99 8297432 : _slp_tree::operator delete (void *node, size_t n)
100 : {
101 8297432 : gcc_assert (n == sizeof (_slp_tree));
102 8297432 : slp_tree_pool->remove_raw (node);
103 8297432 : }
104 :
105 :
106 : /* Initialize a SLP node. */
107 :
108 8297432 : _slp_tree::_slp_tree ()
109 : {
110 8297432 : this->prev_node = NULL;
111 8297432 : if (slp_first_node)
112 7251039 : slp_first_node->prev_node = this;
113 8297432 : this->next_node = slp_first_node;
114 8297432 : slp_first_node = this;
115 8297432 : SLP_TREE_SCALAR_STMTS (this) = vNULL;
116 8297432 : SLP_TREE_SCALAR_OPS (this) = vNULL;
117 8297432 : SLP_TREE_LIVE_LANES (this) = vNULL;
118 8297432 : SLP_TREE_VEC_DEFS (this) = vNULL;
119 8297432 : SLP_TREE_CHILDREN (this) = vNULL;
120 8297432 : SLP_TREE_LOAD_PERMUTATION (this) = vNULL;
121 8297432 : SLP_TREE_LANE_PERMUTATION (this) = vNULL;
122 8297432 : SLP_TREE_DEF_TYPE (this) = vect_uninitialized_def;
123 8297432 : SLP_TREE_CODE (this) = ERROR_MARK;
124 8297432 : SLP_TREE_GS_SCALE (this) = 0;
125 8297432 : SLP_TREE_GS_BASE (this) = NULL_TREE;
126 8297432 : this->ldst_lanes = false;
127 8297432 : this->avoid_stlf_fail = false;
128 8297432 : SLP_TREE_VECTYPE (this) = NULL_TREE;
129 8297432 : SLP_TREE_REPRESENTATIVE (this) = NULL;
130 8297432 : this->cycle_info.id = -1;
131 8297432 : this->cycle_info.reduc_idx = -1;
132 8297432 : SLP_TREE_REF_COUNT (this) = 1;
133 8297432 : this->failed = NULL;
134 8297432 : this->lanes = 0;
135 8297432 : SLP_TREE_TYPE (this) = undef_vec_info_type;
136 8297432 : this->data = NULL;
137 8297432 : this->si = NULL;
138 8297432 : }
139 :
140 : /* Tear down a SLP node. */
141 :
142 8297432 : _slp_tree::~_slp_tree ()
143 : {
144 8297432 : if (this->prev_node)
145 5061030 : this->prev_node->next_node = this->next_node;
146 : else
147 3236402 : slp_first_node = this->next_node;
148 8297432 : if (this->next_node)
149 6263940 : this->next_node->prev_node = this->prev_node;
150 8297432 : SLP_TREE_CHILDREN (this).release ();
151 8297432 : SLP_TREE_SCALAR_STMTS (this).release ();
152 8297432 : SLP_TREE_SCALAR_OPS (this).release ();
153 8297432 : SLP_TREE_LIVE_LANES (this).release ();
154 8297432 : SLP_TREE_VEC_DEFS (this).release ();
155 8297432 : SLP_TREE_LOAD_PERMUTATION (this).release ();
156 8297432 : SLP_TREE_LANE_PERMUTATION (this).release ();
157 8297432 : if (this->failed)
158 2199736 : free (failed);
159 8297432 : if (this->data)
160 1287202 : delete this->data;
161 8297432 : }
162 :
163 : /* Push the single SSA definition in DEF to the vector of vector defs. */
164 :
165 : void
166 522688 : _slp_tree::push_vec_def (gimple *def)
167 : {
168 522688 : if (gphi *phi = dyn_cast <gphi *> (def))
169 57679 : vec_defs.quick_push (gimple_phi_result (phi));
170 : else
171 : {
172 465009 : def_operand_p defop = single_ssa_def_operand (def, SSA_OP_ALL_DEFS);
173 465009 : vec_defs.quick_push (get_def_from_ptr (defop));
174 : }
175 522688 : }
176 :
177 : /* Recursively free the memory allocated for the SLP tree rooted at NODE. */
178 :
179 : void
180 15680010 : vect_free_slp_tree (slp_tree node)
181 : {
182 15680010 : int i;
183 15680010 : slp_tree child;
184 :
185 15680010 : if (--SLP_TREE_REF_COUNT (node) != 0)
186 15680010 : return;
187 :
188 11808526 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
189 4204708 : if (child)
190 3817236 : vect_free_slp_tree (child);
191 :
192 7603818 : delete node;
193 : }
194 :
195 : /* Return a location suitable for dumpings related to the SLP instance. */
196 :
197 : dump_user_location_t
198 3556387 : _slp_instance::location () const
199 : {
200 3556387 : if (!root_stmts.is_empty ())
201 414664 : return root_stmts[0]->stmt;
202 : else
203 3141723 : return SLP_TREE_SCALAR_STMTS (root)[0]->stmt;
204 : }
205 :
206 :
207 : /* Free the memory allocated for the SLP instance. */
208 :
209 : void
210 1643704 : vect_free_slp_instance (slp_instance instance)
211 : {
212 1643704 : vect_free_slp_tree (SLP_INSTANCE_TREE (instance));
213 1643704 : SLP_INSTANCE_LOADS (instance).release ();
214 1643704 : SLP_INSTANCE_ROOT_STMTS (instance).release ();
215 1643704 : SLP_INSTANCE_REMAIN_DEFS (instance).release ();
216 1643704 : instance->subgraph_entries.release ();
217 1643704 : instance->cost_vec.release ();
218 1643704 : free (instance);
219 1643704 : }
220 :
221 :
222 : /* Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR
223 : for a permute node or else ERROR_MARK. */
224 :
225 : slp_tree
226 97773 : vect_create_new_slp_node (unsigned nops, tree_code code)
227 : {
228 97773 : gcc_assert (code == ERROR_MARK || code == VEC_PERM_EXPR);
229 97773 : slp_tree node = new _slp_tree;
230 97773 : SLP_TREE_SCALAR_STMTS (node) = vNULL;
231 97773 : SLP_TREE_CHILDREN (node).create (nops);
232 97773 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
233 97773 : SLP_TREE_CODE (node) = code;
234 97773 : return node;
235 : }
236 :
237 : /* Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children. */
238 :
239 : static slp_tree
240 4007655 : vect_create_new_slp_node (slp_tree node,
241 : vec<stmt_vec_info> scalar_stmts, unsigned nops)
242 : {
243 4007655 : SLP_TREE_SCALAR_STMTS (node) = scalar_stmts;
244 4007655 : SLP_TREE_CHILDREN (node).create (nops);
245 4007655 : SLP_TREE_DEF_TYPE (node) = vect_internal_def;
246 4007655 : SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0];
247 4007655 : SLP_TREE_LANES (node) = scalar_stmts.length ();
248 4007655 : return node;
249 : }
250 :
251 : /* Create an SLP node for SCALAR_STMTS and NOPS children. */
252 :
253 : static slp_tree
254 8384 : vect_create_new_slp_node (vec<stmt_vec_info> scalar_stmts, unsigned nops)
255 : {
256 8384 : return vect_create_new_slp_node (new _slp_tree, scalar_stmts, nops);
257 : }
258 :
259 : /* Create a vect_external_def SLP node inplace at NODE for scalar
260 : operands OPS. */
261 :
262 : static slp_tree
263 1980141 : vect_create_new_slp_node (slp_tree node, vec<tree> ops)
264 : {
265 1980141 : SLP_TREE_SCALAR_OPS (node) = ops;
266 1980141 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
267 0 : SLP_TREE_LANES (node) = ops.length ();
268 1980141 : return node;
269 : }
270 :
271 : /* Create a vect_external_def SLP node for scalar operands OPS. */
272 :
273 : static slp_tree
274 1980141 : vect_create_new_slp_node (vec<tree> ops)
275 : {
276 1980141 : return vect_create_new_slp_node (new _slp_tree, ops);
277 : }
278 :
279 :
280 : /* This structure is used in creation of an SLP tree. Each instance
281 : corresponds to the same operand in a group of scalar stmts in an SLP
282 : node. */
283 : typedef struct _slp_oprnd_info
284 : {
285 : /* Def-stmts for the operands. */
286 : vec<stmt_vec_info> def_stmts;
287 : /* Operands. */
288 : vec<tree> ops;
289 : /* Information about the first statement, its vector def-type, type, the
290 : operand itself in case it's constant, and an indication if it's a pattern
291 : stmt and gather/scatter info. */
292 : tree first_op_type;
293 : enum vect_def_type first_dt;
294 : bool any_pattern;
295 : bool first_gs_p;
296 : gather_scatter_info first_gs_info;
297 : } *slp_oprnd_info;
298 :
299 :
300 : /* Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each
301 : operand. */
302 : static vec<slp_oprnd_info>
303 3687985 : vect_create_oprnd_info (int nops, int group_size)
304 : {
305 3687985 : int i;
306 3687985 : slp_oprnd_info oprnd_info;
307 3687985 : vec<slp_oprnd_info> oprnds_info;
308 :
309 3687985 : oprnds_info.create (nops);
310 13201610 : for (i = 0; i < nops; i++)
311 : {
312 5825640 : oprnd_info = XNEW (struct _slp_oprnd_info);
313 5825640 : oprnd_info->def_stmts.create (group_size);
314 5825640 : oprnd_info->ops.create (group_size);
315 5825640 : oprnd_info->first_dt = vect_uninitialized_def;
316 5825640 : oprnd_info->first_op_type = NULL_TREE;
317 5825640 : oprnd_info->any_pattern = false;
318 5825640 : oprnd_info->first_gs_p = false;
319 5825640 : oprnds_info.quick_push (oprnd_info);
320 : }
321 :
322 3687985 : return oprnds_info;
323 : }
324 :
325 :
326 : /* Free operands info. */
327 :
328 : static void
329 3564546 : vect_free_oprnd_info (vec<slp_oprnd_info> &oprnds_info)
330 : {
331 3564546 : int i;
332 3564546 : slp_oprnd_info oprnd_info;
333 :
334 9258309 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
335 : {
336 5693763 : oprnd_info->def_stmts.release ();
337 5693763 : oprnd_info->ops.release ();
338 5693763 : XDELETE (oprnd_info);
339 : }
340 :
341 3564546 : oprnds_info.release ();
342 3564546 : }
343 :
344 : /* Return the execution frequency of NODE (so that a higher value indicates
345 : a "more important" node when optimizing for speed). */
346 :
347 : static sreal
348 3695625 : vect_slp_node_weight (vec_info *vinfo, slp_tree node)
349 : {
350 3695625 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
351 3695625 : basic_block bb;
352 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
353 : but are laid out close to their children. */
354 3695625 : if (!stmt_info)
355 143099 : bb = vinfo->bbs[0];
356 : else
357 4031362 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
358 3695625 : return bb->count.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
359 : }
360 :
361 : /* Return true if STMTS contains a pattern statement. */
362 :
363 : static bool
364 30202 : vect_contains_pattern_stmt_p (vec<stmt_vec_info> stmts)
365 : {
366 30202 : stmt_vec_info stmt_info;
367 30202 : unsigned int i;
368 88002 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
369 65271 : if (stmt_info && is_pattern_stmt_p (stmt_info))
370 : return true;
371 : return false;
372 : }
373 :
374 : /* Return true when all lanes in the external or constant NODE have
375 : the same value. */
376 :
377 : static bool
378 647938 : vect_slp_tree_uniform_p (slp_tree node)
379 : {
380 647938 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_constant_def
381 : || SLP_TREE_DEF_TYPE (node) == vect_external_def);
382 :
383 : /* Pre-existing vectors. */
384 647938 : if (SLP_TREE_SCALAR_OPS (node).is_empty ())
385 : return false;
386 :
387 : unsigned i;
388 : tree op, first = NULL_TREE;
389 1485540 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
390 1326668 : if (!first)
391 : first = op;
392 678730 : else if (!operand_equal_p (first, op, 0))
393 : return false;
394 :
395 : return true;
396 : }
397 :
398 : /* Find the place of the data-ref in STMT_INFO in the interleaving chain
399 : that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part
400 : of the chain. */
401 :
402 : int
403 754663 : vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info,
404 : stmt_vec_info first_stmt_info)
405 : {
406 754663 : stmt_vec_info next_stmt_info = first_stmt_info;
407 754663 : int result = 0;
408 :
409 754663 : if (first_stmt_info != DR_GROUP_FIRST_ELEMENT (stmt_info))
410 : return -1;
411 :
412 2134984 : do
413 : {
414 2134984 : if (next_stmt_info == stmt_info)
415 : return result;
416 1380321 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
417 1380321 : if (next_stmt_info)
418 1380321 : result += DR_GROUP_GAP (next_stmt_info);
419 : }
420 1380321 : while (next_stmt_info);
421 :
422 : return -1;
423 : }
424 :
425 : /* Check whether it is possible to load COUNT elements of type ELT_TYPE
426 : using the method implemented by duplicate_and_interleave. Return true
427 : if so, returning the number of intermediate vectors in *NVECTORS_OUT
428 : (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
429 : (if nonnull). */
430 :
431 : bool
432 0 : can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count,
433 : tree elt_type, unsigned int *nvectors_out,
434 : tree *vector_type_out,
435 : tree *permutes)
436 : {
437 0 : tree base_vector_type = get_vectype_for_scalar_type (vinfo, elt_type, count);
438 0 : if (!base_vector_type || !VECTOR_MODE_P (TYPE_MODE (base_vector_type)))
439 : return false;
440 :
441 0 : machine_mode base_vector_mode = TYPE_MODE (base_vector_type);
442 0 : poly_int64 elt_bytes = count * GET_MODE_UNIT_SIZE (base_vector_mode);
443 0 : unsigned int nvectors = 1;
444 0 : for (;;)
445 : {
446 0 : scalar_int_mode int_mode;
447 0 : poly_int64 elt_bits = elt_bytes * BITS_PER_UNIT;
448 0 : if (int_mode_for_size (elt_bits, 1).exists (&int_mode))
449 : {
450 : /* Get the natural vector type for this SLP group size. */
451 0 : tree int_type = build_nonstandard_integer_type
452 0 : (GET_MODE_BITSIZE (int_mode), 1);
453 0 : tree vector_type
454 0 : = get_vectype_for_scalar_type (vinfo, int_type, count);
455 0 : poly_int64 half_nelts;
456 0 : if (vector_type
457 0 : && VECTOR_MODE_P (TYPE_MODE (vector_type))
458 0 : && known_eq (GET_MODE_SIZE (TYPE_MODE (vector_type)),
459 : GET_MODE_SIZE (base_vector_mode))
460 0 : && multiple_p (GET_MODE_NUNITS (TYPE_MODE (vector_type)),
461 : 2, &half_nelts))
462 : {
463 : /* Try fusing consecutive sequences of COUNT / NVECTORS elements
464 : together into elements of type INT_TYPE and using the result
465 : to build NVECTORS vectors. */
466 0 : poly_uint64 nelts = GET_MODE_NUNITS (TYPE_MODE (vector_type));
467 0 : vec_perm_builder sel1 (nelts, 2, 3);
468 0 : vec_perm_builder sel2 (nelts, 2, 3);
469 :
470 0 : for (unsigned int i = 0; i < 3; ++i)
471 : {
472 0 : sel1.quick_push (i);
473 0 : sel1.quick_push (i + nelts);
474 0 : sel2.quick_push (half_nelts + i);
475 0 : sel2.quick_push (half_nelts + i + nelts);
476 : }
477 0 : vec_perm_indices indices1 (sel1, 2, nelts);
478 0 : vec_perm_indices indices2 (sel2, 2, nelts);
479 0 : machine_mode vmode = TYPE_MODE (vector_type);
480 0 : if (can_vec_perm_const_p (vmode, vmode, indices1)
481 0 : && can_vec_perm_const_p (vmode, vmode, indices2))
482 : {
483 0 : if (nvectors_out)
484 0 : *nvectors_out = nvectors;
485 0 : if (vector_type_out)
486 0 : *vector_type_out = vector_type;
487 0 : if (permutes)
488 : {
489 0 : permutes[0] = vect_gen_perm_mask_checked (vector_type,
490 : indices1);
491 0 : permutes[1] = vect_gen_perm_mask_checked (vector_type,
492 : indices2);
493 : }
494 0 : return true;
495 : }
496 0 : }
497 : }
498 0 : if (!multiple_p (elt_bytes, 2, &elt_bytes))
499 : return false;
500 0 : nvectors *= 2;
501 : /* We need to be able to fuse COUNT / NVECTORS elements together. */
502 0 : if (!multiple_p (count, nvectors))
503 : return false;
504 : }
505 : }
506 :
507 : /* Return true if DTA and DTB match. */
508 :
509 : static bool
510 17706985 : vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
511 : {
512 17706985 : return (dta == dtb
513 417686 : || ((dta == vect_external_def || dta == vect_constant_def)
514 268264 : && (dtb == vect_external_def || dtb == vect_constant_def)));
515 : }
516 :
517 : #define GATHER_SCATTER_OFFSET (-3)
518 :
519 : /* For most SLP statements, there is a one-to-one mapping between
520 : gimple arguments and child nodes. If that is not true for STMT,
521 : return an array that contains:
522 :
523 : - the number of child nodes, followed by
524 : - for each child node, the index of the argument associated with that node.
525 : The special index -1 is the first operand of an embedded comparison and
526 : the special index -2 is the second operand of an embedded comparison.
527 : The special index -3 is the offset of a gather as analyzed by
528 : vect_check_gather_scatter.
529 :
530 : SWAP is as for vect_get_and_check_slp_defs. */
531 :
532 : static const int *
533 25975970 : vect_get_operand_map (const gimple *stmt, bool gather_scatter_p,
534 : unsigned char swap)
535 : {
536 25975970 : static const int no_arg_map[] = { 0 };
537 25975970 : static const int arg0_map[] = { 1, 0 };
538 25975970 : static const int arg2_map[] = { 1, 2 };
539 25975970 : static const int arg2_arg3_map[] = { 2, 2, 3 };
540 25975970 : static const int arg2_arg4_map[] = { 2, 2, 4 };
541 25975970 : static const int arg2_arg5_arg6_map[] = { 3, 2, 5, 6 };
542 25975970 : static const int arg2_arg4_arg5_map[] = { 3, 2, 4, 5 };
543 25975970 : static const int arg3_arg2_map[] = { 2, 3, 2 };
544 25975970 : static const int op00_map[] = { 1, -1 };
545 25975970 : static const int op1_op0_map[] = { 2, 1, 0 };
546 25975970 : static const int off_map[] = { 1, GATHER_SCATTER_OFFSET };
547 25975970 : static const int off_op0_map[] = { 2, GATHER_SCATTER_OFFSET, 0 };
548 25975970 : static const int off_arg2_arg3_map[] = { 3, GATHER_SCATTER_OFFSET, 2, 3 };
549 25975970 : static const int off_arg3_arg2_map[] = { 3, GATHER_SCATTER_OFFSET, 3, 2 };
550 25975970 : static const int mask_call_maps[6][7] = {
551 : { 1, 1, },
552 : { 2, 1, 2, },
553 : { 3, 1, 2, 3, },
554 : { 4, 1, 2, 3, 4, },
555 : { 5, 1, 2, 3, 4, 5, },
556 : { 6, 1, 2, 3, 4, 5, 6 },
557 : };
558 :
559 25975970 : gcc_checking_assert (!swap
560 : || !is_gimple_assign (stmt)
561 : || TREE_CODE_CLASS
562 : (gimple_assign_rhs_code (stmt)) == tcc_comparison
563 : || commutative_tree_code
564 : (gimple_assign_rhs_code (stmt)));
565 :
566 25975970 : if (auto assign = dyn_cast<const gassign *> (stmt))
567 : {
568 24429721 : tree_code code = gimple_assign_rhs_code (assign);
569 24429721 : if (code == COND_EXPR
570 24429721 : && COMPARISON_CLASS_P (gimple_assign_rhs1 (assign)))
571 0 : gcc_unreachable ();
572 24429721 : else if ((TREE_CODE_CLASS (code) == tcc_comparison
573 22891631 : || commutative_tree_code (code))
574 33571754 : && swap)
575 : return op1_op0_map;
576 24387621 : else if (code == VIEW_CONVERT_EXPR)
577 : return op00_map;
578 24378133 : else if (gather_scatter_p)
579 46969 : return (TREE_CODE (gimple_assign_lhs (assign)) != SSA_NAME
580 46969 : ? off_op0_map : off_map);
581 : }
582 1546249 : else if (auto call = dyn_cast<const gcall *> (stmt))
583 : {
584 170818 : if (gimple_call_internal_p (call))
585 94123 : switch (gimple_call_internal_fn (call))
586 : {
587 13928 : case IFN_MASK_LOAD:
588 13928 : return gather_scatter_p ? off_arg2_arg3_map : arg2_arg3_map;
589 :
590 0 : case IFN_GATHER_LOAD:
591 0 : return arg2_map;
592 :
593 0 : case IFN_MASK_GATHER_LOAD:
594 0 : case IFN_MASK_LEN_GATHER_LOAD:
595 0 : return arg2_arg5_arg6_map;
596 :
597 0 : case IFN_SCATTER_STORE:
598 0 : return arg2_arg4_map;
599 :
600 0 : case IFN_MASK_SCATTER_STORE:
601 0 : case IFN_MASK_LEN_SCATTER_STORE:
602 0 : return arg2_arg4_arg5_map;
603 :
604 9200 : case IFN_MASK_STORE:
605 9200 : return gather_scatter_p ? off_arg3_arg2_map : arg3_arg2_map;
606 :
607 996 : case IFN_MASK_CALL:
608 996 : {
609 996 : unsigned nargs = gimple_call_num_args (call);
610 996 : if (nargs >= 2 && nargs <= 7)
611 996 : return mask_call_maps[nargs-2];
612 : else
613 : return nullptr;
614 : }
615 :
616 278 : case IFN_CLZ:
617 278 : case IFN_CTZ:
618 278 : return arg0_map;
619 :
620 7308 : case IFN_GOMP_SIMD_LANE:
621 7308 : return no_arg_map;
622 :
623 : default:
624 : break;
625 : }
626 : }
627 : return nullptr;
628 : }
629 :
630 : static const int *
631 25956921 : vect_get_operand_map (const stmt_vec_info stmt, unsigned char swap = 0)
632 : {
633 0 : return vect_get_operand_map (stmt->stmt, STMT_VINFO_GATHER_SCATTER_P (stmt),
634 0 : swap);
635 : }
636 :
637 : /* Return the SLP node child index for operand OP of STMT. */
638 :
639 : int
640 1395573 : vect_slp_child_index_for_operand (const stmt_vec_info stmt, int op)
641 : {
642 1395573 : const int *opmap = vect_get_operand_map (stmt);
643 1395573 : if (!opmap)
644 : return op;
645 20323 : for (int i = 1; i < 1 + opmap[0]; ++i)
646 20323 : if (opmap[i] == op)
647 11561 : return i - 1;
648 0 : gcc_unreachable ();
649 : }
650 :
651 : /* Helper class for mapping of GIMPLE operands to SLP children. */
652 : /* ??? Add vect_slp_child_index_for_operand here and amend opmaps
653 : with the full reverse mapping and indicating the position of the
654 : first commutative operand index, eliding the swap_p argument from
655 : vect_get_operand_map. Adjust all consumers. */
656 :
657 : struct slp_oprnds {
658 : slp_oprnds (stmt_vec_info);
659 : tree get_op_for_slp_child (stmt_vec_info, unsigned);
660 : const int *opmap;
661 : const unsigned int num_slp_children;
662 : };
663 :
664 4645434 : slp_oprnds::slp_oprnds (stmt_vec_info stmt_info)
665 4645434 : : opmap (vect_get_operand_map (stmt_info)),
666 4645434 : num_slp_children (opmap ? opmap[0] : gimple_num_args (stmt_info->stmt))
667 : {
668 4645434 : }
669 :
670 : /* For SLP child number N get the corresponding tree operand from GIMPLE
671 : statement described by STMT_INFO. */
672 :
673 : tree
674 5183731 : slp_oprnds::get_op_for_slp_child (stmt_vec_info stmt_info, unsigned n)
675 : {
676 5183731 : gcc_assert (n < num_slp_children);
677 5183731 : int opno = opmap ? opmap[n + 1] : (int) n;
678 5183731 : if (opno == GATHER_SCATTER_OFFSET)
679 0 : gcc_unreachable (); // TODO
680 5183731 : else if (opno < 0)
681 2602 : return TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
682 : else
683 5181129 : return gimple_arg (stmt_info->stmt, opno);
684 : }
685 :
686 : /* Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
687 : they are of a valid type and that they match the defs of the first stmt of
688 : the SLP group (stored in OPRNDS_INFO). This function tries to match stmts
689 : by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP
690 : indicates swap is required for cond_expr stmts. Specifically, SWAP
691 : is 1 if STMT is cond and operands of comparison need to be swapped;
692 : SWAP is 2 if STMT is cond and code of comparison needs to be inverted.
693 :
694 : If there was a fatal error return -1; if the error could be corrected by
695 : swapping operands of father node of this one, return 1; if everything is
696 : ok return 0. */
697 : static int
698 13720414 : vect_get_and_check_slp_defs (vec_info *vinfo, tree vectype, unsigned char swap,
699 : bool *skip_args,
700 : vec<stmt_vec_info> stmts, unsigned stmt_num,
701 : vec<slp_oprnd_info> *oprnds_info,
702 : bool soft_fail)
703 : {
704 13720414 : stmt_vec_info stmt_info = stmts[stmt_num];
705 13720414 : tree oprnd;
706 13720414 : unsigned int i, number_of_oprnds;
707 13720414 : enum vect_def_type dt = vect_uninitialized_def;
708 13720414 : slp_oprnd_info oprnd_info;
709 13720414 : gather_scatter_info gs_info;
710 13720414 : unsigned int gs_op = -1u;
711 13720414 : unsigned int commutative_op = -1U;
712 13720414 : bool first = stmt_num == 0;
713 :
714 13720414 : if (!stmt_info)
715 : {
716 0 : for (auto oi : *oprnds_info)
717 : {
718 0 : oi->def_stmts.quick_push (NULL);
719 0 : oi->ops.quick_push (NULL_TREE);
720 : }
721 : return 0;
722 : }
723 :
724 13720414 : if (!is_a<gcall *> (stmt_info->stmt)
725 : && !is_a<gassign *> (stmt_info->stmt)
726 : && !is_a<gphi *> (stmt_info->stmt))
727 : return -1;
728 :
729 13720414 : number_of_oprnds = gimple_num_args (stmt_info->stmt);
730 13720414 : const int *map = vect_get_operand_map (stmt_info, swap);
731 13720414 : if (map)
732 79255 : number_of_oprnds = *map++;
733 13720414 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
734 : {
735 54372 : if (gimple_call_internal_p (stmt))
736 : {
737 33903 : internal_fn ifn = gimple_call_internal_fn (stmt);
738 33903 : commutative_op = first_commutative_argument (ifn);
739 33903 : if (internal_gather_scatter_fn_p (ifn))
740 : {
741 0 : vect_describe_gather_scatter_call
742 0 : (stmt_info,
743 0 : first ? &(*oprnds_info)[0]->first_gs_info : &gs_info);
744 0 : if (first)
745 0 : (*oprnds_info)[0]->first_gs_p = true;
746 : gs_op = 0;
747 : }
748 : }
749 : }
750 13666042 : else if (gassign *stmt = dyn_cast <gassign *> (stmt_info->stmt))
751 : {
752 16415904 : if (commutative_tree_code (gimple_assign_rhs_code (stmt)))
753 8539154 : commutative_op = 0;
754 : }
755 :
756 13720414 : bool swapped = (swap != 0);
757 13720414 : bool backedge = false;
758 13720414 : enum vect_def_type *dts = XALLOCAVEC (enum vect_def_type, number_of_oprnds);
759 37459204 : for (i = 0; i < number_of_oprnds; i++)
760 : {
761 23740153 : oprnd_info = (*oprnds_info)[i];
762 23740153 : int opno = map ? map[i] : int (i);
763 23740153 : if (opno == GATHER_SCATTER_OFFSET)
764 : {
765 23917 : gcc_assert (STMT_VINFO_GATHER_SCATTER_P (stmt_info));
766 23917 : if (!is_a <loop_vec_info> (vinfo)
767 23917 : || !vect_check_gather_scatter (stmt_info, vectype,
768 : as_a <loop_vec_info> (vinfo),
769 : first ? &oprnd_info->first_gs_info
770 : : &gs_info))
771 1363 : return -1;
772 :
773 23917 : if (first)
774 : {
775 23643 : oprnd_info->first_gs_p = true;
776 23643 : oprnd = oprnd_info->first_gs_info.offset;
777 : }
778 : else
779 : {
780 274 : gs_op = i;
781 274 : oprnd = gs_info.offset;
782 : }
783 : }
784 23716236 : else if (opno < 0)
785 3267 : oprnd = TREE_OPERAND (gimple_arg (stmt_info->stmt, 0), -1 - opno);
786 : else
787 : {
788 23712969 : oprnd = gimple_arg (stmt_info->stmt, opno);
789 23712969 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
790 : {
791 1279705 : edge e = gimple_phi_arg_edge (stmt, opno);
792 2559410 : backedge = (is_a <bb_vec_info> (vinfo)
793 1982387 : ? e->flags & EDGE_DFS_BACK
794 702682 : : dominated_by_p (CDI_DOMINATORS, e->src,
795 702682 : gimple_bb (stmt_info->stmt)));
796 : }
797 : }
798 :
799 23740153 : stmt_vec_info def_stmt_info;
800 23740153 : if (!vect_is_simple_use (oprnd, vinfo, &dts[i], &def_stmt_info))
801 : {
802 1052 : if (dump_enabled_p ())
803 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
804 : "Build SLP failed: can't analyze def for %T\n",
805 : oprnd);
806 :
807 : return -1;
808 : }
809 :
810 23739101 : if (skip_args[i])
811 : {
812 565381 : oprnd_info->def_stmts.quick_push (NULL);
813 565381 : oprnd_info->ops.quick_push (NULL_TREE);
814 565381 : oprnd_info->first_dt = vect_uninitialized_def;
815 565381 : continue;
816 : }
817 :
818 23173720 : oprnd_info->def_stmts.quick_push (def_stmt_info);
819 23173720 : oprnd_info->ops.quick_push (oprnd);
820 :
821 23173720 : if (def_stmt_info
822 23173720 : && is_pattern_stmt_p (def_stmt_info))
823 : {
824 466552 : if (STMT_VINFO_RELATED_STMT (vect_orig_stmt (def_stmt_info))
825 : != def_stmt_info)
826 326451 : oprnd_info->any_pattern = true;
827 : else
828 : /* If we promote this to external use the original stmt def. */
829 140101 : oprnd_info->ops.last ()
830 280202 : = gimple_get_lhs (vect_orig_stmt (def_stmt_info)->stmt);
831 : }
832 :
833 : /* If there's a extern def on a backedge make sure we can
834 : code-generate at the region start.
835 : ??? This is another case that could be fixed by adjusting
836 : how we split the function but at the moment we'd have conflicting
837 : goals there. */
838 23173720 : if (backedge
839 175100 : && dts[i] == vect_external_def
840 332 : && is_a <bb_vec_info> (vinfo)
841 332 : && TREE_CODE (oprnd) == SSA_NAME
842 311 : && !SSA_NAME_IS_DEFAULT_DEF (oprnd)
843 23174031 : && !dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
844 311 : gimple_bb (SSA_NAME_DEF_STMT (oprnd))))
845 : {
846 311 : if (dump_enabled_p ())
847 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
848 : "Build SLP failed: extern def %T only defined "
849 : "on backedge\n", oprnd);
850 : return -1;
851 : }
852 :
853 23173409 : if (first)
854 : {
855 5299418 : tree type = TREE_TYPE (oprnd);
856 5299418 : dt = dts[i];
857 :
858 : /* For the swapping logic below force vect_reduction_def
859 : for the reduction op in a SLP reduction group. */
860 5299418 : if (!STMT_VINFO_DATA_REF (stmt_info)
861 3983215 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
862 5312 : && (int)i == STMT_VINFO_REDUC_IDX (stmt_info)
863 5302031 : && def_stmt_info)
864 2613 : dts[i] = dt = vect_reduction_def;
865 :
866 : /* Check the types of the definition. */
867 5299418 : switch (dt)
868 : {
869 5299418 : case vect_external_def:
870 5299418 : case vect_constant_def:
871 5299418 : case vect_internal_def:
872 5299418 : case vect_reduction_def:
873 5299418 : case vect_double_reduction_def:
874 5299418 : case vect_induction_def:
875 5299418 : case vect_nested_cycle:
876 5299418 : case vect_first_order_recurrence:
877 5299418 : break;
878 :
879 0 : default:
880 : /* FORNOW: Not supported. */
881 0 : if (dump_enabled_p ())
882 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
883 : "Build SLP failed: illegal type of def %T\n",
884 : oprnd);
885 : return -1;
886 : }
887 :
888 5299418 : oprnd_info->first_dt = dt;
889 5299418 : oprnd_info->first_op_type = type;
890 : }
891 : }
892 13719051 : if (first)
893 : return 0;
894 :
895 : /* Now match the operand definition types to that of the first stmt. */
896 27561682 : for (i = 0; i < number_of_oprnds;)
897 : {
898 17864043 : if (skip_args[i])
899 : {
900 44960 : ++i;
901 44960 : continue;
902 : }
903 :
904 17819083 : oprnd_info = (*oprnds_info)[i];
905 17819083 : dt = dts[i];
906 17819083 : stmt_vec_info def_stmt_info = oprnd_info->def_stmts[stmt_num];
907 17819083 : oprnd = oprnd_info->ops[stmt_num];
908 17819083 : tree type = TREE_TYPE (oprnd);
909 :
910 17819083 : if (!types_compatible_p (oprnd_info->first_op_type, type))
911 : {
912 119334 : if (dump_enabled_p ())
913 93 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
914 : "Build SLP failed: different operand types\n");
915 : return 1;
916 : }
917 :
918 17699749 : if ((gs_op == i) != oprnd_info->first_gs_p)
919 : {
920 0 : if (dump_enabled_p ())
921 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
922 : "Build SLP failed: mixed gather and non-gather\n");
923 : return 1;
924 : }
925 17699749 : else if (gs_op == i)
926 : {
927 242 : if (!operand_equal_p (oprnd_info->first_gs_info.base,
928 242 : gs_info.base))
929 : {
930 16 : if (dump_enabled_p ())
931 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
932 : "Build SLP failed: different gather base\n");
933 : return 1;
934 : }
935 226 : if (oprnd_info->first_gs_info.scale != gs_info.scale)
936 : {
937 8 : if (dump_enabled_p ())
938 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
939 : "Build SLP failed: different gather scale\n");
940 : return 1;
941 : }
942 : }
943 :
944 : /* Not first stmt of the group, check that the def-stmt/s match
945 : the def-stmt/s of the first stmt. Allow different definition
946 : types for reduction chains: the first stmt must be a
947 : vect_reduction_def (a phi node), and the rest
948 : end in the reduction chain. */
949 17699725 : if ((!vect_def_types_match (oprnd_info->first_dt, dt)
950 334288 : && !(oprnd_info->first_dt == vect_reduction_def
951 5019 : && !STMT_VINFO_DATA_REF (stmt_info)
952 5019 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
953 4993 : && def_stmt_info
954 4991 : && !STMT_VINFO_DATA_REF (def_stmt_info)
955 4991 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
956 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
957 17370428 : || (!STMT_VINFO_DATA_REF (stmt_info)
958 15580856 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
959 10429 : && ((!def_stmt_info
960 10193 : || STMT_VINFO_DATA_REF (def_stmt_info)
961 18668 : || (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
962 : != REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
963 10429 : != (oprnd_info->first_dt != vect_reduction_def))))
964 : {
965 : /* Try swapping operands if we got a mismatch. For BB
966 : vectorization only in case it will clearly improve things. */
967 331932 : if (i == commutative_op && !swapped
968 329297 : && (!is_a <bb_vec_info> (vinfo)
969 5607 : || (!vect_def_types_match ((*oprnds_info)[i+1]->first_dt,
970 5607 : dts[i+1])
971 1420 : && (vect_def_types_match (oprnd_info->first_dt, dts[i+1])
972 : || vect_def_types_match
973 233 : ((*oprnds_info)[i+1]->first_dt, dts[i])))))
974 : {
975 2635 : if (dump_enabled_p ())
976 198 : dump_printf_loc (MSG_NOTE, vect_location,
977 : "trying swapped operands\n");
978 2635 : std::swap (dts[i], dts[i+1]);
979 2635 : std::swap ((*oprnds_info)[i]->def_stmts[stmt_num],
980 2635 : (*oprnds_info)[i+1]->def_stmts[stmt_num]);
981 2635 : std::swap ((*oprnds_info)[i]->ops[stmt_num],
982 2635 : (*oprnds_info)[i+1]->ops[stmt_num]);
983 : /* After swapping some operands we lost track whether an
984 : operand has any pattern defs so be conservative here. */
985 2635 : if ((*oprnds_info)[i]->any_pattern
986 2635 : || (*oprnds_info)[i+1]->any_pattern)
987 36 : (*oprnds_info)[i]->any_pattern
988 18 : = (*oprnds_info)[i+1]->any_pattern = true;
989 2635 : swapped = true;
990 2635 : continue;
991 : }
992 :
993 326662 : if (is_a <bb_vec_info> (vinfo)
994 309337 : && !oprnd_info->any_pattern
995 309069 : && number_of_oprnds > 1
996 635731 : && !soft_fail)
997 : {
998 : /* Now for commutative ops we should see whether we can
999 : make the other operand matching. */
1000 107223 : if (dump_enabled_p ())
1001 338 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1002 : "treating operand as external\n");
1003 : oprnd_info->first_dt = dt = vect_external_def;
1004 : }
1005 : else
1006 : {
1007 219439 : if (dump_enabled_p ())
1008 410 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1009 : "Build SLP failed: different types\n");
1010 : return 1;
1011 : }
1012 : }
1013 :
1014 : /* Make sure to demote the overall operand to external. */
1015 17370428 : if (dt == vect_external_def)
1016 426444 : oprnd_info->first_dt = vect_external_def;
1017 : /* For a SLP reduction chain we want to duplicate the reduction to
1018 : each of the chain members. That gets us a sane SLP graph (still
1019 : the stmts are not 100% correct wrt the initial values). */
1020 17051207 : else if ((dt == vect_internal_def
1021 17051207 : || dt == vect_reduction_def)
1022 15788112 : && oprnd_info->first_dt == vect_reduction_def
1023 101523 : && !STMT_VINFO_DATA_REF (stmt_info)
1024 101523 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info)
1025 4991 : && !STMT_VINFO_DATA_REF (def_stmt_info)
1026 17056198 : && (REDUC_GROUP_FIRST_ELEMENT (def_stmt_info)
1027 : == REDUC_GROUP_FIRST_ELEMENT (stmt_info)))
1028 : {
1029 4991 : oprnd_info->def_stmts[stmt_num] = oprnd_info->def_stmts[0];
1030 4991 : oprnd_info->ops[stmt_num] = oprnd_info->ops[0];
1031 : }
1032 :
1033 17477651 : ++i;
1034 : }
1035 :
1036 : /* Swap operands. */
1037 9697639 : if (swapped)
1038 : {
1039 41816 : if (dump_enabled_p ())
1040 502 : dump_printf_loc (MSG_NOTE, vect_location,
1041 : "swapped operands to match def types in %G",
1042 : stmt_info->stmt);
1043 : }
1044 :
1045 : return 0;
1046 : }
1047 :
1048 : /* Return true if call statements CALL1 and CALL2 are similar enough
1049 : to be combined into the same SLP group. */
1050 :
1051 : bool
1052 65587 : compatible_calls_p (gcall *call1, gcall *call2, bool allow_two_operators)
1053 : {
1054 65587 : unsigned int nargs = gimple_call_num_args (call1);
1055 65587 : if (nargs != gimple_call_num_args (call2))
1056 : return false;
1057 :
1058 48499 : auto cfn1 = gimple_call_combined_fn (call1);
1059 48499 : auto cfn2 = gimple_call_combined_fn (call2);
1060 48499 : if (cfn1 != cfn2
1061 39 : && (!allow_two_operators
1062 39 : || !((cfn1 == CFN_FMA || cfn1 == CFN_FMS)
1063 2 : && (cfn2 == CFN_FMA || cfn2 == CFN_FMS))))
1064 : return false;
1065 :
1066 48462 : if (gimple_call_internal_p (call1))
1067 : {
1068 7739 : if (!types_compatible_p (TREE_TYPE (gimple_call_lhs (call1)),
1069 7739 : TREE_TYPE (gimple_call_lhs (call2))))
1070 : return false;
1071 16381 : for (unsigned int i = 0; i < nargs; ++i)
1072 8642 : if (!types_compatible_p (TREE_TYPE (gimple_call_arg (call1, i)),
1073 8642 : TREE_TYPE (gimple_call_arg (call2, i))))
1074 : return false;
1075 : }
1076 : else
1077 : {
1078 40723 : if (!operand_equal_p (gimple_call_fn (call1),
1079 40723 : gimple_call_fn (call2), 0))
1080 : return false;
1081 :
1082 33930 : if (gimple_call_fntype (call1) != gimple_call_fntype (call2))
1083 : return false;
1084 : }
1085 :
1086 : /* Check that any unvectorized arguments are equal. */
1087 19049 : if (const int *map = vect_get_operand_map (call1, false, false))
1088 : {
1089 15 : unsigned int nkept = *map++;
1090 15 : unsigned int mapi = 0;
1091 57 : for (unsigned int i = 0; i < nargs; ++i)
1092 42 : if (mapi < nkept && map[mapi] == int (i))
1093 27 : mapi += 1;
1094 15 : else if (!operand_equal_p (gimple_call_arg (call1, i),
1095 15 : gimple_call_arg (call2, i)))
1096 : return false;
1097 : }
1098 :
1099 : return true;
1100 : }
1101 :
1102 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1103 : permutation or are of unsupported types of operation.
1104 : Return false if at least one (or all in case of BB vectorization)
1105 : stmt is unvectorizable or the comparison could not be carried out.
1106 : Return true if they all (or at least one in case of BB vectorization)
1107 : are and indicate in MATCHES[] which stmts are not isomorphic to the
1108 : stmt at *START_I.
1109 :
1110 : This function is designed to be invoked repeatedly on the same
1111 : set of STMTS with increasing *START_I.
1112 :
1113 : When *START_I is zero and the function returns true MATCHES[] will be
1114 : initialized with, in case of BB vectorization, unvectorizable stmts
1115 : marked with -2, stmts isomorphic to *START_I as *START_I and
1116 : other vectorizable stmts with -1 (not isomorphic to *START_I).
1117 : *START_I, when initially zero, is updated to the first vectorizable
1118 : statement, so MATCHES[] can have a prefix with entries valued -2.
1119 :
1120 : When *START_I is not zero MATCHES[] is expected to be pre-initialized
1121 : by a former call with *START_I zero and MATCHES[*START_I] should be -1.
1122 : The function will return true and have the stmts isomorphic to
1123 : MATCHES[*START_I] marked with *START_I.
1124 :
1125 : *TWO_OPERATORS indicates whether the group of isomorphic statements
1126 : uses two related operations like PLUS_EXPR and MINUS_EXPR. If
1127 : TWO_OPERATORS is NULL such case is not considered isomorphic.
1128 :
1129 : SWAP[] indicates whether for a stmt to be isomorphic to its group
1130 : leader, has to have its operands swapped (1) or its predicate inverted (2).
1131 : COND_EXPR is possibly isomorphic to another one after swapping its
1132 : operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
1133 : the first stmt by swapping the two operands of comparison; set SWAP[i]
1134 : to 2 if stmt I is isormorphic to the first stmt by inverting the code
1135 : of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped
1136 : to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1. */
1137 :
1138 : static bool
1139 6319530 : vect_build_slp_tree_3 (vec_info *vinfo, vec<stmt_vec_info> stmts,
1140 : match_elt_t *matches, unsigned char *swap,
1141 : bool *two_operators, tree vectype, unsigned *start_i)
1142 : {
1143 6319530 : stmt_vec_info first_stmt_info = NULL;
1144 6319530 : code_helper first_stmt_code = ERROR_MARK;
1145 6319530 : code_helper alt_stmt_code = ERROR_MARK;
1146 6319530 : code_helper first_cond_code = ERROR_MARK;
1147 6319530 : bool need_same_oprnds = false;
1148 6319530 : tree first_lhs = NULL_TREE;
1149 6319530 : tree first_op1 = NULL_TREE;
1150 6319530 : stmt_vec_info first_load = NULL, prev_first_load = NULL;
1151 6319530 : bool first_stmt_ldst_p = false, first_stmt_ldst_masklen_p = false;
1152 6319530 : bool first_stmt_phi_p = false;
1153 6319530 : int first_reduc_idx = -1;
1154 :
1155 6319530 : basic_block common_bb = NULL;
1156 6319530 : gimple *trapping_stmt = NULL;
1157 6319530 : int first_match = -1;
1158 :
1159 26461482 : for (unsigned i = *start_i; i < stmts.length (); ++i)
1160 : {
1161 20142223 : stmt_vec_info stmt_info = stmts[i];
1162 20142223 : bool ldst_p = false;
1163 20142223 : bool ldst_masklen_p = false;
1164 20142223 : bool phi_p = false;
1165 20142223 : code_helper rhs_code = ERROR_MARK;
1166 :
1167 20142223 : if (*start_i == 0)
1168 : {
1169 18088950 : swap[i] = 0;
1170 18088950 : matches[i] = -1;
1171 : }
1172 2053273 : else if (matches[i] != -1)
1173 20141952 : continue;
1174 :
1175 20051649 : if (!stmt_info)
1176 : {
1177 : /* ??? We shouldn't run into this. */
1178 40987 : gcc_assert (first_match != -1);
1179 40987 : matches[i] = first_match;
1180 40987 : continue;
1181 : }
1182 :
1183 20010662 : gimple *stmt = stmt_info->stmt;
1184 20010662 : if (dump_enabled_p ())
1185 241147 : dump_printf_loc (MSG_NOTE, vect_location, "Build SLP for %G", stmt);
1186 :
1187 : /* Fail to vectorize statements marked as unvectorizable, throw
1188 : or are volatile. */
1189 20010662 : if (!STMT_VINFO_VECTORIZABLE (stmt_info)
1190 19623788 : || stmt_can_throw_internal (cfun, stmt)
1191 38621973 : || gimple_has_volatile_ops (stmt))
1192 : {
1193 404167 : if (dump_enabled_p ())
1194 466 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1195 : "Build SLP failed: unvectorizable statement %G",
1196 : stmt);
1197 404167 : if (is_a <bb_vec_info> (vinfo))
1198 : {
1199 404167 : matches[i] = -2;
1200 404167 : continue;
1201 : }
1202 6319530 : return false;
1203 : }
1204 :
1205 19606495 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
1206 19606495 : tree lhs = gimple_get_lhs (stmt);
1207 19606495 : if (lhs == NULL_TREE && !call_stmt)
1208 : {
1209 34 : if (dump_enabled_p ())
1210 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1211 : "Build SLP failed: not GIMPLE_ASSIGN nor "
1212 : "GIMPLE_CALL %G", stmt);
1213 34 : if (is_a <bb_vec_info> (vinfo))
1214 : {
1215 34 : matches[i] = -2;
1216 34 : continue;
1217 : }
1218 : return false;
1219 : }
1220 :
1221 19606461 : if (call_stmt)
1222 : {
1223 205825 : combined_fn cfn = gimple_call_combined_fn (call_stmt);
1224 205825 : if (cfn != CFN_LAST && cfn != CFN_MASK_CALL)
1225 63799 : rhs_code = cfn;
1226 : else
1227 : rhs_code = CALL_EXPR;
1228 :
1229 205825 : if (cfn == CFN_GATHER_LOAD
1230 205825 : || cfn == CFN_SCATTER_STORE)
1231 : ldst_p = true;
1232 : else if (cfn == CFN_MASK_LOAD
1233 : || cfn == CFN_MASK_GATHER_LOAD
1234 : || cfn == CFN_MASK_LEN_GATHER_LOAD
1235 : || cfn == CFN_MASK_SCATTER_STORE
1236 : || cfn == CFN_MASK_LEN_SCATTER_STORE)
1237 : {
1238 : ldst_p = true;
1239 : ldst_masklen_p = true;
1240 : }
1241 : else if (cfn == CFN_MASK_STORE)
1242 : {
1243 : ldst_p = true;
1244 : ldst_masklen_p = true;
1245 : rhs_code = CFN_MASK_STORE;
1246 : }
1247 : else if (cfn == CFN_GOMP_SIMD_LANE)
1248 : ;
1249 194721 : else if ((cfn != CFN_LAST
1250 : && cfn != CFN_MASK_CALL
1251 52695 : && internal_fn_p (cfn)
1252 39990 : && !vectorizable_internal_fn_p (as_internal_fn (cfn)))
1253 194610 : || gimple_call_tail_p (call_stmt)
1254 194610 : || gimple_call_noreturn_p (call_stmt)
1255 389331 : || gimple_call_chain (call_stmt))
1256 : {
1257 1394 : if (dump_enabled_p ())
1258 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1259 : "Build SLP failed: unsupported call type %G",
1260 : (gimple *) call_stmt);
1261 1394 : if (is_a <bb_vec_info> (vinfo))
1262 : {
1263 1394 : matches[i] = -2;
1264 1394 : continue;
1265 : }
1266 : return false;
1267 : }
1268 : }
1269 19400636 : else if (gimple_code (stmt) == GIMPLE_PHI)
1270 : {
1271 19556992 : rhs_code = ERROR_MARK;
1272 : phi_p = true;
1273 : }
1274 : else
1275 : {
1276 18388159 : rhs_code = gimple_assign_rhs_code (stmt);
1277 18388159 : ldst_p = STMT_VINFO_DATA_REF (stmt_info) != nullptr;
1278 : }
1279 :
1280 18388159 : if (!ldst_p
1281 : && !phi_p
1282 13284899 : && rhs_code.is_tree_code ()
1283 13228661 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_binary
1284 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_unary
1285 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_expression
1286 : && TREE_CODE_CLASS (tree_code (rhs_code)) != tcc_comparison
1287 240296 : && rhs_code != VIEW_CONVERT_EXPR
1288 : && rhs_code != CALL_EXPR
1289 : && rhs_code != BIT_FIELD_REF
1290 18388159 : && rhs_code != SSA_NAME)
1291 : {
1292 48075 : if (dump_enabled_p ())
1293 98 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1294 : "Build SLP failed: operation unsupported %G",
1295 : stmt);
1296 48075 : if (is_a <bb_vec_info> (vinfo))
1297 : {
1298 47930 : matches[i] = -2;
1299 47930 : continue;
1300 : }
1301 : return false;
1302 : }
1303 :
1304 : /* Non-grouped store or load. */
1305 19556992 : if (ldst_p
1306 5307691 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
1307 1134127 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1308 856179 : && rhs_code != CFN_GATHER_LOAD
1309 856179 : && rhs_code != CFN_MASK_GATHER_LOAD
1310 856179 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1311 856179 : && rhs_code != CFN_SCATTER_STORE
1312 856179 : && rhs_code != CFN_MASK_SCATTER_STORE
1313 856179 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1314 856179 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1315 : /* Not grouped loads are handled as externals for BB
1316 : vectorization. Treat them as not vectorizable. */
1317 20393441 : && is_a <bb_vec_info> (vinfo))
1318 : {
1319 : /* Not grouped load. */
1320 0 : if (dump_enabled_p ())
1321 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1322 : "Build SLP failed: not grouped load %G", stmt);
1323 0 : matches[i] = -2;
1324 0 : continue;
1325 : }
1326 :
1327 19556992 : if (!ldst_p && rhs_code == BIT_FIELD_REF)
1328 : {
1329 44767 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
1330 44767 : if (!is_a <bb_vec_info> (vinfo)
1331 44641 : || TREE_CODE (vec) != SSA_NAME
1332 : /* When the element types are not compatible we pun the
1333 : source to the target vectype which requires equal size. */
1334 89384 : || ((!VECTOR_TYPE_P (TREE_TYPE (vec))
1335 41883 : || !types_compatible_p (TREE_TYPE (vectype),
1336 41883 : TREE_TYPE (TREE_TYPE (vec))))
1337 3997 : && !operand_equal_p (TYPE_SIZE (vectype),
1338 3997 : TYPE_SIZE (TREE_TYPE (vec)))))
1339 : {
1340 2288 : if (dump_enabled_p ())
1341 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1342 : "Build SLP failed: "
1343 : "BIT_FIELD_REF not supported\n");
1344 2288 : if (is_a <bb_vec_info> (vinfo))
1345 : {
1346 2162 : matches[i] = -2;
1347 2162 : continue;
1348 : }
1349 : return false;
1350 : }
1351 : }
1352 :
1353 : /* Check the operation. */
1354 19554704 : if (first_match == -1)
1355 : {
1356 6217878 : first_match = i;
1357 6217878 : first_stmt_info = stmt_info;
1358 6217878 : common_bb = gimple_bb (stmt_info->stmt);
1359 6217878 : first_lhs = lhs;
1360 6217878 : first_stmt_code = rhs_code;
1361 6217878 : first_stmt_ldst_p = ldst_p;
1362 6217878 : first_stmt_ldst_masklen_p = ldst_masklen_p;
1363 6217878 : first_stmt_phi_p = phi_p;
1364 6217878 : first_reduc_idx = STMT_VINFO_REDUC_IDX (stmt_info);
1365 :
1366 : /* Shift arguments should be equal in all the packed stmts for a
1367 : vector shift with scalar shift operand. */
1368 6217878 : if (rhs_code == LSHIFT_EXPR
1369 6118623 : || rhs_code == RSHIFT_EXPR
1370 6025049 : || rhs_code == LROTATE_EXPR
1371 12242813 : || rhs_code == RROTATE_EXPR)
1372 : {
1373 : /* First see if we have a vector/vector shift. */
1374 193428 : if (!directly_supported_p (rhs_code, vectype, optab_vector))
1375 : {
1376 : /* No vector/vector shift, arrange for a vector/scalar
1377 : SLP layout. */
1378 180035 : need_same_oprnds = true;
1379 180035 : first_op1 = gimple_assign_rhs2 (stmt);
1380 : }
1381 : }
1382 6024450 : else if (rhs_code == WIDEN_LSHIFT_EXPR)
1383 : {
1384 0 : need_same_oprnds = true;
1385 0 : first_op1 = gimple_assign_rhs2 (stmt);
1386 : }
1387 6024450 : else if (rhs_code == CFN_DIV_POW2)
1388 : {
1389 0 : need_same_oprnds = true;
1390 0 : first_op1 = gimple_call_arg (call_stmt, 1);
1391 : }
1392 6024450 : else if (rhs_code == CFN_GOMP_SIMD_LANE)
1393 : {
1394 3654 : need_same_oprnds = true;
1395 3654 : first_op1 = gimple_call_arg (call_stmt, 1);
1396 : }
1397 : }
1398 : else
1399 : {
1400 13336826 : int comm_arg;
1401 13337265 : if (first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1402 : /* For SLP reduction groups the index isn't necessarily
1403 : uniform but only that of the first stmt matters. */
1404 2525 : && !(first_reduc_idx != -1
1405 2525 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1406 2525 : && REDUC_GROUP_FIRST_ELEMENT (stmt_info))
1407 13336826 : && !(first_reduc_idx != -1
1408 1187 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1409 1187 : && (comm_arg = first_commutative_argument
1410 1187 : (rhs_code, TREE_TYPE (lhs))) >= 0
1411 : && (first_reduc_idx
1412 932 : == 2 * comm_arg + 1 - STMT_VINFO_REDUC_IDX (stmt_info))))
1413 : {
1414 439 : if (dump_enabled_p ())
1415 : {
1416 12 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1417 : "Build SLP failed: different reduc_idx "
1418 : "%d instead of %d in %G",
1419 : STMT_VINFO_REDUC_IDX (stmt_info),
1420 : first_reduc_idx, stmt);
1421 : }
1422 : /* Mismatch. */
1423 439 : continue;
1424 : }
1425 13336387 : if (!ldst_p
1426 13336387 : && two_operators
1427 10487017 : && first_stmt_code != rhs_code
1428 14857580 : && alt_stmt_code == ERROR_MARK)
1429 : alt_stmt_code = rhs_code;
1430 14838692 : if ((!ldst_p
1431 10487017 : && first_stmt_code != rhs_code
1432 1521193 : && (first_stmt_code != IMAGPART_EXPR
1433 202 : || rhs_code != REALPART_EXPR)
1434 1521193 : && (first_stmt_code != REALPART_EXPR
1435 735 : || rhs_code != IMAGPART_EXPR)
1436 : /* Handle mismatches in plus/minus by computing both
1437 : and merging the results. */
1438 1521193 : && !((((first_stmt_code == PLUS_EXPR
1439 1414053 : || first_stmt_code == MINUS_EXPR)
1440 131429 : && (alt_stmt_code == PLUS_EXPR
1441 121488 : || alt_stmt_code == MINUS_EXPR))
1442 1489260 : || ((first_stmt_code == CFN_FMA
1443 1489258 : || first_stmt_code == CFN_FMS)
1444 2 : && (alt_stmt_code == CFN_FMA
1445 2 : || alt_stmt_code == CFN_FMS)))
1446 31935 : && rhs_code == alt_stmt_code)
1447 1531116 : && !(first_stmt_code.is_tree_code ()
1448 1400068 : && rhs_code.is_tree_code ()
1449 1285949 : && (TREE_CODE_CLASS (tree_code (first_stmt_code))
1450 : == tcc_comparison)
1451 140791 : && (swap_tree_comparison (tree_code (first_stmt_code))
1452 140791 : == tree_code (rhs_code))
1453 : && (first_reduc_idx == -1
1454 0 : || REDUC_GROUP_FIRST_ELEMENT (stmt_info))))
1455 : || (ldst_p
1456 5698740 : && (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1457 2849370 : != STMT_VINFO_GROUPED_ACCESS (first_stmt_info)))
1458 : || (ldst_p
1459 2795692 : && (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1460 2795692 : != STMT_VINFO_GATHER_SCATTER_P (first_stmt_info)))
1461 11834220 : || first_stmt_ldst_p != ldst_p
1462 11834090 : || (ldst_p && first_stmt_ldst_masklen_p != ldst_masklen_p)
1463 25170469 : || first_stmt_phi_p != phi_p)
1464 : {
1465 1502305 : if (dump_enabled_p ())
1466 : {
1467 3636 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1468 : "Build SLP failed: different operation "
1469 : "in stmt %G", stmt);
1470 3636 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1471 : "original stmt %G", first_stmt_info->stmt);
1472 : }
1473 : /* Mismatch. */
1474 1502305 : continue;
1475 : }
1476 :
1477 11853269 : if (!ldst_p
1478 9038543 : && first_stmt_code == BIT_FIELD_REF
1479 11863963 : && (TREE_OPERAND (gimple_assign_rhs1 (first_stmt_info->stmt), 0)
1480 29881 : != TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0)))
1481 : {
1482 19187 : if (dump_enabled_p ())
1483 116 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1484 : "Build SLP failed: different BIT_FIELD_REF "
1485 : "arguments in %G", stmt);
1486 : /* Mismatch. */
1487 19187 : continue;
1488 : }
1489 :
1490 11814895 : if (call_stmt
1491 88949 : && first_stmt_code != CFN_MASK_LOAD
1492 11903442 : && first_stmt_code != CFN_MASK_STORE)
1493 : {
1494 88236 : if (!is_a <gcall *> (stmts[0]->stmt)
1495 88236 : || !compatible_calls_p (as_a <gcall *> (stmts[0]->stmt),
1496 : call_stmt, true))
1497 : {
1498 69187 : if (dump_enabled_p ())
1499 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1500 : "Build SLP failed: different calls in %G",
1501 : stmt);
1502 : /* Mismatch. */
1503 69187 : continue;
1504 : }
1505 : }
1506 :
1507 11788318 : if (phi_p
1508 11745708 : && (gimple_bb (first_stmt_info->stmt)
1509 213325 : != gimple_bb (stmt_info->stmt)))
1510 : {
1511 42610 : if (dump_enabled_p ())
1512 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1513 : "Build SLP failed: different BB for PHI %G",
1514 : stmt);
1515 : /* Mismatch. */
1516 42610 : continue;
1517 : }
1518 :
1519 11703098 : if (need_same_oprnds)
1520 : {
1521 64024 : tree other_op1 = gimple_arg (stmt, 1);
1522 64024 : if (!operand_equal_p (first_op1, other_op1, 0))
1523 : {
1524 11397 : if (dump_enabled_p ())
1525 195 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1526 : "Build SLP failed: different shift "
1527 : "arguments in %G", stmt);
1528 : /* Mismatch. */
1529 11397 : continue;
1530 : }
1531 : }
1532 :
1533 11692542 : if (first_lhs
1534 11691701 : && lhs
1535 11691701 : && !types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (first_lhs)))
1536 : {
1537 841 : if (dump_enabled_p ())
1538 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1539 : "Build SLP failed: different vector type "
1540 : "in %G", stmt);
1541 : /* Mismatch. */
1542 841 : continue;
1543 : }
1544 :
1545 : /* We need to ensure all stmts are in the same BB when one stmt could
1546 : trap. */
1547 11690860 : if (trapping_stmt || gimple_could_trap_p (stmt))
1548 : {
1549 1175223 : gcc_assert (!trapping_stmt || common_bb);
1550 1175223 : if (gimple_bb (stmt) != common_bb)
1551 : {
1552 6077 : if (dump_enabled_p ())
1553 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1554 : "Build SLP failed: not all stmts in same "
1555 : "BB but possibly trapping operation in %G",
1556 : trapping_stmt ? trapping_stmt : stmt);
1557 6077 : continue;
1558 : }
1559 : }
1560 : }
1561 :
1562 : /* Grouped store or load. */
1563 17902661 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1564 : {
1565 4114447 : gcc_assert (ldst_p);
1566 4114447 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)))
1567 : {
1568 : /* Store. */
1569 3031913 : gcc_assert (rhs_code == CFN_MASK_STORE
1570 : || REFERENCE_CLASS_P (lhs)
1571 : || DECL_P (lhs));
1572 : }
1573 : else
1574 : {
1575 : /* Load. */
1576 1082534 : first_load = DR_GROUP_FIRST_ELEMENT (stmt_info);
1577 1082534 : if (prev_first_load)
1578 : {
1579 : /* Check that there are no loads from different interleaving
1580 : chains in the same node. */
1581 508141 : if (prev_first_load != first_load)
1582 : {
1583 106457 : if (dump_enabled_p ())
1584 2296 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
1585 : vect_location,
1586 : "Build SLP failed: different "
1587 : "interleaving chains in one node %G",
1588 : stmt);
1589 : /* Mismatch. */
1590 106457 : continue;
1591 : }
1592 : }
1593 : else
1594 : prev_first_load = first_load;
1595 : }
1596 : }
1597 : /* Non-grouped store or load. */
1598 13788214 : else if (ldst_p)
1599 : {
1600 1133816 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))
1601 855868 : && rhs_code != CFN_GATHER_LOAD
1602 : && rhs_code != CFN_MASK_GATHER_LOAD
1603 : && rhs_code != CFN_MASK_LEN_GATHER_LOAD
1604 : && rhs_code != CFN_SCATTER_STORE
1605 : && rhs_code != CFN_MASK_SCATTER_STORE
1606 : && rhs_code != CFN_MASK_LEN_SCATTER_STORE
1607 1989684 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1608 : {
1609 836157 : gcc_assert (!is_a <bb_vec_info> (vinfo));
1610 : /* For loop vectorization we can handle splats the same we
1611 : handle single element interleaving. Likewise we can handle
1612 : a collection of invariant refs. */
1613 1040356 : if (stmt_info != first_stmt_info
1614 836419 : && !(integer_zerop (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)))
1615 262 : && integer_zerop (DR_STEP (STMT_VINFO_DATA_REF
1616 : (first_stmt_info)))))
1617 : {
1618 : /* Not grouped load. */
1619 204199 : if (dump_enabled_p ())
1620 179 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1621 : "Build SLP failed: not grouped load %G",
1622 : stmt);
1623 204199 : continue;
1624 : }
1625 : }
1626 : }
1627 : /* Not memory operation. */
1628 : else
1629 : {
1630 12654398 : if (rhs_code == COND_EXPR)
1631 : {
1632 71641 : tree cond_expr = gimple_assign_rhs1 (stmt);
1633 71641 : enum tree_code cond_code = TREE_CODE (cond_expr);
1634 71641 : enum tree_code swap_code = ERROR_MARK;
1635 71641 : enum tree_code invert_code = ERROR_MARK;
1636 :
1637 71641 : if (i == (unsigned)first_match)
1638 59406 : first_cond_code = TREE_CODE (cond_expr);
1639 12235 : else if (TREE_CODE_CLASS (cond_code) == tcc_comparison)
1640 : {
1641 0 : bool honor_nans = HONOR_NANS (TREE_OPERAND (cond_expr, 0));
1642 0 : swap_code = swap_tree_comparison (cond_code);
1643 0 : invert_code = invert_tree_comparison (cond_code, honor_nans);
1644 : }
1645 :
1646 71641 : if (first_cond_code == cond_code)
1647 : ;
1648 : /* Isomorphic can be achieved by swapping. */
1649 0 : else if (first_cond_code == swap_code)
1650 0 : swap[i] = 1;
1651 : /* Isomorphic can be achieved by inverting. */
1652 0 : else if (first_cond_code == invert_code)
1653 0 : swap[i] = 2;
1654 : else
1655 : {
1656 0 : if (dump_enabled_p ())
1657 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1658 : "Build SLP failed: different"
1659 : " operation %G", stmt);
1660 : /* Mismatch. */
1661 0 : continue;
1662 : }
1663 : }
1664 :
1665 12654398 : if (i != (unsigned) first_match
1666 8894841 : && first_stmt_code != rhs_code
1667 72717 : && first_stmt_code.is_tree_code ()
1668 72715 : && rhs_code.is_tree_code ()
1669 72715 : && TREE_CODE_CLASS ((tree_code)first_stmt_code) == tcc_comparison
1670 12695721 : && (swap_tree_comparison ((tree_code)first_stmt_code)
1671 41323 : == (tree_code)rhs_code))
1672 41323 : swap[i] = 1;
1673 :
1674 12654398 : if (i != (unsigned) first_match
1675 8894841 : && first_reduc_idx != STMT_VINFO_REDUC_IDX (stmt_info)
1676 1786 : && first_reduc_idx != -1
1677 1786 : && STMT_VINFO_REDUC_IDX (stmt_info) != -1
1678 1786 : && rhs_code.is_tree_code ()
1679 1778 : && commutative_tree_code (tree_code (rhs_code))
1680 12656174 : && first_reduc_idx == 1 - STMT_VINFO_REDUC_IDX (stmt_info))
1681 1776 : swap[i] = 1;
1682 : }
1683 :
1684 : /* We need to ensure all stmts are in the same BB when one stmt could
1685 : trap. Not matching stmts are not relevant, so exclude those. */
1686 17592005 : if (!trapping_stmt && gimple_could_trap_p (stmt))
1687 : trapping_stmt = stmt;
1688 17592005 : if (common_bb != gimple_bb (stmt))
1689 : {
1690 62701 : common_bb = NULL;
1691 62701 : gcc_assert (!trapping_stmt);
1692 : }
1693 :
1694 17592005 : matches[i] = first_match;
1695 : }
1696 :
1697 : /* Record if we allowed two distinct operations for the SLP node. */
1698 6319259 : if (((first_stmt_code == PLUS_EXPR
1699 5798146 : || first_stmt_code == MINUS_EXPR)
1700 631591 : && (alt_stmt_code == PLUS_EXPR
1701 624293 : || alt_stmt_code == MINUS_EXPR))
1702 12621475 : || ((first_stmt_code == CFN_FMA
1703 6302167 : || first_stmt_code == CFN_FMS)
1704 49 : && (alt_stmt_code == CFN_FMA
1705 49 : || alt_stmt_code == CFN_FMS)))
1706 17045 : *two_operators = true;
1707 :
1708 6319259 : if (first_match == -1)
1709 : {
1710 : /* If there was no useful stmt, fail. Can only happen during
1711 : the first sweep. */
1712 101381 : gcc_assert (*start_i == 0);
1713 101381 : *start_i = stmts.length () - 1;
1714 101381 : return false;
1715 : }
1716 :
1717 6217878 : *start_i = first_match;
1718 6217878 : return true;
1719 : }
1720 :
1721 : /* Verify if the scalar stmts STMTS are isomorphic, require data
1722 : permutation or are of unsupported types of operation. Return
1723 : true if they are, otherwise return false and indicate in *MATCHES
1724 : the groups of isomorphic stmts. See vect_build_slp_tree_3 for
1725 : details. *TWO_OPERATORS is for the first isomorphic group,
1726 : knowledge whether following isomorphic groups have one or two operators
1727 : is not retained. */
1728 :
1729 : static bool
1730 6191176 : vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap,
1731 : vec<stmt_vec_info> stmts, match_elt_t *matches,
1732 : bool *two_operators, tree *node_vectype)
1733 : {
1734 6191176 : stmt_vec_info first_stmt_info = stmts[0];
1735 6191176 : unsigned int group_size = stmts.length ();
1736 6191176 : tree vectype;
1737 6191176 : if (!vect_get_vector_types_for_stmt (vinfo, first_stmt_info, &vectype,
1738 : group_size))
1739 : {
1740 : /* Fatal mismatch. */
1741 254985 : matches[0] = -1;
1742 254985 : return false;
1743 : }
1744 5936191 : if (is_a <bb_vec_info> (vinfo)
1745 5936191 : && known_le (TYPE_VECTOR_SUBPARTS (vectype), 1U))
1746 : {
1747 374827 : if (dump_enabled_p ())
1748 308 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1749 : "Build SLP failed: not using single lane "
1750 : "vector type %T\n", vectype);
1751 374827 : matches[0] = -1;
1752 374827 : return false;
1753 : }
1754 : /* Check nunits required but continue analysis, producing matches[]
1755 : as if nunits was not an issue. This allows splitting of groups
1756 : to happen. */
1757 5561364 : unsigned HOST_WIDE_INT const_nunits = 0;
1758 5561364 : if (vectype
1759 5561344 : && is_a <bb_vec_info> (vinfo)
1760 11122708 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)))
1761 : {
1762 211055 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
1763 211055 : || const_nunits > group_size)
1764 : {
1765 0 : if (dump_enabled_p ())
1766 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1767 : "Build SLP failed: unrolling required "
1768 : "in basic block SLP\n");
1769 : /* Fatal mismatch. */
1770 0 : matches[0] = -1;
1771 0 : return false;
1772 : }
1773 : }
1774 :
1775 5561364 : gcc_assert (vectype || !gimple_get_lhs (first_stmt_info->stmt));
1776 5561364 : *node_vectype = vectype;
1777 :
1778 5561364 : unsigned start_i = 0;
1779 5561364 : if (!vect_build_slp_tree_3 (vinfo, stmts, matches, swap, two_operators,
1780 : vectype, &start_i))
1781 : {
1782 : /* Fatal mismatch. */
1783 101652 : matches[0] = -1;
1784 101652 : return false;
1785 : }
1786 5459712 : gcc_assert (matches[start_i] == (int)start_i);
1787 : /* Discover further isomorphic groups. */
1788 17744535 : for (start_i = start_i + 1; start_i < group_size; ++start_i)
1789 12284823 : if (matches[start_i] == -1)
1790 : {
1791 758166 : unsigned prev_start_i = start_i;
1792 758166 : bool tem_two_operators;
1793 758166 : bool res = vect_build_slp_tree_3 (vinfo, stmts, matches, swap,
1794 : &tem_two_operators,
1795 : vectype, &start_i);
1796 758166 : gcc_assert (res && matches[start_i] == (int)prev_start_i);
1797 : }
1798 :
1799 : bool res = true;
1800 23280635 : for (unsigned i = 0; i < group_size; ++i)
1801 17820923 : if (matches[i] != 0)
1802 1898971 : res = false;
1803 : if (!res)
1804 : return false;
1805 :
1806 : return true;
1807 : }
1808 :
1809 : /* Traits for the hash_set to record failed SLP builds for a stmt set.
1810 : Note we never remove apart from at destruction time so we do not
1811 : need a special value for deleted that differs from empty. */
1812 : struct bst_traits
1813 : {
1814 : typedef vec <stmt_vec_info> value_type;
1815 : typedef vec <stmt_vec_info> compare_type;
1816 : static inline hashval_t hash (value_type);
1817 : static inline bool equal (value_type existing, value_type candidate);
1818 512172108 : static inline bool is_empty (value_type x) { return !x.exists (); }
1819 114750460 : static inline bool is_deleted (value_type x) { return !x.exists (); }
1820 : static const bool empty_zero_p = true;
1821 0 : static inline void mark_empty (value_type &x) { x.release (); }
1822 : static inline void mark_deleted (value_type &x) { x.release (); }
1823 9857772 : static inline void remove (value_type &x) { x.release (); }
1824 : };
1825 : inline hashval_t
1826 99870420 : bst_traits::hash (value_type x)
1827 : {
1828 99870420 : inchash::hash h;
1829 440896781 : for (unsigned i = 0; i < x.length (); ++i)
1830 341026361 : h.add_int (x[i] ? gimple_uid (x[i]->stmt) : -1);
1831 99870420 : return h.end ();
1832 : }
1833 : inline bool
1834 87458235 : bst_traits::equal (value_type existing, value_type candidate)
1835 : {
1836 262374705 : if (existing.length () != candidate.length ())
1837 : return false;
1838 87900187 : for (unsigned i = 0; i < existing.length (); ++i)
1839 83307672 : if (existing[i] != candidate[i])
1840 : return false;
1841 : return true;
1842 : }
1843 :
1844 : typedef hash_map <vec <stmt_vec_info>, slp_tree,
1845 : simple_hashmap_traits <bst_traits, slp_tree> >
1846 : scalar_stmts_to_slp_tree_map_t;
1847 :
1848 : /* Release BST_MAP. */
1849 :
1850 : static void
1851 1924607 : release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
1852 : {
1853 : /* The map keeps a reference on SLP nodes built, release that. */
1854 11782379 : for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin ();
1855 11782379 : it != bst_map->end (); ++it)
1856 9857772 : if ((*it).second)
1857 9857772 : vect_free_slp_tree ((*it).second);
1858 3849214 : delete bst_map;
1859 1924607 : }
1860 :
1861 : /* ??? This was std::pair<std::pair<tree_code, vect_def_type>, tree>
1862 : but then vec::insert does memmove and that's not compatible with
1863 : std::pair. */
1864 : struct chain_op_t
1865 : {
1866 4389257 : chain_op_t (tree_code code_, vect_def_type dt_, tree op_)
1867 4389257 : : code (code_), dt (dt_), op (op_) {}
1868 : tree_code code;
1869 : vect_def_type dt;
1870 : tree op;
1871 : };
1872 :
1873 : /* Comparator for sorting associatable chains. */
1874 :
1875 : static int
1876 12876843 : dt_sort_cmp (const void *op1_, const void *op2_, void *)
1877 : {
1878 12876843 : auto *op1 = (const chain_op_t *) op1_;
1879 12876843 : auto *op2 = (const chain_op_t *) op2_;
1880 12876843 : if (op1->dt != op2->dt)
1881 2028559 : return (int)op1->dt - (int)op2->dt;
1882 10848284 : return (int)op1->code - (int)op2->code;
1883 : }
1884 :
1885 : /* Linearize the associatable expression chain at START with the
1886 : associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
1887 : filling CHAIN with the result and using WORKLIST as intermediate storage.
1888 : CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
1889 : or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation
1890 : stmts, starting with START. When ALLOW_ALT_CODE is false, do not
1891 : follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf). */
1892 :
1893 : static void
1894 1890517 : vect_slp_linearize_chain (vec_info *vinfo,
1895 : vec<std::pair<tree_code, gimple *> > &worklist,
1896 : vec<chain_op_t> &chain,
1897 : enum tree_code code, gimple *start,
1898 : gimple *&code_stmt, gimple *&alt_code_stmt,
1899 : vec<gimple *> *chain_stmts,
1900 : bool allow_alt_code = true)
1901 : {
1902 : /* For each lane linearize the addition/subtraction (or other
1903 : uniform associatable operation) expression tree. */
1904 1890517 : worklist.safe_push (std::make_pair (code, start));
1905 4389257 : while (!worklist.is_empty ())
1906 : {
1907 2498740 : auto entry = worklist.pop ();
1908 2498740 : gassign *stmt = as_a <gassign *> (entry.second);
1909 2498740 : enum tree_code in_code = entry.first;
1910 4997480 : enum tree_code this_code = gimple_assign_rhs_code (stmt);
1911 : /* Pick some stmts suitable for SLP_TREE_REPRESENTATIVE. */
1912 2498740 : if (!code_stmt
1913 2498740 : && gimple_assign_rhs_code (stmt) == code)
1914 1576853 : code_stmt = stmt;
1915 921887 : else if (!alt_code_stmt
1916 921887 : && gimple_assign_rhs_code (stmt) == MINUS_EXPR)
1917 370243 : alt_code_stmt = stmt;
1918 2498740 : if (chain_stmts)
1919 2417541 : chain_stmts->safe_push (stmt);
1920 7496220 : for (unsigned opnum = 1; opnum <= 2; ++opnum)
1921 : {
1922 4997480 : tree op = gimple_op (stmt, opnum);
1923 4997480 : vect_def_type dt;
1924 4997480 : stmt_vec_info def_stmt_info;
1925 4997480 : bool res = vect_is_simple_use (op, vinfo, &dt, &def_stmt_info);
1926 4997480 : gcc_assert (res);
1927 4997480 : if (dt == vect_internal_def
1928 4997480 : && is_pattern_stmt_p (def_stmt_info))
1929 9250 : op = gimple_get_lhs (def_stmt_info->stmt);
1930 4997480 : gimple *use_stmt;
1931 4997480 : use_operand_p use_p;
1932 4997480 : if (dt == vect_internal_def
1933 4545464 : && single_imm_use (op, &use_p, &use_stmt)
1934 2903967 : && is_gimple_assign (def_stmt_info->stmt)
1935 7706486 : && (gimple_assign_rhs_code (def_stmt_info->stmt) == code
1936 2101100 : || (allow_alt_code
1937 57504 : && code == PLUS_EXPR
1938 36441 : && (gimple_assign_rhs_code (def_stmt_info->stmt)
1939 : == MINUS_EXPR))))
1940 : {
1941 608223 : tree_code op_def_code = this_code;
1942 608223 : if (op_def_code == MINUS_EXPR && opnum == 1)
1943 54816 : op_def_code = PLUS_EXPR;
1944 608223 : if (in_code == MINUS_EXPR)
1945 222 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1946 608223 : worklist.safe_push (std::make_pair (op_def_code,
1947 608223 : def_stmt_info->stmt));
1948 : }
1949 : else
1950 : {
1951 4389257 : tree_code op_def_code = this_code;
1952 4389257 : if (op_def_code == MINUS_EXPR && opnum == 1)
1953 315546 : op_def_code = PLUS_EXPR;
1954 4389257 : if (in_code == MINUS_EXPR)
1955 4230 : op_def_code = op_def_code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR;
1956 4389257 : chain.safe_push (chain_op_t (op_def_code, dt, op));
1957 : }
1958 : }
1959 : }
1960 1890517 : }
1961 :
1962 : /* Distance from the node currently being discovered to the closest upthread
1963 : commutative operation whose operand-zero discovery may still be fixed by
1964 : retrying with swapped operands, or -1U if there is none. */
1965 :
1966 : static unsigned least_upthread_swappable_op_distance = -1U;
1967 :
1968 : static slp_tree
1969 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
1970 : vec<stmt_vec_info> stmts,
1971 : match_elt_t *matches, unsigned *limit,
1972 : unsigned *tree_size,
1973 : scalar_stmts_to_slp_tree_map_t *bst_map);
1974 :
1975 : static slp_tree
1976 6701522 : vect_build_slp_tree (vec_info *vinfo,
1977 : vec<stmt_vec_info> stmts,
1978 : match_elt_t *matches, unsigned *limit, unsigned *tree_size,
1979 : scalar_stmts_to_slp_tree_map_t *bst_map)
1980 : {
1981 6701522 : unsigned int group_size = stmts.length ();
1982 6701522 : if (slp_tree *leader = bst_map->get (stmts))
1983 : {
1984 504497 : if (dump_enabled_p ())
1985 18347 : dump_printf_loc (MSG_NOTE, vect_location, "re-using %sSLP tree %p\n",
1986 18347 : !(*leader)->failed ? "" : "failed ",
1987 : (void *) *leader);
1988 504497 : if (!(*leader)->failed)
1989 : {
1990 455835 : SLP_TREE_REF_COUNT (*leader)++;
1991 455835 : stmts.release ();
1992 455835 : return *leader;
1993 : }
1994 48662 : memcpy (matches, (*leader)->failed, sizeof (match_elt_t) * group_size);
1995 48662 : return NULL;
1996 : }
1997 :
1998 : /* Single-lane SLP doesn't have the chance of run-away, do not account
1999 : it to the limit. */
2000 6197025 : if (stmts.length () > 1)
2001 : {
2002 3469363 : if (*limit == 0)
2003 : {
2004 1355 : if (dump_enabled_p ())
2005 15 : dump_printf_loc (MSG_NOTE, vect_location,
2006 : "SLP discovery limit exceeded\n");
2007 1355 : memset (matches, -1, sizeof (match_elt_t) * group_size);
2008 1355 : return NULL;
2009 : }
2010 3468008 : --*limit;
2011 : }
2012 :
2013 : /* Seed the bst_map with a stub node to be filled by vect_build_slp_tree_2
2014 : so we can pick up backedge destinations during discovery. */
2015 6195670 : slp_tree res = new _slp_tree;
2016 6195670 : SLP_TREE_DEF_TYPE (res) = vect_internal_def;
2017 6195670 : SLP_TREE_SCALAR_STMTS (res) = stmts;
2018 6195670 : bst_map->put (stmts.copy (), res);
2019 :
2020 6195670 : if (dump_enabled_p ())
2021 157017 : dump_printf_loc (MSG_NOTE, vect_location,
2022 : "starting SLP discovery for node %p\n", (void *) res);
2023 :
2024 6195670 : slp_tree res_ = vect_build_slp_tree_2 (vinfo, res, stmts,
2025 : matches, limit, tree_size, bst_map);
2026 6195670 : if (!res_)
2027 : {
2028 2199736 : if (dump_enabled_p ())
2029 8217 : dump_printf_loc (MSG_NOTE, vect_location,
2030 : "SLP discovery for node %p failed\n", (void *) res);
2031 : /* Mark the node invalid so we can detect those when still in use
2032 : as backedge destinations. */
2033 2199736 : SLP_TREE_SCALAR_STMTS (res) = vNULL;
2034 2199736 : SLP_TREE_DEF_TYPE (res) = vect_uninitialized_def;
2035 2199736 : res->failed = XNEWVEC (match_elt_t, group_size);
2036 2199736 : if (flag_checking)
2037 : {
2038 : unsigned i;
2039 3871379 : for (i = 0; i < group_size; ++i)
2040 3871379 : if (matches[i] != 0)
2041 : break;
2042 2199736 : gcc_assert (i < group_size);
2043 : }
2044 2199736 : memcpy (res->failed, matches, sizeof (match_elt_t) * group_size);
2045 : }
2046 : else
2047 : {
2048 3995934 : if (dump_enabled_p ())
2049 148800 : dump_printf_loc (MSG_NOTE, vect_location,
2050 : "SLP discovery for node %p succeeded\n",
2051 : (void *) res);
2052 3995934 : gcc_assert (res_ == res);
2053 : /* Keep a reference for the bst_map use. */
2054 3995934 : SLP_TREE_REF_COUNT (res)++;
2055 : /* For BB vectorization nodes that need splitting should be failed. */
2056 3995934 : gcc_assert (!is_a <bb_vec_info> (vinfo)
2057 : || multiple_p (group_size,
2058 : TYPE_VECTOR_SUBPARTS (res->vectype)));
2059 : }
2060 : return res_;
2061 : }
2062 :
2063 : /* Helper for building an associated SLP node chain. */
2064 :
2065 : static void
2066 162 : vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype,
2067 : slp_tree op0, slp_tree op1,
2068 : stmt_vec_info oper1, stmt_vec_info oper2,
2069 : vec<std::pair<unsigned, unsigned> > lperm)
2070 : {
2071 162 : unsigned group_size = SLP_TREE_LANES (op1);
2072 :
2073 162 : slp_tree child1 = new _slp_tree;
2074 162 : SLP_TREE_DEF_TYPE (child1) = vect_internal_def;
2075 162 : SLP_TREE_VECTYPE (child1) = vectype;
2076 162 : SLP_TREE_LANES (child1) = group_size;
2077 162 : SLP_TREE_CHILDREN (child1).create (2);
2078 162 : SLP_TREE_CHILDREN (child1).quick_push (op0);
2079 162 : SLP_TREE_CHILDREN (child1).quick_push (op1);
2080 162 : SLP_TREE_REPRESENTATIVE (child1) = oper1;
2081 :
2082 162 : slp_tree child2 = new _slp_tree;
2083 162 : SLP_TREE_DEF_TYPE (child2) = vect_internal_def;
2084 162 : SLP_TREE_VECTYPE (child2) = vectype;
2085 162 : SLP_TREE_LANES (child2) = group_size;
2086 162 : SLP_TREE_CHILDREN (child2).create (2);
2087 162 : SLP_TREE_CHILDREN (child2).quick_push (op0);
2088 162 : SLP_TREE_REF_COUNT (op0)++;
2089 162 : SLP_TREE_CHILDREN (child2).quick_push (op1);
2090 162 : SLP_TREE_REF_COUNT (op1)++;
2091 162 : SLP_TREE_REPRESENTATIVE (child2) = oper2;
2092 :
2093 162 : SLP_TREE_DEF_TYPE (perm) = vect_internal_def;
2094 162 : SLP_TREE_CODE (perm) = VEC_PERM_EXPR;
2095 162 : SLP_TREE_VECTYPE (perm) = vectype;
2096 162 : SLP_TREE_LANES (perm) = group_size;
2097 162 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
2098 162 : SLP_TREE_LANE_PERMUTATION (perm) = lperm;
2099 162 : SLP_TREE_CHILDREN (perm).quick_push (child1);
2100 162 : SLP_TREE_CHILDREN (perm).quick_push (child2);
2101 162 : }
2102 :
2103 : /* For isomorphic matches[], indicate a splitting point according to
2104 : CONST_NUNITS. */
2105 :
2106 : static void
2107 124417 : force_split_matches (match_elt_t *matches, unsigned group_size,
2108 : unsigned const_nunits)
2109 : {
2110 124417 : if (dump_enabled_p ())
2111 54 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2112 : "Build SLP failed: unrolling required "
2113 : "in basic block SLP\n");
2114 :
2115 : /* With constant vector elements simulate a mismatch at the
2116 : point we need to split. But indicate the tail is isomorphic. */
2117 124417 : unsigned tail = group_size & (const_nunits - 1);
2118 273790 : for (unsigned i = group_size - tail; i < group_size; ++i)
2119 149373 : matches[i] = (int)(group_size - tail);
2120 124417 : }
2121 :
2122 : /* Recursively build an SLP tree starting from NODE.
2123 : Fail (and return a value not equal to zero) if def-stmts are not
2124 : isomorphic, require data permutation or are of unsupported types of
2125 : operation. Otherwise, return 0.
2126 : The value returned is the depth in the SLP tree where a mismatch
2127 : was found. */
2128 :
2129 : static slp_tree
2130 6195670 : vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node,
2131 : vec<stmt_vec_info> stmts,
2132 : match_elt_t *matches, unsigned *limit,
2133 : unsigned *tree_size,
2134 : scalar_stmts_to_slp_tree_map_t *bst_map)
2135 : {
2136 6195670 : unsigned int group_size = stmts.length ();
2137 6195670 : unsigned nops, i, this_tree_size = 0;
2138 :
2139 6195670 : matches[0] = -1;
2140 :
2141 6195670 : stmt_vec_info stmt_info = stmts[0];
2142 6195670 : if (!is_a<gcall *> (stmt_info->stmt)
2143 : && !is_a<gassign *> (stmt_info->stmt)
2144 : && !is_a<gphi *> (stmt_info->stmt))
2145 : return NULL;
2146 :
2147 6195500 : nops = gimple_num_args (stmt_info->stmt);
2148 6195500 : if (const int *map = vect_get_operand_map (stmt_info))
2149 36818 : nops = map[0];
2150 :
2151 : /* If the SLP node is a PHI (induction or reduction), terminate
2152 : the recursion. */
2153 6195500 : bool *skip_args = XALLOCAVEC (bool, nops);
2154 6195500 : memset (skip_args, 0, sizeof (bool) * nops);
2155 6195500 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2156 2926874 : if (gphi *stmt = dyn_cast <gphi *> (stmt_info->stmt))
2157 : {
2158 320375 : tree scalar_type = TREE_TYPE (PHI_RESULT (stmt));
2159 320375 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
2160 : group_size);
2161 320375 : if (!vectype)
2162 : {
2163 4318 : if (dump_enabled_p ())
2164 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2165 : "Build SLP failed: unsupported data-type in %G\n",
2166 : stmt_info->stmt);
2167 : return NULL;
2168 : }
2169 :
2170 316057 : vect_def_type def_type = STMT_VINFO_DEF_TYPE (stmt_info);
2171 316057 : if (def_type == vect_induction_def)
2172 : {
2173 : /* Induction PHIs are not cycles but walk the initial
2174 : value. Only for inner loops through, for outer loops
2175 : we need to pick up the value from the actual PHIs
2176 : to more easily support peeling and epilogue vectorization. */
2177 207246 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2178 207246 : if (!nested_in_vect_loop_p (loop, stmt_info))
2179 206414 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2180 : else
2181 : loop = loop->inner;
2182 207246 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2183 : }
2184 108811 : else if (def_type == vect_reduction_def
2185 : || def_type == vect_double_reduction_def
2186 : || def_type == vect_nested_cycle
2187 108811 : || def_type == vect_first_order_recurrence)
2188 : {
2189 : /* Else def types have to match. */
2190 : stmt_vec_info other_info;
2191 : bool all_same = true;
2192 246175 : FOR_EACH_VEC_ELT (stmts, i, other_info)
2193 : {
2194 138652 : if (STMT_VINFO_DEF_TYPE (other_info) != def_type)
2195 6195670 : return NULL;
2196 138646 : if (other_info != stmt_info)
2197 26269 : all_same = false;
2198 : }
2199 107523 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2200 : /* Reduction initial values are not explicitly represented. */
2201 107523 : if (def_type != vect_first_order_recurrence
2202 107523 : && gimple_bb (stmt_info->stmt) == loop->header)
2203 104345 : skip_args[loop_preheader_edge (loop)->dest_idx] = true;
2204 : /* Reduction chain backedge defs are filled manually.
2205 : ??? Need a better way to identify a SLP reduction chain PHI.
2206 : Or a better overall way to SLP match those. */
2207 107523 : if (stmts.length () > 1
2208 107523 : && all_same && def_type == vect_reduction_def)
2209 2416 : skip_args[loop_latch_edge (loop)->dest_idx] = true;
2210 : }
2211 1282 : else if (def_type != vect_internal_def)
2212 : return NULL;
2213 : }
2214 :
2215 :
2216 6191176 : bool two_operators = false;
2217 6191176 : unsigned char *swap = XALLOCAVEC (unsigned char, group_size);
2218 6191176 : tree vectype = NULL_TREE;
2219 6191176 : if (!vect_build_slp_tree_1 (vinfo, swap, stmts, matches, &two_operators,
2220 : &vectype))
2221 : return NULL;
2222 :
2223 4623152 : bool soft_fail
2224 4623152 : = (is_a <bb_vec_info> (vinfo)
2225 4623152 : && !multiple_p (group_size, TYPE_VECTOR_SUBPARTS (vectype)));
2226 :
2227 : /* If the SLP node is a load, terminate the recursion unless masked. */
2228 4623152 : if (STMT_VINFO_DATA_REF (stmt_info)
2229 2240165 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
2230 : {
2231 958150 : if (soft_fail)
2232 : {
2233 962 : force_split_matches (matches, group_size,
2234 962 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2235 962 : return NULL;
2236 : }
2237 957188 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
2238 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)));
2239 : else
2240 : {
2241 937770 : (*tree_size)++;
2242 937770 : node = vect_create_new_slp_node (node, stmts, 0);
2243 937770 : SLP_TREE_VECTYPE (node) = vectype;
2244 : /* And compute the load permutation. Whether it is actually
2245 : a permutation depends on the unrolling factor which is
2246 : decided later. */
2247 937770 : vec<unsigned> load_permutation;
2248 937770 : int j;
2249 937770 : stmt_vec_info load_info;
2250 937770 : load_permutation.create (group_size);
2251 937770 : stmt_vec_info first_stmt_info
2252 937770 : = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2253 937770 : ? DR_GROUP_FIRST_ELEMENT (stmt_info) : stmt_info;
2254 937770 : bool any_permute = false;
2255 2269075 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
2256 : {
2257 1331305 : int load_place;
2258 1331305 : if (! load_info)
2259 : {
2260 40747 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2261 : load_place = j;
2262 : else
2263 : load_place = 0;
2264 : }
2265 1290558 : else if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2266 754663 : load_place = vect_get_place_in_interleaving_chain
2267 754663 : (load_info, first_stmt_info);
2268 : else
2269 : /* Recognize the splat case as { 0, 0, ... } but make
2270 : sure to use the appropriate refs for collections
2271 : of invariant refs. */
2272 535895 : load_place = (load_info == stmt_info) ? 0 : j;
2273 795663 : gcc_assert (load_place != -1);
2274 1331305 : any_permute |= load_place != j;
2275 1331305 : load_permutation.quick_push (load_place);
2276 : }
2277 :
2278 937770 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
2279 : {
2280 3353 : gcc_assert (gimple_call_internal_p (stmt, IFN_MASK_LOAD));
2281 3353 : bool has_gaps = false;
2282 3353 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2283 189 : for (stmt_vec_info si = DR_GROUP_NEXT_ELEMENT (first_stmt_info);
2284 846 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2285 657 : if (DR_GROUP_GAP (si) != 1)
2286 80 : has_gaps = true;
2287 : /* We cannot handle permuted masked loads directly, see
2288 : PR114375. We cannot handle strided masked loads or masked
2289 : loads with gaps unless the mask is uniform. */
2290 3353 : if ((STMT_VINFO_GROUPED_ACCESS (stmt_info)
2291 189 : && (DR_GROUP_GAP (first_stmt_info) != 0
2292 129 : || (has_gaps
2293 35 : && STMT_VINFO_SLP_VECT_ONLY (first_stmt_info))))
2294 6626 : || STMT_VINFO_STRIDED_P (stmt_info))
2295 : {
2296 108 : load_permutation.release ();
2297 108 : matches[0] = -1;
2298 934564 : return NULL;
2299 : }
2300 :
2301 : /* For permuted masked loads do an unpermuted masked load of
2302 : the whole group followed by a SLP permute node. */
2303 3245 : if (any_permute
2304 3245 : || (STMT_VINFO_GROUPED_ACCESS (stmt_info)
2305 83 : && DR_GROUP_SIZE (first_stmt_info) != group_size))
2306 : {
2307 : /* Discover the whole unpermuted load. */
2308 39 : vec<stmt_vec_info> stmts2;
2309 39 : unsigned dr_group_size = STMT_VINFO_GROUPED_ACCESS (stmt_info)
2310 68 : ? DR_GROUP_SIZE (first_stmt_info) : 1;
2311 39 : stmts2.create (dr_group_size);
2312 39 : stmts2.quick_grow_cleared (dr_group_size);
2313 39 : unsigned i = 0;
2314 39 : for (stmt_vec_info si = first_stmt_info;
2315 464 : si; si = DR_GROUP_NEXT_ELEMENT (si))
2316 : {
2317 425 : if (si != first_stmt_info)
2318 1586 : for (unsigned k = 1; k < DR_GROUP_GAP (si); ++k)
2319 1200 : stmts2[i++] = NULL;
2320 425 : stmts2[i++] = si;
2321 : }
2322 39 : match_elt_t *matches2
2323 39 : = XALLOCAVEC (match_elt_t, dr_group_size);
2324 39 : slp_tree unperm_load
2325 39 : = vect_build_slp_tree (vinfo, stmts2, matches2, limit,
2326 39 : &this_tree_size, bst_map);
2327 : /* When we are able to do the full masked load emit that
2328 : followed by 'node' being the desired final permutation. */
2329 39 : if (unperm_load)
2330 : {
2331 16 : gcc_assert
2332 : (!SLP_TREE_LOAD_PERMUTATION (unperm_load).exists ());
2333 16 : lane_permutation_t lperm;
2334 16 : lperm.create (group_size);
2335 72 : for (unsigned j = 0; j < load_permutation.length (); ++j)
2336 40 : lperm.quick_push
2337 40 : (std::make_pair (0, load_permutation[j]));
2338 16 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2339 16 : SLP_TREE_CHILDREN (node).safe_push (unperm_load);
2340 16 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2341 16 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2342 16 : load_permutation.release ();
2343 16 : return node;
2344 : }
2345 23 : stmts2.release ();
2346 23 : load_permutation.release ();
2347 23 : matches[0] = -1;
2348 23 : return NULL;
2349 : }
2350 3206 : load_permutation.release ();
2351 : }
2352 : else
2353 : {
2354 934417 : if (!any_permute
2355 803803 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
2356 1228526 : && group_size == DR_GROUP_SIZE (first_stmt_info))
2357 129443 : load_permutation.release ();
2358 934417 : SLP_TREE_LOAD_PERMUTATION (node) = load_permutation;
2359 934417 : return node;
2360 : }
2361 : }
2362 : }
2363 3665002 : else if (gimple_assign_single_p (stmt_info->stmt)
2364 2570978 : && !gimple_vuse (stmt_info->stmt)
2365 3674004 : && gimple_assign_rhs_code (stmt_info->stmt) == BIT_FIELD_REF)
2366 : {
2367 3335 : if (soft_fail)
2368 : {
2369 16 : force_split_matches (matches, group_size,
2370 16 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2371 16 : return NULL;
2372 : }
2373 : /* vect_build_slp_tree_2 determined all BIT_FIELD_REFs reference
2374 : the same SSA name vector of a compatible type to vectype. */
2375 3319 : vec<std::pair<unsigned, unsigned> > lperm = vNULL;
2376 3319 : tree vec = TREE_OPERAND (gimple_assign_rhs1 (stmt_info->stmt), 0);
2377 3319 : stmt_vec_info estmt_info;
2378 11547 : FOR_EACH_VEC_ELT (stmts, i, estmt_info)
2379 : {
2380 8375 : gassign *estmt = as_a <gassign *> (estmt_info->stmt);
2381 8375 : tree bfref = gimple_assign_rhs1 (estmt);
2382 8375 : HOST_WIDE_INT lane;
2383 8375 : if (!known_eq (bit_field_size (bfref),
2384 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (vectype))))
2385 16603 : || !constant_multiple_p (bit_field_offset (bfref),
2386 8375 : bit_field_size (bfref), &lane))
2387 : {
2388 147 : lperm.release ();
2389 147 : matches[0] = -1;
2390 147 : return NULL;
2391 : }
2392 8228 : lperm.safe_push (std::make_pair (0, (unsigned)lane));
2393 : }
2394 3172 : slp_tree vnode = vect_create_new_slp_node (vNULL);
2395 3172 : if (operand_equal_p (TYPE_SIZE (vectype), TYPE_SIZE (TREE_TYPE (vec))))
2396 : /* ??? We record vectype here but we hide eventually necessary
2397 : punning and instead rely on code generation to materialize
2398 : VIEW_CONVERT_EXPRs as necessary. We instead should make
2399 : this explicit somehow. */
2400 1409 : SLP_TREE_VECTYPE (vnode) = vectype;
2401 : else
2402 : {
2403 : /* For different size but compatible elements we can still
2404 : use VEC_PERM_EXPR without punning. */
2405 1763 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (vec))
2406 : && types_compatible_p (TREE_TYPE (vectype),
2407 : TREE_TYPE (TREE_TYPE (vec))));
2408 1763 : SLP_TREE_VECTYPE (vnode) = TREE_TYPE (vec);
2409 : }
2410 3172 : auto nunits = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (vnode));
2411 3172 : unsigned HOST_WIDE_INT const_nunits;
2412 3172 : if (nunits.is_constant (&const_nunits))
2413 3172 : SLP_TREE_LANES (vnode) = const_nunits;
2414 3172 : SLP_TREE_VEC_DEFS (vnode).safe_push (vec);
2415 : /* We are always building a permutation node even if it is an identity
2416 : permute to shield the rest of the vectorizer from the odd node
2417 : representing an actual vector without any scalar ops.
2418 : ??? We could hide it completely with making the permute node
2419 : external? */
2420 3172 : node = vect_create_new_slp_node (node, stmts, 1);
2421 3172 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
2422 3172 : SLP_TREE_LANE_PERMUTATION (node) = lperm;
2423 3172 : SLP_TREE_VECTYPE (node) = vectype;
2424 3172 : SLP_TREE_CHILDREN (node).quick_push (vnode);
2425 3172 : SLP_TREE_REPRESENTATIVE (node) = NULL;
2426 3172 : return node;
2427 : }
2428 : /* When discovery reaches an associatable operation see whether we can
2429 : improve that to match up lanes in a way superior to the operand
2430 : swapping code which at most looks at two defs.
2431 : ??? For BB vectorization we cannot do the brute-force search
2432 : for matching as we can succeed by means of builds from scalars
2433 : and have no good way to "cost" one build against another. */
2434 3661667 : else if (is_a <loop_vec_info> (vinfo)
2435 : /* Do not bother for single-lane SLP. */
2436 2080229 : && group_size > 1
2437 : /* ??? We don't handle !vect_internal_def defs below. */
2438 114429 : && STMT_VINFO_DEF_TYPE (stmt_info) == vect_internal_def
2439 : /* ??? Do not associate a reduction, this will wreck REDUC_IDX
2440 : mapping as long as that exists on the stmt_info level. */
2441 88745 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1
2442 80118 : && is_gimple_assign (stmt_info->stmt)
2443 79793 : && (associative_tree_code (gimple_assign_rhs_code (stmt_info->stmt))
2444 52821 : || gimple_assign_rhs_code (stmt_info->stmt) == MINUS_EXPR)
2445 3690596 : && ((FLOAT_TYPE_P (vectype) && flag_associative_math)
2446 16662 : || (INTEGRAL_TYPE_P (TREE_TYPE (vectype))
2447 14123 : && TYPE_OVERFLOW_WRAPS (TREE_TYPE (vectype)))))
2448 : {
2449 : /* See if we have a chain of (mixed) adds or subtracts or other
2450 : associatable ops. */
2451 21756 : enum tree_code code = gimple_assign_rhs_code (stmt_info->stmt);
2452 21756 : if (code == MINUS_EXPR)
2453 842 : code = PLUS_EXPR;
2454 21756 : stmt_vec_info other_op_stmt_info = NULL;
2455 21756 : stmt_vec_info op_stmt_info = NULL;
2456 21756 : unsigned chain_len = 0;
2457 21756 : auto_vec<chain_op_t> chain;
2458 21756 : auto_vec<std::pair<tree_code, gimple *> > worklist;
2459 21756 : auto_vec<vec<chain_op_t> > chains (group_size);
2460 21756 : auto_vec<slp_tree, 4> children;
2461 21756 : bool hard_fail = true;
2462 22739 : for (unsigned lane = 0; lane < group_size; ++lane)
2463 : {
2464 22413 : if (!stmts[lane])
2465 : {
2466 : /* ??? Below we require lane zero is present. */
2467 0 : if (lane == 0)
2468 : {
2469 : hard_fail = false;
2470 21430 : break;
2471 : }
2472 0 : chains.quick_push (vNULL);
2473 0 : continue;
2474 : }
2475 : /* For each lane linearize the addition/subtraction (or other
2476 : uniform associatable operation) expression tree. */
2477 22413 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
2478 22413 : vect_slp_linearize_chain (vinfo, worklist, chain, code,
2479 22413 : stmts[lane]->stmt, op_stmt, other_op_stmt,
2480 : NULL);
2481 22413 : if (!op_stmt_info && op_stmt)
2482 21105 : op_stmt_info = vinfo->lookup_stmt (op_stmt);
2483 22413 : if (!other_op_stmt_info && other_op_stmt)
2484 878 : other_op_stmt_info = vinfo->lookup_stmt (other_op_stmt);
2485 22413 : if (chain.length () == 2)
2486 : {
2487 : /* In a chain of just two elements resort to the regular
2488 : operand swapping scheme. Likewise if we run into a
2489 : length mismatch process regularly as well as we did not
2490 : process the other lanes we cannot report a good hint what
2491 : lanes to try swapping in the parent. */
2492 : hard_fail = false;
2493 : break;
2494 : }
2495 986 : else if (chain_len == 0)
2496 366 : chain_len = chain.length ();
2497 1240 : else if (chain.length () != chain_len)
2498 : {
2499 : /* ??? Here we could slip in magic to compensate with
2500 : neutral operands. */
2501 3 : matches[lane] = -1;
2502 3 : if (lane != group_size - 1)
2503 3 : matches[0] = -1;
2504 : break;
2505 : }
2506 983 : chains.quick_push (chain.copy ());
2507 983 : chain.truncate (0);
2508 : }
2509 43512 : if (chains.length () == group_size)
2510 : {
2511 : /* We cannot yet use SLP_TREE_CODE to communicate the operation. */
2512 326 : if (!op_stmt_info)
2513 : {
2514 3 : hard_fail = false;
2515 3 : goto out;
2516 : }
2517 : /* Now we have a set of chains with the same length. */
2518 : /* 1. pre-sort according to def_type and operation. */
2519 1194 : for (unsigned lane = 0; lane < group_size; ++lane)
2520 1742 : chains[lane].stablesort (dt_sort_cmp, vinfo);
2521 323 : if (dump_enabled_p ())
2522 : {
2523 155 : dump_printf_loc (MSG_NOTE, vect_location,
2524 : "pre-sorted chains of %s\n",
2525 : get_tree_code_name (code));
2526 655 : for (unsigned lane = 0; lane < group_size; ++lane)
2527 : {
2528 500 : if (!stmts[lane])
2529 0 : dump_printf (MSG_NOTE, "--");
2530 : else
2531 2250 : for (unsigned opnum = 0; opnum < chain_len; ++opnum)
2532 3500 : dump_printf (MSG_NOTE, "%s %T ",
2533 1750 : get_tree_code_name (chains[lane][opnum].code),
2534 1750 : chains[lane][opnum].op);
2535 500 : dump_printf (MSG_NOTE, "\n");
2536 : }
2537 : }
2538 : /* 2. try to build children nodes, associating as necessary. */
2539 : /* 2a. prepare and perform early checks to avoid eating into
2540 : discovery limit unnecessarily. */
2541 323 : vect_def_type *dts = XALLOCAVEC (vect_def_type, chain_len);
2542 1361 : for (unsigned n = 0; n < chain_len; ++n)
2543 : {
2544 1038 : vect_def_type dt = chains[0][n].dt;
2545 1038 : unsigned lane;
2546 3949 : for (lane = 0; lane < group_size; ++lane)
2547 5822 : if (stmts[lane] && chains[lane][n].dt != dt)
2548 : {
2549 0 : if (dt == vect_constant_def
2550 0 : && chains[lane][n].dt == vect_external_def)
2551 : dt = vect_external_def;
2552 0 : else if (dt == vect_external_def
2553 0 : && chains[lane][n].dt == vect_constant_def)
2554 : ;
2555 : else
2556 : break;
2557 : }
2558 1038 : if (lane != group_size)
2559 : {
2560 0 : if (dump_enabled_p ())
2561 0 : dump_printf_loc (MSG_NOTE, vect_location,
2562 : "giving up on chain due to mismatched "
2563 : "def types\n");
2564 0 : matches[lane] = -1;
2565 0 : if (lane != group_size - 1)
2566 0 : matches[0] = -1;
2567 0 : goto out;
2568 : }
2569 1038 : dts[n] = dt;
2570 1038 : if (dt == vect_constant_def
2571 1038 : || dt == vect_external_def)
2572 : {
2573 : /* Check whether we can build the invariant. If we can't
2574 : we never will be able to. */
2575 77 : tree type = TREE_TYPE (chains[0][n].op);
2576 1038 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ()
2577 : && (TREE_CODE (type) == BOOLEAN_TYPE
2578 : || !can_duplicate_and_interleave_p (vinfo, group_size,
2579 : type)))
2580 : {
2581 : matches[0] = -1;
2582 : goto out;
2583 : }
2584 : }
2585 961 : else if (dt != vect_internal_def)
2586 : {
2587 : /* Not sure, we might need sth special.
2588 : gcc.dg/vect/pr96854.c,
2589 : gfortran.dg/vect/fast-math-pr37021.f90
2590 : and gfortran.dg/vect/pr61171.f trigger. */
2591 : /* Soft-fail for now. */
2592 0 : hard_fail = false;
2593 0 : goto out;
2594 : }
2595 : }
2596 : /* 2b. do the actual build. */
2597 1307 : for (unsigned n = 0; n < chain_len; ++n)
2598 : {
2599 1003 : vect_def_type dt = dts[n];
2600 1003 : unsigned lane;
2601 1003 : if (dt == vect_constant_def
2602 1003 : || dt == vect_external_def)
2603 : {
2604 77 : vec<tree> ops;
2605 77 : ops.create (group_size);
2606 397 : for (lane = 0; lane < group_size; ++lane)
2607 243 : if (stmts[lane])
2608 243 : ops.quick_push (chains[lane][n].op);
2609 : else
2610 0 : ops.quick_push (NULL_TREE);
2611 77 : slp_tree child = vect_create_new_slp_node (ops);
2612 77 : SLP_TREE_DEF_TYPE (child) = dt;
2613 77 : children.safe_push (child);
2614 : }
2615 : else
2616 : {
2617 926 : vec<stmt_vec_info> op_stmts;
2618 926 : op_stmts.create (group_size);
2619 926 : slp_tree child = NULL;
2620 : /* Brute-force our way. We have to consider a lane
2621 : failing after fixing an earlier fail up in the
2622 : SLP discovery recursion. So track the current
2623 : permute per lane. */
2624 926 : unsigned *perms = XALLOCAVEC (unsigned, group_size);
2625 926 : memset (perms, 0, sizeof (unsigned) * group_size);
2626 1022 : do
2627 : {
2628 1022 : op_stmts.truncate (0);
2629 4900 : for (lane = 0; lane < group_size; ++lane)
2630 2856 : if (stmts[lane])
2631 2856 : op_stmts.quick_push
2632 2856 : (vinfo->lookup_def (chains[lane][n].op));
2633 : else
2634 0 : op_stmts.quick_push (NULL);
2635 1022 : child = vect_build_slp_tree (vinfo, op_stmts,
2636 : matches, limit,
2637 : &this_tree_size, bst_map);
2638 : /* ??? We're likely getting too many fatal mismatches
2639 : here so maybe we want to ignore them (but then we
2640 : have no idea which lanes fatally mismatched). */
2641 : /* ??? Revisit this with matches[] improvements. */
2642 1022 : if (child || matches[0] != 0)
2643 : break;
2644 : /* Swap another lane we have not yet matched up into
2645 : lanes that did not match. If we run out of
2646 : permute possibilities for a lane terminate the
2647 : search. */
2648 291 : bool term = false;
2649 291 : for (lane = 1; lane < group_size; ++lane)
2650 195 : if (matches[lane] != 0)
2651 : {
2652 167 : if (n + perms[lane] + 1 == chain_len)
2653 : {
2654 : term = true;
2655 : break;
2656 : }
2657 148 : if (dump_enabled_p ())
2658 115 : dump_printf_loc (MSG_NOTE, vect_location,
2659 : "swapping operand %d and %d "
2660 : "of lane %d\n",
2661 : n, n + perms[lane] + 1, lane);
2662 296 : std::swap (chains[lane][n],
2663 148 : chains[lane][n + perms[lane] + 1]);
2664 148 : perms[lane]++;
2665 : }
2666 115 : if (term)
2667 : break;
2668 : }
2669 : while (1);
2670 926 : if (!child)
2671 : {
2672 19 : if (dump_enabled_p ())
2673 18 : dump_printf_loc (MSG_NOTE, vect_location,
2674 : "failed to match up op %d\n", n);
2675 19 : op_stmts.release ();
2676 19 : if (lane != group_size - 1)
2677 9 : matches[0] = -1;
2678 : else
2679 10 : matches[lane] = -1;
2680 19 : goto out;
2681 : }
2682 907 : if (dump_enabled_p ())
2683 : {
2684 409 : dump_printf_loc (MSG_NOTE, vect_location,
2685 : "matched up op %d to\n", n);
2686 409 : vect_print_slp_tree (MSG_NOTE, vect_location, child);
2687 : }
2688 907 : children.safe_push (child);
2689 : }
2690 : }
2691 : /* 3. build SLP nodes to combine the chain. */
2692 1102 : for (unsigned lane = 0; lane < group_size; ++lane)
2693 1608 : if (stmts[lane] && chains[lane][0].code != code)
2694 : {
2695 : /* See if there's any alternate all-PLUS entry. */
2696 : unsigned n;
2697 6 : for (n = 1; n < chain_len; ++n)
2698 : {
2699 30 : for (lane = 0; lane < group_size; ++lane)
2700 48 : if (stmts[lane] && chains[lane][n].code != code)
2701 : break;
2702 6 : if (lane == group_size)
2703 : break;
2704 : }
2705 6 : if (n != chain_len)
2706 : {
2707 : /* Swap that in at first position. */
2708 6 : std::swap (children[0], children[n]);
2709 30 : for (lane = 0; lane < group_size; ++lane)
2710 24 : if (stmts[lane])
2711 24 : std::swap (chains[lane][0], chains[lane][n]);
2712 : }
2713 : else
2714 : {
2715 : /* ??? When this triggers and we end up with two
2716 : vect_constant/external_def up-front things break (ICE)
2717 : spectacularly finding an insertion place for the
2718 : all-constant op. We should have a fully
2719 : vect_internal_def operand though(?) so we can swap
2720 : that into first place and then prepend the all-zero
2721 : constant. */
2722 0 : if (dump_enabled_p ())
2723 0 : dump_printf_loc (MSG_NOTE, vect_location,
2724 : "inserting constant zero to compensate "
2725 : "for (partially) negated first "
2726 : "operand\n");
2727 0 : chain_len++;
2728 0 : for (lane = 0; lane < group_size; ++lane)
2729 0 : if (stmts[lane])
2730 0 : chains[lane].safe_insert
2731 0 : (0, chain_op_t (code, vect_constant_def, NULL_TREE));
2732 0 : vec<tree> zero_ops;
2733 0 : zero_ops.create (group_size);
2734 0 : zero_ops.quick_push (build_zero_cst (TREE_TYPE (vectype)));
2735 0 : for (lane = 1; lane < group_size; ++lane)
2736 0 : if (stmts[lane])
2737 0 : zero_ops.quick_push (zero_ops[0]);
2738 : else
2739 0 : zero_ops.quick_push (NULL_TREE);
2740 0 : slp_tree zero = vect_create_new_slp_node (zero_ops);
2741 0 : SLP_TREE_DEF_TYPE (zero) = vect_constant_def;
2742 0 : children.safe_insert (0, zero);
2743 : }
2744 : break;
2745 : }
2746 979 : for (unsigned i = 1; i < children.length (); ++i)
2747 : {
2748 675 : slp_tree op0 = children[i - 1];
2749 675 : slp_tree op1 = children[i];
2750 675 : bool this_two_op = false;
2751 2391 : for (unsigned lane = 0; lane < group_size; ++lane)
2752 3756 : if (stmts[lane] && chains[lane][i].code != chains[0][i].code)
2753 : {
2754 : this_two_op = true;
2755 : break;
2756 : }
2757 675 : slp_tree child;
2758 675 : if (i == children.length () - 1)
2759 304 : child = vect_create_new_slp_node (node, stmts, 2);
2760 : else
2761 371 : child = vect_create_new_slp_node (2, ERROR_MARK);
2762 675 : if (this_two_op)
2763 : {
2764 162 : vec<std::pair<unsigned, unsigned> > lperm;
2765 162 : lperm.create (group_size);
2766 744 : for (unsigned lane = 0; lane < group_size; ++lane)
2767 840 : lperm.quick_push (std::make_pair
2768 420 : (chains[lane][i].code != chains[0][i].code, lane));
2769 324 : vect_slp_build_two_operator_nodes (child, vectype, op0, op1,
2770 162 : (chains[0][i].code == code
2771 : ? op_stmt_info
2772 : : other_op_stmt_info),
2773 162 : (chains[0][i].code == code
2774 : ? other_op_stmt_info
2775 : : op_stmt_info),
2776 : lperm);
2777 : }
2778 : else
2779 : {
2780 513 : SLP_TREE_DEF_TYPE (child) = vect_internal_def;
2781 513 : SLP_TREE_VECTYPE (child) = vectype;
2782 513 : SLP_TREE_LANES (child) = group_size;
2783 513 : SLP_TREE_CHILDREN (child).quick_push (op0);
2784 513 : SLP_TREE_CHILDREN (child).quick_push (op1);
2785 513 : SLP_TREE_REPRESENTATIVE (child)
2786 1026 : = (chains[0][i].code == code
2787 513 : ? op_stmt_info : other_op_stmt_info);
2788 : }
2789 675 : children[i] = child;
2790 : }
2791 304 : *tree_size += this_tree_size + 1;
2792 1452 : while (!chains.is_empty ())
2793 822 : chains.pop ().release ();
2794 : return node;
2795 : }
2796 21430 : out:
2797 21452 : if (dump_enabled_p ())
2798 2825 : dump_printf_loc (MSG_NOTE, vect_location,
2799 : "failed to line up SLP graph by re-associating "
2800 : "operations in lanes%s\n",
2801 : !hard_fail ? " trying regular discovery" : "");
2802 21457 : while (!children.is_empty ())
2803 5 : vect_free_slp_tree (children.pop ());
2804 21613 : while (!chains.is_empty ())
2805 161 : chains.pop ().release ();
2806 : /* Hard-fail, otherwise we might run into quadratic processing of the
2807 : chains starting one stmt into the chain again. */
2808 21452 : if (hard_fail)
2809 : return NULL;
2810 : /* Fall thru to normal processing. */
2811 21756 : }
2812 :
2813 : /* Get at the operands, verifying they are compatible. */
2814 3683965 : vec<slp_oprnd_info> oprnds_info = vect_create_oprnd_info (nops, group_size);
2815 3683965 : slp_oprnd_info oprnd_info;
2816 21086981 : FOR_EACH_VEC_ELT (stmts, i, stmt_info)
2817 : {
2818 27440828 : int res = vect_get_and_check_slp_defs (vinfo, vectype,
2819 13720414 : swap[i], skip_args,
2820 : stmts, i, &oprnds_info,
2821 : soft_fail);
2822 13720414 : if (res != 0)
2823 : /* ??? This puts -1 back into matches[] and the cache. */
2824 340160 : matches[(res == -1) ? 0 : i] = -1;
2825 13720414 : if (matches[0] == -1)
2826 : break;
2827 : }
2828 17010640 : for (i = 0; i < group_size; ++i)
2829 13576716 : if (matches[i] != 0)
2830 : {
2831 250041 : vect_free_oprnd_info (oprnds_info);
2832 250041 : return NULL;
2833 : }
2834 3433924 : swap = NULL;
2835 :
2836 : /* Perform delayed soft-failing only here so we can factor in mismatches
2837 : determined by vect_get_and_check_slp_defs. */
2838 3433924 : if (soft_fail)
2839 : {
2840 123439 : force_split_matches (matches, group_size,
2841 123439 : TYPE_VECTOR_SUBPARTS (vectype).to_constant ());
2842 123439 : return NULL;
2843 : }
2844 :
2845 3310485 : bool has_two_operators_perm = false;
2846 16552425 : auto_vec<unsigned> two_op_perm_indices[2];
2847 3310485 : vec<stmt_vec_info> two_op_scalar_stmts[2] = {vNULL, vNULL};
2848 :
2849 3326715 : if (two_operators && oprnds_info.length () == 2 && group_size > 2)
2850 : {
2851 4020 : unsigned idx = 0;
2852 4020 : hash_map<gimple *, unsigned> seen;
2853 4020 : vec<slp_oprnd_info> new_oprnds_info
2854 4020 : = vect_create_oprnd_info (1, group_size);
2855 4020 : bool success = true;
2856 :
2857 4020 : enum tree_code code = ERROR_MARK;
2858 4020 : if (oprnds_info[0]->def_stmts[0]
2859 4020 : && is_a<gassign *> (oprnds_info[0]->def_stmts[0]->stmt))
2860 3960 : code = gimple_assign_rhs_code (oprnds_info[0]->def_stmts[0]->stmt);
2861 4020 : basic_block bb = nullptr;
2862 :
2863 7722 : for (unsigned j = 0; j < group_size; ++j)
2864 : {
2865 18008 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2866 : {
2867 14306 : stmt_vec_info stmt_info = oprnd_info->def_stmts[j];
2868 14306 : if (!stmt_info
2869 14139 : || !is_a<gassign *> (stmt_info->stmt)
2870 14139 : || gimple_assign_rhs_code (stmt_info->stmt) != code
2871 25075 : || skip_args[i])
2872 : {
2873 : success = false;
2874 3541 : break;
2875 : }
2876 : /* Avoid mixing lanes with defs in different basic-blocks. */
2877 10769 : if (!bb)
2878 4180 : bb = gimple_bb (vect_orig_stmt (stmt_info)->stmt);
2879 8547 : else if (gimple_bb (vect_orig_stmt (stmt_info)->stmt) != bb)
2880 : {
2881 : success = false;
2882 : break;
2883 : }
2884 :
2885 10765 : bool exists;
2886 10765 : unsigned &stmt_idx
2887 10765 : = seen.get_or_insert (stmt_info->stmt, &exists);
2888 :
2889 10765 : if (!exists)
2890 : {
2891 9384 : new_oprnds_info[0]->def_stmts.safe_push (stmt_info);
2892 9384 : new_oprnds_info[0]->ops.safe_push (oprnd_info->ops[j]);
2893 9384 : stmt_idx = idx;
2894 9384 : idx++;
2895 : }
2896 :
2897 10765 : two_op_perm_indices[i].safe_push (stmt_idx);
2898 : }
2899 :
2900 7243 : if (!success)
2901 : break;
2902 : }
2903 :
2904 4020 : if (success && idx == group_size)
2905 : {
2906 98 : if (dump_enabled_p ())
2907 : {
2908 0 : dump_printf_loc (MSG_NOTE, vect_location,
2909 : "Replace two_operators operands:\n");
2910 :
2911 0 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2912 : {
2913 0 : dump_printf_loc (MSG_NOTE, vect_location,
2914 : "Operand %u:\n", i);
2915 0 : for (unsigned j = 0; j < group_size; j++)
2916 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2917 0 : j, oprnd_info->def_stmts[j]->stmt);
2918 : }
2919 :
2920 0 : dump_printf_loc (MSG_NOTE, vect_location,
2921 : "With a single operand:\n");
2922 0 : for (unsigned j = 0; j < group_size; j++)
2923 0 : dump_printf_loc (MSG_NOTE, vect_location, "\tstmt %u %G",
2924 0 : j, new_oprnds_info[0]->def_stmts[j]->stmt);
2925 : }
2926 :
2927 98 : two_op_scalar_stmts[0].safe_splice (oprnds_info[0]->def_stmts);
2928 98 : two_op_scalar_stmts[1].safe_splice (oprnds_info[1]->def_stmts);
2929 :
2930 98 : new_oprnds_info[0]->first_op_type = oprnds_info[0]->first_op_type;
2931 98 : new_oprnds_info[0]->first_dt = oprnds_info[0]->first_dt;
2932 98 : new_oprnds_info[0]->any_pattern = oprnds_info[0]->any_pattern;
2933 98 : new_oprnds_info[0]->first_gs_p = oprnds_info[0]->first_gs_p;
2934 98 : new_oprnds_info[0]->first_gs_info = oprnds_info[0]->first_gs_info;
2935 :
2936 98 : vect_free_oprnd_info (oprnds_info);
2937 98 : oprnds_info = new_oprnds_info;
2938 98 : nops = 1;
2939 98 : has_two_operators_perm = true;
2940 : }
2941 : else
2942 3922 : vect_free_oprnd_info (new_oprnds_info);
2943 4020 : }
2944 :
2945 6620970 : auto_vec<slp_tree, 4> children;
2946 :
2947 3310485 : stmt_info = stmts[0];
2948 :
2949 3310485 : int reduc_idx = -1;
2950 3310485 : int gs_scale = 0;
2951 3310485 : tree gs_base = NULL_TREE;
2952 :
2953 : /* Create SLP_TREE nodes for the definition node/s. */
2954 8519652 : FOR_EACH_VEC_ELT (oprnds_info, i, oprnd_info)
2955 : {
2956 5321669 : slp_tree child = nullptr;
2957 5321669 : unsigned int j;
2958 5321669 : unsigned old_swap_distance;
2959 5321669 : bool can_swap;
2960 5321669 : bool can_swap_nonmatching;
2961 5321669 : bool *stmt_can_swap;
2962 :
2963 : /* We're skipping certain operands from processing, for example
2964 : outer loop reduction initial defs. */
2965 5321669 : if (skip_args[i])
2966 : {
2967 520421 : children.safe_push (NULL);
2968 5209167 : continue;
2969 : }
2970 :
2971 4801248 : if (oprnd_info->first_dt == vect_uninitialized_def)
2972 : {
2973 : /* COND_EXPR have one too many eventually if the condition
2974 : is a SSA name. */
2975 0 : gcc_assert (i == 3 && nops == 4);
2976 0 : continue;
2977 : }
2978 :
2979 4801248 : if (oprnd_info->first_gs_p)
2980 : {
2981 23593 : gs_scale = oprnd_info->first_gs_info.scale;
2982 23593 : gs_base = oprnd_info->first_gs_info.base;
2983 : }
2984 :
2985 4801248 : if (is_a <bb_vec_info> (vinfo)
2986 1703515 : && oprnd_info->first_dt == vect_internal_def
2987 5711093 : && !oprnd_info->any_pattern)
2988 : {
2989 : /* For BB vectorization, if all defs are the same do not
2990 : bother to continue the build along the single-lane
2991 : graph but use a splat of the scalar value. */
2992 857345 : stmt_vec_info first_def = oprnd_info->def_stmts[0];
2993 923914 : for (j = 1; j < group_size; ++j)
2994 873220 : if (oprnd_info->def_stmts[j] != first_def)
2995 : break;
2996 857345 : if (j == group_size
2997 : /* But avoid doing this for loads where we may be
2998 : able to CSE things, unless the stmt is not
2999 : vectorizable. */
3000 857345 : && (!STMT_VINFO_VECTORIZABLE (first_def)
3001 60924 : || !gimple_vuse (first_def->stmt)))
3002 : {
3003 40899 : if (dump_enabled_p ())
3004 125 : dump_printf_loc (MSG_NOTE, vect_location,
3005 : "Using a splat of the uniform operand %G",
3006 : first_def->stmt);
3007 40899 : oprnd_info->first_dt = vect_external_def;
3008 : }
3009 : }
3010 :
3011 4801248 : if (oprnd_info->first_dt == vect_external_def
3012 4801248 : || oprnd_info->first_dt == vect_constant_def)
3013 : {
3014 1584509 : if (!GET_MODE_SIZE (vinfo->vector_mode).is_constant ())
3015 : {
3016 : tree op0;
3017 : tree uniform_val = op0 = oprnd_info->ops[0];
3018 : for (j = 1; j < oprnd_info->ops.length (); ++j)
3019 : if (oprnd_info->ops[j]
3020 : && !operand_equal_p (uniform_val, oprnd_info->ops[j]))
3021 : {
3022 : uniform_val = NULL_TREE;
3023 : break;
3024 : }
3025 : if (!uniform_val
3026 : && !can_duplicate_and_interleave_p (vinfo,
3027 : oprnd_info->ops.length (),
3028 : TREE_TYPE (op0)))
3029 : {
3030 : matches[j] = -1;
3031 : if (dump_enabled_p ())
3032 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3033 : "Build SLP failed: invalid type of def "
3034 : "for variable-length SLP %T\n", op0);
3035 : goto fail;
3036 : }
3037 : }
3038 1584509 : slp_tree invnode = vect_create_new_slp_node (oprnd_info->ops);
3039 1584509 : SLP_TREE_DEF_TYPE (invnode) = oprnd_info->first_dt;
3040 1584509 : oprnd_info->ops = vNULL;
3041 1584509 : children.safe_push (invnode);
3042 1584509 : continue;
3043 1584509 : }
3044 :
3045 : /* See which SLP operand a reduction chain continues on. We want
3046 : to chain even PHIs but not backedges. */
3047 3216739 : if (STMT_VINFO_REDUC_DEF (oprnd_info->def_stmts[0])
3048 3216739 : || STMT_VINFO_REDUC_IDX (oprnd_info->def_stmts[0]) != -1)
3049 : {
3050 239027 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle)
3051 : {
3052 776 : if (oprnd_info->first_dt == vect_double_reduction_def)
3053 388 : reduc_idx = i;
3054 : }
3055 238251 : else if (is_a <gphi *> (stmt_info->stmt)
3056 238251 : && gimple_phi_num_args
3057 102322 : (as_a <gphi *> (stmt_info->stmt)) != 1)
3058 : ;
3059 136322 : else if (STMT_VINFO_REDUC_IDX (stmt_info) == -1
3060 393 : && STMT_VINFO_DEF_TYPE (stmt_info) != vect_double_reduction_def)
3061 : ;
3062 136322 : else if (reduc_idx == -1)
3063 127830 : reduc_idx = i;
3064 : else
3065 : /* For .COND_* reduction operations the else value can be the
3066 : same as one of the operation operands. The other def
3067 : stmts have been moved, so we can't check easily. Check
3068 : it's a call at least. */
3069 8492 : gcc_assert (is_a <gcall *> (stmt_info->stmt));
3070 : }
3071 :
3072 : /* When we have a masked load with uniform mask discover this
3073 : as a single-lane mask with a splat permute. This way we can
3074 : recognize this as a masked load-lane by stripping the splat. */
3075 3216739 : if (is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
3076 58084 : && gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
3077 : IFN_MASK_LOAD)
3078 4751 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
3079 3216816 : && ! STMT_VINFO_SLP_VECT_ONLY (DR_GROUP_FIRST_ELEMENT (stmt_info)))
3080 : {
3081 35 : vec<stmt_vec_info> def_stmts2;
3082 35 : def_stmts2.create (1);
3083 35 : def_stmts2.quick_push (oprnd_info->def_stmts[0]);
3084 35 : child = vect_build_slp_tree (vinfo, def_stmts2,
3085 : matches, limit,
3086 : &this_tree_size, bst_map);
3087 35 : if (child)
3088 : {
3089 35 : slp_tree pnode = vect_create_new_slp_node (1, VEC_PERM_EXPR);
3090 35 : SLP_TREE_VECTYPE (pnode) = SLP_TREE_VECTYPE (child);
3091 35 : SLP_TREE_LANES (pnode) = group_size;
3092 35 : SLP_TREE_SCALAR_STMTS (pnode).create (group_size);
3093 35 : SLP_TREE_LANE_PERMUTATION (pnode).create (group_size);
3094 245 : for (unsigned k = 0; k < group_size; ++k)
3095 : {
3096 175 : SLP_TREE_SCALAR_STMTS (pnode)
3097 175 : .quick_push (oprnd_info->def_stmts[0]);
3098 175 : SLP_TREE_LANE_PERMUTATION (pnode)
3099 175 : .quick_push (std::make_pair (0u, 0u));
3100 : }
3101 35 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3102 35 : children.safe_push (pnode);
3103 35 : oprnd_info->def_stmts = vNULL;
3104 35 : continue;
3105 35 : }
3106 : else
3107 0 : def_stmts2.release ();
3108 : }
3109 :
3110 6433408 : can_swap = (i == 0
3111 2377761 : && (nops == 2 || nops == 3)
3112 1542320 : && oprnds_info.length () > 1
3113 1542320 : && oprnds_info[1]->first_dt == vect_internal_def
3114 626627 : && (is_gimple_assign (stmt_info->stmt)
3115 51556 : || is_gimple_call (stmt_info->stmt))
3116 : /* Swapping operands for reductions breaks assumptions
3117 : later on. */
3118 3796969 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3119 3216704 : can_swap_nonmatching = can_swap;
3120 3216704 : stmt_can_swap = NULL;
3121 3216704 : if (can_swap)
3122 : {
3123 524401 : stmt_can_swap = XALLOCAVEC (bool, group_size);
3124 8186759 : for (j = 0; j < group_size; ++j)
3125 : {
3126 7662358 : stmt_can_swap[j] = false;
3127 7662358 : if (!stmts[j])
3128 : /* NULL lanes are gaps and have no stmt to swap. */
3129 0 : stmt_can_swap[j] = true;
3130 7662358 : else if (gassign *stmt = dyn_cast <gassign *> (stmts[j]->stmt))
3131 : {
3132 7656309 : tree_code code = gimple_assign_rhs_code (stmt);
3133 15312618 : stmt_can_swap[j] = (commutative_tree_code (code)
3134 7656309 : || commutative_ternary_tree_code (code));
3135 : }
3136 6049 : else if (gcall *call = dyn_cast <gcall *> (stmts[j]->stmt))
3137 : {
3138 6049 : internal_fn fn = (gimple_call_internal_p (call)
3139 6049 : ? gimple_call_internal_fn (call) : IFN_LAST);
3140 12098 : stmt_can_swap[j] = ((commutative_binary_fn_p (fn)
3141 4794 : || commutative_ternary_fn_p (fn))
3142 6085 : && first_commutative_argument (fn) == 0);
3143 : }
3144 :
3145 7662358 : if (j != 0 && !stmt_can_swap[j])
3146 7662358 : can_swap_nonmatching = false;
3147 : }
3148 : }
3149 :
3150 3216704 : old_swap_distance = least_upthread_swappable_op_distance;
3151 3216704 : if (can_swap_nonmatching)
3152 482727 : least_upthread_swappable_op_distance = 1;
3153 2733977 : else if (least_upthread_swappable_op_distance != -1U)
3154 353153 : least_upthread_swappable_op_distance++;
3155 3216704 : child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3156 : matches, limit,
3157 : &this_tree_size, bst_map);
3158 3216704 : least_upthread_swappable_op_distance = old_swap_distance;
3159 3216704 : if (child != NULL)
3160 : {
3161 2707738 : oprnd_info->def_stmts = vNULL;
3162 2707738 : children.safe_push (child);
3163 2707738 : continue;
3164 : }
3165 :
3166 : /* If the SLP build for operand zero failed and operand zero
3167 : and one can be commuted try that for the scalar stmts
3168 : that failed the match. */
3169 508966 : if (/* A first scalar stmt mismatch signals a fatal mismatch. */
3170 508966 : matches[0] == 0
3171 508966 : && can_swap)
3172 : {
3173 : /* See whether we can swap the matching or the non-matching
3174 : stmt operands. */
3175 : bool swap_not_matching = true;
3176 74666 : do
3177 : {
3178 7115569 : for (j = 0; j < group_size; ++j)
3179 : {
3180 7059057 : if ((matches[j] == 0) != !swap_not_matching)
3181 89105 : continue;
3182 : /* Verify if we can swap operands of this stmt. */
3183 6969952 : if (!stmt_can_swap[j])
3184 : {
3185 18154 : if (!swap_not_matching)
3186 7993 : goto fail;
3187 : swap_not_matching = false;
3188 : break;
3189 : }
3190 : }
3191 : }
3192 66673 : while (j != group_size);
3193 :
3194 : /* Swap mismatched definition stmts. */
3195 56512 : if (dump_enabled_p ())
3196 432 : dump_printf_loc (MSG_NOTE, vect_location,
3197 : "Re-trying with swapped operands of stmts ");
3198 7086197 : for (j = 0; j < group_size; ++j)
3199 7029685 : if ((matches[j] == 0) == !swap_not_matching)
3200 : {
3201 13903188 : std::swap (oprnds_info[0]->def_stmts[j],
3202 6951594 : oprnds_info[1]->def_stmts[j]);
3203 13903188 : std::swap (oprnds_info[0]->ops[j],
3204 6951594 : oprnds_info[1]->ops[j]);
3205 6951594 : if (dump_enabled_p ())
3206 1101 : dump_printf (MSG_NOTE, "%d ", j);
3207 : }
3208 56512 : if (dump_enabled_p ())
3209 432 : dump_printf (MSG_NOTE, "\n");
3210 : /* After swapping some operands we lost track whether an
3211 : operand has any pattern defs so be conservative here. */
3212 109875 : if (oprnds_info[0]->any_pattern || oprnds_info[1]->any_pattern)
3213 4226 : oprnds_info[0]->any_pattern = oprnds_info[1]->any_pattern = true;
3214 : /* And try again with scratch 'matches' ... */
3215 56512 : match_elt_t *tem = XALLOCAVEC (match_elt_t, group_size);
3216 56512 : if ((child = vect_build_slp_tree (vinfo, oprnd_info->def_stmts,
3217 : tem, limit,
3218 : &this_tree_size, bst_map)) != NULL)
3219 : {
3220 7762 : oprnd_info->def_stmts = vNULL;
3221 7762 : children.safe_push (child);
3222 7762 : continue;
3223 : }
3224 : }
3225 501204 : fail:
3226 :
3227 : /* If the SLP build failed and we analyze a basic-block
3228 : simply treat nodes we fail to build as externally defined
3229 : (and thus build vectors from the scalar defs).
3230 : The cost model will reject outright expensive cases.
3231 : ??? This doesn't treat cases where permutation ultimatively
3232 : fails (or we don't try permutation below). Ideally we'd
3233 : even compute a permutation that will end up with the maximum
3234 : SLP tree size... */
3235 501204 : if (is_a <bb_vec_info> (vinfo)
3236 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3237 : do extra work to cancel the pattern so the uses see the
3238 : scalar version. */
3239 : /* Skip building vector operands from scalars while operand
3240 : discovery may still be fixed by retrying with swapped operands. */
3241 441994 : && (least_upthread_swappable_op_distance != 1
3242 : /* A first scalar stmt mismatch signals a fatal mismatch
3243 : that the parent commutative retry cannot recover.
3244 : ??? Possibly revisit this with the matches[] improvements
3245 : as we can have matches[0] == -2 here. */
3246 27871 : || matches[0] != 0)
3247 422872 : && !is_pattern_stmt_p (stmt_info)
3248 896756 : && !oprnd_info->any_pattern)
3249 : {
3250 : /* But if there's a leading vector sized set of matching stmts
3251 : fail here so we can split the group. This matches the condition
3252 : vect_analyze_slp_instance uses. */
3253 : /* ??? We might want to split here and combine the results to support
3254 : multiple vector sizes better. */
3255 613644 : for (j = 0; j < group_size; ++j)
3256 613644 : if (matches[j] != 0)
3257 : break;
3258 395187 : if (!known_ge (j, TYPE_VECTOR_SUBPARTS (vectype))
3259 395153 : && vect_slp_can_convert_to_external (oprnd_info->def_stmts))
3260 : {
3261 388702 : if (dump_enabled_p ())
3262 797 : dump_printf_loc (MSG_NOTE, vect_location,
3263 : "Building vector operands from scalars\n");
3264 388702 : this_tree_size++;
3265 388702 : child = vect_create_new_slp_node (oprnd_info->ops);
3266 388702 : children.safe_push (child);
3267 388702 : oprnd_info->ops = vNULL;
3268 388702 : continue;
3269 : }
3270 : }
3271 :
3272 112502 : gcc_assert (child == NULL);
3273 245333 : FOR_EACH_VEC_ELT (children, j, child)
3274 20329 : if (child)
3275 20329 : vect_free_slp_tree (child);
3276 112502 : vect_free_oprnd_info (oprnds_info);
3277 112502 : return NULL;
3278 : }
3279 :
3280 3197983 : vect_free_oprnd_info (oprnds_info);
3281 :
3282 : /* If we have all children of a child built up from uniform scalars
3283 : or does more than one possibly expensive vector construction then
3284 : just throw that away, causing it built up from scalars.
3285 : The exception is the SLP node for the vector store. */
3286 3197983 : if (is_a <bb_vec_info> (vinfo)
3287 1160816 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info)
3288 : /* ??? Rejecting patterns this way doesn't work. We'd have to
3289 : do extra work to cancel the pattern so the uses see the
3290 : scalar version. */
3291 3681173 : && !is_pattern_stmt_p (stmt_info))
3292 : {
3293 : slp_tree child;
3294 : unsigned j;
3295 : bool all_uniform_p = true;
3296 : unsigned n_vector_builds = 0;
3297 1347237 : FOR_EACH_VEC_ELT (children, j, child)
3298 : {
3299 898257 : if (!child)
3300 : ;
3301 898257 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
3302 : all_uniform_p = false;
3303 641457 : else if (!vect_slp_tree_uniform_p (child))
3304 : {
3305 485385 : all_uniform_p = false;
3306 485385 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def)
3307 444873 : n_vector_builds++;
3308 : }
3309 : }
3310 448980 : if (all_uniform_p
3311 448980 : || n_vector_builds > 1
3312 762390 : || (n_vector_builds == children.length ()
3313 35229 : && is_a <gphi *> (stmt_info->stmt)))
3314 : {
3315 : /* Roll back. */
3316 139958 : matches[0] = -1;
3317 443907 : FOR_EACH_VEC_ELT (children, j, child)
3318 303949 : if (child)
3319 303949 : vect_free_slp_tree (child);
3320 :
3321 139958 : if (dump_enabled_p ())
3322 262 : dump_printf_loc (MSG_NOTE, vect_location,
3323 : "Building parent vector operands from "
3324 : "scalars instead\n");
3325 3310485 : return NULL;
3326 : }
3327 : }
3328 :
3329 3058025 : *tree_size += this_tree_size + 1;
3330 :
3331 3058025 : if (two_operators)
3332 : {
3333 : /* ??? We'd likely want to either cache in bst_map sth like
3334 : { a+b, NULL, a+b, NULL } and { NULL, a-b, NULL, a-b } or
3335 : the true { a+b, a+b, a+b, a+b } ... but there we don't have
3336 : explicit stmts to put in so the keying on 'stmts' doesn't
3337 : work (but we have the same issue with nodes that use 'ops'). */
3338 :
3339 7570 : if (has_two_operators_perm)
3340 : {
3341 46 : slp_tree child = children[0];
3342 46 : children.truncate (0);
3343 184 : for (i = 0; i < 2; i++)
3344 : {
3345 92 : slp_tree pnode
3346 92 : = vect_create_new_slp_node (two_op_scalar_stmts[i], 2);
3347 92 : SLP_TREE_CODE (pnode) = VEC_PERM_EXPR;
3348 92 : SLP_TREE_VECTYPE (pnode) = vectype;
3349 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3350 92 : SLP_TREE_CHILDREN (pnode).quick_push (child);
3351 92 : SLP_TREE_REPRESENTATIVE (pnode) = NULL;
3352 92 : lane_permutation_t& perm = SLP_TREE_LANE_PERMUTATION (pnode);
3353 92 : children.safe_push (pnode);
3354 :
3355 808 : for (unsigned j = 0; j < stmts.length (); j++)
3356 624 : perm.safe_push (std::make_pair (0, two_op_perm_indices[i][j]));
3357 : }
3358 :
3359 46 : SLP_TREE_REF_COUNT (child) += 4;
3360 : }
3361 :
3362 7570 : slp_tree one = new _slp_tree;
3363 7570 : slp_tree two = new _slp_tree;
3364 7570 : SLP_TREE_DEF_TYPE (one) = vect_internal_def;
3365 7570 : SLP_TREE_DEF_TYPE (two) = vect_internal_def;
3366 7570 : SLP_TREE_VECTYPE (one) = vectype;
3367 7570 : SLP_TREE_VECTYPE (two) = vectype;
3368 7570 : SLP_TREE_CHILDREN (one).safe_splice (children);
3369 7570 : SLP_TREE_CHILDREN (two).safe_splice (children);
3370 7570 : slp_tree child;
3371 30282 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (two), i, child)
3372 15142 : SLP_TREE_REF_COUNT (child)++;
3373 :
3374 : /* Here we record the original defs since this
3375 : node represents the final lane configuration. */
3376 7570 : node = vect_create_new_slp_node (node, stmts, 2);
3377 7570 : SLP_TREE_VECTYPE (node) = vectype;
3378 7570 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
3379 7570 : SLP_TREE_CHILDREN (node).quick_push (one);
3380 7570 : SLP_TREE_CHILDREN (node).quick_push (two);
3381 7570 : SLP_TREE_REPRESENTATIVE (node) = NULL;
3382 7570 : enum tree_code code0 = ERROR_MARK;
3383 7570 : if (gassign *stmt = dyn_cast <gassign *> (stmts[0]->stmt))
3384 7568 : code0 = gimple_assign_rhs_code (stmt);
3385 7570 : stmt_vec_info ostmt_info;
3386 7570 : unsigned j = 0;
3387 27105 : FOR_EACH_VEC_ELT (stmts, i, ostmt_info)
3388 : {
3389 19535 : int op = 0;
3390 19535 : gassign *ostmt = dyn_cast<gassign *> (ostmt_info->stmt);
3391 19531 : if ((ostmt && gimple_assign_rhs_code (ostmt) != code0)
3392 : || (!ostmt
3393 8 : && gimple_call_combined_fn (stmts[0]->stmt)
3394 4 : != gimple_call_combined_fn (ostmt_info->stmt)))
3395 : {
3396 : op = 1;
3397 : j = i;
3398 : }
3399 19535 : SLP_TREE_LANE_PERMUTATION (node).safe_push (std::make_pair (op, i));
3400 : }
3401 7570 : SLP_TREE_LANES (one) = stmts.length ();
3402 7570 : SLP_TREE_LANES (two) = stmts.length ();
3403 7570 : SLP_TREE_REPRESENTATIVE (one) = stmts[0];
3404 7570 : SLP_TREE_REPRESENTATIVE (two) = stmts[j];
3405 :
3406 7570 : return node;
3407 : }
3408 :
3409 3050455 : node = vect_create_new_slp_node (node, stmts, nops);
3410 3050455 : SLP_TREE_VECTYPE (node) = vectype;
3411 3050455 : SLP_TREE_CHILDREN (node).splice (children);
3412 3050455 : SLP_TREE_GS_SCALE (node) = gs_scale;
3413 3050455 : SLP_TREE_GS_BASE (node) = gs_base;
3414 3050455 : if (reduc_idx != -1)
3415 : {
3416 119689 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) != -1
3417 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_nested_cycle
3418 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def);
3419 119689 : SLP_TREE_REDUC_IDX (node) = reduc_idx;
3420 119689 : node->cycle_info.id = SLP_TREE_CHILDREN (node)[reduc_idx]->cycle_info.id;
3421 : }
3422 : /* When reaching the reduction PHI, create a vect_reduc_info. */
3423 2930766 : else if ((STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
3424 2930766 : || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3425 2930766 : && is_a <gphi *> (STMT_VINFO_STMT (stmt_info)))
3426 : {
3427 104345 : loop_vec_info loop_vinfo = as_a <loop_vec_info> (vinfo);
3428 104345 : gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info) == -1);
3429 104345 : node->cycle_info.id = loop_vinfo->reduc_infos.length ();
3430 104345 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
3431 104345 : loop_vinfo->reduc_infos.safe_push (reduc_info);
3432 104345 : stmt_vec_info reduc_phi = stmt_info;
3433 : /* ??? For double reductions vect_is_simple_reduction stores the
3434 : reduction type and code on the inner loop header PHI. */
3435 104345 : if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
3436 : {
3437 388 : use_operand_p use_p;
3438 388 : gimple *use_stmt;
3439 388 : bool res = single_imm_use (gimple_phi_result (stmt_info->stmt),
3440 : &use_p, &use_stmt);
3441 388 : gcc_assert (res);
3442 388 : reduc_phi = loop_vinfo->lookup_stmt (use_stmt);
3443 : }
3444 104345 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (stmt_info);
3445 104345 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (reduc_phi);
3446 104345 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (reduc_phi);
3447 104345 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
3448 : }
3449 : return node;
3450 9931455 : }
3451 :
3452 : /* Dump a single SLP tree NODE. */
3453 :
3454 : static void
3455 482302 : vect_print_slp_tree (dump_flags_t dump_kind, dump_location_t loc,
3456 : slp_tree node)
3457 : {
3458 482302 : unsigned i, j;
3459 482302 : slp_tree child;
3460 482302 : stmt_vec_info stmt_info;
3461 482302 : tree op;
3462 :
3463 482302 : dump_metadata_t metadata (dump_kind, loc.get_impl_location ());
3464 482302 : dump_user_location_t user_loc = loc.get_user_location ();
3465 482302 : dump_printf_loc (metadata, user_loc,
3466 : "node%s %p (refcnt=%u)",
3467 482302 : SLP_TREE_DEF_TYPE (node) == vect_external_def
3468 : ? " (external)"
3469 : : (SLP_TREE_DEF_TYPE (node) == vect_constant_def
3470 465517 : ? " (constant)"
3471 : : ""), (void *) node,
3472 : SLP_TREE_REF_COUNT (node));
3473 482302 : if (SLP_TREE_VECTYPE (node))
3474 406982 : dump_printf (metadata, " %T", SLP_TREE_VECTYPE (node));
3475 482302 : dump_printf (metadata, "%s",
3476 482302 : node->avoid_stlf_fail ? " (avoid-stlf-fail)" : "");
3477 482302 : if (node->cycle_info.id != -1 || node->cycle_info.reduc_idx != -1)
3478 24625 : dump_printf (metadata, " cycle %d, link %d", node->cycle_info.id,
3479 : node->cycle_info.reduc_idx);
3480 482302 : if (node->si)
3481 5382 : dump_printf (metadata, " @%G", node->si);
3482 : else
3483 476920 : dump_printf (metadata, "\n");
3484 482302 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3485 : {
3486 390973 : if (SLP_TREE_PERMUTE_P (node))
3487 17320 : dump_printf_loc (metadata, user_loc, "op: VEC_PERM_EXPR\n");
3488 : else
3489 373653 : dump_printf_loc (metadata, user_loc, "op template: %G",
3490 373653 : SLP_TREE_REPRESENTATIVE (node)->stmt);
3491 : }
3492 482302 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
3493 935318 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
3494 553381 : if (stmt_info)
3495 547229 : dump_printf_loc (metadata, user_loc, "\t%sstmt %u %G",
3496 547229 : SLP_TREE_LIVE_LANES (node).contains (i)
3497 543455 : ? "[l*]" : (STMT_VINFO_LIVE_P (stmt_info)
3498 543455 : ? "[l] " : ""),
3499 : i, stmt_info->stmt);
3500 : else
3501 6152 : dump_printf_loc (metadata, user_loc, "\tstmt %u ---\n", i);
3502 : else
3503 : {
3504 100365 : dump_printf_loc (metadata, user_loc, "\t{ ");
3505 221590 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (node), i, op)
3506 121225 : dump_printf (metadata, "%T%s ", op,
3507 121225 : i < SLP_TREE_SCALAR_OPS (node).length () - 1 ? "," : "");
3508 100365 : dump_printf (metadata, "}\n");
3509 : }
3510 482302 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
3511 : {
3512 68738 : dump_printf_loc (metadata, user_loc, "\tload permutation {");
3513 156656 : FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (node), i, j)
3514 87918 : dump_printf (dump_kind, " %u", j);
3515 68738 : dump_printf (dump_kind, " }\n");
3516 : }
3517 482302 : if (SLP_TREE_LANE_PERMUTATION (node).exists ())
3518 : {
3519 17328 : dump_printf_loc (metadata, user_loc, "\tlane permutation {");
3520 64872 : for (i = 0; i < SLP_TREE_LANE_PERMUTATION (node).length (); ++i)
3521 47544 : dump_printf (dump_kind, " %u[%u]",
3522 47544 : SLP_TREE_LANE_PERMUTATION (node)[i].first,
3523 47544 : SLP_TREE_LANE_PERMUTATION (node)[i].second);
3524 17328 : dump_printf (dump_kind, " }%s\n",
3525 17328 : node->ldst_lanes ? " (load-lanes)" : "");
3526 : }
3527 482302 : if (SLP_TREE_CHILDREN (node).is_empty ())
3528 182774 : return;
3529 299528 : dump_printf_loc (metadata, user_loc, "\tchildren");
3530 790057 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3531 490529 : dump_printf (dump_kind, " %p", (void *)child);
3532 299528 : dump_printf (dump_kind, "%s\n",
3533 299528 : node->ldst_lanes && !SLP_TREE_LANE_PERMUTATION (node).exists ()
3534 : ? " (store-lanes)" : "");
3535 : }
3536 :
3537 : DEBUG_FUNCTION void
3538 0 : debug (slp_tree node)
3539 : {
3540 0 : debug_dump_context ctx;
3541 0 : vect_print_slp_tree (MSG_NOTE,
3542 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3543 : node);
3544 0 : }
3545 :
3546 : /* Recursive helper for the dot producer below. */
3547 :
3548 : static void
3549 0 : dot_slp_tree (FILE *f, slp_tree node, hash_set<slp_tree> &visited)
3550 : {
3551 0 : if (visited.add (node))
3552 : return;
3553 :
3554 0 : fprintf (f, "\"%p\" [label=\"", (void *)node);
3555 0 : vect_print_slp_tree (MSG_NOTE,
3556 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3557 : node);
3558 0 : fprintf (f, "\"];\n");
3559 :
3560 :
3561 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3562 0 : fprintf (f, "\"%p\" -> \"%p\";", (void *)node, (void *)child);
3563 :
3564 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
3565 0 : if (child)
3566 0 : dot_slp_tree (f, child, visited);
3567 : }
3568 :
3569 : DEBUG_FUNCTION void
3570 0 : dot_slp_tree (const char *fname, slp_tree node)
3571 : {
3572 0 : FILE *f = fopen (fname, "w");
3573 0 : fprintf (f, "digraph {\n");
3574 0 : fflush (f);
3575 0 : {
3576 0 : debug_dump_context ctx (f);
3577 0 : hash_set<slp_tree> visited;
3578 0 : dot_slp_tree (f, node, visited);
3579 0 : }
3580 0 : fflush (f);
3581 0 : fprintf (f, "}\n");
3582 0 : fclose (f);
3583 0 : }
3584 :
3585 : DEBUG_FUNCTION void
3586 0 : dot_slp_tree (const char *fname, const vec<slp_instance> &slp_instances)
3587 : {
3588 0 : FILE *f = fopen (fname, "w");
3589 0 : fprintf (f, "digraph {\n");
3590 0 : fflush (f);
3591 0 : {
3592 0 : debug_dump_context ctx (f);
3593 0 : hash_set<slp_tree> visited;
3594 0 : for (auto inst : slp_instances)
3595 0 : dot_slp_tree (f, SLP_INSTANCE_TREE (inst), visited);
3596 0 : }
3597 0 : fflush (f);
3598 0 : fprintf (f, "}\n");
3599 0 : fclose (f);
3600 0 : }
3601 :
3602 : /* Dump a slp tree NODE using flags specified in DUMP_KIND. */
3603 :
3604 : static void
3605 520735 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3606 : slp_tree node, hash_set<slp_tree> &visited)
3607 : {
3608 520735 : unsigned i;
3609 520735 : slp_tree child;
3610 :
3611 520735 : if (visited.add (node))
3612 520735 : return;
3613 :
3614 478471 : vect_print_slp_tree (dump_kind, loc, node);
3615 :
3616 1442342 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3617 485400 : if (child)
3618 440451 : vect_print_slp_graph (dump_kind, loc, child, visited);
3619 : }
3620 :
3621 : static void
3622 49125 : vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc,
3623 : slp_tree entry)
3624 : {
3625 49125 : hash_set<slp_tree> visited;
3626 49125 : vect_print_slp_graph (dump_kind, loc, entry, visited);
3627 49125 : }
3628 :
3629 : DEBUG_FUNCTION void
3630 0 : debug (slp_instance instance)
3631 : {
3632 0 : debug_dump_context ctx;
3633 0 : vect_print_slp_graph (MSG_NOTE,
3634 0 : dump_location_t::from_location_t (UNKNOWN_LOCATION),
3635 : SLP_INSTANCE_TREE (instance));
3636 0 : }
3637 :
3638 :
3639 : /* Compute the set of scalar stmts participating in external nodes. */
3640 :
3641 : static void
3642 1706611 : vect_slp_gather_extern_scalar_stmts (vec_info *vinfo, slp_tree node,
3643 : hash_set<slp_tree> &visited,
3644 : hash_set<stmt_vec_info> &estmts)
3645 : {
3646 1706611 : if (visited.add (node))
3647 : return;
3648 :
3649 1645472 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def)
3650 : {
3651 : slp_tree child;
3652 : int i;
3653 1930087 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3654 985879 : if (child)
3655 985879 : vect_slp_gather_extern_scalar_stmts (vinfo, child, visited, estmts);
3656 : }
3657 : else
3658 3926576 : for (tree def : SLP_TREE_SCALAR_OPS (node))
3659 : {
3660 1824956 : stmt_vec_info def_stmt = vinfo->lookup_def (def);
3661 1824956 : if (def_stmt)
3662 390897 : estmts.add (def_stmt);
3663 : }
3664 : }
3665 :
3666 : /* Mark the original scalar stmt coverage of the vector SLP graph of VINFO
3667 : with STMT_SLP_TYPE == pure_slp. */
3668 :
3669 : static void
3670 248621 : vect_bb_slp_mark_stmts_vectorized (bb_vec_info vinfo)
3671 : {
3672 : /* Gather the scalar stmt leafs of the SLP graph to stop the below DFS
3673 : walk on. */
3674 248621 : hash_set<stmt_vec_info> scalar_stmts_in_externs;
3675 248621 : hash_set<slp_tree> visited;
3676 1466595 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3677 720732 : vect_slp_gather_extern_scalar_stmts (vinfo, SLP_INSTANCE_TREE (instance),
3678 : visited, scalar_stmts_in_externs);
3679 :
3680 : /* DFS walk scalar stmts to compute the vectorized coverage indicated
3681 : by STMT_SLP_TYPE (stmt) == pure_slp on the original scalar (non-pattern)
3682 : stmts. */
3683 1466595 : for (auto instance : BB_VINFO_SLP_INSTANCES (vinfo))
3684 : {
3685 917069 : for (auto stmt : SLP_INSTANCE_ROOT_STMTS (instance))
3686 90967 : if (!scalar_stmts_in_externs.contains (stmt))
3687 89722 : STMT_SLP_TYPE (stmt) = pure_slp;
3688 720732 : auto_vec<stmt_vec_info> worklist;
3689 4062422 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
3690 : {
3691 1900226 : stmt = vect_orig_stmt (stmt);
3692 1900226 : if (!scalar_stmts_in_externs.contains (stmt)
3693 1900226 : && STMT_SLP_TYPE (stmt) != pure_slp)
3694 : {
3695 1883261 : STMT_SLP_TYPE (stmt) = pure_slp;
3696 1883261 : worklist.safe_push (stmt);
3697 : }
3698 : }
3699 3791975 : while (!worklist.is_empty ())
3700 : {
3701 2355384 : stmt_vec_info stmt = worklist.pop ();
3702 :
3703 : /* Now walk relevant parts of the SSA use-def graph. */
3704 2355384 : slp_oprnds child_ops (stmt);
3705 7346269 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
3706 : {
3707 2635501 : tree op = child_ops.get_op_for_slp_child (stmt, i);
3708 2635501 : stmt_vec_info def = vinfo->lookup_def (op);
3709 2635501 : if (def
3710 947416 : && !scalar_stmts_in_externs.contains (def)
3711 3203370 : && STMT_SLP_TYPE (def) != pure_slp)
3712 : {
3713 472123 : STMT_SLP_TYPE (def) = pure_slp;
3714 472123 : worklist.safe_push (def);
3715 : }
3716 : }
3717 : }
3718 720732 : }
3719 248621 : }
3720 :
3721 : /* Mark the statements of the tree rooted at NODE as relevant (vect_used). */
3722 :
3723 : static void
3724 2614921 : vect_mark_slp_stmts_relevant (slp_tree node, hash_set<slp_tree> &visited)
3725 : {
3726 2614921 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3727 : return;
3728 :
3729 1572888 : if (visited.add (node))
3730 : return;
3731 :
3732 7341408 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
3733 3279218 : if (stmt_info)
3734 : {
3735 3279218 : gcc_assert (!STMT_VINFO_RELEVANT (stmt_info)
3736 : || STMT_VINFO_RELEVANT (stmt_info) == vect_used_in_scope);
3737 3279218 : STMT_VINFO_RELEVANT (stmt_info) = vect_used_in_scope;
3738 : }
3739 :
3740 5690125 : for (auto child: SLP_TREE_CHILDREN (node))
3741 1789485 : if (child)
3742 1789485 : vect_mark_slp_stmts_relevant (child, visited);
3743 : }
3744 :
3745 : static void
3746 825436 : vect_mark_slp_stmts_relevant (slp_tree node)
3747 : {
3748 825436 : hash_set<slp_tree> visited;
3749 825436 : vect_mark_slp_stmts_relevant (node, visited);
3750 825436 : }
3751 :
3752 :
3753 : /* Gather loads in the SLP graph NODE and populate the INST loads array. */
3754 :
3755 : static void
3756 11253104 : vect_gather_slp_loads (vec<slp_tree> &loads, slp_tree node,
3757 : hash_set<slp_tree> &visited)
3758 : {
3759 11253104 : if (!node || visited.add (node))
3760 : return;
3761 :
3762 9404240 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
3763 : return;
3764 :
3765 6951961 : if (!SLP_TREE_PERMUTE_P (node))
3766 : {
3767 6737581 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
3768 6737581 : if (STMT_VINFO_DATA_REF (stmt_info)
3769 2885551 : && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
3770 1647439 : loads.safe_push (node);
3771 : }
3772 :
3773 6951961 : unsigned i;
3774 6951961 : slp_tree child;
3775 15851320 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
3776 8899359 : vect_gather_slp_loads (loads, child, visited);
3777 : }
3778 :
3779 :
3780 : /* Find the last store in SLP INSTANCE. */
3781 :
3782 : stmt_vec_info
3783 2565852 : vect_find_last_scalar_stmt_in_slp (slp_tree node)
3784 : {
3785 2565852 : stmt_vec_info last = NULL;
3786 14517977 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3787 6820421 : if (stmt_vinfo)
3788 : {
3789 6820421 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3790 6820421 : last = last ? get_later_stmt (stmt_vinfo, last) : stmt_vinfo;
3791 : }
3792 :
3793 2565852 : return last;
3794 : }
3795 :
3796 : /* Find the first stmt in NODE. */
3797 :
3798 : stmt_vec_info
3799 657874 : vect_find_first_scalar_stmt_in_slp (slp_tree node)
3800 : {
3801 657874 : stmt_vec_info first = NULL;
3802 :
3803 3559741 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
3804 1586119 : if (stmt_vinfo)
3805 : {
3806 1583330 : stmt_vinfo = vect_orig_stmt (stmt_vinfo);
3807 1583330 : if (!first
3808 1583330 : || get_later_stmt (stmt_vinfo, first) == first)
3809 : first = stmt_vinfo;
3810 : }
3811 :
3812 657874 : return first;
3813 : }
3814 :
3815 : /* Splits a group of stores, currently beginning at FIRST_VINFO, into
3816 : two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
3817 : (also containing the first GROUP1_SIZE stmts, since stores are
3818 : consecutive), the second containing the remainder.
3819 : Return the first stmt in the second group. */
3820 :
3821 : static stmt_vec_info
3822 138107 : vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
3823 : {
3824 138107 : gcc_assert (DR_GROUP_FIRST_ELEMENT (first_vinfo) == first_vinfo);
3825 138107 : gcc_assert (group1_size > 0);
3826 138107 : int group2_size = DR_GROUP_SIZE (first_vinfo) - group1_size;
3827 138107 : gcc_assert (group2_size > 0);
3828 138107 : DR_GROUP_SIZE (first_vinfo) = group1_size;
3829 :
3830 138107 : stmt_vec_info stmt_info = first_vinfo;
3831 462443 : for (unsigned i = group1_size; i > 1; i--)
3832 : {
3833 324336 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info);
3834 324336 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3835 : }
3836 : /* STMT is now the last element of the first group. */
3837 138107 : stmt_vec_info group2 = DR_GROUP_NEXT_ELEMENT (stmt_info);
3838 138107 : DR_GROUP_NEXT_ELEMENT (stmt_info) = 0;
3839 :
3840 138107 : DR_GROUP_SIZE (group2) = group2_size;
3841 409464 : for (stmt_info = group2; stmt_info;
3842 271357 : stmt_info = DR_GROUP_NEXT_ELEMENT (stmt_info))
3843 : {
3844 271357 : DR_GROUP_FIRST_ELEMENT (stmt_info) = group2;
3845 271357 : gcc_assert (DR_GROUP_GAP (stmt_info) == 1);
3846 : }
3847 :
3848 : /* For the second group, the DR_GROUP_GAP is that before the original group,
3849 : plus skipping over the first vector. */
3850 138107 : DR_GROUP_GAP (group2) = DR_GROUP_GAP (first_vinfo) + group1_size;
3851 :
3852 : /* DR_GROUP_GAP of the first group now has to skip over the second group too. */
3853 138107 : DR_GROUP_GAP (first_vinfo) += group2_size;
3854 :
3855 138107 : if (dump_enabled_p ())
3856 88 : dump_printf_loc (MSG_NOTE, vect_location, "Split group into %d and %d\n",
3857 : group1_size, group2_size);
3858 :
3859 138107 : return group2;
3860 : }
3861 :
3862 : /* Calculate the unrolling factor for an SLP instance with GROUP_SIZE
3863 : statements and a vector of NUNITS elements. */
3864 :
3865 : static poly_uint64
3866 2888739 : calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
3867 : {
3868 2888739 : return exact_div (common_multiple (nunits, group_size), group_size);
3869 : }
3870 :
3871 : /* Helper that checks to see if a node is a load node. */
3872 :
3873 : static inline bool
3874 103 : vect_is_slp_load_node (slp_tree root)
3875 : {
3876 103 : return (!SLP_TREE_PERMUTE_P (root)
3877 103 : && SLP_TREE_DEF_TYPE (root) == vect_internal_def
3878 97 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_REPRESENTATIVE (root))
3879 167 : && DR_IS_READ (STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (root))));
3880 : }
3881 :
3882 :
3883 : /* Helper function of optimize_load_redistribution that performs the operation
3884 : recursively. */
3885 :
3886 : static slp_tree
3887 21801 : optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map,
3888 : vec_info *vinfo, unsigned int group_size,
3889 : hash_map<slp_tree, slp_tree> *load_map,
3890 : slp_tree root)
3891 : {
3892 21801 : if (slp_tree *leader = load_map->get (root))
3893 4549 : return *leader;
3894 :
3895 17252 : slp_tree node;
3896 17252 : unsigned i;
3897 :
3898 : /* For now, we don't know anything about externals so do not do anything. */
3899 17252 : if (!root || SLP_TREE_DEF_TYPE (root) != vect_internal_def)
3900 : return NULL;
3901 12116 : else if (SLP_TREE_PERMUTE_P (root))
3902 : {
3903 : /* First convert this node into a load node and add it to the leaves
3904 : list and flatten the permute from a lane to a load one. If it's
3905 : unneeded it will be elided later. */
3906 71 : vec<stmt_vec_info> stmts;
3907 71 : stmts.create (SLP_TREE_LANES (root));
3908 71 : lane_permutation_t lane_perm = SLP_TREE_LANE_PERMUTATION (root);
3909 135 : for (unsigned j = 0; j < lane_perm.length (); j++)
3910 : {
3911 103 : std::pair<unsigned, unsigned> perm = lane_perm[j];
3912 103 : node = SLP_TREE_CHILDREN (root)[perm.first];
3913 :
3914 103 : if (!vect_is_slp_load_node (node)
3915 103 : || SLP_TREE_CHILDREN (node).exists ())
3916 : {
3917 39 : stmts.release ();
3918 39 : goto next;
3919 : }
3920 :
3921 64 : stmts.quick_push (SLP_TREE_SCALAR_STMTS (node)[perm.second]);
3922 : }
3923 :
3924 32 : if (dump_enabled_p ())
3925 0 : dump_printf_loc (MSG_NOTE, vect_location,
3926 : "converting stmts on permute node %p\n",
3927 : (void *) root);
3928 :
3929 32 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
3930 32 : unsigned tree_size = 0, limit = 1;
3931 32 : node = vect_build_slp_tree (vinfo, stmts,
3932 : matches, &limit, &tree_size, bst_map);
3933 32 : if (!node)
3934 0 : stmts.release ();
3935 :
3936 32 : load_map->put (root, node);
3937 32 : return node;
3938 : }
3939 :
3940 12045 : next:
3941 12084 : load_map->put (root, NULL);
3942 :
3943 27523 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3944 : {
3945 15439 : slp_tree value
3946 15439 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3947 : node);
3948 15439 : if (value)
3949 : {
3950 32 : SLP_TREE_REF_COUNT (value)++;
3951 32 : SLP_TREE_CHILDREN (root)[i] = value;
3952 : /* ??? We know the original leafs of the replaced nodes will
3953 : be referenced by bst_map, only the permutes created by
3954 : pattern matching are not. */
3955 32 : if (SLP_TREE_REF_COUNT (node) == 1)
3956 32 : load_map->remove (node);
3957 32 : vect_free_slp_tree (node);
3958 : }
3959 : }
3960 :
3961 : return NULL;
3962 : }
3963 :
3964 : /* Temporary workaround for loads not being CSEd during SLP build. This
3965 : function will traverse the SLP tree rooted in ROOT for INSTANCE and find
3966 : VEC_PERM nodes that blend vectors from multiple nodes that all read from the
3967 : same DR such that the final operation is equal to a permuted load. Such
3968 : NODES are then directly converted into LOADS themselves. The nodes are
3969 : CSEd using BST_MAP. */
3970 :
3971 : static void
3972 4180 : optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map,
3973 : vec_info *vinfo, unsigned int group_size,
3974 : hash_map<slp_tree, slp_tree> *load_map,
3975 : slp_tree root)
3976 : {
3977 4180 : slp_tree node;
3978 4180 : unsigned i;
3979 :
3980 10542 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (root), i , node)
3981 : {
3982 6362 : slp_tree value
3983 6362 : = optimize_load_redistribution_1 (bst_map, vinfo, group_size, load_map,
3984 : node);
3985 6362 : if (value)
3986 : {
3987 0 : SLP_TREE_REF_COUNT (value)++;
3988 0 : SLP_TREE_CHILDREN (root)[i] = value;
3989 : /* ??? We know the original leafs of the replaced nodes will
3990 : be referenced by bst_map, only the permutes created by
3991 : pattern matching are not. */
3992 0 : if (SLP_TREE_REF_COUNT (node) == 1)
3993 0 : load_map->remove (node);
3994 0 : vect_free_slp_tree (node);
3995 : }
3996 : }
3997 4180 : }
3998 :
3999 : /* Helper function of vect_match_slp_patterns.
4000 :
4001 : Attempts to match patterns against the slp tree rooted in REF_NODE using
4002 : VINFO. Patterns are matched in post-order traversal.
4003 :
4004 : If matching is successful the value in REF_NODE is updated and returned, if
4005 : not then it is returned unchanged. */
4006 :
4007 : static bool
4008 6533293 : vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo,
4009 : slp_tree_to_load_perm_map_t *perm_cache,
4010 : slp_compat_nodes_map_t *compat_cache,
4011 : hash_set<slp_tree> *visited)
4012 : {
4013 6533293 : unsigned i;
4014 6533293 : slp_tree node = *ref_node;
4015 6533293 : bool found_p = false;
4016 6533293 : if (!node || visited->add (node))
4017 : return false;
4018 :
4019 : slp_tree child;
4020 10488998 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
4021 4891745 : found_p |= vect_match_slp_patterns_2 (&SLP_TREE_CHILDREN (node)[i],
4022 : vinfo, perm_cache, compat_cache,
4023 : visited);
4024 :
4025 16791759 : for (unsigned x = 0; x < num__slp_patterns; x++)
4026 : {
4027 11194506 : vect_pattern *pattern
4028 11194506 : = slp_patterns[x] (perm_cache, compat_cache, ref_node);
4029 11194506 : if (pattern)
4030 : {
4031 1114 : pattern->build (vinfo);
4032 1114 : delete pattern;
4033 : found_p = true;
4034 : }
4035 : }
4036 :
4037 : return found_p;
4038 : }
4039 :
4040 : /* Applies pattern matching to the given SLP tree rooted in REF_NODE using
4041 : vec_info VINFO.
4042 :
4043 : The modified tree is returned. Patterns are tried in order and multiple
4044 : patterns may match. */
4045 :
4046 : static bool
4047 1641548 : vect_match_slp_patterns (slp_instance instance, vec_info *vinfo,
4048 : hash_set<slp_tree> *visited,
4049 : slp_tree_to_load_perm_map_t *perm_cache,
4050 : slp_compat_nodes_map_t *compat_cache)
4051 : {
4052 1641548 : DUMP_VECT_SCOPE ("vect_match_slp_patterns");
4053 1641548 : slp_tree *ref_node = &SLP_INSTANCE_TREE (instance);
4054 :
4055 1641548 : if (dump_enabled_p ())
4056 32512 : dump_printf_loc (MSG_NOTE, vect_location,
4057 : "Analyzing SLP tree %p for patterns\n",
4058 32512 : (void *) SLP_INSTANCE_TREE (instance));
4059 :
4060 1641548 : return vect_match_slp_patterns_2 (ref_node, vinfo, perm_cache, compat_cache,
4061 1641548 : visited);
4062 : }
4063 :
4064 : /* STMT_INFO is a store group of size GROUP_SIZE that we are considering
4065 : vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether
4066 : the stores are masked.
4067 : Return true if we could use IFN_STORE_LANES instead and if that appears
4068 : to be the better approach. */
4069 :
4070 : static bool
4071 6132 : vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info,
4072 : tree vectype, bool masked_p,
4073 : unsigned int group_size,
4074 : unsigned int new_group_size)
4075 : {
4076 6132 : if (!vectype)
4077 : {
4078 6132 : tree scalar_type = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
4079 6132 : vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
4080 : }
4081 6132 : if (!vectype)
4082 : return false;
4083 : /* Allow the split if one of the two new groups would operate on full
4084 : vectors *within* rather than across one scalar loop iteration.
4085 : This is purely a heuristic, but it should work well for group
4086 : sizes of 3 and 4, where the possible splits are:
4087 :
4088 : 3->2+1: OK if the vector has exactly two elements
4089 : 4->2+2: Likewise
4090 : 4->3+1: Less clear-cut. */
4091 6132 : if (multiple_p (group_size - new_group_size, TYPE_VECTOR_SUBPARTS (vectype))
4092 6132 : || multiple_p (new_group_size, TYPE_VECTOR_SUBPARTS (vectype)))
4093 : return false;
4094 3488 : return vect_store_lanes_supported (vectype, group_size, masked_p) != IFN_LAST;
4095 : }
4096 :
4097 : /* Analyze an SLP instance starting from a group of grouped stores. Call
4098 : vect_build_slp_tree to build a tree of packed stmts if possible.
4099 : Return FALSE if it's impossible to SLP any stmt in the loop. */
4100 :
4101 : static bool
4102 : vect_analyze_slp_instance (vec_info *vinfo,
4103 : scalar_stmts_to_slp_tree_map_t *bst_map,
4104 : stmt_vec_info stmt_info, slp_instance_kind kind,
4105 : unsigned max_tree_size, unsigned *limit,
4106 : bool force_single_lane);
4107 :
4108 : /* Build an interleaving scheme for the store sources RHS_NODES from
4109 : SCALAR_STMTS. */
4110 :
4111 : static slp_tree
4112 8249 : vect_build_slp_store_interleaving (vec<slp_tree> &rhs_nodes,
4113 : vec<stmt_vec_info> &scalar_stmts)
4114 : {
4115 8249 : unsigned int group_size = scalar_stmts.length ();
4116 16498 : slp_tree node = vect_create_new_slp_node (scalar_stmts,
4117 8249 : SLP_TREE_CHILDREN
4118 : (rhs_nodes[0]).length ());
4119 8249 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
4120 8249 : for (unsigned l = 0;
4121 16525 : l < SLP_TREE_CHILDREN (rhs_nodes[0]).length (); ++l)
4122 : {
4123 : /* And a permute merging all RHS SLP trees. */
4124 8276 : slp_tree perm = vect_create_new_slp_node (rhs_nodes.length (),
4125 8276 : VEC_PERM_EXPR);
4126 8276 : SLP_TREE_CHILDREN (node).quick_push (perm);
4127 8276 : SLP_TREE_LANE_PERMUTATION (perm).create (group_size);
4128 8276 : SLP_TREE_VECTYPE (perm) = SLP_TREE_VECTYPE (node);
4129 8276 : SLP_TREE_LANES (perm) = group_size;
4130 8276 : SLP_TREE_REPRESENTATIVE (perm) = NULL;
4131 64708 : for (unsigned j = 0; j < rhs_nodes.length (); ++j)
4132 : {
4133 24078 : SLP_TREE_CHILDREN (perm)
4134 24078 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[l]);
4135 24078 : SLP_TREE_CHILDREN (rhs_nodes[j])[l]->refcnt++;
4136 24078 : for (unsigned k = 0;
4137 50547 : k < SLP_TREE_LANES (rhs_nodes[j]); ++k)
4138 : {
4139 : /* ??? We should populate SLP_TREE_SCALAR_STMTS
4140 : or SLP_TREE_SCALAR_OPS but then we might have
4141 : a mix of both in our children. */
4142 26469 : SLP_TREE_LANE_PERMUTATION (perm)
4143 26469 : .quick_push (std::make_pair (j, k));
4144 : }
4145 : }
4146 :
4147 : /* Now we have a single permute node but we cannot code-generate
4148 : the case with more than two inputs.
4149 : Perform pairwise reduction, reducing the two inputs
4150 : with the least number of lanes to one and then repeat until
4151 : we end up with two inputs. That scheme makes sure we end
4152 : up with permutes satisfying the restriction of requiring at
4153 : most two vector inputs to produce a single vector output
4154 : when the number of lanes is even. */
4155 15802 : while (SLP_TREE_CHILDREN (perm).length () > 2)
4156 : {
4157 : /* When we have three equal sized groups left the pairwise
4158 : reduction does not result in a scheme that avoids using
4159 : three vectors. Instead merge the first two groups
4160 : to the final size with do-not-care elements (chosen
4161 : from the first group) and then merge with the third.
4162 : { A0, B0, x, A1, B1, x, ... }
4163 : -> { A0, B0, C0, A1, B1, C1, ... }
4164 : This handles group size of three (and at least
4165 : power-of-two multiples of that). */
4166 7526 : if (SLP_TREE_CHILDREN (perm).length () == 3
4167 3370 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4168 3370 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[1]))
4169 7526 : && (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[0])
4170 2489 : == SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[2])))
4171 : {
4172 2183 : int ai = 0;
4173 2183 : int bi = 1;
4174 2183 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4175 2183 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4176 2183 : unsigned n = SLP_TREE_LANES (perm);
4177 :
4178 2183 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4179 2183 : SLP_TREE_LANES (permab) = n;
4180 2183 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4181 2183 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4182 : /* ??? Should be NULL but that's not expected. */
4183 2183 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4184 2183 : SLP_TREE_CHILDREN (permab).quick_push (a);
4185 6563 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4186 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4187 2197 : .quick_push (std::make_pair (0, k));
4188 2183 : SLP_TREE_CHILDREN (permab).quick_push (b);
4189 6563 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4190 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4191 2197 : .quick_push (std::make_pair (1, k));
4192 : /* Push the do-not-care lanes. */
4193 4380 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4194 2197 : SLP_TREE_LANE_PERMUTATION (permab)
4195 2197 : .quick_push (std::make_pair (0, k));
4196 :
4197 : /* Put the merged node into 'perm', in place of a. */
4198 2183 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4199 : /* Adjust the references to b in the permutation
4200 : of perm and to the later children which we'll
4201 : remove. */
4202 8774 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4203 : {
4204 6591 : std::pair<unsigned, unsigned> &p
4205 6591 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4206 6591 : if (p.first == (unsigned) bi)
4207 : {
4208 2197 : p.first = ai;
4209 2197 : p.second += SLP_TREE_LANES (a);
4210 : }
4211 4394 : else if (p.first > (unsigned) bi)
4212 2197 : p.first--;
4213 : }
4214 2183 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4215 2183 : break;
4216 : }
4217 :
4218 : /* Pick the two nodes with the least number of lanes,
4219 : prefer the earliest candidate and maintain ai < bi. */
4220 : int ai = -1;
4221 : int bi = -1;
4222 50446 : for (unsigned ci = 0; ci < SLP_TREE_CHILDREN (perm).length (); ++ci)
4223 : {
4224 45103 : if (ai == -1)
4225 5343 : ai = ci;
4226 39760 : else if (bi == -1)
4227 5343 : bi = ci;
4228 34417 : else if ((SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4229 34417 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai]))
4230 34417 : || (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ci])
4231 28593 : < SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi])))
4232 : {
4233 12538 : if (SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[ai])
4234 6269 : <= SLP_TREE_LANES (SLP_TREE_CHILDREN (perm)[bi]))
4235 2892 : bi = ci;
4236 : else
4237 : {
4238 3377 : ai = bi;
4239 3377 : bi = ci;
4240 : }
4241 : }
4242 : }
4243 :
4244 : /* Produce a merge of nodes ai and bi. */
4245 5343 : slp_tree a = SLP_TREE_CHILDREN (perm)[ai];
4246 5343 : slp_tree b = SLP_TREE_CHILDREN (perm)[bi];
4247 5343 : unsigned n = SLP_TREE_LANES (a) + SLP_TREE_LANES (b);
4248 5343 : slp_tree permab = vect_create_new_slp_node (2, VEC_PERM_EXPR);
4249 5343 : SLP_TREE_LANES (permab) = n;
4250 5343 : SLP_TREE_LANE_PERMUTATION (permab).create (n);
4251 5343 : SLP_TREE_VECTYPE (permab) = SLP_TREE_VECTYPE (perm);
4252 : /* ??? Should be NULL but that's not expected. */
4253 5343 : SLP_TREE_REPRESENTATIVE (permab) = SLP_TREE_REPRESENTATIVE (perm);
4254 5343 : SLP_TREE_CHILDREN (permab).quick_push (a);
4255 19553 : for (unsigned k = 0; k < SLP_TREE_LANES (a); ++k)
4256 8867 : SLP_TREE_LANE_PERMUTATION (permab)
4257 8867 : .quick_push (std::make_pair (0, k));
4258 5343 : SLP_TREE_CHILDREN (permab).quick_push (b);
4259 18829 : for (unsigned k = 0; k < SLP_TREE_LANES (b); ++k)
4260 8143 : SLP_TREE_LANE_PERMUTATION (permab)
4261 8143 : .quick_push (std::make_pair (1, k));
4262 :
4263 : /* Put the merged node into 'perm', in place of a. */
4264 5343 : SLP_TREE_CHILDREN (perm)[ai] = permab;
4265 : /* Adjust the references to b in the permutation
4266 : of perm and to the later children which we'll
4267 : remove. */
4268 81265 : for (unsigned k = 0; k < SLP_TREE_LANES (perm); ++k)
4269 : {
4270 75922 : std::pair<unsigned, unsigned> &p
4271 75922 : = SLP_TREE_LANE_PERMUTATION (perm)[k];
4272 75922 : if (p.first == (unsigned) bi)
4273 : {
4274 8143 : p.first = ai;
4275 8143 : p.second += SLP_TREE_LANES (a);
4276 : }
4277 67779 : else if (p.first > (unsigned) bi)
4278 28686 : p.first--;
4279 : }
4280 5343 : SLP_TREE_CHILDREN (perm).ordered_remove (bi);
4281 : }
4282 : }
4283 :
4284 8249 : return node;
4285 : }
4286 :
4287 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4288 : of KIND. Return true if successful. SCALAR_STMTS is owned by this
4289 : function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to
4290 : the caller upon failure. */
4291 :
4292 : static bool
4293 2114337 : vect_build_slp_instance (vec_info *vinfo,
4294 : slp_instance_kind kind,
4295 : vec<stmt_vec_info> &scalar_stmts,
4296 : vec<stmt_vec_info> &root_stmt_infos,
4297 : vec<tree> &remain,
4298 : unsigned max_tree_size, unsigned *limit,
4299 : scalar_stmts_to_slp_tree_map_t *bst_map,
4300 : bool force_single_lane)
4301 : {
4302 : /* If there's no budget left bail out early. */
4303 2114337 : if (*limit == 0)
4304 : {
4305 16098 : scalar_stmts.release ();
4306 16098 : return false;
4307 : }
4308 :
4309 2098239 : if (kind == slp_inst_kind_ctor)
4310 : {
4311 15333 : if (dump_enabled_p ())
4312 98 : dump_printf_loc (MSG_NOTE, vect_location,
4313 : "Analyzing vectorizable constructor: %G\n",
4314 49 : root_stmt_infos[0]->stmt);
4315 : }
4316 2082906 : else if (kind == slp_inst_kind_gcond)
4317 : {
4318 296852 : if (dump_enabled_p ())
4319 6222 : dump_printf_loc (MSG_NOTE, vect_location,
4320 : "Analyzing vectorizable control flow: %G",
4321 3111 : root_stmt_infos[0]->stmt);
4322 : }
4323 1786054 : else if (kind == slp_inst_kind_bb_reduc)
4324 : {
4325 1390808 : if (dump_enabled_p ())
4326 7492 : dump_printf_loc (MSG_NOTE, vect_location,
4327 : "Analyzing vectorizable BB reduction: %G",
4328 3746 : root_stmt_infos[0]->stmt);
4329 : }
4330 :
4331 2098239 : if (dump_enabled_p ())
4332 : {
4333 27560 : dump_printf_loc (MSG_NOTE, vect_location,
4334 : "Starting SLP discovery for\n");
4335 59568 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4336 64016 : dump_printf_loc (MSG_NOTE, vect_location,
4337 32008 : " %G", scalar_stmts[i]->stmt);
4338 : }
4339 :
4340 : /* Build the tree for the SLP instance. */
4341 2098239 : unsigned int group_size = scalar_stmts.length ();
4342 2098239 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4343 2098239 : unsigned tree_size = 0;
4344 :
4345 2098239 : slp_tree node = NULL;
4346 2098239 : if (group_size > 1 && force_single_lane)
4347 : {
4348 0 : matches[0] = 0;
4349 0 : matches[1] = -1;
4350 : }
4351 : else
4352 2098239 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4353 : &tree_size, bst_map);
4354 2098239 : if (node != NULL)
4355 : {
4356 : /* Create a new SLP instance. */
4357 834218 : slp_instance new_instance = XNEW (class _slp_instance);
4358 834218 : SLP_INSTANCE_TREE (new_instance) = node;
4359 834218 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4360 834218 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4361 834218 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4362 834218 : SLP_INSTANCE_KIND (new_instance) = kind;
4363 834218 : new_instance->reduc_phis = NULL;
4364 834218 : new_instance->cost_vec = vNULL;
4365 834218 : new_instance->subgraph_entries = vNULL;
4366 :
4367 834218 : if (dump_enabled_p ())
4368 24237 : dump_printf_loc (MSG_NOTE, vect_location,
4369 : "SLP size %u vs. limit %u.\n",
4370 : tree_size, max_tree_size);
4371 :
4372 834218 : vinfo->slp_instances.safe_push (new_instance);
4373 :
4374 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4375 : the number of SLP lanes of the root in a few places.
4376 : Verify that assumption holds. */
4377 834218 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4378 : == group_size);
4379 :
4380 834218 : if (dump_enabled_p ())
4381 : {
4382 24237 : if (kind == slp_inst_kind_reduc_group)
4383 1509 : dump_printf_loc (MSG_NOTE, vect_location,
4384 : "SLP discovery of size %d reduction group "
4385 : "succeeded\n", group_size);
4386 24237 : dump_printf_loc (MSG_NOTE, vect_location,
4387 : "Final SLP tree for instance %p:\n",
4388 : (void *) new_instance);
4389 24237 : vect_print_slp_graph (MSG_NOTE, vect_location,
4390 : SLP_INSTANCE_TREE (new_instance));
4391 : }
4392 :
4393 834218 : return true;
4394 : }
4395 : /* Failed to SLP. */
4396 :
4397 : /* While we arrive here even with slp_inst_kind_store we should only
4398 : for group_size == 1. The code to split store groups is only in
4399 : vect_analyze_slp_instance now. */
4400 1264021 : gcc_assert (kind != slp_inst_kind_store || group_size == 1);
4401 :
4402 : /* For BB vectorization we get failures only in case of the need of
4403 : unrolling, as otherwise we'll simply get operands built from scalars.
4404 : Iff there is any mismatches in the toplevel stmts those will prevail,
4405 : otherwise we get the non-power-of-two tail of the lanes failed.
4406 : For BB reductions we mainly want to catch the first case so we pick
4407 : a more useful subset of lanes to reduce. Pick the largest matching
4408 : subset of that covers half of the group or more. */
4409 1264021 : if (kind == slp_inst_kind_bb_reduc && matches[0] != -1)
4410 : {
4411 676502 : unsigned *n_matching = XALLOCAVEC (unsigned, group_size);
4412 676502 : memset (n_matching, 0, sizeof (unsigned) * group_size);
4413 2189758 : for (unsigned i = 0; i < group_size; ++i)
4414 1513256 : if (matches[i] != -2 && matches[i] != -1)
4415 1276273 : n_matching[matches[i].v]++;
4416 : unsigned largest_i = 0;
4417 1513256 : for (unsigned i = 1; i < group_size; ++i)
4418 836754 : if (n_matching[i] > n_matching[largest_i])
4419 62904 : largest_i = i;
4420 : /* Pick the largest matching part and put the rest to remain. */
4421 676502 : if (n_matching[largest_i] >= 2
4422 50777 : && n_matching[largest_i] >= group_size / 2)
4423 : {
4424 : /* As we know the matches[] stmts match up, recursing for
4425 : non-power-of-two sizes will just force-fail the tail
4426 : for us at hopefully optimal vector size and succesfully
4427 : finish discovery. */
4428 49413 : vec<stmt_vec_info> scalar_stmts2;
4429 49413 : scalar_stmts2.create (n_matching[largest_i]);
4430 292694 : for (unsigned i = 0; i < group_size; ++i)
4431 193868 : if (matches[i] == (int)largest_i)
4432 132230 : scalar_stmts2.quick_push (scalar_stmts[i]);
4433 : else
4434 61638 : remain.safe_push
4435 65624 : (gimple_get_lhs (vect_orig_stmt (scalar_stmts[i])->stmt));
4436 49413 : if (dump_enabled_p ())
4437 228 : dump_printf_loc (MSG_NOTE, vect_location, "Splitting %d "
4438 : "non-matching lanes to scalar remains\n",
4439 228 : scalar_stmts.length () - scalar_stmts2.length ());
4440 49413 : scalar_stmts.release ();
4441 49413 : return vect_build_slp_instance (vinfo, kind, scalar_stmts2,
4442 : root_stmt_infos, remain,
4443 : max_tree_size, limit, bst_map,
4444 : force_single_lane);
4445 : }
4446 : }
4447 :
4448 : /* Free the allocated memory. */
4449 1214608 : scalar_stmts.release ();
4450 :
4451 : /* Failed to SLP. */
4452 1214608 : if (dump_enabled_p ())
4453 3209 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4454 : return false;
4455 : }
4456 :
4457 : /* Analyze an SLP instance starting from a the start of a reduction chain.
4458 : Call vect_build_slp_tree to build a tree of packed stmts if possible.
4459 : Return FALSE if SLP build fails. */
4460 :
4461 : static bool
4462 73600 : vect_analyze_slp_reduc_chain (loop_vec_info vinfo,
4463 : scalar_stmts_to_slp_tree_map_t *bst_map,
4464 : stmt_vec_info scalar_stmt,
4465 : unsigned max_tree_size, unsigned *limit)
4466 : {
4467 73600 : vec<stmt_vec_info> scalar_stmts = vNULL;
4468 :
4469 73600 : bool fail = false;
4470 : /* ??? We could leave operation code checking to SLP discovery. */
4471 73600 : code_helper code = STMT_VINFO_REDUC_CODE (STMT_VINFO_REDUC_DEF
4472 : (vect_orig_stmt (scalar_stmt)));
4473 73600 : bool first = true;
4474 73600 : stmt_vec_info next_stmt = scalar_stmt;
4475 83803 : do
4476 : {
4477 83803 : stmt_vec_info stmt = next_stmt;
4478 83803 : gimple_match_op op, orig_op;
4479 83803 : if (!gimple_extract_op (STMT_VINFO_STMT (stmt), &op))
4480 0 : gcc_unreachable ();
4481 167606 : tree reduc_def = gimple_arg (STMT_VINFO_STMT (stmt),
4482 83803 : STMT_VINFO_REDUC_IDX (stmt));
4483 83803 : next_stmt = vect_stmt_to_vectorize (vinfo->lookup_def (reduc_def));
4484 83803 : gcc_assert (is_a <gphi *> (STMT_VINFO_STMT (next_stmt))
4485 : || STMT_VINFO_REDUC_IDX (next_stmt) != -1);
4486 89767 : if (!gimple_extract_op (STMT_VINFO_STMT (vect_orig_stmt (stmt)),
4487 : &orig_op))
4488 0 : gcc_unreachable ();
4489 83803 : if (CONVERT_EXPR_CODE_P (op.code)
4490 4999 : && tree_nop_conversion_p (op.type, TREE_TYPE (op.ops[0]))
4491 88790 : && (first
4492 2498 : || is_a <gphi *> (STMT_VINFO_STMT (next_stmt))))
4493 : ;
4494 78876 : else if (code != orig_op.code)
4495 : {
4496 2728 : fail = true;
4497 2728 : break;
4498 : }
4499 : else
4500 76148 : scalar_stmts.safe_push (stmt);
4501 81075 : first = false;
4502 : }
4503 81075 : while (!is_a <gphi *> (STMT_VINFO_STMT (next_stmt)));
4504 73600 : if (fail)
4505 2728 : return false;
4506 :
4507 : /* When the SSA def chain through reduc-idx does not form a natural
4508 : reduction chain try to linearize an associative operation manually. */
4509 70872 : if (scalar_stmts.length () == 1
4510 68202 : && code.is_tree_code ()
4511 62130 : && associative_tree_code ((tree_code)code)
4512 : /* We may not associate if a fold-left reduction is required. */
4513 131572 : && !needs_fold_left_reduction_p (TREE_TYPE (gimple_get_lhs
4514 : (scalar_stmts[0]->stmt)),
4515 : code))
4516 : {
4517 : /* Remember a stmt with the actual reduction operation. */
4518 57426 : stmt_vec_info reduc_scalar_stmt = scalar_stmts[0];
4519 57426 : auto_vec<chain_op_t> chain;
4520 57426 : auto_vec<std::pair<tree_code, gimple *> > worklist;
4521 57426 : gimple *op_stmt = NULL, *other_op_stmt = NULL;
4522 57426 : if (is_a <gassign *> (scalar_stmts[0]->stmt)
4523 : /* We cannot linearize an operation that vect_slp_linearize_chain
4524 : would not put on its worklist. */
4525 57426 : && gimple_assign_rhs_code (scalar_stmts[0]->stmt) == (tree_code)code)
4526 : {
4527 56770 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4528 56770 : scalar_stmts[0]->stmt, op_stmt,
4529 : other_op_stmt,
4530 : NULL);
4531 :
4532 56770 : scalar_stmts.truncate (0);
4533 56770 : stmt_vec_info tail = NULL;
4534 284151 : for (auto el : chain)
4535 : {
4536 114140 : if (el.dt == vect_external_def
4537 114140 : || el.dt == vect_constant_def
4538 114140 : || el.code != (tree_code) code)
4539 : {
4540 299 : scalar_stmts.release ();
4541 299 : return false;
4542 : }
4543 113841 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4544 113841 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4545 111555 : || STMT_VINFO_REDUC_DEF (stmt))
4546 : {
4547 56680 : gcc_assert (tail == NULL);
4548 56680 : tail = stmt;
4549 56680 : continue;
4550 : }
4551 57161 : scalar_stmts.safe_push (stmt);
4552 : }
4553 56471 : gcc_assert (tail);
4554 : }
4555 :
4556 : /* When this linearization didn't produce a chain see if stripping
4557 : a wrapping sign conversion produces one. */
4558 57127 : if (scalar_stmts.length () == 1
4559 57127 : && (code == PLUS_EXPR || code == MULT_EXPR || code == BIT_IOR_EXPR
4560 : || code == BIT_AND_EXPR || code == BIT_XOR_EXPR))
4561 : {
4562 55310 : gimple *stmt = scalar_stmts[0]->stmt;
4563 55310 : if (!is_gimple_assign (stmt)
4564 54150 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
4565 4724 : || TREE_CODE (gimple_assign_rhs1 (stmt)) != SSA_NAME
4566 60034 : || !tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
4567 4724 : TREE_TYPE (gimple_assign_rhs1 (stmt))))
4568 : {
4569 53530 : scalar_stmts.release ();
4570 53530 : return false;
4571 : }
4572 1780 : stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (stmt));
4573 1780 : if (!is_gimple_assign (stmt)
4574 1780 : || gimple_assign_rhs_code (stmt) != (tree_code)code)
4575 : {
4576 1762 : scalar_stmts.release ();
4577 1762 : return false;
4578 : }
4579 18 : chain.truncate (0);
4580 18 : vect_slp_linearize_chain (vinfo, worklist, chain, (tree_code)code,
4581 : stmt, op_stmt, other_op_stmt, NULL);
4582 :
4583 18 : scalar_stmts.truncate (0);
4584 18 : stmt_vec_info tail = NULL;
4585 88 : for (auto el : chain)
4586 : {
4587 42 : if (el.dt == vect_external_def
4588 42 : || el.dt == vect_constant_def
4589 42 : || el.code != (tree_code) code)
4590 : {
4591 8 : scalar_stmts.release ();
4592 8 : return false;
4593 : }
4594 34 : stmt_vec_info stmt = vinfo->lookup_def (el.op);
4595 34 : if (STMT_VINFO_REDUC_IDX (stmt) != -1
4596 34 : || STMT_VINFO_REDUC_DEF (stmt))
4597 : {
4598 0 : gcc_assert (tail == NULL);
4599 0 : tail = stmt;
4600 0 : continue;
4601 : }
4602 34 : scalar_stmts.safe_push (stmt);
4603 : }
4604 : /* Unlike the above this does not include the reduction SSA
4605 : cycle. */
4606 10 : gcc_assert (!tail);
4607 : }
4608 :
4609 1827 : if (scalar_stmts.length () < 2)
4610 : {
4611 1697 : scalar_stmts.release ();
4612 1697 : return false;
4613 : }
4614 :
4615 130 : if (dump_enabled_p ())
4616 : {
4617 40 : dump_printf_loc (MSG_NOTE, vect_location,
4618 : "Starting SLP discovery of reduction chain for\n");
4619 164 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4620 248 : dump_printf_loc (MSG_NOTE, vect_location,
4621 124 : " %G", scalar_stmts[i]->stmt);
4622 : }
4623 :
4624 130 : unsigned int group_size = scalar_stmts.length ();
4625 130 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4626 130 : unsigned tree_size = 0;
4627 130 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4628 130 : &tree_size, bst_map);
4629 130 : if (!node)
4630 : {
4631 46 : scalar_stmts.release ();
4632 46 : return false;
4633 : }
4634 :
4635 84 : unsigned cycle_id = vinfo->reduc_infos.length ();
4636 84 : vect_reduc_info reduc_info = new vect_reduc_info_s ();
4637 84 : vinfo->reduc_infos.safe_push (reduc_info);
4638 84 : VECT_REDUC_INFO_DEF_TYPE (reduc_info) = STMT_VINFO_DEF_TYPE (next_stmt);
4639 84 : VECT_REDUC_INFO_TYPE (reduc_info) = STMT_VINFO_REDUC_TYPE (next_stmt);
4640 84 : VECT_REDUC_INFO_CODE (reduc_info) = STMT_VINFO_REDUC_CODE (next_stmt);
4641 84 : VECT_REDUC_INFO_FN (reduc_info) = IFN_LAST;
4642 84 : reduc_info->is_reduc_chain = true;
4643 :
4644 : /* Build the node for the PHI and possibly the conversions. */
4645 84 : slp_tree phis = vect_create_new_slp_node (2, ERROR_MARK);
4646 84 : SLP_TREE_REPRESENTATIVE (phis) = next_stmt;
4647 84 : phis->cycle_info.id = cycle_id;
4648 84 : SLP_TREE_LANES (phis) = group_size;
4649 84 : if (reduc_scalar_stmt == scalar_stmt)
4650 80 : SLP_TREE_VECTYPE (phis) = SLP_TREE_VECTYPE (node);
4651 : else
4652 4 : SLP_TREE_VECTYPE (phis)
4653 4 : = signed_or_unsigned_type_for (TYPE_UNSIGNED
4654 : (TREE_TYPE (gimple_get_lhs
4655 : (scalar_stmt->stmt))),
4656 : SLP_TREE_VECTYPE (node));
4657 : /* ??? vect_cse_slp_nodes cannot cope with cycles without any
4658 : SLP_TREE_SCALAR_STMTS. */
4659 84 : SLP_TREE_SCALAR_STMTS (phis).create (group_size);
4660 501 : for (unsigned i = 0; i < group_size; ++i)
4661 333 : SLP_TREE_SCALAR_STMTS (phis).quick_push (next_stmt);
4662 :
4663 84 : slp_tree op_input = phis;
4664 84 : if (reduc_scalar_stmt != scalar_stmt)
4665 : {
4666 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4667 4 : SLP_TREE_REPRESENTATIVE (conv)
4668 4 : = vinfo->lookup_def (gimple_arg (reduc_scalar_stmt->stmt,
4669 4 : STMT_VINFO_REDUC_IDX
4670 : (reduc_scalar_stmt)));
4671 4 : SLP_TREE_CHILDREN (conv).quick_push (phis);
4672 4 : conv->cycle_info.id = cycle_id;
4673 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4674 4 : SLP_TREE_LANES (conv) = group_size;
4675 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (node);
4676 4 : SLP_TREE_SCALAR_STMTS (conv) = vNULL;
4677 4 : op_input = conv;
4678 : }
4679 :
4680 84 : slp_tree reduc = vect_create_new_slp_node (2, ERROR_MARK);
4681 84 : SLP_TREE_REPRESENTATIVE (reduc) = reduc_scalar_stmt;
4682 84 : SLP_TREE_CHILDREN (reduc).quick_push (op_input);
4683 84 : SLP_TREE_CHILDREN (reduc).quick_push (node);
4684 84 : reduc->cycle_info.id = cycle_id;
4685 84 : SLP_TREE_REDUC_IDX (reduc) = 0;
4686 84 : SLP_TREE_LANES (reduc) = group_size;
4687 84 : SLP_TREE_VECTYPE (reduc) = SLP_TREE_VECTYPE (node);
4688 : /* ??? For the reduction epilogue we need a live lane. */
4689 84 : SLP_TREE_SCALAR_STMTS (reduc).create (group_size);
4690 84 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (reduc_scalar_stmt);
4691 417 : for (unsigned i = 1; i < group_size; ++i)
4692 249 : SLP_TREE_SCALAR_STMTS (reduc).quick_push (NULL);
4693 :
4694 84 : if (reduc_scalar_stmt != scalar_stmt)
4695 : {
4696 4 : slp_tree conv = vect_create_new_slp_node (1, ERROR_MARK);
4697 4 : SLP_TREE_REPRESENTATIVE (conv) = scalar_stmt;
4698 4 : SLP_TREE_CHILDREN (conv).quick_push (reduc);
4699 4 : conv->cycle_info.id = cycle_id;
4700 4 : SLP_TREE_REDUC_IDX (conv) = 0;
4701 4 : SLP_TREE_LANES (conv) = group_size;
4702 4 : SLP_TREE_VECTYPE (conv) = SLP_TREE_VECTYPE (phis);
4703 : /* ??? For the reduction epilogue we need a live lane. */
4704 4 : SLP_TREE_SCALAR_STMTS (conv).create (group_size);
4705 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (scalar_stmt);
4706 12 : for (unsigned i = 1; i < group_size; ++i)
4707 4 : SLP_TREE_SCALAR_STMTS (conv).quick_push (NULL);
4708 4 : reduc = conv;
4709 : }
4710 :
4711 84 : edge le = loop_latch_edge (LOOP_VINFO_LOOP (vinfo));
4712 84 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4713 84 : SLP_TREE_CHILDREN (phis).quick_push (NULL);
4714 84 : SLP_TREE_CHILDREN (phis)[le->dest_idx] = reduc;
4715 84 : SLP_TREE_REF_COUNT (reduc)++;
4716 :
4717 : /* Create a new SLP instance. */
4718 84 : slp_instance new_instance = XNEW (class _slp_instance);
4719 84 : SLP_INSTANCE_TREE (new_instance) = reduc;
4720 84 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4721 84 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4722 84 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4723 84 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4724 84 : new_instance->reduc_phis = NULL;
4725 84 : new_instance->cost_vec = vNULL;
4726 84 : new_instance->subgraph_entries = vNULL;
4727 :
4728 84 : vinfo->slp_instances.safe_push (new_instance);
4729 :
4730 84 : if (dump_enabled_p ())
4731 : {
4732 30 : dump_printf_loc (MSG_NOTE, vect_location,
4733 : "Final SLP tree for instance %p:\n",
4734 : (void *) new_instance);
4735 30 : vect_print_slp_graph (MSG_NOTE, vect_location,
4736 : SLP_INSTANCE_TREE (new_instance));
4737 : }
4738 :
4739 : return true;
4740 57426 : }
4741 :
4742 13446 : if (scalar_stmts.length () <= 1)
4743 : {
4744 10782 : scalar_stmts.release ();
4745 10782 : return false;
4746 : }
4747 :
4748 2664 : scalar_stmts.reverse ();
4749 2664 : stmt_vec_info reduc_phi_info = next_stmt;
4750 :
4751 : /* Build the tree for the SLP instance. */
4752 2664 : vec<stmt_vec_info> root_stmt_infos = vNULL;
4753 2664 : vec<tree> remain = vNULL;
4754 :
4755 2664 : if (dump_enabled_p ())
4756 : {
4757 219 : dump_printf_loc (MSG_NOTE, vect_location,
4758 : "Starting SLP discovery of reduction chain for\n");
4759 1143 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4760 1848 : dump_printf_loc (MSG_NOTE, vect_location,
4761 924 : " %G", scalar_stmts[i]->stmt);
4762 : }
4763 :
4764 : /* Build the tree for the SLP instance. */
4765 2664 : unsigned int group_size = scalar_stmts.length ();
4766 2664 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4767 2664 : unsigned tree_size = 0;
4768 :
4769 : /* ??? We need this only for SLP discovery. */
4770 10528 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4771 7864 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = scalar_stmts[0];
4772 :
4773 2664 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4774 2664 : &tree_size, bst_map);
4775 :
4776 10528 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4777 7864 : REDUC_GROUP_FIRST_ELEMENT (scalar_stmts[i]) = NULL;
4778 :
4779 2664 : if (node != NULL)
4780 : {
4781 : /* Create a new SLP instance. */
4782 2385 : slp_instance new_instance = XNEW (class _slp_instance);
4783 2385 : SLP_INSTANCE_TREE (new_instance) = node;
4784 2385 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4785 2385 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
4786 2385 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
4787 2385 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_chain;
4788 2385 : new_instance->reduc_phis = NULL;
4789 2385 : new_instance->cost_vec = vNULL;
4790 2385 : new_instance->subgraph_entries = vNULL;
4791 :
4792 2385 : vect_reduc_info reduc_info = info_for_reduction (vinfo, node);
4793 2385 : reduc_info->is_reduc_chain = true;
4794 :
4795 2385 : if (dump_enabled_p ())
4796 153 : dump_printf_loc (MSG_NOTE, vect_location,
4797 : "SLP size %u vs. limit %u.\n",
4798 : tree_size, max_tree_size);
4799 :
4800 : /* Fixup SLP reduction chains. If this is a reduction chain with
4801 : a conversion in front amend the SLP tree with a node for that. */
4802 2385 : gimple *scalar_def = STMT_VINFO_REDUC_DEF (reduc_phi_info)->stmt;
4803 2385 : if (is_gimple_assign (scalar_def)
4804 2385 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (scalar_def)))
4805 : {
4806 43 : stmt_vec_info conv_info = vect_stmt_to_vectorize
4807 43 : (STMT_VINFO_REDUC_DEF (reduc_phi_info));
4808 43 : scalar_stmts = vNULL;
4809 43 : scalar_stmts.create (group_size);
4810 178 : for (unsigned i = 0; i < group_size; ++i)
4811 92 : scalar_stmts.quick_push (conv_info);
4812 43 : slp_tree conv = vect_create_new_slp_node (scalar_stmts, 1);
4813 43 : SLP_TREE_VECTYPE (conv)
4814 43 : = get_vectype_for_scalar_type (vinfo,
4815 43 : TREE_TYPE
4816 : (gimple_assign_lhs (scalar_def)),
4817 : group_size);
4818 43 : SLP_TREE_REDUC_IDX (conv) = 0;
4819 43 : conv->cycle_info.id = node->cycle_info.id;
4820 43 : SLP_TREE_CHILDREN (conv).quick_push (node);
4821 43 : SLP_INSTANCE_TREE (new_instance) = conv;
4822 : }
4823 : /* Fill the backedge child of the PHI SLP node. The
4824 : general matching code cannot find it because the
4825 : scalar code does not reflect how we vectorize the
4826 : reduction. */
4827 2385 : use_operand_p use_p;
4828 2385 : imm_use_iterator imm_iter;
4829 2385 : class loop *loop = LOOP_VINFO_LOOP (vinfo);
4830 9058 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter,
4831 : gimple_get_lhs (scalar_def))
4832 : /* There are exactly two non-debug uses, the reduction
4833 : PHI and the loop-closed PHI node. */
4834 6673 : if (!is_gimple_debug (USE_STMT (use_p))
4835 6673 : && gimple_bb (USE_STMT (use_p)) == loop->header)
4836 : {
4837 2385 : auto_vec<stmt_vec_info, 64> phis (group_size);
4838 2385 : stmt_vec_info phi_info = vinfo->lookup_stmt (USE_STMT (use_p));
4839 9541 : for (unsigned i = 0; i < group_size; ++i)
4840 7156 : phis.quick_push (phi_info);
4841 2385 : slp_tree *phi_node = bst_map->get (phis);
4842 2385 : unsigned dest_idx = loop_latch_edge (loop)->dest_idx;
4843 4770 : SLP_TREE_CHILDREN (*phi_node)[dest_idx]
4844 2385 : = SLP_INSTANCE_TREE (new_instance);
4845 2385 : SLP_INSTANCE_TREE (new_instance)->refcnt++;
4846 2385 : }
4847 :
4848 2385 : vinfo->slp_instances.safe_push (new_instance);
4849 :
4850 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4851 : the number of SLP lanes of the root in a few places.
4852 : Verify that assumption holds. */
4853 2385 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4854 : == group_size);
4855 :
4856 2385 : if (dump_enabled_p ())
4857 : {
4858 153 : dump_printf_loc (MSG_NOTE, vect_location,
4859 : "Final SLP tree for instance %p:\n",
4860 : (void *) new_instance);
4861 153 : vect_print_slp_graph (MSG_NOTE, vect_location,
4862 : SLP_INSTANCE_TREE (new_instance));
4863 : }
4864 :
4865 2385 : return true;
4866 : }
4867 :
4868 : /* Failed to SLP. */
4869 279 : scalar_stmts.release ();
4870 279 : if (dump_enabled_p ())
4871 66 : dump_printf_loc (MSG_NOTE, vect_location,
4872 : "SLP discovery of reduction chain failed\n");
4873 : return false;
4874 : }
4875 :
4876 : /* Analyze an SLP instance starting from SCALAR_STMTS which are a group
4877 : of KIND. Return true if successful. */
4878 :
4879 : static bool
4880 100793 : vect_analyze_slp_reduction (loop_vec_info vinfo,
4881 : stmt_vec_info scalar_stmt,
4882 : unsigned max_tree_size, unsigned *limit,
4883 : scalar_stmts_to_slp_tree_map_t *bst_map,
4884 : bool force_single_lane)
4885 : {
4886 100793 : slp_instance_kind kind = slp_inst_kind_reduc_group;
4887 :
4888 : /* Try to gather a reduction chain. Only attempt if there's budget left
4889 : since chain analysis may build multi-lane trees that consume limit. */
4890 100793 : if (! force_single_lane
4891 73885 : && *limit != 0
4892 73885 : && STMT_VINFO_DEF_TYPE (scalar_stmt) == vect_reduction_def
4893 174393 : && vect_analyze_slp_reduc_chain (vinfo, bst_map, scalar_stmt,
4894 : max_tree_size, limit))
4895 : return true;
4896 :
4897 98324 : vec<stmt_vec_info> scalar_stmts;
4898 98324 : scalar_stmts.create (1);
4899 98324 : scalar_stmts.quick_push (scalar_stmt);
4900 :
4901 98324 : if (dump_enabled_p ())
4902 : {
4903 3945 : dump_printf_loc (MSG_NOTE, vect_location,
4904 : "Starting SLP discovery for\n");
4905 7890 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
4906 7890 : dump_printf_loc (MSG_NOTE, vect_location,
4907 3945 : " %G", scalar_stmts[i]->stmt);
4908 : }
4909 :
4910 : /* Build the tree for the SLP instance. */
4911 98324 : unsigned int group_size = scalar_stmts.length ();
4912 98324 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
4913 98324 : unsigned tree_size = 0;
4914 :
4915 98324 : slp_tree node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
4916 : &tree_size, bst_map);
4917 98324 : if (node != NULL)
4918 : {
4919 : /* Create a new SLP instance. */
4920 96428 : slp_instance new_instance = XNEW (class _slp_instance);
4921 96428 : SLP_INSTANCE_TREE (new_instance) = node;
4922 96428 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4923 96428 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4924 96428 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4925 96428 : SLP_INSTANCE_KIND (new_instance) = kind;
4926 96428 : new_instance->reduc_phis = NULL;
4927 96428 : new_instance->cost_vec = vNULL;
4928 96428 : new_instance->subgraph_entries = vNULL;
4929 :
4930 96428 : if (dump_enabled_p ())
4931 3868 : dump_printf_loc (MSG_NOTE, vect_location,
4932 : "SLP size %u vs. limit %u.\n",
4933 : tree_size, max_tree_size);
4934 :
4935 96428 : vinfo->slp_instances.safe_push (new_instance);
4936 :
4937 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
4938 : the number of SLP lanes of the root in a few places.
4939 : Verify that assumption holds. */
4940 96428 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
4941 : == group_size);
4942 :
4943 96428 : if (dump_enabled_p ())
4944 : {
4945 3868 : dump_printf_loc (MSG_NOTE, vect_location,
4946 : "Final SLP tree for instance %p:\n",
4947 : (void *) new_instance);
4948 3868 : vect_print_slp_graph (MSG_NOTE, vect_location,
4949 : SLP_INSTANCE_TREE (new_instance));
4950 : }
4951 :
4952 96428 : return true;
4953 : }
4954 : /* Failed to SLP. */
4955 :
4956 : /* Free the allocated memory. */
4957 1896 : scalar_stmts.release ();
4958 :
4959 : /* Failed to SLP. */
4960 1896 : if (dump_enabled_p ())
4961 77 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
4962 : return false;
4963 : }
4964 :
4965 : /* Analyze a single SLP reduction group. If successful add a SLP instance
4966 : for it and return true, otherwise return false and have *MATCHES
4967 : populated. */
4968 :
4969 : static bool
4970 24242 : vect_analyze_slp_reduction_group (loop_vec_info loop_vinfo,
4971 : vec<stmt_vec_info> scalar_stmts,
4972 : scalar_stmts_to_slp_tree_map_t *bst_map,
4973 : unsigned max_tree_size, unsigned *limit,
4974 : match_elt_t *matches)
4975 : {
4976 : /* Try to form a reduction group. Size-1 groups are not suitable
4977 : for SLP reduction and should fall back to single-lane reduction. */
4978 24242 : unsigned int group_size = scalar_stmts.length ();
4979 24242 : if (group_size <= 1)
4980 : return false;
4981 17523 : if (!matches)
4982 4547 : matches = XALLOCAVEC (match_elt_t, group_size);
4983 17523 : unsigned tree_size = 0;
4984 17523 : slp_tree node = vect_build_slp_tree (loop_vinfo, scalar_stmts, matches, limit,
4985 : &tree_size, bst_map);
4986 17523 : if (!node)
4987 : return false;
4988 :
4989 : /* Create a new SLP instance. */
4990 2784 : slp_instance new_instance = XNEW (class _slp_instance);
4991 2784 : SLP_INSTANCE_TREE (new_instance) = node;
4992 2784 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
4993 2784 : SLP_INSTANCE_ROOT_STMTS (new_instance) = vNULL;
4994 2784 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = vNULL;
4995 2784 : SLP_INSTANCE_KIND (new_instance) = slp_inst_kind_reduc_group;
4996 2784 : new_instance->reduc_phis = NULL;
4997 2784 : new_instance->cost_vec = vNULL;
4998 2784 : new_instance->subgraph_entries = vNULL;
4999 :
5000 2784 : if (dump_enabled_p ())
5001 203 : dump_printf_loc (MSG_NOTE, vect_location,
5002 : "SLP size %u vs. limit %u.\n",
5003 : tree_size, max_tree_size);
5004 :
5005 2784 : loop_vinfo->slp_instances.safe_push (new_instance);
5006 :
5007 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5008 : the number of SLP lanes of the root in a few places.
5009 : Verify that assumption holds. */
5010 2784 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5011 : == group_size);
5012 :
5013 2784 : if (dump_enabled_p ())
5014 : {
5015 203 : dump_printf_loc (MSG_NOTE, vect_location,
5016 : "SLP discovery of size %d reduction group "
5017 : "succeeded\n", group_size);
5018 203 : dump_printf_loc (MSG_NOTE, vect_location,
5019 : "Final SLP tree for instance %p:\n",
5020 : (void *) new_instance);
5021 203 : vect_print_slp_graph (MSG_NOTE, vect_location,
5022 : SLP_INSTANCE_TREE (new_instance));
5023 : }
5024 :
5025 : return true;
5026 : }
5027 :
5028 : /* Analyze reductions in LOOP_VINFO and populate SLP instances
5029 : accordingly. Returns false if something fails. */
5030 :
5031 : static bool
5032 520454 : vect_analyze_slp_reductions (loop_vec_info loop_vinfo,
5033 : unsigned max_tree_size, unsigned *limit,
5034 : scalar_stmts_to_slp_tree_map_t *bst_map,
5035 : bool force_single_lane)
5036 : {
5037 520454 : if (loop_vinfo->reductions.is_empty ())
5038 : return true;
5039 :
5040 : /* Collect reduction statements we can combine into
5041 : a SLP reduction. */
5042 74884 : vec<stmt_vec_info> scalar_stmts;
5043 74884 : scalar_stmts.create (loop_vinfo->reductions.length ());
5044 331877 : for (auto next_info : loop_vinfo->reductions)
5045 : {
5046 107225 : next_info = vect_stmt_to_vectorize (next_info);
5047 107225 : if ((STMT_VINFO_RELEVANT_P (next_info)
5048 14 : || STMT_VINFO_LIVE_P (next_info))
5049 : /* ??? Make sure we didn't skip a conversion around a
5050 : reduction path. In that case we'd have to reverse
5051 : engineer that conversion stmt following the chain using
5052 : reduc_idx and from the PHI using reduc_def. */
5053 107211 : && (STMT_VINFO_DEF_TYPE (next_info) == vect_reduction_def
5054 107211 : || (STMT_VINFO_DEF_TYPE (next_info)
5055 : == vect_double_reduction_def)))
5056 : {
5057 : /* Do not discover SLP reductions combining lane-reducing
5058 : ops, that will fail later. */
5059 107211 : if (!force_single_lane
5060 107211 : && !lane_reducing_stmt_p (STMT_VINFO_STMT (next_info)))
5061 79588 : scalar_stmts.quick_push (next_info);
5062 : /* Do SLP discovery for single-lane reductions. */
5063 27623 : else if (! vect_analyze_slp_reduction (loop_vinfo, next_info,
5064 : max_tree_size, limit,
5065 : bst_map,
5066 : force_single_lane))
5067 : {
5068 0 : scalar_stmts.release ();
5069 0 : return false;
5070 : }
5071 : }
5072 : }
5073 :
5074 74884 : if (scalar_stmts.length () > 1)
5075 : {
5076 : /* Try to form a reduction group. */
5077 4675 : unsigned int group_size = scalar_stmts.length ();
5078 4675 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
5079 4675 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts, bst_map,
5080 : max_tree_size, limit, matches))
5081 1601 : return true;
5082 :
5083 : /* When analysis as a single SLP reduction group failed try to
5084 : form sub-groups by collecting matching lanes. Do not recurse
5085 : that on failure (to limit compile-time costs), but recurse
5086 : for the initial non-matching parts. Everything not covered
5087 : by a sub-group gets single-reduction treatment. */
5088 : /* ??? Improve this with the matches[] improvements. */
5089 3526 : vec<stmt_vec_info> cands = vNULL;
5090 11397 : while (matches[0] == 0)
5091 : {
5092 11266 : cands.truncate (0);
5093 11266 : cands.reserve (group_size, true);
5094 99737 : for (unsigned i = 0; i < group_size; ++i)
5095 77205 : if (matches[i] == 0)
5096 19596 : cands.quick_push (scalar_stmts[i]);
5097 :
5098 : /* Try to form a reduction group. */
5099 11266 : if (vect_analyze_slp_reduction_group (loop_vinfo, cands, bst_map,
5100 : max_tree_size, limit, NULL))
5101 1205 : cands = vNULL;
5102 : else
5103 : {
5104 : /* Do SLP discovery for single-lane reductions. */
5105 47319 : for (auto stmt_info : cands)
5106 17158 : if (! vect_analyze_slp_reduction (loop_vinfo,
5107 : vect_stmt_to_vectorize
5108 : (stmt_info),
5109 : max_tree_size, limit,
5110 : bst_map, force_single_lane))
5111 : {
5112 22 : scalar_stmts.release ();
5113 22 : cands.release ();
5114 22 : return false;
5115 : }
5116 : }
5117 : /* Remove the handled stmts from scalar_stmts and try again,
5118 : possibly repeating the above with updated matches[]. */
5119 11244 : unsigned j = 0;
5120 88387 : for (unsigned i = 0; i < group_size; ++i)
5121 77143 : if (matches[i] != 0)
5122 : {
5123 57582 : scalar_stmts[j] = scalar_stmts[i];
5124 57582 : ++j;
5125 : }
5126 11244 : scalar_stmts.truncate (j);
5127 11375 : group_size = scalar_stmts.length ();
5128 11244 : if (group_size <= 1)
5129 : break;
5130 8301 : if (vect_analyze_slp_reduction_group (loop_vinfo, scalar_stmts,
5131 : bst_map, max_tree_size, limit,
5132 : matches))
5133 : return true;
5134 : }
5135 : }
5136 : /* Do SLP discovery for single-lane reductions. */
5137 273987 : for (auto stmt_info : scalar_stmts)
5138 56012 : if (! vect_analyze_slp_reduction (loop_vinfo,
5139 : vect_stmt_to_vectorize (stmt_info),
5140 : max_tree_size, limit,
5141 : bst_map, force_single_lane))
5142 : {
5143 1874 : scalar_stmts.release ();
5144 1874 : return false;
5145 : }
5146 :
5147 71409 : scalar_stmts.release ();
5148 71409 : return true;
5149 : }
5150 :
5151 : /* Analyze an SLP instance starting from a group of grouped stores. Call
5152 : vect_build_slp_tree to build a tree of packed stmts if possible.
5153 : Return FALSE if it's impossible to SLP any stmt in the group. */
5154 :
5155 : static bool
5156 1106208 : vect_analyze_slp_instance (vec_info *vinfo,
5157 : scalar_stmts_to_slp_tree_map_t *bst_map,
5158 : stmt_vec_info stmt_info,
5159 : slp_instance_kind kind,
5160 : unsigned max_tree_size, unsigned *limit,
5161 : bool force_single_lane)
5162 : {
5163 1106208 : vec<stmt_vec_info> scalar_stmts;
5164 :
5165 1106208 : if (is_a <bb_vec_info> (vinfo))
5166 1075903 : vect_location = stmt_info->stmt;
5167 :
5168 1106208 : gcc_assert (kind == slp_inst_kind_store);
5169 :
5170 : /* Collect the stores and store them in scalar_stmts. */
5171 1106208 : scalar_stmts.create (DR_GROUP_SIZE (stmt_info));
5172 1106208 : stmt_vec_info next_info = stmt_info;
5173 5495262 : while (next_info)
5174 : {
5175 3282846 : scalar_stmts.quick_push (vect_stmt_to_vectorize (next_info));
5176 3282846 : next_info = DR_GROUP_NEXT_ELEMENT (next_info);
5177 : }
5178 :
5179 1106208 : vec<stmt_vec_info> root_stmt_infos = vNULL;
5180 1106208 : vec<tree> remain = vNULL;
5181 :
5182 : /* Build the tree for the SLP instance. */
5183 :
5184 : /* If there's no budget left bail out early. */
5185 1106208 : if (*limit == 0)
5186 : return false;
5187 :
5188 1106188 : if (dump_enabled_p ())
5189 : {
5190 4220 : dump_printf_loc (MSG_NOTE, vect_location,
5191 : "Starting SLP discovery for\n");
5192 24541 : for (unsigned i = 0; i < scalar_stmts.length (); ++i)
5193 40642 : dump_printf_loc (MSG_NOTE, vect_location,
5194 20321 : " %G", scalar_stmts[i]->stmt);
5195 : }
5196 :
5197 : /* Build the tree for the SLP instance. */
5198 1106188 : unsigned int group_size = scalar_stmts.length ();
5199 1106188 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_size);
5200 1106188 : unsigned tree_size = 0;
5201 1106188 : unsigned i;
5202 :
5203 1106188 : slp_tree node = NULL;
5204 1106188 : if (group_size > 1 && force_single_lane)
5205 : {
5206 1774 : matches[0] = 0;
5207 1774 : matches[1] = -1;
5208 : }
5209 : else
5210 1104414 : node = vect_build_slp_tree (vinfo, scalar_stmts, matches, limit,
5211 : &tree_size, bst_map);
5212 1106188 : if (node != NULL)
5213 : {
5214 : /* Create a new SLP instance. */
5215 699556 : slp_instance new_instance = XNEW (class _slp_instance);
5216 699556 : SLP_INSTANCE_TREE (new_instance) = node;
5217 699556 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5218 699556 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5219 699556 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5220 699556 : SLP_INSTANCE_KIND (new_instance) = kind;
5221 699556 : new_instance->reduc_phis = NULL;
5222 699556 : new_instance->cost_vec = vNULL;
5223 699556 : new_instance->subgraph_entries = vNULL;
5224 :
5225 699556 : if (dump_enabled_p ())
5226 3214 : dump_printf_loc (MSG_NOTE, vect_location,
5227 : "SLP size %u vs. limit %u.\n",
5228 : tree_size, max_tree_size);
5229 :
5230 699556 : vinfo->slp_instances.safe_push (new_instance);
5231 :
5232 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5233 : the number of SLP lanes of the root in a few places.
5234 : Verify that assumption holds. */
5235 699556 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5236 : == group_size);
5237 :
5238 699556 : if (dump_enabled_p ())
5239 : {
5240 3214 : dump_printf_loc (MSG_NOTE, vect_location,
5241 : "Final SLP tree for instance %p:\n",
5242 : (void *) new_instance);
5243 3214 : vect_print_slp_graph (MSG_NOTE, vect_location,
5244 : SLP_INSTANCE_TREE (new_instance));
5245 : }
5246 699556 : return true;
5247 : }
5248 : /* Failed to SLP. */
5249 :
5250 : /* Try to break the group up into pieces. */
5251 : /* ??? Improve this with the matches[] improvements. */
5252 406632 : if (*limit > 0 && kind == slp_inst_kind_store)
5253 : {
5254 : /* ??? We could delay all the actual splitting of store-groups
5255 : until after SLP discovery of the original group completed.
5256 : Then we can recurse to vect_build_slp_instance directly. */
5257 1013835 : for (i = 0; i < group_size; i++)
5258 1013835 : if (matches[i] != 0)
5259 : break;
5260 :
5261 : /* For basic block SLP, try to break the group up into multiples of
5262 : a vector size. */
5263 406631 : if (is_a <bb_vec_info> (vinfo)
5264 406631 : && (i > 1 && i < group_size))
5265 : {
5266 : /* Free the allocated memory. */
5267 134863 : scalar_stmts.release ();
5268 :
5269 134863 : tree scalar_type
5270 134863 : = TREE_TYPE (DR_REF (STMT_VINFO_DATA_REF (stmt_info)));
5271 269726 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type,
5272 134863 : 1 << floor_log2 (i));
5273 134863 : unsigned HOST_WIDE_INT const_nunits;
5274 134863 : if (vectype
5275 134863 : && TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits))
5276 : {
5277 : /* Split into two groups at the first vector boundary. */
5278 134863 : gcc_assert ((const_nunits & (const_nunits - 1)) == 0);
5279 134863 : unsigned group1_size = i & ~(const_nunits - 1);
5280 :
5281 134863 : if (dump_enabled_p ())
5282 83 : dump_printf_loc (MSG_NOTE, vect_location,
5283 : "Splitting SLP group at stmt %u\n", i);
5284 134863 : stmt_vec_info rest = vect_split_slp_store_group (stmt_info,
5285 : group1_size);
5286 134863 : bool res = vect_analyze_slp_instance (vinfo, bst_map, stmt_info,
5287 : kind, max_tree_size,
5288 : limit, false);
5289 : /* Split the rest at the failure point and possibly
5290 : re-analyze the remaining matching part if it has
5291 : at least two lanes. */
5292 134863 : if (group1_size < i
5293 5773 : && (i + 1 < group_size
5294 2563 : || i - group1_size > 1))
5295 : {
5296 3244 : stmt_vec_info rest2 = rest;
5297 3244 : rest = vect_split_slp_store_group (rest, i - group1_size);
5298 3244 : if (i - group1_size > 1)
5299 69 : res |= vect_analyze_slp_instance (vinfo, bst_map, rest2,
5300 : kind, max_tree_size,
5301 : limit, false);
5302 : }
5303 : /* Re-analyze the non-matching tail if it has at least
5304 : two lanes. */
5305 134863 : if (i + 1 < group_size)
5306 22428 : res |= vect_analyze_slp_instance (vinfo, bst_map,
5307 : rest, kind, max_tree_size,
5308 : limit, false);
5309 1106208 : return res;
5310 : }
5311 : }
5312 :
5313 : /* For loop vectorization split the RHS into arbitrary pieces of
5314 : size >= 1. */
5315 271768 : else if (is_a <loop_vec_info> (vinfo)
5316 271768 : && (group_size != 1 && i < group_size))
5317 : {
5318 8364 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
5319 28 : bool masked_p = call
5320 28 : && gimple_call_internal_p (call)
5321 28 : && internal_fn_mask_index (gimple_call_internal_fn (call)) != -1;
5322 : /* There are targets that cannot do even/odd interleaving schemes
5323 : so they absolutely need to use load/store-lanes. For now
5324 : force single-lane SLP for them - they would be happy with
5325 : uniform power-of-two lanes (but depending on element size),
5326 : but even if we can use 'i' as indicator we would need to
5327 : backtrack when later lanes fail to discover with the same
5328 : granularity. We cannot turn any of strided or scatter store
5329 : into store-lanes. */
5330 : /* ??? If this is not in sync with what get_load_store_type
5331 : later decides the SLP representation is not good for other
5332 : store vectorization methods. */
5333 8364 : bool want_store_lanes
5334 8364 : = (! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5335 8364 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5336 6245 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5337 6241 : && compare_step_with_zero (vinfo, stmt_info) > 0
5338 14496 : && vect_slp_prefer_store_lanes_p (vinfo, stmt_info, NULL_TREE,
5339 16728 : masked_p, group_size, i));
5340 8364 : if (want_store_lanes || force_single_lane)
5341 : i = 1;
5342 :
5343 : /* A fatal discovery fail doesn't always mean single-lane SLP
5344 : isn't a possibility, so try. */
5345 6590 : if (i == 0)
5346 : i = 1;
5347 :
5348 8364 : if (dump_enabled_p ())
5349 873 : dump_printf_loc (MSG_NOTE, vect_location,
5350 : "Splitting SLP group at stmt %u\n", i);
5351 :
5352 : /* Analyze the stored values and pinch them together with
5353 : a permute node so we can preserve the whole store group. */
5354 8364 : auto_vec<slp_tree> rhs_nodes;
5355 :
5356 8364 : unsigned int rhs_common_nlanes = 0;
5357 8364 : unsigned int start = 0, end = i;
5358 38324 : while (start < group_size)
5359 : {
5360 30075 : gcc_assert (end - start >= 1);
5361 30075 : vec<stmt_vec_info> substmts;
5362 30075 : substmts.create (end - start);
5363 126402 : for (unsigned j = start; j < end; ++j)
5364 66252 : substmts.quick_push (scalar_stmts[j]);
5365 30075 : node = vect_build_slp_tree (vinfo, substmts,
5366 : matches, limit, &tree_size, bst_map);
5367 30075 : if (node)
5368 : {
5369 24015 : rhs_nodes.safe_push (node);
5370 24015 : if (start == 0)
5371 8249 : rhs_common_nlanes = SLP_TREE_LANES (node);
5372 15766 : else if (rhs_common_nlanes != SLP_TREE_LANES (node))
5373 1419 : rhs_common_nlanes = 0;
5374 24015 : start = end;
5375 24015 : if (want_store_lanes || force_single_lane)
5376 5323 : end = start + 1;
5377 : else
5378 : end = group_size;
5379 : }
5380 : else
5381 : {
5382 6060 : substmts.release ();
5383 6060 : if (end - start == 1)
5384 : {
5385 : /* Single-lane discovery failed. Free resources. */
5386 115 : for (auto node : rhs_nodes)
5387 0 : vect_free_slp_tree (node);
5388 115 : scalar_stmts.release ();
5389 115 : if (dump_enabled_p ())
5390 14 : dump_printf_loc (MSG_NOTE, vect_location,
5391 : "SLP discovery failed\n");
5392 115 : return false;
5393 : }
5394 :
5395 : /* ??? It really happens that we soft-fail SLP
5396 : build at a mismatch but the matching part hard-fails
5397 : later. As we know we arrived here with a group
5398 : larger than one try a group of size one! */
5399 5945 : if (matches[0] != 0)
5400 12 : end = start + 1;
5401 : else
5402 13037 : for (unsigned j = start; j < end; j++)
5403 13037 : if (matches[j - start] != 0)
5404 : {
5405 : end = j;
5406 : break;
5407 : }
5408 : }
5409 : }
5410 :
5411 : /* Now re-assess whether we want store lanes in case the
5412 : discovery ended up producing all single-lane RHSs. */
5413 8249 : if (! want_store_lanes
5414 8249 : && rhs_common_nlanes == 1
5415 7163 : && ! STMT_VINFO_GATHER_SCATTER_P (stmt_info)
5416 7163 : && ! STMT_VINFO_STRIDED_P (stmt_info)
5417 5415 : && ! STMT_VINFO_SLP_VECT_ONLY (stmt_info)
5418 5412 : && compare_step_with_zero (vinfo, stmt_info) > 0
5419 13570 : && (vect_store_lanes_supported (SLP_TREE_VECTYPE (rhs_nodes[0]),
5420 : group_size, masked_p)
5421 : != IFN_LAST))
5422 : want_store_lanes = true;
5423 :
5424 : /* Now we assume we can build the root SLP node from all stores. */
5425 8249 : if (want_store_lanes)
5426 : {
5427 : /* For store-lanes feed the store node with all RHS nodes
5428 : in order. */
5429 0 : node = vect_create_new_slp_node (scalar_stmts,
5430 0 : SLP_TREE_CHILDREN
5431 : (rhs_nodes[0]).length ());
5432 0 : SLP_TREE_VECTYPE (node) = SLP_TREE_VECTYPE (rhs_nodes[0]);
5433 0 : node->ldst_lanes = true;
5434 0 : SLP_TREE_CHILDREN (node)
5435 0 : .reserve_exact (SLP_TREE_CHILDREN (rhs_nodes[0]).length ()
5436 0 : + rhs_nodes.length () - 1);
5437 : /* First store value and possibly mask. */
5438 0 : SLP_TREE_CHILDREN (node)
5439 0 : .splice (SLP_TREE_CHILDREN (rhs_nodes[0]));
5440 : /* Rest of the store values. All mask nodes are the same,
5441 : this should be guaranteed by dataref group discovery. */
5442 0 : for (unsigned j = 1; j < rhs_nodes.length (); ++j)
5443 0 : SLP_TREE_CHILDREN (node)
5444 0 : .quick_push (SLP_TREE_CHILDREN (rhs_nodes[j])[0]);
5445 0 : for (slp_tree child : SLP_TREE_CHILDREN (node))
5446 0 : child->refcnt++;
5447 : }
5448 : else
5449 8249 : node = vect_build_slp_store_interleaving (rhs_nodes, scalar_stmts);
5450 :
5451 32264 : while (!rhs_nodes.is_empty ())
5452 24015 : vect_free_slp_tree (rhs_nodes.pop ());
5453 :
5454 : /* Create a new SLP instance. */
5455 8249 : slp_instance new_instance = XNEW (class _slp_instance);
5456 8249 : SLP_INSTANCE_TREE (new_instance) = node;
5457 8249 : SLP_INSTANCE_LOADS (new_instance) = vNULL;
5458 8249 : SLP_INSTANCE_ROOT_STMTS (new_instance) = root_stmt_infos;
5459 8249 : SLP_INSTANCE_REMAIN_DEFS (new_instance) = remain;
5460 8249 : SLP_INSTANCE_KIND (new_instance) = kind;
5461 8249 : new_instance->reduc_phis = NULL;
5462 8249 : new_instance->cost_vec = vNULL;
5463 8249 : new_instance->subgraph_entries = vNULL;
5464 :
5465 8249 : if (dump_enabled_p ())
5466 859 : dump_printf_loc (MSG_NOTE, vect_location,
5467 : "SLP size %u vs. limit %u.\n",
5468 : tree_size, max_tree_size);
5469 :
5470 8249 : vinfo->slp_instances.safe_push (new_instance);
5471 :
5472 : /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
5473 : the number of SLP lanes of the root in a few places.
5474 : Verify that assumption holds. */
5475 8249 : gcc_assert (SLP_TREE_LANES (SLP_INSTANCE_TREE (new_instance))
5476 : == group_size);
5477 :
5478 8249 : if (dump_enabled_p ())
5479 : {
5480 859 : dump_printf_loc (MSG_NOTE, vect_location,
5481 : "Final SLP tree for instance %p:\n",
5482 : (void *) new_instance);
5483 859 : vect_print_slp_graph (MSG_NOTE, vect_location,
5484 : SLP_INSTANCE_TREE (new_instance));
5485 : }
5486 : return true;
5487 8364 : }
5488 : else
5489 : /* Free the allocated memory. */
5490 263404 : scalar_stmts.release ();
5491 :
5492 : /* Even though the first vector did not all match, we might be able to SLP
5493 : (some) of the remainder. FORNOW ignore this possibility. */
5494 : }
5495 : else
5496 : /* Free the allocated memory. */
5497 1 : scalar_stmts.release ();
5498 :
5499 : /* Failed to SLP. */
5500 263405 : if (dump_enabled_p ())
5501 50 : dump_printf_loc (MSG_NOTE, vect_location, "SLP discovery failed\n");
5502 : return false;
5503 : }
5504 :
5505 : /* qsort comparator ordering SLP load nodes. */
5506 :
5507 : static int
5508 2693607 : vllp_cmp (const void *a_, const void *b_)
5509 : {
5510 2693607 : const slp_tree a = *(const slp_tree *)a_;
5511 2693607 : const slp_tree b = *(const slp_tree *)b_;
5512 2693607 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (a)[0];
5513 2693607 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (b)[0];
5514 2693607 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5515 1547612 : && STMT_VINFO_GROUPED_ACCESS (b0)
5516 4179211 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5517 : {
5518 : /* Same group, order after lanes used. */
5519 349662 : if (SLP_TREE_LANES (a) < SLP_TREE_LANES (b))
5520 : return 1;
5521 340689 : else if (SLP_TREE_LANES (a) > SLP_TREE_LANES (b))
5522 : return -1;
5523 : else
5524 : {
5525 : /* Try to order loads using the same lanes together, breaking
5526 : the tie with the lane number that first differs. */
5527 330952 : if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5528 330952 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5529 : return 0;
5530 330952 : else if (SLP_TREE_LOAD_PERMUTATION (a).exists ()
5531 330952 : && !SLP_TREE_LOAD_PERMUTATION (b).exists ())
5532 : return 1;
5533 326885 : else if (!SLP_TREE_LOAD_PERMUTATION (a).exists ()
5534 326885 : && SLP_TREE_LOAD_PERMUTATION (b).exists ())
5535 : return -1;
5536 : else
5537 : {
5538 319395 : for (unsigned i = 0; i < SLP_TREE_LANES (a); ++i)
5539 319395 : if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5540 319395 : != SLP_TREE_LOAD_PERMUTATION (b)[i])
5541 : {
5542 : /* In-order lane first, that's what the above case for
5543 : no permutation does. */
5544 318083 : if (SLP_TREE_LOAD_PERMUTATION (a)[i] == i)
5545 : return -1;
5546 195407 : else if (SLP_TREE_LOAD_PERMUTATION (b)[i] == i)
5547 : return 1;
5548 103150 : else if (SLP_TREE_LOAD_PERMUTATION (a)[i]
5549 103150 : < SLP_TREE_LOAD_PERMUTATION (b)[i])
5550 : return -1;
5551 : else
5552 38638 : return 1;
5553 : }
5554 : return 0;
5555 : }
5556 : }
5557 : }
5558 : else /* Different groups or non-groups. */
5559 : {
5560 : /* Order groups as their first element to keep them together. */
5561 2343945 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5562 2343945 : a0 = DR_GROUP_FIRST_ELEMENT (a0);
5563 2343945 : if (STMT_VINFO_GROUPED_ACCESS (b0))
5564 2343945 : b0 = DR_GROUP_FIRST_ELEMENT (b0);
5565 2343945 : if (a0 == b0)
5566 : return 0;
5567 : /* Tie using UID. */
5568 2343825 : else if (gimple_uid (STMT_VINFO_STMT (a0))
5569 2343825 : < gimple_uid (STMT_VINFO_STMT (b0)))
5570 : return -1;
5571 : else
5572 : {
5573 1041580 : gcc_assert (gimple_uid (STMT_VINFO_STMT (a0))
5574 : != gimple_uid (STMT_VINFO_STMT (b0)));
5575 : return 1;
5576 : }
5577 : }
5578 : }
5579 :
5580 : /* Return whether if the load permutation of NODE is consecutive starting
5581 : with value START_VAL in the first element. If START_VAL is not given
5582 : the first element's value is used. */
5583 :
5584 : bool
5585 653007 : vect_load_perm_consecutive_p (slp_tree node, unsigned start_val)
5586 : {
5587 653007 : load_permutation_t perm = SLP_TREE_LOAD_PERMUTATION (node);
5588 :
5589 653007 : if (!perm.exists () || !perm.length ())
5590 : return false;
5591 :
5592 653007 : if (start_val == UINT_MAX)
5593 79870 : start_val = perm[0];
5594 :
5595 1289936 : for (unsigned int i = 0; i < perm.length (); i++)
5596 660627 : if (perm[i] != start_val + (unsigned int) i)
5597 : return false;
5598 :
5599 : return true;
5600 : }
5601 :
5602 : /* Process the set of LOADS that are all from the same dataref group. */
5603 :
5604 : static void
5605 162421 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5606 : scalar_stmts_to_slp_tree_map_t *bst_map,
5607 : const array_slice<slp_tree> &loads,
5608 : bool force_single_lane)
5609 : {
5610 : /* We at this point want to lower without a fixed VF or vector
5611 : size in mind which means we cannot actually compute whether we
5612 : need three or more vectors for a load permutation yet. So always
5613 : lower. */
5614 162421 : stmt_vec_info first
5615 162421 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (loads[0])[0]);
5616 162421 : unsigned group_lanes = DR_GROUP_SIZE (first);
5617 :
5618 : /* Verify if all load permutations can be implemented with a suitably
5619 : large element load-lanes operation. */
5620 162421 : unsigned ld_lanes_lanes = SLP_TREE_LANES (loads[0]);
5621 162421 : if (STMT_VINFO_STRIDED_P (first)
5622 159943 : || compare_step_with_zero (loop_vinfo, first) <= 0
5623 157290 : || exact_log2 (ld_lanes_lanes) == -1
5624 : /* ??? For now only support the single-lane case as there is
5625 : missing support on the store-lane side and code generation
5626 : isn't up to the task yet. */
5627 154459 : || ld_lanes_lanes != 1
5628 305818 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (loads[0]),
5629 : group_lanes / ld_lanes_lanes,
5630 : false) == IFN_LAST)
5631 : ld_lanes_lanes = 0;
5632 : else
5633 : /* Verify the loads access the same number of lanes aligned to
5634 : ld_lanes_lanes. */
5635 0 : for (slp_tree load : loads)
5636 : {
5637 0 : if (SLP_TREE_LANES (load) != ld_lanes_lanes)
5638 : {
5639 : ld_lanes_lanes = 0;
5640 : break;
5641 : }
5642 0 : unsigned first = SLP_TREE_LOAD_PERMUTATION (load)[0];
5643 0 : if (first % ld_lanes_lanes != 0)
5644 : {
5645 : ld_lanes_lanes = 0;
5646 : break;
5647 : }
5648 0 : if (!vect_load_perm_consecutive_p (load))
5649 : {
5650 : ld_lanes_lanes = 0;
5651 : break;
5652 : }
5653 : }
5654 :
5655 : /* Only a power-of-two number of lanes matches interleaving with N levels.
5656 : ??? An even number of lanes could be reduced to 1<<ceil_log2(N)-1 lanes
5657 : at each step. */
5658 263383 : if (ld_lanes_lanes == 0 && exact_log2 (group_lanes) == -1 && group_lanes != 3)
5659 : return;
5660 :
5661 267046 : for (slp_tree load : loads)
5662 : {
5663 : /* Leave masked or gather loads alone for now. */
5664 188465 : if (!SLP_TREE_CHILDREN (load).is_empty ())
5665 61166 : continue;
5666 :
5667 : /* For single-element interleaving spanning multiple vectors avoid
5668 : lowering, we want to use VMAT_ELEMENTWISE later. */
5669 188459 : if (ld_lanes_lanes == 0
5670 188459 : && SLP_TREE_LANES (load) == 1
5671 168935 : && !DR_GROUP_NEXT_ELEMENT (first)
5672 268462 : && maybe_gt (group_lanes,
5673 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (load))))
5674 51490 : return;
5675 :
5676 : /* We want to pattern-match special cases here and keep those
5677 : alone. Candidates are splats and load-lane. */
5678 :
5679 : /* We need to lower only loads of less than half of the groups
5680 : lanes, including duplicate lanes. Note this leaves nodes
5681 : with a non-1:1 load permutation around instead of canonicalizing
5682 : those into a load and a permute node. Removing this early
5683 : check would do such canonicalization. */
5684 136969 : if (SLP_TREE_LANES (load) >= (group_lanes + 1) / 2
5685 57588 : && ld_lanes_lanes == 0)
5686 57588 : continue;
5687 :
5688 : /* Build the permute to get the original load permutation order. */
5689 79381 : bool contiguous = vect_load_perm_consecutive_p (load);
5690 79381 : lane_permutation_t final_perm;
5691 79381 : final_perm.create (SLP_TREE_LANES (load));
5692 239087 : for (unsigned i = 0; i < SLP_TREE_LANES (load); ++i)
5693 160650 : final_perm.quick_push (
5694 80325 : std::make_pair (0, SLP_TREE_LOAD_PERMUTATION (load)[i]));
5695 :
5696 : /* When the load permutation accesses a contiguous unpermuted,
5697 : power-of-two aligned and sized chunk leave the load alone.
5698 : We can likely (re-)load it more efficiently rather than
5699 : extracting it from the larger load.
5700 : ??? Long-term some of the lowering should move to where
5701 : the vector types involved are fixed. */
5702 82953 : if (!force_single_lane
5703 79381 : && ld_lanes_lanes == 0
5704 53518 : && contiguous
5705 53258 : && (SLP_TREE_LANES (load) > 1 || loads.size () == 1)
5706 6586 : && pow2p_hwi (SLP_TREE_LANES (load))
5707 6550 : && pow2p_hwi (group_lanes)
5708 3572 : && SLP_TREE_LOAD_PERMUTATION (load)[0] % SLP_TREE_LANES (load) == 0
5709 82953 : && group_lanes % SLP_TREE_LANES (load) == 0)
5710 : {
5711 3572 : final_perm.release ();
5712 3572 : continue;
5713 : }
5714 :
5715 : /* First build (and possibly re-use) a load node for the
5716 : unpermuted group. Gaps in the middle and on the end are
5717 : represented with NULL stmts. */
5718 75809 : vec<stmt_vec_info> stmts;
5719 75809 : stmts.create (group_lanes);
5720 270605 : for (stmt_vec_info s = first; s; s = DR_GROUP_NEXT_ELEMENT (s))
5721 : {
5722 194796 : if (s != first)
5723 124042 : for (unsigned i = 1; i < DR_GROUP_GAP (s); ++i)
5724 5055 : stmts.quick_push (NULL);
5725 194796 : stmts.quick_push (s);
5726 : }
5727 139721 : for (unsigned i = 0; i < DR_GROUP_GAP (first); ++i)
5728 63912 : stmts.quick_push (NULL);
5729 75809 : match_elt_t *matches = XALLOCAVEC (match_elt_t, group_lanes);
5730 75809 : unsigned limit = 1;
5731 75809 : unsigned tree_size = 0;
5732 75809 : slp_tree l0 = vect_build_slp_tree (loop_vinfo, stmts, matches, &limit,
5733 75809 : &tree_size, bst_map);
5734 75809 : gcc_assert (!SLP_TREE_LOAD_PERMUTATION (l0).exists ());
5735 :
5736 75809 : if (ld_lanes_lanes != 0)
5737 : {
5738 : /* ??? If this is not in sync with what get_load_store_type
5739 : later decides the SLP representation is not good for other
5740 : store vectorization methods. */
5741 0 : l0->ldst_lanes = true;
5742 0 : load->ldst_lanes = true;
5743 : }
5744 :
5745 236089 : while (1)
5746 : {
5747 155949 : unsigned group_lanes = SLP_TREE_LANES (l0);
5748 155949 : if (ld_lanes_lanes != 0
5749 155949 : || SLP_TREE_LANES (load) >= (group_lanes + 1) / 2)
5750 : break;
5751 :
5752 : /* Try to lower by reducing the group to half its size using an
5753 : interleaving scheme. For this try to compute whether all
5754 : elements needed for this load are in even or odd elements of
5755 : an even/odd decomposition with N consecutive elements.
5756 : Thus { e, e, o, o, e, e, o, o } would be an even/odd decomposition
5757 : with N == 2. */
5758 : /* ??? Only an even number of lanes can be handed this way, but the
5759 : fallback below could work for any number. We have to make sure
5760 : to round up in that case. */
5761 80140 : gcc_assert ((group_lanes & 1) == 0 || group_lanes == 3);
5762 12147 : unsigned even = 0, odd = 0;
5763 12147 : if ((group_lanes & 1) == 0)
5764 : {
5765 12147 : even = (1 << ceil_log2 (group_lanes)) - 1;
5766 12147 : odd = even;
5767 49337 : for (auto l : final_perm)
5768 : {
5769 12896 : even &= ~l.second;
5770 12896 : odd &= l.second;
5771 : }
5772 : }
5773 :
5774 : /* Now build an even or odd extraction from the unpermuted load. */
5775 80140 : lane_permutation_t perm;
5776 80140 : perm.create ((group_lanes + 1) / 2);
5777 80140 : unsigned even_level = even ? 1 << ctz_hwi (even) : 0;
5778 80140 : unsigned odd_level = odd ? 1 << ctz_hwi (odd) : 0;
5779 80140 : if (even_level
5780 11169 : && group_lanes % (2 * even_level) == 0
5781 : /* ??? When code generating permutes we do not try to pun
5782 : to larger component modes so level != 1 isn't a natural
5783 : even/odd extract. Prefer one if possible. */
5784 11169 : && (even_level == 1 || !odd_level || odd_level != 1))
5785 : {
5786 : /* { 0, 1, ... 4, 5 ..., } */
5787 39506 : for (unsigned i = 0; i < group_lanes / 2 / even_level; ++i)
5788 62345 : for (unsigned j = 0; j < even_level; ++j)
5789 31368 : perm.quick_push (std::make_pair (0, 2 * i * even_level + j));
5790 : }
5791 68971 : else if (odd_level)
5792 : {
5793 : /* { ..., 2, 3, ... 6, 7 } */
5794 3588 : gcc_assert (group_lanes % (2 * odd_level) == 0);
5795 15474 : for (unsigned i = 0; i < group_lanes / 2 / odd_level; ++i)
5796 23826 : for (unsigned j = 0; j < odd_level; ++j)
5797 11940 : perm.quick_push
5798 11940 : (std::make_pair (0, (2 * i + 1) * odd_level + j));
5799 : }
5800 : else
5801 : {
5802 : /* As fallback extract all used lanes and fill to half the
5803 : group size by repeating the last element.
5804 : ??? This is quite a bad strathegy for re-use - we could
5805 : brute force our way to find more optimal filling lanes to
5806 : maximize re-use when looking at all loads from the group. */
5807 68023 : auto_bitmap l;
5808 272148 : for (auto p : final_perm)
5809 68079 : bitmap_set_bit (l, p.second);
5810 68023 : unsigned i = 0;
5811 68023 : bitmap_iterator bi;
5812 136102 : EXECUTE_IF_SET_IN_BITMAP (l, 0, i, bi)
5813 68079 : perm.quick_push (std::make_pair (0, i));
5814 272244 : while (perm.length () < (group_lanes + 1) / 2)
5815 68099 : perm.quick_push (perm.last ());
5816 68023 : }
5817 :
5818 : /* Update final_perm with the intermediate permute. */
5819 161029 : for (unsigned i = 0; i < final_perm.length (); ++i)
5820 : {
5821 80889 : unsigned l = final_perm[i].second;
5822 80889 : unsigned j;
5823 90463 : for (j = 0; j < perm.length (); ++j)
5824 90463 : if (perm[j].second == l)
5825 : {
5826 80889 : final_perm[i].second = j;
5827 80889 : break;
5828 : }
5829 80889 : gcc_assert (j < perm.length ());
5830 : }
5831 :
5832 : /* And create scalar stmts. */
5833 80140 : vec<stmt_vec_info> perm_stmts;
5834 80140 : perm_stmts.create (perm.length ());
5835 339766 : for (unsigned i = 0; i < perm.length (); ++i)
5836 179486 : perm_stmts.quick_push (SLP_TREE_SCALAR_STMTS (l0)[perm[i].second]);
5837 :
5838 80140 : slp_tree p = vect_create_new_slp_node (1, VEC_PERM_EXPR);
5839 80140 : SLP_TREE_CHILDREN (p).quick_push (l0);
5840 80140 : SLP_TREE_LANE_PERMUTATION (p) = perm;
5841 80140 : SLP_TREE_VECTYPE (p) = SLP_TREE_VECTYPE (load);
5842 80140 : SLP_TREE_LANES (p) = perm.length ();
5843 : /* ??? As we have scalar stmts for this intermediate permute we
5844 : could CSE it via bst_map but we do not want to pick up
5845 : another SLP node with a load permutation. We instead should
5846 : have a "local" CSE map here. */
5847 80140 : SLP_TREE_SCALAR_STMTS (p) = perm_stmts;
5848 :
5849 : /* We now have a node for (group_lanes + 1) / 2 lanes. */
5850 80140 : l0 = p;
5851 80140 : }
5852 :
5853 : /* And finally from the ordered reduction node create the
5854 : permute to shuffle the lanes into the original load-permutation
5855 : order. We replace the original load node with this. */
5856 75809 : SLP_TREE_CODE (load) = VEC_PERM_EXPR;
5857 75809 : SLP_TREE_LOAD_PERMUTATION (load).release ();
5858 75809 : SLP_TREE_LANE_PERMUTATION (load) = final_perm;
5859 75809 : SLP_TREE_CHILDREN (load).create (1);
5860 75809 : SLP_TREE_CHILDREN (load).quick_push (l0);
5861 75809 : SLP_TREE_REPRESENTATIVE (load) = NULL;
5862 : }
5863 : }
5864 :
5865 : /* Transform SLP loads in the SLP graph created by SLP discovery to
5866 : group loads from the same group and lower load permutations that
5867 : are unlikely to be supported into a series of permutes.
5868 : In the degenerate case of having only single-lane SLP instances
5869 : this should result in a series of permute nodes emulating an
5870 : interleaving scheme. */
5871 :
5872 : static void
5873 505784 : vect_lower_load_permutations (loop_vec_info loop_vinfo,
5874 : scalar_stmts_to_slp_tree_map_t *bst_map,
5875 : bool force_single_lane)
5876 : {
5877 : /* Gather and sort loads across all instances. */
5878 505784 : hash_set<slp_tree> visited;
5879 505784 : auto_vec<slp_tree> loads;
5880 2322411 : for (auto inst : loop_vinfo->slp_instances)
5881 807127 : vect_gather_slp_loads (loads, SLP_INSTANCE_TREE (inst), visited);
5882 505784 : if (loads.is_empty ())
5883 94073 : return;
5884 411711 : loads.qsort (vllp_cmp);
5885 :
5886 : /* Now process each dataref group separately. */
5887 411711 : unsigned firsti = 0;
5888 1168895 : for (unsigned i = 1; i < loads.length (); ++i)
5889 : {
5890 345473 : slp_tree first = loads[firsti];
5891 345473 : slp_tree next = loads[i];
5892 345473 : stmt_vec_info a0 = SLP_TREE_SCALAR_STMTS (first)[0];
5893 345473 : stmt_vec_info b0 = SLP_TREE_SCALAR_STMTS (next)[0];
5894 345473 : if (STMT_VINFO_GROUPED_ACCESS (a0)
5895 158923 : && STMT_VINFO_GROUPED_ACCESS (b0)
5896 491194 : && DR_GROUP_FIRST_ELEMENT (a0) == DR_GROUP_FIRST_ELEMENT (b0))
5897 63583 : continue;
5898 : /* Now we have one or multiple SLP loads of the same group from
5899 : firsti to i - 1. */
5900 281890 : if (STMT_VINFO_GROUPED_ACCESS (a0))
5901 95340 : vect_lower_load_permutations (loop_vinfo, bst_map,
5902 95340 : make_array_slice (&loads[firsti],
5903 : i - firsti),
5904 : force_single_lane);
5905 : firsti = i;
5906 : }
5907 823422 : if (firsti < loads.length ()
5908 823422 : && STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (loads[firsti])[0]))
5909 67081 : vect_lower_load_permutations (loop_vinfo, bst_map,
5910 67081 : make_array_slice (&loads[firsti],
5911 67081 : loads.length () - firsti),
5912 : force_single_lane);
5913 505784 : }
5914 :
5915 : /* Check if there are stmts in the loop can be vectorized using SLP. Build SLP
5916 : trees of packed scalar stmts if SLP is possible. */
5917 :
5918 : opt_result
5919 1203111 : vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size,
5920 : bool force_single_lane)
5921 : {
5922 1203111 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5923 1203111 : unsigned int i;
5924 1203111 : stmt_vec_info first_element;
5925 1203111 : slp_instance instance;
5926 :
5927 1203111 : DUMP_VECT_SCOPE ("vect_analyze_slp");
5928 :
5929 1203111 : unsigned limit = max_tree_size;
5930 :
5931 1203111 : scalar_stmts_to_slp_tree_map_t *bst_map
5932 1203111 : = new scalar_stmts_to_slp_tree_map_t ();
5933 :
5934 : /* Find SLP sequences starting from groups of grouped stores. */
5935 3354947 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
5936 948848 : if (! vect_analyze_slp_instance (vinfo, bst_map, first_element,
5937 : slp_inst_kind_store, max_tree_size, &limit,
5938 : force_single_lane)
5939 948848 : && loop_vinfo)
5940 : {
5941 123 : release_scalar_stmts_to_slp_tree_map (bst_map);
5942 123 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
5943 : }
5944 :
5945 : /* For loops also start SLP discovery from non-grouped stores. */
5946 1202988 : if (loop_vinfo)
5947 : {
5948 : data_reference_p dr;
5949 1701584 : FOR_EACH_VEC_ELT (vinfo->shared->datarefs, i, dr)
5950 1181130 : if (DR_IS_WRITE (dr))
5951 : {
5952 381540 : stmt_vec_info stmt_info = vinfo->lookup_dr (dr)->stmt;
5953 : /* Grouped stores are already handled above. */
5954 381540 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
5955 103592 : continue;
5956 277948 : vec<stmt_vec_info> stmts;
5957 277948 : vec<stmt_vec_info> roots = vNULL;
5958 277948 : vec<tree> remain = vNULL;
5959 277948 : stmts.create (1);
5960 277948 : stmts.quick_push (stmt_info);
5961 277948 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5962 : stmts, roots, remain, max_tree_size,
5963 : &limit, bst_map, force_single_lane))
5964 : {
5965 3646 : release_scalar_stmts_to_slp_tree_map (bst_map);
5966 3646 : return opt_result::failure_at (vect_location,
5967 : "SLP build failed.\n");
5968 : }
5969 : }
5970 :
5971 : stmt_vec_info stmt_info;
5972 520494 : FOR_EACH_VEC_ELT (LOOP_VINFO_ALTERNATE_DEFS (loop_vinfo), i, stmt_info)
5973 : {
5974 20 : vec<stmt_vec_info> stmts;
5975 20 : vec<stmt_vec_info> roots = vNULL;
5976 20 : vec<tree> remain = vNULL;
5977 20 : stmts.create (1);
5978 20 : stmts.quick_push (stmt_info);
5979 20 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_store,
5980 : stmts, roots, remain, max_tree_size,
5981 : &limit, bst_map, force_single_lane))
5982 : {
5983 0 : release_scalar_stmts_to_slp_tree_map (bst_map);
5984 0 : return opt_result::failure_at (vect_location,
5985 : "SLP build failed.\n");
5986 : }
5987 : }
5988 : }
5989 :
5990 1199342 : if (bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo))
5991 : {
5992 2051714 : for (unsigned i = 0; i < bb_vinfo->roots.length (); ++i)
5993 : {
5994 1372826 : vect_location = bb_vinfo->roots[i].roots[0]->stmt;
5995 : /* Apply patterns. */
5996 4490403 : for (unsigned j = 0; j < bb_vinfo->roots[i].stmts.length (); ++j)
5997 6235154 : bb_vinfo->roots[i].stmts[j]
5998 3226044 : = vect_stmt_to_vectorize (bb_vinfo->roots[i].stmts[j]);
5999 1372826 : if (vect_build_slp_instance (bb_vinfo, bb_vinfo->roots[i].kind,
6000 1372826 : bb_vinfo->roots[i].stmts,
6001 1372826 : bb_vinfo->roots[i].roots,
6002 1372826 : bb_vinfo->roots[i].remain,
6003 : max_tree_size, &limit, bst_map, false))
6004 : {
6005 156798 : bb_vinfo->roots[i].roots = vNULL;
6006 156798 : bb_vinfo->roots[i].remain = vNULL;
6007 : }
6008 1372826 : bb_vinfo->roots[i].stmts = vNULL;
6009 : }
6010 : }
6011 :
6012 1199342 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6013 : {
6014 : /* Find SLP sequences starting from groups of reductions. */
6015 520454 : if (!vect_analyze_slp_reductions (loop_vinfo, max_tree_size, &limit,
6016 : bst_map, force_single_lane))
6017 : {
6018 1896 : release_scalar_stmts_to_slp_tree_map (bst_map);
6019 1896 : return opt_result::failure_at (vect_location, "SLP build failed.\n");
6020 : }
6021 :
6022 : /* Make sure to vectorize only-live stmts, usually inductions. */
6023 2337542 : for (edge e : get_loop_exit_edges (LOOP_VINFO_LOOP (loop_vinfo)))
6024 1512929 : for (auto gsi = gsi_start_phis (e->dest); !gsi_end_p (gsi);
6025 720938 : gsi_next (&gsi))
6026 : {
6027 731061 : gphi *lc_phi = *gsi;
6028 731061 : tree def = gimple_phi_arg_def_from_edge (lc_phi, e);
6029 731061 : stmt_vec_info stmt_info;
6030 731061 : if (TREE_CODE (def) == SSA_NAME
6031 614037 : && !virtual_operand_p (def)
6032 317895 : && (stmt_info = loop_vinfo->lookup_def (def))
6033 285354 : && ((stmt_info = vect_stmt_to_vectorize (stmt_info)), true)
6034 285354 : && STMT_VINFO_RELEVANT (stmt_info) == vect_used_only_live
6035 222015 : && STMT_VINFO_LIVE_P (stmt_info)
6036 222015 : && !VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))
6037 848427 : && STMT_VINFO_REDUC_IDX (stmt_info) == -1)
6038 : {
6039 117278 : vec<stmt_vec_info> stmts;
6040 117278 : vec<stmt_vec_info> roots = vNULL;
6041 117278 : vec<tree> remain = vNULL;
6042 117278 : stmts.create (1);
6043 117278 : stmts.quick_push (vect_stmt_to_vectorize (stmt_info));
6044 117278 : if (! vect_build_slp_instance (vinfo,
6045 : slp_inst_kind_reduc_group,
6046 : stmts, roots, remain,
6047 : max_tree_size, &limit,
6048 : bst_map, force_single_lane))
6049 : {
6050 10123 : release_scalar_stmts_to_slp_tree_map (bst_map);
6051 10123 : return opt_result::failure_at (vect_location,
6052 : "SLP build failed.\n");
6053 : }
6054 : }
6055 10123 : }
6056 :
6057 : /* Find SLP sequences starting from gconds. */
6058 1277190 : for (auto cond : LOOP_VINFO_LOOP_CONDS (loop_vinfo))
6059 : {
6060 298594 : auto cond_info = loop_vinfo->lookup_stmt (cond);
6061 :
6062 298594 : cond_info = vect_stmt_to_vectorize (cond_info);
6063 298594 : vec<stmt_vec_info> roots = vNULL;
6064 298594 : roots.safe_push (cond_info);
6065 298594 : gimple *stmt = STMT_VINFO_STMT (cond_info);
6066 298594 : tree args0 = gimple_cond_lhs (stmt);
6067 298594 : tree args1 = gimple_cond_rhs (stmt);
6068 :
6069 : /* These should be enforced by cond lowering, but if it failed
6070 : bail. */
6071 298594 : if (gimple_cond_code (stmt) != NE_EXPR
6072 297451 : || TREE_TYPE (args0) != boolean_type_node
6073 595446 : || !integer_zerop (args1))
6074 : {
6075 1742 : roots.release ();
6076 1742 : release_scalar_stmts_to_slp_tree_map (bst_map);
6077 1742 : return opt_result::failure_at (vect_location,
6078 : "SLP build failed.\n");
6079 : }
6080 :
6081 : /* An argument without a loop def will be codegened from vectorizing the
6082 : root gcond itself. As such we don't need to try to build an SLP tree
6083 : from them. It's highly likely that the resulting SLP tree here if both
6084 : arguments have a def will be incompatible, but we rely on it being split
6085 : later on. */
6086 296852 : auto varg = loop_vinfo->lookup_def (args0);
6087 296852 : vec<stmt_vec_info> stmts;
6088 296852 : vec<tree> remain = vNULL;
6089 296852 : stmts.create (1);
6090 296852 : stmts.quick_push (vect_stmt_to_vectorize (varg));
6091 :
6092 296852 : if (! vect_build_slp_instance (vinfo, slp_inst_kind_gcond,
6093 : stmts, roots, remain,
6094 : max_tree_size, &limit,
6095 : bst_map, force_single_lane))
6096 : {
6097 909 : roots.release ();
6098 909 : release_scalar_stmts_to_slp_tree_map (bst_map);
6099 909 : return opt_result::failure_at (vect_location,
6100 : "SLP build failed.\n");
6101 : }
6102 : }
6103 : }
6104 :
6105 1184672 : hash_set<slp_tree> visited_patterns;
6106 1184672 : slp_tree_to_load_perm_map_t perm_cache;
6107 1184672 : slp_compat_nodes_map_t compat_cache;
6108 :
6109 : /* See if any patterns can be found in the SLP tree. */
6110 1184672 : bool pattern_found = false;
6111 4010892 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6112 1641548 : pattern_found |= vect_match_slp_patterns (instance, vinfo,
6113 : &visited_patterns, &perm_cache,
6114 : &compat_cache);
6115 :
6116 : /* If any were found optimize permutations of loads. */
6117 1184672 : if (pattern_found)
6118 : {
6119 267 : hash_map<slp_tree, slp_tree> load_map;
6120 4981 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6121 : {
6122 4180 : slp_tree root = SLP_INSTANCE_TREE (instance);
6123 4180 : optimize_load_redistribution (bst_map, vinfo, SLP_TREE_LANES (root),
6124 : &load_map, root);
6125 : }
6126 267 : }
6127 :
6128 : /* Check whether we should force some SLP instances to use load/store-lanes
6129 : and do so by forcing SLP re-discovery with single lanes. We used
6130 : to cancel SLP when this applied to all instances in a loop but now
6131 : we decide this per SLP instance. It's important to do this only
6132 : after SLP pattern recognition. */
6133 1184672 : if (is_a <loop_vec_info> (vinfo))
6134 1312911 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6135 807127 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
6136 303014 : && !SLP_INSTANCE_TREE (instance)->ldst_lanes)
6137 : {
6138 303014 : slp_tree slp_root = SLP_INSTANCE_TREE (instance);
6139 303014 : unsigned int group_size = SLP_TREE_LANES (slp_root);
6140 303014 : tree vectype = SLP_TREE_VECTYPE (slp_root);
6141 :
6142 303014 : stmt_vec_info rep_info = SLP_TREE_REPRESENTATIVE (slp_root);
6143 303014 : gimple *rep = STMT_VINFO_STMT (rep_info);
6144 303014 : bool masked = (is_gimple_call (rep)
6145 2446 : && gimple_call_internal_p (rep)
6146 305440 : && internal_fn_mask_index
6147 2426 : (gimple_call_internal_fn (rep)) != -1);
6148 302994 : if (!STMT_VINFO_GROUPED_ACCESS (rep_info)
6149 30147 : || slp_root->ldst_lanes
6150 333161 : || (vect_store_lanes_supported (vectype, group_size, masked)
6151 : == IFN_LAST))
6152 303014 : continue;
6153 :
6154 0 : auto_vec<slp_tree> loads;
6155 0 : hash_set<slp_tree> visited;
6156 0 : vect_gather_slp_loads (loads, slp_root, visited);
6157 :
6158 : /* Check whether any load in the SLP instance is possibly
6159 : permuted. */
6160 0 : bool loads_permuted = false;
6161 0 : slp_tree load_node;
6162 0 : unsigned j;
6163 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6164 : {
6165 0 : if (!SLP_TREE_LOAD_PERMUTATION (load_node).exists ())
6166 0 : continue;
6167 0 : for (unsigned k = 0; k < SLP_TREE_LANES (load_node); k++)
6168 0 : if (SLP_TREE_LOAD_PERMUTATION (load_node)[k] != k)
6169 : {
6170 : loads_permuted = true;
6171 : break;
6172 : }
6173 : }
6174 :
6175 : /* If the loads and stores can use load/store-lanes force re-discovery
6176 : with single lanes. */
6177 0 : if (loads_permuted)
6178 : {
6179 0 : bool can_use_lanes = true;
6180 : bool prefer_load_lanes = false;
6181 0 : FOR_EACH_VEC_ELT (loads, j, load_node)
6182 0 : if (STMT_VINFO_GROUPED_ACCESS
6183 : (SLP_TREE_REPRESENTATIVE (load_node)))
6184 : {
6185 0 : stmt_vec_info stmt_vinfo = DR_GROUP_FIRST_ELEMENT
6186 : (SLP_TREE_REPRESENTATIVE (load_node));
6187 0 : rep = STMT_VINFO_STMT (stmt_vinfo);
6188 0 : masked = (is_gimple_call (rep)
6189 0 : && gimple_call_internal_p (rep)
6190 0 : && internal_fn_mask_index
6191 0 : (gimple_call_internal_fn (rep)));
6192 : /* Use SLP for strided accesses (or if we can't
6193 : load-lanes). */
6194 0 : if (STMT_VINFO_STRIDED_P (stmt_vinfo)
6195 0 : || compare_step_with_zero (vinfo, stmt_vinfo) <= 0
6196 0 : || vect_load_lanes_supported
6197 0 : (SLP_TREE_VECTYPE (load_node),
6198 0 : DR_GROUP_SIZE (stmt_vinfo), masked) == IFN_LAST
6199 : /* ??? During SLP re-discovery with a single lane
6200 : a masked grouped load will appear permuted and
6201 : discovery will fail. We have to rework this
6202 : on the discovery side - for now avoid ICEing. */
6203 0 : || masked)
6204 : {
6205 : can_use_lanes = false;
6206 : break;
6207 : }
6208 : /* Make sure that the target would prefer store-lanes
6209 : for at least one of the loads.
6210 :
6211 : ??? Perhaps we should instead require this for
6212 : all loads? */
6213 0 : prefer_load_lanes
6214 : = (prefer_load_lanes
6215 0 : || SLP_TREE_LANES (load_node) == group_size
6216 0 : || (vect_slp_prefer_store_lanes_p
6217 0 : (vinfo, stmt_vinfo,
6218 : SLP_TREE_VECTYPE (load_node), masked,
6219 : group_size, SLP_TREE_LANES (load_node))));
6220 : }
6221 :
6222 0 : if (can_use_lanes && prefer_load_lanes)
6223 : {
6224 0 : if (dump_enabled_p ())
6225 0 : dump_printf_loc (MSG_NOTE, vect_location,
6226 : "SLP instance %p can use load/store-lanes,"
6227 : " re-discovering with single-lanes\n",
6228 : (void *) instance);
6229 :
6230 0 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (slp_root);
6231 :
6232 0 : vect_free_slp_instance (instance);
6233 0 : limit = max_tree_size;
6234 0 : bool res = vect_analyze_slp_instance (vinfo, bst_map,
6235 : stmt_info,
6236 : slp_inst_kind_store,
6237 : max_tree_size, &limit,
6238 : true);
6239 0 : gcc_assert (res);
6240 0 : auto new_inst = LOOP_VINFO_SLP_INSTANCES (vinfo).pop ();
6241 0 : LOOP_VINFO_SLP_INSTANCES (vinfo)[i] = new_inst;
6242 : }
6243 : }
6244 0 : }
6245 :
6246 : /* When we end up with load permutations that we cannot possibly handle,
6247 : like those requiring three vector inputs, lower them using interleaving
6248 : like schemes. */
6249 1184672 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
6250 : {
6251 505784 : vect_lower_load_permutations (loop_vinfo, bst_map, force_single_lane);
6252 505784 : if (dump_enabled_p ())
6253 : {
6254 21550 : dump_printf_loc (MSG_NOTE, vect_location,
6255 : "SLP graph after lowering permutations:\n");
6256 21550 : hash_set<slp_tree> visited;
6257 95606 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6258 30983 : vect_print_slp_graph (MSG_NOTE, vect_location,
6259 : SLP_INSTANCE_TREE (instance), visited);
6260 21550 : }
6261 : }
6262 :
6263 1184672 : release_scalar_stmts_to_slp_tree_map (bst_map);
6264 :
6265 1184672 : if (pattern_found && dump_enabled_p ())
6266 : {
6267 20 : dump_printf_loc (MSG_NOTE, vect_location,
6268 : "Pattern matched SLP tree\n");
6269 20 : hash_set<slp_tree> visited;
6270 101 : FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (vinfo), i, instance)
6271 41 : vect_print_slp_graph (MSG_NOTE, vect_location,
6272 : SLP_INSTANCE_TREE (instance), visited);
6273 20 : }
6274 :
6275 1184672 : return opt_result::success ();
6276 1184672 : }
6277 :
6278 : /* Estimates the cost of inserting layout changes into the SLP graph.
6279 : It can also say that the insertion is impossible. */
6280 :
6281 : struct slpg_layout_cost
6282 : {
6283 11293664 : slpg_layout_cost () = default;
6284 : slpg_layout_cost (sreal, bool);
6285 :
6286 532873 : static slpg_layout_cost impossible () { return { sreal::max (), 0 }; }
6287 5624160 : bool is_possible () const { return depth != sreal::max (); }
6288 :
6289 : bool operator== (const slpg_layout_cost &) const;
6290 : bool operator!= (const slpg_layout_cost &) const;
6291 :
6292 : bool is_better_than (const slpg_layout_cost &, bool) const;
6293 :
6294 : void add_parallel_cost (const slpg_layout_cost &);
6295 : void add_serial_cost (const slpg_layout_cost &);
6296 : void split (unsigned int);
6297 :
6298 : /* The longest sequence of layout changes needed during any traversal
6299 : of the partition dag, weighted by execution frequency.
6300 :
6301 : This is the most important metric when optimizing for speed, since
6302 : it helps to ensure that we keep the number of operations on
6303 : critical paths to a minimum. */
6304 : sreal depth = 0;
6305 :
6306 : /* An estimate of the total number of operations needed. It is weighted by
6307 : execution frequency when optimizing for speed but not when optimizing for
6308 : size. In order to avoid double-counting, a node with a fanout of N will
6309 : distribute 1/N of its total cost to each successor.
6310 :
6311 : This is the most important metric when optimizing for size, since
6312 : it helps to keep the total number of operations to a minimum, */
6313 : sreal total = 0;
6314 : };
6315 :
6316 : /* Construct costs for a node with weight WEIGHT. A higher weight
6317 : indicates more frequent execution. IS_FOR_SIZE is true if we are
6318 : optimizing for size rather than speed. */
6319 :
6320 1326236 : slpg_layout_cost::slpg_layout_cost (sreal weight, bool is_for_size)
6321 1333943 : : depth (weight), total (is_for_size && weight > 0 ? 1 : weight)
6322 : {
6323 1326236 : }
6324 :
6325 : bool
6326 0 : slpg_layout_cost::operator== (const slpg_layout_cost &other) const
6327 : {
6328 0 : return depth == other.depth && total == other.total;
6329 : }
6330 :
6331 : bool
6332 0 : slpg_layout_cost::operator!= (const slpg_layout_cost &other) const
6333 : {
6334 0 : return !operator== (other);
6335 : }
6336 :
6337 : /* Return true if these costs are better than OTHER. IS_FOR_SIZE is
6338 : true if we are optimizing for size rather than speed. */
6339 :
6340 : bool
6341 318212 : slpg_layout_cost::is_better_than (const slpg_layout_cost &other,
6342 : bool is_for_size) const
6343 : {
6344 318212 : if (is_for_size)
6345 : {
6346 3295 : if (total != other.total)
6347 1678 : return total < other.total;
6348 1617 : return depth < other.depth;
6349 : }
6350 : else
6351 : {
6352 314917 : if (depth != other.depth)
6353 128738 : return depth < other.depth;
6354 186179 : return total < other.total;
6355 : }
6356 : }
6357 :
6358 : /* Increase the costs to account for something with cost INPUT_COST
6359 : happening in parallel with the current costs. */
6360 :
6361 : void
6362 381704 : slpg_layout_cost::add_parallel_cost (const slpg_layout_cost &input_cost)
6363 : {
6364 381704 : depth = std::max (depth, input_cost.depth);
6365 381704 : total += input_cost.total;
6366 381704 : }
6367 :
6368 : /* Increase the costs to account for something with cost INPUT_COST
6369 : happening in series with the current costs. */
6370 :
6371 : void
6372 1575039 : slpg_layout_cost::add_serial_cost (const slpg_layout_cost &other)
6373 : {
6374 1575039 : depth += other.depth;
6375 1575039 : total += other.total;
6376 1575039 : }
6377 :
6378 : /* Split the total cost among TIMES successors or predecessors. */
6379 :
6380 : void
6381 1250833 : slpg_layout_cost::split (unsigned int times)
6382 : {
6383 1250833 : if (times > 1)
6384 602141 : total /= times;
6385 1250833 : }
6386 :
6387 : /* Information about one node in the SLP graph, for use during
6388 : vect_optimize_slp_pass. */
6389 :
6390 : struct slpg_vertex
6391 : {
6392 10541076 : slpg_vertex (slp_tree node_) : node (node_) {}
6393 :
6394 : /* The node itself. */
6395 : slp_tree node;
6396 :
6397 : /* Which partition the node belongs to, or -1 if none. Nodes outside of
6398 : partitions are flexible; they can have whichever layout consumers
6399 : want them to have. */
6400 : int partition = -1;
6401 :
6402 : /* The number of nodes that directly use the result of this one
6403 : (i.e. the number of nodes that count this one as a child). */
6404 : unsigned int out_degree = 0;
6405 :
6406 : /* The execution frequency of the node. */
6407 : sreal weight = 0;
6408 :
6409 : /* The total execution frequency of all nodes that directly use the
6410 : result of this one. */
6411 : sreal out_weight = 0;
6412 : };
6413 :
6414 : /* Information about one partition of the SLP graph, for use during
6415 : vect_optimize_slp_pass. */
6416 :
6417 : struct slpg_partition_info
6418 : {
6419 : /* The nodes in the partition occupy indices [NODE_BEGIN, NODE_END)
6420 : of m_partitioned_nodes. */
6421 : unsigned int node_begin = 0;
6422 : unsigned int node_end = 0;
6423 :
6424 : /* Which layout we've chosen to use for this partition, or -1 if
6425 : we haven't picked one yet. */
6426 : int layout = -1;
6427 :
6428 : /* The number of predecessors and successors in the partition dag.
6429 : The predecessors always have lower partition numbers and the
6430 : successors always have higher partition numbers.
6431 :
6432 : Note that the directions of these edges are not necessarily the
6433 : same as in the data flow graph. For example, if an SCC has separate
6434 : partitions for an inner loop and an outer loop, the inner loop's
6435 : partition will have at least two incoming edges from the outer loop's
6436 : partition: one for a live-in value and one for a live-out value.
6437 : In data flow terms, one of these edges would also be from the outer loop
6438 : to the inner loop, but the other would be in the opposite direction. */
6439 : unsigned int in_degree = 0;
6440 : unsigned int out_degree = 0;
6441 : };
6442 :
6443 : /* Information about the costs of using a particular layout for a
6444 : particular partition. It can also say that the combination is
6445 : impossible. */
6446 :
6447 : struct slpg_partition_layout_costs
6448 : {
6449 1658995 : bool is_possible () const { return internal_cost.is_possible (); }
6450 69687 : void mark_impossible () { internal_cost = slpg_layout_cost::impossible (); }
6451 :
6452 : /* The costs inherited from predecessor partitions. */
6453 : slpg_layout_cost in_cost;
6454 :
6455 : /* The inherent cost of the layout within the node itself. For example,
6456 : this is nonzero for a load if choosing a particular layout would require
6457 : the load to permute the loaded elements. It is nonzero for a
6458 : VEC_PERM_EXPR if the permutation cannot be eliminated or converted
6459 : to full-vector moves. */
6460 : slpg_layout_cost internal_cost;
6461 :
6462 : /* The costs inherited from successor partitions. */
6463 : slpg_layout_cost out_cost;
6464 : };
6465 :
6466 : /* This class tries to optimize the layout of vectors in order to avoid
6467 : unnecessary shuffling. At the moment, the set of possible layouts are
6468 : restricted to bijective permutations.
6469 :
6470 : The goal of the pass depends on whether we're optimizing for size or
6471 : for speed. When optimizing for size, the goal is to reduce the overall
6472 : number of layout changes (including layout changes implied by things
6473 : like load permutations). When optimizing for speed, the goal is to
6474 : reduce the maximum latency attributable to layout changes on any
6475 : non-cyclical path through the data flow graph.
6476 :
6477 : For example, when optimizing a loop nest for speed, we will prefer
6478 : to make layout changes outside of a loop rather than inside of a loop,
6479 : and will prefer to make layout changes in parallel rather than serially,
6480 : even if that increases the overall number of layout changes.
6481 :
6482 : The high-level procedure is:
6483 :
6484 : (1) Build a graph in which edges go from uses (parents) to definitions
6485 : (children).
6486 :
6487 : (2) Divide the graph into a dag of strongly-connected components (SCCs).
6488 :
6489 : (3) When optimizing for speed, partition the nodes in each SCC based
6490 : on their containing cfg loop. When optimizing for size, treat
6491 : each SCC as a single partition.
6492 :
6493 : This gives us a dag of partitions. The goal is now to assign a
6494 : layout to each partition.
6495 :
6496 : (4) Construct a set of vector layouts that are worth considering.
6497 : Record which nodes must keep their current layout.
6498 :
6499 : (5) Perform a forward walk over the partition dag (from loads to stores)
6500 : accumulating the "forward" cost of using each layout. When visiting
6501 : each partition, assign a tentative choice of layout to the partition
6502 : and use that choice when calculating the cost of using a different
6503 : layout in successor partitions.
6504 :
6505 : (6) Perform a backward walk over the partition dag (from stores to loads),
6506 : accumulating the "backward" cost of using each layout. When visiting
6507 : each partition, make a final choice of layout for that partition based
6508 : on the accumulated forward costs (from (5)) and backward costs
6509 : (from (6)).
6510 :
6511 : (7) Apply the chosen layouts to the SLP graph.
6512 :
6513 : For example, consider the SLP statements:
6514 :
6515 : S1: a_1 = load
6516 : loop:
6517 : S2: a_2 = PHI<a_1, a_3>
6518 : S3: b_1 = load
6519 : S4: a_3 = a_2 + b_1
6520 : exit:
6521 : S5: a_4 = PHI<a_3>
6522 : S6: store a_4
6523 :
6524 : S2 and S4 form an SCC and are part of the same loop. Every other
6525 : statement is in a singleton SCC. In this example there is a one-to-one
6526 : mapping between SCCs and partitions and the partition dag looks like this;
6527 :
6528 : S1 S3
6529 : \ /
6530 : S2+S4
6531 : |
6532 : S5
6533 : |
6534 : S6
6535 :
6536 : S2, S3 and S4 will have a higher execution frequency than the other
6537 : statements, so when optimizing for speed, the goal is to avoid any
6538 : layout changes:
6539 :
6540 : - within S3
6541 : - within S2+S4
6542 : - on the S3->S2+S4 edge
6543 :
6544 : For example, if S3 was originally a reversing load, the goal of the
6545 : pass is to make it an unreversed load and change the layout on the
6546 : S1->S2+S4 and S2+S4->S5 edges to compensate. (Changing the layout
6547 : on S1->S2+S4 and S5->S6 would also be acceptable.)
6548 :
6549 : The difference between SCCs and partitions becomes important if we
6550 : add an outer loop:
6551 :
6552 : S1: a_1 = ...
6553 : loop1:
6554 : S2: a_2 = PHI<a_1, a_6>
6555 : S3: b_1 = load
6556 : S4: a_3 = a_2 + b_1
6557 : loop2:
6558 : S5: a_4 = PHI<a_3, a_5>
6559 : S6: c_1 = load
6560 : S7: a_5 = a_4 + c_1
6561 : exit2:
6562 : S8: a_6 = PHI<a_5>
6563 : S9: store a_6
6564 : exit1:
6565 :
6566 : Here, S2, S4, S5, S7 and S8 form a single SCC. However, when optimizing
6567 : for speed, we usually do not want restrictions in the outer loop to "infect"
6568 : the decision for the inner loop. For example, if an outer-loop node
6569 : in the SCC contains a statement with a fixed layout, that should not
6570 : prevent the inner loop from using a different layout. Conversely,
6571 : the inner loop should not dictate a layout to the outer loop: if the
6572 : outer loop does a lot of computation, then it may not be efficient to
6573 : do all of that computation in the inner loop's preferred layout.
6574 :
6575 : So when optimizing for speed, we partition the SCC into S2+S4+S8 (outer)
6576 : and S5+S7 (inner). We also try to arrange partitions so that:
6577 :
6578 : - the partition for an outer loop comes before the partition for
6579 : an inner loop
6580 :
6581 : - if a sibling loop A dominates a sibling loop B, A's partition
6582 : comes before B's
6583 :
6584 : This gives the following partition dag for the example above:
6585 :
6586 : S1 S3
6587 : \ /
6588 : S2+S4+S8 S6
6589 : | \\ /
6590 : | S5+S7
6591 : |
6592 : S9
6593 :
6594 : There are two edges from S2+S4+S8 to S5+S7: one for the edge S4->S5 and
6595 : one for a reversal of the edge S7->S8.
6596 :
6597 : The backward walk picks a layout for S5+S7 before S2+S4+S8. The choice
6598 : for S2+S4+S8 therefore has to balance the cost of using the outer loop's
6599 : preferred layout against the cost of changing the layout on entry to the
6600 : inner loop (S4->S5) and on exit from the inner loop (S7->S8 reversed).
6601 :
6602 : Although this works well when optimizing for speed, it has the downside
6603 : when optimizing for size that the choice of layout for S5+S7 is completely
6604 : independent of S9, which lessens the chance of reducing the overall number
6605 : of permutations. We therefore do not partition SCCs when optimizing
6606 : for size.
6607 :
6608 : To give a concrete example of the difference between optimizing
6609 : for size and speed, consider:
6610 :
6611 : a[0] = (b[1] << c[3]) - d[1];
6612 : a[1] = (b[0] << c[2]) - d[0];
6613 : a[2] = (b[3] << c[1]) - d[3];
6614 : a[3] = (b[2] << c[0]) - d[2];
6615 :
6616 : There are three different layouts here: one for a, one for b and d,
6617 : and one for c. When optimizing for speed it is better to permute each
6618 : of b, c and d into the order required by a, since those permutations
6619 : happen in parallel. But when optimizing for size, it is better to:
6620 :
6621 : - permute c into the same order as b
6622 : - do the arithmetic
6623 : - permute the result into the order required by a
6624 :
6625 : This gives 2 permutations rather than 3. */
6626 :
6627 : class vect_optimize_slp_pass
6628 : {
6629 : public:
6630 721496 : vect_optimize_slp_pass (vec_info *vinfo) : m_vinfo (vinfo) {}
6631 : void run ();
6632 :
6633 : private:
6634 : /* Graph building. */
6635 : struct loop *containing_loop (slp_tree);
6636 : bool is_cfg_latch_edge (graph_edge *);
6637 : void build_vertices (hash_set<slp_tree> &, slp_tree);
6638 : void build_vertices ();
6639 : void build_graph ();
6640 :
6641 : /* Partitioning. */
6642 : void create_partitions ();
6643 : template<typename T> void for_each_partition_edge (unsigned int, T);
6644 :
6645 : /* Layout selection. */
6646 : bool is_compatible_layout (slp_tree, unsigned int);
6647 : bool is_compatible_layout (const slpg_partition_info &, unsigned int);
6648 : int change_layout_cost (slp_tree, unsigned int, unsigned int);
6649 : slpg_partition_layout_costs &partition_layout_costs (unsigned int,
6650 : unsigned int);
6651 : void change_vec_perm_layout (slp_tree, lane_permutation_t &,
6652 : int, unsigned int);
6653 : int internal_node_cost (slp_tree, int, unsigned int);
6654 : void start_choosing_layouts ();
6655 : bool legitimize ();
6656 :
6657 : /* Cost propagation. */
6658 : slpg_layout_cost edge_layout_cost (graph_edge *, unsigned int,
6659 : unsigned int, unsigned int);
6660 : slpg_layout_cost total_in_cost (unsigned int);
6661 : slpg_layout_cost forward_cost (graph_edge *, unsigned int, unsigned int);
6662 : slpg_layout_cost backward_cost (graph_edge *, unsigned int, unsigned int);
6663 : void forward_pass ();
6664 : void backward_pass ();
6665 :
6666 : /* Rematerialization. */
6667 : slp_tree get_result_with_layout (slp_tree, unsigned int);
6668 : void materialize ();
6669 :
6670 : /* Clean-up. */
6671 : void remove_redundant_permutations ();
6672 :
6673 : /* Masked load lanes discovery. */
6674 : void decide_masked_load_lanes ();
6675 :
6676 : void dump ();
6677 :
6678 : vec_info *m_vinfo;
6679 :
6680 : /* True if we should optimize the graph for size, false if we should
6681 : optimize it for speed. (It wouldn't be easy to make this decision
6682 : more locally.) */
6683 : bool m_optimize_size;
6684 :
6685 : /* A graph of all SLP nodes, with edges leading from uses to definitions.
6686 : In other words, a node's predecessors are its slp_tree parents and
6687 : a node's successors are its slp_tree children. */
6688 : graph *m_slpg = nullptr;
6689 :
6690 : /* The vertices of M_SLPG, indexed by slp_tree::vertex. */
6691 : auto_vec<slpg_vertex> m_vertices;
6692 :
6693 : /* The list of all leaves of M_SLPG. such as external definitions, constants,
6694 : and loads. */
6695 : auto_vec<int> m_leafs;
6696 :
6697 : /* This array has one entry for every vector layout that we're considering.
6698 : Element 0 is null and indicates "no change". Other entries describe
6699 : permutations that are inherent in the current graph and that we would
6700 : like to reverse if possible.
6701 :
6702 : For example, a permutation { 1, 2, 3, 0 } means that something has
6703 : effectively been permuted in that way, such as a load group
6704 : { a[1], a[2], a[3], a[0] } (viewed as a permutation of a[0:3]).
6705 : We'd then like to apply the reverse permutation { 3, 0, 1, 2 }
6706 : in order to put things "back" in order. */
6707 : auto_vec<vec<unsigned> > m_perms;
6708 :
6709 : /* A partitioning of the nodes for which a layout must be chosen.
6710 : Each partition represents an <SCC, cfg loop> pair; that is,
6711 : nodes in different SCCs belong to different partitions, and nodes
6712 : within an SCC can be further partitioned according to a containing
6713 : cfg loop. Partition <SCC1, L1> comes before <SCC2, L2> if:
6714 :
6715 : - SCC1 != SCC2 and SCC1 is a predecessor of SCC2 in a forward walk
6716 : from leaves (such as loads) to roots (such as stores).
6717 :
6718 : - SCC1 == SCC2 and L1's header strictly dominates L2's header. */
6719 : auto_vec<slpg_partition_info> m_partitions;
6720 :
6721 : /* The list of all nodes for which a layout must be chosen. Nodes for
6722 : partition P come before the nodes for partition P+1. Nodes within a
6723 : partition are in reverse postorder. */
6724 : auto_vec<unsigned int> m_partitioned_nodes;
6725 :
6726 : /* Index P * num-layouts + L contains the cost of using layout L
6727 : for partition P. */
6728 : auto_vec<slpg_partition_layout_costs> m_partition_layout_costs;
6729 :
6730 : /* Index N * num-layouts + L, if nonnull, is a node that provides the
6731 : original output of node N adjusted to have layout L. */
6732 : auto_vec<slp_tree> m_node_layouts;
6733 : };
6734 :
6735 : /* Fill the vertices and leafs vector with all nodes in the SLP graph.
6736 : Also record whether we should optimize anything for speed rather
6737 : than size. */
6738 :
6739 : void
6740 11390004 : vect_optimize_slp_pass::build_vertices (hash_set<slp_tree> &visited,
6741 : slp_tree node)
6742 : {
6743 11390004 : unsigned i;
6744 11390004 : slp_tree child;
6745 :
6746 11390004 : if (visited.add (node))
6747 11390004 : return;
6748 :
6749 10541076 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
6750 : {
6751 8062724 : basic_block bb = gimple_bb (vect_orig_stmt (rep)->stmt);
6752 7105052 : if (optimize_bb_for_speed_p (bb))
6753 6975078 : m_optimize_size = false;
6754 : }
6755 :
6756 10541076 : node->vertex = m_vertices.length ();
6757 10541076 : m_vertices.safe_push (slpg_vertex (node));
6758 :
6759 10541076 : bool leaf = true;
6760 10541076 : bool force_leaf = false;
6761 19762758 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
6762 9221682 : if (child)
6763 : {
6764 8296768 : leaf = false;
6765 8296768 : build_vertices (visited, child);
6766 : }
6767 : else
6768 : force_leaf = true;
6769 : /* Since SLP discovery works along use-def edges all cycles have an
6770 : entry - but there's the exception of cycles where we do not handle
6771 : the entry explicitly (but with a NULL SLP node), like some reductions
6772 : and inductions. Force those SLP PHIs to act as leafs to make them
6773 : backwards reachable. */
6774 10541076 : if (leaf || force_leaf)
6775 5227452 : m_leafs.safe_push (node->vertex);
6776 : }
6777 :
6778 : /* Fill the vertices and leafs vector with all nodes in the SLP graph. */
6779 :
6780 : void
6781 1442992 : vect_optimize_slp_pass::build_vertices ()
6782 : {
6783 1442992 : hash_set<slp_tree> visited;
6784 1442992 : unsigned i;
6785 1442992 : slp_instance instance;
6786 1442992 : m_vertices.truncate (0);
6787 1442992 : m_leafs.truncate (0);
6788 7422212 : FOR_EACH_VEC_ELT (m_vinfo->slp_instances, i, instance)
6789 3093236 : build_vertices (visited, SLP_INSTANCE_TREE (instance));
6790 1442992 : }
6791 :
6792 : /* Apply (reverse) bijectite PERM to VEC. */
6793 :
6794 : template <class T>
6795 : static void
6796 232367 : vect_slp_permute (vec<unsigned> perm,
6797 : vec<T> &vec, bool reverse)
6798 : {
6799 232367 : auto_vec<T, 64> saved;
6800 232367 : saved.create (vec.length ());
6801 770243 : for (unsigned i = 0; i < vec.length (); ++i)
6802 537876 : saved.quick_push (vec[i]);
6803 :
6804 232367 : if (reverse)
6805 : {
6806 1520829 : for (unsigned i = 0; i < vec.length (); ++i)
6807 535756 : vec[perm[i]] = saved[i];
6808 767123 : for (unsigned i = 0; i < vec.length (); ++i)
6809 905811 : gcc_assert (vec[perm[i]] == saved[i]);
6810 : }
6811 : else
6812 : {
6813 6240 : for (unsigned i = 0; i < vec.length (); ++i)
6814 2120 : vec[i] = saved[perm[i]];
6815 234487 : for (unsigned i = 0; i < vec.length (); ++i)
6816 3180 : gcc_assert (vec[i] == saved[perm[i]]);
6817 : }
6818 232367 : }
6819 :
6820 : /* Return the cfg loop that contains NODE. */
6821 :
6822 : struct loop *
6823 4102941 : vect_optimize_slp_pass::containing_loop (slp_tree node)
6824 : {
6825 4102941 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
6826 : /* ??? This is imprecise, VEC_PERM nodes do not have a representative
6827 : but are laid out close to their children. */
6828 4102941 : if (!rep)
6829 144951 : return m_vinfo->bbs[0]->loop_father;
6830 4450179 : return gimple_bb (vect_orig_stmt (rep)->stmt)->loop_father;
6831 : }
6832 :
6833 : /* Return true if UD (an edge from a use to a definition) is associated
6834 : with a loop latch edge in the cfg. */
6835 :
6836 : bool
6837 8296768 : vect_optimize_slp_pass::is_cfg_latch_edge (graph_edge *ud)
6838 : {
6839 8296768 : slp_tree use = m_vertices[ud->src].node;
6840 8296768 : slp_tree def = m_vertices[ud->dest].node;
6841 8296768 : if ((SLP_TREE_DEF_TYPE (use) != vect_internal_def
6842 8296768 : || SLP_TREE_PERMUTE_P (use))
6843 7976142 : || SLP_TREE_DEF_TYPE (def) != vect_internal_def)
6844 : return false;
6845 :
6846 4829866 : stmt_vec_info use_rep = vect_orig_stmt (SLP_TREE_REPRESENTATIVE (use));
6847 4829866 : return (is_a<gphi *> (use_rep->stmt)
6848 386998 : && bb_loop_header_p (gimple_bb (use_rep->stmt))
6849 5048672 : && containing_loop (def) == containing_loop (use));
6850 : }
6851 :
6852 : /* Build the graph. Mark edges that correspond to cfg loop latch edges with
6853 : a nonnull data field. */
6854 :
6855 : void
6856 1442992 : vect_optimize_slp_pass::build_graph ()
6857 : {
6858 1442992 : m_optimize_size = true;
6859 1442992 : build_vertices ();
6860 :
6861 2885984 : m_slpg = new_graph (m_vertices.length ());
6862 14870052 : for (slpg_vertex &v : m_vertices)
6863 31500566 : for (slp_tree child : SLP_TREE_CHILDREN (v.node))
6864 9221682 : if (child)
6865 : {
6866 8296768 : graph_edge *ud = add_edge (m_slpg, v.node->vertex, child->vertex);
6867 8296768 : if (is_cfg_latch_edge (ud))
6868 208958 : ud->data = this;
6869 : }
6870 1442992 : }
6871 :
6872 : /* Return true if E corresponds to a loop latch edge in the cfg. */
6873 :
6874 : static bool
6875 4252294 : skip_cfg_latch_edges (graph_edge *e)
6876 : {
6877 4252294 : return e->data;
6878 : }
6879 :
6880 : /* Create the node partitions. */
6881 :
6882 : void
6883 721496 : vect_optimize_slp_pass::create_partitions ()
6884 : {
6885 : /* Calculate a postorder of the graph, ignoring edges that correspond
6886 : to natural latch edges in the cfg. Reading the vector from the end
6887 : to the beginning gives the reverse postorder. */
6888 721496 : auto_vec<int> initial_rpo;
6889 1442992 : graphds_dfs (m_slpg, &m_leafs[0], m_leafs.length (), &initial_rpo,
6890 : false, NULL, skip_cfg_latch_edges);
6891 2164488 : gcc_assert (initial_rpo.length () == m_vertices.length ());
6892 :
6893 : /* Calculate the strongly connected components of the graph. */
6894 721496 : auto_vec<int> scc_grouping;
6895 721496 : unsigned int num_sccs = graphds_scc (m_slpg, NULL, NULL, &scc_grouping);
6896 :
6897 : /* Create a new index order in which all nodes from the same SCC are
6898 : consecutive. Use scc_pos to record the index of the first node in
6899 : each SCC. */
6900 721496 : auto_vec<unsigned int> scc_pos (num_sccs);
6901 721496 : int last_component = -1;
6902 721496 : unsigned int node_count = 0;
6903 7434463 : for (unsigned int node_i : scc_grouping)
6904 : {
6905 5269975 : if (last_component != m_slpg->vertices[node_i].component)
6906 : {
6907 5138981 : last_component = m_slpg->vertices[node_i].component;
6908 10277962 : gcc_assert (last_component == int (scc_pos.length ()));
6909 5138981 : scc_pos.quick_push (node_count);
6910 : }
6911 5269975 : node_count += 1;
6912 : }
6913 1442992 : gcc_assert (node_count == initial_rpo.length ()
6914 : && last_component + 1 == int (num_sccs));
6915 :
6916 : /* Use m_partitioned_nodes to group nodes into SCC order, with the nodes
6917 : inside each SCC following the RPO we calculated above. The fact that
6918 : we ignored natural latch edges when calculating the RPO should ensure
6919 : that, for natural loop nests:
6920 :
6921 : - the first node that we encounter in a cfg loop is the loop header phi
6922 : - the loop header phis are in dominance order
6923 :
6924 : Arranging for this is an optimization (see below) rather than a
6925 : correctness issue. Unnatural loops with a tangled mess of backedges
6926 : will still work correctly, but might give poorer results.
6927 :
6928 : Also update scc_pos so that it gives 1 + the index of the last node
6929 : in the SCC. */
6930 721496 : m_partitioned_nodes.safe_grow (node_count);
6931 6712967 : for (unsigned int old_i = initial_rpo.length (); old_i-- > 0;)
6932 : {
6933 5269975 : unsigned int node_i = initial_rpo[old_i];
6934 5269975 : unsigned int new_i = scc_pos[m_slpg->vertices[node_i].component]++;
6935 5269975 : m_partitioned_nodes[new_i] = node_i;
6936 : }
6937 :
6938 : /* When optimizing for speed, partition each SCC based on the containing
6939 : cfg loop. The order we constructed above should ensure that, for natural
6940 : cfg loops, we'll create sub-SCC partitions for outer loops before
6941 : the corresponding sub-SCC partitions for inner loops. Similarly,
6942 : when one sibling loop A dominates another sibling loop B, we should
6943 : create a sub-SCC partition for A before a sub-SCC partition for B.
6944 :
6945 : As above, nothing depends for correctness on whether this achieves
6946 : a natural nesting, but we should get better results when it does. */
6947 1442992 : m_partitions.reserve (m_vertices.length ());
6948 721496 : unsigned int next_partition_i = 0;
6949 721496 : hash_map<struct loop *, int> loop_partitions;
6950 721496 : unsigned int rpo_begin = 0;
6951 721496 : unsigned int num_partitioned_nodes = 0;
6952 7303469 : for (unsigned int rpo_end : scc_pos)
6953 : {
6954 5138981 : loop_partitions.empty ();
6955 5138981 : unsigned int partition_i = next_partition_i;
6956 10408956 : for (unsigned int rpo_i = rpo_begin; rpo_i < rpo_end; ++rpo_i)
6957 : {
6958 : /* Handle externals and constants optimistically throughout.
6959 : But treat existing vectors as fixed since we do not handle
6960 : permuting them. */
6961 5269975 : unsigned int node_i = m_partitioned_nodes[rpo_i];
6962 5269975 : auto &vertex = m_vertices[node_i];
6963 5269975 : if ((SLP_TREE_DEF_TYPE (vertex.node) == vect_external_def
6964 532245 : && !SLP_TREE_VEC_DEFS (vertex.node).exists ())
6965 5273119 : || SLP_TREE_DEF_TYPE (vertex.node) == vect_constant_def)
6966 1574350 : vertex.partition = -1;
6967 : else
6968 : {
6969 3695625 : bool existed;
6970 3695625 : if (m_optimize_size)
6971 30296 : existed = next_partition_i > partition_i;
6972 : else
6973 : {
6974 3665329 : struct loop *loop = containing_loop (vertex.node);
6975 3665329 : auto &entry = loop_partitions.get_or_insert (loop, &existed);
6976 3665329 : if (!existed)
6977 3535564 : entry = next_partition_i;
6978 3665329 : partition_i = entry;
6979 : }
6980 3695625 : if (!existed)
6981 : {
6982 3565768 : m_partitions.quick_push (slpg_partition_info ());
6983 3565768 : next_partition_i += 1;
6984 : }
6985 3695625 : vertex.partition = partition_i;
6986 3695625 : num_partitioned_nodes += 1;
6987 3695625 : m_partitions[partition_i].node_end += 1;
6988 : }
6989 : }
6990 5138981 : rpo_begin = rpo_end;
6991 : }
6992 :
6993 : /* Assign ranges of consecutive node indices to each partition,
6994 : in partition order. Start with node_end being the same as
6995 : node_begin so that the next loop can use it as a counter. */
6996 721496 : unsigned int node_begin = 0;
6997 5730256 : for (auto &partition : m_partitions)
6998 : {
6999 3565768 : partition.node_begin = node_begin;
7000 3565768 : node_begin += partition.node_end;
7001 3565768 : partition.node_end = partition.node_begin;
7002 : }
7003 721496 : gcc_assert (node_begin == num_partitioned_nodes);
7004 :
7005 : /* Finally build the list of nodes in partition order. */
7006 721496 : m_partitioned_nodes.truncate (num_partitioned_nodes);
7007 6712967 : for (unsigned int node_i = 0; node_i < m_vertices.length (); ++node_i)
7008 : {
7009 5269975 : int partition_i = m_vertices[node_i].partition;
7010 5269975 : if (partition_i >= 0)
7011 : {
7012 3695625 : unsigned int order_i = m_partitions[partition_i].node_end++;
7013 3695625 : m_partitioned_nodes[order_i] = node_i;
7014 : }
7015 : }
7016 721496 : }
7017 :
7018 : /* Look for edges from earlier partitions into node NODE_I and edges from
7019 : node NODE_I into later partitions. Call:
7020 :
7021 : FN (ud, other_node_i)
7022 :
7023 : for each such use-to-def edge ud, where other_node_i is the node at the
7024 : other end of the edge. */
7025 :
7026 : template<typename T>
7027 : void
7028 4160370 : vect_optimize_slp_pass::for_each_partition_edge (unsigned int node_i, T fn)
7029 : {
7030 4160370 : int partition_i = m_vertices[node_i].partition;
7031 4160370 : for (graph_edge *pred = m_slpg->vertices[node_i].pred;
7032 7187235 : pred; pred = pred->pred_next)
7033 : {
7034 3026865 : int src_partition_i = m_vertices[pred->src].partition;
7035 3026865 : if (src_partition_i >= 0 && src_partition_i != partition_i)
7036 2696372 : fn (pred, pred->src);
7037 : }
7038 4160370 : for (graph_edge *succ = m_slpg->vertices[node_i].succ;
7039 8934733 : succ; succ = succ->succ_next)
7040 : {
7041 4774363 : int dest_partition_i = m_vertices[succ->dest].partition;
7042 4774363 : if (dest_partition_i >= 0 && dest_partition_i != partition_i)
7043 2732947 : fn (succ, succ->dest);
7044 : }
7045 4160370 : }
7046 :
7047 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7048 : that NODE would operate on. This test is independent of NODE's actual
7049 : operation. */
7050 :
7051 : bool
7052 1771288 : vect_optimize_slp_pass::is_compatible_layout (slp_tree node,
7053 : unsigned int layout_i)
7054 : {
7055 1771288 : if (layout_i == 0)
7056 : return true;
7057 :
7058 1043550 : if (SLP_TREE_LANES (node) != m_perms[layout_i].length ())
7059 18509 : return false;
7060 :
7061 : return true;
7062 : }
7063 :
7064 : /* Return true if layout LAYOUT_I is compatible with the number of SLP lanes
7065 : that NODE would operate on for each NODE in PARTITION.
7066 : This test is independent of NODE's actual operations. */
7067 :
7068 : bool
7069 25849 : vect_optimize_slp_pass::is_compatible_layout (const slpg_partition_info
7070 : &partition,
7071 : unsigned int layout_i)
7072 : {
7073 52133 : for (unsigned int order_i = partition.node_begin;
7074 52133 : order_i < partition.node_end; ++order_i)
7075 : {
7076 26383 : unsigned int node_i = m_partitioned_nodes[order_i];
7077 26383 : auto &vertex = m_vertices[node_i];
7078 :
7079 : /* The layout is incompatible if it is individually incompatible
7080 : with any node in the partition. */
7081 26383 : if (!is_compatible_layout (vertex.node, layout_i))
7082 : return false;
7083 : }
7084 : return true;
7085 : }
7086 :
7087 : /* Return the cost (in arbitrary units) of going from layout FROM_LAYOUT_I
7088 : to layout TO_LAYOUT_I for a node like NODE. Return -1 if either of the
7089 : layouts is incompatible with NODE or if the change is not possible for
7090 : some other reason.
7091 :
7092 : The properties taken from NODE include the number of lanes and the
7093 : vector type. The actual operation doesn't matter. */
7094 :
7095 : int
7096 745918 : vect_optimize_slp_pass::change_layout_cost (slp_tree node,
7097 : unsigned int from_layout_i,
7098 : unsigned int to_layout_i)
7099 : {
7100 745918 : if (!is_compatible_layout (node, from_layout_i)
7101 745918 : || !is_compatible_layout (node, to_layout_i))
7102 : return -1;
7103 :
7104 745239 : if (from_layout_i == to_layout_i)
7105 : return 0;
7106 :
7107 308631 : auto_vec<slp_tree, 1> children (1);
7108 308631 : children.quick_push (node);
7109 308631 : auto_lane_permutation_t perm (SLP_TREE_LANES (node));
7110 308631 : if (from_layout_i > 0)
7111 871033 : for (unsigned int i : m_perms[from_layout_i])
7112 386911 : perm.quick_push ({ 0, i });
7113 : else
7114 483214 : for (unsigned int i = 0; i < SLP_TREE_LANES (node); ++i)
7115 335957 : perm.quick_push ({ 0, i });
7116 308631 : if (to_layout_i > 0)
7117 148006 : vect_slp_permute (m_perms[to_layout_i], perm, true);
7118 308631 : auto count = vectorizable_slp_permutation_1 (m_vinfo, nullptr, node, perm,
7119 : children, false);
7120 308631 : if (count >= 0)
7121 303232 : return MAX (count, 1);
7122 :
7123 : /* ??? In principle we could try changing via layout 0, giving two
7124 : layout changes rather than 1. Doing that would require
7125 : corresponding support in get_result_with_layout. */
7126 : return -1;
7127 308631 : }
7128 :
7129 : /* Return the costs of assigning layout LAYOUT_I to partition PARTITION_I. */
7130 :
7131 : inline slpg_partition_layout_costs &
7132 1125203 : vect_optimize_slp_pass::partition_layout_costs (unsigned int partition_i,
7133 : unsigned int layout_i)
7134 : {
7135 2250406 : return m_partition_layout_costs[partition_i * m_perms.length () + layout_i];
7136 : }
7137 :
7138 : /* Change PERM in one of two ways:
7139 :
7140 : - if IN_LAYOUT_I < 0, accept input operand I in the layout that has been
7141 : chosen for child I of NODE.
7142 :
7143 : - if IN_LAYOUT >= 0, accept all inputs operands with that layout.
7144 :
7145 : In both cases, arrange for the output to have layout OUT_LAYOUT_I */
7146 :
7147 : void
7148 38300 : vect_optimize_slp_pass::
7149 : change_vec_perm_layout (slp_tree node, lane_permutation_t &perm,
7150 : int in_layout_i, unsigned int out_layout_i)
7151 : {
7152 224470 : for (auto &entry : perm)
7153 : {
7154 109570 : int this_in_layout_i = in_layout_i;
7155 109570 : if (this_in_layout_i < 0)
7156 : {
7157 74759 : slp_tree in_node = SLP_TREE_CHILDREN (node)[entry.first];
7158 74759 : unsigned int in_partition_i = m_vertices[in_node->vertex].partition;
7159 74759 : if (in_partition_i == -1u)
7160 329 : continue;
7161 74430 : this_in_layout_i = m_partitions[in_partition_i].layout;
7162 : }
7163 109241 : if (this_in_layout_i > 0)
7164 24965 : entry.second = m_perms[this_in_layout_i][entry.second];
7165 : }
7166 38300 : if (out_layout_i > 0)
7167 10611 : vect_slp_permute (m_perms[out_layout_i], perm, true);
7168 38300 : }
7169 :
7170 : /* Check whether the target allows NODE to be rearranged so that the node's
7171 : output has layout OUT_LAYOUT_I. Return the cost of the change if so,
7172 : in the same arbitrary units as for change_layout_cost. Return -1 otherwise.
7173 :
7174 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I < 0, also check whether
7175 : NODE can adapt to the layout changes that have (perhaps provisionally)
7176 : been chosen for NODE's children, so that no extra permutations are
7177 : needed on either the input or the output of NODE.
7178 :
7179 : If NODE is a VEC_PERM_EXPR and IN_LAYOUT_I >= 0, instead assume
7180 : that all inputs will be forced into layout IN_LAYOUT_I beforehand.
7181 :
7182 : IN_LAYOUT_I has no meaning for other types of node.
7183 :
7184 : Keeping the node as-is is always valid. If the target doesn't appear
7185 : to support the node as-is, but might realistically support other layouts,
7186 : then layout 0 instead has the cost of a worst-case permutation. On the
7187 : one hand, this ensures that every node has at least one valid layout,
7188 : avoiding what would otherwise be an awkward special case. On the other,
7189 : it still encourages the pass to change an invalid pre-existing layout
7190 : choice into a valid one. */
7191 :
7192 : int
7193 248558 : vect_optimize_slp_pass::internal_node_cost (slp_tree node, int in_layout_i,
7194 : unsigned int out_layout_i)
7195 : {
7196 248558 : const int fallback_cost = 1;
7197 :
7198 248558 : if (SLP_TREE_PERMUTE_P (node))
7199 : {
7200 31804 : auto_lane_permutation_t tmp_perm;
7201 31804 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7202 :
7203 : /* Check that the child nodes support the chosen layout. Checking
7204 : the first child is enough, since any second child would have the
7205 : same shape. */
7206 31804 : auto first_child = SLP_TREE_CHILDREN (node)[0];
7207 31804 : if (in_layout_i > 0
7208 31804 : && !is_compatible_layout (first_child, in_layout_i))
7209 : return -1;
7210 :
7211 31130 : change_vec_perm_layout (node, tmp_perm, in_layout_i, out_layout_i);
7212 62260 : int count = vectorizable_slp_permutation_1 (m_vinfo, nullptr,
7213 : node, tmp_perm,
7214 31130 : SLP_TREE_CHILDREN (node),
7215 : false);
7216 31130 : if (count < 0)
7217 : {
7218 2563 : if (in_layout_i == 0 && out_layout_i == 0)
7219 : {
7220 : /* Use the fallback cost if the node could in principle support
7221 : some nonzero layout for both the inputs and the outputs.
7222 : Otherwise assume that the node will be rejected later
7223 : and rebuilt from scalars. */
7224 638 : if (SLP_TREE_LANES (node) == SLP_TREE_LANES (first_child))
7225 : return fallback_cost;
7226 353 : return 0;
7227 : }
7228 : return -1;
7229 : }
7230 :
7231 : /* We currently have no way of telling whether the new layout is cheaper
7232 : or more expensive than the old one. But at least in principle,
7233 : it should be worth making zero permutations (whole-vector shuffles)
7234 : cheaper than real permutations, in case the pass is able to remove
7235 : the latter. */
7236 28567 : return count == 0 ? 0 : 1;
7237 31804 : }
7238 :
7239 216754 : stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node);
7240 216754 : if (rep
7241 214883 : && STMT_VINFO_DATA_REF (rep)
7242 73879 : && DR_IS_READ (STMT_VINFO_DATA_REF (rep))
7243 264439 : && SLP_TREE_LOAD_PERMUTATION (node).exists ())
7244 : {
7245 39208 : auto_load_permutation_t tmp_perm;
7246 39208 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7247 39208 : if (out_layout_i > 0)
7248 15061 : vect_slp_permute (m_perms[out_layout_i], tmp_perm, true);
7249 :
7250 39208 : poly_uint64 vf = 1;
7251 39208 : if (auto loop_vinfo = dyn_cast<loop_vec_info> (m_vinfo))
7252 12176 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
7253 39208 : unsigned int n_perms;
7254 39208 : if (!vect_transform_slp_perm_load_1 (m_vinfo, node, tmp_perm, vNULL,
7255 : nullptr, vf, true, false, &n_perms))
7256 : {
7257 2011 : auto rep = SLP_TREE_REPRESENTATIVE (node);
7258 2011 : if (out_layout_i == 0)
7259 : {
7260 : /* Use the fallback cost if the load is an N-to-N permutation.
7261 : Otherwise assume that the node will be rejected later
7262 : and rebuilt from scalars. */
7263 1479 : if (STMT_VINFO_GROUPED_ACCESS (rep)
7264 2958 : && (DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (rep))
7265 1479 : == SLP_TREE_LANES (node)))
7266 671 : return fallback_cost;
7267 : return 0;
7268 : }
7269 : return -1;
7270 : }
7271 :
7272 : /* See the comment above the corresponding VEC_PERM_EXPR handling. */
7273 37197 : return n_perms == 0 ? 0 : 1;
7274 39208 : }
7275 :
7276 : return 0;
7277 : }
7278 :
7279 : /* Decide which element layouts we should consider using. Calculate the
7280 : weights associated with inserting layout changes on partition edges.
7281 : Also mark partitions that cannot change layout, by setting their
7282 : layout to zero. */
7283 :
7284 : void
7285 721496 : vect_optimize_slp_pass::start_choosing_layouts ()
7286 : {
7287 : /* Used to assign unique permutation indices. */
7288 721496 : using perm_hash = unbounded_hashmap_traits<
7289 : vec_free_hash_base<int_hash_base<unsigned>>,
7290 : int_hash<int, -1, -2>
7291 : >;
7292 721496 : hash_map<vec<unsigned>, int, perm_hash> layout_ids;
7293 :
7294 : /* Layout 0 is "no change". */
7295 721496 : m_perms.safe_push (vNULL);
7296 :
7297 : /* Create layouts from existing permutations. */
7298 721496 : auto_load_permutation_t tmp_perm;
7299 5860113 : for (unsigned int node_i : m_partitioned_nodes)
7300 : {
7301 : /* Leafs also double as entries to the reverse graph. Allow the
7302 : layout of those to be changed. */
7303 3695625 : auto &vertex = m_vertices[node_i];
7304 3695625 : auto &partition = m_partitions[vertex.partition];
7305 3695625 : if (!m_slpg->vertices[node_i].succ)
7306 946553 : partition.layout = 0;
7307 :
7308 : /* Loads and VEC_PERM_EXPRs are the only things generating permutes. */
7309 3695625 : slp_tree node = vertex.node;
7310 3695625 : stmt_vec_info dr_stmt = SLP_TREE_REPRESENTATIVE (node);
7311 3695625 : slp_tree child;
7312 3695625 : unsigned HOST_WIDE_INT imin, imax = 0;
7313 3695625 : bool any_permute = false;
7314 3695625 : tmp_perm.truncate (0);
7315 3695625 : if (SLP_TREE_LOAD_PERMUTATION (node).exists ())
7316 : {
7317 : /* If splitting out a SLP_TREE_LANE_PERMUTATION can make the node
7318 : unpermuted, record a layout that reverses this permutation.
7319 :
7320 : We would need more work to cope with loads that are internally
7321 : permuted and also have inputs (such as masks for
7322 : IFN_MASK_LOADs). */
7323 637968 : gcc_assert (partition.layout == 0 && !m_slpg->vertices[node_i].succ);
7324 637968 : if (!STMT_VINFO_GROUPED_ACCESS (dr_stmt))
7325 : {
7326 451148 : partition.layout = -1;
7327 3673367 : continue;
7328 : }
7329 186820 : dr_stmt = DR_GROUP_FIRST_ELEMENT (dr_stmt);
7330 186820 : imin = DR_GROUP_SIZE (dr_stmt) + 1;
7331 186820 : tmp_perm.safe_splice (SLP_TREE_LOAD_PERMUTATION (node));
7332 : }
7333 5995158 : else if (SLP_TREE_PERMUTE_P (node)
7334 139950 : && SLP_TREE_CHILDREN (node).length () == 1
7335 120156 : && (child = SLP_TREE_CHILDREN (node)[0])
7336 3197607 : && (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (child))
7337 120156 : .is_constant (&imin)))
7338 : {
7339 : /* If the child has the same vector size as this node,
7340 : reversing the permutation can make the permutation a no-op.
7341 : In other cases it can change a true permutation into a
7342 : full-vector extract. */
7343 120156 : tmp_perm.reserve (SLP_TREE_LANES (node));
7344 444876 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7345 204564 : tmp_perm.quick_push (SLP_TREE_LANE_PERMUTATION (node)[j].second);
7346 : }
7347 : else
7348 2937501 : continue;
7349 :
7350 822730 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7351 : {
7352 515754 : unsigned idx = tmp_perm[j];
7353 515754 : imin = MIN (imin, idx);
7354 515754 : imax = MAX (imax, idx);
7355 515754 : if (idx - tmp_perm[0] != j)
7356 164212 : any_permute = true;
7357 : }
7358 : /* If the span doesn't match we'd disrupt VF computation, avoid
7359 : that for now. */
7360 306976 : if (imax - imin + 1 != SLP_TREE_LANES (node))
7361 91740 : continue;
7362 : /* If there's no permute no need to split one out. In this case
7363 : we can consider turning a load into a permuted load, if that
7364 : turns out to be cheaper than alternatives. */
7365 215236 : if (!any_permute)
7366 : {
7367 192822 : partition.layout = -1;
7368 192822 : continue;
7369 : }
7370 :
7371 : /* For now only handle true permutes, like
7372 : vect_attempt_slp_rearrange_stmts did. This allows us to be lazy
7373 : when permuting constants and invariants keeping the permute
7374 : bijective. */
7375 22414 : auto_sbitmap load_index (SLP_TREE_LANES (node));
7376 22414 : bitmap_clear (load_index);
7377 112521 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7378 67693 : bitmap_set_bit (load_index, tmp_perm[j] - imin);
7379 : unsigned j;
7380 89261 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
7381 67003 : if (!bitmap_bit_p (load_index, j))
7382 : break;
7383 22414 : if (j != SLP_TREE_LANES (node))
7384 156 : continue;
7385 :
7386 22258 : vec<unsigned> perm = vNULL;
7387 22258 : perm.safe_grow (SLP_TREE_LANES (node), true);
7388 111238 : for (unsigned j = 0; j < SLP_TREE_LANES (node); ++j)
7389 66722 : perm[j] = tmp_perm[j] - imin;
7390 :
7391 44516 : if (int (m_perms.length ()) >= param_vect_max_layout_candidates)
7392 : {
7393 : /* Continue to use existing layouts, but don't add any more. */
7394 0 : int *entry = layout_ids.get (perm);
7395 0 : partition.layout = entry ? *entry : 0;
7396 0 : perm.release ();
7397 : }
7398 : else
7399 : {
7400 22258 : bool existed;
7401 22258 : int &layout_i = layout_ids.get_or_insert (perm, &existed);
7402 22258 : if (existed)
7403 7891 : perm.release ();
7404 : else
7405 : {
7406 14367 : layout_i = m_perms.length ();
7407 14367 : m_perms.safe_push (perm);
7408 : }
7409 22258 : partition.layout = layout_i;
7410 : }
7411 22414 : }
7412 :
7413 : /* Initially assume that every layout is possible and has zero cost
7414 : in every partition. */
7415 721496 : m_partition_layout_costs.safe_grow_cleared (m_partitions.length ()
7416 1442992 : * m_perms.length ());
7417 :
7418 : /* We have to mark outgoing permutations facing non-associating-reduction
7419 : graph entries that are not represented as to be materialized.
7420 : slp_inst_kind_bb_reduc currently only covers associatable reductions. */
7421 3711106 : for (slp_instance instance : m_vinfo->slp_instances)
7422 1546618 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor)
7423 : {
7424 7293 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7425 7293 : m_partitions[m_vertices[node_i].partition].layout = 0;
7426 : }
7427 1539325 : else if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_reduc_chain)
7428 : {
7429 2351 : stmt_vec_info stmt_info
7430 2351 : = SLP_TREE_REPRESENTATIVE (SLP_INSTANCE_TREE (instance));
7431 2351 : vect_reduc_info reduc_info
7432 2351 : = info_for_reduction (as_a <loop_vec_info> (m_vinfo),
7433 : SLP_INSTANCE_TREE (instance));
7434 2351 : if (needs_fold_left_reduction_p (TREE_TYPE
7435 : (gimple_get_lhs (stmt_info->stmt)),
7436 : VECT_REDUC_INFO_CODE (reduc_info)))
7437 : {
7438 124 : unsigned int node_i = SLP_INSTANCE_TREE (instance)->vertex;
7439 124 : m_partitions[m_vertices[node_i].partition].layout = 0;
7440 : }
7441 : }
7442 :
7443 : /* Check which layouts each node and partition can handle. Calculate the
7444 : weights associated with inserting layout changes on edges. */
7445 5860113 : for (unsigned int node_i : m_partitioned_nodes)
7446 : {
7447 3695625 : auto &vertex = m_vertices[node_i];
7448 3695625 : auto &partition = m_partitions[vertex.partition];
7449 3695625 : slp_tree node = vertex.node;
7450 :
7451 3695625 : vertex.weight = vect_slp_node_weight (m_vinfo, node);
7452 :
7453 3695625 : if (stmt_vec_info rep = SLP_TREE_REPRESENTATIVE (node))
7454 : {
7455 : /* We do not handle stores with a permutation, so all
7456 : incoming permutations must have been materialized.
7457 :
7458 : We also don't handle masked grouped loads, which lack a
7459 : permutation vector. In this case the memory locations
7460 : form an implicit second input to the loads, on top of the
7461 : explicit mask input, and the memory input's layout cannot
7462 : be changed.
7463 :
7464 : On the other hand, we do support permuting gather loads and
7465 : masked gather loads, where each scalar load is independent
7466 : of the others. This can be useful if the address/index input
7467 : benefits from permutation. */
7468 3552526 : if (STMT_VINFO_DATA_REF (rep)
7469 1710641 : && STMT_VINFO_GROUPED_ACCESS (rep)
7470 4559565 : && !SLP_TREE_LOAD_PERMUTATION (node).exists ())
7471 820219 : partition.layout = 0;
7472 :
7473 : /* We cannot change the layout of an operation that is
7474 : not independent on lanes. Note this is an explicit
7475 : negative list since that's much shorter than the respective
7476 : positive one but it's critical to keep maintaining it. */
7477 3552526 : if (is_gimple_call (STMT_VINFO_STMT (rep)))
7478 32990 : switch (gimple_call_combined_fn (STMT_VINFO_STMT (rep)))
7479 : {
7480 1098 : case CFN_COMPLEX_ADD_ROT90:
7481 1098 : case CFN_COMPLEX_ADD_ROT270:
7482 1098 : case CFN_COMPLEX_MUL:
7483 1098 : case CFN_COMPLEX_MUL_CONJ:
7484 1098 : case CFN_VEC_ADDSUB:
7485 1098 : case CFN_VEC_FMADDSUB:
7486 1098 : case CFN_VEC_FMSUBADD:
7487 1098 : partition.layout = 0;
7488 : default:;
7489 : }
7490 : }
7491 :
7492 8347227 : auto process_edge = [&](graph_edge *ud, unsigned int other_node_i)
7493 : {
7494 4651602 : auto &other_vertex = m_vertices[other_node_i];
7495 :
7496 : /* Count the number of edges from earlier partitions and the number
7497 : of edges to later partitions. */
7498 4651602 : if (other_vertex.partition < vertex.partition)
7499 2325801 : partition.in_degree += 1;
7500 : else
7501 2325801 : partition.out_degree += 1;
7502 :
7503 : /* If the current node uses the result of OTHER_NODE_I, accumulate
7504 : the effects of that. */
7505 4651602 : if (ud->src == int (node_i))
7506 : {
7507 2325801 : other_vertex.out_weight += vertex.weight;
7508 2325801 : other_vertex.out_degree += 1;
7509 : }
7510 8347227 : };
7511 3695625 : for_each_partition_edge (node_i, process_edge);
7512 : }
7513 721496 : }
7514 :
7515 : /* Return the incoming costs for node NODE_I, assuming that each input keeps
7516 : its current (provisional) choice of layout. The inputs do not necessarily
7517 : have the same layout as each other. */
7518 :
7519 : slpg_layout_cost
7520 4333 : vect_optimize_slp_pass::total_in_cost (unsigned int node_i)
7521 : {
7522 4333 : auto &vertex = m_vertices[node_i];
7523 4333 : slpg_layout_cost cost;
7524 14405 : auto add_cost = [&](graph_edge *, unsigned int other_node_i)
7525 : {
7526 10072 : auto &other_vertex = m_vertices[other_node_i];
7527 10072 : if (other_vertex.partition < vertex.partition)
7528 : {
7529 6726 : auto &other_partition = m_partitions[other_vertex.partition];
7530 13452 : auto &other_costs = partition_layout_costs (other_vertex.partition,
7531 6726 : other_partition.layout);
7532 6726 : slpg_layout_cost this_cost = other_costs.in_cost;
7533 6726 : this_cost.add_serial_cost (other_costs.internal_cost);
7534 6726 : this_cost.split (other_partition.out_degree);
7535 6726 : cost.add_parallel_cost (this_cost);
7536 : }
7537 14405 : };
7538 4333 : for_each_partition_edge (node_i, add_cost);
7539 4333 : return cost;
7540 : }
7541 :
7542 : /* Return the cost of switching between layout LAYOUT1_I (at node NODE1_I)
7543 : and layout LAYOUT2_I on cross-partition use-to-def edge UD. Return
7544 : slpg_layout_cost::impossible () if the change isn't possible. */
7545 :
7546 : slpg_layout_cost
7547 745918 : vect_optimize_slp_pass::
7548 : edge_layout_cost (graph_edge *ud, unsigned int node1_i, unsigned int layout1_i,
7549 : unsigned int layout2_i)
7550 : {
7551 745918 : auto &def_vertex = m_vertices[ud->dest];
7552 745918 : auto &use_vertex = m_vertices[ud->src];
7553 745918 : auto def_layout_i = ud->dest == int (node1_i) ? layout1_i : layout2_i;
7554 745918 : auto use_layout_i = ud->dest == int (node1_i) ? layout2_i : layout1_i;
7555 745918 : auto factor = change_layout_cost (def_vertex.node, def_layout_i,
7556 : use_layout_i);
7557 745918 : if (factor < 0)
7558 6078 : return slpg_layout_cost::impossible ();
7559 :
7560 : /* We have a choice of putting the layout change at the site of the
7561 : definition or at the site of the use. Prefer the former when
7562 : optimizing for size or when the execution frequency of the
7563 : definition is no greater than the combined execution frequencies of
7564 : the uses. When putting the layout change at the site of the definition,
7565 : divvy up the cost among all consumers. */
7566 739840 : if (m_optimize_size || def_vertex.weight <= def_vertex.out_weight)
7567 : {
7568 697132 : slpg_layout_cost cost = { def_vertex.weight * factor, m_optimize_size };
7569 697132 : cost.split (def_vertex.out_degree);
7570 697132 : return cost;
7571 : }
7572 42708 : return { use_vertex.weight * factor, m_optimize_size };
7573 : }
7574 :
7575 : /* UD represents a use-def link between FROM_NODE_I and a node in a later
7576 : partition; FROM_NODE_I could be the definition node or the use node.
7577 : The node at the other end of the link wants to use layout TO_LAYOUT_I.
7578 : Return the cost of any necessary fix-ups on edge UD, or return
7579 : slpg_layout_cost::impossible () if the change isn't possible.
7580 :
7581 : At this point, FROM_NODE_I's partition has chosen the cheapest
7582 : layout based on the information available so far, but this choice
7583 : is only provisional. */
7584 :
7585 : slpg_layout_cost
7586 199250 : vect_optimize_slp_pass::forward_cost (graph_edge *ud, unsigned int from_node_i,
7587 : unsigned int to_layout_i)
7588 : {
7589 199250 : auto &from_vertex = m_vertices[from_node_i];
7590 199250 : unsigned int from_partition_i = from_vertex.partition;
7591 199250 : slpg_partition_info &from_partition = m_partitions[from_partition_i];
7592 199250 : gcc_assert (from_partition.layout >= 0);
7593 :
7594 : /* First calculate the cost on the assumption that FROM_PARTITION sticks
7595 : with its current layout preference. */
7596 199250 : slpg_layout_cost cost = slpg_layout_cost::impossible ();
7597 199250 : auto edge_cost = edge_layout_cost (ud, from_node_i,
7598 199250 : from_partition.layout, to_layout_i);
7599 199250 : if (edge_cost.is_possible ())
7600 : {
7601 392084 : auto &from_costs = partition_layout_costs (from_partition_i,
7602 196042 : from_partition.layout);
7603 196042 : cost = from_costs.in_cost;
7604 196042 : cost.add_serial_cost (from_costs.internal_cost);
7605 196042 : cost.split (from_partition.out_degree);
7606 196042 : cost.add_serial_cost (edge_cost);
7607 : }
7608 3208 : else if (from_partition.layout == 0)
7609 : /* We must allow the source partition to have layout 0 as a fallback,
7610 : in case all other options turn out to be impossible. */
7611 3208 : return cost;
7612 :
7613 : /* Take the minimum of that cost and the cost that applies if
7614 : FROM_PARTITION instead switches to TO_LAYOUT_I. */
7615 196042 : auto &direct_layout_costs = partition_layout_costs (from_partition_i,
7616 : to_layout_i);
7617 196042 : if (direct_layout_costs.is_possible ())
7618 : {
7619 171997 : slpg_layout_cost direct_cost = direct_layout_costs.in_cost;
7620 171997 : direct_cost.add_serial_cost (direct_layout_costs.internal_cost);
7621 171997 : direct_cost.split (from_partition.out_degree);
7622 171997 : if (!cost.is_possible ()
7623 171997 : || direct_cost.is_better_than (cost, m_optimize_size))
7624 34133 : cost = direct_cost;
7625 : }
7626 :
7627 196042 : return cost;
7628 : }
7629 :
7630 : /* UD represents a use-def link between TO_NODE_I and a node in an earlier
7631 : partition; TO_NODE_I could be the definition node or the use node.
7632 : The node at the other end of the link wants to use layout FROM_LAYOUT_I;
7633 : return the cost of any necessary fix-ups on edge UD, or
7634 : slpg_layout_cost::impossible () if the choice cannot be made.
7635 :
7636 : At this point, TO_NODE_I's partition has a fixed choice of layout. */
7637 :
7638 : slpg_layout_cost
7639 178936 : vect_optimize_slp_pass::backward_cost (graph_edge *ud, unsigned int to_node_i,
7640 : unsigned int from_layout_i)
7641 : {
7642 178936 : auto &to_vertex = m_vertices[to_node_i];
7643 178936 : unsigned int to_partition_i = to_vertex.partition;
7644 178936 : slpg_partition_info &to_partition = m_partitions[to_partition_i];
7645 178936 : gcc_assert (to_partition.layout >= 0);
7646 :
7647 : /* If TO_NODE_I is a VEC_PERM_EXPR consumer, see whether it can be
7648 : adjusted for this input having layout FROM_LAYOUT_I. Assume that
7649 : any other inputs keep their current choice of layout. */
7650 178936 : auto &to_costs = partition_layout_costs (to_partition_i,
7651 : to_partition.layout);
7652 178936 : if (ud->src == int (to_node_i)
7653 178654 : && SLP_TREE_PERMUTE_P (to_vertex.node))
7654 : {
7655 11390 : auto &from_partition = m_partitions[m_vertices[ud->dest].partition];
7656 11390 : auto old_layout = from_partition.layout;
7657 11390 : from_partition.layout = from_layout_i;
7658 22780 : int factor = internal_node_cost (to_vertex.node, -1,
7659 11390 : to_partition.layout);
7660 11390 : from_partition.layout = old_layout;
7661 11390 : if (factor >= 0)
7662 : {
7663 10489 : slpg_layout_cost cost = to_costs.out_cost;
7664 10489 : cost.add_serial_cost ({ to_vertex.weight * factor,
7665 : m_optimize_size });
7666 10489 : cost.split (to_partition.in_degree);
7667 10489 : return cost;
7668 : }
7669 : }
7670 :
7671 : /* Compute the cost if we insert any necessary layout change on edge UD. */
7672 168447 : auto edge_cost = edge_layout_cost (ud, to_node_i,
7673 168447 : to_partition.layout, from_layout_i);
7674 168447 : if (edge_cost.is_possible ())
7675 : {
7676 168447 : slpg_layout_cost cost = to_costs.out_cost;
7677 168447 : cost.add_serial_cost (to_costs.internal_cost);
7678 168447 : cost.split (to_partition.in_degree);
7679 168447 : cost.add_serial_cost (edge_cost);
7680 168447 : return cost;
7681 : }
7682 :
7683 0 : return slpg_layout_cost::impossible ();
7684 : }
7685 :
7686 : /* Make a forward pass through the partitions, accumulating input costs.
7687 : Make a tentative (provisional) choice of layout for each partition,
7688 : ensuring that this choice still allows later partitions to keep
7689 : their original layout. */
7690 :
7691 : void
7692 6578 : vect_optimize_slp_pass::forward_pass ()
7693 : {
7694 135507 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
7695 : ++partition_i)
7696 : {
7697 128929 : auto &partition = m_partitions[partition_i];
7698 :
7699 : /* If the partition consists of a single VEC_PERM_EXPR, precompute
7700 : the incoming cost that would apply if every predecessor partition
7701 : keeps its current layout. This is used within the loop below. */
7702 128929 : slpg_layout_cost in_cost;
7703 128929 : slp_tree single_node = nullptr;
7704 128929 : if (partition.node_end == partition.node_begin + 1)
7705 : {
7706 122512 : unsigned int node_i = m_partitioned_nodes[partition.node_begin];
7707 122512 : single_node = m_vertices[node_i].node;
7708 122512 : if (SLP_TREE_PERMUTE_P (single_node))
7709 4333 : in_cost = total_in_cost (node_i);
7710 : }
7711 :
7712 : /* Go through the possible layouts. Decide which ones are valid
7713 : for this partition and record which of the valid layouts has
7714 : the lowest cost. */
7715 128929 : unsigned int min_layout_i = 0;
7716 128929 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7717 397253 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7718 : {
7719 268324 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7720 268324 : if (!layout_costs.is_possible ())
7721 69687 : continue;
7722 :
7723 : /* If the recorded layout is already 0 then the layout cannot
7724 : change. */
7725 268324 : if (partition.layout == 0 && layout_i != 0)
7726 : {
7727 48225 : layout_costs.mark_impossible ();
7728 48225 : continue;
7729 : }
7730 :
7731 220099 : bool is_possible = true;
7732 448238 : for (unsigned int order_i = partition.node_begin;
7733 448238 : order_i < partition.node_end; ++order_i)
7734 : {
7735 246706 : unsigned int node_i = m_partitioned_nodes[order_i];
7736 246706 : auto &vertex = m_vertices[node_i];
7737 :
7738 : /* Reject the layout if it is individually incompatible
7739 : with any node in the partition. */
7740 246706 : if (!is_compatible_layout (vertex.node, layout_i))
7741 : {
7742 17057 : is_possible = false;
7743 18567 : break;
7744 : }
7745 :
7746 611561 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7747 : {
7748 381912 : auto &other_vertex = m_vertices[other_node_i];
7749 381912 : if (other_vertex.partition < vertex.partition)
7750 : {
7751 : /* Accumulate the incoming costs from earlier
7752 : partitions, plus the cost of any layout changes
7753 : on UD itself. */
7754 199250 : auto cost = forward_cost (ud, other_node_i, layout_i);
7755 199250 : if (!cost.is_possible ())
7756 3208 : is_possible = false;
7757 : else
7758 196042 : layout_costs.in_cost.add_parallel_cost (cost);
7759 : }
7760 : else
7761 : /* Reject the layout if it would make layout 0 impossible
7762 : for later partitions. This amounts to testing that the
7763 : target supports reversing the layout change on edges
7764 : to later partitions.
7765 :
7766 : In principle, it might be possible to push a layout
7767 : change all the way down a graph, so that it never
7768 : needs to be reversed and so that the target doesn't
7769 : need to support the reverse operation. But it would
7770 : be awkward to bail out if we hit a partition that
7771 : does not support the new layout, especially since
7772 : we are not dealing with a lattice. */
7773 182662 : is_possible &= edge_layout_cost (ud, other_node_i, 0,
7774 182662 : layout_i).is_possible ();
7775 611561 : };
7776 229649 : for_each_partition_edge (node_i, add_cost);
7777 :
7778 : /* Accumulate the cost of using LAYOUT_I within NODE,
7779 : both for the inputs and the outputs. */
7780 229649 : int factor = internal_node_cost (vertex.node, layout_i,
7781 : layout_i);
7782 229649 : if (factor < 0)
7783 : {
7784 1510 : is_possible = false;
7785 1510 : break;
7786 : }
7787 228139 : else if (factor)
7788 36235 : layout_costs.internal_cost.add_serial_cost
7789 36235 : ({ vertex.weight * factor, m_optimize_size });
7790 : }
7791 220099 : if (!is_possible)
7792 : {
7793 21462 : layout_costs.mark_impossible ();
7794 21462 : continue;
7795 : }
7796 :
7797 : /* Combine the incoming and partition-internal costs. */
7798 198637 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7799 198637 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7800 :
7801 : /* If this partition consists of a single VEC_PERM_EXPR, see
7802 : if the VEC_PERM_EXPR can be changed to support output layout
7803 : LAYOUT_I while keeping all the provisional choices of input
7804 : layout. */
7805 198637 : if (single_node && SLP_TREE_PERMUTE_P (single_node))
7806 : {
7807 7519 : int factor = internal_node_cost (single_node, -1, layout_i);
7808 7519 : if (factor >= 0)
7809 : {
7810 6799 : auto weight = m_vertices[single_node->vertex].weight;
7811 6799 : slpg_layout_cost internal_cost
7812 6799 : = { weight * factor, m_optimize_size };
7813 :
7814 6799 : slpg_layout_cost alt_cost = in_cost;
7815 6799 : alt_cost.add_serial_cost (internal_cost);
7816 6799 : if (alt_cost.is_better_than (combined_cost, m_optimize_size))
7817 : {
7818 1636 : combined_cost = alt_cost;
7819 1636 : layout_costs.in_cost = in_cost;
7820 1636 : layout_costs.internal_cost = internal_cost;
7821 : }
7822 : }
7823 : }
7824 :
7825 : /* Record the layout with the lowest cost. Prefer layout 0 in
7826 : the event of a tie between it and another layout. */
7827 198637 : if (!min_layout_cost.is_possible ()
7828 69708 : || combined_cost.is_better_than (min_layout_cost,
7829 : m_optimize_size))
7830 : {
7831 146391 : min_layout_i = layout_i;
7832 146391 : min_layout_cost = combined_cost;
7833 : }
7834 : }
7835 :
7836 : /* This loop's handling of earlier partitions should ensure that
7837 : choosing the original layout for the current partition is no
7838 : less valid than it was in the original graph, even with the
7839 : provisional layout choices for those earlier partitions. */
7840 128929 : gcc_assert (min_layout_cost.is_possible ());
7841 128929 : partition.layout = min_layout_i;
7842 : }
7843 6578 : }
7844 :
7845 : /* Make a backward pass through the partitions, accumulating output costs.
7846 : Make a final choice of layout for each partition. */
7847 :
7848 : void
7849 6578 : vect_optimize_slp_pass::backward_pass ()
7850 : {
7851 142085 : for (unsigned int partition_i = m_partitions.length (); partition_i-- > 0;)
7852 : {
7853 128929 : auto &partition = m_partitions[partition_i];
7854 :
7855 128929 : unsigned int min_layout_i = 0;
7856 128929 : slpg_layout_cost min_layout_cost = slpg_layout_cost::impossible ();
7857 397253 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
7858 : {
7859 268324 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
7860 268324 : if (!layout_costs.is_possible ())
7861 69687 : continue;
7862 :
7863 : /* Accumulate the costs from successor partitions. */
7864 198637 : bool is_possible = true;
7865 423856 : for (unsigned int order_i = partition.node_begin;
7866 423856 : order_i < partition.node_end; ++order_i)
7867 : {
7868 225219 : unsigned int node_i = m_partitioned_nodes[order_i];
7869 225219 : auto &vertex = m_vertices[node_i];
7870 599714 : auto add_cost = [&](graph_edge *ud, unsigned int other_node_i)
7871 : {
7872 374495 : auto &other_vertex = m_vertices[other_node_i];
7873 374495 : auto &other_partition = m_partitions[other_vertex.partition];
7874 374495 : if (other_vertex.partition > vertex.partition)
7875 : {
7876 : /* Accumulate the incoming costs from later
7877 : partitions, plus the cost of any layout changes
7878 : on UD itself. */
7879 178936 : auto cost = backward_cost (ud, other_node_i, layout_i);
7880 178936 : if (!cost.is_possible ())
7881 0 : is_possible = false;
7882 : else
7883 178936 : layout_costs.out_cost.add_parallel_cost (cost);
7884 : }
7885 : else
7886 : /* Make sure that earlier partitions can (if necessary
7887 : or beneficial) keep the layout that they chose in
7888 : the forward pass. This ensures that there is at
7889 : least one valid choice of layout. */
7890 195559 : is_possible &= edge_layout_cost (ud, other_node_i,
7891 195559 : other_partition.layout,
7892 195559 : layout_i).is_possible ();
7893 599714 : };
7894 225219 : for_each_partition_edge (node_i, add_cost);
7895 : }
7896 198637 : if (!is_possible)
7897 : {
7898 0 : layout_costs.mark_impossible ();
7899 0 : continue;
7900 : }
7901 :
7902 : /* Locally combine the costs from the forward and backward passes.
7903 : (This combined cost is not passed on, since that would lead
7904 : to double counting.) */
7905 198637 : slpg_layout_cost combined_cost = layout_costs.in_cost;
7906 198637 : combined_cost.add_serial_cost (layout_costs.internal_cost);
7907 198637 : combined_cost.add_serial_cost (layout_costs.out_cost);
7908 :
7909 : /* Record the layout with the lowest cost. Prefer layout 0 in
7910 : the event of a tie between it and another layout. */
7911 198637 : if (!min_layout_cost.is_possible ()
7912 69708 : || combined_cost.is_better_than (min_layout_cost,
7913 : m_optimize_size))
7914 : {
7915 140033 : min_layout_i = layout_i;
7916 140033 : min_layout_cost = combined_cost;
7917 : }
7918 : }
7919 :
7920 128929 : gcc_assert (min_layout_cost.is_possible ());
7921 128929 : partition.layout = min_layout_i;
7922 : }
7923 6578 : }
7924 :
7925 : /* Return a node that applies layout TO_LAYOUT_I to the original form of NODE.
7926 : NODE already has the layout that was selected for its partition. */
7927 :
7928 : slp_tree
7929 180717 : vect_optimize_slp_pass::get_result_with_layout (slp_tree node,
7930 : unsigned int to_layout_i)
7931 : {
7932 180717 : unsigned int result_i = node->vertex * m_perms.length () + to_layout_i;
7933 180717 : slp_tree result = m_node_layouts[result_i];
7934 180717 : if (result)
7935 : return result;
7936 :
7937 179800 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
7938 179800 : || (SLP_TREE_DEF_TYPE (node) == vect_external_def
7939 : /* We can't permute vector defs in place. */
7940 22161 : && SLP_TREE_VEC_DEFS (node).is_empty ()))
7941 : {
7942 : /* If the vector is uniform or unchanged, there's nothing to do. */
7943 47031 : if (to_layout_i == 0 || vect_slp_tree_uniform_p (node))
7944 : result = node;
7945 : else
7946 : {
7947 3681 : auto scalar_ops = SLP_TREE_SCALAR_OPS (node).copy ();
7948 3681 : result = vect_create_new_slp_node (scalar_ops);
7949 3681 : vect_slp_permute (m_perms[to_layout_i], scalar_ops, true);
7950 : }
7951 : }
7952 : else
7953 : {
7954 132769 : unsigned int partition_i = m_vertices[node->vertex].partition;
7955 132769 : unsigned int from_layout_i = m_partitions[partition_i].layout;
7956 132769 : if (from_layout_i == to_layout_i)
7957 131552 : return node;
7958 :
7959 : /* If NODE is itself a VEC_PERM_EXPR, try to create a parallel
7960 : permutation instead of a serial one. Leave the new permutation
7961 : in TMP_PERM on success. */
7962 1217 : auto_lane_permutation_t tmp_perm;
7963 1217 : unsigned int num_inputs = 1;
7964 1217 : if (SLP_TREE_PERMUTE_P (node))
7965 : {
7966 103 : tmp_perm.safe_splice (SLP_TREE_LANE_PERMUTATION (node));
7967 103 : if (from_layout_i != 0)
7968 43 : vect_slp_permute (m_perms[from_layout_i], tmp_perm, false);
7969 103 : if (to_layout_i != 0)
7970 64 : vect_slp_permute (m_perms[to_layout_i], tmp_perm, true);
7971 103 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
7972 : tmp_perm,
7973 103 : SLP_TREE_CHILDREN (node),
7974 : false) >= 0)
7975 103 : num_inputs = SLP_TREE_CHILDREN (node).length ();
7976 : else
7977 0 : tmp_perm.truncate (0);
7978 : }
7979 :
7980 1217 : if (dump_enabled_p ())
7981 : {
7982 71 : if (tmp_perm.length () > 0)
7983 6 : dump_printf_loc (MSG_NOTE, vect_location,
7984 : "duplicating permutation node %p with"
7985 : " layout %d\n",
7986 : (void *) node, to_layout_i);
7987 : else
7988 65 : dump_printf_loc (MSG_NOTE, vect_location,
7989 : "inserting permutation node in place of %p\n",
7990 : (void *) node);
7991 : }
7992 :
7993 1217 : unsigned int num_lanes = SLP_TREE_LANES (node);
7994 1217 : result = vect_create_new_slp_node (num_inputs, VEC_PERM_EXPR);
7995 1217 : if (SLP_TREE_SCALAR_STMTS (node).exists ())
7996 : {
7997 1217 : auto &stmts = SLP_TREE_SCALAR_STMTS (result);
7998 1217 : stmts.safe_splice (SLP_TREE_SCALAR_STMTS (node));
7999 1217 : if (from_layout_i != 0)
8000 500 : vect_slp_permute (m_perms[from_layout_i], stmts, false);
8001 1217 : if (to_layout_i != 0)
8002 721 : vect_slp_permute (m_perms[to_layout_i], stmts, true);
8003 : }
8004 1217 : SLP_TREE_LANES (result) = num_lanes;
8005 1217 : SLP_TREE_VECTYPE (result) = SLP_TREE_VECTYPE (node);
8006 1217 : result->vertex = -1;
8007 :
8008 1217 : auto &lane_perm = SLP_TREE_LANE_PERMUTATION (result);
8009 1217 : if (tmp_perm.length ())
8010 : {
8011 103 : lane_perm.safe_splice (tmp_perm);
8012 103 : SLP_TREE_CHILDREN (result).safe_splice (SLP_TREE_CHILDREN (node));
8013 : }
8014 : else
8015 : {
8016 1114 : lane_perm.create (num_lanes);
8017 4538 : for (unsigned j = 0; j < num_lanes; ++j)
8018 2310 : lane_perm.quick_push ({ 0, j });
8019 1114 : if (from_layout_i != 0)
8020 457 : vect_slp_permute (m_perms[from_layout_i], lane_perm, false);
8021 1114 : if (to_layout_i != 0)
8022 657 : vect_slp_permute (m_perms[to_layout_i], lane_perm, true);
8023 1114 : SLP_TREE_CHILDREN (result).safe_push (node);
8024 : }
8025 4940 : for (slp_tree child : SLP_TREE_CHILDREN (result))
8026 1289 : child->refcnt++;
8027 1217 : }
8028 48248 : m_node_layouts[result_i] = result;
8029 48248 : return result;
8030 : }
8031 :
8032 : /* Apply the chosen vector layouts to the SLP graph. */
8033 :
8034 : void
8035 13799 : vect_optimize_slp_pass::materialize ()
8036 : {
8037 : /* We no longer need the costs, so avoid having two O(N * P) arrays
8038 : live at the same time. */
8039 13799 : m_partition_layout_costs.release ();
8040 41397 : m_node_layouts.safe_grow_cleared (m_vertices.length () * m_perms.length ());
8041 :
8042 27598 : auto_sbitmap fully_folded (m_vertices.length ());
8043 13799 : bitmap_clear (fully_folded);
8044 200868 : for (unsigned int node_i : m_partitioned_nodes)
8045 : {
8046 159471 : auto &vertex = m_vertices[node_i];
8047 159471 : slp_tree node = vertex.node;
8048 159471 : int layout_i = m_partitions[vertex.partition].layout;
8049 159471 : gcc_assert (layout_i >= 0);
8050 :
8051 : /* Rearrange the scalar statements to match the chosen layout. */
8052 159471 : if (layout_i > 0)
8053 26688 : vect_slp_permute (m_perms[layout_i],
8054 26688 : SLP_TREE_SCALAR_STMTS (node), true);
8055 :
8056 : /* Update load and lane permutations. */
8057 159471 : if (SLP_TREE_PERMUTE_P (node))
8058 : {
8059 : /* First try to absorb the input vector layouts. If that fails,
8060 : force the inputs to have layout LAYOUT_I too. We checked that
8061 : that was possible before deciding to use nonzero output layouts.
8062 : (Note that at this stage we don't really have any guarantee that
8063 : the target supports the original VEC_PERM_EXPR.) */
8064 6532 : auto &perm = SLP_TREE_LANE_PERMUTATION (node);
8065 6532 : auto_lane_permutation_t tmp_perm;
8066 6532 : tmp_perm.safe_splice (perm);
8067 6532 : change_vec_perm_layout (node, tmp_perm, -1, layout_i);
8068 6532 : if (vectorizable_slp_permutation_1 (m_vinfo, nullptr, node,
8069 : tmp_perm,
8070 6532 : SLP_TREE_CHILDREN (node),
8071 : false) >= 0)
8072 : {
8073 5894 : if (dump_enabled_p ()
8074 6822 : && !std::equal (tmp_perm.begin (), tmp_perm.end (),
8075 : perm.begin ()))
8076 58 : dump_printf_loc (MSG_NOTE, vect_location,
8077 : "absorbing input layouts into %p\n",
8078 : (void *) node);
8079 33335 : std::copy (tmp_perm.begin (), tmp_perm.end (), perm.begin ());
8080 5894 : bitmap_set_bit (fully_folded, node_i);
8081 : }
8082 : else
8083 : {
8084 : /* Not MSG_MISSED because it would make no sense to users. */
8085 638 : if (dump_enabled_p ())
8086 46 : dump_printf_loc (MSG_NOTE, vect_location,
8087 : "failed to absorb input layouts into %p\n",
8088 : (void *) node);
8089 638 : change_vec_perm_layout (nullptr, perm, layout_i, layout_i);
8090 : }
8091 6532 : }
8092 : else
8093 : {
8094 152939 : gcc_assert (!SLP_TREE_LANE_PERMUTATION (node).exists ());
8095 152939 : auto &load_perm = SLP_TREE_LOAD_PERMUTATION (node);
8096 152939 : if (layout_i > 0)
8097 : /* ??? When we handle non-bijective permutes the idea
8098 : is that we can force the load-permutation to be
8099 : { min, min + 1, min + 2, ... max }. But then the
8100 : scalar defs might no longer match the lane content
8101 : which means wrong-code with live lane vectorization.
8102 : So we possibly have to have NULL entries for those. */
8103 25878 : vect_slp_permute (m_perms[layout_i], load_perm, true);
8104 : }
8105 : }
8106 :
8107 : /* Do this before any nodes disappear, since it involves a walk
8108 : over the leaves. */
8109 13799 : remove_redundant_permutations ();
8110 :
8111 : /* Replace each child with a correctly laid-out version. */
8112 200868 : for (unsigned int node_i : m_partitioned_nodes)
8113 : {
8114 : /* Skip nodes that have already been handled above. */
8115 159471 : if (bitmap_bit_p (fully_folded, node_i))
8116 5894 : continue;
8117 :
8118 153577 : auto &vertex = m_vertices[node_i];
8119 153577 : int in_layout_i = m_partitions[vertex.partition].layout;
8120 153577 : gcc_assert (in_layout_i >= 0);
8121 :
8122 : unsigned j;
8123 : slp_tree child;
8124 457054 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (vertex.node), j, child)
8125 : {
8126 186767 : if (!child)
8127 6050 : continue;
8128 :
8129 180717 : slp_tree new_child = get_result_with_layout (child, in_layout_i);
8130 180717 : if (new_child != child)
8131 : {
8132 5482 : vect_free_slp_tree (child);
8133 5482 : SLP_TREE_CHILDREN (vertex.node)[j] = new_child;
8134 5482 : new_child->refcnt += 1;
8135 : }
8136 : }
8137 : }
8138 13799 : }
8139 :
8140 : /* Elide load permutations that are not necessary. Such permutations might
8141 : be pre-existing, rather than created by the layout optimizations. */
8142 :
8143 : void
8144 721496 : vect_optimize_slp_pass::remove_redundant_permutations ()
8145 : {
8146 4778214 : for (unsigned int node_i : m_leafs)
8147 : {
8148 2613726 : slp_tree node = m_vertices[node_i].node;
8149 2613726 : if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
8150 1975758 : continue;
8151 :
8152 : /* In basic block vectorization we allow any subchain of an interleaving
8153 : chain.
8154 : FORNOW: not in loop SLP because of realignment complications. */
8155 637968 : if (is_a <bb_vec_info> (m_vinfo))
8156 : {
8157 191723 : bool subchain_p = true;
8158 : stmt_vec_info next_load_info = NULL;
8159 : stmt_vec_info load_info;
8160 : unsigned j;
8161 191723 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), j, load_info)
8162 : {
8163 158222 : if (j != 0
8164 158222 : && (next_load_info != load_info
8165 70240 : || ! load_info
8166 70240 : || DR_GROUP_GAP (load_info) != 1))
8167 : {
8168 : subchain_p = false;
8169 : break;
8170 : }
8171 126892 : next_load_info = DR_GROUP_NEXT_ELEMENT (load_info);
8172 : }
8173 64831 : if (subchain_p)
8174 : {
8175 33501 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8176 33501 : continue;
8177 : }
8178 : }
8179 : else
8180 : {
8181 573137 : loop_vec_info loop_vinfo = as_a<loop_vec_info> (m_vinfo);
8182 573137 : bool this_load_permuted = !vect_load_perm_consecutive_p (node, 0);
8183 : /* When this isn't a grouped access we know it's single element
8184 : and contiguous. */
8185 573137 : if (!STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (node)[0]))
8186 : {
8187 451148 : if (!this_load_permuted
8188 451148 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8189 450375 : || SLP_TREE_LANES (node) == 1))
8190 450387 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8191 451148 : continue;
8192 : }
8193 121989 : stmt_vec_info first_stmt_info
8194 121989 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (node)[0]);
8195 122522 : if (!this_load_permuted
8196 : /* The load requires permutation when unrolling exposes
8197 : a gap either because the group is larger than the SLP
8198 : group-size or because there is a gap between the groups. */
8199 121989 : && (known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U)
8200 99301 : || ((SLP_TREE_LANES (node) == DR_GROUP_SIZE (first_stmt_info))
8201 147 : && DR_GROUP_GAP (first_stmt_info) == 0)))
8202 : {
8203 533 : SLP_TREE_LOAD_PERMUTATION (node).release ();
8204 533 : continue;
8205 : }
8206 : }
8207 : }
8208 721496 : }
8209 :
8210 : /* Print the partition graph and layout information to the dump file. */
8211 :
8212 : void
8213 691 : vect_optimize_slp_pass::dump ()
8214 : {
8215 691 : dump_printf_loc (MSG_NOTE, vect_location,
8216 : "SLP optimize permutations:\n");
8217 1395 : for (unsigned int layout_i = 1; layout_i < m_perms.length (); ++layout_i)
8218 : {
8219 704 : dump_printf_loc (MSG_NOTE, vect_location, " %d: { ", layout_i);
8220 704 : const char *sep = "";
8221 5976 : for (unsigned int idx : m_perms[layout_i])
8222 : {
8223 3864 : dump_printf (MSG_NOTE, "%s%d", sep, idx);
8224 3864 : sep = ", ";
8225 : }
8226 704 : dump_printf (MSG_NOTE, " }\n");
8227 : }
8228 691 : dump_printf_loc (MSG_NOTE, vect_location,
8229 : "SLP optimize partitions:\n");
8230 5996 : for (unsigned int partition_i = 0; partition_i < m_partitions.length ();
8231 : ++partition_i)
8232 : {
8233 5305 : auto &partition = m_partitions[partition_i];
8234 5305 : dump_printf_loc (MSG_NOTE, vect_location, " -------------\n");
8235 5305 : dump_printf_loc (MSG_NOTE, vect_location,
8236 : " partition %d (layout %d):\n",
8237 : partition_i, partition.layout);
8238 5305 : dump_printf_loc (MSG_NOTE, vect_location, " nodes:\n");
8239 10849 : for (unsigned int order_i = partition.node_begin;
8240 10849 : order_i < partition.node_end; ++order_i)
8241 : {
8242 5544 : auto &vertex = m_vertices[m_partitioned_nodes[order_i]];
8243 11088 : dump_printf_loc (MSG_NOTE, vect_location, " - %p:\n",
8244 5544 : (void *) vertex.node);
8245 5544 : dump_printf_loc (MSG_NOTE, vect_location,
8246 : " weight: %f\n",
8247 : vertex.weight.to_double ());
8248 5544 : if (vertex.out_degree)
8249 4344 : dump_printf_loc (MSG_NOTE, vect_location,
8250 : " out weight: %f (degree %d)\n",
8251 : vertex.out_weight.to_double (),
8252 : vertex.out_degree);
8253 5544 : if (SLP_TREE_PERMUTE_P (vertex.node))
8254 510 : dump_printf_loc (MSG_NOTE, vect_location,
8255 : " op: VEC_PERM_EXPR\n");
8256 5034 : else if (auto rep = SLP_TREE_REPRESENTATIVE (vertex.node))
8257 5016 : dump_printf_loc (MSG_NOTE, vect_location,
8258 : " op template: %G", rep->stmt);
8259 : }
8260 5305 : dump_printf_loc (MSG_NOTE, vect_location, " edges:\n");
8261 10849 : for (unsigned int order_i = partition.node_begin;
8262 10849 : order_i < partition.node_end; ++order_i)
8263 : {
8264 5544 : unsigned int node_i = m_partitioned_nodes[order_i];
8265 5544 : auto &vertex = m_vertices[node_i];
8266 16782 : auto print_edge = [&](graph_edge *, unsigned int other_node_i)
8267 : {
8268 11238 : auto &other_vertex = m_vertices[other_node_i];
8269 11238 : if (other_vertex.partition < vertex.partition)
8270 5619 : dump_printf_loc (MSG_NOTE, vect_location,
8271 : " - %p [%d] --> %p\n",
8272 5619 : (void *) other_vertex.node,
8273 : other_vertex.partition,
8274 5619 : (void *) vertex.node);
8275 : else
8276 5619 : dump_printf_loc (MSG_NOTE, vect_location,
8277 : " - %p --> [%d] %p\n",
8278 5619 : (void *) vertex.node,
8279 : other_vertex.partition,
8280 5619 : (void *) other_vertex.node);
8281 16782 : };
8282 5544 : for_each_partition_edge (node_i, print_edge);
8283 : }
8284 :
8285 16114 : for (unsigned int layout_i = 0; layout_i < m_perms.length (); ++layout_i)
8286 : {
8287 10809 : auto &layout_costs = partition_layout_costs (partition_i, layout_i);
8288 10809 : if (layout_costs.is_possible ())
8289 : {
8290 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8291 : " layout %d:%s\n", layout_i,
8292 8952 : partition.layout == int (layout_i)
8293 : ? " (*)" : "");
8294 8952 : slpg_layout_cost combined_cost = layout_costs.in_cost;
8295 8952 : combined_cost.add_serial_cost (layout_costs.internal_cost);
8296 8952 : combined_cost.add_serial_cost (layout_costs.out_cost);
8297 : #define TEMPLATE "{depth: %f, total: %f}"
8298 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8299 : " " TEMPLATE "\n",
8300 : layout_costs.in_cost.depth.to_double (),
8301 : layout_costs.in_cost.total.to_double ());
8302 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8303 : " + " TEMPLATE "\n",
8304 : layout_costs.internal_cost.depth.to_double (),
8305 : layout_costs.internal_cost.total.to_double ());
8306 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8307 : " + " TEMPLATE "\n",
8308 : layout_costs.out_cost.depth.to_double (),
8309 : layout_costs.out_cost.total.to_double ());
8310 8952 : dump_printf_loc (MSG_NOTE, vect_location,
8311 : " = " TEMPLATE "\n",
8312 : combined_cost.depth.to_double (),
8313 : combined_cost.total.to_double ());
8314 : #undef TEMPLATE
8315 : }
8316 : else
8317 1857 : dump_printf_loc (MSG_NOTE, vect_location,
8318 : " layout %d: rejected\n", layout_i);
8319 : }
8320 : }
8321 691 : }
8322 :
8323 : /* Masked load lanes discovery. */
8324 :
8325 : void
8326 721496 : vect_optimize_slp_pass::decide_masked_load_lanes ()
8327 : {
8328 7435589 : for (auto v : m_vertices)
8329 : {
8330 5271101 : slp_tree node = v.node;
8331 5271101 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8332 3693602 : || SLP_TREE_PERMUTE_P (node))
8333 1718575 : continue;
8334 3552526 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
8335 1710641 : if (! STMT_VINFO_GROUPED_ACCESS (stmt_info)
8336 : /* The mask has to be uniform. */
8337 1007039 : || STMT_VINFO_SLP_VECT_ONLY (stmt_info)
8338 1006961 : || ! is_a <gcall *> (STMT_VINFO_STMT (stmt_info))
8339 3552611 : || ! gimple_call_internal_p (STMT_VINFO_STMT (stmt_info),
8340 : IFN_MASK_LOAD))
8341 3552493 : continue;
8342 33 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
8343 66 : if (STMT_VINFO_STRIDED_P (stmt_info)
8344 33 : || compare_step_with_zero (m_vinfo, stmt_info) <= 0
8345 63 : || vect_load_lanes_supported (SLP_TREE_VECTYPE (node),
8346 30 : DR_GROUP_SIZE (stmt_info),
8347 : true) == IFN_LAST)
8348 33 : continue;
8349 :
8350 : /* Uniform masks need to be suitably represented. */
8351 0 : slp_tree mask = SLP_TREE_CHILDREN (node)[0];
8352 0 : if (!SLP_TREE_PERMUTE_P (mask)
8353 0 : || SLP_TREE_CHILDREN (mask).length () != 1)
8354 0 : continue;
8355 0 : bool match = true;
8356 0 : for (auto perm : SLP_TREE_LANE_PERMUTATION (mask))
8357 0 : if (perm.first != 0 || perm.second != 0)
8358 : {
8359 : match = false;
8360 : break;
8361 : }
8362 0 : if (!match)
8363 0 : continue;
8364 :
8365 : /* Now see if the consumer side matches. */
8366 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8367 0 : pred; pred = pred->pred_next)
8368 : {
8369 0 : slp_tree pred_node = m_vertices[pred->src].node;
8370 : /* All consumers should be a permute with a single outgoing lane. */
8371 0 : if (!SLP_TREE_PERMUTE_P (pred_node)
8372 0 : || SLP_TREE_LANES (pred_node) != 1)
8373 : {
8374 : match = false;
8375 : break;
8376 : }
8377 0 : gcc_assert (SLP_TREE_CHILDREN (pred_node).length () == 1);
8378 : }
8379 0 : if (!match)
8380 0 : continue;
8381 : /* Now we can mark the nodes as to use load lanes. */
8382 0 : node->ldst_lanes = true;
8383 0 : for (graph_edge *pred = m_slpg->vertices[node->vertex].pred;
8384 0 : pred; pred = pred->pred_next)
8385 0 : m_vertices[pred->src].node->ldst_lanes = true;
8386 : /* The catch is we have to massage the mask. We have arranged
8387 : analyzed uniform masks to be represented by a splat VEC_PERM
8388 : which we can now simply elide as we cannot easily re-do SLP
8389 : discovery here. */
8390 0 : slp_tree new_mask = SLP_TREE_CHILDREN (mask)[0];
8391 0 : SLP_TREE_REF_COUNT (new_mask)++;
8392 0 : SLP_TREE_CHILDREN (node)[0] = new_mask;
8393 0 : vect_free_slp_tree (mask);
8394 : }
8395 721496 : }
8396 :
8397 : /* Perform legitimizing attempts. This is intended to improve the
8398 : situation when layout 0 is not valid which is a situation the cost
8399 : based propagation does not handle well.
8400 : Return true if further layout optimization is possible, false if
8401 : the layout configuration should be considered final. */
8402 :
8403 : bool
8404 13799 : vect_optimize_slp_pass::legitimize ()
8405 : {
8406 : /* Perform a very simple legitimizing attempt by attempting to choose
8407 : a single layout for all partitions that will make all permutations
8408 : a noop. That should also be the optimal layout choice in case
8409 : layout zero is legitimate.
8410 : ??? Disconnected components of the SLP graph could have distinct
8411 : single layouts. */
8412 13799 : int single_layout_i = -1;
8413 13799 : unsigned deferred_up_to = -1U;
8414 42650 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8415 : ++partition_i)
8416 : {
8417 35412 : auto &partition = m_partitions[partition_i];
8418 35412 : if (single_layout_i == -1)
8419 : {
8420 17489 : single_layout_i = partition.layout;
8421 17489 : deferred_up_to = partition_i;
8422 : }
8423 17923 : else if (partition.layout == single_layout_i || partition.layout == -1)
8424 : ;
8425 : else
8426 : single_layout_i = 0;
8427 32188 : if (single_layout_i == 0)
8428 : return true;
8429 :
8430 28933 : if (single_layout_i != -1
8431 28933 : && !is_compatible_layout (partition, single_layout_i))
8432 : return true;
8433 : }
8434 :
8435 7238 : if (single_layout_i <= 0)
8436 : return true;
8437 :
8438 7827 : for (unsigned partition_i = 0; partition_i < deferred_up_to; ++partition_i)
8439 606 : if (!is_compatible_layout (m_partitions[partition_i],
8440 : single_layout_i))
8441 : return true;
8442 :
8443 22192 : for (unsigned partition_i = 0; partition_i < m_partitions.length ();
8444 : ++partition_i)
8445 : {
8446 14971 : auto &partition = m_partitions[partition_i];
8447 14971 : partition.layout = single_layout_i;
8448 : }
8449 :
8450 : return false;
8451 : }
8452 :
8453 : /* Main entry point for the SLP graph optimization pass. */
8454 :
8455 : void
8456 721496 : vect_optimize_slp_pass::run ()
8457 : {
8458 721496 : build_graph ();
8459 721496 : create_partitions ();
8460 721496 : start_choosing_layouts ();
8461 721496 : if (m_perms.length () > 1)
8462 : {
8463 13799 : if (legitimize ())
8464 : {
8465 6578 : forward_pass ();
8466 6578 : backward_pass ();
8467 : }
8468 13799 : if (dump_enabled_p ())
8469 691 : dump ();
8470 13799 : materialize ();
8471 55764 : while (!m_perms.is_empty ())
8472 28166 : m_perms.pop ().release ();
8473 : }
8474 : else
8475 707697 : remove_redundant_permutations ();
8476 721496 : free_graph (m_slpg);
8477 721496 : build_graph ();
8478 721496 : decide_masked_load_lanes ();
8479 721496 : free_graph (m_slpg);
8480 721496 : }
8481 :
8482 : /* Apply CSE to NODE and its children using BST_MAP. */
8483 :
8484 : static void
8485 5691384 : vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree& node)
8486 : {
8487 5691384 : bool put_p = false;
8488 5691384 : if (SLP_TREE_DEF_TYPE (node) == vect_internal_def
8489 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8490 : lack scalar stmts and thus CSE doesn't work via bst_map. Ideally
8491 : we'd have sth that works for all internal and external nodes. */
8492 5691384 : && !SLP_TREE_SCALAR_STMTS (node).is_empty ())
8493 : {
8494 4085633 : slp_tree *leader = bst_map->get (SLP_TREE_SCALAR_STMTS (node));
8495 4085633 : if (leader)
8496 : {
8497 : /* We've visited this node already. */
8498 423531 : if (!*leader || *leader == node)
8499 : return;
8500 :
8501 4363 : if (dump_enabled_p ())
8502 919 : dump_printf_loc (MSG_NOTE, vect_location,
8503 : "re-using SLP tree %p for %p\n",
8504 : (void *)*leader, (void *)node);
8505 4363 : vect_free_slp_tree (node);
8506 4363 : (*leader)->refcnt += 1;
8507 4363 : node = *leader;
8508 4363 : return;
8509 : }
8510 :
8511 : /* Avoid creating a cycle by populating the map only after recursion. */
8512 3662102 : bst_map->put (SLP_TREE_SCALAR_STMTS (node).copy (), nullptr);
8513 3662102 : node->refcnt += 1;
8514 3662102 : put_p = true;
8515 : /* And recurse. */
8516 : }
8517 :
8518 15736908 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8519 4607217 : if (child)
8520 4144766 : vect_cse_slp_nodes (bst_map, child);
8521 :
8522 : /* Now record the node for CSE in other siblings. */
8523 5267853 : if (put_p)
8524 3662102 : *bst_map->get (SLP_TREE_SCALAR_STMTS (node)) = node;
8525 : }
8526 :
8527 : /* Associate stmts with possible starts of a subset of lanes of NODE
8528 : in PART_STARTS. */
8529 :
8530 : static void
8531 2159509 : vect_cse_gather_part_starts (hash_set<slp_tree> &visited,
8532 : vec<vec<slp_tree>> part_starts, slp_tree node)
8533 : {
8534 : /* CSEing external nodes complicates scheduling since we materialize
8535 : those at the latest position, so avoid that. */
8536 2159509 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8537 2159509 : || visited.add (node))
8538 : return;
8539 :
8540 : /* Besides some VEC_PERM_EXPR, two-operator nodes also lack scalar stmts
8541 : and thus CSE doesn't work. For now gather two-lane aligned starts
8542 : of nodes with a multiple of two number of lanes. */
8543 1160446 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8544 1153038 : && SLP_TREE_LANES (node) > 2
8545 151575 : && (SLP_TREE_LANES (node) & 1) == 0)
8546 : {
8547 151575 : auto_vec<unsigned, 8> uids;
8548 629509 : for (unsigned i = 0; i < SLP_TREE_LANES (node); i += 2)
8549 : {
8550 477934 : stmt_vec_info s = SLP_TREE_SCALAR_STMTS (node)[i];
8551 477934 : if (!s)
8552 0 : continue;
8553 477934 : unsigned uid = gimple_uid (s->stmt);
8554 477934 : if (!uids.contains (uid))
8555 : {
8556 473920 : uids.safe_push (uid);
8557 473920 : part_starts[uid].safe_push (node);
8558 : }
8559 : }
8560 151575 : }
8561 :
8562 4516328 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8563 1325088 : if (child)
8564 1325088 : vect_cse_gather_part_starts (visited, part_starts, child);
8565 : }
8566 :
8567 : /* Apply CSE to NODE and its children using lowparts of nodes in BST_MAP. */
8568 :
8569 : static void
8570 2155842 : vect_cse_slp_node_parts (hash_set<slp_tree> &visited,
8571 : const vec<vec<slp_tree>> part_starts,
8572 : vec<slp_tree> &drops, slp_tree node)
8573 : {
8574 2155842 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
8575 2155842 : || visited.add (node))
8576 : return;
8577 :
8578 : /* Besides some VEC_PERM_EXPR, two-operator nodes also
8579 : lack scalar stmts and thus CSE doesn't work. */
8580 1158762 : unsigned HOST_WIDE_INT c;
8581 1158762 : if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
8582 1151358 : && SLP_TREE_SCALAR_STMTS (node)[0]
8583 : /* Avoid touching loads which need care with load permutations
8584 : and specialities like load-lane representations. */
8585 1151358 : && (SLP_TREE_PERMUTE_P (node)
8586 1143807 : || !STMT_VINFO_DATA_REF (SLP_TREE_REPRESENTATIVE (node))))
8587 37495 : for (slp_tree cand
8588 433895 : : part_starts[gimple_uid (SLP_TREE_SCALAR_STMTS (node)[0]->stmt)])
8589 : /* ??? There is a possible ordering/optimality problem in that
8590 : the CSE then can keep a wider feeding live even though it itself
8591 : becomes dead by means of CSE. Which might be solvable by doing
8592 : the CSE in a wide-to-narrow order. */
8593 37960 : if (SLP_TREE_LANES (cand) > SLP_TREE_LANES (node)
8594 : /* We can do high/lo extracts and full vector copies. */
8595 : && constant_multiple_p
8596 4617 : (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (cand)),
8597 38452 : TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)), &c)
8598 42577 : && c <= 2)
8599 : {
8600 4125 : unsigned HOST_WIDE_INT s;
8601 4125 : bool const_p
8602 4125 : = TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (node)).is_constant (&s);
8603 4125 : unsigned i;
8604 14817 : for (i = 0; i <= SLP_TREE_LANES (cand) - SLP_TREE_LANES (node);)
8605 : {
8606 : unsigned j;
8607 15780 : for (j = 0; j < SLP_TREE_LANES (node); ++j)
8608 15315 : if (!SLP_TREE_SCALAR_STMTS (node)[j]
8609 15315 : || (SLP_TREE_SCALAR_STMTS (cand)[i+j]
8610 15315 : != SLP_TREE_SCALAR_STMTS (node)[j]))
8611 : break;
8612 11157 : if (j == SLP_TREE_LANES (node))
8613 : break;
8614 10692 : if (!const_p)
8615 : {
8616 : i = SLP_TREE_LANES (cand);
8617 : break;
8618 : }
8619 : /* We can extract only aligned on node vector type boundary. */
8620 10692 : i += s;
8621 : }
8622 4125 : if (i > SLP_TREE_LANES (cand) - SLP_TREE_LANES (node))
8623 3660 : continue;
8624 : /* Found node within cand at i. Put a permute in place
8625 : of it, selecting the subset from cand. */
8626 465 : if (dump_enabled_p ())
8627 41 : dump_printf (MSG_NOTE, "CSEd node %p as %spart of node %p\n",
8628 : (void *)node, i == 0 ? "low" : "high", (void *)cand);
8629 2290 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8630 : /* Delay SLP tree release since we might still reference a node
8631 : from the part_starts map. */
8632 895 : drops.safe_push (child);
8633 465 : SLP_TREE_CHILDREN (node).truncate (1);
8634 465 : SLP_TREE_REF_COUNT (cand)++;
8635 465 : SLP_TREE_CHILDREN (node)[0] = cand;
8636 465 : SLP_TREE_CODE (node) = VEC_PERM_EXPR;
8637 465 : SLP_TREE_REPRESENTATIVE (node) = NULL;
8638 465 : SLP_TREE_LANE_PERMUTATION (node).create (SLP_TREE_LANES (node));
8639 1992 : for (unsigned j = i; j < i + SLP_TREE_LANES (node); ++j)
8640 1062 : SLP_TREE_LANE_PERMUTATION (node).quick_push (std::make_pair (0, j));
8641 2155842 : return;
8642 : }
8643 :
8644 4506780 : for (slp_tree &child : SLP_TREE_CHILDREN (node))
8645 1321421 : if (child)
8646 1321421 : vect_cse_slp_node_parts (visited, part_starts, drops, child);
8647 : }
8648 :
8649 : /* Optimize the SLP graph of VINFO. */
8650 :
8651 : void
8652 1117270 : vect_optimize_slp (vec_info *vinfo)
8653 : {
8654 1117270 : if (vinfo->slp_instances.is_empty ())
8655 834156 : return;
8656 721496 : vect_optimize_slp_pass (vinfo).run ();
8657 :
8658 : /* Apply CSE again to nodes after permute optimization. */
8659 721496 : scalar_stmts_to_slp_tree_map_t *bst_map
8660 721496 : = new scalar_stmts_to_slp_tree_map_t ();
8661 :
8662 3711106 : for (auto inst : vinfo->slp_instances)
8663 1546618 : vect_cse_slp_nodes (bst_map, SLP_INSTANCE_TREE (inst));
8664 :
8665 721496 : release_scalar_stmts_to_slp_tree_map (bst_map);
8666 :
8667 721496 : if (!is_a <bb_vec_info> (vinfo))
8668 : return;
8669 :
8670 : /* Attempt to merge SLP sub-graphs that intersect in low or highparts of
8671 : each other. Build the reverse mapping from stmt to SLP node for
8672 : lanes starting at the low or high part.
8673 : ??? In the future we can extend this to do a two-step permute
8674 : and extract or extract and permute to put the high/low part in
8675 : place on the original vector or permute the hogh/low part to
8676 : match up the target lane order. */
8677 283114 : hash_set<slp_tree> visited;
8678 283114 : vec<vec<slp_tree>> start_for_part;
8679 566228 : start_for_part.create (vinfo->stmt_vec_infos.length () + 1);
8680 566228 : start_for_part.quick_grow_cleared (vinfo->stmt_vec_infos.length () + 1);
8681 1683763 : for (auto inst : vinfo->slp_instances)
8682 834421 : vect_cse_gather_part_starts (visited,
8683 : start_for_part, SLP_INSTANCE_TREE (inst));
8684 :
8685 : /* Now replace low/highpart copies with extracting permutes. */
8686 283114 : auto_vec<slp_tree> drops;
8687 283114 : visited.empty ();
8688 1683763 : for (auto inst : vinfo->slp_instances)
8689 834421 : vect_cse_slp_node_parts (visited, start_for_part, drops,
8690 : SLP_INSTANCE_TREE (inst));
8691 :
8692 : /* Now perform delayed releases of nodes. */
8693 284499 : for (slp_tree node : drops)
8694 895 : vect_free_slp_tree (node);
8695 :
8696 24404581 : for (auto v : start_for_part)
8697 23555239 : v.release ();
8698 283114 : start_for_part.release ();
8699 283114 : }
8700 :
8701 : /* Gather loads reachable from the individual SLP graph entries. */
8702 :
8703 : void
8704 1117270 : vect_gather_slp_loads (vec_info *vinfo)
8705 : {
8706 1117270 : unsigned i;
8707 1117270 : slp_instance instance;
8708 2663888 : FOR_EACH_VEC_ELT (vinfo->slp_instances, i, instance)
8709 : {
8710 1546618 : hash_set<slp_tree> visited;
8711 1546618 : vect_gather_slp_loads (SLP_INSTANCE_LOADS (instance),
8712 : SLP_INSTANCE_TREE (instance), visited);
8713 1546618 : }
8714 1117270 : }
8715 :
8716 : /* For NODE update VF based on the number of lanes and the vector types
8717 : used. */
8718 :
8719 : static void
8720 4525353 : vect_update_slp_vf_for_node (slp_tree node, poly_uint64 &vf,
8721 : hash_set<slp_tree> &visited)
8722 : {
8723 4525353 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
8724 1640294 : return;
8725 3262619 : if (visited.add (node))
8726 : return;
8727 :
8728 10978813 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8729 3718226 : vect_update_slp_vf_for_node (child, vf, visited);
8730 :
8731 : /* We do not visit SLP nodes for constants or externals - those neither
8732 : have a vector type set yet (vectorizable_* does this).
8733 : Note that when we stop using fixed size vectors externs and constants
8734 : shouldn't influence the (minimum) vectorization factor, instead
8735 : vectorizable_* should honor the vectorization factor when trying to
8736 : assign vector types to constants and externals and cause iteration
8737 : to a higher vectorization factor when required. */
8738 2885079 : tree vectype = SLP_TREE_VECTYPE (node);
8739 2885079 : if (!vectype)
8740 : /* OMP SIMD calls w/o LHS have no SLP_TREE_VECTYPE set. */
8741 : return;
8742 2885059 : poly_uint64 node_vf
8743 2885059 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8744 : SLP_TREE_LANES (node));
8745 2885059 : vf = force_common_multiple (vf, node_vf);
8746 :
8747 : /* For permute nodes that are fed from externs or constants we have to
8748 : consider their number of lanes as well. Likewise for store-lanes. */
8749 2885059 : if (SLP_TREE_PERMUTE_P (node) || node->ldst_lanes)
8750 717405 : for (slp_tree child : SLP_TREE_CHILDREN (node))
8751 193362 : if (SLP_TREE_DEF_TYPE (child) != vect_internal_def)
8752 : {
8753 3680 : poly_uint64 child_vf
8754 3680 : = calculate_unrolling_factor (TYPE_VECTOR_SUBPARTS (vectype),
8755 : SLP_TREE_LANES (child));
8756 3680 : vf = force_common_multiple (vf, child_vf);
8757 : }
8758 : }
8759 :
8760 : /* For each possible SLP instance decide whether to SLP it and calculate overall
8761 : unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at
8762 : least one instance. */
8763 :
8764 : bool
8765 505784 : vect_make_slp_decision (loop_vec_info loop_vinfo)
8766 : {
8767 505784 : unsigned int i;
8768 505784 : poly_uint64 unrolling_factor = 1;
8769 505784 : const vec<slp_instance> &slp_instances
8770 : = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
8771 505784 : slp_instance instance;
8772 505784 : int decided_to_slp = 0;
8773 :
8774 505784 : DUMP_VECT_SCOPE ("vect_make_slp_decision");
8775 :
8776 505784 : hash_set<slp_tree> visited;
8777 1818695 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
8778 : {
8779 807127 : slp_tree root = SLP_INSTANCE_TREE (instance);
8780 :
8781 : /* All unroll factors have the form:
8782 :
8783 : GET_MODE_SIZE (vinfo->vector_mode) * X
8784 :
8785 : for some rational X, so they must have a common multiple. */
8786 807127 : vect_update_slp_vf_for_node (root, unrolling_factor, visited);
8787 :
8788 : /* If all instances ended up with vector(1) T roots make sure to
8789 : not vectorize. RVV for example relies on loop vectorization
8790 : when some instances are essentially kept scalar. See PR121048. */
8791 807127 : if (SLP_TREE_VECTYPE (root)
8792 807127 : && known_gt (TYPE_VECTOR_SUBPARTS (SLP_TREE_VECTYPE (root)), 1U))
8793 654644 : decided_to_slp++;
8794 : }
8795 :
8796 505784 : LOOP_VINFO_VECT_FACTOR (loop_vinfo) = unrolling_factor;
8797 :
8798 505784 : if (decided_to_slp && dump_enabled_p ())
8799 : {
8800 20403 : dump_printf_loc (MSG_NOTE, vect_location,
8801 : "Decided to SLP %d instances. Unrolling factor ",
8802 : decided_to_slp);
8803 20403 : dump_dec (MSG_NOTE, unrolling_factor);
8804 20403 : dump_printf (MSG_NOTE, "\n");
8805 : }
8806 :
8807 505784 : return (decided_to_slp > 0);
8808 505784 : }
8809 :
8810 : /* Initialize a bb_vec_info struct for the statements in BBS basic blocks. */
8811 :
8812 2349498 : _bb_vec_info::_bb_vec_info (vec<basic_block> _bbs, vec_info_shared *shared)
8813 : : vec_info (vec_info::bb, shared),
8814 2349498 : roots (vNULL)
8815 : {
8816 : /* The region we are operating on. bbs[0] is the entry, excluding
8817 : its PHI nodes. In the future we might want to track an explicit
8818 : entry edge to cover bbs[0] PHI nodes and have a region entry
8819 : insert location. */
8820 2349498 : bbs = _bbs.address ();
8821 2349498 : nbbs = _bbs.length ();
8822 :
8823 18748341 : for (unsigned i = 0; i < nbbs; ++i)
8824 : {
8825 16398843 : if (i != 0)
8826 21304247 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8827 7254902 : gsi_next (&si))
8828 : {
8829 7254902 : gphi *phi = si.phi ();
8830 7254902 : gimple_set_uid (phi, 0);
8831 7254902 : add_stmt (phi);
8832 : }
8833 32797686 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8834 152524186 : !gsi_end_p (gsi); gsi_next (&gsi))
8835 : {
8836 136125343 : gimple *stmt = gsi_stmt (gsi);
8837 136125343 : gimple_set_uid (stmt, 0);
8838 136125343 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
8839 88685003 : continue;
8840 47440340 : add_stmt (stmt);
8841 : }
8842 : }
8843 2349498 : }
8844 :
8845 :
8846 : /* Free BB_VINFO struct, as well as all the stmt_vec_info structs of all the
8847 : stmts in the basic block. */
8848 :
8849 2349498 : _bb_vec_info::~_bb_vec_info ()
8850 : {
8851 : /* Reset region marker. */
8852 18748341 : for (unsigned i = 0; i < nbbs; ++i)
8853 : {
8854 16398843 : if (i != 0)
8855 21318364 : for (gphi_iterator si = gsi_start_phis (bbs[i]); !gsi_end_p (si);
8856 7269019 : gsi_next (&si))
8857 : {
8858 7269019 : gphi *phi = si.phi ();
8859 7269019 : gimple_set_uid (phi, -1);
8860 : }
8861 32797686 : for (gimple_stmt_iterator gsi = gsi_start_bb (bbs[i]);
8862 152396513 : !gsi_end_p (gsi); gsi_next (&gsi))
8863 : {
8864 135997670 : gimple *stmt = gsi_stmt (gsi);
8865 135997670 : gimple_set_uid (stmt, -1);
8866 : }
8867 : }
8868 :
8869 3722324 : for (unsigned i = 0; i < roots.length (); ++i)
8870 : {
8871 1372826 : roots[i].stmts.release ();
8872 1372826 : roots[i].roots.release ();
8873 1372826 : roots[i].remain.release ();
8874 : }
8875 2349498 : roots.release ();
8876 2349498 : }
8877 :
8878 : /* Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE,
8879 : given then that child nodes have already been processed, and that
8880 : their def types currently match their SLP node's def type. */
8881 :
8882 : static bool
8883 2743751 : vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node,
8884 : slp_instance node_instance,
8885 : stmt_vector_for_cost *cost_vec)
8886 : {
8887 : /* Handle purely internal nodes. */
8888 2743751 : if (SLP_TREE_PERMUTE_P (node))
8889 : {
8890 82924 : gcc_checking_assert (!SLP_TREE_REPRESENTATIVE (node));
8891 82924 : if (!vectorizable_slp_permutation (vinfo, NULL, node, cost_vec))
8892 : return false;
8893 :
8894 : stmt_vec_info slp_stmt_info;
8895 : unsigned int i;
8896 204201 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, slp_stmt_info)
8897 : {
8898 122869 : if (slp_stmt_info
8899 117167 : && STMT_VINFO_LIVE_P (slp_stmt_info)
8900 122869 : && !vectorizable_live_operation (vinfo, slp_stmt_info, node,
8901 : node_instance, i,
8902 : false, cost_vec))
8903 : return false;
8904 : }
8905 81332 : SLP_TREE_TYPE (node) = permute_info_type;
8906 81332 : return true;
8907 : }
8908 :
8909 2660827 : return vect_analyze_stmt (vinfo, node, node_instance, cost_vec);
8910 : }
8911 :
8912 : static int
8913 912283 : sort_ints (const void *a_, const void *b_)
8914 : {
8915 912283 : int a = *(const int *)a_;
8916 912283 : int b = *(const int *)b_;
8917 912283 : return a - b;
8918 : }
8919 :
8920 : /* Verify if we can externalize a set of internal defs. */
8921 :
8922 : static bool
8923 411208 : vect_slp_can_convert_to_external (const vec<stmt_vec_info> &stmts)
8924 : {
8925 : /* Constant generation uses get_later_stmt which can only handle
8926 : defs from the same BB or a set of defs that can be ordered
8927 : with a dominance query. */
8928 411208 : basic_block bb = NULL;
8929 411208 : bool all_same = true;
8930 411208 : auto_vec<int> bbs;
8931 822416 : bbs.reserve_exact (stmts.length ());
8932 2168634 : for (stmt_vec_info stmt : stmts)
8933 : {
8934 935010 : if (!stmt)
8935 : return false;
8936 935010 : else if (!bb)
8937 411208 : bb = gimple_bb (stmt->stmt);
8938 523802 : else if (gimple_bb (stmt->stmt) != bb)
8939 157117 : all_same = false;
8940 935010 : bbs.quick_push (gimple_bb (stmt->stmt)->index);
8941 : }
8942 411208 : if (all_same)
8943 : return true;
8944 :
8945 : /* Produce a vector of unique BB indexes for the defs. */
8946 140194 : bbs.qsort (sort_ints);
8947 140194 : unsigned i, j;
8948 303536 : for (i = 1, j = 1; i < bbs.length (); ++i)
8949 163342 : if (bbs[i] != bbs[j-1])
8950 144097 : bbs[j++] = bbs[i];
8951 140194 : gcc_assert (j >= 2);
8952 140194 : bbs.truncate (j);
8953 :
8954 280388 : if (bbs.length () == 2)
8955 138502 : return (dominated_by_p (CDI_DOMINATORS,
8956 138502 : BASIC_BLOCK_FOR_FN (cfun, bbs[0]),
8957 138502 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]))
8958 270580 : || dominated_by_p (CDI_DOMINATORS,
8959 132078 : BASIC_BLOCK_FOR_FN (cfun, bbs[1]),
8960 132078 : BASIC_BLOCK_FOR_FN (cfun, bbs[0])));
8961 :
8962 : /* ??? For more than two BBs we can sort the vector and verify the
8963 : result is a total order. But we can't use vec::qsort with a
8964 : compare function using a dominance query since there's no way to
8965 : signal failure and any fallback for an unordered pair would
8966 : fail qsort_chk later.
8967 : For now simply hope that ordering after BB index provides the
8968 : best candidate total order. If required we can implement our
8969 : own mergesort or export an entry without checking. */
8970 418437 : for (unsigned i = 1; i < bbs.length (); ++i)
8971 5571 : if (!dominated_by_p (CDI_DOMINATORS,
8972 5571 : BASIC_BLOCK_FOR_FN (cfun, bbs[i]),
8973 5571 : BASIC_BLOCK_FOR_FN (cfun, bbs[i-1])))
8974 : return false;
8975 :
8976 : return true;
8977 411208 : }
8978 :
8979 : /* Try to build NODE from scalars, returning true on success.
8980 : NODE_INSTANCE is the SLP instance that contains NODE. */
8981 :
8982 : static bool
8983 645966 : vect_slp_convert_to_external (vec_info *vinfo, slp_tree node,
8984 : slp_instance node_instance)
8985 : {
8986 645966 : stmt_vec_info stmt_info;
8987 645966 : unsigned int i;
8988 :
8989 645966 : if (!is_a <bb_vec_info> (vinfo)
8990 90174 : || node == SLP_INSTANCE_TREE (node_instance)
8991 30243 : || !SLP_TREE_SCALAR_STMTS (node).exists ()
8992 30202 : || vect_contains_pattern_stmt_p (SLP_TREE_SCALAR_STMTS (node))
8993 : /* Force the mask use to be built from scalars instead. */
8994 22731 : || VECTOR_BOOLEAN_TYPE_P (SLP_TREE_VECTYPE (node))
8995 668438 : || !vect_slp_can_convert_to_external (SLP_TREE_SCALAR_STMTS (node)))
8996 : return false;
8997 :
8998 22472 : if (dump_enabled_p ())
8999 84 : dump_printf_loc (MSG_NOTE, vect_location,
9000 : "Building vector operands of %p from scalars instead\n",
9001 : (void *) node);
9002 :
9003 : /* Don't remove and free the child nodes here, since they could be
9004 : referenced by other structures. The analysis and scheduling phases
9005 : (need to) ignore child nodes of anything that isn't vect_internal_def. */
9006 22472 : unsigned int group_size = SLP_TREE_LANES (node);
9007 22472 : SLP_TREE_DEF_TYPE (node) = vect_external_def;
9008 : /* Invariants get their vector type from the uses. */
9009 22472 : SLP_TREE_VECTYPE (node) = NULL_TREE;
9010 22472 : SLP_TREE_SCALAR_OPS (node).safe_grow (group_size, true);
9011 22472 : SLP_TREE_LOAD_PERMUTATION (node).release ();
9012 101984 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9013 : {
9014 57040 : tree lhs = gimple_get_lhs (vect_orig_stmt (stmt_info)->stmt);
9015 57040 : SLP_TREE_SCALAR_OPS (node)[i] = lhs;
9016 : }
9017 : return true;
9018 : }
9019 :
9020 : /* Return true if all elements of the slice are the same. */
9021 : bool
9022 451629 : vect_scalar_ops_slice::all_same_p () const
9023 : {
9024 503553 : for (unsigned int i = 1; i < length; ++i)
9025 416167 : if (!operand_equal_p (op (0), op (i)))
9026 : return false;
9027 : return true;
9028 : }
9029 :
9030 : hashval_t
9031 426730 : vect_scalar_ops_slice_hash::hash (const value_type &s)
9032 : {
9033 426730 : hashval_t hash = 0;
9034 1631983 : for (unsigned i = 0; i < s.length; ++i)
9035 1205253 : hash = iterative_hash_expr (s.op (i), hash);
9036 426730 : return hash;
9037 : }
9038 :
9039 : bool
9040 232721 : vect_scalar_ops_slice_hash::equal (const value_type &s1,
9041 : const compare_type &s2)
9042 : {
9043 232721 : if (s1.length != s2.length)
9044 : return false;
9045 400790 : for (unsigned i = 0; i < s1.length; ++i)
9046 349798 : if (!operand_equal_p (s1.op (i), s2.op (i)))
9047 : return false;
9048 : return true;
9049 : }
9050 :
9051 : /* Like vect_get_num_copies but N copies of the vector might have
9052 : excess elements in the last vector. Stores the number of excess
9053 : elements in the last vector in *EXCESS_ELTS.
9054 : Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed. */
9055 :
9056 : bool
9057 1605472 : vect_get_num_copies_for_invariant (vec_info *vinfo, slp_tree node,
9058 : unsigned *nvectors,
9059 : unsigned *excess_elts)
9060 : {
9061 1605472 : poly_uint64 vf;
9062 :
9063 1605472 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
9064 382351 : vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
9065 : else
9066 1223121 : vf = 1;
9067 1605472 : vf *= SLP_TREE_LANES (node);
9068 :
9069 1605472 : tree vectype = SLP_TREE_VECTYPE (node);
9070 1605472 : uint64_t rem;
9071 1605472 : if (can_div_away_from_zero_p (vf, TYPE_VECTOR_SUBPARTS (vectype),
9072 : nvectors)
9073 1605472 : && (TYPE_VECTOR_SUBPARTS (vectype) * *nvectors - vf) .is_constant (&rem))
9074 : {
9075 1605472 : *excess_elts = rem;
9076 1605472 : return true;
9077 : }
9078 : return false;
9079 : }
9080 :
9081 : /* Compute the prologue cost for invariant or constant operands represented
9082 : by NODE. */
9083 :
9084 : static void
9085 1100533 : vect_prologue_cost_for_slp (slp_tree node, unsigned nvectors,
9086 : stmt_vector_for_cost *cost_vec)
9087 : {
9088 : /* There's a special case of an existing vector, that costs nothing. */
9089 1100533 : if (SLP_TREE_SCALAR_OPS (node).length () == 0
9090 1100533 : && !SLP_TREE_VEC_DEFS (node).is_empty ())
9091 2255 : return;
9092 : /* Without looking at the actual initializer a vector of
9093 : constants can be implemented as load from the constant pool.
9094 : When all elements are the same we can use a splat. */
9095 1098278 : tree vectype = SLP_TREE_VECTYPE (node);
9096 1098278 : unsigned group_size = SLP_TREE_LANES (node);
9097 1098278 : unsigned HOST_WIDE_INT const_nunits;
9098 1098278 : unsigned nelt_limit;
9099 1098278 : auto ops = &SLP_TREE_SCALAR_OPS (node);
9100 1098278 : auto_vec<unsigned int> starts (nvectors);
9101 1098278 : if (TYPE_VECTOR_SUBPARTS (vectype).is_constant (&const_nunits)
9102 1098278 : && ! multiple_p (const_nunits, group_size))
9103 : {
9104 67810 : nelt_limit = const_nunits;
9105 67810 : hash_set<vect_scalar_ops_slice_hash> vector_ops;
9106 347875 : for (unsigned int i = 0; i < nvectors; ++i)
9107 212255 : if (!vector_ops.add ({ ops, i * nelt_limit, nelt_limit }))
9108 161263 : starts.quick_push (i * nelt_limit);
9109 67810 : }
9110 : else
9111 : {
9112 : /* If either the vector has variable length or the vectors
9113 : are composed of repeated whole groups we only need to
9114 : cost construction once. All vectors will be the same. */
9115 1030468 : nelt_limit = group_size;
9116 1030468 : starts.quick_push (0);
9117 : }
9118 : /* ??? We're just tracking whether vectors in a single node are the same.
9119 : Ideally we'd do something more global. */
9120 4486565 : for (unsigned int start : starts)
9121 : {
9122 1191731 : vect_cost_for_stmt kind;
9123 1191731 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
9124 : kind = vector_load;
9125 451629 : else if (vect_scalar_ops_slice { ops, start, nelt_limit }.all_same_p ())
9126 : kind = scalar_to_vec;
9127 : else
9128 364243 : kind = vec_construct;
9129 1191731 : record_stmt_cost (cost_vec, 1, kind, nullptr, node,
9130 : vectype, 0, vect_prologue);
9131 : }
9132 1098278 : }
9133 :
9134 : /* Analyze statements contained in SLP tree NODE after recursively analyzing
9135 : the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE.
9136 :
9137 : Return true if the operations are supported. */
9138 :
9139 : static bool
9140 5094170 : vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node,
9141 : slp_instance node_instance,
9142 : hash_set<slp_tree> &visited_set,
9143 : vec<slp_tree> &visited_vec,
9144 : stmt_vector_for_cost *cost_vec)
9145 : {
9146 5094170 : int i, j;
9147 5094170 : slp_tree child;
9148 :
9149 : /* Costing and analysis of invariants is delayed. */
9150 5094170 : if (!node
9151 4695690 : || SLP_TREE_DEF_TYPE (node) == vect_constant_def
9152 3898139 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
9153 : return true;
9154 :
9155 3411732 : if (SLP_TREE_DEF_TYPE (node) == vect_uninitialized_def)
9156 : {
9157 5 : if (dump_enabled_p ())
9158 0 : dump_printf_loc (MSG_NOTE, vect_location,
9159 : "Failed cyclic SLP reference in %p\n", (void *) node);
9160 : return false;
9161 : }
9162 3411727 : gcc_assert (SLP_TREE_DEF_TYPE (node) == vect_internal_def);
9163 :
9164 : /* If we already analyzed the exact same set of scalar stmts we're done.
9165 : We share the generated vector stmts for those. */
9166 3411727 : if (visited_set.add (node))
9167 : return true;
9168 3108717 : visited_vec.safe_push (node);
9169 :
9170 3108717 : bool res = true;
9171 3108717 : unsigned visited_rec_start = visited_vec.length ();
9172 3108717 : unsigned cost_vec_rec_start = cost_vec->length ();
9173 3108717 : bool seen_non_constant_child = false;
9174 8999530 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9175 : {
9176 3715754 : res = vect_slp_analyze_node_operations (vinfo, child, node_instance,
9177 : visited_set, visited_vec,
9178 : cost_vec);
9179 3715754 : if (!res)
9180 : break;
9181 3351006 : if (child && SLP_TREE_DEF_TYPE (child) != vect_constant_def)
9182 3351006 : seen_non_constant_child = true;
9183 : }
9184 : /* We're having difficulties scheduling nodes with just constant
9185 : operands and no scalar stmts since we then cannot compute a stmt
9186 : insertion place. */
9187 3108717 : if (res
9188 3108717 : && !seen_non_constant_child
9189 3108717 : && SLP_TREE_SCALAR_STMTS (node).is_empty ())
9190 : {
9191 218 : if (dump_enabled_p ())
9192 6 : dump_printf_loc (MSG_NOTE, vect_location,
9193 : "Cannot vectorize all-constant op node %p\n",
9194 : (void *) node);
9195 : res = false;
9196 : }
9197 :
9198 3108505 : if (res)
9199 2743751 : res = vect_slp_analyze_node_operations_1 (vinfo, node, node_instance,
9200 : cost_vec);
9201 : /* If analysis failed we have to pop all recursive visited nodes
9202 : plus ourselves. */
9203 3108717 : if (!res)
9204 : {
9205 3184622 : while (visited_vec.length () >= visited_rec_start)
9206 946345 : visited_set.remove (visited_vec.pop ());
9207 645966 : cost_vec->truncate (cost_vec_rec_start);
9208 : }
9209 :
9210 : /* When the node can be vectorized cost invariant nodes it references.
9211 : This is not done in DFS order to allow the referring node
9212 : vectorizable_* calls to nail down the invariant nodes vector type
9213 : and possibly unshare it if it needs a different vector type than
9214 : other referrers. */
9215 3108717 : if (res)
9216 9468339 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), j, child)
9217 2994784 : if (child
9218 2670223 : && (SLP_TREE_DEF_TYPE (child) == vect_constant_def
9219 2670223 : || SLP_TREE_DEF_TYPE (child) == vect_external_def)
9220 : /* Perform usual caching, note code-generation still
9221 : code-gens these nodes multiple times but we expect
9222 : to CSE them later. */
9223 4137356 : && !visited_set.add (child))
9224 : {
9225 1139533 : visited_vec.safe_push (child);
9226 : /* ??? After auditing more code paths make a "default"
9227 : and push the vector type from NODE to all children
9228 : if it is not already set. */
9229 : /* Compute the number of vectors to be generated. */
9230 1139533 : tree vector_type = SLP_TREE_VECTYPE (child);
9231 1139533 : if (!vector_type)
9232 : {
9233 : /* Masked loads can have an undefined (default SSA definition)
9234 : else operand. We do not need to cost it. */
9235 39000 : vec<tree> ops = SLP_TREE_SCALAR_OPS (child);
9236 40431 : if (SLP_TREE_TYPE (node) == load_vec_info_type
9237 40431 : && ((ops.length ()
9238 1431 : && TREE_CODE (ops[0]) == SSA_NAME
9239 0 : && SSA_NAME_IS_DEFAULT_DEF (ops[0])
9240 0 : && VAR_P (SSA_NAME_VAR (ops[0])))
9241 1431 : || SLP_TREE_DEF_TYPE (child) == vect_constant_def))
9242 1431 : continue;
9243 :
9244 : /* For shifts with a scalar argument we don't need
9245 : to cost or code-generate anything.
9246 : ??? Represent this more explicitly. */
9247 37569 : gcc_assert (SLP_TREE_TYPE (node) == shift_vec_info_type
9248 : && j == 1);
9249 37569 : continue;
9250 37569 : }
9251 :
9252 : /* Make sure we can generate them and then cost them. */
9253 1100533 : unsigned nvectors, excess_elts;
9254 1100533 : if (!vect_get_num_copies_for_invariant (vinfo, child, &nvectors,
9255 : &excess_elts))
9256 0 : return false;
9257 1100533 : vect_prologue_cost_for_slp (child, nvectors, cost_vec);
9258 : }
9259 :
9260 : /* If this node or any of its children can't be vectorized, try pruning
9261 : the tree here rather than felling the whole thing. */
9262 645966 : if (!res && vect_slp_convert_to_external (vinfo, node, node_instance))
9263 : {
9264 : /* We'll need to revisit this for invariant costing and number
9265 : of vectorized stmt setting. */
9266 : res = true;
9267 : }
9268 :
9269 : return res;
9270 : }
9271 :
9272 : /* Mark lanes of NODE that are live outside of the basic-block vectorized
9273 : region and that can be vectorized using vectorizable_live_operation
9274 : with STMT_VINFO_LIVE_P. Not handled live operations will cause the
9275 : scalar code computing it to be retained. */
9276 :
9277 : static void
9278 1002841 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node,
9279 : slp_instance instance,
9280 : stmt_vector_for_cost *cost_vec,
9281 : hash_set<stmt_vec_info> &svisited,
9282 : hash_set<slp_tree> &visited)
9283 : {
9284 1002841 : if (visited.add (node))
9285 58615 : return;
9286 :
9287 : unsigned i;
9288 : stmt_vec_info stmt_info;
9289 : gimple *last_stmt = NULL;
9290 3390104 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info)
9291 : {
9292 2445878 : if (!stmt_info || svisited.contains (stmt_info))
9293 95054 : continue;
9294 2410857 : stmt_vec_info orig_stmt_info = vect_orig_stmt (stmt_info);
9295 2410857 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)
9296 36987 : && STMT_VINFO_RELATED_STMT (orig_stmt_info) != stmt_info)
9297 : /* Only the pattern root stmt computes the original scalar value. */
9298 30417 : continue;
9299 2380440 : if (!PURE_SLP_STMT (orig_stmt_info))
9300 : /* Iff the stmt is not part of the vector coverage because it or
9301 : uses of it are used by SLP graph leafs as extern input there is
9302 : no point in trying to live code-generate from a vector stmt as
9303 : the scalar stmt will survive anyway. */
9304 29616 : continue;
9305 2350824 : bool mark_visited = true;
9306 2350824 : gimple *orig_stmt = orig_stmt_info->stmt;
9307 2350824 : ssa_op_iter op_iter;
9308 2350824 : def_operand_p def_p;
9309 5272872 : FOR_EACH_PHI_OR_STMT_DEF (def_p, orig_stmt, op_iter, SSA_OP_DEF)
9310 : {
9311 : /* We have to verify whether we can insert the lane extract
9312 : before all uses. The following is a conservative approximation.
9313 : We cannot put this into vectorizable_live_operation because
9314 : iterating over all use stmts from inside a FOR_EACH_IMM_USE_STMT
9315 : doesn't work.
9316 : Note that while the fact that we emit code for loads at the
9317 : first load should make this a non-problem leafs we construct
9318 : from scalars are vectorized after the last scalar def.
9319 : ??? If we'd actually compute the insert location during
9320 : analysis we could use sth less conservative than the last
9321 : scalar stmt in the node for the dominance check. */
9322 : /* ??? What remains is "live" uses in vector CTORs in the same
9323 : SLP graph which is where those uses can end up code-generated
9324 : right after their definition instead of close to their original
9325 : use. But that would restrict us to code-generate lane-extracts
9326 : from the latest stmt in a node. So we compensate for this
9327 : during code-generation, simply not replacing uses for those
9328 : hopefully rare cases. */
9329 571224 : imm_use_iterator use_iter;
9330 :
9331 571224 : bool live_p = false;
9332 571224 : bool can_insert = true;
9333 571224 : use_operand_p use_p;
9334 1714342 : FOR_EACH_IMM_USE_FAST (use_p, use_iter, DEF_FROM_PTR (def_p))
9335 : {
9336 1151194 : gimple *use_stmt = USE_STMT (use_p);
9337 1151194 : stmt_vec_info use_stmt_info;
9338 1151194 : if (!(!is_gimple_debug (use_stmt)
9339 886616 : && (!(use_stmt_info = bb_vinfo->lookup_stmt (use_stmt))
9340 875920 : || !PURE_SLP_STMT (use_stmt_info))))
9341 938892 : continue;
9342 212302 : live_p = true;
9343 212302 : if (!last_stmt)
9344 65740 : last_stmt
9345 66002 : = (node->si ? node->si
9346 262 : : vect_find_last_scalar_stmt_in_slp (node)->stmt);
9347 212302 : if (is_a <gphi *> (use_stmt))
9348 : {
9349 72594 : if (!dominated_by_p (CDI_DOMINATORS,
9350 36297 : phi_arg_edge_from_use (use_p)->src,
9351 36297 : gimple_bb (last_stmt)))
9352 : can_insert = false;
9353 : }
9354 : /* As we instert after last_stmt it may not be the use_stmt
9355 : itself. */
9356 176005 : else if (last_stmt == use_stmt
9357 176005 : || !vect_stmt_dominates_stmt_p (last_stmt, use_stmt))
9358 : can_insert = false;
9359 : if (!can_insert)
9360 : {
9361 8076 : if (dump_enabled_p ())
9362 82 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9363 : "Cannot determine insertion place for "
9364 : "lane extract of %T at node %p\n",
9365 : DEF_FROM_PTR (def_p), (void *)node);
9366 : can_insert = false;
9367 : break;
9368 : }
9369 571224 : }
9370 571224 : if (live_p && can_insert)
9371 : {
9372 : /* Only record a live stmt when we can replace all uses. We
9373 : record from which SLP tree we vectorize the uses, so we'll
9374 : cost once and can deal with the case that not all SLP nodes
9375 : may be suitable for code-generation of all live uses.
9376 : ??? But we never split up the work between multiple SLP
9377 : nodes. */
9378 106268 : STMT_VINFO_LIVE_P (stmt_info) = true;
9379 106268 : if (!vectorizable_live_operation (bb_vinfo, stmt_info, node,
9380 : instance, i, false, cost_vec))
9381 : {
9382 0 : STMT_VINFO_LIVE_P (stmt_info) = false;
9383 0 : mark_visited = false;
9384 : }
9385 : }
9386 : }
9387 2350824 : if (mark_visited)
9388 2350824 : svisited.add (stmt_info);
9389 : }
9390 :
9391 : slp_tree child;
9392 2759844 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
9393 985891 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9394 282109 : vect_bb_slp_mark_live_stmts (bb_vinfo, child, instance, cost_vec,
9395 : svisited, visited);
9396 : }
9397 :
9398 : /* Traverse all slp instances of BB_VINFO, and mark lanes of every node that
9399 : are live outside of the basic-block vectorized region and that can be
9400 : vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. */
9401 :
9402 : static void
9403 248621 : vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
9404 : {
9405 248621 : if (bb_vinfo->slp_instances.is_empty ())
9406 0 : return;
9407 :
9408 248621 : hash_set<slp_tree> visited;
9409 248621 : hash_set<stmt_vec_info> svisited;
9410 1466595 : for (slp_instance instance : bb_vinfo->slp_instances)
9411 : {
9412 720732 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9413 52685 : STMT_VINFO_LIVE_P (SLP_INSTANCE_ROOT_STMTS (instance)[0]) = true;
9414 720732 : vect_location = instance->location ();
9415 720732 : vect_bb_slp_mark_live_stmts (bb_vinfo, SLP_INSTANCE_TREE (instance),
9416 : instance, &instance->cost_vec,
9417 : svisited, visited);
9418 : }
9419 248621 : }
9420 :
9421 : /* Determine whether we can vectorize the reduction epilogue for INSTANCE. */
9422 :
9423 : static bool
9424 90929 : vectorizable_bb_reduc_epilogue (slp_instance instance,
9425 : stmt_vector_for_cost *cost_vec)
9426 : {
9427 90929 : gassign *stmt = as_a <gassign *> (instance->root_stmts[0]->stmt);
9428 90929 : enum tree_code reduc_code = gimple_assign_rhs_code (stmt);
9429 90929 : if (reduc_code == MINUS_EXPR)
9430 0 : reduc_code = PLUS_EXPR;
9431 90929 : internal_fn reduc_fn;
9432 90929 : tree vectype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (instance));
9433 90929 : if (!vectype
9434 90917 : || !useless_type_conversion_p (TREE_TYPE (gimple_assign_lhs (stmt)),
9435 90917 : TREE_TYPE (vectype))
9436 53835 : || (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype), 2u)
9437 975 : && (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
9438 975 : || reduc_fn == IFN_LAST
9439 975 : || !direct_internal_fn_supported_p (reduc_fn, vectype,
9440 : OPTIMIZE_FOR_BOTH)))
9441 : /* Two-element reductions do not need special-handling for fold-left,
9442 : other cases are not yet implemented. remain_defs also have to
9443 : be included here. */
9444 181894 : || (needs_fold_left_reduction_p (TREE_TYPE (vectype), reduc_code)
9445 5959 : && (!instance->remain_defs.is_empty ()
9446 1887 : || SLP_TREE_LANES (SLP_INSTANCE_TREE (instance)) != 2)))
9447 : {
9448 39180 : if (dump_enabled_p ())
9449 60 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
9450 : "not vectorized: basic block reduction epilogue "
9451 : "operation unsupported.\n");
9452 : return false;
9453 : }
9454 :
9455 : /* There's no way to cost a horizontal vector reduction via REDUC_FN so
9456 : cost log2 vector operations plus shuffles and one extraction. */
9457 51749 : unsigned steps = floor_log2 (vect_nunits_for_cost (vectype));
9458 51749 : record_stmt_cost (cost_vec, steps, vector_stmt, instance->root_stmts[0],
9459 : vectype, 0, vect_body);
9460 51749 : record_stmt_cost (cost_vec, steps, vec_perm, instance->root_stmts[0],
9461 : vectype, 0, vect_body);
9462 51749 : record_stmt_cost (cost_vec, 1, vec_to_scalar, instance->root_stmts[0],
9463 : vectype, 0, vect_body);
9464 :
9465 : /* Since we replace all stmts of a possibly longer scalar reduction
9466 : chain account for the extra scalar stmts for that. */
9467 51749 : if (!instance->remain_defs.is_empty ())
9468 32648 : record_stmt_cost (cost_vec, instance->remain_defs.length (), scalar_stmt,
9469 16324 : instance->root_stmts[0], 0, vect_body);
9470 : return true;
9471 : }
9472 :
9473 : /* Prune from ROOTS all stmts that are computed as part of lanes of NODE
9474 : and recurse to children. */
9475 :
9476 : static void
9477 364513 : vect_slp_prune_covered_roots (slp_tree node, hash_set<stmt_vec_info> &roots,
9478 : hash_set<slp_tree> &visited)
9479 : {
9480 364513 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def
9481 364513 : || visited.add (node))
9482 : return;
9483 :
9484 1025182 : for (auto stmt : SLP_TREE_SCALAR_STMTS (node))
9485 455914 : if (stmt)
9486 488583 : roots.remove (vect_orig_stmt (stmt));
9487 :
9488 771396 : for (auto child : SLP_TREE_CHILDREN (node))
9489 257130 : if (child)
9490 255680 : vect_slp_prune_covered_roots (child, roots, visited);
9491 : }
9492 :
9493 : /* Hand over COST_VEC to the target COSTS grouped by SLP node. */
9494 :
9495 : static void
9496 1038341 : add_slp_costs (vector_costs *costs, stmt_vector_for_cost& cost_vec)
9497 : {
9498 3858490 : for (unsigned start = 0; start < cost_vec.length ();)
9499 : {
9500 2820149 : unsigned end = start + 1;
9501 3544512 : while (end < cost_vec.length ()
9502 6060221 : && cost_vec[start].node == cost_vec[end].node)
9503 724363 : end++;
9504 2820149 : if (cost_vec[start].node)
9505 2777353 : costs->add_slp_cost (cost_vec[start].node,
9506 2777353 : array_slice<stmt_info_for_cost>
9507 2777353 : (cost_vec.begin () + start, end - start));
9508 : else
9509 42796 : costs->vector_costs::add_slp_cost (cost_vec[start].node,
9510 42796 : array_slice<stmt_info_for_cost>
9511 42796 : (cost_vec.begin () + start, end - start));
9512 2820149 : start = end;
9513 : }
9514 1038341 : }
9515 :
9516 : /* Analyze statements in SLP instances of VINFO. Return true if the
9517 : operations are supported. */
9518 :
9519 : bool
9520 701474 : vect_slp_analyze_operations (vec_info *vinfo)
9521 : {
9522 701474 : slp_instance instance;
9523 701474 : int i;
9524 :
9525 701474 : DUMP_VECT_SCOPE ("vect_slp_analyze_operations");
9526 :
9527 701474 : hash_set<slp_tree> visited;
9528 2518770 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9529 : {
9530 1378416 : auto_vec<slp_tree> visited_vec;
9531 1378416 : stmt_vector_for_cost cost_vec;
9532 1378416 : cost_vec.create (2);
9533 1378416 : if (is_a <bb_vec_info> (vinfo))
9534 825436 : vect_location = instance->location ();
9535 1378416 : if (!vect_slp_analyze_node_operations (vinfo,
9536 : SLP_INSTANCE_TREE (instance),
9537 : instance, visited, visited_vec,
9538 : &cost_vec)
9539 : /* CTOR instances require vectorized defs for the SLP tree root. */
9540 1119665 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_ctor
9541 6521 : && (SLP_TREE_DEF_TYPE (SLP_INSTANCE_TREE (instance))
9542 : != vect_internal_def
9543 : /* Make sure we vectorized with the expected type. */
9544 6521 : || !useless_type_conversion_p
9545 6521 : (TREE_TYPE (TREE_TYPE (gimple_assign_rhs1
9546 : (instance->root_stmts[0]->stmt))),
9547 6521 : TREE_TYPE (SLP_TREE_VECTYPE
9548 : (SLP_INSTANCE_TREE (instance))))))
9549 : /* Check we can vectorize the reduction. */
9550 1119650 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_bb_reduc
9551 90929 : && !vectorizable_bb_reduc_epilogue (instance, &cost_vec))
9552 : /* Check we can vectorize the gcond. */
9553 2458886 : || (SLP_INSTANCE_KIND (instance) == slp_inst_kind_gcond
9554 66418 : && !vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
9555 66418 : SLP_INSTANCE_ROOT_STMTS (instance)[0],
9556 : NULL,
9557 : SLP_INSTANCE_TREE (instance),
9558 : &cost_vec)))
9559 : {
9560 361720 : cost_vec.release ();
9561 361720 : slp_tree node = SLP_INSTANCE_TREE (instance);
9562 361720 : stmt_vec_info stmt_info;
9563 361720 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9564 270834 : stmt_info = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9565 90886 : else if (!SLP_TREE_SCALAR_STMTS (node).is_empty ()
9566 90886 : && SLP_TREE_SCALAR_STMTS (node)[0])
9567 : stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
9568 : else
9569 3 : stmt_info = SLP_TREE_REPRESENTATIVE (node);
9570 361720 : if (is_a <loop_vec_info> (vinfo))
9571 : {
9572 262594 : if (dump_enabled_p ())
9573 7365 : dump_printf_loc (MSG_NOTE, vect_location,
9574 : "unsupported SLP instance starting from: %G",
9575 : stmt_info->stmt);
9576 262594 : return false;
9577 : }
9578 99126 : if (dump_enabled_p ())
9579 120 : dump_printf_loc (MSG_NOTE, vect_location,
9580 : "removing SLP instance operations starting from: %G",
9581 : stmt_info->stmt);
9582 236571 : while (!visited_vec.is_empty ())
9583 : {
9584 137445 : slp_tree node = visited_vec.pop ();
9585 137445 : SLP_TREE_TYPE (node) = undef_vec_info_type;
9586 137445 : if (node->data)
9587 : {
9588 8604 : delete node->data;
9589 8604 : node->data = nullptr;
9590 : }
9591 137445 : visited.remove (node);
9592 : }
9593 99126 : vect_free_slp_instance (instance);
9594 99126 : vinfo->slp_instances.ordered_remove (i);
9595 : }
9596 : else
9597 : {
9598 1016696 : i++;
9599 1016696 : if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
9600 : {
9601 290386 : add_slp_costs (loop_vinfo->vector_costs, cost_vec);
9602 290386 : cost_vec.release ();
9603 : }
9604 : else
9605 : /* For BB vectorization remember the SLP graph entry
9606 : cost for later. */
9607 726310 : instance->cost_vec = cost_vec;
9608 : }
9609 1378416 : }
9610 :
9611 : /* Now look for SLP instances with a root that are covered by other
9612 : instances and remove them. */
9613 438880 : hash_set<stmt_vec_info> roots;
9614 1820871 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9615 1003998 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
9616 60887 : roots.add (SLP_INSTANCE_ROOT_STMTS (instance)[0]);
9617 438880 : if (!roots.is_empty ())
9618 : {
9619 24850 : visited.empty ();
9620 133683 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); ++i)
9621 108833 : vect_slp_prune_covered_roots (SLP_INSTANCE_TREE (instance), roots,
9622 : visited);
9623 133683 : for (i = 0; vinfo->slp_instances.iterate (i, &instance); )
9624 108833 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ()
9625 60887 : && !roots.contains (SLP_INSTANCE_ROOT_STMTS (instance)[0]))
9626 : {
9627 5540 : stmt_vec_info root = SLP_INSTANCE_ROOT_STMTS (instance)[0];
9628 5540 : if (dump_enabled_p ())
9629 287 : dump_printf_loc (MSG_NOTE, vect_location,
9630 : "removing SLP instance operations starting "
9631 : "from: %G", root->stmt);
9632 5540 : vect_free_slp_instance (instance);
9633 5540 : vinfo->slp_instances.ordered_remove (i);
9634 : }
9635 : else
9636 103293 : ++i;
9637 : }
9638 :
9639 877760 : return !vinfo->slp_instances.is_empty ();
9640 1140354 : }
9641 :
9642 : /* Get the SLP instance leader from INSTANCE_LEADER thereby transitively
9643 : closing the eventual chain. */
9644 :
9645 : static slp_instance
9646 811669 : get_ultimate_leader (slp_instance instance,
9647 : hash_map<slp_instance, slp_instance> &instance_leader)
9648 : {
9649 811669 : auto_vec<slp_instance *, 8> chain;
9650 811669 : slp_instance *tem;
9651 922670 : while (*(tem = instance_leader.get (instance)) != instance)
9652 : {
9653 111001 : chain.safe_push (tem);
9654 111001 : instance = *tem;
9655 : }
9656 922670 : while (!chain.is_empty ())
9657 111001 : *chain.pop () = instance;
9658 811669 : return instance;
9659 811669 : }
9660 :
9661 : namespace {
9662 : /* Subroutine of vect_bb_partition_graph_r. Map KEY to INSTANCE in
9663 : KEY_TO_INSTANCE, making INSTANCE the leader of any previous mapping
9664 : for KEY. Return true if KEY was already in KEY_TO_INSTANCE.
9665 :
9666 : INSTANCE_LEADER is as for get_ultimate_leader. */
9667 :
9668 : template<typename T>
9669 : bool
9670 3587079 : vect_map_to_instance (slp_instance instance, T key,
9671 : hash_map<T, slp_instance> &key_to_instance,
9672 : hash_map<slp_instance, slp_instance> &instance_leader)
9673 : {
9674 : bool existed_p;
9675 3587079 : slp_instance &key_instance = key_to_instance.get_or_insert (key, &existed_p);
9676 3587079 : if (!existed_p)
9677 : ;
9678 245149 : else if (key_instance != instance)
9679 : {
9680 : /* If we're running into a previously marked key make us the
9681 : leader of the current ultimate leader. This keeps the
9682 : leader chain acyclic and works even when the current instance
9683 : connects two previously independent graph parts. */
9684 90937 : slp_instance key_leader
9685 90937 : = get_ultimate_leader (key_instance, instance_leader);
9686 90937 : if (key_leader != instance)
9687 27239 : instance_leader.put (key_leader, instance);
9688 : }
9689 3587079 : key_instance = instance;
9690 3587079 : return existed_p;
9691 : }
9692 : }
9693 :
9694 : /* Worker of vect_bb_partition_graph, recurse on NODE. */
9695 :
9696 : static void
9697 1002841 : vect_bb_partition_graph_r (bb_vec_info bb_vinfo,
9698 : slp_instance instance, slp_tree node,
9699 : hash_map<stmt_vec_info, slp_instance> &stmt_to_instance,
9700 : hash_map<slp_tree, slp_instance> &node_to_instance,
9701 : hash_map<slp_instance, slp_instance> &instance_leader)
9702 : {
9703 5581609 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
9704 2584238 : if (stmt_info)
9705 2584238 : vect_map_to_instance (instance, stmt_info, stmt_to_instance,
9706 : instance_leader);
9707 :
9708 1002841 : if (vect_map_to_instance (instance, node, node_to_instance,
9709 : instance_leader))
9710 : return;
9711 :
9712 3589571 : for (auto child : SLP_TREE_CHILDREN (node))
9713 985891 : if (child && SLP_TREE_DEF_TYPE (child) == vect_internal_def)
9714 282109 : vect_bb_partition_graph_r (bb_vinfo, instance, child, stmt_to_instance,
9715 : node_to_instance, instance_leader);
9716 : }
9717 :
9718 : /* Partition the SLP graph into pieces that can be costed independently. */
9719 :
9720 : static void
9721 248621 : vect_bb_partition_graph (bb_vec_info bb_vinfo)
9722 : {
9723 248621 : DUMP_VECT_SCOPE ("vect_bb_partition_graph");
9724 :
9725 : /* First walk the SLP graph assigning each involved scalar stmt a
9726 : corresponding SLP graph entry and upon visiting a previously
9727 : marked stmt, make the stmts leader the current SLP graph entry. */
9728 248621 : hash_map<stmt_vec_info, slp_instance> stmt_to_instance;
9729 248621 : hash_map<slp_tree, slp_instance> node_to_instance;
9730 248621 : hash_map<slp_instance, slp_instance> instance_leader;
9731 248621 : slp_instance instance;
9732 1217974 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9733 : {
9734 720732 : instance_leader.put (instance, instance);
9735 720732 : vect_bb_partition_graph_r (bb_vinfo,
9736 : instance, SLP_INSTANCE_TREE (instance),
9737 : stmt_to_instance, node_to_instance,
9738 : instance_leader);
9739 : }
9740 :
9741 : /* Then collect entries to each independent subgraph. */
9742 1217974 : for (unsigned i = 0; bb_vinfo->slp_instances.iterate (i, &instance); ++i)
9743 : {
9744 720732 : slp_instance leader = get_ultimate_leader (instance, instance_leader);
9745 720732 : leader->subgraph_entries.safe_push (instance);
9746 720732 : if (dump_enabled_p ()
9747 720732 : && leader != instance)
9748 102 : dump_printf_loc (MSG_NOTE, vect_location,
9749 : "instance %p is leader of %p\n",
9750 : (void *) leader, (void *) instance);
9751 : }
9752 248621 : }
9753 :
9754 : /* Compute the scalar cost of the SLP node NODE and its children
9755 : and return it. Do not account defs that are marked in LIFE and
9756 : update LIFE according to uses of NODE. */
9757 :
9758 : static void
9759 716525 : vect_bb_slp_scalar_cost (bb_vec_info vinfo,
9760 : vec<stmt_vec_info> &worklist,
9761 : stmt_vector_for_cost *cost_vec,
9762 : hash_set<stmt_vec_info> &visited)
9763 : {
9764 3341023 : while (!worklist.is_empty ())
9765 : {
9766 2624498 : stmt_vec_info stmt = worklist.pop ();
9767 2950340 : if (!PURE_SLP_STMT (stmt))
9768 334448 : continue;
9769 :
9770 : /* When the stmt is live but not actually vectorized we have
9771 : to keep the feeding scalar defs. */
9772 2304853 : if (!STMT_VINFO_LIVE_P (vect_stmt_to_vectorize (stmt)))
9773 : {
9774 2194759 : bool live_p = false;
9775 2194759 : ssa_op_iter op_iter;
9776 2194759 : def_operand_p def_p;
9777 4828011 : FOR_EACH_PHI_OR_STMT_DEF (def_p, stmt->stmt, op_iter, SSA_OP_DEF)
9778 : {
9779 438493 : imm_use_iterator use_iter;
9780 438493 : gimple *use_stmt;
9781 1123671 : FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
9782 685178 : if (!is_gimple_debug (use_stmt))
9783 : {
9784 515961 : stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
9785 515961 : if (!use_stmt_info || !PURE_SLP_STMT (use_stmt_info))
9786 : {
9787 15803 : if (dump_enabled_p ())
9788 : {
9789 92 : dump_printf_loc (MSG_NOTE, vect_location,
9790 : "stmt considered live: %G",
9791 : stmt->stmt);
9792 92 : dump_printf_loc (MSG_NOTE, vect_location,
9793 : "because of use in: %G",
9794 : use_stmt);
9795 : }
9796 : live_p = true;
9797 : }
9798 438493 : }
9799 : }
9800 2194759 : if (live_p)
9801 8606 : continue;
9802 : }
9803 :
9804 : /* The following assert verifies that vect_bb_partition_graph
9805 : partitions the SLP graph in a way that each scalar stmt of
9806 : the coverage of the SLP graph belongs to exactly one subgraph.
9807 : ??? This is currently not guaranteed since the function
9808 : works purely on SLP_TREE_SCALAR_STMTS, resulting in the assert
9809 : tripping or scalar stmts costed multiple times, making vectorization
9810 : more profitable than it really is. */
9811 : /* gcc_checking_assert (!gimple_visited_p (stmt->stmt)); */
9812 :
9813 2290050 : if (vect_nop_conversion_p (stmt))
9814 : ;
9815 : /* For single-argument PHIs assume coalescing which means zero
9816 : cost for the scalar and the vector PHIs. This avoids
9817 : artificially favoring the vector path (but may pessimize it
9818 : in some cases). */
9819 2258698 : else if (is_a <gphi *> (stmt->stmt)
9820 2258698 : && gimple_phi_num_args (as_a <gphi *> (stmt->stmt)) == 1)
9821 : ;
9822 : else
9823 : {
9824 2248168 : vect_cost_for_stmt kind;
9825 2248168 : if (STMT_VINFO_DATA_REF (stmt))
9826 : {
9827 2021471 : data_reference_p dr = STMT_VINFO_DATA_REF (stmt);
9828 2021471 : tree base = get_base_address (DR_REF (dr));
9829 : /* When the scalar access is to a non-global not
9830 : address-taken decl that is not BLKmode assume we can
9831 : access it with a single non-load/store instruction. */
9832 2021471 : if (DECL_P (base)
9833 1550496 : && !is_global_var (base)
9834 1471743 : && !TREE_ADDRESSABLE (base)
9835 2580180 : && DECL_MODE (base) != BLKmode)
9836 : kind = scalar_stmt;
9837 1877097 : else if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt)))
9838 : kind = scalar_load;
9839 : else
9840 1628830 : kind = scalar_store;
9841 : }
9842 : else
9843 : kind = scalar_stmt;
9844 : /* Cost each scalar stmt only once. */
9845 2248168 : gimple_set_visited (stmt->stmt, true);
9846 2248168 : record_stmt_cost (cost_vec, 1, kind, stmt, NULL_TREE, 0, vect_body);
9847 : }
9848 :
9849 : /* Now walk relevant parts of the SSA use-def graph. */
9850 2290050 : slp_oprnds child_ops (stmt);
9851 7128330 : for (unsigned i = 0; i < child_ops.num_slp_children; ++i)
9852 : {
9853 2548230 : tree op = child_ops.get_op_for_slp_child (stmt, i);
9854 2548230 : stmt_vec_info def = vinfo->lookup_def (op);
9855 2548230 : if (def && !visited.add (def))
9856 765469 : worklist.safe_push (def);
9857 : }
9858 : }
9859 716525 : }
9860 :
9861 :
9862 : /* Comparator for the loop-index sorted cost vectors. */
9863 :
9864 : static int
9865 20187638 : li_cost_vec_cmp (const void *a_, const void *b_, void *)
9866 : {
9867 20187638 : auto *a = (const std::pair<unsigned, stmt_info_for_cost *> *)a_;
9868 20187638 : auto *b = (const std::pair<unsigned, stmt_info_for_cost *> *)b_;
9869 20187638 : if (a->first < b->first)
9870 : return -1;
9871 18740510 : else if (a->first == b->first)
9872 17585797 : return 0;
9873 : return 1;
9874 : }
9875 :
9876 : /* Check if vectorization of the basic block is profitable for the
9877 : subgraph denoted by SLP_INSTANCES. */
9878 :
9879 : static bool
9880 689490 : vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo,
9881 : vec<slp_instance> slp_instances,
9882 : loop_p orig_loop)
9883 : {
9884 689490 : slp_instance instance;
9885 689490 : int i;
9886 689490 : unsigned int vec_inside_cost = 0, vec_outside_cost = 0, scalar_cost = 0;
9887 689490 : unsigned int vec_prologue_cost = 0, vec_epilogue_cost = 0;
9888 :
9889 689490 : if (dump_enabled_p ())
9890 : {
9891 125 : dump_printf_loc (MSG_NOTE, vect_location, "Costing subgraph:\n");
9892 260 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9893 135 : dump_printf_loc (MSG_NOTE, vect_location, " entry instance %p -> "
9894 : "node %p\n", (void *)instance,
9895 135 : (void *)SLP_INSTANCE_TREE (instance));
9896 125 : hash_set<slp_tree> visited;
9897 510 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9898 135 : vect_print_slp_graph (MSG_NOTE, vect_location,
9899 : SLP_INSTANCE_TREE (instance), visited);
9900 125 : }
9901 :
9902 : /* Then DFS walk scalar stmts, performing costing and handling
9903 : still live scalar stmts via the previously computed vector coverage. */
9904 689490 : stmt_vector_for_cost scalar_costs = vNULL;
9905 689490 : stmt_vector_for_cost vector_costs = vNULL;
9906 689490 : hash_set<slp_tree> visited;
9907 689490 : hash_set<stmt_vec_info> svisited;
9908 2095505 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
9909 : {
9910 716525 : auto_vec<stmt_vec_info> worklist;
9911 716525 : if (SLP_INSTANCE_ROOT_STMTS (instance).exists ())
9912 103790 : record_stmt_cost (&scalar_costs,
9913 51895 : SLP_INSTANCE_ROOT_STMTS (instance).length (),
9914 : scalar_stmt,
9915 51895 : SLP_INSTANCE_ROOT_STMTS (instance)[0], 0, vect_body);
9916 4024039 : for (auto stmt : SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)))
9917 : {
9918 1874464 : stmt = vect_orig_stmt (stmt);
9919 1874464 : if (!svisited.add (stmt))
9920 1859029 : worklist.safe_push (stmt);
9921 : }
9922 716525 : vect_bb_slp_scalar_cost (bb_vinfo, worklist, &scalar_costs, svisited);
9923 716525 : vector_costs.safe_splice (instance->cost_vec);
9924 716525 : instance->cost_vec.release ();
9925 716525 : }
9926 :
9927 689490 : if (dump_enabled_p ())
9928 125 : dump_printf_loc (MSG_NOTE, vect_location, "Cost model analysis: \n");
9929 :
9930 : /* When costing non-loop vectorization we need to consider each covered
9931 : loop independently and make sure vectorization is profitable. For
9932 : now we assume a loop may be not entered or executed an arbitrary
9933 : number of iterations (??? static information can provide more
9934 : precise info here) which means we can simply cost each containing
9935 : loops stmts separately. */
9936 :
9937 : /* First produce cost vectors sorted by loop index. */
9938 689490 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9939 1378980 : li_scalar_costs (scalar_costs.length ());
9940 689490 : auto_vec<std::pair<unsigned, stmt_info_for_cost *> >
9941 1378967 : li_vector_costs (vector_costs.length ());
9942 689490 : stmt_info_for_cost *cost;
9943 3679043 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
9944 : {
9945 2300063 : unsigned l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9946 2300063 : li_scalar_costs.quick_push (std::make_pair (l, cost));
9947 : }
9948 : /* Use a random used loop as fallback in case the first vector_costs
9949 : entry does not have a location associated with it. */
9950 689490 : unsigned l = li_scalar_costs[0].first;
9951 2650475 : FOR_EACH_VEC_ELT (vector_costs, i, cost)
9952 : {
9953 : /* Use SLP node placement according to the computed schedule. */
9954 1960985 : if (cost->node && cost->node->si)
9955 1221515 : l = gimple_bb (cost->node->si)->loop_father->num;
9956 : /* For schedules at region boundary use the region entry loop. */
9957 739470 : else if (cost->node)
9958 588907 : l = bb_vinfo->bbs[0]->loop_father->num;
9959 : /* SLP instance root stmts do not have an associated SLP node. */
9960 150563 : else if (cost->stmt_info)
9961 150563 : l = gimple_bb (cost->stmt_info->stmt)->loop_father->num;
9962 : /* And since vect_prologue_cost_for_slp can end up costing with
9963 : neither, inherit from the previous node. */
9964 1960985 : li_vector_costs.quick_push (std::make_pair (l, cost));
9965 : }
9966 689490 : li_scalar_costs.stablesort (li_cost_vec_cmp, NULL);
9967 689490 : li_vector_costs.stablesort (li_cost_vec_cmp, NULL);
9968 :
9969 689490 : unsigned total_vec_outside_cost = 0;
9970 689490 : unsigned total_vec_inside_cost = 0;
9971 689490 : unsigned total_scalar_cost = 0;
9972 :
9973 : /* Now cost the portions individually. */
9974 689490 : unsigned vi = 0;
9975 689490 : unsigned si = 0;
9976 689490 : bool profitable = true;
9977 1380311 : while (si < li_scalar_costs.length ()
9978 2130097 : || vi < li_vector_costs.length ())
9979 : {
9980 749799 : unsigned sl
9981 1498267 : = si < li_scalar_costs.length () ? li_scalar_costs[si].first : -1U;
9982 749799 : unsigned vl
9983 1499514 : = vi < li_vector_costs.length () ? li_vector_costs[vi].first : -1U;
9984 :
9985 749799 : class vector_costs *scalar_target_cost_data = nullptr;
9986 749799 : scalar_cost = 0;
9987 749799 : if (sl <= vl)
9988 : {
9989 704857 : if (dump_enabled_p ())
9990 147 : dump_printf_loc (MSG_NOTE, vect_location,
9991 : "Scalar cost for part in loop %d\n", sl);
9992 704857 : scalar_target_cost_data = init_cost (bb_vinfo, true);
9993 2300063 : do
9994 : {
9995 2300063 : add_stmt_cost (scalar_target_cost_data,
9996 2300063 : li_scalar_costs[si].second);
9997 2300063 : si++;
9998 : }
9999 2300063 : while (si < li_scalar_costs.length ()
10000 4615493 : && li_scalar_costs[si].first == sl);
10001 704857 : scalar_target_cost_data->finish_cost (nullptr);
10002 704857 : scalar_cost = scalar_target_cost_data->body_cost ();
10003 704857 : total_scalar_cost += scalar_cost;
10004 704857 : if (sl < vl)
10005 : {
10006 1844 : if (dump_enabled_p ())
10007 2 : dump_printf_loc (MSG_NOTE, vect_location,
10008 : "Scalar %d loop part does not "
10009 : "have corresponding vector part\n", sl);
10010 1844 : delete scalar_target_cost_data;
10011 650053 : continue;
10012 : }
10013 : }
10014 :
10015 : /* Complete the target-specific vector cost calculation. */
10016 747955 : if (dump_enabled_p ())
10017 149 : dump_printf_loc (MSG_NOTE, vect_location,
10018 : "Vector cost for part in loop %d\n", vl);
10019 747955 : class vector_costs *vect_target_cost_data = init_cost (bb_vinfo, false);
10020 747955 : auto_vec<stmt_info_for_cost> tem;
10021 1960985 : do
10022 : {
10023 1960985 : tem.safe_push (*li_vector_costs[vi].second);
10024 1960985 : vi++;
10025 : }
10026 1960985 : while (vi < li_vector_costs.length ()
10027 3980448 : && li_vector_costs[vi].first == vl);
10028 747955 : add_slp_costs (vect_target_cost_data, tem);
10029 747955 : vect_target_cost_data->finish_cost (scalar_target_cost_data);
10030 747955 : vec_prologue_cost = vect_target_cost_data->prologue_cost ();
10031 747955 : vec_inside_cost = vect_target_cost_data->body_cost ();
10032 747955 : vec_epilogue_cost = vect_target_cost_data->epilogue_cost ();
10033 747955 : if (scalar_target_cost_data)
10034 703013 : delete scalar_target_cost_data;
10035 747955 : delete vect_target_cost_data;
10036 :
10037 747955 : vec_outside_cost = vec_prologue_cost + vec_epilogue_cost;
10038 :
10039 747955 : total_vec_outside_cost += vec_outside_cost;
10040 747955 : total_vec_inside_cost += vec_inside_cost;
10041 :
10042 747955 : if (sl > vl && dump_enabled_p ())
10043 4 : dump_printf_loc (MSG_NOTE, vect_location,
10044 : "Vector %d loop part does not "
10045 : "have corresponding scalar part\n", vl);
10046 :
10047 : /* When this is vector costs for the region entry delay costing
10048 : and instead only require the total costs to be profitable. */
10049 747955 : if (vl == (unsigned) bb_vinfo->bbs[0]->loop_father->num)
10050 648209 : continue;
10051 :
10052 99746 : if (dump_enabled_p ())
10053 : {
10054 52 : dump_printf_loc (MSG_NOTE, vect_location,
10055 : "Cost model analysis for part in loop %d:\n",
10056 52 : std::min (sl, vl));
10057 52 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
10058 : vec_inside_cost + vec_outside_cost);
10059 52 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", scalar_cost);
10060 : }
10061 :
10062 : /* Vectorization is profitable if its cost is more than the cost of scalar
10063 : version. Note that we err on the vector side for equal cost because
10064 : the cost estimate is otherwise quite pessimistic (constant uses are
10065 : free on the scalar side but cost a load on the vector side for
10066 : example). */
10067 99746 : if (vec_outside_cost + vec_inside_cost > scalar_cost)
10068 46549 : profitable = false;
10069 747955 : }
10070 :
10071 689490 : if (dump_enabled_p ())
10072 : {
10073 125 : dump_printf_loc (MSG_NOTE, vect_location,
10074 : "Cost model analysis for whole subgraph:\n");
10075 125 : dump_printf (MSG_NOTE, " Vector cost: %d\n",
10076 : total_vec_inside_cost + total_vec_outside_cost);
10077 125 : dump_printf (MSG_NOTE, " Scalar cost: %d\n", total_scalar_cost);
10078 : }
10079 :
10080 : /* For the case where the outermost region had no scalar cost require
10081 : overall profitability. */
10082 689490 : if (total_vec_outside_cost + total_vec_inside_cost > total_scalar_cost)
10083 211817 : profitable = false;
10084 :
10085 : /* Unset visited flag. This is delayed when the subgraph is profitable
10086 : and we process the loop for remaining unvectorized if-converted code. */
10087 689490 : if (!orig_loop || !profitable)
10088 2987501 : FOR_EACH_VEC_ELT (scalar_costs, i, cost)
10089 2298136 : gimple_set_visited (cost->stmt_info->stmt, false);
10090 :
10091 689490 : scalar_costs.release ();
10092 689490 : vector_costs.release ();
10093 :
10094 689490 : return profitable;
10095 689490 : }
10096 :
10097 : /* qsort comparator for lane defs. */
10098 :
10099 : static int
10100 128 : vld_cmp (const void *a_, const void *b_)
10101 : {
10102 128 : auto *a = (const std::pair<unsigned, tree> *)a_;
10103 128 : auto *b = (const std::pair<unsigned, tree> *)b_;
10104 128 : return a->first - b->first;
10105 : }
10106 :
10107 : /* Return true if USE_STMT is a vector lane insert into VEC and set
10108 : *THIS_LANE to the lane number that is set. */
10109 :
10110 : static bool
10111 295 : vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
10112 : {
10113 295 : gassign *use_ass = dyn_cast <gassign *> (use_stmt);
10114 110 : if (!use_ass
10115 110 : || gimple_assign_rhs_code (use_ass) != BIT_INSERT_EXPR
10116 35 : || (vec
10117 35 : ? gimple_assign_rhs1 (use_ass) != vec
10118 26 : : ((vec = gimple_assign_rhs1 (use_ass)), false))
10119 61 : || !useless_type_conversion_p (TREE_TYPE (TREE_TYPE (vec)),
10120 61 : TREE_TYPE (gimple_assign_rhs2 (use_ass)))
10121 61 : || !constant_multiple_p
10122 61 : (tree_to_poly_uint64 (gimple_assign_rhs3 (use_ass)),
10123 356 : tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (TREE_TYPE (vec)))),
10124 : this_lane))
10125 : return false;
10126 : return true;
10127 : }
10128 :
10129 : /* Find any vectorizable constructors and add them to the grouped_store
10130 : array. */
10131 :
10132 : static void
10133 2349498 : vect_slp_check_for_roots (bb_vec_info bb_vinfo)
10134 : {
10135 18748341 : for (unsigned i = 0; i < bb_vinfo->nbbs; ++i)
10136 32797686 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[i]);
10137 152524186 : !gsi_end_p (gsi); gsi_next (&gsi))
10138 : {
10139 136125343 : gassign *assign = dyn_cast<gassign *> (gsi_stmt (gsi));
10140 : /* This can be used to start SLP discovery for early breaks for BB early breaks
10141 : when we get that far. */
10142 136125343 : if (!assign)
10143 104858507 : continue;
10144 :
10145 33451706 : tree rhs = gimple_assign_rhs1 (assign);
10146 33451706 : enum tree_code code = gimple_assign_rhs_code (assign);
10147 33451706 : use_operand_p use_p;
10148 33451706 : gimple *use_stmt;
10149 33451706 : if (code == CONSTRUCTOR)
10150 : {
10151 1799600 : if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10152 1846837 : || maybe_ne (TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)),
10153 103121 : CONSTRUCTOR_NELTS (rhs))
10154 47237 : || VECTOR_TYPE_P (TREE_TYPE (CONSTRUCTOR_ELT (rhs, 0)->value))
10155 1846817 : || uniform_vector_p (rhs))
10156 1784275 : continue;
10157 :
10158 : unsigned j;
10159 : tree val;
10160 76477 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10161 61152 : if (TREE_CODE (val) != SSA_NAME
10162 61152 : || !bb_vinfo->lookup_def (val))
10163 : break;
10164 37824 : if (j != CONSTRUCTOR_NELTS (rhs))
10165 3587 : continue;
10166 :
10167 15325 : vec<stmt_vec_info> roots = vNULL;
10168 15325 : roots.safe_push (bb_vinfo->lookup_stmt (assign));
10169 15325 : vec<stmt_vec_info> stmts;
10170 15325 : stmts.create (CONSTRUCTOR_NELTS (rhs));
10171 86160 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), j, val)
10172 55510 : stmts.quick_push
10173 55510 : (vect_stmt_to_vectorize (bb_vinfo->lookup_def (val)));
10174 15325 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10175 15325 : stmts, roots));
10176 : }
10177 31652106 : else if (code == BIT_INSERT_EXPR
10178 1022 : && VECTOR_TYPE_P (TREE_TYPE (rhs))
10179 721 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).is_constant ()
10180 721 : && TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs)).to_constant () > 1
10181 718 : && integer_zerop (gimple_assign_rhs3 (assign))
10182 392 : && useless_type_conversion_p
10183 392 : (TREE_TYPE (TREE_TYPE (rhs)),
10184 392 : TREE_TYPE (gimple_assign_rhs2 (assign)))
10185 31652830 : && bb_vinfo->lookup_def (gimple_assign_rhs2 (assign)))
10186 : {
10187 : /* We start to match on insert to lane zero but since the
10188 : inserts need not be ordered we'd have to search both
10189 : the def and the use chains. */
10190 257 : tree vectype = TREE_TYPE (rhs);
10191 257 : unsigned nlanes = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
10192 257 : auto_vec<std::pair<unsigned, tree> > lane_defs (nlanes);
10193 257 : auto_sbitmap lanes (nlanes);
10194 257 : bitmap_clear (lanes);
10195 257 : bitmap_set_bit (lanes, 0);
10196 257 : tree def = gimple_assign_lhs (assign);
10197 257 : lane_defs.quick_push
10198 257 : (std::make_pair (0, gimple_assign_rhs2 (assign)));
10199 257 : unsigned lanes_found = 1;
10200 : /* Start with the use chains, the last stmt will be the root. */
10201 257 : stmt_vec_info last = bb_vinfo->lookup_stmt (assign);
10202 257 : vec<stmt_vec_info> roots = vNULL;
10203 257 : roots.safe_push (last);
10204 268 : do
10205 : {
10206 268 : use_operand_p use_p;
10207 268 : gimple *use_stmt;
10208 268 : if (!single_imm_use (def, &use_p, &use_stmt))
10209 : break;
10210 256 : unsigned this_lane;
10211 256 : if (!bb_vinfo->lookup_stmt (use_stmt)
10212 256 : || !vect_slp_is_lane_insert (use_stmt, def, &this_lane)
10213 291 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (use_stmt)))
10214 : break;
10215 35 : if (bitmap_bit_p (lanes, this_lane))
10216 : break;
10217 15 : lanes_found++;
10218 15 : bitmap_set_bit (lanes, this_lane);
10219 15 : gassign *use_ass = as_a <gassign *> (use_stmt);
10220 15 : lane_defs.quick_push (std::make_pair
10221 15 : (this_lane, gimple_assign_rhs2 (use_ass)));
10222 15 : last = bb_vinfo->lookup_stmt (use_ass);
10223 15 : roots.safe_push (last);
10224 15 : def = gimple_assign_lhs (use_ass);
10225 : }
10226 15 : while (lanes_found < nlanes);
10227 257 : if (roots.length () > 1)
10228 7 : std::swap(roots[0], roots[roots.length () - 1]);
10229 257 : if (lanes_found < nlanes)
10230 : {
10231 : /* Now search the def chain. */
10232 253 : def = gimple_assign_rhs1 (assign);
10233 255 : do
10234 : {
10235 255 : if (TREE_CODE (def) != SSA_NAME
10236 255 : || !has_single_use (def))
10237 : break;
10238 59 : gimple *def_stmt = SSA_NAME_DEF_STMT (def);
10239 59 : unsigned this_lane;
10240 59 : if (!bb_vinfo->lookup_stmt (def_stmt)
10241 39 : || !vect_slp_is_lane_insert (def_stmt,
10242 : NULL_TREE, &this_lane)
10243 85 : || !bb_vinfo->lookup_def (gimple_assign_rhs2 (def_stmt)))
10244 : break;
10245 26 : if (bitmap_bit_p (lanes, this_lane))
10246 : break;
10247 6 : lanes_found++;
10248 6 : bitmap_set_bit (lanes, this_lane);
10249 12 : lane_defs.quick_push (std::make_pair
10250 6 : (this_lane,
10251 6 : gimple_assign_rhs2 (def_stmt)));
10252 6 : roots.safe_push (bb_vinfo->lookup_stmt (def_stmt));
10253 6 : def = gimple_assign_rhs1 (def_stmt);
10254 : }
10255 6 : while (lanes_found < nlanes);
10256 : }
10257 257 : if (lanes_found == nlanes)
10258 : {
10259 : /* Sort lane_defs after the lane index and register the root. */
10260 8 : lane_defs.qsort (vld_cmp);
10261 8 : vec<stmt_vec_info> stmts;
10262 8 : stmts.create (nlanes);
10263 44 : for (unsigned i = 0; i < nlanes; ++i)
10264 28 : stmts.quick_push (bb_vinfo->lookup_def (lane_defs[i].second));
10265 8 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_ctor,
10266 8 : stmts, roots));
10267 : }
10268 : else
10269 249 : roots.release ();
10270 257 : }
10271 31651849 : else if (!VECTOR_TYPE_P (TREE_TYPE (rhs))
10272 30579180 : && (associative_tree_code (code) || code == MINUS_EXPR)
10273 : /* Ops with constants at the tail can be stripped here. */
10274 6730933 : && TREE_CODE (rhs) == SSA_NAME
10275 6667390 : && TREE_CODE (gimple_assign_rhs2 (assign)) == SSA_NAME
10276 : /* Should be the chain end. */
10277 34549495 : && (!single_imm_use (gimple_assign_lhs (assign),
10278 : &use_p, &use_stmt)
10279 2335058 : || !is_gimple_assign (use_stmt)
10280 1733335 : || (gimple_assign_rhs_code (use_stmt) != code
10281 1164810 : && ((code != PLUS_EXPR && code != MINUS_EXPR)
10282 674807 : || (gimple_assign_rhs_code (use_stmt)
10283 674807 : != (code == PLUS_EXPR ? MINUS_EXPR : PLUS_EXPR))))))
10284 : {
10285 : /* We start the match at the end of a possible association
10286 : chain. */
10287 2211911 : auto_vec<chain_op_t> chain;
10288 2211911 : auto_vec<std::pair<tree_code, gimple *> > worklist;
10289 2211911 : auto_vec<gimple *> chain_stmts;
10290 2211911 : gimple *code_stmt = NULL, *alt_code_stmt = NULL;
10291 2211911 : if (code == MINUS_EXPR)
10292 369074 : code = PLUS_EXPR;
10293 2211911 : internal_fn reduc_fn;
10294 2612506 : if (!reduction_fn_for_scalar_code (code, &reduc_fn)
10295 2211911 : || reduc_fn == IFN_LAST)
10296 400595 : continue;
10297 1811316 : vect_slp_linearize_chain (bb_vinfo, worklist, chain, code, assign,
10298 : /* ??? */
10299 : code_stmt, alt_code_stmt, &chain_stmts,
10300 : false);
10301 3622632 : if (chain.length () > 1)
10302 : {
10303 : /* Sort the chain according to def_type and operation. */
10304 1811316 : chain.sort (dt_sort_cmp, bb_vinfo);
10305 : /* ??? Now we'd want to strip externals and constants
10306 : but record those to be handled in the epilogue. */
10307 : /* ??? For now do not allow mixing ops or externs/constants. */
10308 1811316 : bool invalid = false;
10309 1811316 : unsigned remain_cnt = 0;
10310 7469127 : for (unsigned i = 0; i < chain.length (); ++i)
10311 : {
10312 4215569 : if (chain[i].code != code)
10313 : {
10314 : invalid = true;
10315 : break;
10316 : }
10317 3846495 : if (chain[i].dt != vect_internal_def
10318 : /* Avoid stmts where the def is not the LHS, like
10319 : ASMs. */
10320 7320902 : || (gimple_get_lhs (bb_vinfo->lookup_def
10321 3474407 : (chain[i].op)->stmt)
10322 3474407 : != chain[i].op))
10323 375184 : remain_cnt++;
10324 : }
10325 1811316 : if (!invalid && chain.length () - remain_cnt > 1)
10326 : {
10327 1357493 : vec<stmt_vec_info> stmts;
10328 1357493 : vec<tree> remain = vNULL;
10329 1357493 : stmts.create (chain.length ());
10330 1357493 : if (remain_cnt > 0)
10331 52313 : remain.create (remain_cnt);
10332 4526416 : for (unsigned i = 0; i < chain.length (); ++i)
10333 : {
10334 3168923 : stmt_vec_info stmt_info;
10335 3168923 : if (chain[i].dt == vect_internal_def
10336 3168923 : && ((stmt_info = bb_vinfo->lookup_def (chain[i].op)),
10337 3062255 : gimple_get_lhs (stmt_info->stmt) == chain[i].op))
10338 3062039 : stmts.quick_push (stmt_info);
10339 : else
10340 106884 : remain.quick_push (chain[i].op);
10341 : }
10342 1357493 : vec<stmt_vec_info> roots;
10343 1357493 : roots.create (chain_stmts.length ());
10344 4526416 : for (unsigned i = 0; i < chain_stmts.length (); ++i)
10345 1811430 : roots.quick_push (bb_vinfo->lookup_stmt (chain_stmts[i]));
10346 1357493 : bb_vinfo->roots.safe_push (slp_root (slp_inst_kind_bb_reduc,
10347 1357493 : stmts, roots, remain));
10348 : }
10349 : }
10350 2211911 : }
10351 : }
10352 2349498 : }
10353 :
10354 : /* Walk the grouped store chains and replace entries with their
10355 : pattern variant if any. */
10356 :
10357 : static void
10358 678888 : vect_fixup_store_groups_with_patterns (vec_info *vinfo)
10359 : {
10360 678888 : stmt_vec_info first_element;
10361 678888 : unsigned i;
10362 :
10363 1597431 : FOR_EACH_VEC_ELT (vinfo->grouped_stores, i, first_element)
10364 : {
10365 : /* We also have CTORs in this array. */
10366 918543 : if (!STMT_VINFO_GROUPED_ACCESS (first_element))
10367 0 : continue;
10368 918543 : if (STMT_VINFO_IN_PATTERN_P (first_element))
10369 : {
10370 255 : stmt_vec_info orig = first_element;
10371 255 : first_element = STMT_VINFO_RELATED_STMT (first_element);
10372 255 : DR_GROUP_FIRST_ELEMENT (first_element) = first_element;
10373 255 : DR_GROUP_SIZE (first_element) = DR_GROUP_SIZE (orig);
10374 255 : DR_GROUP_GAP (first_element) = DR_GROUP_GAP (orig);
10375 255 : DR_GROUP_NEXT_ELEMENT (first_element) = DR_GROUP_NEXT_ELEMENT (orig);
10376 255 : vinfo->grouped_stores[i] = first_element;
10377 : }
10378 918543 : stmt_vec_info prev = first_element;
10379 2582831 : while (DR_GROUP_NEXT_ELEMENT (prev))
10380 : {
10381 1664288 : stmt_vec_info elt = DR_GROUP_NEXT_ELEMENT (prev);
10382 1664288 : if (STMT_VINFO_IN_PATTERN_P (elt))
10383 : {
10384 870 : stmt_vec_info orig = elt;
10385 870 : elt = STMT_VINFO_RELATED_STMT (elt);
10386 870 : DR_GROUP_NEXT_ELEMENT (prev) = elt;
10387 870 : DR_GROUP_GAP (elt) = DR_GROUP_GAP (orig);
10388 870 : DR_GROUP_NEXT_ELEMENT (elt) = DR_GROUP_NEXT_ELEMENT (orig);
10389 : }
10390 1664288 : DR_GROUP_FIRST_ELEMENT (elt) = first_element;
10391 1664288 : prev = elt;
10392 : }
10393 : }
10394 678888 : }
10395 :
10396 : /* Check if the region described by BB_VINFO can be vectorized, returning
10397 : true if so. When returning false, set FATAL to true if the same failure
10398 : would prevent vectorization at other vector sizes, false if it is still
10399 : worth trying other sizes. N_STMTS is the number of statements in the
10400 : region. */
10401 :
10402 : static bool
10403 2349498 : vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal,
10404 : vec<int> *dataref_groups)
10405 : {
10406 2349498 : DUMP_VECT_SCOPE ("vect_slp_analyze_bb");
10407 :
10408 2349498 : slp_instance instance;
10409 2349498 : int i;
10410 :
10411 : /* The first group of checks is independent of the vector size. */
10412 2349498 : fatal = true;
10413 :
10414 : /* Analyze the data references. */
10415 :
10416 2349498 : if (!vect_analyze_data_refs (bb_vinfo, NULL))
10417 : {
10418 0 : if (dump_enabled_p ())
10419 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10420 : "not vectorized: unhandled data-ref in basic "
10421 : "block.\n");
10422 : return false;
10423 : }
10424 :
10425 2349498 : if (!vect_analyze_data_ref_accesses (bb_vinfo, dataref_groups))
10426 : {
10427 0 : if (dump_enabled_p ())
10428 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10429 : "not vectorized: unhandled data access in "
10430 : "basic block.\n");
10431 : return false;
10432 : }
10433 :
10434 2349498 : vect_slp_check_for_roots (bb_vinfo);
10435 :
10436 : /* If there are no grouped stores and no constructors in the region
10437 : there is no need to continue with pattern recog as vect_analyze_slp
10438 : will fail anyway. */
10439 2349498 : if (bb_vinfo->grouped_stores.is_empty ()
10440 1994518 : && bb_vinfo->roots.is_empty ())
10441 : {
10442 1670610 : if (dump_enabled_p ())
10443 1033 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10444 : "not vectorized: no grouped stores in "
10445 : "basic block.\n");
10446 : return false;
10447 : }
10448 :
10449 : /* While the rest of the analysis below depends on it in some way. */
10450 678888 : fatal = false;
10451 :
10452 678888 : vect_pattern_recog (bb_vinfo);
10453 :
10454 : /* Update store groups from pattern processing. */
10455 678888 : vect_fixup_store_groups_with_patterns (bb_vinfo);
10456 :
10457 : /* Check the SLP opportunities in the basic block, analyze and build SLP
10458 : trees. */
10459 678888 : if (!vect_analyze_slp (bb_vinfo, n_stmts, false))
10460 : {
10461 0 : if (dump_enabled_p ())
10462 : {
10463 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10464 : "Failed to SLP the basic block.\n");
10465 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10466 : "not vectorized: failed to find SLP opportunities "
10467 : "in basic block.\n");
10468 : }
10469 : return false;
10470 : }
10471 :
10472 : /* Optimize permutations. */
10473 678888 : vect_optimize_slp (bb_vinfo);
10474 :
10475 : /* Gather the loads reachable from the SLP graph entries. */
10476 678888 : vect_gather_slp_loads (bb_vinfo);
10477 :
10478 678888 : vect_record_base_alignments (bb_vinfo);
10479 :
10480 : /* Analyze and verify the alignment of data references and the
10481 : dependence in the SLP instances. */
10482 2192197 : for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); )
10483 : {
10484 834421 : vect_location = instance->location ();
10485 834421 : if (! vect_slp_analyze_instance_alignment (bb_vinfo, instance)
10486 834421 : || ! vect_slp_analyze_instance_dependence (bb_vinfo, instance))
10487 : {
10488 8985 : slp_tree node = SLP_INSTANCE_TREE (instance);
10489 8985 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
10490 8985 : if (dump_enabled_p ())
10491 4 : dump_printf_loc (MSG_NOTE, vect_location,
10492 : "removing SLP instance operations starting from: %G",
10493 : stmt_info->stmt);
10494 8985 : vect_free_slp_instance (instance);
10495 8985 : BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i);
10496 8985 : continue;
10497 8985 : }
10498 :
10499 : /* Mark all the statements that we want to vectorize as relevant. */
10500 825436 : vect_mark_slp_stmts_relevant (SLP_INSTANCE_TREE (instance));
10501 :
10502 825436 : i++;
10503 : }
10504 2630606 : if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ())
10505 : return false;
10506 :
10507 281108 : if (!vect_slp_analyze_operations (bb_vinfo))
10508 : {
10509 32487 : if (dump_enabled_p ())
10510 73 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10511 : "not vectorized: bad operation in basic block.\n");
10512 : return false;
10513 : }
10514 :
10515 : /* Compute vector stmt placement. */
10516 248621 : if (!vect_schedule_slp (bb_vinfo, BB_VINFO_SLP_INSTANCES (bb_vinfo), true))
10517 : {
10518 0 : if (dump_enabled_p ())
10519 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10520 : "not vectorized: cannot schedule SLP graph\n");
10521 : return false;
10522 : }
10523 :
10524 : /* Mark all the statements that we vectorize. */
10525 248621 : vect_bb_slp_mark_stmts_vectorized (bb_vinfo);
10526 :
10527 : /* Compute vectorizable live stmts. */
10528 248621 : vect_bb_slp_mark_live_stmts (bb_vinfo);
10529 :
10530 248621 : vect_bb_partition_graph (bb_vinfo);
10531 :
10532 248621 : return true;
10533 : }
10534 :
10535 : /* Subroutine of vect_slp_bb. Try to vectorize the statements for all
10536 : basic blocks in BBS, returning true on success.
10537 : The region has N_STMTS statements and has the datarefs given by DATAREFS. */
10538 :
10539 : static bool
10540 1981444 : vect_slp_region (vec<basic_block> bbs, vec<data_reference_p> datarefs,
10541 : vec<int> *dataref_groups, unsigned int n_stmts,
10542 : loop_p orig_loop)
10543 : {
10544 1981444 : bb_vec_info bb_vinfo;
10545 1981444 : auto_vector_modes vector_modes;
10546 :
10547 : /* Autodetect first vector size we try. */
10548 1981444 : machine_mode next_vector_mode = VOIDmode;
10549 1981444 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, false);
10550 1981444 : unsigned int mode_i = 0;
10551 :
10552 1981444 : vec_info_shared shared;
10553 :
10554 1981444 : machine_mode autodetected_vector_mode = VOIDmode;
10555 2717552 : while (1)
10556 : {
10557 2349498 : bool vectorized = false;
10558 2349498 : bool fatal = false;
10559 2349498 : bb_vinfo = new _bb_vec_info (bbs, &shared);
10560 :
10561 2349498 : bool first_time_p = shared.datarefs.is_empty ();
10562 2349498 : BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
10563 2349498 : if (first_time_p)
10564 2006247 : bb_vinfo->shared->save_datarefs ();
10565 : else
10566 343251 : bb_vinfo->shared->check_datarefs ();
10567 2349498 : bb_vinfo->vector_mode = next_vector_mode;
10568 :
10569 2349498 : if (vect_slp_analyze_bb_1 (bb_vinfo, n_stmts, fatal, dataref_groups))
10570 : {
10571 248621 : if (dump_enabled_p ())
10572 : {
10573 1668 : dump_printf_loc (MSG_NOTE, vect_location,
10574 : "***** Analysis succeeded with vector mode"
10575 834 : " %s\n", GET_MODE_NAME (bb_vinfo->vector_mode));
10576 834 : dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
10577 : }
10578 :
10579 248621 : bb_vinfo->shared->check_datarefs ();
10580 :
10581 248621 : bool force_clear = false;
10582 248621 : auto_vec<slp_instance> profitable_subgraphs;
10583 1466595 : for (slp_instance instance : BB_VINFO_SLP_INSTANCES (bb_vinfo))
10584 : {
10585 720732 : if (instance->subgraph_entries.is_empty ())
10586 239384 : continue;
10587 :
10588 693493 : dump_user_location_t saved_vect_location = vect_location;
10589 693493 : vect_location = instance->location ();
10590 693493 : if (!unlimited_cost_model (NULL)
10591 689499 : && !param_vect_allow_possibly_not_worthwhile_vectorizations
10592 1382983 : && !vect_bb_vectorization_profitable_p
10593 689490 : (bb_vinfo, instance->subgraph_entries, orig_loop))
10594 : {
10595 212145 : if (dump_enabled_p ())
10596 51 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10597 : "not vectorized: vectorization is not "
10598 : "profitable.\n");
10599 212145 : vect_location = saved_vect_location;
10600 212145 : continue;
10601 : }
10602 :
10603 481348 : vect_location = saved_vect_location;
10604 481348 : if (!dbg_cnt (vect_slp))
10605 : {
10606 0 : force_clear = true;
10607 0 : continue;
10608 : }
10609 :
10610 481348 : profitable_subgraphs.safe_push (instance);
10611 : }
10612 :
10613 : /* When we're vectorizing an if-converted loop body make sure
10614 : we vectorized all if-converted code. */
10615 406293 : if ((!profitable_subgraphs.is_empty () || force_clear) && orig_loop)
10616 : {
10617 126 : gcc_assert (bb_vinfo->nbbs == 1);
10618 252 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb_vinfo->bbs[0]);
10619 5753 : !gsi_end_p (gsi); gsi_next (&gsi))
10620 : {
10621 : /* The costing above left us with DCEable vectorized scalar
10622 : stmts having the visited flag set on profitable
10623 : subgraphs. Do the delayed clearing of the flag here. */
10624 5627 : if (gimple_visited_p (gsi_stmt (gsi)))
10625 : {
10626 1816 : gimple_set_visited (gsi_stmt (gsi), false);
10627 1816 : continue;
10628 : }
10629 3811 : if (flag_vect_cost_model == VECT_COST_MODEL_UNLIMITED)
10630 813 : continue;
10631 :
10632 8370 : if (gassign *ass = dyn_cast <gassign *> (gsi_stmt (gsi)))
10633 3456 : if (gimple_assign_rhs_code (ass) == COND_EXPR)
10634 : {
10635 151 : if (!profitable_subgraphs.is_empty ()
10636 66 : && dump_enabled_p ())
10637 0 : dump_printf_loc (MSG_NOTE, vect_location,
10638 : "not profitable because of "
10639 : "unprofitable if-converted scalar "
10640 : "code\n");
10641 85 : profitable_subgraphs.truncate (0);
10642 : }
10643 : }
10644 : }
10645 :
10646 : /* Finally schedule the profitable subgraphs. */
10647 1045230 : for (slp_instance instance : profitable_subgraphs)
10648 : {
10649 481265 : if (!vectorized && dump_enabled_p ())
10650 795 : dump_printf_loc (MSG_NOTE, vect_location,
10651 : "Basic block will be vectorized "
10652 : "using SLP\n");
10653 481265 : vectorized = true;
10654 :
10655 : /* Dump before scheduling as store vectorization will remove
10656 : the original stores and mess with the instance tree
10657 : so querying its location will eventually ICE. */
10658 481265 : if (flag_checking)
10659 1936317 : for (slp_instance sub : instance->subgraph_entries)
10660 492522 : gcc_assert (SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub)));
10661 481265 : unsigned HOST_WIDE_INT bytes;
10662 481265 : if (dump_enabled_p ())
10663 3863 : for (slp_instance sub : instance->subgraph_entries)
10664 : {
10665 1040 : tree vtype = SLP_TREE_VECTYPE (SLP_INSTANCE_TREE (sub));
10666 2080 : if (GET_MODE_SIZE (TYPE_MODE (vtype)).is_constant (&bytes))
10667 1040 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10668 1040 : sub->location (),
10669 : "basic block part vectorized using %wu "
10670 : "byte vectors\n", bytes);
10671 : else
10672 : dump_printf_loc (MSG_OPTIMIZED_LOCATIONS,
10673 : sub->location (),
10674 : "basic block part vectorized using "
10675 : "variable length vectors\n");
10676 : }
10677 :
10678 481265 : dump_user_location_t saved_vect_location = vect_location;
10679 481265 : vect_location = instance->location ();
10680 :
10681 481265 : vect_schedule_slp (bb_vinfo, instance->subgraph_entries, false);
10682 :
10683 481265 : vect_location = saved_vect_location;
10684 : }
10685 :
10686 : /* Generate the invariant statements. */
10687 248621 : if (!gimple_seq_empty_p (bb_vinfo->inv_pattern_def_seq))
10688 : {
10689 23 : if (dump_enabled_p ())
10690 0 : dump_printf_loc (MSG_NOTE, vect_location,
10691 : "------>generating invariant statements\n");
10692 :
10693 23 : bb_vinfo->insert_seq_on_entry (NULL,
10694 : bb_vinfo->inv_pattern_def_seq);
10695 : }
10696 248621 : }
10697 : else
10698 : {
10699 2100877 : if (dump_enabled_p ())
10700 1355 : dump_printf_loc (MSG_NOTE, vect_location,
10701 : "***** Analysis failed with vector mode %s\n",
10702 1355 : GET_MODE_NAME (bb_vinfo->vector_mode));
10703 : }
10704 :
10705 2349498 : if (mode_i == 0)
10706 1981444 : autodetected_vector_mode = bb_vinfo->vector_mode;
10707 :
10708 2349498 : if (!fatal)
10709 3386355 : while (mode_i < vector_modes.length ()
10710 1920442 : && vect_chooses_same_modes_p (bb_vinfo, vector_modes[mode_i]))
10711 : {
10712 357969 : if (dump_enabled_p ())
10713 1818 : dump_printf_loc (MSG_NOTE, vect_location,
10714 : "***** The result for vector mode %s would"
10715 : " be the same\n",
10716 909 : GET_MODE_NAME (vector_modes[mode_i]));
10717 357969 : mode_i += 1;
10718 : }
10719 :
10720 2349498 : delete bb_vinfo;
10721 :
10722 2349498 : if (mode_i < vector_modes.length ()
10723 2147676 : && VECTOR_MODE_P (autodetected_vector_mode)
10724 2120578 : && (related_vector_mode (vector_modes[mode_i],
10725 : GET_MODE_INNER (autodetected_vector_mode))
10726 1060289 : == autodetected_vector_mode)
10727 4497174 : && (related_vector_mode (autodetected_vector_mode,
10728 534527 : GET_MODE_INNER (vector_modes[mode_i]))
10729 1069054 : == vector_modes[mode_i]))
10730 : {
10731 534527 : if (dump_enabled_p ())
10732 214 : dump_printf_loc (MSG_NOTE, vect_location,
10733 : "***** Skipping vector mode %s, which would"
10734 : " repeat the analysis for %s\n",
10735 214 : GET_MODE_NAME (vector_modes[mode_i]),
10736 214 : GET_MODE_NAME (autodetected_vector_mode));
10737 534527 : mode_i += 1;
10738 : }
10739 :
10740 2349498 : if (vectorized
10741 2191892 : || mode_i == vector_modes.length ()
10742 1990114 : || autodetected_vector_mode == VOIDmode
10743 : /* If vect_slp_analyze_bb_1 signaled that analysis for all
10744 : vector sizes will fail do not bother iterating. */
10745 3252225 : || fatal)
10746 3962888 : return vectorized;
10747 :
10748 : /* Try the next biggest vector size. */
10749 368054 : next_vector_mode = vector_modes[mode_i++];
10750 368054 : if (dump_enabled_p ())
10751 249 : dump_printf_loc (MSG_NOTE, vect_location,
10752 : "***** Re-trying analysis with vector mode %s\n",
10753 249 : GET_MODE_NAME (next_vector_mode));
10754 368054 : }
10755 1981444 : }
10756 :
10757 :
10758 : /* Main entry for the BB vectorizer. Analyze and transform BBS, returns
10759 : true if anything in the basic-block was vectorized. */
10760 :
10761 : static bool
10762 1981444 : vect_slp_bbs (const vec<basic_block> &bbs, loop_p orig_loop)
10763 : {
10764 1981444 : vec<data_reference_p> datarefs = vNULL;
10765 1981444 : auto_vec<int> dataref_groups;
10766 1981444 : int insns = 0;
10767 1981444 : int current_group = 0;
10768 :
10769 13057665 : for (unsigned i = 0; i < bbs.length (); i++)
10770 : {
10771 11076221 : basic_block bb = bbs[i];
10772 97426602 : for (gimple_stmt_iterator gsi = gsi_after_labels (bb); !gsi_end_p (gsi);
10773 86350381 : gsi_next (&gsi))
10774 : {
10775 86350381 : gimple *stmt = gsi_stmt (gsi);
10776 86350381 : if (is_gimple_debug (stmt))
10777 55252149 : continue;
10778 :
10779 31098232 : insns++;
10780 :
10781 31098232 : if (gimple_location (stmt) != UNKNOWN_LOCATION)
10782 27884561 : vect_location = stmt;
10783 :
10784 31098232 : if (!vect_find_stmt_data_reference (NULL, stmt, &datarefs,
10785 : &dataref_groups, current_group))
10786 5332846 : ++current_group;
10787 : }
10788 : /* New BBs always start a new DR group. */
10789 11076221 : ++current_group;
10790 : }
10791 :
10792 1981444 : return vect_slp_region (bbs, datarefs, &dataref_groups, insns, orig_loop);
10793 1981444 : }
10794 :
10795 : /* Special entry for the BB vectorizer. Analyze and transform a single
10796 : if-converted BB with ORIG_LOOPs body being the not if-converted
10797 : representation. Returns true if anything in the basic-block was
10798 : vectorized. */
10799 :
10800 : bool
10801 16796 : vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
10802 : {
10803 16796 : auto_vec<basic_block> bbs;
10804 16796 : bbs.safe_push (bb);
10805 16796 : return vect_slp_bbs (bbs, orig_loop);
10806 16796 : }
10807 :
10808 : /* Main entry for the BB vectorizer. Analyze and transform BB, returns
10809 : true if anything in the basic-block was vectorized. */
10810 :
10811 : bool
10812 932281 : vect_slp_function (function *fun)
10813 : {
10814 932281 : bool r = false;
10815 932281 : int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fun));
10816 932281 : auto_bitmap exit_bbs;
10817 932281 : bitmap_set_bit (exit_bbs, EXIT_BLOCK);
10818 932281 : edge entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
10819 932281 : unsigned n = rev_post_order_and_mark_dfs_back_seme (fun, entry, exit_bbs,
10820 932281 : true, rpo, NULL);
10821 :
10822 : /* For the moment split the function into pieces to avoid making
10823 : the iteration on the vector mode moot. Split at points we know
10824 : to not handle well which is CFG merges (SLP discovery doesn't
10825 : handle non-loop-header PHIs) and loop exits. Since pattern
10826 : recog requires reverse iteration to visit uses before defs
10827 : simply chop RPO into pieces. */
10828 932281 : auto_vec<basic_block> bbs;
10829 12018123 : for (unsigned i = 0; i < n; i++)
10830 : {
10831 11085842 : basic_block bb = BASIC_BLOCK_FOR_FN (fun, rpo[i]);
10832 11085842 : bool split = false;
10833 :
10834 : /* Split when a BB is not dominated by the first block. */
10835 20916328 : if (!bbs.is_empty ()
10836 9830486 : && !dominated_by_p (CDI_DOMINATORS, bb, bbs[0]))
10837 : {
10838 725651 : if (dump_enabled_p ())
10839 146 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10840 : "splitting region at dominance boundary bb%d\n",
10841 : bb->index);
10842 : split = true;
10843 : }
10844 : /* Split when the loop determined by the first block
10845 : is exited. This is because we eventually insert
10846 : invariants at region begin. */
10847 19465026 : else if (!bbs.is_empty ()
10848 9104835 : && bbs[0]->loop_father != bb->loop_father
10849 2366736 : && !flow_loop_nested_p (bbs[0]->loop_father, bb->loop_father))
10850 : {
10851 3868 : if (dump_enabled_p ())
10852 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10853 : "splitting region at loop %d exit at bb%d\n",
10854 3 : bbs[0]->loop_father->num, bb->index);
10855 : split = true;
10856 : }
10857 10356323 : else if (!bbs.is_empty ()
10858 9100967 : && bb->loop_father->header == bb
10859 482291 : && bb->loop_father->dont_vectorize)
10860 : {
10861 6190 : if (dump_enabled_p ())
10862 72 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10863 : "splitting region at dont-vectorize loop %d "
10864 : "entry at bb%d\n",
10865 : bb->loop_father->num, bb->index);
10866 : split = true;
10867 : }
10868 :
10869 221 : if (split && !bbs.is_empty ())
10870 : {
10871 735709 : r |= vect_slp_bbs (bbs, NULL);
10872 735709 : bbs.truncate (0);
10873 : }
10874 :
10875 11085842 : if (bbs.is_empty ())
10876 : {
10877 : /* We need to be able to insert at the head of the region which
10878 : we cannot for region starting with a returns-twice call. */
10879 1991065 : if (gcall *first = safe_dyn_cast <gcall *> (first_stmt (bb)))
10880 423512 : if (gimple_call_flags (first) & ECF_RETURNS_TWICE)
10881 : {
10882 306 : if (dump_enabled_p ())
10883 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10884 : "skipping bb%d as start of region as it "
10885 : "starts with returns-twice call\n",
10886 : bb->index);
10887 26417 : continue;
10888 : }
10889 : /* If the loop this BB belongs to is marked as not to be vectorized
10890 : honor that also for BB vectorization. */
10891 1990759 : if (bb->loop_father->dont_vectorize)
10892 26111 : continue;
10893 : }
10894 :
10895 11059425 : bbs.safe_push (bb);
10896 :
10897 : /* When we have a stmt ending this block and defining a
10898 : value we have to insert on edges when inserting after it for
10899 : a vector containing its definition. Avoid this for now. */
10900 22118850 : if (gimple *last = *gsi_last_bb (bb))
10901 9012996 : if (gimple_get_lhs (last)
10902 9012996 : && is_ctrl_altering_stmt (last))
10903 : {
10904 296665 : if (dump_enabled_p ())
10905 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
10906 : "splitting region at control altering "
10907 : "definition %G", last);
10908 296665 : r |= vect_slp_bbs (bbs, NULL);
10909 296665 : bbs.truncate (0);
10910 : }
10911 : }
10912 :
10913 932281 : if (!bbs.is_empty ())
10914 932274 : r |= vect_slp_bbs (bbs, NULL);
10915 :
10916 932281 : free (rpo);
10917 :
10918 932281 : return r;
10919 932281 : }
10920 :
10921 : /* Build a variable-length vector in which the elements in ELTS are repeated
10922 : to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
10923 : RESULTS and add any new instructions to SEQ.
10924 :
10925 : The approach we use is:
10926 :
10927 : (1) Find a vector mode VM with integer elements of mode IM.
10928 :
10929 : (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10930 : ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
10931 : from small vectors to IM.
10932 :
10933 : (3) Duplicate each ELTS'[I] into a vector of mode VM.
10934 :
10935 : (4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
10936 : correct byte contents.
10937 :
10938 : (5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
10939 :
10940 : We try to find the largest IM for which this sequence works, in order
10941 : to cut down on the number of interleaves. */
10942 :
10943 : void
10944 0 : duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type,
10945 : const vec<tree> &elts, unsigned int nresults,
10946 : vec<tree> &results)
10947 : {
10948 0 : unsigned int nelts = elts.length ();
10949 0 : tree element_type = TREE_TYPE (vector_type);
10950 :
10951 : /* (1) Find a vector mode VM with integer elements of mode IM. */
10952 0 : unsigned int nvectors = 1;
10953 0 : tree new_vector_type;
10954 0 : tree permutes[2];
10955 0 : if (!can_duplicate_and_interleave_p (vinfo, nelts, element_type,
10956 : &nvectors, &new_vector_type,
10957 : permutes))
10958 0 : gcc_unreachable ();
10959 :
10960 : /* Get a vector type that holds ELTS[0:NELTS/NELTS']. */
10961 0 : unsigned int partial_nelts = nelts / nvectors;
10962 0 : tree partial_vector_type = build_vector_type (element_type, partial_nelts);
10963 :
10964 0 : tree_vector_builder partial_elts;
10965 0 : auto_vec<tree, 32> pieces (nvectors * 2);
10966 0 : pieces.quick_grow_cleared (nvectors * 2);
10967 0 : for (unsigned int i = 0; i < nvectors; ++i)
10968 : {
10969 : /* (2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
10970 : ELTS' has mode IM. */
10971 0 : partial_elts.new_vector (partial_vector_type, partial_nelts, 1);
10972 0 : for (unsigned int j = 0; j < partial_nelts; ++j)
10973 0 : partial_elts.quick_push (elts[i * partial_nelts + j]);
10974 0 : tree t = gimple_build_vector (seq, &partial_elts);
10975 0 : t = gimple_build (seq, VIEW_CONVERT_EXPR,
10976 0 : TREE_TYPE (new_vector_type), t);
10977 :
10978 : /* (3) Duplicate each ELTS'[I] into a vector of mode VM. */
10979 0 : pieces[i] = gimple_build_vector_from_val (seq, new_vector_type, t);
10980 : }
10981 :
10982 : /* (4) Use a tree of VEC_PERM_EXPRs to create a single VM with the
10983 : correct byte contents.
10984 :
10985 : Conceptually, we need to repeat the following operation log2(nvectors)
10986 : times, where hi_start = nvectors / 2:
10987 :
10988 : out[i * 2] = VEC_PERM_EXPR (in[i], in[i + hi_start], lo_permute);
10989 : out[i * 2 + 1] = VEC_PERM_EXPR (in[i], in[i + hi_start], hi_permute);
10990 :
10991 : However, if each input repeats every N elements and the VF is
10992 : a multiple of N * 2, the HI result is the same as the LO result.
10993 : This will be true for the first N1 iterations of the outer loop,
10994 : followed by N2 iterations for which both the LO and HI results
10995 : are needed. I.e.:
10996 :
10997 : N1 + N2 = log2(nvectors)
10998 :
10999 : Each "N1 iteration" doubles the number of redundant vectors and the
11000 : effect of the process as a whole is to have a sequence of nvectors/2**N1
11001 : vectors that repeats 2**N1 times. Rather than generate these redundant
11002 : vectors, we halve the number of vectors for each N1 iteration. */
11003 : unsigned int in_start = 0;
11004 : unsigned int out_start = nvectors;
11005 : unsigned int new_nvectors = nvectors;
11006 0 : for (unsigned int in_repeat = 1; in_repeat < nvectors; in_repeat *= 2)
11007 : {
11008 0 : unsigned int hi_start = new_nvectors / 2;
11009 0 : unsigned int out_i = 0;
11010 0 : for (unsigned int in_i = 0; in_i < new_nvectors; ++in_i)
11011 : {
11012 0 : if ((in_i & 1) != 0
11013 0 : && multiple_p (TYPE_VECTOR_SUBPARTS (new_vector_type),
11014 : 2 * in_repeat))
11015 0 : continue;
11016 :
11017 0 : tree output = make_ssa_name (new_vector_type);
11018 0 : tree input1 = pieces[in_start + (in_i / 2)];
11019 0 : tree input2 = pieces[in_start + (in_i / 2) + hi_start];
11020 0 : gassign *stmt = gimple_build_assign (output, VEC_PERM_EXPR,
11021 : input1, input2,
11022 : permutes[in_i & 1]);
11023 0 : gimple_seq_add_stmt (seq, stmt);
11024 0 : pieces[out_start + out_i] = output;
11025 0 : out_i += 1;
11026 : }
11027 0 : std::swap (in_start, out_start);
11028 0 : new_nvectors = out_i;
11029 : }
11030 :
11031 : /* (5) Use VIEW_CONVERT_EXPR to cast the final VM to the required type. */
11032 0 : results.reserve (nresults);
11033 0 : for (unsigned int i = 0; i < nresults; ++i)
11034 0 : if (i < new_nvectors)
11035 0 : results.quick_push (gimple_build (seq, VIEW_CONVERT_EXPR, vector_type,
11036 0 : pieces[in_start + i]));
11037 : else
11038 0 : results.quick_push (results[i - new_nvectors]);
11039 0 : }
11040 :
11041 :
11042 : /* For constant and loop invariant defs in OP_NODE this function creates
11043 : vector defs that will be used in the vectorized stmts and stores them
11044 : to SLP_TREE_VEC_DEFS of OP_NODE. */
11045 :
11046 : static void
11047 504199 : vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
11048 : {
11049 504199 : unsigned HOST_WIDE_INT nunits;
11050 504199 : tree vec_cst;
11051 504199 : unsigned j, number_of_places_left_in_vector;
11052 504199 : tree vector_type;
11053 504199 : tree vop;
11054 504199 : int group_size = op_node->ops.length ();
11055 504199 : unsigned int vec_num, i;
11056 504199 : unsigned number_of_copies = 1;
11057 504199 : bool constant_p;
11058 504199 : gimple_seq ctor_seq = NULL;
11059 504199 : auto_vec<tree, 16> permute_results;
11060 :
11061 : /* We always want SLP_TREE_VECTYPE (op_node) here correctly set. */
11062 504199 : vector_type = SLP_TREE_VECTYPE (op_node);
11063 :
11064 504199 : unsigned int number_of_vectors, excess_elts;
11065 504199 : bool res = vect_get_num_copies_for_invariant (vinfo, op_node,
11066 : &number_of_vectors,
11067 : &excess_elts);
11068 504199 : gcc_assert (res);
11069 504199 : SLP_TREE_VEC_DEFS (op_node).create (number_of_vectors);
11070 504199 : auto_vec<tree> voprnds (number_of_vectors);
11071 :
11072 : /* NUMBER_OF_COPIES is the number of times we need to use the same values in
11073 : created vectors. It is greater than 1 if unrolling is performed.
11074 :
11075 : For example, we have two scalar operands, s1 and s2 (e.g., group of
11076 : strided accesses of size two), while NUNITS is four (i.e., four scalars
11077 : of this type can be packed in a vector). The output vector will contain
11078 : two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
11079 : will be 2).
11080 :
11081 : If GROUP_SIZE > NUNITS, the scalars will be split into several vectors
11082 : containing the operands.
11083 :
11084 : For example, NUNITS is four as before, and the group size is 8
11085 : (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
11086 : {s5, s6, s7, s8}. */
11087 :
11088 : /* When using duplicate_and_interleave, we just need one element for
11089 : each scalar statement. */
11090 504199 : if (!TYPE_VECTOR_SUBPARTS (vector_type).is_constant (&nunits))
11091 : nunits = group_size;
11092 :
11093 504199 : number_of_copies = (nunits * number_of_vectors - excess_elts) / group_size;
11094 :
11095 504199 : constant_p = true;
11096 504199 : tree uniform_elt = NULL_TREE;
11097 504199 : tree_vector_builder elts (vector_type, nunits, 1);
11098 504199 : elts.quick_grow (nunits);
11099 : /* Zero-pad the last vector if necessary. */
11100 504199 : number_of_places_left_in_vector = nunits - excess_elts;
11101 504457 : for (i = nunits; i > number_of_places_left_in_vector; --i)
11102 258 : elts[i-1] = build_zero_cst (TREE_TYPE (vector_type));
11103 : stmt_vec_info insert_after = NULL;
11104 1504962 : for (j = 0; j < number_of_copies; j++)
11105 : {
11106 1000763 : tree op;
11107 3832161 : for (i = group_size - 1; op_node->ops.iterate (i, &op); i--)
11108 : {
11109 : /* Create 'vect_ = {op0,op1,...,opn}'. */
11110 1830635 : tree orig_op = op;
11111 1830635 : if (number_of_places_left_in_vector == nunits)
11112 : uniform_elt = op;
11113 1192590 : else if (uniform_elt && operand_equal_p (uniform_elt, op))
11114 757583 : op = elts[number_of_places_left_in_vector];
11115 : else
11116 : uniform_elt = NULL_TREE;
11117 1830635 : number_of_places_left_in_vector--;
11118 1830635 : if (!types_compatible_p (TREE_TYPE (vector_type), TREE_TYPE (op)))
11119 : {
11120 280329 : if (CONSTANT_CLASS_P (op))
11121 : {
11122 101703 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
11123 : {
11124 : /* Can't use VIEW_CONVERT_EXPR for booleans because
11125 : of possibly different sizes of scalar value and
11126 : vector element. */
11127 66 : if (integer_zerop (op))
11128 66 : op = build_int_cst (TREE_TYPE (vector_type), 0);
11129 0 : else if (integer_onep (op))
11130 0 : op = build_all_ones_cst (TREE_TYPE (vector_type));
11131 : else
11132 0 : gcc_unreachable ();
11133 : }
11134 : else
11135 101637 : op = fold_unary (VIEW_CONVERT_EXPR,
11136 : TREE_TYPE (vector_type), op);
11137 101703 : gcc_assert (op && CONSTANT_CLASS_P (op));
11138 : }
11139 : else
11140 : {
11141 178626 : tree new_temp = make_ssa_name (TREE_TYPE (vector_type));
11142 178626 : gimple *init_stmt;
11143 178626 : if (VECTOR_BOOLEAN_TYPE_P (vector_type))
11144 : {
11145 427 : tree true_val
11146 427 : = build_all_ones_cst (TREE_TYPE (vector_type));
11147 427 : tree false_val
11148 427 : = build_zero_cst (TREE_TYPE (vector_type));
11149 427 : gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
11150 427 : init_stmt = gimple_build_assign (new_temp, COND_EXPR,
11151 : op, true_val,
11152 : false_val);
11153 : }
11154 : else
11155 : {
11156 178199 : op = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (vector_type),
11157 : op);
11158 178199 : init_stmt
11159 178199 : = gimple_build_assign (new_temp, VIEW_CONVERT_EXPR,
11160 : op);
11161 : }
11162 178626 : gimple_seq_add_stmt (&ctor_seq, init_stmt);
11163 178626 : op = new_temp;
11164 : }
11165 : }
11166 1830635 : elts[number_of_places_left_in_vector] = op;
11167 1830635 : if (!CONSTANT_CLASS_P (op))
11168 322846 : constant_p = false;
11169 : /* For BB vectorization we have to compute an insert location
11170 : when a def is inside the analyzed region since we cannot
11171 : simply insert at the BB start in this case. */
11172 1830635 : stmt_vec_info opdef;
11173 1830635 : if (TREE_CODE (orig_op) == SSA_NAME
11174 184966 : && !SSA_NAME_IS_DEFAULT_DEF (orig_op)
11175 164402 : && is_a <bb_vec_info> (vinfo)
11176 1937040 : && (opdef = vinfo->lookup_def (orig_op)))
11177 : {
11178 86282 : if (!insert_after)
11179 : insert_after = opdef;
11180 : else
11181 47573 : insert_after = get_later_stmt (insert_after, opdef);
11182 : }
11183 :
11184 1830635 : if (number_of_places_left_in_vector == 0)
11185 : {
11186 638074 : auto type_nunits = TYPE_VECTOR_SUBPARTS (vector_type);
11187 638074 : if (uniform_elt)
11188 664244 : vec_cst = gimple_build_vector_from_val (&ctor_seq, vector_type,
11189 332122 : elts[0]);
11190 611904 : else if (constant_p
11191 611904 : ? multiple_p (type_nunits, nunits)
11192 111991 : : known_eq (type_nunits, nunits))
11193 305952 : vec_cst = gimple_build_vector (&ctor_seq, &elts);
11194 : else
11195 : {
11196 0 : if (permute_results.is_empty ())
11197 0 : duplicate_and_interleave (vinfo, &ctor_seq, vector_type,
11198 : elts, number_of_vectors,
11199 : permute_results);
11200 0 : vec_cst = permute_results[number_of_vectors - j - 1];
11201 : }
11202 638074 : if (!gimple_seq_empty_p (ctor_seq))
11203 : {
11204 139827 : if (insert_after)
11205 : {
11206 38709 : gimple_stmt_iterator gsi;
11207 38709 : if (gimple_code (insert_after->stmt) == GIMPLE_PHI)
11208 : {
11209 618 : gsi = gsi_after_labels (gimple_bb (insert_after->stmt));
11210 618 : gsi_insert_seq_before (&gsi, ctor_seq,
11211 : GSI_CONTINUE_LINKING);
11212 : }
11213 38091 : else if (!stmt_ends_bb_p (insert_after->stmt))
11214 : {
11215 38091 : gsi = gsi_for_stmt (insert_after->stmt);
11216 38091 : gsi_insert_seq_after (&gsi, ctor_seq,
11217 : GSI_CONTINUE_LINKING);
11218 : }
11219 : else
11220 : {
11221 : /* When we want to insert after a def where the
11222 : defining stmt throws then insert on the fallthru
11223 : edge. */
11224 0 : edge e = find_fallthru_edge
11225 0 : (gimple_bb (insert_after->stmt)->succs);
11226 0 : basic_block new_bb
11227 0 : = gsi_insert_seq_on_edge_immediate (e, ctor_seq);
11228 0 : gcc_assert (!new_bb);
11229 : }
11230 : }
11231 : else
11232 101118 : vinfo->insert_seq_on_entry (NULL, ctor_seq);
11233 139827 : ctor_seq = NULL;
11234 : }
11235 638074 : voprnds.quick_push (vec_cst);
11236 638074 : insert_after = NULL;
11237 638074 : number_of_places_left_in_vector = nunits;
11238 638074 : constant_p = true;
11239 638074 : elts.new_vector (vector_type, nunits, 1);
11240 638074 : elts.quick_grow (nunits);
11241 : }
11242 : }
11243 : }
11244 :
11245 504199 : gcc_assert (number_of_places_left_in_vector == nunits);
11246 :
11247 : /* Since the vectors are created in the reverse order, we should invert
11248 : them. */
11249 504199 : vec_num = voprnds.length ();
11250 504199 : gcc_checking_assert (vec_num > 0);
11251 :
11252 1142273 : for (j = vec_num; j != 0; j--)
11253 : {
11254 638074 : vop = voprnds[j - 1];
11255 638074 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11256 : }
11257 :
11258 : /* In case that VF is greater than the unrolling factor needed for the SLP
11259 : group of stmts, NUMBER_OF_VECTORS to be created is greater than
11260 : NUMBER_OF_SCALARS/NUNITS or NUNITS/NUMBER_OF_SCALARS, and hence we have
11261 : to replicate the vectors. */
11262 1008398 : while (number_of_vectors > SLP_TREE_VEC_DEFS (op_node).length ())
11263 0 : for (i = 0; SLP_TREE_VEC_DEFS (op_node).iterate (i, &vop) && i < vec_num;
11264 : i++)
11265 0 : SLP_TREE_VEC_DEFS (op_node).quick_push (vop);
11266 504199 : }
11267 :
11268 : /* Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE
11269 : if there is no definition for it in the scalar IL or it is not known. */
11270 :
11271 : tree
11272 73612 : vect_get_slp_scalar_def (slp_tree slp_node, unsigned n)
11273 : {
11274 73612 : if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def)
11275 : {
11276 2693 : if (!SLP_TREE_SCALAR_STMTS (slp_node).exists ())
11277 : return NULL_TREE;
11278 2693 : stmt_vec_info def = SLP_TREE_SCALAR_STMTS (slp_node)[n];
11279 2693 : if (!def)
11280 : return NULL_TREE;
11281 2693 : return gimple_get_lhs (STMT_VINFO_STMT (def));
11282 : }
11283 : else
11284 70919 : return SLP_TREE_SCALAR_OPS (slp_node)[n];
11285 : }
11286 :
11287 : /* Get the Ith vectorized definition from SLP_NODE. */
11288 :
11289 : tree
11290 141725 : vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
11291 : {
11292 141725 : return SLP_TREE_VEC_DEFS (slp_node)[i];
11293 : }
11294 :
11295 : /* Get the vectorized definitions of SLP_NODE in *VEC_DEFS. */
11296 :
11297 : void
11298 945157 : vect_get_slp_defs (slp_tree slp_node, vec<tree> *vec_defs)
11299 : {
11300 1890314 : vec_defs->create (SLP_TREE_VEC_DEFS (slp_node).length ());
11301 945157 : vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node));
11302 945157 : }
11303 :
11304 : /* Get N vectorized definitions for SLP_NODE. */
11305 :
11306 : void
11307 3083 : vect_get_slp_defs (vec_info *,
11308 : slp_tree slp_node, vec<vec<tree> > *vec_oprnds, unsigned n)
11309 : {
11310 3083 : if (n == -1U)
11311 3083 : n = SLP_TREE_CHILDREN (slp_node).length ();
11312 :
11313 11001 : for (unsigned i = 0; i < n; ++i)
11314 : {
11315 7918 : slp_tree child = SLP_TREE_CHILDREN (slp_node)[i];
11316 7918 : vec<tree> vec_defs = vNULL;
11317 7918 : vect_get_slp_defs (child, &vec_defs);
11318 7918 : vec_oprnds->quick_push (vec_defs);
11319 : }
11320 3083 : }
11321 :
11322 : /* A subroutine of vect_transform_slp_perm_load with two extra arguments:
11323 : - PERM gives the permutation that the caller wants to use for NODE,
11324 : which might be different from SLP_LOAD_PERMUTATION.
11325 : - DUMP_P controls whether the function dumps information. */
11326 :
11327 : static bool
11328 137664 : vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node,
11329 : load_permutation_t &perm,
11330 : const vec<tree> &dr_chain,
11331 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11332 : bool analyze_only, bool dump_p,
11333 : unsigned *n_perms, unsigned int *n_loads,
11334 : bool dce_chain)
11335 : {
11336 137664 : stmt_vec_info stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
11337 137664 : int vec_index = 0;
11338 137664 : tree vectype = SLP_TREE_VECTYPE (node);
11339 137664 : unsigned int group_size = SLP_TREE_LANES (node);
11340 137664 : unsigned int mask_element;
11341 137664 : unsigned dr_group_size;
11342 137664 : machine_mode mode;
11343 :
11344 137664 : if (!STMT_VINFO_GROUPED_ACCESS (stmt_info))
11345 : {
11346 : /* We have both splats of the same non-grouped load and groups
11347 : of distinct invariant loads entering here. */
11348 1521 : unsigned max_idx = 0;
11349 8413 : for (auto idx : perm)
11350 3850 : max_idx = idx > max_idx ? idx : max_idx;
11351 1521 : dr_group_size = max_idx + 1;
11352 : }
11353 : else
11354 : {
11355 136143 : stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
11356 136143 : dr_group_size = DR_GROUP_SIZE (stmt_info);
11357 : }
11358 :
11359 137664 : mode = TYPE_MODE (vectype);
11360 137664 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11361 137664 : unsigned int nstmts = vect_get_num_copies (vinfo, node);
11362 :
11363 : /* Initialize the vect stmts of NODE to properly insert the generated
11364 : stmts later. */
11365 137664 : if (! analyze_only)
11366 56872 : for (unsigned i = SLP_TREE_VEC_DEFS (node).length (); i < nstmts; i++)
11367 22008 : SLP_TREE_VEC_DEFS (node).quick_push (NULL_TREE);
11368 :
11369 : /* Generate permutation masks for every NODE. Number of masks for each NODE
11370 : is equal to GROUP_SIZE.
11371 : E.g., we have a group of three nodes with three loads from the same
11372 : location in each node, and the vector size is 4. I.e., we have a
11373 : a0b0c0a1b1c1... sequence and we need to create the following vectors:
11374 : for a's: a0a0a0a1 a1a1a2a2 a2a3a3a3
11375 : for b's: b0b0b0b1 b1b1b2b2 b2b3b3b3
11376 : ...
11377 :
11378 : The masks for a's should be: {0,0,0,3} {3,3,6,6} {6,9,9,9}.
11379 : The last mask is illegal since we assume two operands for permute
11380 : operation, and the mask element values can't be outside that range.
11381 : Hence, the last mask must be converted into {2,5,5,5}.
11382 : For the first two permutations we need the first and the second input
11383 : vectors: {a0,b0,c0,a1} and {b1,c1,a2,b2}, and for the last permutation
11384 : we need the second and the third vectors: {b1,c1,a2,b2} and
11385 : {c2,a3,b3,c3}. */
11386 :
11387 137664 : int vect_stmts_counter = 0;
11388 137664 : unsigned int index = 0;
11389 137664 : int first_vec_index = -1;
11390 137664 : int second_vec_index = -1;
11391 137664 : bool noop_p = true;
11392 137664 : *n_perms = 0;
11393 :
11394 137664 : vec_perm_builder mask;
11395 137664 : unsigned int nelts_to_build;
11396 137664 : unsigned int nvectors_per_build;
11397 137664 : unsigned int in_nlanes;
11398 137664 : bool repeating_p = (group_size == dr_group_size
11399 173084 : && multiple_p (nunits, group_size));
11400 137664 : if (repeating_p)
11401 : {
11402 : /* A single vector contains a whole number of copies of the node, so:
11403 : (a) all permutes can use the same mask; and
11404 : (b) the permutes only need a single vector input. */
11405 32930 : mask.new_vector (nunits, group_size, 3);
11406 32930 : nelts_to_build = mask.encoded_nelts ();
11407 : /* It's possible to obtain zero nstmts during analyze_only, so make
11408 : it at least one to ensure the later computation for n_perms
11409 : proceed. */
11410 32930 : nvectors_per_build = nstmts > 0 ? nstmts : 1;
11411 32930 : in_nlanes = dr_group_size * 3;
11412 : }
11413 : else
11414 : {
11415 : /* We need to construct a separate mask for each vector statement. */
11416 104734 : unsigned HOST_WIDE_INT const_nunits, const_vf;
11417 104734 : if (!nunits.is_constant (&const_nunits)
11418 104734 : || !vf.is_constant (&const_vf))
11419 : return false;
11420 104734 : mask.new_vector (const_nunits, const_nunits, 1);
11421 104734 : nelts_to_build = const_vf * group_size;
11422 104734 : nvectors_per_build = 1;
11423 104734 : in_nlanes = const_vf * dr_group_size;
11424 : }
11425 137664 : auto_sbitmap used_in_lanes (in_nlanes);
11426 137664 : bitmap_clear (used_in_lanes);
11427 137664 : auto_bitmap used_defs;
11428 :
11429 137664 : unsigned int count = mask.encoded_nelts ();
11430 137664 : mask.quick_grow (count);
11431 137664 : vec_perm_indices indices;
11432 :
11433 724618 : for (unsigned int j = 0; j < nelts_to_build; j++)
11434 : {
11435 597922 : unsigned int iter_num = j / group_size;
11436 597922 : unsigned int stmt_num = j % group_size;
11437 597922 : unsigned int i = (iter_num * dr_group_size + perm[stmt_num]);
11438 597922 : bitmap_set_bit (used_in_lanes, i);
11439 597922 : if (repeating_p)
11440 : {
11441 : first_vec_index = 0;
11442 : mask_element = i;
11443 : }
11444 : else
11445 : {
11446 : /* Enforced before the loop when !repeating_p. */
11447 383068 : unsigned int const_nunits = nunits.to_constant ();
11448 383068 : vec_index = i / const_nunits;
11449 383068 : mask_element = i % const_nunits;
11450 383068 : if (vec_index == first_vec_index
11451 383068 : || first_vec_index == -1)
11452 : {
11453 : first_vec_index = vec_index;
11454 : }
11455 153537 : else if (vec_index == second_vec_index
11456 153537 : || second_vec_index == -1)
11457 : {
11458 147130 : second_vec_index = vec_index;
11459 147130 : mask_element += const_nunits;
11460 : }
11461 : else
11462 : {
11463 6407 : if (dump_p)
11464 280 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11465 : "permutation requires at "
11466 : "least three vectors %G",
11467 : stmt_info->stmt);
11468 6407 : gcc_assert (analyze_only);
11469 : return false;
11470 : }
11471 :
11472 376661 : gcc_assert (mask_element < 2 * const_nunits);
11473 : }
11474 :
11475 591515 : if (mask_element != index)
11476 380706 : noop_p = false;
11477 591515 : mask[index++] = mask_element;
11478 :
11479 591515 : if (index == count)
11480 : {
11481 162413 : if (!noop_p)
11482 : {
11483 132122 : indices.new_vector (mask, second_vec_index == -1 ? 1 : 2, nunits);
11484 132122 : if (!can_vec_perm_const_p (mode, mode, indices))
11485 : {
11486 4561 : if (dump_p)
11487 : {
11488 90 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11489 : "unsupported vect permute { ");
11490 876 : for (i = 0; i < count; ++i)
11491 : {
11492 786 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
11493 786 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
11494 : }
11495 90 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
11496 : }
11497 4561 : gcc_assert (analyze_only);
11498 : return false;
11499 : }
11500 :
11501 127561 : tree mask_vec = NULL_TREE;
11502 127561 : if (!analyze_only)
11503 20286 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
11504 :
11505 127561 : if (second_vec_index == -1)
11506 36362 : second_vec_index = first_vec_index;
11507 :
11508 257987 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11509 : {
11510 130426 : ++*n_perms;
11511 130426 : if (analyze_only)
11512 109857 : continue;
11513 : /* Generate the permute statement if necessary. */
11514 20569 : tree first_vec = dr_chain[first_vec_index + ri];
11515 20569 : tree second_vec = dr_chain[second_vec_index + ri];
11516 20569 : gassign *stmt = as_a<gassign *> (stmt_info->stmt);
11517 20569 : tree perm_dest
11518 20569 : = vect_create_destination_var (gimple_assign_lhs (stmt),
11519 : vectype);
11520 20569 : perm_dest = make_ssa_name (perm_dest);
11521 20569 : gimple *perm_stmt
11522 20569 : = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec,
11523 : second_vec, mask_vec);
11524 20569 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt,
11525 : gsi);
11526 20569 : if (dce_chain)
11527 : {
11528 19646 : bitmap_set_bit (used_defs, first_vec_index + ri);
11529 19646 : bitmap_set_bit (used_defs, second_vec_index + ri);
11530 : }
11531 :
11532 : /* Store the vector statement in NODE. */
11533 20569 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = perm_dest;
11534 : }
11535 : }
11536 30291 : else if (!analyze_only)
11537 : {
11538 2878 : for (unsigned int ri = 0; ri < nvectors_per_build; ++ri)
11539 : {
11540 1439 : tree first_vec = dr_chain[first_vec_index + ri];
11541 : /* If mask was NULL_TREE generate the requested
11542 : identity transform. */
11543 1439 : if (dce_chain)
11544 1432 : bitmap_set_bit (used_defs, first_vec_index + ri);
11545 :
11546 : /* Store the vector statement in NODE. */
11547 1439 : SLP_TREE_VEC_DEFS (node)[vect_stmts_counter++] = first_vec;
11548 : }
11549 : }
11550 :
11551 : index = 0;
11552 : first_vec_index = -1;
11553 : second_vec_index = -1;
11554 : noop_p = true;
11555 : }
11556 : }
11557 :
11558 126696 : if (n_loads)
11559 : {
11560 88706 : if (repeating_p)
11561 10242 : *n_loads = nstmts;
11562 : else
11563 : {
11564 : /* Enforced above when !repeating_p. */
11565 78464 : unsigned int const_nunits = nunits.to_constant ();
11566 78464 : *n_loads = 0;
11567 78464 : bool load_seen = false;
11568 1056011 : for (unsigned i = 0; i < in_nlanes; ++i)
11569 : {
11570 977547 : if (i % const_nunits == 0)
11571 : {
11572 417449 : if (load_seen)
11573 124142 : *n_loads += 1;
11574 : load_seen = false;
11575 : }
11576 977547 : if (bitmap_bit_p (used_in_lanes, i))
11577 271979 : load_seen = true;
11578 : }
11579 78464 : if (load_seen)
11580 52407 : *n_loads += 1;
11581 : }
11582 : }
11583 :
11584 126696 : if (dce_chain)
11585 211347 : for (unsigned i = 0; i < dr_chain.length (); ++i)
11586 73683 : if (!bitmap_bit_p (used_defs, i))
11587 : {
11588 40932 : tree def = dr_chain[i];
11589 41338 : do
11590 : {
11591 41338 : gimple *stmt = SSA_NAME_DEF_STMT (def);
11592 41338 : if (is_gimple_assign (stmt)
11593 41338 : && (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
11594 41338 : || gimple_assign_rhs_code (stmt) == CONSTRUCTOR))
11595 5019 : def = single_ssa_tree_operand (stmt, SSA_OP_USE);
11596 : else
11597 : def = NULL;
11598 41338 : gimple_stmt_iterator rgsi = gsi_for_stmt (stmt);
11599 41338 : gsi_remove (&rgsi, true);
11600 41338 : release_defs (stmt);
11601 : }
11602 41338 : while (def);
11603 : }
11604 :
11605 : return true;
11606 137664 : }
11607 :
11608 : /* Generate vector permute statements from a list of loads in DR_CHAIN.
11609 : If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
11610 : permute statements for the SLP node NODE. Store the number of vector
11611 : permute instructions in *N_PERMS and the number of vector load
11612 : instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions
11613 : that were not needed. */
11614 :
11615 : bool
11616 98456 : vect_transform_slp_perm_load (vec_info *vinfo,
11617 : slp_tree node, const vec<tree> &dr_chain,
11618 : gimple_stmt_iterator *gsi, poly_uint64 vf,
11619 : bool analyze_only, unsigned *n_perms,
11620 : unsigned int *n_loads, bool dce_chain)
11621 : {
11622 98456 : return vect_transform_slp_perm_load_1 (vinfo, node,
11623 98456 : SLP_TREE_LOAD_PERMUTATION (node),
11624 : dr_chain, gsi, vf, analyze_only,
11625 : dump_enabled_p (), n_perms, n_loads,
11626 98456 : dce_chain);
11627 : }
11628 :
11629 : /* Produce the next vector result for SLP permutation NODE by adding a vector
11630 : statement at GSI. If MASK_VEC is nonnull, add:
11631 :
11632 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
11633 :
11634 : otherwise add:
11635 :
11636 : <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
11637 : { N, N+1, N+2, ... }>
11638 :
11639 : where N == IDENTITY_OFFSET which is either zero or equal to the
11640 : number of elements of the result. */
11641 :
11642 : static void
11643 32415 : vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
11644 : slp_tree node, tree first_def, tree second_def,
11645 : tree mask_vec, poly_uint64 identity_offset)
11646 : {
11647 32415 : tree vectype = SLP_TREE_VECTYPE (node);
11648 :
11649 : /* ??? We SLP match existing vector element extracts but
11650 : allow punning which we need to re-instantiate at uses
11651 : but have no good way of explicitly representing. */
11652 32415 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)), TYPE_SIZE (vectype))
11653 32415 : && !types_compatible_p (TREE_TYPE (first_def), vectype))
11654 : {
11655 11 : gassign *conv_stmt
11656 11 : = gimple_build_assign (make_ssa_name (vectype),
11657 : build1 (VIEW_CONVERT_EXPR, vectype, first_def));
11658 11 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11659 11 : first_def = gimple_assign_lhs (conv_stmt);
11660 : }
11661 32415 : gassign *perm_stmt;
11662 32415 : if (mask_vec)
11663 : {
11664 28470 : if (operand_equal_p (TYPE_SIZE (TREE_TYPE (first_def)),
11665 28470 : TYPE_SIZE (vectype))
11666 28470 : && !types_compatible_p (TREE_TYPE (second_def), vectype))
11667 : {
11668 8 : gassign *conv_stmt
11669 8 : = gimple_build_assign (make_ssa_name (vectype),
11670 : build1 (VIEW_CONVERT_EXPR,
11671 : vectype, second_def));
11672 8 : vect_finish_stmt_generation (vinfo, NULL, conv_stmt, gsi);
11673 8 : second_def = gimple_assign_lhs (conv_stmt);
11674 : }
11675 28470 : tree perm_dest = make_ssa_name (vectype);
11676 28470 : perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR,
11677 : first_def, second_def,
11678 : mask_vec);
11679 : }
11680 : else
11681 : {
11682 3945 : auto def_nunits = TYPE_VECTOR_SUBPARTS (TREE_TYPE (first_def));
11683 3945 : unsigned HOST_WIDE_INT vecno;
11684 3945 : poly_uint64 eltno;
11685 3945 : if (!can_div_trunc_p (poly_uint64 (identity_offset), def_nunits,
11686 : &vecno, &eltno))
11687 : gcc_unreachable ();
11688 3945 : tree def = vecno & 1 ? second_def : first_def;
11689 3945 : if (!types_compatible_p (TREE_TYPE (def), vectype))
11690 : {
11691 : /* For identity permutes we still need to handle the case
11692 : of offsetted extracts or concats. */
11693 362 : tree perm_dest = make_ssa_name (vectype);
11694 362 : unsigned HOST_WIDE_INT c;
11695 362 : if (known_le (TYPE_VECTOR_SUBPARTS (vectype), def_nunits))
11696 : {
11697 358 : unsigned HOST_WIDE_INT elsz
11698 358 : = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (def))));
11699 716 : tree lowpart = build3 (BIT_FIELD_REF, vectype, def,
11700 358 : TYPE_SIZE (vectype),
11701 358 : bitsize_int (eltno * elsz));
11702 358 : perm_stmt = gimple_build_assign (perm_dest, lowpart);
11703 : }
11704 4 : else if (constant_multiple_p (TYPE_VECTOR_SUBPARTS (vectype),
11705 4 : def_nunits, &c) && c == 2)
11706 : {
11707 4 : gcc_assert (known_eq (identity_offset, 0U));
11708 4 : tree ctor = build_constructor_va (vectype, 2,
11709 : NULL_TREE, first_def,
11710 : NULL_TREE, second_def);
11711 4 : perm_stmt = gimple_build_assign (perm_dest, ctor);
11712 : }
11713 : else
11714 0 : gcc_unreachable ();
11715 : }
11716 : else
11717 : {
11718 3583 : gcc_assert (known_eq (eltno, 0U));
11719 3583 : node->push_vec_def (def);
11720 3583 : return;
11721 : }
11722 : }
11723 28832 : vect_finish_stmt_generation (vinfo, NULL, perm_stmt, gsi);
11724 : /* Store the vector statement in NODE. */
11725 28832 : node->push_vec_def (perm_stmt);
11726 : }
11727 :
11728 : /* Subroutine of vectorizable_slp_permutation. Check whether the target
11729 : can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
11730 : If GSI is nonnull, emit the permutation there.
11731 :
11732 : When GSI is null, the only purpose of NODE is to give properties
11733 : of the result, such as the vector type and number of SLP lanes.
11734 : The node does not need to be a VEC_PERM_EXPR.
11735 :
11736 : If the target supports the operation, return the number of individual
11737 : VEC_PERM_EXPRs needed, otherwise return -1. Print information to the
11738 : dump file if DUMP_P is true. */
11739 :
11740 : static int
11741 446166 : vectorizable_slp_permutation_1 (vec_info *vinfo, gimple_stmt_iterator *gsi,
11742 : slp_tree node, lane_permutation_t &perm,
11743 : vec<slp_tree> &children, bool dump_p)
11744 : {
11745 446166 : tree vectype = SLP_TREE_VECTYPE (node);
11746 :
11747 : /* ??? We currently only support all same vector input types
11748 : while the SLP IL should really do a concat + select and thus accept
11749 : arbitrary mismatches. */
11750 446166 : slp_tree child;
11751 446166 : unsigned i;
11752 446166 : poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
11753 446166 : bool repeating_p = multiple_p (nunits, SLP_TREE_LANES (node));
11754 : /* True if we're permuting a single input of 2N vectors down
11755 : to N vectors. This case doesn't generalize beyond 2 since
11756 : VEC_PERM_EXPR only takes 2 inputs. */
11757 446166 : bool pack_p = false;
11758 : /* If we're permuting inputs of N vectors each into X*N outputs,
11759 : this is the value of X, otherwise it is 1. */
11760 446166 : unsigned int unpack_factor = 1;
11761 446166 : tree op_vectype = NULL_TREE;
11762 447770 : FOR_EACH_VEC_ELT (children, i, child)
11763 447670 : if (SLP_TREE_VECTYPE (child))
11764 : {
11765 : op_vectype = SLP_TREE_VECTYPE (child);
11766 : break;
11767 : }
11768 446166 : if (!op_vectype)
11769 100 : op_vectype = vectype;
11770 937578 : FOR_EACH_VEC_ELT (children, i, child)
11771 : {
11772 491412 : if ((SLP_TREE_DEF_TYPE (child) != vect_internal_def
11773 18321 : && !vect_maybe_update_slp_op_vectype (child, op_vectype))
11774 491412 : || !types_compatible_p (SLP_TREE_VECTYPE (child), op_vectype)
11775 982824 : || !types_compatible_p (TREE_TYPE (vectype), TREE_TYPE (op_vectype)))
11776 : {
11777 0 : if (dump_p)
11778 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11779 : "Unsupported vector types in lane permutation\n");
11780 446166 : return -1;
11781 : }
11782 491412 : auto op_nunits = TYPE_VECTOR_SUBPARTS (op_vectype);
11783 491412 : unsigned int this_unpack_factor;
11784 : /* Detect permutations of external, pre-existing vectors. The external
11785 : node's SLP_TREE_LANES stores the total number of units in the vector,
11786 : or zero if the vector has variable length.
11787 :
11788 : We are expected to keep the original VEC_PERM_EXPR for such cases.
11789 : There is no repetition to model. */
11790 491412 : if (SLP_TREE_DEF_TYPE (child) == vect_external_def
11791 491412 : && SLP_TREE_SCALAR_OPS (child).is_empty ())
11792 : repeating_p = false;
11793 : /* Check whether the input has twice as many lanes per vector. */
11794 475835 : else if (children.length () == 1
11795 475835 : && known_eq (SLP_TREE_LANES (child) * nunits,
11796 : SLP_TREE_LANES (node) * op_nunits * 2))
11797 : pack_p = true;
11798 : /* Check whether the output has N times as many lanes per vector. */
11799 491412 : else if (constant_multiple_p (SLP_TREE_LANES (node) * op_nunits,
11800 431699 : SLP_TREE_LANES (child) * nunits,
11801 : &this_unpack_factor)
11802 396822 : && (i == 0 || unpack_factor == this_unpack_factor))
11803 : unpack_factor = this_unpack_factor;
11804 : else
11805 : repeating_p = false;
11806 : }
11807 :
11808 892332 : gcc_assert (perm.length () == SLP_TREE_LANES (node));
11809 :
11810 : /* Load-lanes permute. This permute only acts as a forwarder to
11811 : select the correct vector def of the load-lanes load which
11812 : has the permuted vectors in its vector defs like
11813 : { v0, w0, r0, v1, w1, r1 ... } for a ld3. All costs are
11814 : accounted for in the costing for the actual load so we
11815 : return zero here. */
11816 446166 : if (node->ldst_lanes)
11817 : {
11818 0 : gcc_assert (children.length () == 1);
11819 0 : if (!gsi)
11820 : /* This is a trivial op always supported. */
11821 : return 0;
11822 0 : slp_tree child = children[0];
11823 0 : unsigned vec_idx = (SLP_TREE_LANE_PERMUTATION (node)[0].second
11824 0 : / SLP_TREE_LANES (node));
11825 0 : unsigned vec_num = SLP_TREE_LANES (child) / SLP_TREE_LANES (node);
11826 0 : unsigned nvectors = vect_get_num_copies (vinfo, node);
11827 0 : for (unsigned i = 0; i < nvectors; ++i)
11828 : {
11829 0 : tree def = SLP_TREE_VEC_DEFS (child)[i * vec_num + vec_idx];
11830 0 : node->push_vec_def (def);
11831 : }
11832 : return 0;
11833 : }
11834 :
11835 : /* Set REPEATING_P to true if the permutations are cyclical wrt UNPACK_FACTOR
11836 : and if we can generate the vectors in a vector-length agnostic way.
11837 : This requires UNPACK_STEP == NUNITS / UNPACK_FACTOR to be known at
11838 : compile time.
11839 :
11840 : The significance of UNPACK_STEP is that, when PACK_P is false,
11841 : output vector I operates on a window of UNPACK_STEP elements from each
11842 : input, starting at lane UNPACK_STEP * (I % UNPACK_FACTOR). For example,
11843 : when UNPACK_FACTOR is 2, the first output vector operates on lanes
11844 : [0, NUNITS / 2 - 1] of each input vector and the second output vector
11845 : operates on lanes [NUNITS / 2, NUNITS - 1] of each input vector.
11846 :
11847 : When REPEATING_P is true, NOUTPUTS holds the total number of outputs
11848 : that we actually need to generate. */
11849 446166 : uint64_t noutputs = 0;
11850 446166 : poly_uint64 unpack_step = 0;
11851 446166 : loop_vec_info linfo = dyn_cast <loop_vec_info> (vinfo);
11852 183423 : if (!linfo
11853 485221 : || !multiple_p (nunits, unpack_factor, &unpack_step)
11854 182485 : || !constant_multiple_p (LOOP_VINFO_VECT_FACTOR (linfo)
11855 182485 : * SLP_TREE_LANES (node), nunits, &noutputs))
11856 : repeating_p = false;
11857 :
11858 : /* We can handle the conditions described for REPEATING_P above for
11859 : both variable- and constant-length vectors. The fallback requires
11860 : us to generate every element of every permute vector explicitly,
11861 : which is only possible for constant-length permute vectors.
11862 :
11863 : Set:
11864 :
11865 : - NPATTERNS and NELTS_PER_PATTERN to the encoding of the permute
11866 : mask vectors that we want to build.
11867 :
11868 : - NCOPIES to the number of copies of PERM that we need in order
11869 : to build the necessary permute mask vectors. */
11870 182485 : uint64_t npatterns;
11871 182485 : unsigned nelts_per_pattern;
11872 182485 : uint64_t ncopies;
11873 182485 : if (repeating_p)
11874 : {
11875 : /* We need permute mask vectors that have the form:
11876 :
11877 : { X1, ..., Xn, X1 + n, ..., Xn + n, X1 + 2n, ..., Xn + 2n, ... }
11878 :
11879 : In other words, the original n-element permute in PERM is
11880 : "unrolled" to fill a full vector. The stepped vector encoding
11881 : that we use for permutes requires 3n elements. */
11882 143430 : npatterns = SLP_TREE_LANES (node);
11883 143430 : nelts_per_pattern = ncopies = 3;
11884 : }
11885 : else
11886 : {
11887 : /* Calculate every element of every permute mask vector explicitly,
11888 : instead of relying on the pattern described above. */
11889 302736 : if (!nunits.is_constant (&npatterns)
11890 302736 : || !TYPE_VECTOR_SUBPARTS (op_vectype).is_constant ())
11891 : {
11892 : if (dump_p)
11893 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11894 : "unsupported permutation %p on variable-length"
11895 : " vectors\n", (void *) node);
11896 : return -1;
11897 : }
11898 302736 : nelts_per_pattern = ncopies = 1;
11899 302736 : if (linfo && !LOOP_VINFO_VECT_FACTOR (linfo).is_constant (&ncopies))
11900 : {
11901 : if (dump_p)
11902 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
11903 : "unsupported permutation %p for variable VF\n",
11904 : (void *) node);
11905 : return -1;
11906 : }
11907 : pack_p = false;
11908 : unpack_factor = 1;
11909 : }
11910 446166 : unsigned olanes = unpack_factor * ncopies * SLP_TREE_LANES (node);
11911 446166 : gcc_assert (repeating_p || multiple_p (olanes, nunits));
11912 :
11913 : /* Compute the { { SLP operand, vector index}, lane } permutation sequence
11914 : from the { SLP operand, scalar lane } permutation as recorded in the
11915 : SLP node as intermediate step. This part should already work
11916 : with SLP children with arbitrary number of lanes. */
11917 446166 : auto_vec<std::pair<std::pair<unsigned, unsigned>, poly_uint64>> vperm;
11918 446166 : auto_vec<poly_uint64> active_lane;
11919 446166 : vperm.create (olanes);
11920 446166 : active_lane.safe_grow_cleared (children.length (), true);
11921 1347020 : for (unsigned int ui = 0; ui < unpack_factor; ++ui)
11922 : {
11923 1926240 : for (unsigned j = 0; j < children.length (); ++j)
11924 508432 : active_lane[j] = ui * unpack_step;
11925 1311218 : for (unsigned i = 0; i < ncopies; ++i)
11926 : {
11927 5404462 : for (unsigned pi = 0; pi < perm.length (); ++pi)
11928 : {
11929 1845701 : std::pair<unsigned, unsigned> p = perm[pi];
11930 1845701 : tree vtype = SLP_TREE_VECTYPE (children[p.first]);
11931 1845701 : if (repeating_p)
11932 839097 : vperm.quick_push ({{p.first, 0},
11933 839097 : p.second + active_lane[p.first]});
11934 : else
11935 : {
11936 : /* We checked above that the vectors are constant-length. */
11937 1006604 : unsigned vnunits = TYPE_VECTOR_SUBPARTS (vtype)
11938 1006604 : .to_constant ();
11939 1006604 : unsigned lane = active_lane[p.first].to_constant ();
11940 1006604 : unsigned vi = (lane + p.second) / vnunits;
11941 1006604 : unsigned vl = (lane + p.second) % vnunits;
11942 1006604 : vperm.quick_push ({{p.first, vi}, vl});
11943 : }
11944 : }
11945 : /* Advance to the next group. */
11946 1832565 : for (unsigned j = 0; j < children.length (); ++j)
11947 976035 : active_lane[j] += SLP_TREE_LANES (children[j]);
11948 : }
11949 : }
11950 :
11951 446166 : if (dump_p)
11952 : {
11953 9031 : dump_printf_loc (MSG_NOTE, vect_location,
11954 : "vectorizing permutation %p", (void *)node);
11955 32682 : for (unsigned i = 0; i < perm.length (); ++i)
11956 23651 : dump_printf (MSG_NOTE, " op%u[%u]", perm[i].first, perm[i].second);
11957 9031 : if (repeating_p)
11958 7574 : dump_printf (MSG_NOTE, " (repeat %d)", SLP_TREE_LANES (node));
11959 9031 : dump_printf (MSG_NOTE, "\n");
11960 9031 : dump_printf_loc (MSG_NOTE, vect_location, "as");
11961 90762 : for (unsigned i = 0; i < vperm.length (); ++i)
11962 : {
11963 81731 : if (i != 0
11964 81731 : && (repeating_p
11965 55351 : ? multiple_p (i, npatterns)
11966 81731 : : multiple_p (i, TYPE_VECTOR_SUBPARTS (vectype))))
11967 24385 : dump_printf (MSG_NOTE, ",");
11968 81731 : dump_printf (MSG_NOTE, " vops%u[%u][",
11969 81731 : vperm[i].first.first, vperm[i].first.second);
11970 81731 : dump_dec (MSG_NOTE, vperm[i].second);
11971 81731 : dump_printf (MSG_NOTE, "]");
11972 : }
11973 9031 : dump_printf (MSG_NOTE, "\n");
11974 : }
11975 :
11976 : /* We can only handle two-vector permutes, everything else should
11977 : be lowered on the SLP level. The following is closely inspired
11978 : by vect_transform_slp_perm_load and is supposed to eventually
11979 : replace it.
11980 : ??? As intermediate step do code-gen in the SLP tree representation
11981 : somehow? */
11982 446166 : std::pair<unsigned, unsigned> first_vec = std::make_pair (-1U, -1U);
11983 446166 : std::pair<unsigned, unsigned> second_vec = std::make_pair (-1U, -1U);
11984 446166 : unsigned int index = 0;
11985 446166 : poly_uint64 mask_element;
11986 446166 : vec_perm_builder mask;
11987 446166 : mask.new_vector (nunits, npatterns, nelts_per_pattern);
11988 446166 : unsigned int count = mask.encoded_nelts ();
11989 446166 : mask.quick_grow (count);
11990 446166 : vec_perm_indices indices;
11991 446166 : unsigned nperms = 0;
11992 : /* When REPEATING_P is true, we only have UNPACK_FACTOR unique permute
11993 : vectors to check during analysis, but we need to generate NOUTPUTS
11994 : vectors during transformation. */
11995 446166 : unsigned total_nelts = olanes;
11996 446166 : unsigned process_nelts = olanes;
11997 446166 : if (repeating_p)
11998 : {
11999 143430 : total_nelts = (total_nelts / unpack_factor) * noutputs;
12000 143430 : if (gsi)
12001 9960 : process_nelts = total_nelts;
12002 : }
12003 446166 : unsigned last_ei = (total_nelts - 1) % process_nelts;
12004 2296206 : for (unsigned i = 0; i < process_nelts; ++i)
12005 : {
12006 : /* VI is the input vector index when generating code for REPEATING_P. */
12007 1860232 : unsigned vi = i / olanes * (pack_p ? 2 : 1);
12008 1860232 : unsigned ei = i % olanes;
12009 1860232 : mask_element = vperm[ei].second;
12010 1860232 : if (pack_p)
12011 : {
12012 : /* In this case, we have N outputs and the single child provides 2N
12013 : inputs. Output X permutes inputs 2X and 2X+1.
12014 :
12015 : The mask indices are taken directly from the SLP permutation node.
12016 : Index X selects from the first vector if (X / NUNITS) % 2 == 0;
12017 : X selects from the second vector otherwise. These conditions
12018 : are only known at compile time for constant-length vectors. */
12019 : first_vec = std::make_pair (0, 0);
12020 : second_vec = std::make_pair (0, 1);
12021 : }
12022 1689982 : else if (first_vec.first == -1U
12023 1689982 : || first_vec == vperm[ei].first)
12024 1492483 : first_vec = vperm[ei].first;
12025 197499 : else if (second_vec.first == -1U
12026 197499 : || second_vec == vperm[ei].first)
12027 : {
12028 197093 : second_vec = vperm[ei].first;
12029 197093 : mask_element += nunits;
12030 : }
12031 : else
12032 : {
12033 406 : if (dump_p)
12034 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12035 : "permutation requires at "
12036 : "least three vectors\n");
12037 406 : gcc_assert (!gsi);
12038 : return -1;
12039 : }
12040 :
12041 1859826 : mask[index++] = mask_element;
12042 :
12043 1859826 : if (index == count)
12044 : {
12045 591165 : indices.new_vector (mask, second_vec.first == -1U ? 1 : 2,
12046 : TYPE_VECTOR_SUBPARTS (op_vectype));
12047 591165 : bool identity_p = (indices.series_p (0, 1, mask[0], 1)
12048 920748 : && constant_multiple_p (mask[0], nunits));
12049 591165 : machine_mode vmode = TYPE_MODE (vectype);
12050 591165 : machine_mode op_vmode = TYPE_MODE (op_vectype);
12051 591165 : unsigned HOST_WIDE_INT c;
12052 591165 : if ((!identity_p
12053 541648 : && !can_vec_perm_const_p (vmode, op_vmode, indices))
12054 591165 : || (identity_p
12055 49517 : && !known_le (nunits,
12056 : TYPE_VECTOR_SUBPARTS (op_vectype))
12057 8 : && (!constant_multiple_p (nunits,
12058 9794 : TYPE_VECTOR_SUBPARTS (op_vectype),
12059 8 : &c) || c != 2)))
12060 : {
12061 9786 : if (dump_p)
12062 : {
12063 154 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
12064 : vect_location,
12065 : "unsupported vect permute { ");
12066 1596 : for (i = 0; i < count; ++i)
12067 : {
12068 1442 : dump_dec (MSG_MISSED_OPTIMIZATION, mask[i]);
12069 1442 : dump_printf (MSG_MISSED_OPTIMIZATION, " ");
12070 : }
12071 154 : dump_printf (MSG_MISSED_OPTIMIZATION, "}\n");
12072 : }
12073 9786 : gcc_assert (!gsi);
12074 446166 : return -1;
12075 : }
12076 :
12077 581379 : if (!identity_p)
12078 531862 : nperms += CEIL (total_nelts, process_nelts) - (ei > last_ei);
12079 581379 : if (gsi)
12080 : {
12081 32415 : if (second_vec.first == -1U)
12082 7765 : second_vec = first_vec;
12083 :
12084 32415 : slp_tree
12085 32415 : first_node = children[first_vec.first],
12086 32415 : second_node = children[second_vec.first];
12087 :
12088 32415 : tree mask_vec = NULL_TREE;
12089 32415 : if (!identity_p)
12090 28470 : mask_vec = vect_gen_perm_mask_checked (vectype, indices);
12091 :
12092 32415 : tree first_def
12093 32415 : = vect_get_slp_vect_def (first_node, first_vec.second + vi);
12094 32415 : tree second_def
12095 32415 : = vect_get_slp_vect_def (second_node, second_vec.second + vi);
12096 32415 : vect_add_slp_permutation (vinfo, gsi, node, first_def,
12097 32415 : second_def, mask_vec, mask[0]);
12098 : }
12099 :
12100 : index = 0;
12101 : first_vec = std::make_pair (-1U, -1U);
12102 : second_vec = std::make_pair (-1U, -1U);
12103 : }
12104 : }
12105 :
12106 435974 : return nperms;
12107 446166 : }
12108 :
12109 : /* Vectorize the SLP permutations in NODE as specified
12110 : in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
12111 : child number and lane number.
12112 : Interleaving of two two-lane two-child SLP subtrees (not supported):
12113 : [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
12114 : A blend of two four-lane two-child SLP subtrees:
12115 : [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
12116 : Highpart of a four-lane one-child SLP subtree (not supported):
12117 : [ { 0, 2 }, { 0, 3 } ]
12118 : Where currently only a subset is supported by code generating below. */
12119 :
12120 : bool
12121 99770 : vectorizable_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi,
12122 : slp_tree node, stmt_vector_for_cost *cost_vec)
12123 : {
12124 99770 : tree vectype = SLP_TREE_VECTYPE (node);
12125 99770 : lane_permutation_t &perm = SLP_TREE_LANE_PERMUTATION (node);
12126 99770 : int nperms = vectorizable_slp_permutation_1 (vinfo, gsi, node, perm,
12127 99770 : SLP_TREE_CHILDREN (node),
12128 : dump_enabled_p ());
12129 99770 : if (nperms < 0)
12130 : return false;
12131 :
12132 98178 : if (!gsi && nperms != 0)
12133 74435 : record_stmt_cost (cost_vec, nperms, vec_perm, node, vectype, 0, vect_body);
12134 :
12135 : return true;
12136 : }
12137 :
12138 : /* Vectorize SLP NODE. Only compute the vector insertion places when
12139 : PLACE_ONLY is true. When placing, return false if there is no possible
12140 : schedule. */
12141 :
12142 : static bool
12143 3135925 : vect_schedule_slp_node (vec_info *vinfo,
12144 : slp_tree node, slp_instance instance, bool place_only)
12145 : {
12146 3135925 : int i;
12147 3135925 : slp_tree child;
12148 :
12149 : /* Vectorize externals and constants. */
12150 3135925 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def
12151 3135925 : || SLP_TREE_DEF_TYPE (node) == vect_external_def)
12152 : {
12153 1214398 : if (place_only)
12154 : {
12155 701274 : if (SLP_TREE_DEF_TYPE (node) == vect_constant_def)
12156 : return true;
12157 320591 : gimple *last_stmt = NULL;
12158 320591 : vec<tree> &defs = (!SLP_TREE_SCALAR_OPS (node).is_empty ()
12159 : ? SLP_TREE_SCALAR_OPS (node)
12160 320591 : : SLP_TREE_VEC_DEFS (node));
12161 1717223 : for (tree def : defs)
12162 : /* If the stmt is not inside the region do not
12163 : use it as possible insertion point. */
12164 755450 : if (auto stmt_info = vinfo->lookup_def (def))
12165 : {
12166 391852 : gimple *stmt = stmt_info->stmt;
12167 391852 : if (!last_stmt)
12168 : last_stmt = stmt;
12169 220862 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12170 : last_stmt = stmt;
12171 67254 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12172 : ;
12173 : else
12174 0 : gcc_unreachable ();
12175 : }
12176 320591 : node->si = last_stmt;
12177 320591 : return true;
12178 : }
12179 :
12180 : /* ??? vectorizable_shift can end up using a scalar operand which is
12181 : currently denoted as !SLP_TREE_VECTYPE. No need to vectorize the
12182 : node in this case. */
12183 513124 : if (!SLP_TREE_VECTYPE (node))
12184 : return true;
12185 :
12186 : /* There are two reasons vector defs might already exist. The first
12187 : is that we are vectorizing an existing vector def. The second is
12188 : when performing BB vectorization shared constant/external nodes
12189 : are not split apart during partitioning so during the code-gen
12190 : DFS walk we can end up visiting them twice. */
12191 505022 : if (! SLP_TREE_VEC_DEFS (node).exists ())
12192 504199 : vect_create_constant_vectors (vinfo, node);
12193 : return true;
12194 : }
12195 :
12196 1921527 : stmt_vec_info stmt_info = SLP_TREE_REPRESENTATIVE (node);
12197 :
12198 1921527 : gcc_assert (SLP_TREE_VEC_DEFS (node).is_empty ());
12199 1921527 : if (!place_only && SLP_TREE_VECTYPE (node))
12200 977250 : SLP_TREE_VEC_DEFS (node).create (vect_get_num_copies (vinfo, node));
12201 :
12202 1921527 : gimple *last_stmt;
12203 1921527 : gimple_stmt_iterator si;
12204 : /* ??? When !place_only we'd like to re-use place_only computed info,
12205 : but this is a bit awkward due to using gsi_insert_before and the
12206 : requirement to insert after vector defs. So we compute last_stmt
12207 : during pre-scheduling and si during scheduling. */
12208 1921527 : if (!SLP_TREE_PERMUTE_P (node) && STMT_VINFO_DATA_REF (stmt_info))
12209 : {
12210 : /* Vectorized loads go before the first scalar load to make it
12211 : ready early, vectorized stores go before the last scalar
12212 : stmt which is where all uses are ready.
12213 : In theory, if we delay dependence checking until after
12214 : placing, we can schedule at other points, but then
12215 : dependence checking would need to honor that. On the
12216 : other hand dependence checking could request a different
12217 : scheduling point as well, if dependences require that. */
12218 1489310 : stmt_vec_info last_stmt_info = NULL;
12219 1489310 : if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info)))
12220 270976 : last_stmt_info = vect_find_first_scalar_stmt_in_slp (node);
12221 : else /* DR_IS_WRITE */
12222 : {
12223 1218334 : last_stmt_info = vect_find_last_scalar_stmt_in_slp (node);
12224 1218334 : if (place_only)
12225 1336106 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12226 : {
12227 668055 : if (child->si
12228 668055 : && !vect_stmt_dominates_stmt_p (child->si,
12229 : last_stmt_info->stmt))
12230 : return false;
12231 : }
12232 : }
12233 1489306 : last_stmt = last_stmt_info->stmt;
12234 1489306 : si = gsi_for_stmt (last_stmt);
12235 1489306 : }
12236 432217 : else if (!SLP_TREE_PERMUTE_P (node)
12237 410574 : && (SLP_TREE_TYPE (node) == cycle_phi_info_type
12238 : || SLP_TREE_TYPE (node) == induc_vec_info_type
12239 : || SLP_TREE_TYPE (node) == phi_info_type))
12240 : {
12241 : /* For PHI node vectorization we do not use the insertion iterator. */
12242 106581 : last_stmt = SLP_TREE_SCALAR_STMTS (node)[0]->stmt;
12243 106581 : if (place_only)
12244 182464 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12245 : {
12246 129101 : if (child->si
12247 225071 : && !dominated_by_p (CDI_DOMINATORS,
12248 : gimple_phi_arg_edge
12249 95970 : (as_a <gphi *> (last_stmt), i)->src,
12250 95970 : gimple_bb (child->si)))
12251 : return false;
12252 : }
12253 106579 : si = gsi_none ();
12254 : }
12255 : else
12256 : {
12257 : /* Emit other stmts after the children vectorized defs which is
12258 : earliest possible. */
12259 : last_stmt = NULL;
12260 902120 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12261 576486 : if (place_only)
12262 : {
12263 188802 : gimple *vstmt = child->si;
12264 188802 : if (!vstmt)
12265 : {
12266 : /* vect_constant_def and defs at region boundary do not
12267 : constrain placement. */
12268 : gcc_assert (SLP_TREE_DEF_TYPE (child) == vect_constant_def
12269 : /* ??? Region boundary is not representated
12270 : by a NULL stmt. */
12271 : || true);
12272 : }
12273 147897 : else if (!last_stmt)
12274 : last_stmt = vstmt;
12275 42530 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12276 : last_stmt = vstmt;
12277 10720 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12278 : ;
12279 : else
12280 : /* Non-trapping stmts from different BBs might be combined,
12281 : and if we later CSE a low/high part we can run into this. */
12282 : return false;
12283 : }
12284 387684 : else if (SLP_TREE_DEF_TYPE (child) == vect_internal_def)
12285 : {
12286 : /* For fold-left reductions we are retaining the scalar
12287 : reduction PHI but we still have SLP_TREE_NUM_VEC_STMTS
12288 : set so the representation isn't perfect. Resort to the
12289 : last scalar def here. */
12290 308313 : if (SLP_TREE_VEC_DEFS (child).is_empty ())
12291 : {
12292 959 : gcc_assert (SLP_TREE_TYPE (child) == cycle_phi_info_type);
12293 959 : gphi *phi = as_a <gphi *>
12294 959 : (vect_find_last_scalar_stmt_in_slp (child)->stmt);
12295 959 : if (!last_stmt)
12296 : last_stmt = phi;
12297 727 : else if (vect_stmt_dominates_stmt_p (last_stmt, phi))
12298 : last_stmt = phi;
12299 716 : else if (vect_stmt_dominates_stmt_p (phi, last_stmt))
12300 : ;
12301 : else
12302 0 : gcc_unreachable ();
12303 : }
12304 : /* We are emitting all vectorized stmts in the same place and
12305 : the last one is the last.
12306 : ??? Unless we have a load permutation applied and that
12307 : figures to re-use an earlier generated load. */
12308 308313 : unsigned j;
12309 308313 : tree vdef;
12310 729185 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12311 420872 : if (TREE_CODE (vdef) == SSA_NAME
12312 420872 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12313 : {
12314 420820 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12315 420820 : if (!last_stmt)
12316 : last_stmt = vstmt;
12317 214928 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12318 : last_stmt = vstmt;
12319 47775 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12320 : ;
12321 : else
12322 0 : gcc_unreachable ();
12323 : }
12324 : }
12325 79371 : else if (!SLP_TREE_VECTYPE (child))
12326 : {
12327 : /* For externals we use unvectorized at all scalar defs. */
12328 : unsigned j;
12329 : tree def;
12330 17321 : FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_OPS (child), j, def)
12331 : /* If the stmt is not inside the region do not
12332 : use it as possible insertion point. */
12333 9847 : if (auto stmt_info = vinfo->lookup_def (def))
12334 : {
12335 138 : gimple *stmt = stmt_info->stmt;
12336 138 : if (!last_stmt)
12337 : last_stmt = stmt;
12338 118 : else if (vect_stmt_dominates_stmt_p (last_stmt, stmt))
12339 : last_stmt = stmt;
12340 26 : else if (vect_stmt_dominates_stmt_p (stmt, last_stmt))
12341 : ;
12342 : else
12343 0 : gcc_unreachable ();
12344 : }
12345 : }
12346 : else
12347 : {
12348 : /* For externals we have to look at all defs since their
12349 : insertion place is decided per vector. But beware
12350 : of pre-existing vectors where we need to make sure
12351 : we do not insert before the region boundary. */
12352 143659 : if (SLP_TREE_SCALAR_OPS (child).is_empty ()
12353 596 : && !vinfo->lookup_def (SLP_TREE_VEC_DEFS (child)[0]))
12354 : ;
12355 : else
12356 : {
12357 : unsigned j;
12358 : tree vdef;
12359 749394 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (child), j, vdef)
12360 101148 : if (TREE_CODE (vdef) == SSA_NAME
12361 101148 : && !SSA_NAME_IS_DEFAULT_DEF (vdef))
12362 : {
12363 20662 : gimple *vstmt = SSA_NAME_DEF_STMT (vdef);
12364 20662 : if (!last_stmt)
12365 : last_stmt = vstmt;
12366 11290 : else if (vect_stmt_dominates_stmt_p (last_stmt, vstmt))
12367 : last_stmt = vstmt;
12368 8967 : else if (vect_stmt_dominates_stmt_p (vstmt, last_stmt))
12369 : ;
12370 : else
12371 0 : gcc_unreachable ();
12372 : }
12373 : }
12374 : }
12375 :
12376 : /* When we face existing vector defs that are external, make sure
12377 : to insert at region boundary. */
12378 325634 : if (last_stmt
12379 320881 : && is_a <bb_vec_info> (vinfo)
12380 125209 : && vinfo->bbs[0] != gimple_bb (last_stmt)
12381 427056 : && dominated_by_p (CDI_DOMINATORS, vinfo->bbs[0],
12382 101422 : gimple_bb (last_stmt)))
12383 : last_stmt = NULL;
12384 :
12385 : /* We split regions to vectorize at control altering stmts
12386 : with a definition so this can only be an external. */
12387 325628 : gcc_checking_assert (!last_stmt
12388 : || !is_ctrl_altering_stmt (last_stmt));
12389 :
12390 325634 : if (is_a <bb_vec_info> (vinfo)
12391 128359 : && !SLP_TREE_PERMUTE_P (node)
12392 121906 : && (!last_stmt
12393 119041 : || !dominated_by_p (CDI_DOMINATORS, gimple_bb (last_stmt),
12394 119041 : gimple_bb (stmt_info->stmt)))
12395 350884 : && gimple_could_trap_p (stmt_info->stmt))
12396 : {
12397 : /* We've constrained possibly trapping operations to all come
12398 : from the same basic-block, if vectorized defs would allow earlier
12399 : scheduling still force vectorized stmts to the original block.
12400 : This is only necessary for BB vectorization since for loop vect
12401 : all operations are in a single BB and scalar stmt based
12402 : placement doesn't play well with epilogue vectorization. */
12403 678 : if (last_stmt
12404 1258 : && !dominated_by_p (CDI_DOMINATORS,
12405 580 : gimple_bb (stmt_info->stmt),
12406 580 : gimple_bb (last_stmt)))
12407 : {
12408 4 : gcc_assert (place_only);
12409 : return false;
12410 : }
12411 674 : si = gsi_after_labels (gimple_bb (stmt_info->stmt));
12412 674 : last_stmt = gsi_stmt (si);
12413 : }
12414 : /* When there is no in-region child def to guide placement, insert
12415 : at region boundary. */
12416 324956 : else if (!last_stmt)
12417 : {
12418 4661 : si = gsi_after_labels (vinfo->bbs[0]);
12419 : /* last_stmt NULL marks the region start. */
12420 : }
12421 320295 : else if (is_a <gphi *> (last_stmt))
12422 31602 : si = gsi_after_labels (gimple_bb (last_stmt));
12423 : else
12424 : {
12425 288693 : si = gsi_for_stmt (last_stmt);
12426 : /* We use gsi_insert_before, so when last_stmt is a vector
12427 : def we have to advance (or use gsi_insert_after). */
12428 288693 : gsi_next (&si);
12429 :
12430 288693 : if (auto loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
12431 : {
12432 : /* We'll have to fix this up for loop vect. */
12433 181030 : gcc_assert (!place_only);
12434 : /* Avoid scheduling stmts to random places in the CFG, any
12435 : stmt dominance check we performed is possibly wrong as UIDs
12436 : are not initialized for all of the function for loop
12437 : vectorization. Instead append to the loop preheader. */
12438 181030 : if ((LOOP_VINFO_LOOP (loop_vinfo)->header
12439 181030 : != gimple_bb (last_stmt))
12440 183995 : && dominated_by_p (CDI_DOMINATORS,
12441 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12442 2965 : gimple_bb (last_stmt)))
12443 1146 : si = gsi_end_bb (loop_preheader_edge
12444 573 : (LOOP_VINFO_LOOP (loop_vinfo))->src);
12445 : /* Avoid scheduling internal defs outside of the loop when
12446 : we might have only implicitly tracked loop mask/len defs. */
12447 76 : if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
12448 181030 : || LOOP_VINFO_FULLY_WITH_LENGTH_P (loop_vinfo))
12449 : {
12450 76 : gimple_stmt_iterator si2
12451 76 : = gsi_after_labels (LOOP_VINFO_LOOP (loop_vinfo)->header);
12452 76 : if ((gsi_end_p (si2)
12453 0 : && (LOOP_VINFO_LOOP (loop_vinfo)->header
12454 0 : != gimple_bb (last_stmt))
12455 0 : && dominated_by_p (CDI_DOMINATORS,
12456 : LOOP_VINFO_LOOP (loop_vinfo)->header,
12457 0 : gimple_bb (last_stmt)))
12458 76 : || (!gsi_end_p (si2)
12459 76 : && last_stmt != *si2
12460 73 : && vect_stmt_dominates_stmt_p (last_stmt, *si2)))
12461 3 : si = si2;
12462 : }
12463 : }
12464 : }
12465 : }
12466 :
12467 1921515 : if (place_only)
12468 : {
12469 944259 : if (dump_enabled_p () && last_stmt)
12470 3976 : dump_printf_loc (MSG_NOTE, vect_location,
12471 : "placing node %p at %G:", (void *)node, last_stmt);
12472 : /* Verify we either get a stmt anchor or region start. */
12473 944259 : gcc_assert ((last_stmt && gimple_bb (last_stmt))
12474 : || (!last_stmt && gsi_bb (si)));
12475 944259 : node->si = last_stmt;
12476 944259 : return true;
12477 : }
12478 :
12479 : /* ??? Asserting vect_stmt_dominates_stmt_p (gsi_stmt (si), node->si)
12480 : does not work because in some cases we advance si from last_stmt (as
12481 : we want to insert after vector stmts) and because vector stmts of
12482 : children have been inserted possibly at the same location constraint,
12483 : moving si even further. */
12484 977256 : if (flag_checking && node->si && gimple_bb (node->si) && !gsi_end_p (si))
12485 : {
12486 601850 : auto gsi2 = si;
12487 601850 : while (1)
12488 : {
12489 601850 : if (vect_stmt_dominates_stmt_p (gsi_stmt (gsi2), node->si))
12490 : break;
12491 : /* As we have possibly advanced si it might now point to the
12492 : scalar stmt immediately following node->si. That's OK. */
12493 23523 : if (gsi_stmt (gsi2) != gsi_stmt (si)
12494 23523 : && gimple_uid (gsi_stmt (gsi2)) != 0)
12495 0 : gcc_unreachable ();
12496 23523 : gsi_prev (&gsi2);
12497 23523 : if (gsi_end_p (gsi2))
12498 : {
12499 537 : if (is_a <gphi *> (node->si)
12500 537 : && gimple_bb (node->si) == gsi_bb (si))
12501 : break;
12502 0 : gcc_unreachable ();
12503 : }
12504 : }
12505 : }
12506 :
12507 977256 : if (dump_enabled_p ())
12508 : {
12509 74283 : if (stmt_info)
12510 70861 : dump_printf_loc (MSG_NOTE, vect_location,
12511 : "------>vectorizing SLP node starting from: %G",
12512 : stmt_info->stmt);
12513 : else
12514 : {
12515 3422 : dump_printf_loc (MSG_NOTE, vect_location,
12516 : "------>vectorizing SLP node:\n");
12517 3422 : vect_print_slp_tree (MSG_NOTE, vect_location, node);
12518 : }
12519 : }
12520 977256 : vect_transform_stmt (vinfo, stmt_info, &si, node, instance);
12521 977256 : return true;
12522 : }
12523 :
12524 : /* Replace scalar calls from SLP node NODE with setting of their lhs to zero.
12525 : For loop vectorization this is done in vectorizable_call, but for SLP
12526 : it needs to be deferred until end of vect_schedule_slp, because multiple
12527 : SLP instances may refer to the same scalar stmt. */
12528 :
12529 : static void
12530 623408 : vect_remove_slp_scalar_calls (vec_info *vinfo,
12531 : slp_tree node, hash_set<slp_tree> &visited)
12532 : {
12533 623408 : gimple *new_stmt;
12534 623408 : gimple_stmt_iterator gsi;
12535 623408 : tree lhs;
12536 :
12537 623408 : if (!node || SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12538 197343 : return;
12539 :
12540 470945 : if (visited.add (node))
12541 : return;
12542 :
12543 1595509 : for (auto child : SLP_TREE_CHILDREN (node))
12544 531074 : vect_remove_slp_scalar_calls (vinfo, child, visited);
12545 :
12546 1770031 : for (auto stmt_info : SLP_TREE_SCALAR_STMTS (node))
12547 : {
12548 500472 : if (!stmt_info)
12549 4140 : continue;
12550 496332 : stmt_info = vect_orig_stmt (stmt_info);
12551 496332 : gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt);
12552 5176 : if (!stmt || gimple_bb (stmt) == NULL)
12553 491198 : continue;
12554 5134 : lhs = gimple_call_lhs (stmt);
12555 5134 : if (lhs)
12556 4563 : new_stmt = gimple_build_assign (lhs, build_zero_cst (TREE_TYPE (lhs)));
12557 : else
12558 571 : new_stmt = gimple_build_nop ();
12559 5134 : unlink_stmt_vdef (stmt_info->stmt);
12560 5134 : gsi = gsi_for_stmt (stmt);
12561 5134 : vinfo->replace_stmt (&gsi, stmt_info, new_stmt);
12562 5134 : if (lhs)
12563 4563 : SSA_NAME_DEF_STMT (lhs) = new_stmt;
12564 : }
12565 : }
12566 :
12567 : static void
12568 92334 : vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
12569 : {
12570 92334 : hash_set<slp_tree> visited;
12571 92334 : vect_remove_slp_scalar_calls (vinfo, node, visited);
12572 92334 : }
12573 :
12574 : /* Vectorize the instance root. */
12575 :
12576 : void
12577 9250 : vectorize_slp_instance_root_stmt (vec_info *vinfo, slp_tree node, slp_instance instance)
12578 : {
12579 9250 : if (instance->kind == slp_inst_kind_ctor)
12580 : {
12581 5525 : tree new_def;
12582 5525 : if (SLP_TREE_VEC_DEFS (node).length () == 1)
12583 : {
12584 5480 : new_def = SLP_TREE_VEC_DEFS (node)[0];
12585 5480 : tree root_lhs = gimple_get_lhs (instance->root_stmts[0]->stmt);
12586 5480 : if (!useless_type_conversion_p (TREE_TYPE (root_lhs),
12587 5480 : TREE_TYPE (new_def)))
12588 0 : new_def = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (root_lhs),
12589 : new_def);
12590 : }
12591 : else
12592 : {
12593 45 : gcc_assert (SLP_TREE_VEC_DEFS (node).length () > 1);
12594 45 : tree child_def;
12595 45 : int j;
12596 45 : vec<constructor_elt, va_gc> *v;
12597 45 : vec_alloc (v, SLP_TREE_VEC_DEFS (node).length ());
12598 :
12599 : /* A CTOR can handle V16HI composition from VNx8HI so we
12600 : do not need to convert vector elements if the types
12601 : do not match. */
12602 180 : FOR_EACH_VEC_ELT (SLP_TREE_VEC_DEFS (node), j, child_def)
12603 90 : CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, child_def);
12604 45 : tree rtype
12605 45 : = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmts[0]->stmt));
12606 45 : new_def = build_constructor (rtype, v);
12607 : }
12608 :
12609 5525 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12610 5525 : gimple_assign_set_rhs_from_tree (&rgsi, new_def);
12611 5525 : update_stmt (gsi_stmt (rgsi));
12612 5525 : return;
12613 : }
12614 3725 : else if (instance->kind == slp_inst_kind_bb_reduc)
12615 : {
12616 : /* Largely inspired by reduction chain epilogue handling in
12617 : vect_create_epilog_for_reduction. */
12618 2129 : vec<tree> vec_defs = vNULL;
12619 2129 : vect_get_slp_defs (node, &vec_defs);
12620 2129 : enum tree_code reduc_code
12621 2129 : = gimple_assign_rhs_code (instance->root_stmts[0]->stmt);
12622 : /* ??? We actually have to reflect signs somewhere. */
12623 2129 : if (reduc_code == MINUS_EXPR)
12624 0 : reduc_code = PLUS_EXPR;
12625 2129 : gimple_seq epilogue = NULL;
12626 : /* We may end up with more than one vector result, reduce them
12627 : to one vector. */
12628 2129 : tree vec_def = vec_defs[0];
12629 2129 : tree vectype = TREE_TYPE (vec_def);
12630 2129 : tree compute_vectype = vectype;
12631 2129 : bool pun_for_overflow_p = (ANY_INTEGRAL_TYPE_P (vectype)
12632 968 : && TYPE_OVERFLOW_UNDEFINED (vectype)
12633 2714 : && operation_can_overflow (reduc_code));
12634 437 : if (pun_for_overflow_p)
12635 : {
12636 437 : compute_vectype = unsigned_type_for (vectype);
12637 437 : vec_def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12638 : compute_vectype, vec_def);
12639 : }
12640 2394 : for (unsigned i = 1; i < vec_defs.length (); ++i)
12641 : {
12642 265 : tree def = vec_defs[i];
12643 265 : if (pun_for_overflow_p)
12644 39 : def = gimple_build (&epilogue, VIEW_CONVERT_EXPR,
12645 : compute_vectype, def);
12646 265 : vec_def = gimple_build (&epilogue, reduc_code, compute_vectype,
12647 : vec_def, def);
12648 : }
12649 2129 : vec_defs.release ();
12650 : /* ??? Support other schemes than direct internal fn or two
12651 : element vectors. */
12652 2129 : tree scalar_def;
12653 2129 : internal_fn reduc_fn;
12654 3105 : if (!reduction_fn_for_scalar_code (reduc_code, &reduc_fn)
12655 2129 : || reduc_fn == IFN_LAST
12656 4258 : || !direct_internal_fn_supported_p (reduc_fn, compute_vectype,
12657 : OPTIMIZE_FOR_BOTH))
12658 : {
12659 976 : gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (compute_vectype), 2u));
12660 976 : tree tem0 = gimple_build (&epilogue, BIT_FIELD_REF,
12661 976 : TREE_TYPE (compute_vectype), vec_def,
12662 976 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12663 976 : bitsize_zero_node);
12664 976 : tree tem1 = gimple_build (&epilogue, BIT_FIELD_REF,
12665 976 : TREE_TYPE (compute_vectype), vec_def,
12666 976 : TYPE_SIZE (TREE_TYPE (compute_vectype)),
12667 976 : TYPE_SIZE (TREE_TYPE (compute_vectype)));
12668 976 : scalar_def = gimple_build (&epilogue, reduc_code,
12669 976 : TREE_TYPE (compute_vectype), tem0, tem1);
12670 : }
12671 : else
12672 1153 : scalar_def = gimple_build (&epilogue, as_combined_fn (reduc_fn),
12673 1153 : TREE_TYPE (compute_vectype), vec_def);
12674 2129 : if (!SLP_INSTANCE_REMAIN_DEFS (instance).is_empty ())
12675 : {
12676 850 : tree rem_def = NULL_TREE;
12677 1984 : for (auto def : SLP_INSTANCE_REMAIN_DEFS (instance))
12678 : {
12679 1134 : def = gimple_convert (&epilogue, TREE_TYPE (scalar_def), def);
12680 1134 : if (!rem_def)
12681 : rem_def = def;
12682 : else
12683 284 : rem_def = gimple_build (&epilogue, reduc_code,
12684 284 : TREE_TYPE (scalar_def),
12685 : rem_def, def);
12686 : }
12687 850 : scalar_def = gimple_build (&epilogue, reduc_code,
12688 850 : TREE_TYPE (scalar_def),
12689 : scalar_def, rem_def);
12690 : }
12691 2129 : scalar_def = gimple_convert (&epilogue,
12692 2129 : TREE_TYPE (vectype), scalar_def);
12693 2129 : gimple_stmt_iterator rgsi = gsi_for_stmt (instance->root_stmts[0]->stmt);
12694 2129 : gsi_insert_seq_before (&rgsi, epilogue, GSI_SAME_STMT);
12695 2129 : gimple_assign_set_rhs_from_tree (&rgsi, scalar_def);
12696 2129 : update_stmt (gsi_stmt (rgsi));
12697 2129 : return;
12698 : }
12699 1596 : else if (instance->kind == slp_inst_kind_gcond)
12700 : {
12701 : /* Only support a single root for now as we can't codegen CFG yet and so we
12702 : can't support lane > 1 at this time. */
12703 1596 : gcc_assert (instance->root_stmts.length () == 1);
12704 1596 : auto root_stmt_info = instance->root_stmts[0];
12705 1596 : auto last_stmt = STMT_VINFO_STMT (vect_orig_stmt (root_stmt_info));
12706 1596 : gimple_stmt_iterator rgsi = gsi_for_stmt (last_stmt);
12707 1596 : gcc_assert (!SLP_TREE_VEC_DEFS (node).is_empty ());
12708 1596 : bool res = vectorizable_early_exit (as_a <loop_vec_info> (vinfo),
12709 : root_stmt_info, &rgsi, node, NULL);
12710 1596 : gcc_assert (res);
12711 1596 : return;
12712 : }
12713 : else
12714 0 : gcc_unreachable ();
12715 : }
12716 :
12717 : struct slp_scc_info
12718 : {
12719 : bool on_stack;
12720 : bool res;
12721 : int dfs;
12722 : int lowlink;
12723 : };
12724 :
12725 : /* Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs.
12726 : When PLACE_ONLY, return false if there is no possible schedule. */
12727 :
12728 : static bool
12729 3135925 : vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance,
12730 : hash_map<slp_tree, slp_scc_info> &scc_info,
12731 : int &maxdfs, vec<slp_tree> &stack, bool place_only)
12732 : {
12733 3135925 : bool existed_p;
12734 3135925 : slp_scc_info *info = &scc_info.get_or_insert (node, &existed_p);
12735 3135925 : gcc_assert (!existed_p);
12736 3135925 : info->dfs = maxdfs;
12737 3135925 : info->lowlink = maxdfs;
12738 3135925 : maxdfs++;
12739 :
12740 : /* Leaf. */
12741 3135925 : if (SLP_TREE_DEF_TYPE (node) != vect_internal_def)
12742 : {
12743 1214398 : info->on_stack = false;
12744 1214398 : bool res = vect_schedule_slp_node (vinfo, node, instance, place_only);
12745 1214398 : gcc_assert (res);
12746 1214398 : info->res = res;
12747 1214398 : return res;
12748 : }
12749 :
12750 1921527 : info->on_stack = true;
12751 1921527 : info->res = true;
12752 1921527 : stack.safe_push (node);
12753 :
12754 1921527 : bool res = true;
12755 1921527 : unsigned i;
12756 1921527 : slp_tree child;
12757 : /* DFS recurse. */
12758 7533450 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child)
12759 : {
12760 2040150 : if (!child)
12761 56398 : continue;
12762 1983752 : slp_scc_info *child_info = scc_info.get (child);
12763 1983752 : if (!child_info)
12764 : {
12765 1834155 : res &= vect_schedule_scc (vinfo, child, instance, scc_info,
12766 : maxdfs, stack, place_only);
12767 : /* Recursion might have re-allocated the node. */
12768 1834155 : info = scc_info.get (node);
12769 1834155 : child_info = scc_info.get (child);
12770 1834155 : info->lowlink = MIN (info->lowlink, child_info->lowlink);
12771 : }
12772 149597 : else if (child_info->on_stack)
12773 35528 : info->lowlink = MIN (info->lowlink, child_info->dfs);
12774 : else
12775 114069 : res &= child_info->res;
12776 : }
12777 1921527 : if (info->lowlink != info->dfs)
12778 : return res;
12779 :
12780 1882124 : auto_vec<slp_tree, 4> phis_to_fixup;
12781 :
12782 : /* Singleton. */
12783 1882124 : if (stack.last () == node)
12784 : {
12785 1853229 : stack.pop ();
12786 1853229 : info->on_stack = false;
12787 1853229 : res &= vect_schedule_slp_node (vinfo, node, instance, place_only);
12788 1853229 : info->res = res;
12789 1853229 : if (!SLP_TREE_PERMUTE_P (node)
12790 1853229 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (node)->stmt))
12791 71784 : phis_to_fixup.quick_push (node);
12792 : }
12793 : else
12794 : {
12795 : /* SCC. */
12796 28895 : int last_idx = stack.length () - 1;
12797 68298 : while (stack[last_idx] != node)
12798 39403 : last_idx--;
12799 : /* We can break the cycle at PHIs who have at least one child
12800 : code generated. Then we could re-start the DFS walk until
12801 : all nodes in the SCC are covered (we might have new entries
12802 : for only back-reachable nodes). But it's simpler to just
12803 : iterate and schedule those that are ready. */
12804 28895 : unsigned todo = stack.length () - last_idx;
12805 28895 : auto_vec<slp_tree, 4> saved_scc (todo);
12806 28895 : saved_scc.quick_grow (todo);
12807 86685 : memcpy (saved_scc.address (), &stack.address ()[last_idx],
12808 : sizeof (slp_tree) * todo);
12809 29703 : do
12810 : {
12811 131927 : for (int idx = stack.length () - 1; idx >= last_idx; --idx)
12812 : {
12813 72521 : slp_tree entry = stack[idx];
12814 72521 : if (!entry)
12815 2553 : continue;
12816 69968 : bool phi = (!SLP_TREE_PERMUTE_P (entry)
12817 69968 : && is_a <gphi *> (SLP_TREE_REPRESENTATIVE (entry)->stmt));
12818 69968 : bool ready = !phi;
12819 212874 : FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (entry), i, child)
12820 132720 : if (!child)
12821 : {
12822 22932 : gcc_assert (phi);
12823 : ready = true;
12824 : break;
12825 : }
12826 109788 : else if (scc_info.get (child)->on_stack)
12827 : {
12828 30787 : if (!phi)
12829 : {
12830 : ready = false;
12831 : break;
12832 : }
12833 : }
12834 : else
12835 : {
12836 79001 : if (phi)
12837 : {
12838 : ready = true;
12839 : break;
12840 : }
12841 : }
12842 47036 : if (ready)
12843 : {
12844 68298 : res &= vect_schedule_slp_node (vinfo, entry, instance,
12845 : place_only);
12846 68298 : scc_info.get (entry)->on_stack = false;
12847 68298 : stack[idx] = NULL;
12848 68298 : todo--;
12849 68298 : if (phi)
12850 35356 : phis_to_fixup.safe_push (entry);
12851 : }
12852 : }
12853 : }
12854 29703 : while (todo != 0);
12855 :
12856 : /* Push the scheduling result to all of the SCC. */
12857 154983 : for (slp_tree entry : saved_scc)
12858 68298 : scc_info.get (entry)->res = res;
12859 :
12860 : /* Pop the SCC. */
12861 28895 : stack.truncate (last_idx);
12862 28895 : }
12863 :
12864 1882124 : if (place_only)
12865 : return res;
12866 :
12867 : /* Now fixup the backedge def of the vectorized PHIs in this SCC. */
12868 : slp_tree phi_node;
12869 999250 : FOR_EACH_VEC_ELT (phis_to_fixup, i, phi_node)
12870 : {
12871 53775 : gphi *phi = as_a <gphi *> (SLP_TREE_REPRESENTATIVE (phi_node)->stmt);
12872 53775 : edge_iterator ei;
12873 53775 : edge e;
12874 167552 : FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
12875 : {
12876 113777 : unsigned dest_idx = e->dest_idx;
12877 113777 : child = SLP_TREE_CHILDREN (phi_node)[dest_idx];
12878 113777 : if (!child || SLP_TREE_DEF_TYPE (child) != vect_internal_def)
12879 66380 : continue;
12880 47397 : unsigned n = SLP_TREE_VEC_DEFS (phi_node).length ();
12881 : /* Simply fill all args. */
12882 47397 : if (STMT_VINFO_DEF_TYPE (SLP_TREE_REPRESENTATIVE (phi_node))
12883 : != vect_first_order_recurrence)
12884 100182 : for (unsigned i = 0; i < n; ++i)
12885 : {
12886 52828 : tree phidef = SLP_TREE_VEC_DEFS (phi_node)[i];
12887 52828 : gphi *phi = as_a <gphi *> (SSA_NAME_DEF_STMT (phidef));
12888 52828 : add_phi_arg (phi, vect_get_slp_vect_def (child, i),
12889 : e, gimple_phi_arg_location (phi, dest_idx));
12890 : }
12891 : else
12892 : {
12893 : /* Unless it is a first order recurrence which needs
12894 : args filled in for both the PHI node and the permutes. */
12895 43 : gimple *perm
12896 43 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[0]);
12897 43 : gimple *rphi = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (perm));
12898 43 : add_phi_arg (as_a <gphi *> (rphi),
12899 : vect_get_slp_vect_def (child, n - 1),
12900 : e, gimple_phi_arg_location (phi, dest_idx));
12901 166 : for (unsigned i = 0; i < n; ++i)
12902 : {
12903 80 : gimple *perm
12904 80 : = SSA_NAME_DEF_STMT (SLP_TREE_VEC_DEFS (phi_node)[i]);
12905 80 : if (i > 0)
12906 37 : gimple_assign_set_rhs1 (perm,
12907 : vect_get_slp_vect_def (child, i - 1));
12908 80 : gimple_assign_set_rhs2 (perm,
12909 : vect_get_slp_vect_def (child, i));
12910 80 : update_stmt (perm);
12911 : }
12912 : }
12913 : }
12914 : }
12915 :
12916 945475 : gcc_assert (res);
12917 : return true;
12918 1882124 : }
12919 :
12920 : /* Generate vector code for SLP_INSTANCES in the loop/basic block. Perform
12921 : vector stmt placement only when PLACE_ONLY is true, removing SLP graph
12922 : entries that cannot be scheduled. If placing, return false if a schedule
12923 : cannot be computed for any entry. */
12924 :
12925 : bool
12926 793397 : vect_schedule_slp (vec_info *vinfo, vec<slp_instance> &slp_instances,
12927 : bool place_only)
12928 : {
12929 793397 : slp_instance instance;
12930 793397 : unsigned int i;
12931 :
12932 793397 : hash_map<slp_tree, slp_scc_info> scc_info;
12933 793397 : int maxdfs = 0;
12934 2099023 : for (i = 0; slp_instances.iterate (i, &instance); )
12935 : {
12936 1305626 : slp_tree node = SLP_INSTANCE_TREE (instance);
12937 1305626 : if (!place_only && dump_enabled_p ())
12938 : {
12939 16561 : dump_printf_loc (MSG_NOTE, vect_location,
12940 : "Vectorizing SLP tree:\n");
12941 : /* ??? Dump all? */
12942 16561 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12943 520 : dump_printf_loc (MSG_NOTE, vect_location, "Root stmt: %G",
12944 520 : SLP_INSTANCE_ROOT_STMTS (instance)[0]->stmt);
12945 16561 : vect_print_slp_graph (MSG_NOTE, vect_location,
12946 : SLP_INSTANCE_TREE (instance));
12947 : }
12948 : /* Schedule the tree of INSTANCE, scheduling SCCs in a way to
12949 : have a PHI be the node breaking the cycle. */
12950 1305626 : bool res;
12951 1305626 : if (slp_scc_info *info = scc_info.get (node))
12952 3856 : res = info->res;
12953 : else
12954 : {
12955 1301770 : auto_vec<slp_tree> stack;
12956 1301770 : res = vect_schedule_scc (vinfo, node, instance, scc_info,
12957 : maxdfs, stack, place_only);
12958 1301770 : }
12959 :
12960 1305626 : if (!SLP_INSTANCE_ROOT_STMTS (instance).is_empty ())
12961 : {
12962 61965 : if (place_only)
12963 : {
12964 52715 : gimple *root_stmt = instance->root_stmts[0]->stmt;
12965 52715 : res &= (!node->si
12966 : /* As we instert after node->si it may not be the
12967 : root_stmt itself. */
12968 52715 : || (node->si != root_stmt
12969 51821 : && vect_stmt_dominates_stmt_p (node->si, root_stmt)));
12970 : }
12971 : else
12972 9250 : vectorize_slp_instance_root_stmt (vinfo, node, instance);
12973 : }
12974 :
12975 1305626 : if (!place_only && dump_enabled_p ())
12976 16561 : dump_printf_loc (MSG_NOTE, vect_location,
12977 : "vectorizing stmts using SLP.\n");
12978 :
12979 1305626 : if (!res)
12980 : {
12981 38 : gcc_assert (place_only);
12982 38 : if (dump_enabled_p ())
12983 24 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
12984 : "not vectorized: cannot schedule SLP graph "
12985 : "entry %p\n", (void *)instance);
12986 38 : vect_free_slp_instance (instance);
12987 38 : slp_instances.ordered_remove (i);
12988 38 : continue;
12989 : }
12990 1305588 : ++i;
12991 : }
12992 :
12993 793397 : if (place_only)
12994 497242 : return !slp_instances.is_empty ();
12995 :
12996 1923029 : FOR_EACH_VEC_ELT (slp_instances, i, instance)
12997 : {
12998 584856 : slp_tree root = SLP_INSTANCE_TREE (instance);
12999 584856 : stmt_vec_info store_info;
13000 584856 : unsigned int j;
13001 :
13002 : /* Remove scalar call stmts. Do not do this for basic-block
13003 : vectorization as not all uses may be vectorized.
13004 : ??? Why should this be necessary? DCE should be able to
13005 : remove the stmts itself.
13006 : ??? For BB vectorization we can as well remove scalar
13007 : stmts starting from the SLP tree root if they have no
13008 : uses. */
13009 584856 : if (is_a <loop_vec_info> (vinfo))
13010 92334 : vect_remove_slp_scalar_calls (vinfo, root);
13011 :
13012 : /* Remove vectorized stores original scalar stmts. */
13013 2617192 : for (j = 0; SLP_TREE_SCALAR_STMTS (root).iterate (j, &store_info); j++)
13014 : {
13015 1482057 : if (!store_info
13016 1482040 : || !STMT_VINFO_DATA_REF (store_info)
13017 1453262 : || !DR_IS_WRITE (STMT_VINFO_DATA_REF (store_info)))
13018 : break;
13019 :
13020 1447480 : store_info = vect_orig_stmt (store_info);
13021 : /* Free the attached stmt_vec_info and remove the stmt. */
13022 1447480 : vinfo->remove_stmt (store_info);
13023 :
13024 : /* Invalidate SLP_TREE_REPRESENTATIVE in case we released it
13025 : to not crash in vect_free_slp_tree later. */
13026 1447480 : if (SLP_TREE_REPRESENTATIVE (root) == store_info)
13027 549948 : SLP_TREE_REPRESENTATIVE (root) = NULL;
13028 : }
13029 : }
13030 :
13031 : return true;
13032 793397 : }
|
