Line data Source code
1 : /* Data References Analysis and Manipulation Utilities for Vectorization.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : and Ira Rosen <irar@il.ibm.com>
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #define INCLUDE_ALGORITHM
23 : #include "config.h"
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "target.h"
28 : #include "rtl.h"
29 : #include "tree.h"
30 : #include "gimple.h"
31 : #include "predict.h"
32 : #include "memmodel.h"
33 : #include "tm_p.h"
34 : #include "ssa.h"
35 : #include "optabs-tree.h"
36 : #include "cgraph.h"
37 : #include "dumpfile.h"
38 : #include "pretty-print.h"
39 : #include "alias.h"
40 : #include "fold-const.h"
41 : #include "stor-layout.h"
42 : #include "tree-eh.h"
43 : #include "gimplify.h"
44 : #include "gimple-iterator.h"
45 : #include "gimplify-me.h"
46 : #include "tree-ssa-loop-ivopts.h"
47 : #include "tree-ssa-loop-manip.h"
48 : #include "tree-ssa-loop.h"
49 : #include "cfgloop.h"
50 : #include "tree-scalar-evolution.h"
51 : #include "tree-vectorizer.h"
52 : #include "expr.h"
53 : #include "builtins.h"
54 : #include "tree-cfg.h"
55 : #include "tree-hash-traits.h"
56 : #include "vec-perm-indices.h"
57 : #include "internal-fn.h"
58 : #include "gimple-fold.h"
59 : #include "optabs-query.h"
60 :
61 : /* Return true if load- or store-lanes optab OPTAB is implemented for
62 : COUNT vectors of type VECTYPE. NAME is the name of OPTAB.
63 :
64 : If it is implemented and ELSVALS is nonzero store the possible else
65 : values in the vector it points to. */
66 :
67 : static bool
68 374350 : vect_lanes_optab_supported_p (const char *name, convert_optab optab,
69 : tree vectype, unsigned HOST_WIDE_INT count,
70 : vec<int> *elsvals = nullptr)
71 : {
72 374350 : machine_mode mode, array_mode;
73 374350 : bool limit_p;
74 :
75 374350 : mode = TYPE_MODE (vectype);
76 374350 : if (!targetm.array_mode (mode, count).exists (&array_mode))
77 : {
78 748700 : poly_uint64 bits = count * GET_MODE_BITSIZE (mode);
79 374350 : limit_p = !targetm.array_mode_supported_p (mode, count);
80 374350 : if (!int_mode_for_size (bits, limit_p).exists (&array_mode))
81 : {
82 320466 : if (dump_enabled_p ())
83 12868 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
84 : "no array mode for %s[%wu]\n",
85 12868 : GET_MODE_NAME (mode), count);
86 320466 : return false;
87 : }
88 : }
89 :
90 53884 : enum insn_code icode;
91 53884 : if ((icode = convert_optab_handler (optab, array_mode, mode))
92 : == CODE_FOR_nothing)
93 : {
94 53884 : if (dump_enabled_p ())
95 4260 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
96 : "cannot use %s<%s><%s>\n", name,
97 4260 : GET_MODE_NAME (array_mode), GET_MODE_NAME (mode));
98 : return false;
99 : }
100 :
101 0 : if (dump_enabled_p ())
102 0 : dump_printf_loc (MSG_NOTE, vect_location,
103 0 : "can use %s<%s><%s>\n", name, GET_MODE_NAME (array_mode),
104 0 : GET_MODE_NAME (mode));
105 :
106 0 : if (elsvals)
107 0 : get_supported_else_vals (icode,
108 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD_LANES),
109 : *elsvals);
110 :
111 : return true;
112 : }
113 :
114 :
115 : /* Insert DDR into LOOP_VINFO list of ddrs that may alias and need to be
116 : tested at run-time. Return TRUE if DDR was successfully inserted.
117 : Return false if versioning is not supported. */
118 :
119 : static opt_result
120 181094 : vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
121 : {
122 181094 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
123 :
124 181094 : if ((unsigned) param_vect_max_version_for_alias_checks == 0)
125 54 : return opt_result::failure_at (vect_location,
126 : "will not create alias checks, as"
127 : " --param vect-max-version-for-alias-checks"
128 : " == 0\n");
129 :
130 181040 : opt_result res
131 181040 : = runtime_alias_check_p (ddr, loop,
132 : optimize_loop_nest_for_speed_p (loop));
133 181040 : if (!res)
134 143 : return res;
135 :
136 180897 : LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo).safe_push (ddr);
137 180897 : return opt_result::success ();
138 : }
139 :
140 : /* Record that loop LOOP_VINFO needs to check that VALUE is nonzero. */
141 :
142 : static void
143 1503 : vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value)
144 : {
145 1503 : const vec<tree> &checks = LOOP_VINFO_CHECK_NONZERO (loop_vinfo);
146 2258 : for (unsigned int i = 0; i < checks.length(); ++i)
147 755 : if (checks[i] == value)
148 : return;
149 :
150 1503 : if (dump_enabled_p ())
151 418 : dump_printf_loc (MSG_NOTE, vect_location,
152 : "need run-time check that %T is nonzero\n",
153 : value);
154 1503 : LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value);
155 : }
156 :
157 : /* Return true if we know that the order of vectorized DR_INFO_A and
158 : vectorized DR_INFO_B will be the same as the order of DR_INFO_A and
159 : DR_INFO_B. At least one of the accesses is a write. */
160 :
161 : static bool
162 151190 : vect_preserves_scalar_order_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b)
163 : {
164 151190 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
165 151190 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
166 :
167 : /* Single statements are always kept in their original order. */
168 151190 : if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
169 251186 : && !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
170 : return true;
171 :
172 : /* If there is a loop invariant read involved we might vectorize it in
173 : the prologue, breaking scalar order with respect to the in-loop store. */
174 27956 : if ((DR_IS_READ (dr_info_a->dr) && integer_zerop (DR_STEP (dr_info_a->dr)))
175 84790 : || (DR_IS_READ (dr_info_b->dr) && integer_zerop (DR_STEP (dr_info_b->dr))))
176 : return false;
177 :
178 : /* STMT_A and STMT_B belong to overlapping groups. All loads are
179 : emitted at the position of the first scalar load.
180 : Stores in a group are emitted at the position of the last scalar store.
181 : Compute that position and check whether the resulting order matches
182 : the current one. */
183 56283 : stmt_vec_info il_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
184 56283 : if (il_a)
185 : {
186 50643 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_a)))
187 213417 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
188 188010 : s = DR_GROUP_NEXT_ELEMENT (s))
189 188010 : il_a = get_later_stmt (il_a, s);
190 : else /* DR_IS_READ */
191 96064 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
192 70828 : s = DR_GROUP_NEXT_ELEMENT (s))
193 70828 : if (get_later_stmt (il_a, s) == il_a)
194 2456 : il_a = s;
195 : }
196 : else
197 : il_a = stmtinfo_a;
198 56283 : stmt_vec_info il_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
199 56283 : if (il_b)
200 : {
201 47960 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_b)))
202 274936 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
203 236408 : s = DR_GROUP_NEXT_ELEMENT (s))
204 236408 : il_b = get_later_stmt (il_b, s);
205 : else /* DR_IS_READ */
206 43074 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
207 33642 : s = DR_GROUP_NEXT_ELEMENT (s))
208 33642 : if (get_later_stmt (il_b, s) == il_b)
209 365 : il_b = s;
210 : }
211 : else
212 : il_b = stmtinfo_b;
213 56283 : bool a_after_b = (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a);
214 56283 : return (get_later_stmt (il_a, il_b) == il_a) == a_after_b;
215 : }
216 :
217 : /* A subroutine of vect_analyze_data_ref_dependence. Handle
218 : DDR_COULD_BE_INDEPENDENT_P ddr DDR that has a known set of dependence
219 : distances. These distances are conservatively correct but they don't
220 : reflect a guaranteed dependence.
221 :
222 : Return true if this function does all the work necessary to avoid
223 : an alias or false if the caller should use the dependence distances
224 : to limit the vectorization factor in the usual way. LOOP_DEPTH is
225 : the depth of the loop described by LOOP_VINFO and the other arguments
226 : are as for vect_analyze_data_ref_dependence. */
227 :
228 : static bool
229 9244 : vect_analyze_possibly_independent_ddr (data_dependence_relation *ddr,
230 : loop_vec_info loop_vinfo,
231 : int loop_depth, unsigned int *max_vf)
232 : {
233 9244 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
234 36994 : for (lambda_vector &dist_v : DDR_DIST_VECTS (ddr))
235 : {
236 18217 : int dist = dist_v[loop_depth];
237 18217 : if (dist != 0 && !(dist > 0 && DDR_REVERSED_P (ddr)))
238 : {
239 : /* If the user asserted safelen >= DIST consecutive iterations
240 : can be executed concurrently, assume independence.
241 :
242 : ??? An alternative would be to add the alias check even
243 : in this case, and vectorize the fallback loop with the
244 : maximum VF set to safelen. However, if the user has
245 : explicitly given a length, it's less likely that that
246 : would be a win. */
247 8987 : if (loop->safelen >= 2 && abs_hwi (dist) <= loop->safelen)
248 : {
249 32 : if ((unsigned int) loop->safelen < *max_vf)
250 2 : *max_vf = loop->safelen;
251 32 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
252 32 : continue;
253 : }
254 :
255 : /* For dependence distances of 2 or more, we have the option
256 : of limiting VF or checking for an alias at runtime.
257 : Prefer to check at runtime if we can, to avoid limiting
258 : the VF unnecessarily when the bases are in fact independent.
259 :
260 : Note that the alias checks will be removed if the VF ends up
261 : being small enough. */
262 8955 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
263 8955 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
264 8955 : return (!STMT_VINFO_GATHER_SCATTER_P (dr_info_a->stmt)
265 8955 : && !STMT_VINFO_GATHER_SCATTER_P (dr_info_b->stmt)
266 17918 : && vect_mark_for_runtime_alias_test (ddr, loop_vinfo));
267 : }
268 : }
269 : return true;
270 : }
271 :
272 :
273 : /* Function vect_analyze_data_ref_dependence.
274 :
275 : FIXME: I needed to change the sense of the returned flag.
276 :
277 : Return FALSE if there (might) exist a dependence between a memory-reference
278 : DRA and a memory-reference DRB. When versioning for alias may check a
279 : dependence at run-time, return TRUE. Adjust *MAX_VF according to
280 : the data dependence. */
281 :
282 : static opt_result
283 1519191 : vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
284 : loop_vec_info loop_vinfo,
285 : unsigned int *max_vf)
286 : {
287 1519191 : unsigned int i;
288 1519191 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
289 1519191 : struct data_reference *dra = DDR_A (ddr);
290 1519191 : struct data_reference *drb = DDR_B (ddr);
291 1519191 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (dra);
292 1519191 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (drb);
293 1519191 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
294 1519191 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
295 1519191 : lambda_vector dist_v;
296 1519191 : unsigned int loop_depth;
297 :
298 : /* If user asserted safelen consecutive iterations can be
299 : executed concurrently, assume independence. */
300 1710747 : auto apply_safelen = [&]()
301 : {
302 191556 : if (loop->safelen >= 2)
303 : {
304 7648 : if ((unsigned int) loop->safelen < *max_vf)
305 1913 : *max_vf = loop->safelen;
306 7648 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
307 7648 : return true;
308 : }
309 : return false;
310 1519191 : };
311 :
312 : /* In loop analysis all data references should be vectorizable. */
313 1519191 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
314 1519191 : || !STMT_VINFO_VECTORIZABLE (stmtinfo_b))
315 0 : gcc_unreachable ();
316 :
317 : /* Independent data accesses. */
318 1519191 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
319 1238656 : return opt_result::success ();
320 :
321 280535 : if (dra == drb
322 280535 : || (DR_IS_READ (dra) && DR_IS_READ (drb)))
323 0 : return opt_result::success ();
324 :
325 : /* We do not have to consider dependences between accesses that belong
326 : to the same group, unless the stride could be smaller than the
327 : group size. */
328 280535 : if (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
329 121756 : && (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
330 121756 : == DR_GROUP_FIRST_ELEMENT (stmtinfo_b))
331 300337 : && !STMT_VINFO_STRIDED_P (stmtinfo_a))
332 2543 : return opt_result::success ();
333 :
334 : /* Even if we have an anti-dependence then, as the vectorized loop covers at
335 : least two scalar iterations, there is always also a true dependence.
336 : As the vectorizer does not re-order loads and stores we can ignore
337 : the anti-dependence if TBAA can disambiguate both DRs similar to the
338 : case with known negative distance anti-dependences (positive
339 : distance anti-dependences would violate TBAA constraints). */
340 135306 : if (((DR_IS_READ (dra) && DR_IS_WRITE (drb))
341 142686 : || (DR_IS_WRITE (dra) && DR_IS_READ (drb)))
342 432197 : && !alias_sets_conflict_p (get_alias_set (DR_REF (dra)),
343 : get_alias_set (DR_REF (drb))))
344 6259 : return opt_result::success ();
345 :
346 271733 : if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
347 261261 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
348 : {
349 13162 : if (apply_safelen ())
350 1393 : return opt_result::success ();
351 :
352 11769 : return opt_result::failure_at
353 11769 : (stmtinfo_a->stmt,
354 : "possible alias involving gather/scatter between %T and %T\n",
355 : DR_REF (dra), DR_REF (drb));
356 : }
357 :
358 : /* Unknown data dependence. */
359 258571 : if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
360 : {
361 177833 : if (apply_safelen ())
362 6255 : return opt_result::success ();
363 :
364 171578 : if (dump_enabled_p ())
365 7632 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
366 : "versioning for alias required: "
367 : "can't determine dependence between %T and %T\n",
368 : DR_REF (dra), DR_REF (drb));
369 :
370 : /* Add to list of ddrs that need to be tested at run-time. */
371 171578 : return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
372 : }
373 :
374 : /* Known data dependence. */
375 80738 : if (DDR_NUM_DIST_VECTS (ddr) == 0)
376 : {
377 561 : if (apply_safelen ())
378 0 : return opt_result::success ();
379 :
380 561 : if (dump_enabled_p ())
381 156 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
382 : "versioning for alias required: "
383 : "bad dist vector for %T and %T\n",
384 : DR_REF (dra), DR_REF (drb));
385 : /* Add to list of ddrs that need to be tested at run-time. */
386 561 : return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
387 : }
388 :
389 80177 : loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
390 :
391 80177 : if (DDR_COULD_BE_INDEPENDENT_P (ddr)
392 80177 : && vect_analyze_possibly_independent_ddr (ddr, loop_vinfo,
393 : loop_depth, max_vf))
394 9236 : return opt_result::success ();
395 :
396 134505 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
397 : {
398 70963 : int dist = dist_v[loop_depth];
399 :
400 70963 : if (dump_enabled_p ())
401 4757 : dump_printf_loc (MSG_NOTE, vect_location,
402 : "dependence distance = %d.\n", dist);
403 :
404 70963 : if (dist == 0)
405 : {
406 58706 : if (dump_enabled_p ())
407 3654 : dump_printf_loc (MSG_NOTE, vect_location,
408 : "dependence distance == 0 between %T and %T\n",
409 : DR_REF (dra), DR_REF (drb));
410 :
411 : /* When we perform grouped accesses and perform implicit CSE
412 : by detecting equal accesses and doing disambiguation with
413 : runtime alias tests like for
414 : .. = a[i];
415 : .. = a[i+1];
416 : a[i] = ..;
417 : a[i+1] = ..;
418 : *p = ..;
419 : .. = a[i];
420 : .. = a[i+1];
421 : where we will end up loading { a[i], a[i+1] } once, make
422 : sure that inserting group loads before the first load and
423 : stores after the last store will do the right thing.
424 : Similar for groups like
425 : a[i] = ...;
426 : ... = a[i];
427 : a[i+1] = ...;
428 : where loads from the group interleave with the store. */
429 58706 : if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
430 0 : return opt_result::failure_at (stmtinfo_a->stmt,
431 : "READ_WRITE dependence"
432 : " in interleaving.\n");
433 :
434 58706 : if (loop->safelen < 2)
435 : {
436 54973 : tree indicator = dr_zero_step_indicator (dra);
437 54973 : if (!indicator || integer_zerop (indicator))
438 0 : return opt_result::failure_at (stmtinfo_a->stmt,
439 : "access also has a zero step\n");
440 54973 : else if (TREE_CODE (indicator) != INTEGER_CST)
441 1503 : vect_check_nonzero_value (loop_vinfo, indicator);
442 : }
443 58706 : continue;
444 58706 : }
445 :
446 12257 : if (dist > 0 && DDR_REVERSED_P (ddr))
447 : {
448 : /* If DDR_REVERSED_P the order of the data-refs in DDR was
449 : reversed (to make distance vector positive), and the actual
450 : distance is negative. */
451 3611 : if (dump_enabled_p ())
452 114 : dump_printf_loc (MSG_NOTE, vect_location,
453 : "dependence distance negative.\n");
454 : /* When doing outer loop vectorization, we need to check if there is
455 : a backward dependence at the inner loop level if the dependence
456 : at the outer loop is reversed. See PR81740. */
457 3611 : if (nested_in_vect_loop_p (loop, stmtinfo_a)
458 3599 : || nested_in_vect_loop_p (loop, stmtinfo_b))
459 : {
460 12 : unsigned inner_depth = index_in_loop_nest (loop->inner->num,
461 12 : DDR_LOOP_NEST (ddr));
462 12 : if (dist_v[inner_depth] < 0)
463 9 : return opt_result::failure_at (stmtinfo_a->stmt,
464 : "not vectorized, dependence "
465 : "between data-refs %T and %T\n",
466 : DR_REF (dra), DR_REF (drb));
467 : }
468 3602 : if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
469 18 : return opt_result::failure_at (stmtinfo_a->stmt,
470 : "dependence in interleaving.\n");
471 : /* Record a negative dependence distance to later limit the
472 : amount of stmt copying / unrolling we can perform.
473 : Only need to handle read-after-write dependence. */
474 3584 : if (DR_IS_READ (drb)
475 147 : && (STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) == 0
476 36 : || STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) > (unsigned)dist))
477 147 : STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) = dist;
478 3584 : continue;
479 3584 : }
480 :
481 8646 : unsigned int abs_dist = abs (dist);
482 8646 : if (abs_dist >= 2 && abs_dist < *max_vf)
483 : {
484 : /* The dependence distance requires reduction of the maximal
485 : vectorization factor. */
486 431 : *max_vf = abs_dist;
487 431 : if (dump_enabled_p ())
488 32 : dump_printf_loc (MSG_NOTE, vect_location,
489 : "adjusting maximal vectorization factor to %i\n",
490 : *max_vf);
491 : }
492 :
493 8646 : if (abs_dist >= *max_vf)
494 : {
495 : /* Dependence distance does not create dependence, as far as
496 : vectorization is concerned, in this case. */
497 1274 : if (dump_enabled_p ())
498 737 : dump_printf_loc (MSG_NOTE, vect_location,
499 : "dependence distance >= VF.\n");
500 1274 : continue;
501 : }
502 :
503 7372 : return opt_result::failure_at (stmtinfo_a->stmt,
504 : "not vectorized, possible dependence "
505 : "between data-refs %T and %T\n",
506 : DR_REF (dra), DR_REF (drb));
507 : }
508 :
509 63542 : return opt_result::success ();
510 : }
511 :
512 : /* Function vect_analyze_early_break_dependences.
513 :
514 : Examine all the data references in the loop and make sure that if we have
515 : multiple exits that we are able to safely move stores such that they become
516 : safe for vectorization. The function also calculates the place where to move
517 : the instructions to and computes what the new vUSE chain should be.
518 :
519 : This works in tandem with the CFG that will be produced by
520 : slpeel_tree_duplicate_loop_to_edge_cfg later on.
521 :
522 : This function tries to validate whether an early break vectorization
523 : is possible for the current instruction sequence. Returns True i
524 : possible, otherwise False.
525 :
526 : Requirements:
527 : - Any memory access must be to a fixed size buffer.
528 : - There must not be any loads and stores to the same object.
529 : - Multiple loads are allowed as long as they don't alias.
530 :
531 : NOTE:
532 : This implementation is very conservative. Any overlapping loads/stores
533 : that take place before the early break statement gets rejected aside from
534 : WAR dependencies.
535 :
536 : i.e.:
537 :
538 : a[i] = 8
539 : c = a[i]
540 : if (b[i])
541 : ...
542 :
543 : is not allowed, but
544 :
545 : c = a[i]
546 : a[i] = 8
547 : if (b[i])
548 : ...
549 :
550 : is which is the common case. */
551 :
552 : static opt_result
553 151309 : vect_analyze_early_break_dependences (loop_vec_info loop_vinfo)
554 : {
555 151309 : DUMP_VECT_SCOPE ("vect_analyze_early_break_dependences");
556 :
557 : /* List of all load data references found during traversal. */
558 151309 : auto_vec<data_reference *> bases;
559 151309 : basic_block dest_bb = NULL;
560 :
561 151309 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
562 151309 : class loop *loop_nest = loop_outer (loop);
563 :
564 151309 : if (dump_enabled_p ())
565 1708 : dump_printf_loc (MSG_NOTE, vect_location,
566 : "loop contains multiple exits, analyzing"
567 : " statement dependencies.\n");
568 :
569 151309 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
570 26929 : if (dump_enabled_p ())
571 298 : dump_printf_loc (MSG_NOTE, vect_location,
572 : "alternate exit has been chosen as main exit.\n");
573 :
574 : /* Since we don't support general control flow, the location we'll move the
575 : side-effects to is always the latch connected exit. When we support
576 : general control flow we can do better but for now this is fine. Move
577 : side-effects to the in-loop destination of the last early exit. For the
578 : PEELED case we move the side-effects to the latch block as this is
579 : guaranteed to be the last block to be executed when a vector iteration
580 : finished. */
581 151309 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
582 26929 : dest_bb = loop->latch;
583 : else
584 124380 : dest_bb = single_pred (loop->latch);
585 :
586 : /* We start looking from dest_bb, for the non-PEELED case we don't want to
587 : move any stores already present, but we do want to read and validate the
588 : loads. */
589 151309 : basic_block bb = dest_bb;
590 :
591 : /* We move stores across all loads to the beginning of dest_bb, so
592 : the first block processed below doesn't need dependence checking. */
593 151309 : bool check_deps = false;
594 :
595 544885 : do
596 : {
597 348097 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
598 :
599 : /* Now analyze all the remaining statements and try to determine which
600 : instructions are allowed/needed to be moved. */
601 2608194 : while (!gsi_end_p (gsi))
602 : {
603 2266040 : gimple *stmt = gsi_stmt (gsi);
604 2266040 : gsi_prev (&gsi);
605 2266040 : if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
606 2006004 : continue;
607 :
608 1175314 : stmt_vec_info orig_stmt_vinfo = loop_vinfo->lookup_stmt (stmt);
609 1175314 : stmt_vec_info stmt_vinfo
610 1175314 : = vect_stmt_to_vectorize (orig_stmt_vinfo);
611 1175314 : auto dr_ref = STMT_VINFO_DATA_REF (stmt_vinfo);
612 1175314 : if (!dr_ref)
613 : {
614 : /* Trapping statements after the last early exit are fine. */
615 909165 : if (check_deps)
616 : {
617 545542 : bool could_trap_p = false;
618 545542 : gimple *cur_stmt = STMT_VINFO_STMT (stmt_vinfo);
619 545542 : could_trap_p = gimple_could_trap_p (cur_stmt);
620 545542 : if (STMT_VINFO_IN_PATTERN_P (orig_stmt_vinfo))
621 : {
622 204142 : gimple_stmt_iterator gsi2;
623 204142 : auto stmt_seq
624 204142 : = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_vinfo);
625 204142 : for (gsi2 = gsi_start (stmt_seq);
626 411309 : !could_trap_p && !gsi_end_p (gsi2); gsi_next (&gsi2))
627 : {
628 207167 : cur_stmt = gsi_stmt (gsi2);
629 207167 : could_trap_p = gimple_could_trap_p (cur_stmt);
630 : }
631 : }
632 :
633 545542 : if (could_trap_p)
634 : {
635 5563 : if (dump_enabled_p ())
636 153 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
637 : "cannot vectorize as operation may trap.\n");
638 5563 : return opt_result::failure_at (cur_stmt,
639 : "can't safely apply code motion to dependencies"
640 : " to vectorize the early exit. %G may trap.\n",
641 : cur_stmt);
642 : }
643 : }
644 :
645 903602 : continue;
646 903602 : }
647 :
648 : /* We know everything below dest_bb is safe since we know we
649 : had a full vector iteration when reaching it. Either by
650 : the loop entry / IV exit test being last or because this
651 : is the loop latch itself. */
652 266149 : if (!check_deps)
653 11676 : continue;
654 :
655 : /* Check if vector accesses to the object will be within bounds.
656 : must be a constant or assume loop will be versioned or niters
657 : bounded by VF so accesses are within range. We only need to check
658 : the reads since writes are moved to a safe place where if we get
659 : there we know they are safe to perform. */
660 254473 : if (DR_IS_READ (dr_ref))
661 : {
662 238991 : dr_set_safe_speculative_read_required (stmt_vinfo, true);
663 238991 : bool inbounds = ref_within_array_bound (stmt, DR_REF (dr_ref));
664 238991 : DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_vinfo)) = inbounds;
665 :
666 238991 : if (dump_enabled_p ())
667 2645 : dump_printf_loc (MSG_NOTE, vect_location,
668 : "marking DR (read) as possibly needing peeling "
669 : "for alignment at %G", stmt);
670 : }
671 :
672 254473 : if (DR_IS_READ (dr_ref))
673 238991 : bases.safe_push (dr_ref);
674 15482 : else if (DR_IS_WRITE (dr_ref))
675 : {
676 : /* We are moving writes down in the CFG. To be sure that this
677 : is valid after vectorization we have to check all the loads
678 : we are sinking the stores past to see if any of them may
679 : alias or are the same object.
680 :
681 : Same objects will not be an issue because unless the store
682 : is marked volatile the value can be forwarded. If the
683 : store is marked volatile we don't vectorize the loop
684 : anyway.
685 :
686 : That leaves the check for aliasing. We don't really need
687 : to care about the stores aliasing with each other since the
688 : stores are moved in order so the effects are still observed
689 : correctly. This leaves the check for WAR dependencies
690 : which we would be introducing here if the DR can alias.
691 : The check is quadratic in loads/stores but I have not found
692 : a better API to do this. I believe all loads and stores
693 : must be checked. We also must check them when we
694 : encountered the store, since we don't care about loads past
695 : the store. */
696 :
697 47987 : for (auto dr_read : bases)
698 15173 : if (dr_may_alias_p (dr_ref, dr_read, loop_nest))
699 : {
700 380 : if (dump_enabled_p ())
701 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
702 : vect_location,
703 : "early breaks not supported: "
704 : "overlapping loads and stores "
705 : "found before the break "
706 : "statement.\n");
707 :
708 380 : return opt_result::failure_at (stmt,
709 : "can't safely apply code motion to dependencies"
710 : " to vectorize the early exit. %G may alias with"
711 : " %G\n", stmt, dr_read->stmt);
712 : }
713 : }
714 :
715 508186 : if (gimple_vdef (stmt))
716 : {
717 15102 : if (dump_enabled_p ())
718 287 : dump_printf_loc (MSG_NOTE, vect_location,
719 : "==> recording stmt %G", stmt);
720 :
721 15102 : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (stmt);
722 : }
723 732075 : else if (gimple_vuse (stmt))
724 : {
725 238991 : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).safe_insert (0, stmt);
726 238991 : if (dump_enabled_p ())
727 2645 : dump_printf_loc (MSG_NOTE, vect_location,
728 : "marked statement for vUSE update: %G", stmt);
729 : }
730 : }
731 :
732 342154 : if (!single_pred_p (bb))
733 : {
734 145366 : gcc_assert (bb == loop->header);
735 145366 : break;
736 : }
737 :
738 : /* If we possibly sink through a virtual PHI make sure to elide that. */
739 196788 : if (gphi *vphi = get_virtual_phi (bb))
740 107 : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (vphi);
741 :
742 : /* All earlier blocks need dependence checking. */
743 196788 : check_deps = true;
744 196788 : bb = single_pred (bb);
745 196788 : }
746 : while (1);
747 :
748 : /* We don't allow outer -> inner loop transitions which should have been
749 : trapped already during loop form analysis. */
750 145366 : gcc_assert (dest_bb->loop_father == loop);
751 :
752 : /* Check that the destination block we picked has only one pred. To relax this we
753 : have to take special care when moving the statements. We don't currently support
754 : such control flow however this check is there to simplify how we handle
755 : labels that may be present anywhere in the IL. This check is to ensure that the
756 : labels aren't significant for the CFG. */
757 145366 : if (!single_pred (dest_bb))
758 0 : return opt_result::failure_at (vect_location,
759 : "chosen loop exit block (BB %d) does not have a "
760 : "single predecessor which is currently not "
761 : "supported for early break vectorization.\n",
762 : dest_bb->index);
763 :
764 145366 : LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo) = dest_bb;
765 : /* Check if loop has a side-effect (stores), force scalar epilogue. */
766 654281 : for (auto dr : LOOP_VINFO_DATAREFS (loop_vinfo))
767 245195 : if (DR_IS_WRITE (dr))
768 : {
769 13224 : LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
770 13224 : break;
771 : }
772 :
773 145366 : if (!LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).is_empty ())
774 : {
775 : /* All uses shall be updated to that of the first load. Entries are
776 : stored in reverse order. */
777 134569 : tree vuse = gimple_vuse (LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).last ());
778 372606 : for (auto g : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
779 : {
780 238037 : if (dump_enabled_p ())
781 2582 : dump_printf_loc (MSG_NOTE, vect_location,
782 : "will update use: %T, mem_ref: %G", vuse, g);
783 : }
784 : }
785 :
786 145366 : if (dump_enabled_p ())
787 1551 : dump_printf_loc (MSG_NOTE, vect_location,
788 : "recorded statements to be moved to BB %d\n",
789 1551 : LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo)->index);
790 :
791 145366 : return opt_result::success ();
792 151309 : }
793 :
794 : /* Function vect_analyze_data_ref_dependences.
795 :
796 : Examine all the data references in the loop, and make sure there do not
797 : exist any data dependences between them. Set *MAX_VF according to
798 : the maximum vectorization factor the data dependences allow. */
799 :
800 : opt_result
801 403047 : vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
802 : unsigned int *max_vf)
803 : {
804 403047 : unsigned int i;
805 403047 : struct data_dependence_relation *ddr;
806 :
807 403047 : DUMP_VECT_SCOPE ("vect_analyze_data_ref_dependences");
808 :
809 403047 : if (!LOOP_VINFO_DDRS (loop_vinfo).exists ())
810 : {
811 167293 : LOOP_VINFO_DDRS (loop_vinfo)
812 167293 : .create (LOOP_VINFO_DATAREFS (loop_vinfo).length ()
813 167293 : * LOOP_VINFO_DATAREFS (loop_vinfo).length ());
814 : /* We do not need read-read dependences. */
815 334586 : bool res = compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo),
816 : &LOOP_VINFO_DDRS (loop_vinfo),
817 167293 : LOOP_VINFO_LOOP_NEST (loop_vinfo),
818 : false);
819 167293 : gcc_assert (res);
820 : }
821 :
822 403047 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = true;
823 :
824 : /* For epilogues we either have no aliases or alias versioning
825 : was applied to original loop. Therefore we may just get max_vf
826 : using VF of original loop. */
827 403047 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
828 12917 : *max_vf = LOOP_VINFO_ORIG_MAX_VECT_FACTOR (loop_vinfo);
829 : else
830 1889964 : FOR_EACH_VEC_ELT (LOOP_VINFO_DDRS (loop_vinfo), i, ddr)
831 : {
832 1519191 : opt_result res
833 1519191 : = vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf);
834 1519191 : if (!res)
835 19357 : return res;
836 : }
837 :
838 : /* If we have early break statements in the loop, check to see if they
839 : are of a form we can vectorizer. */
840 383690 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
841 151309 : return vect_analyze_early_break_dependences (loop_vinfo);
842 :
843 232381 : return opt_result::success ();
844 : }
845 :
846 :
847 : /* Function vect_slp_analyze_data_ref_dependence.
848 :
849 : Classify the dependence between the memory-references DRA and DRB of DDR
850 : for VINFO using the classical (affine) data-dependence test. Return
851 : chrec_known if they are provably independent, chrec_dont_know if the test
852 : cannot analyze them (in which case the caller can still try to disambiguate
853 : them with the alias oracle), and the dependence (NULL_TREE) otherwise. */
854 :
855 : static tree
856 6952344 : vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
857 : struct data_dependence_relation *ddr)
858 : {
859 6952344 : struct data_reference *dra = DDR_A (ddr);
860 6952344 : struct data_reference *drb = DDR_B (ddr);
861 6952344 : dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
862 6952344 : dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
863 :
864 : /* We need to check dependences of statements marked as unvectorizable
865 : as well, they still can prohibit vectorization. */
866 :
867 : /* Independent data accesses. */
868 6952344 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
869 : return chrec_known;
870 :
871 1131637 : if (dra == drb)
872 : return chrec_known;
873 :
874 : /* Read-read is OK. */
875 9338 : if (DR_IS_READ (dra) && DR_IS_READ (drb))
876 : return chrec_known;
877 :
878 : /* If dra and drb are part of the same interleaving chain consider
879 : them independent. */
880 9338 : if (STMT_VINFO_GROUPED_ACCESS (dr_info_a->stmt)
881 9338 : && (DR_GROUP_FIRST_ELEMENT (dr_info_a->stmt)
882 9338 : == DR_GROUP_FIRST_ELEMENT (dr_info_b->stmt)))
883 : return chrec_known;
884 :
885 : /* Unknown data dependence. */
886 9338 : if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
887 : {
888 9338 : if (dump_enabled_p ())
889 60 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
890 : "can't determine dependence between %T and %T\n",
891 : DR_REF (dra), DR_REF (drb));
892 : }
893 0 : else if (dump_enabled_p ())
894 0 : dump_printf_loc (MSG_NOTE, vect_location,
895 : "determined dependence between %T and %T\n",
896 : DR_REF (dra), DR_REF (drb));
897 :
898 9338 : return DDR_ARE_DEPENDENT (ddr);
899 : }
900 :
901 :
902 : /* Analyze dependences involved in the transform of a store SLP NODE. */
903 :
904 : static bool
905 677623 : vect_slp_analyze_store_dependences (vec_info *vinfo, slp_tree node)
906 : {
907 : /* This walks over all stmts involved in the SLP store done
908 : in NODE verifying we can sink them up to the last stmt in the
909 : group. */
910 677623 : stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
911 677623 : gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info)));
912 :
913 3815830 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
914 : {
915 1791910 : stmt_vec_info access_info
916 1791910 : = vect_orig_stmt (stmt_vinfo);
917 1791910 : if (access_info == last_access_info)
918 669545 : continue;
919 1122365 : data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
920 1122365 : ao_ref ref;
921 1122365 : bool ref_initialized_p = false;
922 1122365 : for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
923 10743859 : gsi_stmt (gsi) != last_access_info->stmt; gsi_next (&gsi))
924 : {
925 9630443 : gimple *stmt = gsi_stmt (gsi);
926 17049686 : if (! gimple_vuse (stmt))
927 2677768 : continue;
928 :
929 : /* If we couldn't record a (single) data reference for this stmt,
930 : or the classical dependence test cannot analyze it, we have to
931 : resort to the alias oracle. */
932 6952675 : stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
933 6952675 : data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
934 6952675 : if (dr_b)
935 : {
936 6952064 : gcc_assert (!gimple_visited_p (stmt));
937 :
938 6952064 : ddr_p ddr = initialize_data_dependence_relation (dr_a,
939 6952064 : dr_b, vNULL);
940 6952064 : tree dep = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
941 6952064 : free_dependence_relation (ddr);
942 6952064 : if (dep == chrec_known)
943 6942762 : continue;
944 9302 : if (dep != chrec_dont_know)
945 8949 : return false;
946 : /* Unknown dependence - fall through to the alias oracle. */
947 : }
948 :
949 : /* We are moving a store - this means we cannot use TBAA for
950 : disambiguation. */
951 9913 : if (!ref_initialized_p)
952 : {
953 9628 : ao_ref_init (&ref, DR_REF (dr_a));
954 9628 : ref_initialized_p = true;
955 : }
956 9913 : if (stmt_may_clobber_ref_p_1 (stmt, &ref, false)
957 9913 : || ref_maybe_used_by_stmt_p (stmt, &ref, false))
958 : return false;
959 : }
960 : }
961 : return true;
962 : }
963 :
964 : /* Analyze dependences involved in the transform of a load SLP NODE. STORES
965 : contain the vector of scalar stores of this instance if we are
966 : disambiguating the loads. */
967 :
968 : static bool
969 177436 : vect_slp_analyze_load_dependences (vec_info *vinfo, slp_tree node,
970 : vec<stmt_vec_info> stores,
971 : stmt_vec_info last_store_info)
972 : {
973 : /* This walks over all stmts involved in the SLP load done
974 : in NODE verifying we can hoist them up to the first stmt in the
975 : group. */
976 177436 : stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node);
977 177436 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info)));
978 :
979 976200 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
980 : {
981 443928 : if (! stmt_vinfo)
982 187174 : continue;
983 443928 : stmt_vec_info access_info
984 443928 : = vect_orig_stmt (stmt_vinfo);
985 443928 : if (access_info == first_access_info)
986 187174 : continue;
987 256754 : data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
988 256754 : ao_ref ref;
989 256754 : bool ref_initialized_p = false;
990 256754 : hash_set<stmt_vec_info> grp_visited;
991 256754 : for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
992 3340937 : gsi_stmt (gsi) != first_access_info->stmt; gsi_prev (&gsi))
993 : {
994 3084219 : gimple *stmt = gsi_stmt (gsi);
995 5145349 : if (! gimple_vdef (stmt))
996 3026414 : continue;
997 :
998 282611 : stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
999 :
1000 : /* If we run into a store of this same instance (we've just
1001 : marked those) then delay dependence checking until we run
1002 : into the last store because this is where it will have
1003 : been sunk to (and we verified that we can do that already). */
1004 282611 : if (gimple_visited_p (stmt))
1005 : {
1006 224806 : if (stmt_info != last_store_info)
1007 224804 : continue;
1008 :
1009 10 : for (stmt_vec_info &store_info : stores)
1010 : {
1011 4 : data_reference *store_dr = STMT_VINFO_DATA_REF (store_info);
1012 4 : ddr_p ddr = initialize_data_dependence_relation
1013 4 : (dr_a, store_dr, vNULL);
1014 4 : tree dep
1015 4 : = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
1016 4 : free_dependence_relation (ddr);
1017 4 : if (dep == chrec_known)
1018 4 : continue;
1019 0 : if (dep != chrec_dont_know)
1020 36 : return false;
1021 : /* The classical dependence test cannot analyze this;
1022 : resort to the alias oracle. We are hoisting a load
1023 : so TBAA may be used for disambiguation. */
1024 0 : if (!ref_initialized_p)
1025 : {
1026 0 : ao_ref_init (&ref, DR_REF (dr_a));
1027 0 : ref_initialized_p = true;
1028 : }
1029 0 : if (stmt_may_clobber_ref_p_1 (store_info->stmt, &ref, true))
1030 : return false;
1031 : }
1032 2 : continue;
1033 2 : }
1034 :
1035 119643 : auto check_hoist = [&] (stmt_vec_info stmt_info) -> bool
1036 : {
1037 : /* We are hoisting a load - this means we can use TBAA for
1038 : disambiguation. */
1039 61838 : if (!ref_initialized_p)
1040 : {
1041 11638 : ao_ref_init (&ref, DR_REF (dr_a));
1042 11638 : ref_initialized_p = true;
1043 : }
1044 61838 : if (stmt_may_clobber_ref_p_1 (stmt_info->stmt, &ref, true))
1045 : {
1046 : /* If we couldn't record a (single) data reference for this
1047 : stmt we have to give up now. */
1048 276 : data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
1049 276 : if (!dr_b)
1050 : return false;
1051 276 : ddr_p ddr = initialize_data_dependence_relation (dr_a,
1052 276 : dr_b, vNULL);
1053 276 : tree dep
1054 276 : = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
1055 276 : free_dependence_relation (ddr);
1056 : /* The alias oracle above could not rule out a conflict;
1057 : only a proven-independent (chrec_known) result lets us
1058 : hoist the load past this store. */
1059 276 : if (dep != chrec_known)
1060 36 : return false;
1061 : }
1062 : /* No dependence. */
1063 : return true;
1064 57805 : };
1065 57805 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1066 : {
1067 : /* When we run into a store group we have to honor
1068 : that earlier stores might be moved here. We don't
1069 : know exactly which and where to since we lack a
1070 : back-mapping from DR to SLP node, so assume all
1071 : earlier stores are sunk here. It's enough to
1072 : consider the last stmt of a group for this.
1073 : ??? Both this and the fact that we disregard that
1074 : the conflicting instance might be removed later
1075 : is overly conservative. */
1076 57174 : if (!grp_visited.add (DR_GROUP_FIRST_ELEMENT (stmt_info)))
1077 11560 : for (auto store_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
1078 137139 : store_info != NULL;
1079 125579 : store_info = DR_GROUP_NEXT_ELEMENT (store_info))
1080 125615 : if ((store_info == stmt_info
1081 114064 : || get_later_stmt (store_info, stmt_info) == stmt_info)
1082 175271 : && !check_hoist (store_info))
1083 : return false;
1084 : }
1085 : else
1086 : {
1087 631 : if (!check_hoist (stmt_info))
1088 : return false;
1089 : }
1090 : }
1091 256754 : }
1092 : return true;
1093 : }
1094 :
1095 :
1096 : /* Function vect_analyze_data_ref_dependences.
1097 :
1098 : Examine all the data references in the basic-block, and make sure there
1099 : do not exist any data dependences between them. Set *MAX_VF according to
1100 : the maximum vectorization factor the data dependences allow. */
1101 :
1102 : bool
1103 834421 : vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
1104 : {
1105 834421 : DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
1106 :
1107 : /* The stores of this instance are at the root of the SLP tree. */
1108 834421 : slp_tree store = NULL;
1109 834421 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store)
1110 677623 : store = SLP_INSTANCE_TREE (instance);
1111 :
1112 : /* Verify we can sink stores to the vectorized stmt insert location. */
1113 677623 : stmt_vec_info last_store_info = NULL;
1114 677623 : if (store)
1115 : {
1116 677623 : if (! vect_slp_analyze_store_dependences (vinfo, store))
1117 : return false;
1118 :
1119 : /* Mark stores in this instance and remember the last one. */
1120 668674 : last_store_info = vect_find_last_scalar_stmt_in_slp (store);
1121 3788082 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
1122 1782060 : gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), true);
1123 : }
1124 :
1125 825472 : bool res = true;
1126 :
1127 : /* Verify we can sink loads to the vectorized stmt insert location,
1128 : special-casing stores of this instance. */
1129 1278216 : for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
1130 177436 : if (! vect_slp_analyze_load_dependences (vinfo, load,
1131 : store
1132 : ? SLP_TREE_SCALAR_STMTS (store)
1133 : : vNULL, last_store_info))
1134 : {
1135 : res = false;
1136 : break;
1137 : }
1138 :
1139 : /* Unset the visited flag. */
1140 825472 : if (store)
1141 3788082 : for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
1142 1782060 : gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), false);
1143 :
1144 : /* If this is a SLP instance with a store check if there's a dependent
1145 : load that cannot be forwarded from a previous iteration of a loop
1146 : both are in. This is to avoid situations like that in PR115777. */
1147 825472 : if (res && store)
1148 : {
1149 668650 : stmt_vec_info store_info
1150 668650 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (store)[0]);
1151 668650 : class loop *store_loop = gimple_bb (store_info->stmt)->loop_father;
1152 668650 : if (! loop_outer (store_loop))
1153 569253 : return res;
1154 99397 : vec<loop_p> loop_nest;
1155 99397 : loop_nest.create (1);
1156 99397 : loop_nest.quick_push (store_loop);
1157 99397 : data_reference *drs = nullptr;
1158 183562 : for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
1159 : {
1160 37133 : if (! STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (load)[0]))
1161 0 : continue;
1162 37133 : stmt_vec_info load_info
1163 37133 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (load)[0]);
1164 37133 : if (gimple_bb (load_info->stmt)->loop_father != store_loop)
1165 5039 : continue;
1166 :
1167 : /* For now concern ourselves with write-after-read as we also
1168 : only look for re-use of the store within the same SLP instance.
1169 : We can still get a RAW here when the instance contains a PHI
1170 : with a backedge though, thus this test. */
1171 32094 : if (! vect_stmt_dominates_stmt_p (STMT_VINFO_STMT (load_info),
1172 : STMT_VINFO_STMT (store_info)))
1173 11647 : continue;
1174 :
1175 20447 : if (! drs)
1176 : {
1177 19583 : drs = create_data_ref (loop_preheader_edge (store_loop),
1178 : store_loop,
1179 19583 : DR_REF (STMT_VINFO_DATA_REF (store_info)),
1180 : store_info->stmt, false, false);
1181 19583 : if (! DR_BASE_ADDRESS (drs)
1182 16434 : || TREE_CODE (DR_STEP (drs)) != INTEGER_CST)
1183 : break;
1184 : }
1185 17019 : data_reference *drl
1186 17019 : = create_data_ref (loop_preheader_edge (store_loop),
1187 : store_loop,
1188 17019 : DR_REF (STMT_VINFO_DATA_REF (load_info)),
1189 : load_info->stmt, true, false);
1190 :
1191 : /* See whether the DRs have a known constant distance throughout
1192 : the containing loop iteration. */
1193 32334 : if (! DR_BASE_ADDRESS (drl)
1194 14952 : || ! operand_equal_p (DR_STEP (drs), DR_STEP (drl))
1195 8937 : || ! operand_equal_p (DR_BASE_ADDRESS (drs),
1196 8937 : DR_BASE_ADDRESS (drl))
1197 18733 : || ! operand_equal_p (DR_OFFSET (drs), DR_OFFSET (drl)))
1198 : {
1199 15315 : free_data_ref (drl);
1200 15315 : continue;
1201 : }
1202 :
1203 : /* If the next iteration load overlaps with a non-power-of-two offset
1204 : we are surely failing any STLF attempt. */
1205 1704 : HOST_WIDE_INT step = TREE_INT_CST_LOW (DR_STEP (drl));
1206 1704 : unsigned HOST_WIDE_INT sizes
1207 1704 : = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drs))))
1208 1704 : * DR_GROUP_SIZE (store_info));
1209 1704 : unsigned HOST_WIDE_INT sizel
1210 1704 : = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drl))))
1211 1704 : * DR_GROUP_SIZE (load_info));
1212 1704 : if (ranges_overlap_p (TREE_INT_CST_LOW (DR_INIT (drl)) + step, sizel,
1213 1704 : TREE_INT_CST_LOW (DR_INIT (drs)), sizes))
1214 : {
1215 858 : unsigned HOST_WIDE_INT dist
1216 858 : = absu_hwi (TREE_INT_CST_LOW (DR_INIT (drl)) + step
1217 858 : - TREE_INT_CST_LOW (DR_INIT (drs)));
1218 858 : poly_uint64 loadsz = tree_to_poly_uint64
1219 858 : (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (load)));
1220 858 : poly_uint64 storesz = tree_to_poly_uint64
1221 858 : (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (store)));
1222 : /* When the overlap aligns with vector sizes used for the loads
1223 : and the vector stores are larger or equal to the loads
1224 : forwarding should work. */
1225 1716 : if (maybe_gt (loadsz, storesz) || ! multiple_p (dist, loadsz))
1226 70 : load->avoid_stlf_fail = true;
1227 : }
1228 1704 : free_data_ref (drl);
1229 : }
1230 99397 : if (drs)
1231 19583 : free_data_ref (drs);
1232 99397 : loop_nest.release ();
1233 : }
1234 :
1235 : return res;
1236 : }
1237 :
1238 : /* Return the misalignment of DR_INFO accessed in VECTYPE with OFFSET
1239 : applied. */
1240 :
1241 : int
1242 6937378 : dr_misalignment (dr_vec_info *dr_info, tree vectype, poly_int64 offset)
1243 : {
1244 6937378 : HOST_WIDE_INT diff = 0;
1245 : /* Alignment is only analyzed for the first element of a DR group,
1246 : use that but adjust misalignment by the offset of the access. */
1247 6937378 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
1248 : {
1249 2350823 : dr_vec_info *first_dr
1250 2350823 : = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
1251 : /* vect_analyze_data_ref_accesses guarantees that DR_INIT are
1252 : INTEGER_CSTs and the first element in the group has the lowest
1253 : address. */
1254 2350823 : diff = (TREE_INT_CST_LOW (DR_INIT (dr_info->dr))
1255 2350823 : - TREE_INT_CST_LOW (DR_INIT (first_dr->dr)));
1256 2350823 : gcc_assert (diff >= 0);
1257 : dr_info = first_dr;
1258 : }
1259 :
1260 6937378 : int misalign = dr_info->misalignment;
1261 6937378 : gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED);
1262 6937378 : if (misalign == DR_MISALIGNMENT_UNKNOWN)
1263 : return misalign;
1264 :
1265 : /* If the access is only aligned for a vector type with smaller alignment
1266 : requirement the access has unknown misalignment. */
1267 4214682 : if (maybe_lt (dr_info->target_alignment * BITS_PER_UNIT,
1268 4214682 : targetm.vectorize.preferred_vector_alignment (vectype)))
1269 : return DR_MISALIGNMENT_UNKNOWN;
1270 :
1271 : /* Apply the offset from the DR group start and the externally supplied
1272 : offset which can for example result from a negative stride access. */
1273 4214673 : poly_int64 misalignment = misalign + diff + offset;
1274 :
1275 : /* Below we reject compile-time non-constant target alignments, but if
1276 : our misalignment is zero, then we are known to already be aligned
1277 : w.r.t. any such possible target alignment. */
1278 4214673 : if (known_eq (misalignment, 0))
1279 : return 0;
1280 :
1281 657005 : unsigned HOST_WIDE_INT target_alignment_c;
1282 657005 : if (!dr_info->target_alignment.is_constant (&target_alignment_c)
1283 657005 : || !known_misalignment (misalignment, target_alignment_c, &misalign))
1284 : return DR_MISALIGNMENT_UNKNOWN;
1285 657005 : return misalign;
1286 : }
1287 :
1288 : /* Record the base alignment guarantee given by DRB, which occurs
1289 : in STMT_INFO. */
1290 :
1291 : static void
1292 4933562 : vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
1293 : innermost_loop_behavior *drb)
1294 : {
1295 4933562 : bool existed;
1296 4933562 : std::pair<stmt_vec_info, innermost_loop_behavior *> &entry
1297 4933562 : = vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
1298 4933562 : if (!existed || entry.second->base_alignment < drb->base_alignment)
1299 : {
1300 1494474 : entry = std::make_pair (stmt_info, drb);
1301 1494474 : if (dump_enabled_p ())
1302 34500 : dump_printf_loc (MSG_NOTE, vect_location,
1303 : "recording new base alignment for %T\n"
1304 : " alignment: %d\n"
1305 : " misalignment: %d\n"
1306 : " based on: %G",
1307 : drb->base_address,
1308 : drb->base_alignment,
1309 : drb->base_misalignment,
1310 : stmt_info->stmt);
1311 : }
1312 4933562 : }
1313 :
1314 : /* If the region we're going to vectorize is reached, all unconditional
1315 : data references occur at least once. We can therefore pool the base
1316 : alignment guarantees from each unconditional reference. Do this by
1317 : going through all the data references in VINFO and checking whether
1318 : the containing statement makes the reference unconditionally. If so,
1319 : record the alignment of the base address in VINFO so that it can be
1320 : used for all other references with the same base. */
1321 :
1322 : void
1323 1117229 : vect_record_base_alignments (vec_info *vinfo)
1324 : {
1325 1117229 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
1326 438341 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
1327 16411880 : for (data_reference *dr : vinfo->shared->datarefs)
1328 : {
1329 13166495 : dr_vec_info *dr_info = vinfo->lookup_dr (dr);
1330 13166495 : stmt_vec_info stmt_info = dr_info->stmt;
1331 13166495 : if (!DR_IS_CONDITIONAL_IN_STMT (dr)
1332 13157492 : && STMT_VINFO_VECTORIZABLE (stmt_info)
1333 4952592 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1334 : {
1335 4932008 : vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
1336 :
1337 : /* If DR is nested in the loop that is being vectorized, we can also
1338 : record the alignment of the base wrt the outer loop. */
1339 14137573 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1340 1554 : vect_record_base_alignment
1341 1554 : (vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
1342 : }
1343 : }
1344 1117229 : }
1345 :
1346 : /* Function vect_compute_data_ref_alignment
1347 :
1348 : Compute the misalignment of the data reference DR_INFO when vectorizing
1349 : with VECTYPE.
1350 :
1351 : Output:
1352 : 1. initialized misalignment info for DR_INFO
1353 :
1354 : FOR NOW: No analysis is actually performed. Misalignment is calculated
1355 : only for trivial cases. TODO. */
1356 :
1357 : static void
1358 1671851 : vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
1359 : tree vectype)
1360 : {
1361 1671851 : stmt_vec_info stmt_info = dr_info->stmt;
1362 1671851 : vec_base_alignments *base_alignments = &vinfo->base_alignments;
1363 1671851 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
1364 1671851 : class loop *loop = NULL;
1365 1671851 : tree ref = DR_REF (dr_info->dr);
1366 :
1367 1671851 : if (dump_enabled_p ())
1368 54473 : dump_printf_loc (MSG_NOTE, vect_location,
1369 : "vect_compute_data_ref_alignment:\n");
1370 :
1371 1671851 : if (loop_vinfo)
1372 866447 : loop = LOOP_VINFO_LOOP (loop_vinfo);
1373 :
1374 : /* Initialize misalignment to unknown. */
1375 1671851 : SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
1376 :
1377 1671851 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1378 : return;
1379 :
1380 1650169 : innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
1381 1650169 : bool step_preserves_misalignment_p;
1382 :
1383 1650169 : poly_uint64 vector_alignment
1384 1650169 : = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
1385 : BITS_PER_UNIT);
1386 :
1387 1650169 : if (loop_vinfo
1388 1650169 : && dr_safe_speculative_read_required (stmt_info))
1389 : {
1390 : /* The required target alignment must be a power-of-2 value and is
1391 : computed as the product of vector element size, VF and group size.
1392 : We compute the constant part first as VF may be a variable. For
1393 : variable VF, the power-of-2 check of VF is deferred to runtime. */
1394 328643 : auto align_factor_c
1395 328643 : = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
1396 328643 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1397 90520 : align_factor_c *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
1398 :
1399 328643 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1400 328643 : poly_uint64 new_alignment = vf * align_factor_c;
1401 :
1402 657286 : if ((vf.is_constant () && pow2p_hwi (new_alignment.to_constant ()))
1403 : || (!vf.is_constant () && pow2p_hwi (align_factor_c)))
1404 : {
1405 265232 : if (dump_enabled_p ())
1406 : {
1407 3329 : dump_printf_loc (MSG_NOTE, vect_location,
1408 : "alignment increased due to early break to ");
1409 3329 : dump_dec (MSG_NOTE, new_alignment);
1410 3329 : dump_printf (MSG_NOTE, " bytes.\n");
1411 : }
1412 265232 : vector_alignment = new_alignment;
1413 : }
1414 : }
1415 :
1416 1650169 : SET_DR_TARGET_ALIGNMENT (dr_info, vector_alignment);
1417 :
1418 : /* If the main loop has peeled for alignment we have no way of knowing
1419 : whether the data accesses in the epilogues are aligned. We can't at
1420 : compile time answer the question whether we have entered the main loop or
1421 : not. Fixes PR 92351. */
1422 1650169 : if (loop_vinfo)
1423 : {
1424 844765 : loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
1425 844765 : if (orig_loop_vinfo
1426 33685 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo) != 0)
1427 : return;
1428 : }
1429 :
1430 1649952 : unsigned HOST_WIDE_INT vect_align_c;
1431 1649952 : if (!vector_alignment.is_constant (&vect_align_c))
1432 : return;
1433 :
1434 : /* No step for BB vectorization. */
1435 1649952 : if (!loop)
1436 : {
1437 805404 : gcc_assert (integer_zerop (drb->step));
1438 : step_preserves_misalignment_p = true;
1439 : }
1440 :
1441 : else
1442 : {
1443 : /* We can only use base and misalignment information relative to
1444 : an innermost loop if the misalignment stays the same throughout the
1445 : execution of the loop. As above, this is the case if the stride of
1446 : the dataref evenly divides by the alignment. Make sure to check
1447 : previous epilogues and the main loop. */
1448 : step_preserves_misalignment_p = true;
1449 : auto lvinfo = loop_vinfo;
1450 1723418 : while (lvinfo)
1451 : {
1452 878870 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (lvinfo);
1453 878870 : step_preserves_misalignment_p
1454 878870 : &= multiple_p (drb->step_alignment * vf, vect_align_c);
1455 878870 : lvinfo = LOOP_VINFO_ORIG_LOOP_INFO (lvinfo);
1456 : }
1457 :
1458 844548 : if (!step_preserves_misalignment_p && dump_enabled_p ())
1459 328 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1460 : "step doesn't divide the vector alignment.\n");
1461 :
1462 : /* In case the dataref is in an inner-loop of the loop that is being
1463 : vectorized (LOOP), we use the base and misalignment information
1464 : relative to the outer-loop (LOOP). This is ok only if the
1465 : misalignment stays the same throughout the execution of the
1466 : inner-loop, which is why we have to check that the stride of the
1467 : dataref in the inner-loop evenly divides by the vector alignment. */
1468 844548 : if (step_preserves_misalignment_p
1469 844548 : && nested_in_vect_loop_p (loop, stmt_info))
1470 : {
1471 1553 : step_preserves_misalignment_p
1472 1553 : = (DR_STEP_ALIGNMENT (dr_info->dr) % vect_align_c) == 0;
1473 :
1474 1553 : if (dump_enabled_p ())
1475 : {
1476 499 : if (step_preserves_misalignment_p)
1477 358 : dump_printf_loc (MSG_NOTE, vect_location,
1478 : "inner step divides the vector alignment.\n");
1479 : else
1480 141 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1481 : "inner step doesn't divide the vector"
1482 : " alignment.\n");
1483 : }
1484 : }
1485 : }
1486 :
1487 1649952 : unsigned int base_alignment = drb->base_alignment;
1488 1649952 : unsigned int base_misalignment = drb->base_misalignment;
1489 :
1490 : /* Calculate the maximum of the pooled base address alignment and the
1491 : alignment that we can compute for DR itself. */
1492 1649952 : std::pair<stmt_vec_info, innermost_loop_behavior *> *entry
1493 1649952 : = base_alignments->get (drb->base_address);
1494 1649952 : if (entry
1495 1645347 : && base_alignment < (*entry).second->base_alignment
1496 1653541 : && (loop_vinfo
1497 2679 : || (dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt_info->stmt),
1498 2679 : gimple_bb (entry->first->stmt))
1499 2565 : && (gimple_bb (stmt_info->stmt) != gimple_bb (entry->first->stmt)
1500 2280 : || (entry->first->dr_aux.group <= dr_info->group)))))
1501 : {
1502 3458 : base_alignment = entry->second->base_alignment;
1503 3458 : base_misalignment = entry->second->base_misalignment;
1504 : }
1505 :
1506 1649952 : if (drb->offset_alignment < vect_align_c
1507 1581978 : || !step_preserves_misalignment_p
1508 : /* We need to know whether the step wrt the vectorized loop is
1509 : negative when computing the starting misalignment below. */
1510 1574229 : || TREE_CODE (drb->step) != INTEGER_CST)
1511 : {
1512 104744 : if (dump_enabled_p ())
1513 3636 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1514 : "Unknown alignment for access: %T\n", ref);
1515 : return;
1516 : }
1517 :
1518 1545208 : if (base_alignment < vect_align_c)
1519 : {
1520 775602 : unsigned int max_alignment;
1521 775602 : tree base = get_base_for_alignment (drb->base_address, &max_alignment);
1522 775602 : if (max_alignment < vect_align_c
1523 772958 : || (loop_vinfo && LOOP_VINFO_EPILOGUE_P (loop_vinfo))
1524 1548560 : || !vect_can_force_dr_alignment_p (base,
1525 751466 : vect_align_c * BITS_PER_UNIT))
1526 : {
1527 564103 : if (dump_enabled_p ())
1528 15663 : dump_printf_loc (MSG_NOTE, vect_location,
1529 : "can't force alignment of ref: %T\n", ref);
1530 564103 : return;
1531 : }
1532 :
1533 : /* Force the alignment of the decl.
1534 : NOTE: This is the only change to the code we make during
1535 : the analysis phase, before deciding to vectorize the loop. */
1536 211499 : if (dump_enabled_p ())
1537 8327 : dump_printf_loc (MSG_NOTE, vect_location,
1538 : "force alignment of %T\n", ref);
1539 :
1540 211499 : dr_info->base_decl = base;
1541 211499 : dr_info->base_misaligned = true;
1542 211499 : base_misalignment = 0;
1543 : }
1544 981105 : poly_int64 misalignment
1545 981105 : = base_misalignment + wi::to_poly_offset (drb->init).force_shwi ();
1546 :
1547 981105 : unsigned int const_misalignment;
1548 981105 : if (!known_misalignment (misalignment, vect_align_c, &const_misalignment))
1549 : {
1550 : if (dump_enabled_p ())
1551 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1552 : "Non-constant misalignment for access: %T\n", ref);
1553 : return;
1554 : }
1555 :
1556 981105 : SET_DR_MISALIGNMENT (dr_info, const_misalignment);
1557 :
1558 981105 : if (dump_enabled_p ())
1559 33841 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1560 : "misalign = %d bytes of ref %T\n",
1561 : const_misalignment, ref);
1562 :
1563 : return;
1564 : }
1565 :
1566 : /* Return whether DR_INFO, which is related to DR_PEEL_INFO in
1567 : that it only differs in DR_INIT, is aligned if DR_PEEL_INFO
1568 : is made aligned via peeling. */
1569 :
1570 : static bool
1571 2040882 : vect_dr_aligned_if_related_peeled_dr_is (dr_vec_info *dr_info,
1572 : dr_vec_info *dr_peel_info)
1573 : {
1574 2040882 : if (multiple_p (DR_TARGET_ALIGNMENT (dr_peel_info),
1575 2041504 : DR_TARGET_ALIGNMENT (dr_info)))
1576 : {
1577 2040260 : poly_offset_int diff
1578 2040260 : = (wi::to_poly_offset (DR_INIT (dr_peel_info->dr))
1579 2040260 : - wi::to_poly_offset (DR_INIT (dr_info->dr)));
1580 2040260 : if (known_eq (diff, 0)
1581 2040260 : || multiple_p (diff, DR_TARGET_ALIGNMENT (dr_info)))
1582 771979 : return true;
1583 : }
1584 : return false;
1585 : }
1586 :
1587 : /* Return whether DR_INFO is aligned if DR_PEEL_INFO is made
1588 : aligned via peeling. */
1589 :
1590 : static bool
1591 207438 : vect_dr_aligned_if_peeled_dr_is (dr_vec_info *dr_info,
1592 : dr_vec_info *dr_peel_info)
1593 : {
1594 207438 : if (!operand_equal_p (DR_BASE_ADDRESS (dr_info->dr),
1595 207438 : DR_BASE_ADDRESS (dr_peel_info->dr), 0)
1596 53080 : || !operand_equal_p (DR_OFFSET (dr_info->dr),
1597 53080 : DR_OFFSET (dr_peel_info->dr), 0)
1598 259616 : || !operand_equal_p (DR_STEP (dr_info->dr),
1599 52178 : DR_STEP (dr_peel_info->dr), 0))
1600 : return false;
1601 :
1602 51780 : return vect_dr_aligned_if_related_peeled_dr_is (dr_info, dr_peel_info);
1603 : }
1604 :
1605 : /* Compute the value for dr_info->misalign so that the access appears
1606 : aligned. This is used by peeling to compensate for dr_misalignment
1607 : applying the offset for negative step. */
1608 :
1609 : int
1610 22530 : vect_dr_misalign_for_aligned_access (dr_vec_info *dr_info)
1611 : {
1612 22530 : if (tree_int_cst_sgn (DR_STEP (dr_info->dr)) >= 0)
1613 : return 0;
1614 :
1615 201 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
1616 201 : poly_int64 misalignment
1617 201 : = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
1618 201 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1619 :
1620 201 : unsigned HOST_WIDE_INT target_alignment_c;
1621 201 : int misalign;
1622 201 : if (!dr_info->target_alignment.is_constant (&target_alignment_c)
1623 201 : || !known_misalignment (misalignment, target_alignment_c, &misalign))
1624 : return DR_MISALIGNMENT_UNKNOWN;
1625 201 : return misalign;
1626 : }
1627 :
1628 : /* Function vect_update_misalignment_for_peel.
1629 : Sets DR_INFO's misalignment
1630 : - to 0 if it has the same alignment as DR_PEEL_INFO,
1631 : - to the misalignment computed using NPEEL if DR_INFO's salignment is known,
1632 : - to -1 (unknown) otherwise.
1633 :
1634 : DR_INFO - the data reference whose misalignment is to be adjusted.
1635 : DR_PEEL_INFO - the data reference whose misalignment is being made
1636 : zero in the vector loop by the peel.
1637 : NPEEL - the number of iterations in the peel loop if the misalignment
1638 : of DR_PEEL_INFO is known at compile time. */
1639 :
1640 : static void
1641 2799 : vect_update_misalignment_for_peel (dr_vec_info *dr_info,
1642 : dr_vec_info *dr_peel_info, int npeel)
1643 : {
1644 : /* If dr_info is aligned of dr_peel_info is, then mark it so. */
1645 2799 : if (vect_dr_aligned_if_peeled_dr_is (dr_info, dr_peel_info))
1646 : {
1647 448 : SET_DR_MISALIGNMENT (dr_info,
1648 : vect_dr_misalign_for_aligned_access (dr_peel_info));
1649 448 : return;
1650 : }
1651 :
1652 2351 : unsigned HOST_WIDE_INT alignment;
1653 2351 : if (DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)
1654 2351 : && known_alignment_for_access_p (dr_info,
1655 2351 : STMT_VINFO_VECTYPE (dr_info->stmt))
1656 222 : && known_alignment_for_access_p (dr_peel_info,
1657 222 : STMT_VINFO_VECTYPE (dr_peel_info->stmt)))
1658 : {
1659 202 : int misal = dr_info->misalignment;
1660 202 : misal += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
1661 202 : misal &= alignment - 1;
1662 202 : set_dr_misalignment (dr_info, misal);
1663 202 : return;
1664 : }
1665 :
1666 2149 : if (dump_enabled_p ())
1667 40 : dump_printf_loc (MSG_NOTE, vect_location, "Setting misalignment " \
1668 : "to unknown (-1).\n");
1669 2149 : SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
1670 : }
1671 :
1672 : /* Return true if alignment is relevant for DR_INFO. */
1673 :
1674 : static bool
1675 2266760 : vect_relevant_for_alignment_p (dr_vec_info *dr_info)
1676 : {
1677 2266760 : stmt_vec_info stmt_info = dr_info->stmt;
1678 :
1679 2266760 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
1680 : return false;
1681 :
1682 : /* For interleaving, only the alignment of the first access matters. */
1683 2265308 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1684 2614860 : && DR_GROUP_FIRST_ELEMENT (stmt_info) != stmt_info)
1685 : return false;
1686 :
1687 : /* Scatter-gather and invariant accesses continue to address individual
1688 : scalars, so vector-level alignment is irrelevant. */
1689 2143728 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1690 2143728 : || integer_zerop (DR_STEP (dr_info->dr)))
1691 : return false;
1692 :
1693 : /* Strided accesses perform only component accesses, alignment is
1694 : irrelevant for them. */
1695 2072490 : if (STMT_VINFO_STRIDED_P (stmt_info)
1696 2072490 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
1697 83952 : return false;
1698 :
1699 : return true;
1700 : }
1701 :
1702 : /* Given an memory reference EXP return whether its alignment is less
1703 : than its size. */
1704 :
1705 : static bool
1706 1656267 : not_size_aligned (tree exp)
1707 : {
1708 1656267 : if (!tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (exp))))
1709 : return true;
1710 :
1711 1656267 : return (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (exp)))
1712 1656267 : > get_object_alignment (exp));
1713 : }
1714 :
1715 : /* Function vector_alignment_reachable_p
1716 :
1717 : Return true if vector alignment for DR_INFO is reachable by peeling
1718 : a few loop iterations. Return false otherwise. */
1719 :
1720 : static bool
1721 645728 : vector_alignment_reachable_p (dr_vec_info *dr_info, poly_uint64 vf)
1722 : {
1723 645728 : stmt_vec_info stmt_info = dr_info->stmt;
1724 645728 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1725 645728 : poly_uint64 nelements = TYPE_VECTOR_SUBPARTS (vectype);
1726 1291456 : poly_uint64 vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
1727 645728 : unsigned elem_size = vector_element_size (vector_size, nelements);
1728 645728 : unsigned group_size = 1;
1729 :
1730 645728 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1731 : {
1732 : /* For interleaved access we peel only if number of iterations in
1733 : the prolog loop ({VF - misalignment}), is a multiple of the
1734 : number of the interleaved accesses. */
1735 :
1736 : /* FORNOW: handle only known alignment. */
1737 88310 : if (!known_alignment_for_access_p (dr_info, vectype))
1738 645728 : return false;
1739 :
1740 52301 : unsigned mis_in_elements = dr_misalignment (dr_info, vectype) / elem_size;
1741 64897 : if (!multiple_p (nelements - mis_in_elements, DR_GROUP_SIZE (stmt_info)))
1742 : return false;
1743 :
1744 12596 : group_size = DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
1745 : }
1746 :
1747 : /* If the vectorization factor does not guarantee DR advancement of
1748 : a multiple of the target alignment no peeling will help. */
1749 570014 : if (!multiple_p (elem_size * group_size * vf, dr_target_alignment (dr_info)))
1750 160 : return false;
1751 :
1752 : /* If misalignment is known at the compile time then allow peeling
1753 : only if natural alignment is reachable through peeling. */
1754 569854 : if (known_alignment_for_access_p (dr_info, vectype)
1755 887843 : && !aligned_access_p (dr_info, vectype))
1756 : {
1757 16165 : HOST_WIDE_INT elmsize =
1758 16165 : int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
1759 16165 : if (dump_enabled_p ())
1760 : {
1761 772 : dump_printf_loc (MSG_NOTE, vect_location,
1762 : "data size = %wd. misalignment = %d.\n", elmsize,
1763 : dr_misalignment (dr_info, vectype));
1764 : }
1765 16165 : if (dr_misalignment (dr_info, vectype) % elmsize)
1766 : {
1767 72 : if (dump_enabled_p ())
1768 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1769 : "data size does not divide the misalignment.\n");
1770 : return false;
1771 : }
1772 : }
1773 :
1774 569782 : if (!known_alignment_for_access_p (dr_info, vectype))
1775 : {
1776 251865 : tree type = TREE_TYPE (DR_REF (dr_info->dr));
1777 251865 : bool is_packed = not_size_aligned (DR_REF (dr_info->dr));
1778 251865 : if (dump_enabled_p ())
1779 17487 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1780 : "Unknown misalignment, %snaturally aligned\n",
1781 : is_packed ? "not " : "");
1782 251865 : return targetm.vectorize.vector_alignment_reachable (type, is_packed);
1783 : }
1784 :
1785 : return true;
1786 : }
1787 :
1788 :
1789 : /* Calculate the cost of the memory access represented by DR_INFO. */
1790 :
1791 : static void
1792 778597 : vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
1793 : dr_alignment_support alignment_support_scheme,
1794 : int misalignment,
1795 : unsigned int *inside_cost,
1796 : unsigned int *outside_cost,
1797 : stmt_vector_for_cost *body_cost_vec,
1798 : stmt_vector_for_cost *prologue_cost_vec)
1799 : {
1800 778597 : stmt_vec_info stmt_info = dr_info->stmt;
1801 :
1802 778597 : if (DR_IS_READ (dr_info->dr))
1803 548345 : vect_get_load_cost (vinfo, stmt_info, NULL, 1,
1804 : alignment_support_scheme, misalignment, true,
1805 : inside_cost, outside_cost, prologue_cost_vec,
1806 : body_cost_vec, false);
1807 : else
1808 230252 : vect_get_store_cost (vinfo,stmt_info, NULL, 1,
1809 : alignment_support_scheme, misalignment, inside_cost,
1810 : body_cost_vec);
1811 :
1812 778597 : if (dump_enabled_p ())
1813 32144 : dump_printf_loc (MSG_NOTE, vect_location,
1814 : "vect_get_data_access_cost: inside_cost = %d, "
1815 : "outside_cost = %d.\n", *inside_cost, *outside_cost);
1816 778597 : }
1817 :
1818 :
1819 : typedef struct _vect_peel_info
1820 : {
1821 : dr_vec_info *dr_info;
1822 : int npeel;
1823 : unsigned int count;
1824 : } *vect_peel_info;
1825 :
1826 : typedef struct _vect_peel_extended_info
1827 : {
1828 : vec_info *vinfo;
1829 : struct _vect_peel_info peel_info;
1830 : unsigned int inside_cost;
1831 : unsigned int outside_cost;
1832 : } *vect_peel_extended_info;
1833 :
1834 :
1835 : /* Peeling hashtable helpers. */
1836 :
1837 : struct peel_info_hasher : free_ptr_hash <_vect_peel_info>
1838 : {
1839 : static inline hashval_t hash (const _vect_peel_info *);
1840 : static inline bool equal (const _vect_peel_info *, const _vect_peel_info *);
1841 : };
1842 :
1843 : inline hashval_t
1844 789640 : peel_info_hasher::hash (const _vect_peel_info *peel_info)
1845 : {
1846 789640 : return (hashval_t) peel_info->npeel;
1847 : }
1848 :
1849 : inline bool
1850 410489 : peel_info_hasher::equal (const _vect_peel_info *a, const _vect_peel_info *b)
1851 : {
1852 410489 : return (a->npeel == b->npeel);
1853 : }
1854 :
1855 :
1856 : /* Insert DR_INFO into peeling hash table with NPEEL as key. */
1857 :
1858 : static void
1859 380057 : vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
1860 : loop_vec_info loop_vinfo, dr_vec_info *dr_info,
1861 : int npeel, bool supportable_if_not_aligned)
1862 : {
1863 380057 : struct _vect_peel_info elem, *slot;
1864 380057 : _vect_peel_info **new_slot;
1865 :
1866 380057 : elem.npeel = npeel;
1867 380057 : slot = peeling_htab->find (&elem);
1868 380057 : if (slot)
1869 163573 : slot->count++;
1870 : else
1871 : {
1872 216484 : slot = XNEW (struct _vect_peel_info);
1873 216484 : slot->npeel = npeel;
1874 216484 : slot->dr_info = dr_info;
1875 216484 : slot->count = 1;
1876 216484 : new_slot = peeling_htab->find_slot (slot, INSERT);
1877 216484 : *new_slot = slot;
1878 : }
1879 :
1880 : /* If this DR is not supported with unknown misalignment then bias
1881 : this slot when the cost model is disabled. */
1882 380057 : if (!supportable_if_not_aligned
1883 380057 : && unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
1884 5361 : slot->count += VECT_MAX_COST;
1885 380057 : }
1886 :
1887 :
1888 : /* Traverse peeling hash table to find peeling option that aligns maximum
1889 : number of data accesses. */
1890 :
1891 : int
1892 38949 : vect_peeling_hash_get_most_frequent (_vect_peel_info **slot,
1893 : _vect_peel_extended_info *max)
1894 : {
1895 38949 : vect_peel_info elem = *slot;
1896 :
1897 38949 : if (elem->count > max->peel_info.count
1898 23774 : || (elem->count == max->peel_info.count
1899 18850 : && max->peel_info.npeel > elem->npeel))
1900 : {
1901 15191 : max->peel_info.npeel = elem->npeel;
1902 15191 : max->peel_info.count = elem->count;
1903 15191 : max->peel_info.dr_info = elem->dr_info;
1904 : }
1905 :
1906 38949 : return 1;
1907 : }
1908 :
1909 : /* Get the costs of peeling NPEEL iterations for LOOP_VINFO, checking
1910 : data access costs for all data refs. If UNKNOWN_MISALIGNMENT is true,
1911 : npeel is computed at runtime but DR0_INFO's misalignment will be zero
1912 : after peeling. */
1913 :
1914 : static void
1915 430080 : vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
1916 : dr_vec_info *dr0_info,
1917 : unsigned int *inside_cost,
1918 : unsigned int *outside_cost,
1919 : stmt_vector_for_cost *body_cost_vec,
1920 : stmt_vector_for_cost *prologue_cost_vec,
1921 : unsigned int npeel)
1922 : {
1923 430080 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
1924 :
1925 430080 : bool dr0_alignment_known_p
1926 : = (dr0_info
1927 788647 : && known_alignment_for_access_p (dr0_info,
1928 358567 : STMT_VINFO_VECTYPE (dr0_info->stmt)));
1929 :
1930 2107200 : for (data_reference *dr : datarefs)
1931 : {
1932 816960 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
1933 816960 : if (!vect_relevant_for_alignment_p (dr_info))
1934 38363 : continue;
1935 :
1936 778597 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
1937 778597 : dr_alignment_support alignment_support_scheme;
1938 778597 : int misalignment;
1939 778597 : unsigned HOST_WIDE_INT alignment;
1940 :
1941 778597 : bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
1942 778597 : size_zero_node) < 0;
1943 778597 : poly_int64 off = 0;
1944 778597 : if (negative)
1945 24417 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
1946 24417 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1947 :
1948 778597 : if (npeel == 0)
1949 395853 : misalignment = dr_misalignment (dr_info, vectype, off);
1950 382744 : else if (dr_info == dr0_info
1951 382744 : || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
1952 : misalignment = 0;
1953 129215 : else if (!dr0_alignment_known_p
1954 9657 : || !known_alignment_for_access_p (dr_info, vectype)
1955 138872 : || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
1956 : misalignment = DR_MISALIGNMENT_UNKNOWN;
1957 : else
1958 : {
1959 8628 : misalignment = dr_misalignment (dr_info, vectype, off);
1960 8628 : misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
1961 8628 : misalignment &= alignment - 1;
1962 : }
1963 778597 : alignment_support_scheme
1964 778597 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
1965 : misalignment);
1966 :
1967 778597 : vect_get_data_access_cost (loop_vinfo, dr_info,
1968 : alignment_support_scheme, misalignment,
1969 : inside_cost, outside_cost,
1970 : body_cost_vec, prologue_cost_vec);
1971 : }
1972 430080 : }
1973 :
1974 : /* Traverse peeling hash table and calculate cost for each peeling option.
1975 : Find the one with the lowest cost. */
1976 :
1977 : int
1978 157898 : vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
1979 : _vect_peel_extended_info *min)
1980 : {
1981 157898 : vect_peel_info elem = *slot;
1982 157898 : unsigned int inside_cost = 0, outside_cost = 0;
1983 157898 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
1984 157898 : stmt_vector_for_cost prologue_cost_vec, body_cost_vec;
1985 :
1986 157898 : prologue_cost_vec.create (2);
1987 157898 : body_cost_vec.create (2);
1988 :
1989 157898 : vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost,
1990 : &outside_cost, &body_cost_vec,
1991 157898 : &prologue_cost_vec, elem->npeel);
1992 :
1993 157898 : body_cost_vec.release ();
1994 157898 : prologue_cost_vec.release ();
1995 :
1996 157898 : outside_cost += vect_get_known_peeling_cost (loop_vinfo, elem->npeel);
1997 :
1998 157898 : if (inside_cost < min->inside_cost
1999 2607 : || (inside_cost == min->inside_cost
2000 1259 : && outside_cost < min->outside_cost))
2001 : {
2002 155297 : min->inside_cost = inside_cost;
2003 155297 : min->outside_cost = outside_cost;
2004 155297 : min->peel_info.dr_info = elem->dr_info;
2005 155297 : min->peel_info.npeel = elem->npeel;
2006 155297 : min->peel_info.count = elem->count;
2007 : }
2008 :
2009 157898 : return 1;
2010 : }
2011 :
2012 :
2013 : /* Choose best peeling option by traversing peeling hash table and either
2014 : choosing an option with the lowest cost (if cost model is enabled) or the
2015 : option that aligns as many accesses as possible. */
2016 :
2017 : static struct _vect_peel_extended_info
2018 168475 : vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_htab,
2019 : loop_vec_info loop_vinfo)
2020 : {
2021 168475 : struct _vect_peel_extended_info res;
2022 :
2023 168475 : res.peel_info.dr_info = NULL;
2024 168475 : res.vinfo = loop_vinfo;
2025 :
2026 168475 : if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
2027 : {
2028 153353 : res.inside_cost = INT_MAX;
2029 153353 : res.outside_cost = INT_MAX;
2030 153353 : peeling_htab->traverse <_vect_peel_extended_info *,
2031 311251 : vect_peeling_hash_get_lowest_cost> (&res);
2032 : }
2033 : else
2034 : {
2035 15122 : res.peel_info.count = 0;
2036 15122 : peeling_htab->traverse <_vect_peel_extended_info *,
2037 54071 : vect_peeling_hash_get_most_frequent> (&res);
2038 15122 : res.inside_cost = 0;
2039 15122 : res.outside_cost = 0;
2040 : }
2041 :
2042 168475 : return res;
2043 : }
2044 :
2045 : /* Return if vectorization is definitely, possibly, or unlikely to be
2046 : supportable after loop peeling. */
2047 :
2048 : static enum peeling_support
2049 85753 : vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
2050 : unsigned npeel)
2051 : {
2052 85753 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
2053 85753 : enum dr_alignment_support supportable_dr_alignment;
2054 :
2055 85753 : bool dr0_alignment_known_p
2056 171506 : = known_alignment_for_access_p (dr0_info,
2057 85753 : STMT_VINFO_VECTYPE (dr0_info->stmt));
2058 85753 : bool has_unsupported_dr_p = false;
2059 85753 : unsigned int dr0_step = tree_to_shwi (DR_STEP (dr0_info->dr));
2060 85753 : int known_unsupported_misalignment = DR_MISALIGNMENT_UNKNOWN;
2061 :
2062 : /* Check if each data ref can be vectorized after peeling. */
2063 365082 : for (data_reference *dr : datarefs)
2064 : {
2065 124667 : if (dr == dr0_info->dr)
2066 84803 : continue;
2067 :
2068 39864 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2069 39864 : if (!vect_relevant_for_alignment_p (dr_info)
2070 39864 : || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
2071 7119 : continue;
2072 :
2073 32745 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
2074 32745 : int misalignment;
2075 32745 : unsigned HOST_WIDE_INT alignment;
2076 32745 : if (!dr0_alignment_known_p
2077 2522 : || !known_alignment_for_access_p (dr_info, vectype)
2078 35267 : || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
2079 : misalignment = DR_MISALIGNMENT_UNKNOWN;
2080 : else
2081 : {
2082 2508 : misalignment = dr_misalignment (dr_info, vectype);
2083 2508 : misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
2084 2508 : misalignment &= alignment - 1;
2085 : }
2086 32745 : supportable_dr_alignment
2087 32745 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2088 : misalignment);
2089 32745 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2090 : {
2091 31418 : has_unsupported_dr_p = true;
2092 :
2093 : /* If unaligned unsupported DRs exist, we do following checks to see
2094 : if they can be mutually aligned to support vectorization. If yes,
2095 : we can try peeling and create a runtime (mutual alignment) check
2096 : to guard the peeled loop. If no, return PEELING_UNSUPPORTED. */
2097 :
2098 : /* 1) If unaligned unsupported DRs have different alignment steps, the
2099 : probability of DRs being mutually aligned is very low, and it's
2100 : quite complex to check mutual alignment at runtime. We return
2101 : PEELING_UNSUPPORTED in this case. */
2102 31418 : if (tree_to_shwi (DR_STEP (dr)) != dr0_step)
2103 85753 : return peeling_unsupported;
2104 :
2105 : /* 2) Based on above same alignment step condition, if one known
2106 : misaligned DR has zero misalignment, or different misalignment
2107 : amount from another known misaligned DR, peeling is unable to
2108 : help make all these DRs aligned together. We won't try peeling
2109 : with versioning anymore. */
2110 27170 : int curr_dr_misalignment = dr_misalignment (dr_info, vectype);
2111 27170 : if (curr_dr_misalignment == 0)
2112 : return peeling_unsupported;
2113 14574 : if (known_unsupported_misalignment != DR_MISALIGNMENT_UNKNOWN)
2114 : {
2115 8 : if (curr_dr_misalignment != DR_MISALIGNMENT_UNKNOWN
2116 8 : && curr_dr_misalignment != known_unsupported_misalignment)
2117 : return peeling_unsupported;
2118 : }
2119 : else
2120 : known_unsupported_misalignment = curr_dr_misalignment;
2121 : }
2122 : }
2123 :
2124 : /* Vectorization is known to be supportable with peeling alone when there is
2125 : no unsupported DR. */
2126 68909 : return has_unsupported_dr_p ? peeling_maybe_supported
2127 : : peeling_known_supported;
2128 : }
2129 :
2130 : /* Compare two data-references DRA and DRB to group them into chunks
2131 : with related alignment. */
2132 :
2133 : static int
2134 4680548 : dr_align_group_sort_cmp (const void *dra_, const void *drb_)
2135 : {
2136 4680548 : data_reference_p dra = *(data_reference_p *)const_cast<void *>(dra_);
2137 4680548 : data_reference_p drb = *(data_reference_p *)const_cast<void *>(drb_);
2138 4680548 : int cmp;
2139 :
2140 : /* Stabilize sort. */
2141 4680548 : if (dra == drb)
2142 : return 0;
2143 :
2144 : /* Ordering of DRs according to base. */
2145 4680548 : cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
2146 : DR_BASE_ADDRESS (drb));
2147 4680548 : if (cmp != 0)
2148 : return cmp;
2149 :
2150 : /* And according to DR_OFFSET. */
2151 2067467 : cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
2152 2067467 : if (cmp != 0)
2153 : return cmp;
2154 :
2155 : /* And after step. */
2156 2053254 : cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
2157 2053254 : if (cmp != 0)
2158 : return cmp;
2159 :
2160 : /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */
2161 2048019 : cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
2162 2048019 : if (cmp == 0)
2163 247978 : return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
2164 : return cmp;
2165 : }
2166 :
2167 : /* Function vect_enhance_data_refs_alignment
2168 :
2169 : This pass will use loop versioning and loop peeling in order to enhance
2170 : the alignment of data references in the loop.
2171 :
2172 : FOR NOW: we assume that whatever versioning/peeling takes place, only the
2173 : original loop is to be vectorized. Any other loops that are created by
2174 : the transformations performed in this pass - are not supposed to be
2175 : vectorized. This restriction will be relaxed.
2176 :
2177 : This pass will require a cost model to guide it whether to apply peeling
2178 : or versioning or a combination of the two. For example, the scheme that
2179 : intel uses when given a loop with several memory accesses, is as follows:
2180 : choose one memory access ('p') which alignment you want to force by doing
2181 : peeling. Then, either (1) generate a loop in which 'p' is aligned and all
2182 : other accesses are not necessarily aligned, or (2) use loop versioning to
2183 : generate one loop in which all accesses are aligned, and another loop in
2184 : which only 'p' is necessarily aligned.
2185 :
2186 : ("Automatic Intra-Register Vectorization for the Intel Architecture",
2187 : Aart J.C. Bik, Milind Girkar, Paul M. Grey and Ximmin Tian, International
2188 : Journal of Parallel Programming, Vol. 30, No. 2, April 2002.)
2189 :
2190 : Devising a cost model is the most critical aspect of this work. It will
2191 : guide us on which access to peel for, whether to use loop versioning, how
2192 : many versions to create, etc. The cost model will probably consist of
2193 : generic considerations as well as target specific considerations (on
2194 : powerpc for example, misaligned stores are more painful than misaligned
2195 : loads).
2196 :
2197 : Here are the general steps involved in alignment enhancements:
2198 :
2199 : -- original loop, before alignment analysis:
2200 : for (i=0; i<N; i++){
2201 : x = q[i]; # DR_MISALIGNMENT(q) = unknown
2202 : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2203 : }
2204 :
2205 : -- After vect_compute_data_refs_alignment:
2206 : for (i=0; i<N; i++){
2207 : x = q[i]; # DR_MISALIGNMENT(q) = 3
2208 : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2209 : }
2210 :
2211 : -- Possibility 1: we do loop versioning:
2212 : if (p is aligned) {
2213 : for (i=0; i<N; i++){ # loop 1A
2214 : x = q[i]; # DR_MISALIGNMENT(q) = 3
2215 : p[i] = y; # DR_MISALIGNMENT(p) = 0
2216 : }
2217 : }
2218 : else {
2219 : for (i=0; i<N; i++){ # loop 1B
2220 : x = q[i]; # DR_MISALIGNMENT(q) = 3
2221 : p[i] = y; # DR_MISALIGNMENT(p) = unaligned
2222 : }
2223 : }
2224 :
2225 : -- Possibility 2: we do loop peeling:
2226 : for (i = 0; i < 3; i++){ # (scalar loop, not to be vectorized).
2227 : x = q[i];
2228 : p[i] = y;
2229 : }
2230 : for (i = 3; i < N; i++){ # loop 2A
2231 : x = q[i]; # DR_MISALIGNMENT(q) = 0
2232 : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2233 : }
2234 :
2235 : -- Possibility 3: combination of loop peeling and versioning:
2236 : if (p & q are mutually aligned) {
2237 : for (i=0; i<3; i++){ # (peeled loop iterations).
2238 : x = q[i];
2239 : p[i] = y;
2240 : }
2241 : for (i=3; i<N; i++){ # loop 3A
2242 : x = q[i]; # DR_MISALIGNMENT(q) = 0
2243 : p[i] = y; # DR_MISALIGNMENT(p) = 0
2244 : }
2245 : }
2246 : else {
2247 : for (i=0; i<N; i++){ # (scalar loop, not to be vectorized).
2248 : x = q[i]; # DR_MISALIGNMENT(q) = 3
2249 : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2250 : }
2251 : }
2252 :
2253 : These loops are later passed to loop_transform to be vectorized. The
2254 : vectorizer will use the alignment information to guide the transformation
2255 : (whether to generate regular loads/stores, or with special handling for
2256 : misalignment). */
2257 :
2258 : opt_result
2259 405173 : vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
2260 : {
2261 405173 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2262 405173 : dr_vec_info *first_store = NULL;
2263 405173 : dr_vec_info *dr0_info = NULL;
2264 405173 : struct data_reference *dr;
2265 405173 : unsigned int i;
2266 405173 : bool do_peeling = false;
2267 405173 : bool do_versioning = false;
2268 405173 : bool try_peeling_with_versioning = false;
2269 405173 : unsigned int npeel = 0;
2270 405173 : bool one_misalignment_known = false;
2271 405173 : bool one_misalignment_unknown = false;
2272 405173 : bool one_dr_unsupportable = false;
2273 405173 : dr_vec_info *unsupportable_dr_info = NULL;
2274 405173 : unsigned int dr0_same_align_drs = 0, first_store_same_align_drs = 0;
2275 405173 : hash_table<peel_info_hasher> peeling_htab (1);
2276 :
2277 405173 : DUMP_VECT_SCOPE ("vect_enhance_data_refs_alignment");
2278 :
2279 : /* Reset data so we can safely be called multiple times. */
2280 405173 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
2281 405173 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = 0;
2282 :
2283 405173 : if (LOOP_VINFO_DATAREFS (loop_vinfo).is_empty ())
2284 14946 : return opt_result::success ();
2285 :
2286 : /* Sort the vector of datarefs so DRs that have the same or dependent
2287 : alignment are next to each other. */
2288 390227 : auto_vec<data_reference_p> datarefs
2289 390227 : = LOOP_VINFO_DATAREFS (loop_vinfo).copy ();
2290 390227 : datarefs.qsort (dr_align_group_sort_cmp);
2291 :
2292 : /* Compute the number of DRs that become aligned when we peel
2293 : a dataref so it becomes aligned. */
2294 780454 : auto_vec<unsigned> n_same_align_refs (datarefs.length ());
2295 390227 : n_same_align_refs.quick_grow_cleared (datarefs.length ());
2296 390227 : unsigned i0;
2297 801587 : for (i0 = 0; i0 < datarefs.length (); ++i0)
2298 404471 : if (DR_BASE_ADDRESS (datarefs[i0]))
2299 : break;
2300 2501456 : for (i = i0 + 1; i <= datarefs.length (); ++i)
2301 : {
2302 860501 : if (i == datarefs.length ()
2303 477163 : || !operand_equal_p (DR_BASE_ADDRESS (datarefs[i0]),
2304 477163 : DR_BASE_ADDRESS (datarefs[i]), 0)
2305 222419 : || !operand_equal_p (DR_OFFSET (datarefs[i0]),
2306 222419 : DR_OFFSET (datarefs[i]), 0)
2307 1081619 : || !operand_equal_p (DR_STEP (datarefs[i0]),
2308 221118 : DR_STEP (datarefs[i]), 0))
2309 : {
2310 : /* The subgroup [i0, i-1] now only differs in DR_INIT and
2311 : possibly DR_TARGET_ALIGNMENT. Still the whole subgroup
2312 : will get known misalignment if we align one of the refs
2313 : with the largest DR_TARGET_ALIGNMENT. */
2314 1500430 : for (unsigned j = i0; j < i; ++j)
2315 : {
2316 860501 : dr_vec_info *dr_infoj = loop_vinfo->lookup_dr (datarefs[j]);
2317 4570605 : for (unsigned k = i0; k < i; ++k)
2318 : {
2319 2849603 : if (k == j)
2320 860501 : continue;
2321 1989102 : dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]);
2322 1989102 : if (vect_dr_aligned_if_related_peeled_dr_is (dr_infok,
2323 : dr_infoj))
2324 728852 : n_same_align_refs[j]++;
2325 : }
2326 : }
2327 : i0 = i;
2328 : }
2329 : }
2330 :
2331 : /* See if we can relax the flags on speculative reads for early break. Do
2332 : this outside of the other loops below because they can exit early leading
2333 : to the flag not being cleared for known in bounds cases. */
2334 390227 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2335 390227 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
2336 964309 : for (auto dr : datarefs)
2337 : {
2338 372460 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2339 372460 : if (!vect_relevant_for_alignment_p (dr_info))
2340 : continue;
2341 :
2342 372460 : stmt_vec_info stmt_info = dr_info->stmt;
2343 :
2344 : /* With variable VF, unsafe speculative read can be avoided for known
2345 : inbounds DRs as long as partial vectors are used. */
2346 372460 : if (!vf.is_constant ()
2347 : && dr_safe_speculative_read_required (stmt_info)
2348 : && DR_SCALAR_KNOWN_BOUNDS (dr_info))
2349 : {
2350 : dr_set_safe_speculative_read_required (stmt_info, false);
2351 : LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
2352 : }
2353 : }
2354 :
2355 : /* While cost model enhancements are expected in the future, the high level
2356 : view of the code at this time is as follows:
2357 :
2358 : A) If there is a misaligned access then see if doing peeling alone can
2359 : make all data references satisfy vect_supportable_dr_alignment. If so,
2360 : update data structures and return.
2361 :
2362 : B) If peeling alone wasn't possible and there is a data reference with an
2363 : unknown misalignment that does not satisfy vect_supportable_dr_alignment
2364 : then we may use either of the following two approaches.
2365 :
2366 : B1) Try peeling with versioning: Add a runtime loop versioning check to
2367 : see if all unsupportable data references are mutually aligned, which
2368 : means they will be uniformly aligned after a certain amount of loop
2369 : peeling. If peeling and versioning can be used together, set
2370 : LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT_P to TRUE and return.
2371 :
2372 : B2) Try versioning alone: Add a runtime loop versioning check to see if
2373 : all unsupportable data references are already uniformly aligned
2374 : without loop peeling. If versioning can be applied alone, set
2375 : LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT_P to FALSE and return.
2376 :
2377 : Above B1 is more powerful and more likely to be adopted than B2. But B2
2378 : is still available and useful in some cases, for example, the cost model
2379 : does not allow much peeling.
2380 :
2381 : C) If none of above was successful then the alignment was not enhanced,
2382 : just return. */
2383 :
2384 : /* (1) Peeling to force alignment. */
2385 :
2386 : /* (1.1) Decide whether to perform peeling, how many iterations to peel, and
2387 : if vectorization may be supported by peeling with versioning.
2388 : Considerations:
2389 : - How many accesses will become aligned due to the peeling
2390 : - How many accesses will become unaligned due to the peeling,
2391 : and the cost of misaligned accesses.
2392 : - The cost of peeling (the extra runtime checks, the increase
2393 : in code size). */
2394 :
2395 1101384 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2396 : {
2397 757148 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2398 757148 : if (!vect_relevant_for_alignment_p (dr_info))
2399 111420 : continue;
2400 :
2401 645728 : stmt_vec_info stmt_info = dr_info->stmt;
2402 645728 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2403 :
2404 645728 : do_peeling = vector_alignment_reachable_p (dr_info, vf);
2405 645728 : if (do_peeling)
2406 : {
2407 567590 : if (known_alignment_for_access_p (dr_info, vectype))
2408 : {
2409 317917 : unsigned int npeel_tmp = 0;
2410 317917 : bool negative = tree_int_cst_compare (DR_STEP (dr),
2411 317917 : size_zero_node) < 0;
2412 :
2413 : /* If known_alignment_for_access_p then we have set
2414 : DR_MISALIGNMENT which is only done if we know it at compiler
2415 : time, so it is safe to assume target alignment is constant.
2416 : */
2417 317917 : unsigned int target_align =
2418 317917 : DR_TARGET_ALIGNMENT (dr_info).to_constant ();
2419 317917 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (dr_info);
2420 317917 : poly_int64 off = 0;
2421 317917 : if (negative)
2422 2604 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2423 317917 : unsigned int mis = dr_misalignment (dr_info, vectype, off);
2424 317917 : mis = negative ? mis : -mis;
2425 317917 : if (mis != 0)
2426 15152 : npeel_tmp = (mis & (target_align - 1)) / dr_size;
2427 :
2428 : /* For multiple types, it is possible that the bigger type access
2429 : will have more than one peeling option. E.g., a loop with two
2430 : types: one of size (vector size / 4), and the other one of
2431 : size (vector size / 8). Vectorization factor will 8. If both
2432 : accesses are misaligned by 3, the first one needs one scalar
2433 : iteration to be aligned, and the second one needs 5. But the
2434 : first one will be aligned also by peeling 5 scalar
2435 : iterations, and in that case both accesses will be aligned.
2436 : Hence, except for the immediate peeling amount, we also want
2437 : to try to add full vector size, while we don't exceed
2438 : vectorization factor.
2439 : We do this automatically for cost model, since we calculate
2440 : cost for every peeling option. */
2441 317917 : poly_uint64 nscalars = npeel_tmp;
2442 317917 : if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
2443 : {
2444 41714 : unsigned group_size = 1;
2445 41714 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2446 1917 : group_size = DR_GROUP_SIZE (stmt_info);
2447 41714 : nscalars = vf * group_size;
2448 : }
2449 :
2450 : /* Save info about DR in the hash table. Also include peeling
2451 : amounts according to the explanation above. Indicate
2452 : the alignment status when the ref is not aligned.
2453 : ??? Rather than using unknown alignment here we should
2454 : prune all entries from the peeling hashtable which cause
2455 : DRs to be not supported. */
2456 317917 : bool supportable_if_not_aligned
2457 : = vect_supportable_dr_alignment
2458 317917 : (loop_vinfo, dr_info, vectype, DR_MISALIGNMENT_UNKNOWN);
2459 697974 : while (known_le (npeel_tmp, nscalars))
2460 : {
2461 380057 : vect_peeling_hash_insert (&peeling_htab, loop_vinfo,
2462 : dr_info, npeel_tmp,
2463 : supportable_if_not_aligned);
2464 380057 : npeel_tmp += MAX (1, target_align / dr_size);
2465 : }
2466 :
2467 317917 : one_misalignment_known = true;
2468 : }
2469 : else
2470 : {
2471 : /* If we don't know any misalignment values, we prefer
2472 : peeling for data-ref that has the maximum number of data-refs
2473 : with the same alignment, unless the target prefers to align
2474 : stores over load. */
2475 249673 : unsigned same_align_drs = n_same_align_refs[i];
2476 249673 : if (!dr0_info
2477 249673 : || dr0_same_align_drs < same_align_drs)
2478 : {
2479 : dr0_same_align_drs = same_align_drs;
2480 : dr0_info = dr_info;
2481 : }
2482 : /* For data-refs with the same number of related
2483 : accesses prefer the one where the misalign
2484 : computation will be invariant in the outermost loop. */
2485 79395 : else if (dr0_same_align_drs == same_align_drs)
2486 : {
2487 77938 : class loop *ivloop0, *ivloop;
2488 77938 : ivloop0 = outermost_invariant_loop_for_expr
2489 77938 : (loop, DR_BASE_ADDRESS (dr0_info->dr));
2490 77938 : ivloop = outermost_invariant_loop_for_expr
2491 77938 : (loop, DR_BASE_ADDRESS (dr));
2492 77938 : if ((ivloop && !ivloop0)
2493 77938 : || (ivloop && ivloop0
2494 77930 : && flow_loop_nested_p (ivloop, ivloop0)))
2495 : dr0_info = dr_info;
2496 : }
2497 :
2498 249673 : one_misalignment_unknown = true;
2499 :
2500 : /* Check for data refs with unsupportable alignment that
2501 : can be peeled. */
2502 249673 : enum dr_alignment_support supportable_dr_alignment
2503 249673 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2504 : DR_MISALIGNMENT_UNKNOWN);
2505 249673 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2506 : {
2507 104259 : one_dr_unsupportable = true;
2508 104259 : unsupportable_dr_info = dr_info;
2509 : }
2510 :
2511 249673 : if (!first_store && DR_IS_WRITE (dr))
2512 : {
2513 54074 : first_store = dr_info;
2514 54074 : first_store_same_align_drs = same_align_drs;
2515 : }
2516 : }
2517 : }
2518 : else
2519 : {
2520 78138 : if (!aligned_access_p (dr_info, vectype))
2521 : {
2522 45991 : if (dump_enabled_p ())
2523 2107 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2524 : "vector alignment may not be reachable\n");
2525 : break;
2526 : }
2527 : }
2528 : }
2529 :
2530 : /* Check if we can possibly peel the loop. */
2531 390227 : if (!vect_can_advance_ivs_p (loop_vinfo)
2532 386349 : || !slpeel_can_duplicate_loop_p (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
2533 386349 : loop_preheader_edge (loop))
2534 386349 : || loop->inner
2535 : /* We don't currently maintain the LCSSA for prologue peeled inversed
2536 : loops. */
2537 774981 : || (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)
2538 32180 : && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)))
2539 : do_peeling = false;
2540 :
2541 390227 : struct _vect_peel_extended_info peel_for_known_alignment;
2542 390227 : struct _vect_peel_extended_info peel_for_unknown_alignment;
2543 390227 : struct _vect_peel_extended_info best_peel;
2544 :
2545 390227 : peel_for_unknown_alignment.inside_cost = INT_MAX;
2546 390227 : peel_for_unknown_alignment.outside_cost = INT_MAX;
2547 390227 : peel_for_unknown_alignment.peel_info.count = 0;
2548 :
2549 390227 : if (do_peeling
2550 390227 : && one_misalignment_unknown)
2551 : {
2552 : /* Check if the target requires to prefer stores over loads, i.e., if
2553 : misaligned stores are more expensive than misaligned loads (taking
2554 : drs with same alignment into account). */
2555 153675 : unsigned int load_inside_cost = 0;
2556 153675 : unsigned int load_outside_cost = 0;
2557 153675 : unsigned int store_inside_cost = 0;
2558 153675 : unsigned int store_outside_cost = 0;
2559 153675 : unsigned int estimated_npeels = vect_vf_for_cost (loop_vinfo) / 2;
2560 :
2561 153675 : stmt_vector_for_cost dummy;
2562 153675 : dummy.create (2);
2563 153675 : vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info,
2564 : &load_inside_cost,
2565 : &load_outside_cost,
2566 : &dummy, &dummy, estimated_npeels);
2567 153675 : dummy.release ();
2568 :
2569 153675 : if (first_store)
2570 : {
2571 46994 : dummy.create (2);
2572 46994 : vect_get_peeling_costs_all_drs (loop_vinfo, first_store,
2573 : &store_inside_cost,
2574 : &store_outside_cost,
2575 : &dummy, &dummy,
2576 : estimated_npeels);
2577 46994 : dummy.release ();
2578 : }
2579 : else
2580 : {
2581 106681 : store_inside_cost = INT_MAX;
2582 106681 : store_outside_cost = INT_MAX;
2583 : }
2584 :
2585 153675 : if (load_inside_cost > store_inside_cost
2586 153675 : || (load_inside_cost == store_inside_cost
2587 46537 : && load_outside_cost > store_outside_cost))
2588 : {
2589 153675 : dr0_info = first_store;
2590 153675 : dr0_same_align_drs = first_store_same_align_drs;
2591 153675 : peel_for_unknown_alignment.inside_cost = store_inside_cost;
2592 153675 : peel_for_unknown_alignment.outside_cost = store_outside_cost;
2593 : }
2594 : else
2595 : {
2596 153675 : peel_for_unknown_alignment.inside_cost = load_inside_cost;
2597 153675 : peel_for_unknown_alignment.outside_cost = load_outside_cost;
2598 : }
2599 :
2600 153675 : peel_for_unknown_alignment.outside_cost
2601 153675 : += vect_get_known_peeling_cost (loop_vinfo, estimated_npeels);
2602 :
2603 153675 : peel_for_unknown_alignment.peel_info.count = dr0_same_align_drs + 1;
2604 : }
2605 :
2606 390227 : peel_for_unknown_alignment.peel_info.npeel = 0;
2607 390227 : peel_for_unknown_alignment.peel_info.dr_info = dr0_info;
2608 :
2609 390227 : best_peel = peel_for_unknown_alignment;
2610 :
2611 390227 : peel_for_known_alignment.inside_cost = INT_MAX;
2612 390227 : peel_for_known_alignment.outside_cost = INT_MAX;
2613 390227 : peel_for_known_alignment.peel_info.count = 0;
2614 390227 : peel_for_known_alignment.peel_info.dr_info = NULL;
2615 :
2616 390227 : if (do_peeling && one_misalignment_known)
2617 : {
2618 : /* Peeling is possible, but there is no data access that is not supported
2619 : unless aligned. So we try to choose the best possible peeling from
2620 : the hash table. */
2621 168475 : peel_for_known_alignment = vect_peeling_hash_choose_best_peeling
2622 168475 : (&peeling_htab, loop_vinfo);
2623 : }
2624 :
2625 : /* Compare costs of peeling for known and unknown alignment. */
2626 390227 : if (peel_for_known_alignment.peel_info.dr_info != NULL
2627 168475 : && peel_for_unknown_alignment.inside_cost
2628 : >= peel_for_known_alignment.inside_cost)
2629 : {
2630 154011 : best_peel = peel_for_known_alignment;
2631 :
2632 : /* If the best peeling for known alignment has NPEEL == 0, perform no
2633 : peeling at all except if there is an unsupportable dr that we can
2634 : align. */
2635 154011 : if (best_peel.peel_info.npeel == 0 && !one_dr_unsupportable)
2636 : do_peeling = false;
2637 : }
2638 :
2639 : /* If there is an unsupportable data ref, prefer this over all choices so far
2640 : since we'd have to discard a chosen peeling except when it accidentally
2641 : aligned the unsupportable data ref. */
2642 244890 : if (one_dr_unsupportable)
2643 : dr0_info = unsupportable_dr_info;
2644 303874 : else if (do_peeling)
2645 : {
2646 : /* Calculate the penalty for no peeling, i.e. leaving everything as-is.
2647 : TODO: Use nopeel_outside_cost or get rid of it? */
2648 71513 : unsigned nopeel_inside_cost = 0;
2649 71513 : unsigned nopeel_outside_cost = 0;
2650 :
2651 71513 : stmt_vector_for_cost dummy;
2652 71513 : dummy.create (2);
2653 71513 : vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
2654 : &nopeel_outside_cost, &dummy, &dummy, 0);
2655 71513 : dummy.release ();
2656 :
2657 : /* Add epilogue costs. As we do not peel for alignment here, no prologue
2658 : costs will be recorded. */
2659 71513 : nopeel_outside_cost += vect_get_known_peeling_cost (loop_vinfo, 0);
2660 :
2661 71513 : npeel = best_peel.peel_info.npeel;
2662 71513 : dr0_info = best_peel.peel_info.dr_info;
2663 :
2664 : /* If no peeling is not more expensive than the best peeling we
2665 : have so far, don't perform any peeling. */
2666 71513 : if (nopeel_inside_cost <= best_peel.inside_cost)
2667 64030 : do_peeling = false;
2668 : }
2669 :
2670 157866 : if (do_peeling)
2671 : {
2672 85753 : stmt_vec_info stmt_info = dr0_info->stmt;
2673 85753 : if (known_alignment_for_access_p (dr0_info,
2674 : STMT_VINFO_VECTYPE (stmt_info)))
2675 : {
2676 7460 : bool negative = tree_int_cst_compare (DR_STEP (dr0_info->dr),
2677 7460 : size_zero_node) < 0;
2678 7460 : if (!npeel)
2679 : {
2680 : /* Since it's known at compile time, compute the number of
2681 : iterations in the peeled loop (the peeling factor) for use in
2682 : updating DR_MISALIGNMENT values. The peeling factor is the
2683 : vectorization factor minus the misalignment as an element
2684 : count. */
2685 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2686 0 : poly_int64 off = 0;
2687 0 : if (negative)
2688 0 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
2689 0 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
2690 0 : unsigned int mis
2691 0 : = dr_misalignment (dr0_info, vectype, off);
2692 0 : mis = negative ? mis : -mis;
2693 : /* If known_alignment_for_access_p then we have set
2694 : DR_MISALIGNMENT which is only done if we know it at compiler
2695 : time, so it is safe to assume target alignment is constant.
2696 : */
2697 0 : unsigned int target_align =
2698 0 : DR_TARGET_ALIGNMENT (dr0_info).to_constant ();
2699 0 : npeel = ((mis & (target_align - 1))
2700 0 : / vect_get_scalar_dr_size (dr0_info));
2701 : }
2702 :
2703 : /* For interleaved data access every iteration accesses all the
2704 : members of the group, therefore we divide the number of iterations
2705 : by the group size. */
2706 7460 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2707 293 : npeel /= DR_GROUP_SIZE (stmt_info);
2708 :
2709 7460 : if (dump_enabled_p ())
2710 296 : dump_printf_loc (MSG_NOTE, vect_location,
2711 : "Try peeling by %d\n", npeel);
2712 : }
2713 :
2714 : /* Check how peeling for alignment can support vectorization. Function
2715 : vect_peeling_supportable returns one of the three possible values:
2716 : - PEELING_KNOWN_SUPPORTED: indicates that we know all unsupported
2717 : datarefs can be aligned after peeling. We can use peeling alone.
2718 : - PEELING_MAYBE_SUPPORTED: indicates that peeling may be able to make
2719 : these datarefs aligned but we are not sure about it at compile time.
2720 : We will try peeling with versioning to add a runtime check to guard
2721 : the peeled loop.
2722 : - PEELING_UNSUPPORTED: indicates that peeling is almost impossible to
2723 : support vectorization. We will stop trying peeling. */
2724 85753 : switch (vect_peeling_supportable (loop_vinfo, dr0_info, npeel))
2725 : {
2726 : case peeling_known_supported:
2727 : break;
2728 13392 : case peeling_maybe_supported:
2729 13392 : try_peeling_with_versioning = true;
2730 13392 : break;
2731 16844 : case peeling_unsupported:
2732 16844 : do_peeling = false;
2733 16844 : break;
2734 : }
2735 :
2736 : /* Check if all datarefs are supportable and log. */
2737 85753 : if (do_peeling
2738 85753 : && npeel == 0
2739 85753 : && known_alignment_for_access_p (dr0_info,
2740 : STMT_VINFO_VECTYPE (stmt_info)))
2741 3 : return opt_result::success ();
2742 :
2743 : /* Cost model #1 - honor --param vect-max-peeling-for-alignment. */
2744 85750 : if (do_peeling)
2745 : {
2746 68906 : unsigned max_allowed_peel
2747 68906 : = param_vect_max_peeling_for_alignment;
2748 68906 : if (loop_cost_model (loop) <= VECT_COST_MODEL_CHEAP)
2749 : max_allowed_peel = 0;
2750 14850 : if (max_allowed_peel != (unsigned)-1)
2751 : {
2752 54077 : unsigned max_peel = npeel;
2753 54077 : if (max_peel == 0)
2754 : {
2755 51056 : poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr0_info);
2756 51056 : unsigned HOST_WIDE_INT target_align_c;
2757 51056 : if (target_align.is_constant (&target_align_c))
2758 102112 : max_peel =
2759 51056 : target_align_c / vect_get_scalar_dr_size (dr0_info) - 1;
2760 : else
2761 : {
2762 : do_peeling = false;
2763 : if (dump_enabled_p ())
2764 : dump_printf_loc (MSG_NOTE, vect_location,
2765 : "Disable peeling, max peels set and vector"
2766 : " alignment unknown\n");
2767 : }
2768 : }
2769 54077 : if (max_peel > max_allowed_peel)
2770 : {
2771 54069 : do_peeling = false;
2772 54069 : if (dump_enabled_p ())
2773 53 : dump_printf_loc (MSG_NOTE, vect_location,
2774 : "Disable peeling, max peels reached: %d\n", max_peel);
2775 : }
2776 : }
2777 : }
2778 :
2779 : /* Cost model #2 - if peeling may result in a remaining loop not
2780 : iterating enough to be vectorized then do not peel. Since this
2781 : is a cost heuristic rather than a correctness decision, use the
2782 : most likely runtime value for variable vectorization factors. */
2783 53 : if (do_peeling
2784 14837 : && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
2785 : {
2786 3252 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
2787 3252 : unsigned int max_peel = npeel == 0 ? assumed_vf - 1 : npeel;
2788 3252 : if ((unsigned HOST_WIDE_INT) LOOP_VINFO_INT_NITERS (loop_vinfo)
2789 3252 : < assumed_vf + max_peel)
2790 : do_peeling = false;
2791 : }
2792 :
2793 : if (do_peeling)
2794 : {
2795 : /* (1.2) Update the DR_MISALIGNMENT of each data reference DR_i.
2796 : If the misalignment of DR_i is identical to that of dr0 then set
2797 : DR_MISALIGNMENT (DR_i) to zero. If the misalignment of DR_i and
2798 : dr0 are known at compile time then increment DR_MISALIGNMENT (DR_i)
2799 : by the peeling factor times the element size of DR_i (MOD the
2800 : vectorization factor times the size). Otherwise, the
2801 : misalignment of DR_i must be set to unknown. */
2802 31161 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2803 17157 : if (dr != dr0_info->dr)
2804 : {
2805 3153 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2806 3153 : if (!vect_relevant_for_alignment_p (dr_info))
2807 354 : continue;
2808 :
2809 2799 : vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
2810 : }
2811 : }
2812 :
2813 85750 : if (do_peeling && !try_peeling_with_versioning)
2814 : {
2815 : /* Update data structures if peeling will be applied alone. */
2816 12932 : LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
2817 12932 : if (npeel)
2818 12932 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = npeel;
2819 : else
2820 10801 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = -1;
2821 12932 : SET_DR_MISALIGNMENT (dr0_info,
2822 : vect_dr_misalign_for_aligned_access (dr0_info));
2823 12932 : if (dump_enabled_p ())
2824 : {
2825 360 : dump_printf_loc (MSG_NOTE, vect_location,
2826 : "Alignment of access forced using peeling.\n");
2827 360 : dump_printf_loc (MSG_NOTE, vect_location,
2828 : "Peeling for alignment will be applied.\n");
2829 : }
2830 :
2831 : /* The inside-loop cost will be accounted for in vectorizable_load
2832 : and vectorizable_store correctly with adjusted alignments.
2833 : Drop the body_cst_vec on the floor here. */
2834 12932 : return opt_result::success ();
2835 : }
2836 : }
2837 :
2838 : /* (2) Versioning to force alignment. */
2839 :
2840 : /* Try versioning if:
2841 : 1) optimize loop for speed and the cost-model is not cheap
2842 : 2) there is at least one unsupported misaligned data ref with an unknown
2843 : misalignment, and
2844 : 3) all misaligned data refs with a known misalignment are supported, and
2845 : 4) the number of runtime alignment checks is within reason. */
2846 :
2847 377292 : do_versioning
2848 377292 : = (optimize_loop_nest_for_speed_p (loop)
2849 376842 : && !loop->inner /* FORNOW */
2850 752539 : && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP);
2851 :
2852 : if (do_versioning)
2853 : {
2854 368755 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2855 : {
2856 277175 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2857 277175 : if (!vect_relevant_for_alignment_p (dr_info))
2858 195233 : continue;
2859 :
2860 191034 : stmt_vec_info stmt_info = dr_info->stmt;
2861 191034 : if (STMT_VINFO_STRIDED_P (stmt_info))
2862 : {
2863 : do_versioning = false;
2864 5115 : break;
2865 : }
2866 :
2867 189924 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2868 189924 : bool negative = tree_int_cst_compare (DR_STEP (dr),
2869 189924 : size_zero_node) < 0;
2870 189924 : poly_int64 off = 0;
2871 189924 : if (negative)
2872 3405 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
2873 3405 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
2874 189924 : int misalignment;
2875 189924 : if ((misalignment = dr_misalignment (dr_info, vectype, off)) == 0)
2876 109092 : continue;
2877 :
2878 80832 : enum dr_alignment_support supportable_dr_alignment
2879 80832 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2880 : misalignment);
2881 80832 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2882 : {
2883 15795 : if (misalignment != DR_MISALIGNMENT_UNKNOWN
2884 15795 : || (LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
2885 12314 : >= (unsigned) param_vect_max_version_for_alignment_checks))
2886 : {
2887 : do_versioning = false;
2888 5115 : break;
2889 : }
2890 :
2891 : /* Forcing alignment in the first iteration is no good if
2892 : we don't keep it across iterations. For now, just disable
2893 : versioning in this case.
2894 : ?? We could actually unroll the loop to achieve the required
2895 : overall step alignment, and forcing the alignment could be
2896 : done by doing some iterations of the non-vectorized loop. */
2897 11906 : if (!multiple_p (vf * DR_STEP_ALIGNMENT (dr),
2898 11906 : DR_TARGET_ALIGNMENT (dr_info)))
2899 : {
2900 : do_versioning = false;
2901 : break;
2902 : }
2903 :
2904 : /* Use "mask = DR_TARGET_ALIGNMENT - 1" to test rightmost address
2905 : bits for runtime alignment check. For example, for 16 bytes
2906 : target alignment the mask is 15 = 0xf. */
2907 11906 : poly_uint64 mask = DR_TARGET_ALIGNMENT (dr_info) - 1;
2908 :
2909 : /* FORNOW: use the same mask to test all potentially unaligned
2910 : references in the loop. */
2911 11906 : if (maybe_ne (LOOP_VINFO_PTR_MASK (loop_vinfo), 0U)
2912 11906 : && maybe_ne (LOOP_VINFO_PTR_MASK (loop_vinfo), mask))
2913 : {
2914 : do_versioning = false;
2915 : break;
2916 : }
2917 :
2918 11790 : LOOP_VINFO_PTR_MASK (loop_vinfo) = mask;
2919 11790 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).safe_push (stmt_info);
2920 : }
2921 : }
2922 :
2923 : /* Versioning requires at least one misaligned data reference. */
2924 96695 : if (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
2925 : do_versioning = false;
2926 6048 : else if (!do_versioning)
2927 540 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
2928 : }
2929 :
2930 : /* If we are trying peeling with versioning but versioning is disabled for
2931 : some reason, peeling should be turned off together. */
2932 377292 : if (try_peeling_with_versioning && !do_versioning)
2933 : do_peeling = false;
2934 :
2935 365072 : if (do_versioning)
2936 : {
2937 : const vec<stmt_vec_info> &may_misalign_stmts
2938 : = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
2939 : stmt_vec_info stmt_info;
2940 :
2941 : /* It can now be assumed that the data references in the statements
2942 : in LOOP_VINFO_MAY_MISALIGN_STMTS will be aligned in the version
2943 : of the loop being vectorized. */
2944 14658 : FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
2945 : {
2946 9150 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
2947 9150 : SET_DR_MISALIGNMENT (dr_info,
2948 : vect_dr_misalign_for_aligned_access (dr_info));
2949 9150 : if (dump_enabled_p ())
2950 152 : dump_printf_loc (MSG_NOTE, vect_location,
2951 : "Alignment of access forced using versioning.\n");
2952 : }
2953 :
2954 5508 : if (do_peeling)
2955 : {
2956 : /* This point is reached if peeling and versioning are used together
2957 : to ensure alignment. Update data structures to make sure the loop
2958 : is correctly peeled and a right runtime check is added for loop
2959 : versioning. */
2960 1072 : gcc_assert (try_peeling_with_versioning);
2961 1072 : LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
2962 1072 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = -1;
2963 1072 : LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = true;
2964 1072 : if (dump_enabled_p ())
2965 11 : dump_printf_loc (MSG_NOTE, vect_location,
2966 : "Both peeling and versioning will be applied.\n");
2967 : }
2968 : else
2969 : {
2970 : /* This point is reached if versioning is used alone. */
2971 4436 : LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = false;
2972 4436 : if (dump_enabled_p ())
2973 88 : dump_printf_loc (MSG_NOTE, vect_location,
2974 : "Versioning for alignment will be applied.\n");
2975 : }
2976 :
2977 5508 : return opt_result::success ();
2978 : }
2979 :
2980 : /* This point is reached if neither peeling nor versioning is being done. */
2981 371784 : gcc_assert (! (do_peeling || do_versioning));
2982 :
2983 371784 : return opt_result::success ();
2984 795400 : }
2985 :
2986 :
2987 : /* Function vect_analyze_data_refs_alignment
2988 :
2989 : Analyze the alignment of the data-references in the loop. */
2990 :
2991 : void
2992 438341 : vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
2993 : {
2994 438341 : DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment");
2995 :
2996 438341 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
2997 438341 : struct data_reference *dr;
2998 438341 : unsigned int i;
2999 :
3000 438341 : vect_record_base_alignments (loop_vinfo);
3001 2297844 : FOR_EACH_VEC_ELT (datarefs, i, dr)
3002 : {
3003 998071 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
3004 998071 : if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
3005 : {
3006 998071 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)
3007 1291994 : && DR_GROUP_FIRST_ELEMENT (dr_info->stmt) != dr_info->stmt)
3008 131624 : continue;
3009 :
3010 866447 : vect_compute_data_ref_alignment (loop_vinfo, dr_info,
3011 : STMT_VINFO_VECTYPE (dr_info->stmt));
3012 : }
3013 : }
3014 438341 : }
3015 :
3016 :
3017 : /* Analyze alignment of DRs of stmts in NODE. */
3018 :
3019 : static bool
3020 858353 : vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
3021 : {
3022 : /* Alignment is maintained in the first element of the group. */
3023 858353 : stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
3024 858353 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
3025 858353 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
3026 858353 : tree vectype = SLP_TREE_VECTYPE (node);
3027 858353 : poly_uint64 vector_alignment
3028 858353 : = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
3029 : BITS_PER_UNIT);
3030 858353 : if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
3031 805311 : vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
3032 : /* Re-analyze alignment when we're facing a vectorization with a bigger
3033 : alignment requirement. */
3034 53042 : else if (known_lt (dr_info->target_alignment, vector_alignment))
3035 : {
3036 93 : poly_uint64 old_target_alignment = dr_info->target_alignment;
3037 93 : int old_misalignment = dr_info->misalignment;
3038 93 : vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
3039 : /* But keep knowledge about a smaller alignment. */
3040 93 : if (old_misalignment != DR_MISALIGNMENT_UNKNOWN
3041 39 : && dr_info->misalignment == DR_MISALIGNMENT_UNKNOWN)
3042 : {
3043 4 : dr_info->target_alignment = old_target_alignment;
3044 4 : dr_info->misalignment = old_misalignment;
3045 : }
3046 : }
3047 : /* When we ever face unordered target alignments the first one wins in terms
3048 : of analyzing and the other will become unknown in dr_misalignment. */
3049 858353 : return true;
3050 : }
3051 :
3052 : /* Function vect_slp_analyze_instance_alignment
3053 :
3054 : Analyze the alignment of the data-references in the SLP instance.
3055 : Return FALSE if a data reference is found that cannot be vectorized. */
3056 :
3057 : bool
3058 834421 : vect_slp_analyze_instance_alignment (vec_info *vinfo,
3059 : slp_instance instance)
3060 : {
3061 834421 : DUMP_VECT_SCOPE ("vect_slp_analyze_instance_alignment");
3062 :
3063 834421 : slp_tree node;
3064 834421 : unsigned i;
3065 1015151 : FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
3066 180730 : if (! vect_slp_analyze_node_alignment (vinfo, node))
3067 : return false;
3068 :
3069 834421 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
3070 834421 : && ! vect_slp_analyze_node_alignment
3071 677623 : (vinfo, SLP_INSTANCE_TREE (instance)))
3072 : return false;
3073 :
3074 : return true;
3075 : }
3076 :
3077 :
3078 : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
3079 : accesses of legal size, step, etc. Detect gaps, single element
3080 : interleaving, and other special cases. Set grouped access info.
3081 : Collect groups of strided stores for further use in SLP analysis.
3082 : Worker for vect_analyze_group_access. */
3083 :
3084 : static bool
3085 13342359 : vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
3086 : {
3087 13342359 : data_reference *dr = dr_info->dr;
3088 13342359 : tree step = DR_STEP (dr);
3089 13342359 : tree scalar_type = TREE_TYPE (DR_REF (dr));
3090 13342359 : HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
3091 13342359 : stmt_vec_info stmt_info = dr_info->stmt;
3092 13342359 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3093 13342359 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3094 13342359 : HOST_WIDE_INT dr_step = -1;
3095 13342359 : HOST_WIDE_INT groupsize, last_accessed_element = 1;
3096 13342359 : bool slp_impossible = false;
3097 :
3098 : /* For interleaving, GROUPSIZE is STEP counted in elements, i.e., the
3099 : size of the interleaving group (including gaps). */
3100 13342359 : if (tree_fits_shwi_p (step))
3101 : {
3102 13332711 : dr_step = tree_to_shwi (step);
3103 : /* Check that STEP is a multiple of type size. Otherwise there is
3104 : a non-element-sized gap at the end of the group which we
3105 : cannot represent in DR_GROUP_GAP or DR_GROUP_SIZE.
3106 : ??? As we can handle non-constant step fine here we should
3107 : simply remove uses of DR_GROUP_GAP between the last and first
3108 : element and instead rely on DR_STEP. DR_GROUP_SIZE then would
3109 : simply not include that gap. */
3110 13332711 : if ((dr_step % type_size) != 0)
3111 : {
3112 502 : if (dump_enabled_p ())
3113 27 : dump_printf_loc (MSG_NOTE, vect_location,
3114 : "Step %T is not a multiple of the element size"
3115 : " for %T\n",
3116 : step, DR_REF (dr));
3117 : return false;
3118 : }
3119 13332209 : groupsize = absu_hwi (dr_step) / type_size;
3120 : }
3121 : else
3122 : groupsize = 0;
3123 :
3124 : /* Not consecutive access is possible only if it is a part of interleaving. */
3125 13341857 : if (!DR_GROUP_FIRST_ELEMENT (stmt_info))
3126 : {
3127 : /* Check if it this DR is a part of interleaving, and is a single
3128 : element of the group that is accessed in the loop. */
3129 :
3130 : /* Gaps are supported only for loads. STEP must be a multiple of the type
3131 : size. */
3132 8966246 : if (DR_IS_READ (dr)
3133 5377378 : && (dr_step % type_size) == 0
3134 : && groupsize > 0
3135 : /* This could be UINT_MAX but as we are generating code in a very
3136 : inefficient way we have to cap earlier.
3137 : See PR91403 for example. */
3138 5377378 : && groupsize <= 4096)
3139 : {
3140 73726 : DR_GROUP_FIRST_ELEMENT (stmt_info) = stmt_info;
3141 73726 : DR_GROUP_SIZE (stmt_info) = groupsize;
3142 73726 : DR_GROUP_GAP (stmt_info) = groupsize - 1;
3143 73726 : if (dump_enabled_p ())
3144 1501 : dump_printf_loc (MSG_NOTE, vect_location,
3145 : "Detected single element interleaving %T"
3146 : " step %T\n",
3147 : DR_REF (dr), step);
3148 :
3149 : return true;
3150 : }
3151 :
3152 8892520 : if (dump_enabled_p ())
3153 3373 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3154 : "not consecutive access %G", stmt_info->stmt);
3155 :
3156 8892520 : if (bb_vinfo)
3157 : {
3158 : /* Mark the statement as unvectorizable. */
3159 8873634 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
3160 8873634 : return true;
3161 : }
3162 :
3163 18886 : if (dump_enabled_p ())
3164 314 : dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
3165 18886 : STMT_VINFO_STRIDED_P (stmt_info) = true;
3166 18886 : return true;
3167 : }
3168 :
3169 4375611 : if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
3170 : {
3171 : /* First stmt in the interleaving chain. Check the chain. */
3172 1585273 : stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3173 1585273 : struct data_reference *data_ref = dr;
3174 1585273 : unsigned int count = 1;
3175 1585273 : tree prev_init = DR_INIT (data_ref);
3176 1585273 : HOST_WIDE_INT diff, gaps = 0;
3177 :
3178 : /* By construction, all group members have INTEGER_CST DR_INITs. */
3179 4375620 : while (next)
3180 : {
3181 : /* We never have the same DR multiple times. */
3182 2790409 : gcc_assert (tree_int_cst_compare (DR_INIT (data_ref),
3183 : DR_INIT (STMT_VINFO_DATA_REF (next))) != 0);
3184 :
3185 2790409 : data_ref = STMT_VINFO_DATA_REF (next);
3186 :
3187 : /* All group members have the same STEP by construction. */
3188 2790409 : gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0));
3189 :
3190 : /* Check that the distance between two accesses is equal to the type
3191 : size. Otherwise, we have gaps. */
3192 2790409 : diff = (TREE_INT_CST_LOW (DR_INIT (data_ref))
3193 2790409 : - TREE_INT_CST_LOW (prev_init)) / type_size;
3194 2790409 : if (diff < 1 || diff > UINT_MAX)
3195 : {
3196 : /* For artificial testcases with array accesses with large
3197 : constant indices we can run into overflow issues which
3198 : can end up fooling the groupsize constraint below so
3199 : check the individual gaps (which are represented as
3200 : unsigned int) as well. */
3201 0 : if (dump_enabled_p ())
3202 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3203 : "interleaved access with gap larger "
3204 : "than representable\n");
3205 : return false;
3206 : }
3207 2790409 : if (diff != 1)
3208 : {
3209 : /* FORNOW: SLP of accesses with gaps is not supported. */
3210 125895 : slp_impossible = true;
3211 125895 : if (DR_IS_WRITE (data_ref))
3212 : {
3213 62 : if (dump_enabled_p ())
3214 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3215 : "interleaved store with gaps\n");
3216 : return false;
3217 : }
3218 :
3219 125833 : gaps += diff - 1;
3220 : }
3221 :
3222 2790347 : last_accessed_element += diff;
3223 :
3224 : /* Store the gap from the previous member of the group. If there is no
3225 : gap in the access, DR_GROUP_GAP is always 1. */
3226 2790347 : DR_GROUP_GAP (next) = diff;
3227 :
3228 2790347 : prev_init = DR_INIT (data_ref);
3229 2790347 : next = DR_GROUP_NEXT_ELEMENT (next);
3230 : /* Count the number of data-refs in the chain. */
3231 2790347 : count++;
3232 : }
3233 :
3234 1585211 : if (groupsize == 0)
3235 1513730 : groupsize = count + gaps;
3236 :
3237 : /* This could be UINT_MAX but as we are generating code in a very
3238 : inefficient way we have to cap earlier. See PR78699 for example. */
3239 1585211 : if (groupsize > 4096)
3240 : {
3241 1 : if (dump_enabled_p ())
3242 1 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3243 : "group is too large\n");
3244 : return false;
3245 : }
3246 :
3247 : /* Check that the size of the interleaving is equal to count for stores,
3248 : i.e., that there are no gaps. */
3249 1585210 : if (groupsize != count
3250 129125 : && !DR_IS_READ (dr))
3251 : {
3252 12132 : groupsize = count;
3253 12132 : STMT_VINFO_STRIDED_P (stmt_info) = true;
3254 : }
3255 :
3256 : /* If there is a gap after the last load in the group it is the
3257 : difference between the groupsize and the last accessed
3258 : element.
3259 : When there is no gap, this difference should be 0. */
3260 1585210 : DR_GROUP_GAP (stmt_info) = groupsize - last_accessed_element;
3261 :
3262 1585210 : DR_GROUP_SIZE (stmt_info) = groupsize;
3263 1585210 : if (dump_enabled_p ())
3264 : {
3265 8189 : dump_printf_loc (MSG_NOTE, vect_location,
3266 : "Detected interleaving ");
3267 8189 : if (DR_IS_READ (dr))
3268 4421 : dump_printf (MSG_NOTE, "load ");
3269 3768 : else if (STMT_VINFO_STRIDED_P (stmt_info))
3270 466 : dump_printf (MSG_NOTE, "strided store ");
3271 : else
3272 3302 : dump_printf (MSG_NOTE, "store ");
3273 8189 : dump_printf (MSG_NOTE, "of size %u\n",
3274 : (unsigned)groupsize);
3275 8189 : dump_printf_loc (MSG_NOTE, vect_location, "\t%G", stmt_info->stmt);
3276 8189 : next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3277 40117 : while (next)
3278 : {
3279 31928 : if (DR_GROUP_GAP (next) != 1)
3280 329 : dump_printf_loc (MSG_NOTE, vect_location,
3281 : "\t<gap of %d elements>\n",
3282 329 : DR_GROUP_GAP (next) - 1);
3283 31928 : dump_printf_loc (MSG_NOTE, vect_location, "\t%G", next->stmt);
3284 31928 : next = DR_GROUP_NEXT_ELEMENT (next);
3285 : }
3286 8189 : if (DR_GROUP_GAP (stmt_info) != 0)
3287 404 : dump_printf_loc (MSG_NOTE, vect_location,
3288 : "\t<gap of %d elements>\n",
3289 404 : DR_GROUP_GAP (stmt_info));
3290 : }
3291 :
3292 : /* SLP: create an SLP data structure for every interleaving group of
3293 : stores for further analysis in vect_analyse_slp. */
3294 1585210 : if (DR_IS_WRITE (dr) && !slp_impossible)
3295 : {
3296 948848 : if (loop_vinfo)
3297 30305 : LOOP_VINFO_GROUPED_STORES (loop_vinfo).safe_push (stmt_info);
3298 948848 : if (bb_vinfo)
3299 918543 : BB_VINFO_GROUPED_STORES (bb_vinfo).safe_push (stmt_info);
3300 : }
3301 : }
3302 :
3303 : return true;
3304 : }
3305 :
3306 : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
3307 : accesses of legal size, step, etc. Detect gaps, single element
3308 : interleaving, and other special cases. Set grouped access info.
3309 : Collect groups of strided stores for further use in SLP analysis. */
3310 :
3311 : static bool
3312 13342359 : vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
3313 : {
3314 13342359 : if (!vect_analyze_group_access_1 (vinfo, dr_info))
3315 : {
3316 : /* Dissolve the group if present. */
3317 565 : stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
3318 796 : while (stmt_info)
3319 : {
3320 231 : stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3321 231 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3322 231 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3323 231 : stmt_info = next;
3324 : }
3325 : return false;
3326 : }
3327 : return true;
3328 : }
3329 :
3330 : /* Analyze the access pattern of the data-reference DR_INFO.
3331 : In case of non-consecutive accesses call vect_analyze_group_access() to
3332 : analyze groups of accesses. */
3333 :
3334 : static bool
3335 14125829 : vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
3336 : {
3337 14125829 : data_reference *dr = dr_info->dr;
3338 14125829 : tree step = DR_STEP (dr);
3339 14125829 : tree scalar_type = TREE_TYPE (DR_REF (dr));
3340 14125829 : stmt_vec_info stmt_info = dr_info->stmt;
3341 14125829 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3342 14125829 : class loop *loop = NULL;
3343 :
3344 14125829 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
3345 : return true;
3346 :
3347 14019778 : if (loop_vinfo)
3348 996315 : loop = LOOP_VINFO_LOOP (loop_vinfo);
3349 :
3350 14019778 : if (loop_vinfo && !step)
3351 : {
3352 0 : if (dump_enabled_p ())
3353 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3354 : "bad data-ref access in loop\n");
3355 : return false;
3356 : }
3357 :
3358 : /* Allow loads with zero step in inner-loop vectorization. */
3359 14019778 : if (loop_vinfo && integer_zerop (step))
3360 : {
3361 14140 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3362 14140 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3363 14140 : if (!nested_in_vect_loop_p (loop, stmt_info))
3364 13880 : return DR_IS_READ (dr);
3365 : /* Allow references with zero step for outer loops marked
3366 : with pragma omp simd only - it guarantees absence of
3367 : loop-carried dependencies between inner loop iterations. */
3368 260 : if (loop->safelen < 2)
3369 : {
3370 224 : if (dump_enabled_p ())
3371 6 : dump_printf_loc (MSG_NOTE, vect_location,
3372 : "zero step in inner loop of nest\n");
3373 : return false;
3374 : }
3375 : }
3376 :
3377 14005638 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
3378 : {
3379 : /* Interleaved accesses are not yet supported within outer-loop
3380 : vectorization for references in the inner-loop. */
3381 5888 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3382 5888 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3383 :
3384 : /* For the rest of the analysis we use the outer-loop step. */
3385 5888 : step = STMT_VINFO_DR_STEP (stmt_info);
3386 5888 : if (integer_zerop (step))
3387 : {
3388 1294 : if (dump_enabled_p ())
3389 241 : dump_printf_loc (MSG_NOTE, vect_location,
3390 : "zero step in outer loop.\n");
3391 1294 : return DR_IS_READ (dr);
3392 : }
3393 : }
3394 :
3395 : /* Consecutive? */
3396 14004380 : if (TREE_CODE (step) == INTEGER_CST)
3397 : {
3398 13964707 : HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
3399 13964707 : if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type))
3400 13964707 : || (dr_step < 0
3401 28325 : && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step)))
3402 : {
3403 : /* Mark that it is not interleaving. */
3404 629063 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3405 629063 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3406 629063 : return true;
3407 : }
3408 : }
3409 :
3410 13375317 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
3411 : {
3412 3393 : if (dump_enabled_p ())
3413 163 : dump_printf_loc (MSG_NOTE, vect_location,
3414 : "grouped access in outer loop.\n");
3415 : return false;
3416 : }
3417 :
3418 :
3419 : /* Assume this is a DR handled by non-constant strided load case. */
3420 13371924 : if (TREE_CODE (step) != INTEGER_CST)
3421 39213 : return (STMT_VINFO_STRIDED_P (stmt_info)
3422 39213 : && (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
3423 9648 : || vect_analyze_group_access (vinfo, dr_info)));
3424 :
3425 : /* Not consecutive access - check if it's a part of interleaving group. */
3426 13332711 : return vect_analyze_group_access (vinfo, dr_info);
3427 : }
3428 :
3429 : /* Compare two data-references DRA and DRB to group them into chunks
3430 : suitable for grouping. */
3431 :
3432 : static int
3433 375202200 : dr_group_sort_cmp (const void *dra_, const void *drb_)
3434 : {
3435 375202200 : dr_vec_info *dra_info = *(dr_vec_info **)const_cast<void *>(dra_);
3436 375202200 : dr_vec_info *drb_info = *(dr_vec_info **)const_cast<void *>(drb_);
3437 375202200 : data_reference_p dra = dra_info->dr;
3438 375202200 : data_reference_p drb = drb_info->dr;
3439 375202200 : int cmp;
3440 :
3441 : /* Stabilize sort. */
3442 375202200 : if (dra == drb)
3443 : return 0;
3444 :
3445 : /* Different group IDs lead never belong to the same group. */
3446 375202200 : if (dra_info->group != drb_info->group)
3447 269154184 : return dra_info->group < drb_info->group ? -1 : 1;
3448 :
3449 : /* Ordering of DRs according to base. */
3450 106048016 : cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
3451 : DR_BASE_ADDRESS (drb));
3452 106048016 : if (cmp != 0)
3453 : return cmp;
3454 :
3455 : /* And according to DR_OFFSET. */
3456 56545628 : cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
3457 56545628 : if (cmp != 0)
3458 : return cmp;
3459 :
3460 : /* Put reads before writes. */
3461 56181676 : if (DR_IS_READ (dra) != DR_IS_READ (drb))
3462 3062457 : return DR_IS_READ (dra) ? -1 : 1;
3463 :
3464 : /* Then sort after access size. */
3465 53119219 : cmp = data_ref_compare_tree (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra))),
3466 53119219 : TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb))));
3467 53119219 : if (cmp != 0)
3468 : return cmp;
3469 :
3470 : /* And after step. */
3471 46064509 : cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
3472 46064509 : if (cmp != 0)
3473 : return cmp;
3474 :
3475 : /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */
3476 46056931 : cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
3477 46056931 : if (cmp == 0)
3478 514347 : return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
3479 : return cmp;
3480 : }
3481 :
3482 : /* If OP is the result of a conversion, return the unconverted value,
3483 : otherwise return null. */
3484 :
3485 : static tree
3486 342 : strip_conversion (tree op)
3487 : {
3488 342 : if (TREE_CODE (op) != SSA_NAME)
3489 : return NULL_TREE;
3490 342 : gimple *stmt = SSA_NAME_DEF_STMT (op);
3491 342 : if (!is_gimple_assign (stmt)
3492 342 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)))
3493 : return NULL_TREE;
3494 186 : return gimple_assign_rhs1 (stmt);
3495 : }
3496 :
3497 : /* Return true if vectorizable_* routines can handle statements STMT1_INFO
3498 : and STMT2_INFO being in a single group. When ALLOW_SLP_P, masked loads can
3499 : be grouped in SLP mode. */
3500 :
3501 : static bool
3502 7427720 : can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info,
3503 : bool allow_slp_p)
3504 : {
3505 7427720 : if (gimple_assign_single_p (stmt1_info->stmt))
3506 7426328 : return gimple_assign_single_p (stmt2_info->stmt);
3507 :
3508 1392 : gcall *call1 = dyn_cast <gcall *> (stmt1_info->stmt);
3509 1392 : if (call1 && gimple_call_internal_p (call1))
3510 : {
3511 : /* Check for two masked loads or two masked stores. */
3512 1392 : gcall *call2 = dyn_cast <gcall *> (stmt2_info->stmt);
3513 1376 : if (!call2 || !gimple_call_internal_p (call2))
3514 : return false;
3515 1376 : internal_fn ifn = gimple_call_internal_fn (call1);
3516 1376 : if (ifn != IFN_MASK_LOAD && ifn != IFN_MASK_STORE)
3517 : return false;
3518 1376 : if (ifn != gimple_call_internal_fn (call2))
3519 : return false;
3520 :
3521 : /* Check that the masks are the same. Cope with casts of masks,
3522 : like those created by build_mask_conversion. */
3523 1376 : tree mask1 = gimple_call_arg (call1, 2);
3524 1376 : tree mask2 = gimple_call_arg (call2, 2);
3525 1376 : if (!operand_equal_p (mask1, mask2, 0) && !allow_slp_p)
3526 : {
3527 249 : mask1 = strip_conversion (mask1);
3528 249 : if (!mask1)
3529 : return false;
3530 93 : mask2 = strip_conversion (mask2);
3531 93 : if (!mask2)
3532 : return false;
3533 93 : if (!operand_equal_p (mask1, mask2, 0))
3534 : return false;
3535 : }
3536 1183 : return true;
3537 : }
3538 :
3539 : return false;
3540 : }
3541 :
3542 : /* Function vect_analyze_data_ref_accesses.
3543 :
3544 : Analyze the access pattern of all the data references in the loop.
3545 :
3546 : FORNOW: the only access pattern that is considered vectorizable is a
3547 : simple step 1 (consecutive) access.
3548 :
3549 : FORNOW: handle only arrays and pointer accesses. */
3550 :
3551 : opt_result
3552 2807910 : vect_analyze_data_ref_accesses (vec_info *vinfo,
3553 : vec<int> *dataref_groups)
3554 : {
3555 2807910 : unsigned int i;
3556 2807910 : vec<data_reference_p> datarefs = vinfo->shared->datarefs;
3557 :
3558 2807910 : DUMP_VECT_SCOPE ("vect_analyze_data_ref_accesses");
3559 :
3560 2807910 : if (datarefs.is_empty ())
3561 1128262 : return opt_result::success ();
3562 :
3563 : /* Sort the array of datarefs to make building the interleaving chains
3564 : linear. Don't modify the original vector's order, it is needed for
3565 : determining what dependencies are reversed. */
3566 1679648 : vec<dr_vec_info *> datarefs_copy;
3567 1679648 : datarefs_copy.create (datarefs.length ());
3568 19594762 : for (unsigned i = 0; i < datarefs.length (); i++)
3569 : {
3570 16235466 : dr_vec_info *dr_info = vinfo->lookup_dr (datarefs[i]);
3571 : /* If the caller computed DR grouping use that, otherwise group by
3572 : basic blocks. */
3573 16235466 : if (dataref_groups)
3574 15118864 : dr_info->group = (*dataref_groups)[i];
3575 : else
3576 1116602 : dr_info->group = gimple_bb (DR_STMT (datarefs[i]))->index;
3577 16235466 : datarefs_copy.quick_push (dr_info);
3578 : }
3579 1679648 : datarefs_copy.qsort (dr_group_sort_cmp);
3580 1679648 : hash_set<stmt_vec_info> to_fixup;
3581 :
3582 : /* Build the interleaving chains. */
3583 15337043 : for (i = 0; i < datarefs_copy.length () - 1;)
3584 : {
3585 11977747 : dr_vec_info *dr_info_a = datarefs_copy[i];
3586 11977747 : data_reference_p dra = dr_info_a->dr;
3587 11977747 : int dra_group_id = dr_info_a->group;
3588 11977747 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
3589 11977747 : stmt_vec_info lastinfo = NULL;
3590 11977747 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
3591 10042726 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a))
3592 : {
3593 2003896 : ++i;
3594 2003896 : continue;
3595 : }
3596 26457036 : for (i = i + 1; i < datarefs_copy.length (); ++i)
3597 : {
3598 12551922 : dr_vec_info *dr_info_b = datarefs_copy[i];
3599 12551922 : data_reference_p drb = dr_info_b->dr;
3600 12551922 : int drb_group_id = dr_info_b->group;
3601 12551922 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
3602 12551922 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b)
3603 12201417 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
3604 : break;
3605 :
3606 : /* ??? Imperfect sorting (non-compatible types, non-modulo
3607 : accesses, same accesses) can lead to a group to be artificially
3608 : split here as we don't just skip over those. If it really
3609 : matters we can push those to a worklist and re-iterate
3610 : over them. The we can just skip ahead to the next DR here. */
3611 :
3612 : /* DRs in a different DR group should not be put into the same
3613 : interleaving group. */
3614 12197745 : if (dra_group_id != drb_group_id)
3615 : break;
3616 :
3617 : /* Check that the data-refs have same first location (except init)
3618 : and they are both either store or load (not load and store,
3619 : not masked loads or stores). */
3620 7703119 : if (DR_IS_READ (dra) != DR_IS_READ (drb)
3621 6336568 : || data_ref_compare_tree (DR_BASE_ADDRESS (dra),
3622 : DR_BASE_ADDRESS (drb)) != 0
3623 4622046 : || data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)) != 0
3624 12305319 : || !can_group_stmts_p (stmtinfo_a, stmtinfo_b, true))
3625 : break;
3626 :
3627 : /* Check that the data-refs have the same constant size. */
3628 4602175 : tree sza = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra)));
3629 4602175 : tree szb = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb)));
3630 4602175 : if (!tree_fits_uhwi_p (sza)
3631 4602175 : || !tree_fits_uhwi_p (szb)
3632 9204350 : || !tree_int_cst_equal (sza, szb))
3633 : break;
3634 :
3635 : /* Check that the data-refs have the same step. */
3636 4237870 : if (data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb)) != 0)
3637 : break;
3638 :
3639 : /* Check the types are compatible.
3640 : ??? We don't distinguish this during sorting. */
3641 4237102 : if (!types_compatible_p (TREE_TYPE (DR_REF (dra)),
3642 4237102 : TREE_TYPE (DR_REF (drb))))
3643 : break;
3644 :
3645 : /* Check that the DR_INITs are compile-time constants. */
3646 3049578 : if (!tree_fits_shwi_p (DR_INIT (dra))
3647 3049578 : || !tree_fits_shwi_p (DR_INIT (drb)))
3648 : break;
3649 :
3650 : /* Different .GOMP_SIMD_LANE calls still give the same lane,
3651 : just hold extra information. */
3652 3049578 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
3653 1240 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
3654 3050818 : && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
3655 : break;
3656 :
3657 : /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb). */
3658 3048338 : HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
3659 3048338 : HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
3660 3048338 : HOST_WIDE_INT init_prev
3661 3048338 : = TREE_INT_CST_LOW (DR_INIT (datarefs_copy[i-1]->dr));
3662 3048338 : gcc_assert (init_a <= init_b
3663 : && init_a <= init_prev
3664 : && init_prev <= init_b);
3665 :
3666 : /* Do not place the same access in the interleaving chain twice. */
3667 3048338 : if (init_b == init_prev)
3668 : {
3669 31572 : gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1]->dr))
3670 : < gimple_uid (DR_STMT (drb)));
3671 : /* Simply link in duplicates and fix up the chain below. */
3672 : }
3673 : else
3674 : {
3675 : /* If init_b == init_a + the size of the type * k, we have an
3676 : interleaving, and DRA is accessed before DRB. */
3677 3016766 : unsigned HOST_WIDE_INT type_size_a = tree_to_uhwi (sza);
3678 3016766 : if (type_size_a == 0
3679 3016766 : || (((unsigned HOST_WIDE_INT)init_b - init_a)
3680 3016766 : % type_size_a != 0))
3681 : break;
3682 :
3683 : /* If we have a store, the accesses are adjacent. This splits
3684 : groups into chunks we support (we don't support vectorization
3685 : of stores with gaps). */
3686 3014995 : if (!DR_IS_READ (dra)
3687 1928363 : && (((unsigned HOST_WIDE_INT)init_b - init_prev)
3688 : != type_size_a))
3689 : break;
3690 :
3691 : /* For datarefs with big gap, it's better to split them into different
3692 : groups.
3693 : .i.e a[0], a[1], a[2], .. a[7], a[100], a[101],..., a[107] */
3694 2826646 : if ((unsigned HOST_WIDE_INT)(init_b - init_prev)
3695 : > MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT)
3696 : break;
3697 :
3698 : /* If the step (if not zero or non-constant) is smaller than the
3699 : difference between data-refs' inits this splits groups into
3700 : suitable sizes. */
3701 2816655 : if (tree_fits_shwi_p (DR_STEP (dra)))
3702 : {
3703 2810294 : unsigned HOST_WIDE_INT step
3704 2810294 : = absu_hwi (tree_to_shwi (DR_STEP (dra)));
3705 2810294 : if (step != 0
3706 170370 : && step <= ((unsigned HOST_WIDE_INT)init_b - init_a))
3707 : break;
3708 : }
3709 : }
3710 :
3711 2825790 : if (dump_enabled_p ())
3712 32846 : dump_printf_loc (MSG_NOTE, vect_location,
3713 32846 : DR_IS_READ (dra)
3714 : ? "Detected interleaving load %T and %T\n"
3715 : : "Detected interleaving store %T and %T\n",
3716 : DR_REF (dra), DR_REF (drb));
3717 :
3718 : /* Link the found element into the group list. */
3719 2825790 : if (!DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
3720 : {
3721 1562092 : DR_GROUP_FIRST_ELEMENT (stmtinfo_a) = stmtinfo_a;
3722 1562092 : lastinfo = stmtinfo_a;
3723 : }
3724 2825790 : DR_GROUP_FIRST_ELEMENT (stmtinfo_b) = stmtinfo_a;
3725 2825790 : DR_GROUP_NEXT_ELEMENT (lastinfo) = stmtinfo_b;
3726 2825790 : lastinfo = stmtinfo_b;
3727 :
3728 2825790 : if (! STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
3729 : {
3730 2825520 : STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a)
3731 2825520 : = !can_group_stmts_p (stmtinfo_a, stmtinfo_b, false);
3732 :
3733 2825520 : if (dump_enabled_p () && STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
3734 114 : dump_printf_loc (MSG_NOTE, vect_location,
3735 : "Load suitable for SLP vectorization only.\n");
3736 : }
3737 :
3738 2825790 : if (init_b == init_prev
3739 31572 : && !to_fixup.add (DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
3740 2843738 : && dump_enabled_p ())
3741 214 : dump_printf_loc (MSG_NOTE, vect_location,
3742 : "Queuing group with duplicate access for fixup\n");
3743 : }
3744 : }
3745 :
3746 : /* Fixup groups with duplicate entries by splitting it. */
3747 1725776 : while (1)
3748 : {
3749 1725776 : hash_set<stmt_vec_info>::iterator it = to_fixup.begin ();
3750 1725776 : if (!(it != to_fixup.end ()))
3751 : break;
3752 46128 : stmt_vec_info grp = *it;
3753 46128 : to_fixup.remove (grp);
3754 :
3755 : /* Find the earliest duplicate group member. */
3756 46128 : unsigned first_duplicate = -1u;
3757 46128 : stmt_vec_info next, g = grp;
3758 285235 : while ((next = DR_GROUP_NEXT_ELEMENT (g)))
3759 : {
3760 192979 : if (tree_int_cst_equal (DR_INIT (STMT_VINFO_DR_INFO (next)->dr),
3761 192979 : DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
3762 192979 : && gimple_uid (STMT_VINFO_STMT (next)) < first_duplicate)
3763 : first_duplicate = gimple_uid (STMT_VINFO_STMT (next));
3764 192979 : g = next;
3765 : }
3766 46128 : if (first_duplicate == -1U)
3767 17948 : continue;
3768 :
3769 : /* Then move all stmts after the first duplicate to a new group.
3770 : Note this is a heuristic but one with the property that *it
3771 : is fixed up completely. */
3772 28180 : g = grp;
3773 28180 : stmt_vec_info newgroup = NULL, ng = grp;
3774 246779 : while ((next = DR_GROUP_NEXT_ELEMENT (g)))
3775 : {
3776 190419 : if (gimple_uid (STMT_VINFO_STMT (next)) >= first_duplicate)
3777 : {
3778 183986 : DR_GROUP_NEXT_ELEMENT (g) = DR_GROUP_NEXT_ELEMENT (next);
3779 183986 : if (!newgroup)
3780 : {
3781 28180 : newgroup = next;
3782 28180 : STMT_VINFO_SLP_VECT_ONLY (newgroup)
3783 28180 : = STMT_VINFO_SLP_VECT_ONLY (grp);
3784 : }
3785 : else
3786 155806 : DR_GROUP_NEXT_ELEMENT (ng) = next;
3787 183986 : ng = next;
3788 183986 : DR_GROUP_FIRST_ELEMENT (ng) = newgroup;
3789 : }
3790 : else
3791 : g = DR_GROUP_NEXT_ELEMENT (g);
3792 : }
3793 28180 : DR_GROUP_NEXT_ELEMENT (ng) = NULL;
3794 :
3795 : /* Fixup the new group which still may contain duplicates. */
3796 28180 : to_fixup.add (newgroup);
3797 : }
3798 :
3799 1679648 : dr_vec_info *dr_info;
3800 17892809 : FOR_EACH_VEC_ELT (datarefs_copy, i, dr_info)
3801 : {
3802 16221230 : if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
3803 16221230 : && !vect_analyze_data_ref_access (vinfo, dr_info))
3804 : {
3805 8123 : if (dump_enabled_p ())
3806 292 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3807 : "not vectorized: complicated access pattern.\n");
3808 :
3809 8123 : if (is_a <bb_vec_info> (vinfo))
3810 : {
3811 : /* Mark the statement as not vectorizable. */
3812 54 : STMT_VINFO_VECTORIZABLE (dr_info->stmt) = false;
3813 54 : continue;
3814 : }
3815 : else
3816 : {
3817 8069 : datarefs_copy.release ();
3818 8069 : return opt_result::failure_at (dr_info->stmt->stmt,
3819 : "not vectorized:"
3820 : " complicated access pattern.\n");
3821 : }
3822 : }
3823 : }
3824 :
3825 1671579 : datarefs_copy.release ();
3826 1671579 : return opt_result::success ();
3827 1679648 : }
3828 :
3829 : /* Function vect_vfa_segment_size.
3830 :
3831 : Input:
3832 : DR_INFO: The data reference.
3833 : LENGTH_FACTOR: segment length to consider.
3834 :
3835 : Return a value suitable for the dr_with_seg_len::seg_len field.
3836 : This is the "distance travelled" by the pointer from the first
3837 : iteration in the segment to the last. Note that it does not include
3838 : the size of the access; in effect it only describes the first byte. */
3839 :
3840 : static tree
3841 151724 : vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
3842 : {
3843 151724 : length_factor = size_binop (MINUS_EXPR,
3844 : fold_convert (sizetype, length_factor),
3845 : size_one_node);
3846 151724 : return size_binop (MULT_EXPR, fold_convert (sizetype, DR_STEP (dr_info->dr)),
3847 : length_factor);
3848 : }
3849 :
3850 : /* Return a value that, when added to abs (vect_vfa_segment_size (DR_INFO)),
3851 : gives the worst-case number of bytes covered by the segment. */
3852 :
3853 : static unsigned HOST_WIDE_INT
3854 152206 : vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
3855 : {
3856 152206 : stmt_vec_info stmt_vinfo = dr_info->stmt;
3857 152206 : tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
3858 152206 : unsigned HOST_WIDE_INT ref_size = tree_to_uhwi (TYPE_SIZE_UNIT (ref_type));
3859 152206 : unsigned HOST_WIDE_INT access_size = ref_size;
3860 152206 : if (DR_GROUP_FIRST_ELEMENT (stmt_vinfo))
3861 : {
3862 43509 : gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == stmt_vinfo);
3863 43509 : access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
3864 : }
3865 152206 : tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
3866 152206 : int misalignment;
3867 304412 : if (((misalignment = dr_misalignment (dr_info, vectype)), true)
3868 152206 : && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
3869 : == dr_explicit_realign_optimized))
3870 : {
3871 : /* We might access a full vector's worth. */
3872 0 : access_size += tree_to_uhwi (TYPE_SIZE_UNIT (vectype)) - ref_size;
3873 : }
3874 152206 : return access_size;
3875 : }
3876 :
3877 : /* Get the minimum alignment for all the scalar accesses that DR_INFO
3878 : describes. */
3879 :
3880 : static unsigned int
3881 152206 : vect_vfa_align (dr_vec_info *dr_info)
3882 : {
3883 0 : return dr_alignment (dr_info->dr);
3884 : }
3885 :
3886 : /* Function vect_no_alias_p.
3887 :
3888 : Given data references A and B with equal base and offset, see whether
3889 : the alias relation can be decided at compilation time. Return 1 if
3890 : it can and the references alias, 0 if it can and the references do
3891 : not alias, and -1 if we cannot decide at compile time. SEGMENT_LENGTH_A,
3892 : SEGMENT_LENGTH_B, ACCESS_SIZE_A and ACCESS_SIZE_B are the equivalent
3893 : of dr_with_seg_len::{seg_len,access_size} for A and B. */
3894 :
3895 : static int
3896 4590 : vect_compile_time_alias (dr_vec_info *a, dr_vec_info *b,
3897 : tree segment_length_a, tree segment_length_b,
3898 : unsigned HOST_WIDE_INT access_size_a,
3899 : unsigned HOST_WIDE_INT access_size_b)
3900 : {
3901 4590 : poly_offset_int offset_a = wi::to_poly_offset (DR_INIT (a->dr));
3902 4590 : poly_offset_int offset_b = wi::to_poly_offset (DR_INIT (b->dr));
3903 4590 : poly_uint64 const_length_a;
3904 4590 : poly_uint64 const_length_b;
3905 :
3906 : /* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
3907 : bytes, e.g., int a[3] -> a[1] range is [a+4, a+16) instead of
3908 : [a, a+12) */
3909 4590 : if (tree_int_cst_compare (DR_STEP (a->dr), size_zero_node) < 0)
3910 : {
3911 228 : const_length_a = (-wi::to_poly_wide (segment_length_a)).force_uhwi ();
3912 228 : offset_a -= const_length_a;
3913 : }
3914 : else
3915 4362 : const_length_a = tree_to_poly_uint64 (segment_length_a);
3916 4590 : if (tree_int_cst_compare (DR_STEP (b->dr), size_zero_node) < 0)
3917 : {
3918 386 : const_length_b = (-wi::to_poly_wide (segment_length_b)).force_uhwi ();
3919 386 : offset_b -= const_length_b;
3920 : }
3921 : else
3922 4204 : const_length_b = tree_to_poly_uint64 (segment_length_b);
3923 :
3924 4590 : const_length_a += access_size_a;
3925 4590 : const_length_b += access_size_b;
3926 :
3927 4590 : if (ranges_known_overlap_p (offset_a, const_length_a,
3928 : offset_b, const_length_b))
3929 : return 1;
3930 :
3931 536 : if (!ranges_maybe_overlap_p (offset_a, const_length_a,
3932 : offset_b, const_length_b))
3933 536 : return 0;
3934 :
3935 : return -1;
3936 : }
3937 :
3938 : /* Return true if the minimum nonzero dependence distance for loop LOOP_DEPTH
3939 : in DDR is >= VF. */
3940 :
3941 : static bool
3942 89019 : dependence_distance_ge_vf (data_dependence_relation *ddr,
3943 : unsigned int loop_depth, poly_uint64 vf)
3944 : {
3945 89019 : if (DDR_ARE_DEPENDENT (ddr) != NULL_TREE
3946 89019 : || DDR_NUM_DIST_VECTS (ddr) == 0)
3947 : return false;
3948 :
3949 : /* If the dependence is exact, we should have limited the VF instead. */
3950 6004 : gcc_checking_assert (DDR_COULD_BE_INDEPENDENT_P (ddr));
3951 :
3952 : unsigned int i;
3953 : lambda_vector dist_v;
3954 12039 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
3955 : {
3956 12004 : HOST_WIDE_INT dist = dist_v[loop_depth];
3957 12004 : if (dist != 0
3958 6004 : && !(dist > 0 && DDR_REVERSED_P (ddr))
3959 18008 : && maybe_lt ((unsigned HOST_WIDE_INT) abs_hwi (dist), vf))
3960 : return false;
3961 : }
3962 :
3963 35 : if (dump_enabled_p ())
3964 2 : dump_printf_loc (MSG_NOTE, vect_location,
3965 : "dependence distance between %T and %T is >= VF\n",
3966 2 : DR_REF (DDR_A (ddr)), DR_REF (DDR_B (ddr)));
3967 :
3968 : return true;
3969 : }
3970 :
3971 : /* Dump LOWER_BOUND using flags DUMP_KIND. Dumps are known to be enabled. */
3972 :
3973 : static void
3974 432 : dump_lower_bound (dump_flags_t dump_kind, const vec_lower_bound &lower_bound)
3975 : {
3976 432 : dump_printf (dump_kind, "%s (%T) >= ",
3977 432 : lower_bound.unsigned_p ? "unsigned" : "abs",
3978 432 : lower_bound.expr);
3979 432 : dump_dec (dump_kind, lower_bound.min_value);
3980 432 : }
3981 :
3982 : /* Record that the vectorized loop requires the vec_lower_bound described
3983 : by EXPR, UNSIGNED_P and MIN_VALUE. */
3984 :
3985 : static void
3986 6738 : vect_check_lower_bound (loop_vec_info loop_vinfo, tree expr, bool unsigned_p,
3987 : poly_uint64 min_value)
3988 : {
3989 6738 : vec<vec_lower_bound> &lower_bounds
3990 : = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo);
3991 7710 : for (unsigned int i = 0; i < lower_bounds.length (); ++i)
3992 5953 : if (operand_equal_p (lower_bounds[i].expr, expr, 0))
3993 : {
3994 4981 : unsigned_p &= lower_bounds[i].unsigned_p;
3995 4981 : min_value = upper_bound (lower_bounds[i].min_value, min_value);
3996 4981 : if (lower_bounds[i].unsigned_p != unsigned_p
3997 4981 : || maybe_lt (lower_bounds[i].min_value, min_value))
3998 : {
3999 830 : lower_bounds[i].unsigned_p = unsigned_p;
4000 830 : lower_bounds[i].min_value = min_value;
4001 830 : if (dump_enabled_p ())
4002 : {
4003 252 : dump_printf_loc (MSG_NOTE, vect_location,
4004 : "updating run-time check to ");
4005 252 : dump_lower_bound (MSG_NOTE, lower_bounds[i]);
4006 252 : dump_printf (MSG_NOTE, "\n");
4007 : }
4008 : }
4009 4981 : return;
4010 : }
4011 :
4012 1757 : vec_lower_bound lower_bound (expr, unsigned_p, min_value);
4013 1757 : if (dump_enabled_p ())
4014 : {
4015 180 : dump_printf_loc (MSG_NOTE, vect_location, "need a run-time check that ");
4016 180 : dump_lower_bound (MSG_NOTE, lower_bound);
4017 180 : dump_printf (MSG_NOTE, "\n");
4018 : }
4019 1757 : LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).safe_push (lower_bound);
4020 : }
4021 :
4022 : /* Return true if it's unlikely that the step of the vectorized form of DR_INFO
4023 : will span fewer than GAP bytes. */
4024 :
4025 : static bool
4026 5440 : vect_small_gap_p (loop_vec_info loop_vinfo, dr_vec_info *dr_info,
4027 : poly_int64 gap)
4028 : {
4029 5440 : stmt_vec_info stmt_info = dr_info->stmt;
4030 5440 : HOST_WIDE_INT count
4031 5440 : = estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
4032 5440 : if (DR_GROUP_FIRST_ELEMENT (stmt_info))
4033 4680 : count *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
4034 5440 : return (estimated_poly_value (gap)
4035 5440 : <= count * vect_get_scalar_dr_size (dr_info));
4036 : }
4037 :
4038 : /* Return true if we know that there is no alias between DR_INFO_A and
4039 : DR_INFO_B when abs (DR_STEP (DR_INFO_A->dr)) >= N for some N.
4040 : When returning true, set *LOWER_BOUND_OUT to this N. */
4041 :
4042 : static bool
4043 19749 : vectorizable_with_step_bound_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b,
4044 : poly_uint64 *lower_bound_out)
4045 : {
4046 : /* Check that there is a constant gap of known sign between DR_A
4047 : and DR_B. */
4048 19749 : data_reference *dr_a = dr_info_a->dr;
4049 19749 : data_reference *dr_b = dr_info_b->dr;
4050 19749 : poly_int64 init_a, init_b;
4051 19749 : if (!operand_equal_p (DR_BASE_ADDRESS (dr_a), DR_BASE_ADDRESS (dr_b), 0)
4052 8808 : || !operand_equal_p (DR_OFFSET (dr_a), DR_OFFSET (dr_b), 0)
4053 8122 : || !operand_equal_p (DR_STEP (dr_a), DR_STEP (dr_b), 0)
4054 8112 : || !poly_int_tree_p (DR_INIT (dr_a), &init_a)
4055 8112 : || !poly_int_tree_p (DR_INIT (dr_b), &init_b)
4056 19749 : || !ordered_p (init_a, init_b))
4057 : return false;
4058 :
4059 : /* Sort DR_A and DR_B by the address they access. */
4060 8112 : if (maybe_lt (init_b, init_a))
4061 : {
4062 119 : std::swap (init_a, init_b);
4063 119 : std::swap (dr_info_a, dr_info_b);
4064 119 : std::swap (dr_a, dr_b);
4065 : }
4066 :
4067 : /* If the two accesses could be dependent within a scalar iteration,
4068 : make sure that we'd retain their order. */
4069 8112 : if (maybe_gt (init_a + vect_get_scalar_dr_size (dr_info_a), init_b)
4070 8112 : && !vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
4071 : return false;
4072 :
4073 : /* There is no alias if abs (DR_STEP) is greater than or equal to
4074 : the bytes spanned by the combination of the two accesses. */
4075 8112 : *lower_bound_out = init_b + vect_get_scalar_dr_size (dr_info_b) - init_a;
4076 8112 : return true;
4077 : }
4078 :
4079 : /* Function vect_prune_runtime_alias_test_list.
4080 :
4081 : Prune a list of ddrs to be tested at run-time by versioning for alias.
4082 : Merge several alias checks into one if possible.
4083 : Return FALSE if resulting list of ddrs is longer then allowed by
4084 : PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE. */
4085 :
4086 : opt_result
4087 438341 : vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
4088 : {
4089 438341 : typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
4090 438341 : hash_set <tree_pair_hash> compared_objects;
4091 :
4092 438341 : const vec<ddr_p> &may_alias_ddrs = LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo);
4093 438341 : vec<dr_with_seg_len_pair_t> &comp_alias_ddrs
4094 : = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo);
4095 438341 : const vec<vec_object_pair> &check_unequal_addrs
4096 : = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo);
4097 438341 : poly_uint64 vect_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4098 438341 : tree scalar_loop_iters = LOOP_VINFO_NITERS (loop_vinfo);
4099 :
4100 438341 : ddr_p ddr;
4101 438341 : unsigned int i;
4102 438341 : tree length_factor;
4103 :
4104 438341 : DUMP_VECT_SCOPE ("vect_prune_runtime_alias_test_list");
4105 :
4106 : /* Step values are irrelevant for aliasing if the number of vector
4107 : iterations is equal to the number of scalar iterations (which can
4108 : happen for fully-SLP loops). */
4109 438341 : bool vf_one_p = known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U);
4110 :
4111 438341 : if (!vf_one_p)
4112 : {
4113 : /* Convert the checks for nonzero steps into bound tests. */
4114 : tree value;
4115 434962 : FOR_EACH_VEC_ELT (LOOP_VINFO_CHECK_NONZERO (loop_vinfo), i, value)
4116 1655 : vect_check_lower_bound (loop_vinfo, value, true, 1);
4117 : }
4118 :
4119 438341 : if (may_alias_ddrs.is_empty ())
4120 410633 : return opt_result::success ();
4121 :
4122 27708 : comp_alias_ddrs.create (may_alias_ddrs.length ());
4123 :
4124 27708 : unsigned int loop_depth
4125 27708 : = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
4126 27708 : LOOP_VINFO_LOOP_NEST (loop_vinfo));
4127 :
4128 : /* First, we collect all data ref pairs for aliasing checks. */
4129 112661 : FOR_EACH_VEC_ELT (may_alias_ddrs, i, ddr)
4130 : {
4131 89019 : poly_uint64 lower_bound;
4132 89019 : tree segment_length_a, segment_length_b;
4133 89019 : unsigned HOST_WIDE_INT access_size_a, access_size_b;
4134 89019 : unsigned HOST_WIDE_INT align_a, align_b;
4135 :
4136 : /* Ignore the alias if the VF we chose ended up being no greater
4137 : than the dependence distance. */
4138 89019 : if (dependence_distance_ge_vf (ddr, loop_depth, vect_factor))
4139 13440 : continue;
4140 :
4141 88984 : if (DDR_OBJECT_A (ddr))
4142 : {
4143 106 : vec_object_pair new_pair (DDR_OBJECT_A (ddr), DDR_OBJECT_B (ddr));
4144 106 : if (!compared_objects.add (new_pair))
4145 : {
4146 22 : if (dump_enabled_p ())
4147 16 : dump_printf_loc (MSG_NOTE, vect_location,
4148 : "checking that %T and %T"
4149 : " have different addresses\n",
4150 : new_pair.first, new_pair.second);
4151 22 : LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).safe_push (new_pair);
4152 : }
4153 106 : continue;
4154 106 : }
4155 :
4156 88878 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
4157 88878 : stmt_vec_info stmt_info_a = dr_info_a->stmt;
4158 :
4159 88878 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
4160 88878 : stmt_vec_info stmt_info_b = dr_info_b->stmt;
4161 :
4162 88878 : bool preserves_scalar_order_p
4163 88878 : = vect_preserves_scalar_order_p (dr_info_a, dr_info_b);
4164 88878 : bool ignore_step_p
4165 : = (vf_one_p
4166 88878 : && (preserves_scalar_order_p
4167 3965 : || operand_equal_p (DR_STEP (dr_info_a->dr),
4168 3965 : DR_STEP (dr_info_b->dr))));
4169 :
4170 : /* Skip the pair if inter-iteration dependencies are irrelevant
4171 : and intra-iteration dependencies are guaranteed to be honored. */
4172 7680 : if (ignore_step_p
4173 7921 : && (preserves_scalar_order_p
4174 3206 : || vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
4175 : &lower_bound)))
4176 : {
4177 7680 : if (dump_enabled_p ())
4178 2416 : dump_printf_loc (MSG_NOTE, vect_location,
4179 : "no need for alias check between "
4180 : "%T and %T when VF is 1\n",
4181 2416 : DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
4182 7680 : continue;
4183 : }
4184 :
4185 : /* See whether we can handle the alias using a bounds check on
4186 : the step, and whether that's likely to be the best approach.
4187 : (It might not be, for example, if the minimum step is much larger
4188 : than the number of bytes handled by one vector iteration.) */
4189 81198 : if (!ignore_step_p
4190 80957 : && TREE_CODE (DR_STEP (dr_info_a->dr)) != INTEGER_CST
4191 16543 : && vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
4192 : &lower_bound)
4193 86345 : && (vect_small_gap_p (loop_vinfo, dr_info_a, lower_bound)
4194 5147 : || vect_small_gap_p (loop_vinfo, dr_info_b, lower_bound)))
4195 : {
4196 5083 : bool unsigned_p = dr_known_forward_stride_p (dr_info_a->dr);
4197 5083 : if (dump_enabled_p ())
4198 : {
4199 3394 : dump_printf_loc (MSG_NOTE, vect_location, "no alias between "
4200 : "%T and %T when the step %T is outside ",
4201 : DR_REF (dr_info_a->dr),
4202 1697 : DR_REF (dr_info_b->dr),
4203 1697 : DR_STEP (dr_info_a->dr));
4204 1697 : if (unsigned_p)
4205 540 : dump_printf (MSG_NOTE, "[0");
4206 : else
4207 : {
4208 1157 : dump_printf (MSG_NOTE, "(");
4209 1157 : dump_dec (MSG_NOTE, poly_int64 (-lower_bound));
4210 : }
4211 1697 : dump_printf (MSG_NOTE, ", ");
4212 1697 : dump_dec (MSG_NOTE, lower_bound);
4213 1697 : dump_printf (MSG_NOTE, ")\n");
4214 : }
4215 5083 : vect_check_lower_bound (loop_vinfo, DR_STEP (dr_info_a->dr),
4216 : unsigned_p, lower_bound);
4217 5083 : continue;
4218 5083 : }
4219 :
4220 76115 : stmt_vec_info dr_group_first_a = DR_GROUP_FIRST_ELEMENT (stmt_info_a);
4221 76115 : if (dr_group_first_a)
4222 : {
4223 21446 : stmt_info_a = dr_group_first_a;
4224 21446 : dr_info_a = STMT_VINFO_DR_INFO (stmt_info_a);
4225 : }
4226 :
4227 76115 : stmt_vec_info dr_group_first_b = DR_GROUP_FIRST_ELEMENT (stmt_info_b);
4228 76115 : if (dr_group_first_b)
4229 : {
4230 22063 : stmt_info_b = dr_group_first_b;
4231 22063 : dr_info_b = STMT_VINFO_DR_INFO (stmt_info_b);
4232 : }
4233 :
4234 76115 : if (ignore_step_p)
4235 : {
4236 241 : segment_length_a = size_zero_node;
4237 241 : segment_length_b = size_zero_node;
4238 : }
4239 : else
4240 : {
4241 75874 : if (!operand_equal_p (DR_STEP (dr_info_a->dr),
4242 75874 : DR_STEP (dr_info_b->dr), 0))
4243 : {
4244 17214 : length_factor = scalar_loop_iters;
4245 17214 : if (TREE_CODE (length_factor) == SCEV_NOT_KNOWN)
4246 12 : return opt_result::failure_at (vect_location,
4247 : "Unsupported alias check on"
4248 : " uncounted loop\n");
4249 : }
4250 : else
4251 58660 : length_factor = size_int (vect_factor);
4252 75862 : segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
4253 75862 : segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
4254 : }
4255 76103 : access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
4256 76103 : access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
4257 76103 : align_a = vect_vfa_align (dr_info_a);
4258 76103 : align_b = vect_vfa_align (dr_info_b);
4259 :
4260 : /* See whether the alias is known at compilation time. */
4261 76103 : if (operand_equal_p (DR_BASE_ADDRESS (dr_info_a->dr),
4262 76103 : DR_BASE_ADDRESS (dr_info_b->dr), 0)
4263 6464 : && operand_equal_p (DR_OFFSET (dr_info_a->dr),
4264 6464 : DR_OFFSET (dr_info_b->dr), 0)
4265 4756 : && TREE_CODE (DR_STEP (dr_info_a->dr)) == INTEGER_CST
4266 4682 : && TREE_CODE (DR_STEP (dr_info_b->dr)) == INTEGER_CST
4267 4672 : && poly_int_tree_p (segment_length_a)
4268 80715 : && poly_int_tree_p (segment_length_b))
4269 : {
4270 4590 : int res = vect_compile_time_alias (dr_info_a, dr_info_b,
4271 : segment_length_a,
4272 : segment_length_b,
4273 : access_size_a,
4274 : access_size_b);
4275 4590 : if (res >= 0 && dump_enabled_p ())
4276 : {
4277 208 : dump_printf_loc (MSG_NOTE, vect_location,
4278 : "can tell at compile time that %T and %T",
4279 104 : DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
4280 104 : if (res == 0)
4281 57 : dump_printf (MSG_NOTE, " do not alias\n");
4282 : else
4283 47 : dump_printf (MSG_NOTE, " alias\n");
4284 : }
4285 :
4286 4590 : if (res == 0)
4287 536 : continue;
4288 :
4289 4054 : if (res == 1)
4290 4054 : return opt_result::failure_at (stmt_info_b->stmt,
4291 : "not vectorized:"
4292 : " compilation time alias: %G%G",
4293 : stmt_info_a->stmt,
4294 : stmt_info_b->stmt);
4295 : }
4296 :
4297 : /* dr_with_seg_len requires the alignment to apply to the segment length
4298 : and access size, not just the start address. The access size can be
4299 : smaller than the pointer alignment for grouped accesses and bitfield
4300 : references; see PR115192 and PR116125 respectively. */
4301 71513 : align_a = std::min (align_a, least_bit_hwi (access_size_a));
4302 71513 : align_b = std::min (align_b, least_bit_hwi (access_size_b));
4303 :
4304 71513 : dr_with_seg_len dr_a (dr_info_a->dr, segment_length_a,
4305 71513 : access_size_a, align_a);
4306 71513 : dr_with_seg_len dr_b (dr_info_b->dr, segment_length_b,
4307 71513 : access_size_b, align_b);
4308 : /* Canonicalize the order to be the one that's needed for accurate
4309 : RAW, WAR and WAW flags, in cases where the data references are
4310 : well-ordered. The order doesn't really matter otherwise,
4311 : but we might as well be consistent. */
4312 71513 : if (get_later_stmt (stmt_info_a, stmt_info_b) == stmt_info_a)
4313 5315 : std::swap (dr_a, dr_b);
4314 :
4315 71513 : dr_with_seg_len_pair_t dr_with_seg_len_pair
4316 : (dr_a, dr_b, (preserves_scalar_order_p
4317 : ? dr_with_seg_len_pair_t::WELL_ORDERED
4318 78594 : : dr_with_seg_len_pair_t::REORDERED));
4319 :
4320 71513 : comp_alias_ddrs.safe_push (dr_with_seg_len_pair);
4321 : }
4322 :
4323 23642 : prune_runtime_alias_test_list (&comp_alias_ddrs, vect_factor);
4324 :
4325 47284 : unsigned int count = (comp_alias_ddrs.length ()
4326 23642 : + check_unequal_addrs.length ());
4327 :
4328 23642 : if (count
4329 23642 : && (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo))
4330 : == VECT_COST_MODEL_VERY_CHEAP))
4331 13747 : return opt_result::failure_at
4332 13747 : (vect_location, "would need a runtime alias check\n");
4333 :
4334 9895 : if (dump_enabled_p ())
4335 1988 : dump_printf_loc (MSG_NOTE, vect_location,
4336 : "improved number of alias checks from %d to %d\n",
4337 : may_alias_ddrs.length (), count);
4338 9895 : unsigned limit = param_vect_max_version_for_alias_checks;
4339 9895 : if (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo)) == VECT_COST_MODEL_CHEAP)
4340 938 : limit = param_vect_max_version_for_alias_checks * 6 / 10;
4341 9895 : if (count > limit)
4342 162 : return opt_result::failure_at
4343 162 : (vect_location,
4344 : "number of versioning for alias run-time tests exceeds %d "
4345 : "(--param vect-max-version-for-alias-checks)\n", limit);
4346 :
4347 9733 : return opt_result::success ();
4348 438341 : }
4349 :
4350 : /* Structure to hold information about a supported gather/scatter
4351 : configuration. */
4352 : struct gather_scatter_config
4353 : {
4354 : internal_fn ifn;
4355 : tree offset_vectype;
4356 : int scale;
4357 : vec<int> elsvals;
4358 : };
4359 :
4360 : /* Determine which gather/scatter IFN is supported for the given parameters.
4361 : IFN_MASK_GATHER_LOAD, IFN_GATHER_LOAD, and IFN_MASK_LEN_GATHER_LOAD
4362 : are mutually exclusive, so we only need to find one. Return the
4363 : supported IFN or IFN_LAST if none are supported. */
4364 :
4365 : static internal_fn
4366 1179890 : vect_gather_scatter_which_ifn (bool read_p, bool masked_p,
4367 : tree vectype, tree memory_type,
4368 : tree offset_vectype, int scale,
4369 : vec<int> *elsvals)
4370 : {
4371 : /* Work out which functions to try. */
4372 1179890 : internal_fn ifn, alt_ifn, alt_ifn2;
4373 1179890 : if (read_p)
4374 : {
4375 883028 : ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
4376 : alt_ifn = IFN_MASK_GATHER_LOAD;
4377 : alt_ifn2 = IFN_MASK_LEN_GATHER_LOAD;
4378 : }
4379 : else
4380 : {
4381 296862 : ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
4382 : alt_ifn = IFN_MASK_SCATTER_STORE;
4383 : alt_ifn2 = IFN_MASK_LEN_SCATTER_STORE;
4384 : }
4385 :
4386 1179890 : if (!offset_vectype)
4387 : return IFN_LAST;
4388 :
4389 1179890 : if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
4390 : offset_vectype, scale, elsvals))
4391 : return ifn;
4392 1179890 : if (internal_gather_scatter_fn_supported_p (alt_ifn, vectype, memory_type,
4393 : offset_vectype, scale, elsvals))
4394 : return alt_ifn;
4395 1179890 : if (internal_gather_scatter_fn_supported_p (alt_ifn2, vectype, memory_type,
4396 : offset_vectype, scale, elsvals))
4397 0 : return alt_ifn2;
4398 :
4399 : return IFN_LAST;
4400 : }
4401 :
4402 : /* Collect all supported offset vector types for a gather load or scatter
4403 : store. READ_P is true for loads and false for stores. MASKED_P is true
4404 : if the load or store is conditional. VECTYPE is the data vector type.
4405 : MEMORY_TYPE is the type of the memory elements being loaded or stored,
4406 : and OFFSET_TYPE is the type of the offset.
4407 : SCALE is the amount by which the offset should be multiplied.
4408 :
4409 : Return a vector of all configurations the target supports (which can
4410 : be none). */
4411 :
4412 : static auto_vec<gather_scatter_config>
4413 84304 : vect_gather_scatter_get_configs (vec_info *vinfo, bool read_p, bool masked_p,
4414 : tree vectype, tree memory_type,
4415 : tree offset_type, int scale)
4416 : {
4417 84304 : auto_vec<gather_scatter_config> configs;
4418 :
4419 84304 : auto_vec<tree, 8> offset_types_to_try;
4420 :
4421 : /* Try all sizes from the offset type's precision up to POINTER_SIZE. */
4422 84304 : for (unsigned int bits = TYPE_PRECISION (offset_type);
4423 397701 : bits <= POINTER_SIZE;
4424 299961 : bits *= 2)
4425 : {
4426 : /* Signed variant. */
4427 299961 : offset_types_to_try.safe_push
4428 299961 : (build_nonstandard_integer_type (bits, 0));
4429 : /* Unsigned variant. */
4430 299961 : offset_types_to_try.safe_push
4431 299961 : (build_nonstandard_integer_type (bits, 1));
4432 : }
4433 :
4434 : /* Once we find which IFN works for one offset type, we know that it
4435 : will work for other offset types as well. Then we can perform
4436 : the checks for the remaining offset types with only that IFN.
4437 : However, we might need to try different offset types to find which
4438 : IFN is supported, since the check is offset-type-specific. */
4439 : internal_fn ifn = IFN_LAST;
4440 :
4441 : /* Try each offset type. */
4442 684226 : for (unsigned int i = 0; i < offset_types_to_try.length (); i++)
4443 : {
4444 599922 : tree offset_type = offset_types_to_try[i];
4445 599922 : tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
4446 599922 : if (!offset_vectype)
4447 9998 : continue;
4448 :
4449 : /* Try multiple scale values. Start with exact match, then try
4450 : smaller common scales that a target might support . */
4451 589924 : int scales_to_try[] = {scale, 1, 2, 4, 8};
4452 :
4453 3539544 : for (unsigned int j = 0;
4454 3539544 : j < sizeof (scales_to_try) / sizeof (*scales_to_try);
4455 : j++)
4456 : {
4457 2949620 : int try_scale = scales_to_try[j];
4458 :
4459 : /* Skip scales >= requested scale (except for exact match). */
4460 2949620 : if (j > 0 && try_scale >= scale)
4461 1769730 : continue;
4462 :
4463 : /* Skip if requested scale is not a multiple of this scale. */
4464 1180034 : if (j > 0 && scale % try_scale != 0)
4465 144 : continue;
4466 :
4467 1179890 : vec<int> elsvals = vNULL;
4468 :
4469 : /* If we haven't determined which IFN is supported yet, try all three
4470 : to find which one the target supports. */
4471 1179890 : if (ifn == IFN_LAST)
4472 : {
4473 1179890 : ifn = vect_gather_scatter_which_ifn (read_p, masked_p,
4474 : vectype, memory_type,
4475 : offset_vectype, try_scale,
4476 : &elsvals);
4477 1179890 : if (ifn != IFN_LAST)
4478 : {
4479 : /* Found which IFN is supported. Save this configuration. */
4480 0 : gather_scatter_config config;
4481 0 : config.ifn = ifn;
4482 0 : config.offset_vectype = offset_vectype;
4483 0 : config.scale = try_scale;
4484 0 : config.elsvals = elsvals;
4485 0 : configs.safe_push (config);
4486 : }
4487 : }
4488 : else
4489 : {
4490 : /* We already know which IFN is supported, just check if this
4491 : offset type and scale work with it. */
4492 0 : if (internal_gather_scatter_fn_supported_p (ifn, vectype,
4493 : memory_type,
4494 : offset_vectype,
4495 : try_scale,
4496 : &elsvals))
4497 : {
4498 0 : gather_scatter_config config;
4499 0 : config.ifn = ifn;
4500 0 : config.offset_vectype = offset_vectype;
4501 0 : config.scale = try_scale;
4502 0 : config.elsvals = elsvals;
4503 0 : configs.safe_push (config);
4504 : }
4505 : }
4506 : }
4507 : }
4508 :
4509 84304 : return configs;
4510 84304 : }
4511 :
4512 : /* Check whether we can use an internal function for a gather load
4513 : or scatter store. READ_P is true for loads and false for stores.
4514 : MASKED_P is true if the load or store is conditional. MEMORY_TYPE is
4515 : the type of the memory elements being loaded or stored. OFFSET_TYPE
4516 : is the type of the offset that is being applied to the invariant
4517 : base address. If OFFSET_TYPE is scalar the function chooses an
4518 : appropriate vector type for it. SCALE is the amount by which the
4519 : offset should be multiplied *after* it has been converted to address width.
4520 : If the target does not support the requested SCALE, SUPPORTED_SCALE
4521 : will contain the scale that is actually supported
4522 : (which may be smaller, requiring additional multiplication).
4523 : Otherwise SUPPORTED_SCALE is 0.
4524 :
4525 : Return true if the function is supported, storing the function id in
4526 : *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.
4527 : If we support an offset vector type with different signedness than
4528 : OFFSET_TYPE store it in SUPPORTED_OFFSET_VECTYPE.
4529 :
4530 : If we can use gather/scatter and ELSVALS is nonzero, store the possible
4531 : else values in ELSVALS. */
4532 :
4533 : bool
4534 84304 : vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
4535 : tree vectype, tree memory_type, tree offset_type,
4536 : int scale, int *supported_scale,
4537 : internal_fn *ifn_out,
4538 : tree *offset_vectype_out,
4539 : tree *supported_offset_vectype,
4540 : vec<int> *elsvals)
4541 : {
4542 84304 : *supported_offset_vectype = NULL_TREE;
4543 84304 : *supported_scale = 0;
4544 84304 : unsigned int memory_bits = tree_to_uhwi (TYPE_SIZE (memory_type));
4545 84304 : unsigned int element_bits = vector_element_bits (vectype);
4546 84304 : if (element_bits != memory_bits)
4547 : /* For now the vector elements must be the same width as the
4548 : memory elements. */
4549 : return false;
4550 :
4551 : /* Get the original offset vector type for comparison. */
4552 84304 : tree offset_vectype = VECTOR_TYPE_P (offset_type)
4553 84304 : ? offset_type : get_vectype_for_scalar_type (vinfo, offset_type);
4554 :
4555 : /* If there is no offset vectype, bail. */
4556 71224 : if (!offset_vectype)
4557 : return false;
4558 :
4559 84304 : offset_type = TREE_TYPE (offset_vectype);
4560 :
4561 : /* Get all supported configurations for this data vector type. */
4562 84304 : auto_vec<gather_scatter_config> configs
4563 : = vect_gather_scatter_get_configs (vinfo, read_p, masked_p, vectype,
4564 84304 : memory_type, offset_type, scale);
4565 :
4566 84304 : if (configs.is_empty ())
4567 : return false;
4568 :
4569 : /* Selection priority:
4570 : 1 - Exact scale match + offset type match
4571 : 2 - Exact scale match + sign-swapped offset
4572 : 3 - Smaller scale + offset type match
4573 : 4 - Smaller scale + sign-swapped offset
4574 : Within each category, prefer smaller offset types. */
4575 :
4576 : /* First pass: exact scale match with no conversion. */
4577 0 : for (unsigned int i = 0; i < configs.length (); i++)
4578 : {
4579 0 : if (configs[i].scale == scale
4580 0 : && TYPE_SIGN (configs[i].offset_vectype)
4581 0 : == TYPE_SIGN (offset_vectype))
4582 : {
4583 0 : *ifn_out = configs[i].ifn;
4584 0 : *offset_vectype_out = configs[i].offset_vectype;
4585 0 : if (elsvals)
4586 0 : *elsvals = configs[i].elsvals;
4587 : return true;
4588 : }
4589 : }
4590 :
4591 : /* No direct match. This means we try to find either
4592 : - a sign-swapped offset vectype or
4593 : - a different scale and 2x larger offset type
4594 : - a different scale and larger sign-swapped offset vectype. */
4595 0 : unsigned int offset_precision = TYPE_PRECISION (TREE_TYPE (offset_vectype));
4596 0 : unsigned int needed_precision
4597 0 : = TYPE_UNSIGNED (offset_vectype) ? offset_precision * 2 : POINTER_SIZE;
4598 0 : needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
4599 :
4600 : /* Second pass: No direct match. This means we try to find a sign-swapped
4601 : offset vectype. */
4602 0 : for (unsigned int i = 0; i < configs.length (); i++)
4603 : {
4604 0 : unsigned int precision
4605 0 : = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
4606 0 : if (configs[i].scale == scale
4607 0 : && precision >= needed_precision
4608 0 : && (supportable_convert_operation (CONVERT_EXPR,
4609 0 : configs[i].offset_vectype,
4610 : offset_vectype)
4611 0 : || (needed_precision == offset_precision
4612 0 : && tree_nop_conversion_p (configs[i].offset_vectype,
4613 : offset_vectype))))
4614 : {
4615 0 : *ifn_out = configs[i].ifn;
4616 0 : *offset_vectype_out = offset_vectype;
4617 0 : *supported_offset_vectype = configs[i].offset_vectype;
4618 0 : if (elsvals)
4619 0 : *elsvals = configs[i].elsvals;
4620 : return true;
4621 : }
4622 : }
4623 :
4624 : /* Third pass: Try a smaller scale with the same signedness. */
4625 0 : needed_precision = offset_precision * 2;
4626 0 : needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
4627 :
4628 0 : for (unsigned int i = 0; i < configs.length (); i++)
4629 : {
4630 0 : unsigned int precision
4631 0 : = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
4632 0 : if (configs[i].scale < scale
4633 0 : && TYPE_SIGN (configs[i].offset_vectype)
4634 0 : == TYPE_SIGN (offset_vectype)
4635 0 : && precision >= needed_precision)
4636 : {
4637 0 : *ifn_out = configs[i].ifn;
4638 0 : *offset_vectype_out = configs[i].offset_vectype;
4639 0 : *supported_scale = configs[i].scale;
4640 : /* Only set SUPPORTED_OFFSET_VECTYPE if this is a real
4641 : conversion. */
4642 0 : if (!useless_type_conversion_p (offset_vectype,
4643 0 : configs[i].offset_vectype))
4644 0 : *supported_offset_vectype = configs[i].offset_vectype;
4645 0 : if (elsvals)
4646 0 : *elsvals = configs[i].elsvals;
4647 : return true;
4648 : }
4649 : }
4650 :
4651 : /* Fourth pass: Try a smaller scale and sign-swapped offset vectype. */
4652 0 : needed_precision
4653 0 : = TYPE_UNSIGNED (offset_vectype) ? offset_precision * 2 : POINTER_SIZE;
4654 0 : needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
4655 :
4656 84304 : for (unsigned int i = 0; i < configs.length (); i++)
4657 : {
4658 0 : unsigned int precision
4659 0 : = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
4660 0 : if (configs[i].scale < scale
4661 0 : && precision >= needed_precision
4662 0 : && (supportable_convert_operation (CONVERT_EXPR,
4663 0 : configs[i].offset_vectype,
4664 : offset_vectype)
4665 0 : || (needed_precision == offset_precision
4666 0 : && tree_nop_conversion_p (configs[i].offset_vectype,
4667 : offset_vectype))))
4668 : {
4669 0 : *ifn_out = configs[i].ifn;
4670 0 : *offset_vectype_out = offset_vectype;
4671 0 : *supported_offset_vectype = configs[i].offset_vectype;
4672 0 : *supported_scale = configs[i].scale;
4673 0 : if (elsvals)
4674 0 : *elsvals = configs[i].elsvals;
4675 : return true;
4676 : }
4677 : }
4678 :
4679 : return false;
4680 84304 : }
4681 :
4682 : /* STMT_INFO is a call to an internal gather load or scatter store function.
4683 : Describe the operation in INFO. */
4684 :
4685 : void
4686 0 : vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
4687 : gather_scatter_info *info)
4688 : {
4689 0 : gcall *call = as_a <gcall *> (stmt_info->stmt);
4690 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4691 0 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
4692 :
4693 0 : info->ifn = gimple_call_internal_fn (call);
4694 0 : info->decl = NULL_TREE;
4695 0 : info->base = gimple_call_arg (call, 0);
4696 0 : info->alias_ptr = gimple_call_arg
4697 0 : (call, internal_fn_alias_ptr_index (info->ifn));
4698 0 : info->offset = gimple_call_arg
4699 0 : (call, internal_fn_offset_index (info->ifn));
4700 0 : info->offset_vectype = NULL_TREE;
4701 0 : info->scale = TREE_INT_CST_LOW (gimple_call_arg
4702 : (call, internal_fn_scale_index (info->ifn)));
4703 0 : info->element_type = TREE_TYPE (vectype);
4704 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
4705 0 : }
4706 :
4707 : /* Return true if a non-affine read or write in STMT_INFO is suitable for a
4708 : gather load or scatter store with VECTYPE. Describe the operation in *INFO
4709 : if so. If it is suitable and ELSVALS is nonzero store the supported else
4710 : values in the vector it points to. */
4711 :
4712 : bool
4713 368197 : vect_check_gather_scatter (stmt_vec_info stmt_info, tree vectype,
4714 : loop_vec_info loop_vinfo,
4715 : gather_scatter_info *info, vec<int> *elsvals)
4716 : {
4717 368197 : HOST_WIDE_INT scale = 1;
4718 368197 : poly_int64 pbitpos, pbitsize;
4719 368197 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
4720 368197 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
4721 368197 : tree offtype = NULL_TREE;
4722 368197 : tree decl = NULL_TREE, base, off;
4723 368197 : tree memory_type = TREE_TYPE (DR_REF (dr));
4724 368197 : machine_mode pmode;
4725 368197 : int punsignedp, reversep, pvolatilep = 0;
4726 368197 : internal_fn ifn;
4727 368197 : tree offset_vectype;
4728 368197 : bool masked_p = false;
4729 :
4730 : /* See whether this is already a call to a gather/scatter internal function.
4731 : If not, see whether it's a masked load or store. */
4732 368197 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
4733 4285 : if (call && gimple_call_internal_p (call))
4734 : {
4735 4285 : ifn = gimple_call_internal_fn (call);
4736 4285 : if (internal_gather_scatter_fn_p (ifn))
4737 : {
4738 0 : vect_describe_gather_scatter_call (stmt_info, info);
4739 :
4740 : /* In pattern recog we simply used a ZERO else value that
4741 : we need to correct here. To that end just re-use the
4742 : (already successful) check if we support a gather IFN
4743 : and have it populate the else values. */
4744 0 : if (DR_IS_READ (dr) && internal_fn_mask_index (ifn) >= 0 && elsvals)
4745 0 : supports_vec_gather_load_p (TYPE_MODE (vectype), elsvals);
4746 : return true;
4747 : }
4748 4285 : masked_p = (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE);
4749 : }
4750 :
4751 : /* True if we should aim to use internal functions rather than
4752 : built-in functions. */
4753 368197 : bool use_ifn_p = (DR_IS_READ (dr)
4754 368197 : ? supports_vec_gather_load_p (TYPE_MODE (vectype),
4755 : elsvals)
4756 368197 : : supports_vec_scatter_store_p (TYPE_MODE (vectype)));
4757 :
4758 368197 : base = DR_REF (dr);
4759 : /* For masked loads/stores, DR_REF (dr) is an artificial MEM_REF,
4760 : see if we can use the def stmt of the address. */
4761 368197 : if (masked_p
4762 4285 : && TREE_CODE (base) == MEM_REF
4763 4285 : && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
4764 4285 : && integer_zerop (TREE_OPERAND (base, 1))
4765 372482 : && !expr_invariant_in_loop_p (loop, TREE_OPERAND (base, 0)))
4766 : {
4767 4285 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base, 0));
4768 4285 : if (is_gimple_assign (def_stmt)
4769 4285 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
4770 613 : base = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
4771 : }
4772 :
4773 : /* The gather and scatter builtins need address of the form
4774 : loop_invariant + vector * {1, 2, 4, 8}
4775 : or
4776 : loop_invariant + sign_extend (vector) * { 1, 2, 4, 8 }.
4777 : Unfortunately DR_BASE_ADDRESS/DR_OFFSET can be a mixture
4778 : of loop invariants/SSA_NAMEs defined in the loop, with casts,
4779 : multiplications and additions in it. To get a vector, we need
4780 : a single SSA_NAME that will be defined in the loop and will
4781 : contain everything that is not loop invariant and that can be
4782 : vectorized. The following code attempts to find such a preexistng
4783 : SSA_NAME OFF and put the loop invariants into a tree BASE
4784 : that can be gimplified before the loop. */
4785 368197 : base = get_inner_reference (base, &pbitsize, &pbitpos, &off, &pmode,
4786 : &punsignedp, &reversep, &pvolatilep);
4787 368197 : if (reversep)
4788 : return false;
4789 :
4790 : /* PR 107346. Packed structs can have fields at offsets that are not
4791 : multiples of BITS_PER_UNIT. Do not use gather/scatters in such cases. */
4792 368197 : if (!multiple_p (pbitpos, BITS_PER_UNIT))
4793 : return false;
4794 :
4795 : /* We need to be able to form an address to the base which for example
4796 : isn't possible for hard registers. */
4797 368197 : if (may_be_nonaddressable_p (base))
4798 : return false;
4799 :
4800 368189 : poly_int64 pbytepos = exact_div (pbitpos, BITS_PER_UNIT);
4801 :
4802 368189 : if (TREE_CODE (base) == MEM_REF)
4803 : {
4804 299089 : if (!integer_zerop (TREE_OPERAND (base, 1)))
4805 : {
4806 34911 : if (off == NULL_TREE)
4807 34594 : off = wide_int_to_tree (sizetype, mem_ref_offset (base));
4808 : else
4809 317 : off = size_binop (PLUS_EXPR, off,
4810 : fold_convert (sizetype, TREE_OPERAND (base, 1)));
4811 : }
4812 299089 : base = TREE_OPERAND (base, 0);
4813 : }
4814 : else
4815 69100 : base = build_fold_addr_expr (base);
4816 :
4817 368189 : if (off == NULL_TREE)
4818 238431 : off = size_zero_node;
4819 :
4820 : /* BASE must be loop invariant. If it is not invariant, but OFF is, then we
4821 : * can fix that by swapping BASE and OFF. */
4822 368189 : if (!expr_invariant_in_loop_p (loop, base))
4823 : {
4824 273528 : if (!expr_invariant_in_loop_p (loop, off))
4825 : return false;
4826 :
4827 273283 : std::swap (base, off);
4828 : }
4829 :
4830 367944 : base = fold_convert (sizetype, base);
4831 367944 : base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
4832 367944 : int tmp_scale;
4833 367944 : tree tmp_offset_vectype;
4834 :
4835 : /* OFF at this point may be either a SSA_NAME or some tree expression
4836 : from get_inner_reference. Try to peel off loop invariants from it
4837 : into BASE as long as possible. */
4838 367944 : STRIP_NOPS (off);
4839 951441 : while (offtype == NULL_TREE)
4840 : {
4841 826590 : enum tree_code code;
4842 826590 : tree op0, op1, add = NULL_TREE;
4843 :
4844 826590 : if (TREE_CODE (off) == SSA_NAME)
4845 : {
4846 634053 : gimple *def_stmt = SSA_NAME_DEF_STMT (off);
4847 :
4848 634053 : if (expr_invariant_in_loop_p (loop, off))
4849 0 : return false;
4850 :
4851 634053 : if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
4852 : break;
4853 :
4854 494637 : op0 = gimple_assign_rhs1 (def_stmt);
4855 494637 : code = gimple_assign_rhs_code (def_stmt);
4856 494637 : op1 = gimple_assign_rhs2 (def_stmt);
4857 : }
4858 : else
4859 : {
4860 192537 : if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS)
4861 : return false;
4862 192537 : code = TREE_CODE (off);
4863 192537 : extract_ops_from_tree (off, &code, &op0, &op1);
4864 : }
4865 687174 : switch (code)
4866 : {
4867 212863 : case POINTER_PLUS_EXPR:
4868 212863 : case PLUS_EXPR:
4869 212863 : if (expr_invariant_in_loop_p (loop, op0))
4870 : {
4871 140613 : add = op0;
4872 140613 : off = op1;
4873 196130 : do_add:
4874 196130 : add = fold_convert (sizetype, add);
4875 196130 : if (scale != 1)
4876 50168 : add = size_binop (MULT_EXPR, add, size_int (scale));
4877 196130 : base = size_binop (PLUS_EXPR, base, add);
4878 583497 : continue;
4879 : }
4880 72250 : if (expr_invariant_in_loop_p (loop, op1))
4881 : {
4882 55223 : add = op1;
4883 55223 : off = op0;
4884 55223 : goto do_add;
4885 : }
4886 : break;
4887 491 : case MINUS_EXPR:
4888 491 : if (expr_invariant_in_loop_p (loop, op1))
4889 : {
4890 294 : add = fold_convert (sizetype, op1);
4891 294 : add = size_binop (MINUS_EXPR, size_zero_node, add);
4892 294 : off = op0;
4893 294 : goto do_add;
4894 : }
4895 : break;
4896 207564 : case MULT_EXPR:
4897 207564 : if (scale == 1 && tree_fits_shwi_p (op1))
4898 : {
4899 173554 : int new_scale = tree_to_shwi (op1);
4900 : /* Only treat this as a scaling operation if the target
4901 : supports it for at least some offset type. */
4902 173554 : if (use_ifn_p
4903 0 : && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4904 : masked_p, vectype, memory_type,
4905 : signed_char_type_node,
4906 : new_scale, &tmp_scale,
4907 : &ifn,
4908 : &offset_vectype,
4909 : &tmp_offset_vectype,
4910 : elsvals)
4911 173554 : && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4912 : masked_p, vectype, memory_type,
4913 : unsigned_char_type_node,
4914 : new_scale, &tmp_scale,
4915 : &ifn,
4916 : &offset_vectype,
4917 : &tmp_offset_vectype,
4918 : elsvals))
4919 : break;
4920 173554 : scale = new_scale;
4921 173554 : off = op0;
4922 173554 : continue;
4923 173554 : }
4924 : break;
4925 0 : case SSA_NAME:
4926 0 : off = op0;
4927 0 : continue;
4928 220583 : CASE_CONVERT:
4929 441150 : if (!POINTER_TYPE_P (TREE_TYPE (op0))
4930 441150 : && !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
4931 : break;
4932 :
4933 : /* Don't include the conversion if the target is happy with
4934 : the current offset type. */
4935 220583 : if (use_ifn_p
4936 0 : && TREE_CODE (off) == SSA_NAME
4937 0 : && !POINTER_TYPE_P (TREE_TYPE (off))
4938 220583 : && vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4939 : masked_p, vectype, memory_type,
4940 0 : TREE_TYPE (off),
4941 : scale, &tmp_scale,
4942 : &ifn,
4943 : &offset_vectype,
4944 : &tmp_offset_vectype,
4945 : elsvals))
4946 : break;
4947 :
4948 220583 : if (TYPE_PRECISION (TREE_TYPE (op0))
4949 220583 : == TYPE_PRECISION (TREE_TYPE (off)))
4950 : {
4951 88962 : off = op0;
4952 88962 : continue;
4953 : }
4954 :
4955 : /* Include the conversion if it is widening and we're using
4956 : the IFN path or the target can handle the converted from
4957 : offset or the current size is not already the same as the
4958 : data vector element size. */
4959 131621 : if ((TYPE_PRECISION (TREE_TYPE (op0))
4960 131621 : < TYPE_PRECISION (TREE_TYPE (off)))
4961 131621 : && (use_ifn_p
4962 130753 : || (DR_IS_READ (dr)
4963 83402 : ? (targetm.vectorize.builtin_gather
4964 83402 : && targetm.vectorize.builtin_gather (vectype,
4965 83402 : TREE_TYPE (op0),
4966 : scale))
4967 47351 : : (targetm.vectorize.builtin_scatter
4968 47351 : && targetm.vectorize.builtin_scatter (vectype,
4969 47351 : TREE_TYPE (op0),
4970 : scale)))
4971 129653 : || !operand_equal_p (TYPE_SIZE (TREE_TYPE (off)),
4972 129653 : TYPE_SIZE (TREE_TYPE (vectype)), 0)))
4973 : {
4974 124851 : off = op0;
4975 124851 : offtype = TREE_TYPE (off);
4976 124851 : STRIP_NOPS (off);
4977 124851 : continue;
4978 : }
4979 : break;
4980 : default:
4981 : break;
4982 0 : }
4983 : break;
4984 : }
4985 :
4986 : /* If at the end OFF still isn't a SSA_NAME or isn't
4987 : defined in the loop, punt. */
4988 367944 : if (TREE_CODE (off) != SSA_NAME
4989 367944 : || expr_invariant_in_loop_p (loop, off))
4990 : return false;
4991 :
4992 361212 : if (offtype == NULL_TREE)
4993 236715 : offtype = TREE_TYPE (off);
4994 :
4995 361212 : if (use_ifn_p)
4996 : {
4997 0 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
4998 : vectype, memory_type, offtype,
4999 : scale, &tmp_scale,
5000 : &ifn, &offset_vectype,
5001 : &tmp_offset_vectype,
5002 : elsvals))
5003 0 : ifn = IFN_LAST;
5004 : decl = NULL_TREE;
5005 : }
5006 : else
5007 : {
5008 361212 : if (DR_IS_READ (dr))
5009 : {
5010 267998 : if (targetm.vectorize.builtin_gather)
5011 267998 : decl = targetm.vectorize.builtin_gather (vectype, offtype, scale);
5012 : }
5013 : else
5014 : {
5015 93214 : if (targetm.vectorize.builtin_scatter)
5016 93214 : decl = targetm.vectorize.builtin_scatter (vectype, offtype, scale);
5017 : }
5018 361212 : ifn = IFN_LAST;
5019 : /* The offset vector type will be read from DECL when needed. */
5020 361212 : offset_vectype = NULL_TREE;
5021 : }
5022 :
5023 361212 : gcc_checking_assert (expr_invariant_in_loop_p (loop, base));
5024 361212 : gcc_checking_assert (!expr_invariant_in_loop_p (loop, off));
5025 :
5026 361212 : info->ifn = ifn;
5027 361212 : info->decl = decl;
5028 361212 : info->base = base;
5029 :
5030 722424 : info->alias_ptr = build_int_cst
5031 361212 : (reference_alias_ptr_type (DR_REF (dr)),
5032 361212 : get_object_alignment (DR_REF (dr)));
5033 :
5034 361212 : info->offset = off;
5035 361212 : info->offset_vectype = offset_vectype;
5036 361212 : info->scale = scale;
5037 361212 : info->element_type = TREE_TYPE (vectype);
5038 361212 : info->memory_type = memory_type;
5039 361212 : return true;
5040 : }
5041 :
5042 : /* Find the data references in STMT, analyze them with respect to LOOP and
5043 : append them to DATAREFS. Return false if datarefs in this stmt cannot
5044 : be handled. */
5045 :
5046 : opt_result
5047 33666824 : vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
5048 : vec<data_reference_p> *datarefs,
5049 : vec<int> *dataref_groups, int group_id)
5050 : {
5051 : /* We can ignore clobbers for dataref analysis - they are removed during
5052 : loop vectorization and BB vectorization checks dependences with a
5053 : stmt walk. */
5054 33666824 : if (gimple_clobber_p (stmt))
5055 1223990 : return opt_result::success ();
5056 :
5057 60349547 : if (gimple_has_volatile_ops (stmt))
5058 323122 : return opt_result::failure_at (stmt, "not vectorized: volatile type: %G",
5059 : stmt);
5060 :
5061 32119712 : if (stmt_can_throw_internal (cfun, stmt))
5062 765521 : return opt_result::failure_at (stmt,
5063 : "not vectorized:"
5064 : " statement can throw an exception: %G",
5065 : stmt);
5066 :
5067 31354191 : auto_vec<data_reference_p, 2> refs;
5068 31354191 : opt_result res = find_data_references_in_stmt (loop, stmt, &refs);
5069 31354191 : if (!res)
5070 3808142 : return res;
5071 :
5072 27546049 : if (refs.is_empty ())
5073 15833239 : return opt_result::success ();
5074 :
5075 11712810 : if (refs.length () > 1)
5076 : {
5077 1257073 : while (!refs.is_empty ())
5078 838351 : free_data_ref (refs.pop ());
5079 418722 : return opt_result::failure_at (stmt,
5080 : "not vectorized: more than one "
5081 : "data ref in stmt: %G", stmt);
5082 : }
5083 :
5084 11294088 : data_reference_p dr = refs.pop ();
5085 11294088 : if (gcall *call = dyn_cast <gcall *> (stmt))
5086 25747 : if (!gimple_call_internal_p (call)
5087 25747 : || (gimple_call_internal_fn (call) != IFN_MASK_LOAD
5088 23101 : && gimple_call_internal_fn (call) != IFN_MASK_STORE))
5089 : {
5090 22478 : free_data_ref (dr);
5091 22478 : return opt_result::failure_at (stmt,
5092 : "not vectorized: dr in a call %G", stmt);
5093 : }
5094 :
5095 11271610 : if (TREE_CODE (DR_REF (dr)) == COMPONENT_REF
5096 11271610 : && DECL_BIT_FIELD (TREE_OPERAND (DR_REF (dr), 1)))
5097 : {
5098 57778 : free_data_ref (dr);
5099 57778 : return opt_result::failure_at (stmt,
5100 : "not vectorized:"
5101 : " statement is an unsupported"
5102 : " bitfield access %G", stmt);
5103 : }
5104 :
5105 11213832 : if (DR_BASE_ADDRESS (dr)
5106 11121493 : && TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
5107 : {
5108 998 : free_data_ref (dr);
5109 998 : return opt_result::failure_at (stmt,
5110 : "not vectorized:"
5111 : " base addr of dr is a constant\n");
5112 : }
5113 :
5114 : /* Check whether this may be a SIMD lane access and adjust the
5115 : DR to make it easier for us to handle it. */
5116 11212834 : if (loop
5117 614882 : && loop->simduid
5118 10694 : && (!DR_BASE_ADDRESS (dr)
5119 2952 : || !DR_OFFSET (dr)
5120 2952 : || !DR_INIT (dr)
5121 2952 : || !DR_STEP (dr)))
5122 : {
5123 7742 : struct data_reference *newdr
5124 15484 : = create_data_ref (NULL, loop_containing_stmt (stmt), DR_REF (dr), stmt,
5125 7742 : DR_IS_READ (dr), DR_IS_CONDITIONAL_IN_STMT (dr));
5126 7742 : if (DR_BASE_ADDRESS (newdr)
5127 7742 : && DR_OFFSET (newdr)
5128 7742 : && DR_INIT (newdr)
5129 7742 : && DR_STEP (newdr)
5130 7742 : && TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
5131 15484 : && integer_zerop (DR_STEP (newdr)))
5132 : {
5133 7742 : tree base_address = DR_BASE_ADDRESS (newdr);
5134 7742 : tree off = DR_OFFSET (newdr);
5135 7742 : tree step = ssize_int (1);
5136 7742 : if (integer_zerop (off)
5137 7742 : && TREE_CODE (base_address) == POINTER_PLUS_EXPR)
5138 : {
5139 82 : off = TREE_OPERAND (base_address, 1);
5140 82 : base_address = TREE_OPERAND (base_address, 0);
5141 : }
5142 7742 : STRIP_NOPS (off);
5143 7742 : if (TREE_CODE (off) == MULT_EXPR
5144 7742 : && tree_fits_uhwi_p (TREE_OPERAND (off, 1)))
5145 : {
5146 7491 : step = TREE_OPERAND (off, 1);
5147 7491 : off = TREE_OPERAND (off, 0);
5148 7491 : STRIP_NOPS (off);
5149 : }
5150 539 : if (CONVERT_EXPR_P (off)
5151 7742 : && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
5152 7203 : < TYPE_PRECISION (TREE_TYPE (off))))
5153 7203 : off = TREE_OPERAND (off, 0);
5154 7742 : if (TREE_CODE (off) == SSA_NAME)
5155 : {
5156 7219 : gimple *def = SSA_NAME_DEF_STMT (off);
5157 : /* Look through widening conversion. */
5158 7219 : if (is_gimple_assign (def)
5159 7219 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
5160 : {
5161 0 : tree rhs1 = gimple_assign_rhs1 (def);
5162 0 : if (TREE_CODE (rhs1) == SSA_NAME
5163 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
5164 0 : && (TYPE_PRECISION (TREE_TYPE (off))
5165 0 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
5166 0 : def = SSA_NAME_DEF_STMT (rhs1);
5167 : }
5168 7219 : if (is_gimple_call (def)
5169 7083 : && gimple_call_internal_p (def)
5170 14302 : && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
5171 : {
5172 7083 : tree arg = gimple_call_arg (def, 0);
5173 7083 : tree reft = TREE_TYPE (DR_REF (newdr));
5174 7083 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
5175 7083 : arg = SSA_NAME_VAR (arg);
5176 7083 : if (arg == loop->simduid
5177 : /* For now. */
5178 7083 : && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
5179 : {
5180 7058 : DR_BASE_ADDRESS (newdr) = base_address;
5181 7058 : DR_OFFSET (newdr) = ssize_int (0);
5182 7058 : DR_STEP (newdr) = step;
5183 7058 : DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
5184 7058 : DR_STEP_ALIGNMENT (newdr) = highest_pow2_factor (step);
5185 : /* Mark as simd-lane access. */
5186 7058 : tree arg2 = gimple_call_arg (def, 1);
5187 7058 : newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
5188 7058 : free_data_ref (dr);
5189 7058 : datarefs->safe_push (newdr);
5190 7058 : if (dataref_groups)
5191 0 : dataref_groups->safe_push (group_id);
5192 7058 : return opt_result::success ();
5193 : }
5194 : }
5195 : }
5196 : }
5197 684 : free_data_ref (newdr);
5198 : }
5199 :
5200 11205776 : datarefs->safe_push (dr);
5201 11205776 : if (dataref_groups)
5202 10597952 : dataref_groups->safe_push (group_id);
5203 11205776 : return opt_result::success ();
5204 31354191 : }
5205 :
5206 : /* Function vect_analyze_data_refs.
5207 :
5208 : Find all the data references in the loop or basic block.
5209 :
5210 : The general structure of the analysis of data refs in the vectorizer is as
5211 : follows:
5212 : 1- vect_analyze_data_refs(loop/bb): call
5213 : compute_data_dependences_for_loop/bb to find and analyze all data-refs
5214 : in the loop/bb and their dependences.
5215 : 2- vect_analyze_dependences(): apply dependence testing using ddrs.
5216 : 3- vect_analyze_drs_alignment(): check that ref_stmt.alignment is ok.
5217 : 4- vect_analyze_drs_access(): check that ref_stmt.step is ok.
5218 :
5219 : */
5220 :
5221 : opt_result
5222 2883192 : vect_analyze_data_refs (vec_info *vinfo, bool *fatal)
5223 : {
5224 2883192 : class loop *loop = NULL;
5225 2883192 : unsigned int i;
5226 2883192 : struct data_reference *dr;
5227 2883192 : tree scalar_type;
5228 :
5229 2883192 : DUMP_VECT_SCOPE ("vect_analyze_data_refs");
5230 :
5231 2883192 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
5232 533694 : loop = LOOP_VINFO_LOOP (loop_vinfo);
5233 :
5234 : /* Go through the data-refs, check that the analysis succeeded. Update
5235 : pointer from stmt_vec_info struct to DR and vectype. */
5236 :
5237 2883192 : vec<data_reference_p> datarefs = vinfo->shared->datarefs;
5238 19131016 : FOR_EACH_VEC_ELT (datarefs, i, dr)
5239 : {
5240 16323106 : enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
5241 :
5242 16323106 : gcc_assert (DR_REF (dr));
5243 16323106 : stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
5244 16323106 : gcc_assert (!stmt_info->dr_aux.dr);
5245 16323106 : stmt_info->dr_aux.dr = dr;
5246 16323106 : stmt_info->dr_aux.stmt = stmt_info;
5247 :
5248 : /* Check that analysis of the data-ref succeeded. */
5249 16323106 : if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
5250 16199053 : || !DR_STEP (dr))
5251 : {
5252 248106 : bool maybe_gather
5253 124053 : = DR_IS_READ (dr)
5254 124053 : && !TREE_THIS_VOLATILE (DR_REF (dr));
5255 248106 : bool maybe_scatter
5256 : = DR_IS_WRITE (dr)
5257 124053 : && !TREE_THIS_VOLATILE (DR_REF (dr));
5258 :
5259 : /* If target supports vector gather loads or scatter stores,
5260 : see if they can't be used. */
5261 124053 : if (is_a <loop_vec_info> (vinfo)
5262 124053 : && !nested_in_vect_loop_p (loop, stmt_info))
5263 : {
5264 120628 : if (maybe_gather || maybe_scatter)
5265 : {
5266 120628 : if (maybe_gather)
5267 : gatherscatter = GATHER;
5268 : else
5269 25360 : gatherscatter = SCATTER;
5270 : }
5271 : }
5272 :
5273 25360 : if (gatherscatter == SG_NONE)
5274 : {
5275 3425 : if (dump_enabled_p ())
5276 5 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5277 : "not vectorized: data ref analysis "
5278 : "failed %G", stmt_info->stmt);
5279 3425 : if (is_a <bb_vec_info> (vinfo))
5280 : {
5281 : /* In BB vectorization the ref can still participate
5282 : in dependence analysis, we just can't vectorize it. */
5283 3073 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5284 3073 : continue;
5285 : }
5286 352 : return opt_result::failure_at (stmt_info->stmt,
5287 : "not vectorized:"
5288 : " data ref analysis failed: %G",
5289 : stmt_info->stmt);
5290 : }
5291 : }
5292 :
5293 : /* See if this was detected as SIMD lane access. */
5294 16319681 : if (dr->aux == (void *)-1
5295 16319681 : || dr->aux == (void *)-2
5296 16310788 : || dr->aux == (void *)-3
5297 16309948 : || dr->aux == (void *)-4)
5298 : {
5299 10533 : if (nested_in_vect_loop_p (loop, stmt_info))
5300 0 : return opt_result::failure_at (stmt_info->stmt,
5301 : "not vectorized:"
5302 : " data ref analysis failed: %G",
5303 : stmt_info->stmt);
5304 10533 : STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
5305 10533 : = -(uintptr_t) dr->aux;
5306 : }
5307 :
5308 16319681 : tree base = get_base_address (DR_REF (dr));
5309 16319681 : if (base && VAR_P (base) && DECL_NONALIASED (base))
5310 : {
5311 9164 : if (dump_enabled_p ())
5312 186 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5313 : "not vectorized: base object not addressable "
5314 : "for stmt: %G", stmt_info->stmt);
5315 9164 : if (is_a <bb_vec_info> (vinfo))
5316 : {
5317 : /* In BB vectorization the ref can still participate
5318 : in dependence analysis, we just can't vectorize it. */
5319 9163 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5320 9163 : continue;
5321 : }
5322 1 : return opt_result::failure_at (stmt_info->stmt,
5323 : "not vectorized: base object not"
5324 : " addressable for stmt: %G",
5325 : stmt_info->stmt);
5326 : }
5327 :
5328 16310517 : if (is_a <loop_vec_info> (vinfo)
5329 1203889 : && DR_STEP (dr)
5330 17393778 : && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
5331 : {
5332 45320 : if (nested_in_vect_loop_p (loop, stmt_info))
5333 402 : return opt_result::failure_at (stmt_info->stmt,
5334 : "not vectorized: "
5335 : "not suitable for strided load %G",
5336 : stmt_info->stmt);
5337 44918 : STMT_VINFO_STRIDED_P (stmt_info) = true;
5338 : }
5339 :
5340 : /* Update DR field in stmt_vec_info struct. */
5341 :
5342 : /* If the dataref is in an inner-loop of the loop that is considered for
5343 : for vectorization, we also want to analyze the access relative to
5344 : the outer-loop (DR contains information only relative to the
5345 : inner-most enclosing loop). We do that by building a reference to the
5346 : first location accessed by the inner-loop, and analyze it relative to
5347 : the outer-loop. */
5348 16310115 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
5349 : {
5350 : /* Build a reference to the first location accessed by the
5351 : inner loop: *(BASE + INIT + OFFSET). By construction,
5352 : this address must be invariant in the inner loop, so we
5353 : can consider it as being used in the outer loop. */
5354 12014 : tree base = unshare_expr (DR_BASE_ADDRESS (dr));
5355 12014 : tree offset = unshare_expr (DR_OFFSET (dr));
5356 12014 : tree init = unshare_expr (DR_INIT (dr));
5357 12014 : tree init_offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset),
5358 : init, offset);
5359 12014 : tree init_addr = fold_build_pointer_plus (base, init_offset);
5360 12014 : tree init_ref = build2 (MEM_REF, TREE_TYPE (DR_REF (dr)),
5361 : init_addr, build_zero_cst (ptr_type_node));
5362 :
5363 12014 : if (dump_enabled_p ())
5364 1228 : dump_printf_loc (MSG_NOTE, vect_location,
5365 : "analyze in outer loop: %T\n", init_ref);
5366 :
5367 12014 : opt_result res
5368 12014 : = dr_analyze_innermost (&STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info),
5369 12014 : init_ref, loop, stmt_info->stmt);
5370 12014 : if (!res)
5371 : /* dr_analyze_innermost already explained the failure. */
5372 162 : return res;
5373 :
5374 11852 : if (dump_enabled_p ())
5375 1224 : dump_printf_loc (MSG_NOTE, vect_location,
5376 : "\touter base_address: %T\n"
5377 : "\touter offset from base address: %T\n"
5378 : "\touter constant offset from base address: %T\n"
5379 : "\touter step: %T\n"
5380 : "\touter base alignment: %d\n\n"
5381 : "\touter base misalignment: %d\n"
5382 : "\touter offset alignment: %d\n"
5383 : "\touter step alignment: %d\n",
5384 : STMT_VINFO_DR_BASE_ADDRESS (stmt_info),
5385 : STMT_VINFO_DR_OFFSET (stmt_info),
5386 : STMT_VINFO_DR_INIT (stmt_info),
5387 : STMT_VINFO_DR_STEP (stmt_info),
5388 : STMT_VINFO_DR_BASE_ALIGNMENT (stmt_info),
5389 : STMT_VINFO_DR_BASE_MISALIGNMENT (stmt_info),
5390 : STMT_VINFO_DR_OFFSET_ALIGNMENT (stmt_info),
5391 : STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
5392 : }
5393 :
5394 : /* Set vectype for STMT. */
5395 16309953 : scalar_type = TREE_TYPE (DR_REF (dr));
5396 16309953 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
5397 16309953 : if (!vectype)
5398 : {
5399 2149517 : if (dump_enabled_p ())
5400 : {
5401 2284 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5402 : "not vectorized: no vectype for stmt: %G",
5403 : stmt_info->stmt);
5404 2284 : dump_printf (MSG_MISSED_OPTIMIZATION, " scalar_type: ");
5405 2284 : dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_DETAILS,
5406 : scalar_type);
5407 2284 : dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
5408 : }
5409 :
5410 2149517 : if (is_a <bb_vec_info> (vinfo))
5411 : {
5412 : /* No vector type is fine, the ref can still participate
5413 : in dependence analysis, we just can't vectorize it. */
5414 2083165 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5415 2083165 : continue;
5416 : }
5417 66352 : if (fatal)
5418 66352 : *fatal = false;
5419 66352 : return opt_result::failure_at (stmt_info->stmt,
5420 : "not vectorized:"
5421 : " no vectype for stmt: %G"
5422 : " scalar_type: %T\n",
5423 : stmt_info->stmt, scalar_type);
5424 : }
5425 : else
5426 : {
5427 14160436 : if (dump_enabled_p ())
5428 85663 : dump_printf_loc (MSG_NOTE, vect_location,
5429 : "got vectype for stmt: %G%T\n",
5430 : stmt_info->stmt, vectype);
5431 : }
5432 :
5433 : /* Leave the BB vectorizer to pick the vector type later, based on
5434 : the final dataref group size and SLP node size. */
5435 14160436 : if (is_a <loop_vec_info> (vinfo))
5436 1136973 : STMT_VINFO_VECTYPE (stmt_info) = vectype;
5437 :
5438 14160436 : if (gatherscatter != SG_NONE)
5439 : {
5440 114705 : gather_scatter_info gs_info;
5441 114705 : if (!vect_check_gather_scatter (stmt_info, vectype,
5442 : as_a <loop_vec_info> (vinfo),
5443 : &gs_info)
5444 225311 : || !get_vectype_for_scalar_type (vinfo,
5445 110606 : TREE_TYPE (gs_info.offset)))
5446 : {
5447 8013 : if (fatal)
5448 8013 : *fatal = false;
5449 8013 : return opt_result::failure_at
5450 8716 : (stmt_info->stmt,
5451 : (gatherscatter == GATHER)
5452 : ? "not vectorized: not suitable for gather load %G"
5453 : : "not vectorized: not suitable for scatter store %G",
5454 : stmt_info->stmt);
5455 : }
5456 106692 : STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
5457 : }
5458 : }
5459 :
5460 : /* We used to stop processing and prune the list here. Verify we no
5461 : longer need to. */
5462 4487558 : gcc_assert (i == datarefs.length ());
5463 :
5464 2807910 : return opt_result::success ();
5465 : }
5466 :
5467 :
5468 : /* Function vect_get_new_vect_var.
5469 :
5470 : Returns a name for a new variable. The current naming scheme appends the
5471 : prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to
5472 : the name of vectorizer generated variables, and appends that to NAME if
5473 : provided. */
5474 :
5475 : tree
5476 1946760 : vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)
5477 : {
5478 1946760 : const char *prefix;
5479 1946760 : tree new_vect_var;
5480 :
5481 1946760 : switch (var_kind)
5482 : {
5483 : case vect_simple_var:
5484 : prefix = "vect";
5485 : break;
5486 22994 : case vect_scalar_var:
5487 22994 : prefix = "stmp";
5488 22994 : break;
5489 20469 : case vect_mask_var:
5490 20469 : prefix = "mask";
5491 20469 : break;
5492 1400745 : case vect_pointer_var:
5493 1400745 : prefix = "vectp";
5494 1400745 : break;
5495 0 : default:
5496 0 : gcc_unreachable ();
5497 : }
5498 :
5499 1946760 : if (name)
5500 : {
5501 1098537 : char* tmp = concat (prefix, "_", name, NULL);
5502 1098537 : new_vect_var = create_tmp_reg (type, tmp);
5503 1098537 : free (tmp);
5504 : }
5505 : else
5506 848223 : new_vect_var = create_tmp_reg (type, prefix);
5507 :
5508 1946760 : return new_vect_var;
5509 : }
5510 :
5511 : /* Like vect_get_new_vect_var but return an SSA name. */
5512 :
5513 : tree
5514 7164 : vect_get_new_ssa_name (tree type, enum vect_var_kind var_kind, const char *name)
5515 : {
5516 7164 : const char *prefix;
5517 7164 : tree new_vect_var;
5518 :
5519 7164 : switch (var_kind)
5520 : {
5521 : case vect_simple_var:
5522 : prefix = "vect";
5523 : break;
5524 315 : case vect_scalar_var:
5525 315 : prefix = "stmp";
5526 315 : break;
5527 0 : case vect_pointer_var:
5528 0 : prefix = "vectp";
5529 0 : break;
5530 0 : default:
5531 0 : gcc_unreachable ();
5532 : }
5533 :
5534 7164 : if (name)
5535 : {
5536 6682 : char* tmp = concat (prefix, "_", name, NULL);
5537 6682 : new_vect_var = make_temp_ssa_name (type, NULL, tmp);
5538 6682 : free (tmp);
5539 : }
5540 : else
5541 482 : new_vect_var = make_temp_ssa_name (type, NULL, prefix);
5542 :
5543 7164 : return new_vect_var;
5544 : }
5545 :
5546 : /* Duplicate points-to info on NAME from DR_INFO. */
5547 :
5548 : static void
5549 441971 : vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
5550 : {
5551 441971 : if (DR_PTR_INFO (dr_info->dr))
5552 : {
5553 295950 : duplicate_ssa_name_ptr_info (name, DR_PTR_INFO (dr_info->dr));
5554 : /* DR_PTR_INFO is for a base SSA name, not including constant or
5555 : variable offsets in the ref so its alignment info does not apply. */
5556 295950 : mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (name));
5557 : }
5558 146021 : else if (!SSA_NAME_PTR_INFO (name))
5559 : {
5560 146021 : tree base = get_base_address (dr_info->dr->ref);
5561 146021 : if (VAR_P (base)
5562 : || TREE_CODE (base) == PARM_DECL
5563 : || TREE_CODE (base) == RESULT_DECL)
5564 : {
5565 133417 : struct ptr_info_def *pi = get_ptr_info (name);
5566 133417 : pt_solution_set_var (&pi->pt, base);
5567 : }
5568 : }
5569 441971 : }
5570 :
5571 : /* Function vect_create_addr_base_for_vector_ref.
5572 :
5573 : Create an expression that computes the address of the first memory location
5574 : that will be accessed for a data reference.
5575 :
5576 : Input:
5577 : STMT_INFO: The statement containing the data reference.
5578 : NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
5579 : OFFSET: Optional. If supplied, it is be added to the initial address.
5580 : LOOP: Specify relative to which loop-nest should the address be computed.
5581 : For example, when the dataref is in an inner-loop nested in an
5582 : outer-loop that is now being vectorized, LOOP can be either the
5583 : outer-loop, or the inner-loop. The first memory location accessed
5584 : by the following dataref ('in' points to short):
5585 :
5586 : for (i=0; i<N; i++)
5587 : for (j=0; j<M; j++)
5588 : s += in[i+j]
5589 :
5590 : is as follows:
5591 : if LOOP=i_loop: &in (relative to i_loop)
5592 : if LOOP=j_loop: &in+i*2B (relative to j_loop)
5593 :
5594 : Output:
5595 : 1. Return an SSA_NAME whose value is the address of the memory location of
5596 : the first vector of the data reference.
5597 : 2. If new_stmt_list is not NULL_TREE after return then the caller must insert
5598 : these statement(s) which define the returned SSA_NAME.
5599 :
5600 : FORNOW: We are only handling array accesses with step 1. */
5601 :
5602 : tree
5603 691479 : vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
5604 : gimple_seq *new_stmt_list,
5605 : tree offset)
5606 : {
5607 691479 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
5608 691479 : struct data_reference *dr = dr_info->dr;
5609 691479 : const char *base_name;
5610 691479 : tree addr_base;
5611 691479 : tree dest;
5612 691479 : gimple_seq seq = NULL;
5613 691479 : tree vect_ptr_type;
5614 691479 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5615 691479 : innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
5616 :
5617 691479 : tree data_ref_base = unshare_expr (drb->base_address);
5618 691479 : tree vector_offset = NULL_TREE;
5619 691479 : if (loop_vinfo && dr_info->offset)
5620 18053 : vector_offset = unshare_expr (dr_info->offset);
5621 691479 : tree base_offset = unshare_expr (vector_offset
5622 : ? drb->offset
5623 673426 : : get_dr_vinfo_offset (vinfo, dr_info,
5624 : true));
5625 691479 : tree init = unshare_expr (drb->init);
5626 :
5627 691479 : if (loop_vinfo)
5628 132204 : base_name = get_name (data_ref_base);
5629 : else
5630 : {
5631 559275 : base_offset = ssize_int (0);
5632 559275 : init = ssize_int (0);
5633 559275 : base_name = get_name (DR_REF (dr));
5634 : }
5635 :
5636 : /* Create base_offset */
5637 691479 : base_offset = size_binop (PLUS_EXPR,
5638 : fold_convert (sizetype, base_offset),
5639 : fold_convert (sizetype, init));
5640 :
5641 691479 : if (offset)
5642 : {
5643 3115 : offset = fold_convert (sizetype, offset);
5644 3115 : base_offset = fold_build2 (PLUS_EXPR, sizetype,
5645 : base_offset, offset);
5646 : }
5647 :
5648 : /* base + base_offset */
5649 691479 : if (loop_vinfo)
5650 132204 : addr_base = fold_build_pointer_plus (data_ref_base, base_offset);
5651 : else
5652 1118550 : addr_base = build1 (ADDR_EXPR,
5653 559275 : build_pointer_type (TREE_TYPE (DR_REF (dr))),
5654 : /* Strip zero offset components since we don't need
5655 : them and they can confuse late diagnostics if
5656 : we CSE them wrongly. See PR106904 for example. */
5657 : unshare_expr (strip_zero_offset_components
5658 : (DR_REF (dr))));
5659 :
5660 691479 : vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
5661 691479 : dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
5662 :
5663 : /* Keep vectorizer-added offsets separate from the original scalar access
5664 : address. Forming "base + scalar offset" first gives the target a better
5665 : chance of sharing it with other address calculations, such as the
5666 : misalignment check used for masked alignment peeling. */
5667 691479 : if (vector_offset)
5668 : {
5669 18053 : tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
5670 : vect_pointer_var, base_name);
5671 18053 : gimple_seq addr_seq = NULL;
5672 18053 : addr_base = force_gimple_operand (addr_base, &addr_seq, true,
5673 : scalar_dest);
5674 18053 : gimple_seq_add_seq (&seq, addr_seq);
5675 18053 : addr_base = fold_build_pointer_plus (addr_base,
5676 : fold_convert (sizetype,
5677 : vector_offset));
5678 : }
5679 :
5680 691479 : gimple_seq addr_seq = NULL;
5681 691479 : addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
5682 691479 : gimple_seq_add_seq (&seq, addr_seq);
5683 691479 : gimple_seq_add_seq (new_stmt_list, seq);
5684 :
5685 691479 : if (TREE_CODE (addr_base) == SSA_NAME
5686 : /* We should only duplicate pointer info to newly created SSA names. */
5687 698504 : && SSA_NAME_VAR (addr_base) == dest)
5688 : {
5689 177431 : gcc_assert (!SSA_NAME_PTR_INFO (addr_base));
5690 177431 : vect_duplicate_ssa_name_ptr_info (addr_base, dr_info);
5691 : }
5692 :
5693 691479 : if (dump_enabled_p ())
5694 26049 : dump_printf_loc (MSG_NOTE, vect_location, "created %T\n", addr_base);
5695 :
5696 691479 : return addr_base;
5697 : }
5698 :
5699 :
5700 : /* Function vect_create_data_ref_ptr.
5701 :
5702 : Create a new pointer-to-AGGR_TYPE variable (ap), that points to the first
5703 : location accessed in the loop by STMT_INFO, along with the def-use update
5704 : chain to appropriately advance the pointer through the loop iterations.
5705 : Also set aliasing information for the pointer. This pointer is used by
5706 : the callers to this function to create a memory reference expression for
5707 : vector load/store access.
5708 :
5709 : Input:
5710 : 1. STMT_INFO: a stmt that references memory. Expected to be of the form
5711 : GIMPLE_ASSIGN <name, data-ref> or
5712 : GIMPLE_ASSIGN <data-ref, name>.
5713 : 2. AGGR_TYPE: the type of the reference, which should be either a vector
5714 : or an array.
5715 : 3. AT_LOOP: the loop where the vector memref is to be created.
5716 : 4. OFFSET (optional): a byte offset to be added to the initial address
5717 : accessed by the data-ref in STMT_INFO.
5718 : 5. BSI: location where the new stmts are to be placed if there is no loop
5719 : 6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain
5720 : pointing to the initial address.
5721 : 8. IV_STEP (optional, defaults to NULL): the amount that should be added
5722 : to the IV during each iteration of the loop. NULL says to move
5723 : by one copy of AGGR_TYPE up or down, depending on the step of the
5724 : data reference.
5725 :
5726 : Output:
5727 : 1. Declare a new ptr to vector_type, and have it point to the base of the
5728 : data reference (initial addressed accessed by the data reference).
5729 : For example, for vector of type V8HI, the following code is generated:
5730 :
5731 : v8hi *ap;
5732 : ap = (v8hi *)initial_address;
5733 :
5734 : if OFFSET is not supplied:
5735 : initial_address = &a[init];
5736 : if OFFSET is supplied:
5737 : initial_address = &a[init] + OFFSET;
5738 : if BYTE_OFFSET is supplied:
5739 : initial_address = &a[init] + BYTE_OFFSET;
5740 :
5741 : Return the initial_address in INITIAL_ADDRESS.
5742 :
5743 : 2. If ONLY_INIT is true, just return the initial pointer. Otherwise, also
5744 : update the pointer in each iteration of the loop.
5745 :
5746 : Return the increment stmt that updates the pointer in PTR_INCR.
5747 :
5748 : 3. Return the pointer. */
5749 :
5750 : tree
5751 691213 : vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
5752 : tree aggr_type, class loop *at_loop, tree offset,
5753 : tree *initial_address, gimple_stmt_iterator *gsi,
5754 : gimple **ptr_incr, bool only_init,
5755 : tree iv_step)
5756 : {
5757 691213 : const char *base_name;
5758 691213 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5759 691213 : class loop *loop = NULL;
5760 691213 : bool nested_in_vect_loop = false;
5761 691213 : class loop *containing_loop = NULL;
5762 691213 : tree aggr_ptr_type;
5763 691213 : tree aggr_ptr;
5764 691213 : tree new_temp;
5765 691213 : gimple_seq new_stmt_list = NULL;
5766 691213 : edge pe = NULL;
5767 691213 : basic_block new_bb;
5768 691213 : tree aggr_ptr_init;
5769 691213 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
5770 691213 : struct data_reference *dr = dr_info->dr;
5771 691213 : tree aptr;
5772 691213 : gimple_stmt_iterator incr_gsi;
5773 691213 : bool insert_after;
5774 691213 : tree indx_before_incr, indx_after_incr;
5775 691213 : gimple *incr;
5776 691213 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5777 :
5778 691213 : gcc_assert (iv_step != NULL_TREE
5779 : || TREE_CODE (aggr_type) == ARRAY_TYPE
5780 : || TREE_CODE (aggr_type) == VECTOR_TYPE);
5781 :
5782 691213 : if (loop_vinfo)
5783 : {
5784 131938 : loop = LOOP_VINFO_LOOP (loop_vinfo);
5785 131938 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
5786 131938 : containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
5787 131938 : pe = loop_preheader_edge (loop);
5788 : }
5789 : else
5790 : {
5791 559275 : gcc_assert (bb_vinfo);
5792 559275 : only_init = true;
5793 559275 : if (ptr_incr)
5794 0 : *ptr_incr = NULL;
5795 : }
5796 :
5797 : /* Create an expression for the first address accessed by this load
5798 : in LOOP. */
5799 691213 : base_name = get_name (DR_BASE_ADDRESS (dr));
5800 :
5801 691213 : if (dump_enabled_p ())
5802 : {
5803 25948 : tree dr_base_type = TREE_TYPE (DR_BASE_OBJECT (dr));
5804 25948 : dump_printf_loc (MSG_NOTE, vect_location,
5805 : "create %s-pointer variable to type: %T",
5806 25948 : get_tree_code_name (TREE_CODE (aggr_type)),
5807 : aggr_type);
5808 25948 : if (TREE_CODE (dr_base_type) == ARRAY_TYPE)
5809 13759 : dump_printf (MSG_NOTE, " vectorizing an array ref: ");
5810 12189 : else if (TREE_CODE (dr_base_type) == VECTOR_TYPE)
5811 0 : dump_printf (MSG_NOTE, " vectorizing a vector ref: ");
5812 12189 : else if (TREE_CODE (dr_base_type) == RECORD_TYPE)
5813 1660 : dump_printf (MSG_NOTE, " vectorizing a record based array ref: ");
5814 : else
5815 10529 : dump_printf (MSG_NOTE, " vectorizing a pointer ref: ");
5816 25948 : dump_printf (MSG_NOTE, "%T\n", DR_BASE_OBJECT (dr));
5817 : }
5818 :
5819 : /* (1) Create the new aggregate-pointer variable.
5820 : Vector and array types inherit the alias set of their component
5821 : type by default so we need to use a ref-all pointer if the data
5822 : reference does not conflict with the created aggregated data
5823 : reference because it is not addressable. */
5824 691213 : bool need_ref_all = false;
5825 691213 : if (!alias_sets_conflict_p (get_alias_set (aggr_type),
5826 : get_alias_set (DR_REF (dr))))
5827 : need_ref_all = true;
5828 : /* Likewise for any of the data references in the stmt group. */
5829 588506 : else if (DR_GROUP_SIZE (stmt_info) > 1)
5830 : {
5831 472493 : stmt_vec_info sinfo = DR_GROUP_FIRST_ELEMENT (stmt_info);
5832 1322827 : do
5833 : {
5834 1322827 : struct data_reference *sdr = STMT_VINFO_DATA_REF (sinfo);
5835 1322827 : if (!alias_sets_conflict_p (get_alias_set (aggr_type),
5836 : get_alias_set (DR_REF (sdr))))
5837 : {
5838 : need_ref_all = true;
5839 : break;
5840 : }
5841 1297283 : sinfo = DR_GROUP_NEXT_ELEMENT (sinfo);
5842 : }
5843 1297283 : while (sinfo);
5844 : }
5845 691213 : aggr_ptr_type = build_pointer_type_for_mode (aggr_type, VOIDmode,
5846 : need_ref_all);
5847 691213 : aggr_ptr = vect_get_new_vect_var (aggr_ptr_type, vect_pointer_var, base_name);
5848 :
5849 :
5850 : /* Note: If the dataref is in an inner-loop nested in LOOP, and we are
5851 : vectorizing LOOP (i.e., outer-loop vectorization), we need to create two
5852 : def-use update cycles for the pointer: one relative to the outer-loop
5853 : (LOOP), which is what steps (3) and (4) below do. The other is relative
5854 : to the inner-loop (which is the inner-most loop containing the dataref),
5855 : and this is done be step (5) below.
5856 :
5857 : When vectorizing inner-most loops, the vectorized loop (LOOP) is also the
5858 : inner-most loop, and so steps (3),(4) work the same, and step (5) is
5859 : redundant. Steps (3),(4) create the following:
5860 :
5861 : vp0 = &base_addr;
5862 : LOOP: vp1 = phi(vp0,vp2)
5863 : ...
5864 : ...
5865 : vp2 = vp1 + step
5866 : goto LOOP
5867 :
5868 : If there is an inner-loop nested in loop, then step (5) will also be
5869 : applied, and an additional update in the inner-loop will be created:
5870 :
5871 : vp0 = &base_addr;
5872 : LOOP: vp1 = phi(vp0,vp2)
5873 : ...
5874 : inner: vp3 = phi(vp1,vp4)
5875 : vp4 = vp3 + inner_step
5876 : if () goto inner
5877 : ...
5878 : vp2 = vp1 + step
5879 : if () goto LOOP */
5880 :
5881 : /* (2) Calculate the initial address of the aggregate-pointer, and set
5882 : the aggregate-pointer to point to it before the loop. */
5883 :
5884 : /* Create: (&(base[init_val]+offset) in the loop preheader. */
5885 :
5886 691213 : new_temp = vect_create_addr_base_for_vector_ref (vinfo,
5887 : stmt_info, &new_stmt_list,
5888 : offset);
5889 691213 : if (new_stmt_list)
5890 : {
5891 177313 : if (pe)
5892 : {
5893 56222 : new_bb = gsi_insert_seq_on_edge_immediate (pe, new_stmt_list);
5894 56222 : gcc_assert (!new_bb);
5895 : }
5896 : else
5897 121091 : gsi_insert_seq_before (gsi, new_stmt_list, GSI_SAME_STMT);
5898 : }
5899 :
5900 691213 : *initial_address = new_temp;
5901 691213 : aggr_ptr_init = new_temp;
5902 :
5903 : /* (3) Handle the updating of the aggregate-pointer inside the loop.
5904 : This is needed when ONLY_INIT is false, and also when AT_LOOP is the
5905 : inner-loop nested in LOOP (during outer-loop vectorization). */
5906 :
5907 : /* No update in loop is required. */
5908 691213 : if (only_init && (!loop_vinfo || at_loop == loop))
5909 : aptr = aggr_ptr_init;
5910 : else
5911 : {
5912 : /* Accesses to invariant addresses should be handled specially
5913 : by the caller. */
5914 131930 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
5915 131930 : gcc_assert (!integer_zerop (step));
5916 :
5917 131930 : if (iv_step == NULL_TREE)
5918 : {
5919 : /* The step of the aggregate pointer is the type size,
5920 : negated for downward accesses. */
5921 0 : iv_step = TYPE_SIZE_UNIT (aggr_type);
5922 0 : if (tree_int_cst_sgn (step) == -1)
5923 0 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
5924 : }
5925 :
5926 131930 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
5927 :
5928 263860 : create_iv (aggr_ptr_init, PLUS_EXPR,
5929 : iv_step, aggr_ptr, loop, &incr_gsi, insert_after,
5930 : &indx_before_incr, &indx_after_incr,
5931 : !loop_vinfo
5932 131930 : || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
5933 131930 : incr = gsi_stmt (incr_gsi);
5934 :
5935 : /* Copy the points-to information if it exists. */
5936 131930 : vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
5937 131930 : vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
5938 131930 : if (ptr_incr)
5939 0 : *ptr_incr = incr;
5940 :
5941 131930 : aptr = indx_before_incr;
5942 : }
5943 :
5944 691213 : if (!nested_in_vect_loop || only_init)
5945 : return aptr;
5946 :
5947 :
5948 : /* (4) Handle the updating of the aggregate-pointer inside the inner-loop
5949 : nested in LOOP, if exists. */
5950 :
5951 340 : gcc_assert (nested_in_vect_loop);
5952 340 : if (!only_init)
5953 : {
5954 340 : standard_iv_increment_position (containing_loop, &incr_gsi,
5955 : &insert_after);
5956 340 : create_iv (aptr, PLUS_EXPR, DR_STEP (dr),
5957 : aggr_ptr, containing_loop, &incr_gsi, insert_after,
5958 : &indx_before_incr, &indx_after_incr);
5959 340 : incr = gsi_stmt (incr_gsi);
5960 :
5961 : /* Copy the points-to information if it exists. */
5962 340 : vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
5963 340 : vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
5964 340 : if (ptr_incr)
5965 0 : *ptr_incr = incr;
5966 :
5967 340 : return indx_before_incr;
5968 : }
5969 : else
5970 : gcc_unreachable ();
5971 : }
5972 :
5973 :
5974 : /* Function bump_vector_ptr
5975 :
5976 : Increment DATAREF_PTR by UPDATE.
5977 :
5978 : Input:
5979 : DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated
5980 : in the loop.
5981 : GSI - location where the new update stmt is to be placed.
5982 : STMT_INFO - the original scalar memory-access stmt that is being vectorized.
5983 : UPDATE - The offset by which to bump the pointer.
5984 :
5985 : Output: Return NEW_DATAREF_PTR as illustrated above.
5986 :
5987 : */
5988 :
5989 : tree
5990 237029 : bump_vector_ptr (vec_info *vinfo,
5991 : tree dataref_ptr, gimple_stmt_iterator *gsi,
5992 : stmt_vec_info stmt_info, tree update)
5993 : {
5994 237029 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
5995 237029 : gimple *incr_stmt;
5996 237029 : tree new_dataref_ptr;
5997 :
5998 237029 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
5999 114426 : new_dataref_ptr = copy_ssa_name (dataref_ptr);
6000 122603 : else if (is_gimple_min_invariant (dataref_ptr))
6001 : /* When possible avoid emitting a separate increment stmt that will
6002 : force the addressed object addressable. */
6003 245206 : return build1 (ADDR_EXPR, TREE_TYPE (dataref_ptr),
6004 122603 : fold_build2 (MEM_REF,
6005 : TREE_TYPE (TREE_TYPE (dataref_ptr)),
6006 : dataref_ptr,
6007 122603 : fold_convert (ptr_type_node, update)));
6008 : else
6009 0 : new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
6010 114426 : incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
6011 : dataref_ptr, update);
6012 114426 : vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
6013 : /* Fold the increment, avoiding excessive chains use-def chains of
6014 : those, leading to compile-time issues for passes until the next
6015 : forwprop pass which would do this as well. */
6016 114426 : gimple_stmt_iterator fold_gsi = gsi_for_stmt (incr_stmt);
6017 114426 : if (fold_stmt (&fold_gsi, follow_all_ssa_edges))
6018 : {
6019 74601 : incr_stmt = gsi_stmt (fold_gsi);
6020 74601 : update_stmt (incr_stmt);
6021 : }
6022 :
6023 : /* Copy the points-to information if it exists. */
6024 114426 : duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
6025 :
6026 114426 : return new_dataref_ptr;
6027 : }
6028 :
6029 :
6030 : /* Copy memory reference info such as base/clique from the SRC reference
6031 : to the DEST MEM_REF. */
6032 :
6033 : void
6034 943458 : vect_copy_ref_info (tree dest, tree src)
6035 : {
6036 943458 : if (TREE_CODE (dest) != MEM_REF)
6037 : return;
6038 :
6039 : tree src_base = src;
6040 1911980 : while (handled_component_p (src_base))
6041 973225 : src_base = TREE_OPERAND (src_base, 0);
6042 938755 : if (TREE_CODE (src_base) != MEM_REF
6043 938755 : && TREE_CODE (src_base) != TARGET_MEM_REF)
6044 : return;
6045 :
6046 509347 : MR_DEPENDENCE_CLIQUE (dest) = MR_DEPENDENCE_CLIQUE (src_base);
6047 509347 : MR_DEPENDENCE_BASE (dest) = MR_DEPENDENCE_BASE (src_base);
6048 : }
6049 :
6050 :
6051 : /* Function vect_create_destination_var.
6052 :
6053 : Create a new temporary of type VECTYPE. */
6054 :
6055 : tree
6056 528545 : vect_create_destination_var (tree scalar_dest, tree vectype)
6057 : {
6058 528545 : tree vec_dest;
6059 528545 : const char *name;
6060 528545 : char *new_name;
6061 528545 : tree type;
6062 528545 : enum vect_var_kind kind;
6063 :
6064 528545 : kind = vectype
6065 1034096 : ? VECTOR_BOOLEAN_TYPE_P (vectype)
6066 505551 : ? vect_mask_var
6067 : : vect_simple_var
6068 : : vect_scalar_var;
6069 22994 : type = vectype ? vectype : TREE_TYPE (scalar_dest);
6070 :
6071 528545 : gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);
6072 :
6073 528545 : name = get_name (scalar_dest);
6074 528545 : if (name)
6075 178258 : new_name = xasprintf ("%s_%u", name, SSA_NAME_VERSION (scalar_dest));
6076 : else
6077 350287 : new_name = xasprintf ("_%u", SSA_NAME_VERSION (scalar_dest));
6078 528545 : vec_dest = vect_get_new_vect_var (type, kind, new_name);
6079 528545 : free (new_name);
6080 :
6081 528545 : return vec_dest;
6082 : }
6083 :
6084 : /* Function vect_grouped_store_supported.
6085 :
6086 : Returns TRUE if interleave high and interleave low permutations
6087 : are supported, and FALSE otherwise. */
6088 :
6089 : bool
6090 2782 : vect_grouped_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
6091 : {
6092 2782 : machine_mode mode = TYPE_MODE (vectype);
6093 :
6094 : /* vect_permute_store_chain requires the group size to be equal to 3 or
6095 : be a power of two. */
6096 2782 : if (count != 3 && exact_log2 (count) == -1)
6097 : {
6098 551 : if (dump_enabled_p ())
6099 15 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6100 : "the size of the group of accesses"
6101 : " is not a power of 2 or not equal to 3\n");
6102 : return false;
6103 : }
6104 :
6105 : /* Check that the permutation is supported. */
6106 2231 : if (VECTOR_MODE_P (mode))
6107 : {
6108 2231 : unsigned int i;
6109 2231 : if (count == 3)
6110 : {
6111 950 : unsigned int j0 = 0, j1 = 0, j2 = 0;
6112 950 : unsigned int i, j;
6113 :
6114 950 : unsigned int nelt;
6115 1900 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6116 : {
6117 : if (dump_enabled_p ())
6118 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6119 : "cannot handle groups of 3 stores for"
6120 : " variable-length vectors\n");
6121 : return false;
6122 : }
6123 :
6124 950 : vec_perm_builder sel (nelt, nelt, 1);
6125 950 : sel.quick_grow (nelt);
6126 950 : vec_perm_indices indices;
6127 3575 : for (j = 0; j < 3; j++)
6128 : {
6129 2700 : int nelt0 = ((3 - j) * nelt) % 3;
6130 2700 : int nelt1 = ((3 - j) * nelt + 1) % 3;
6131 2700 : int nelt2 = ((3 - j) * nelt + 2) % 3;
6132 9606 : for (i = 0; i < nelt; i++)
6133 : {
6134 6906 : if (3 * i + nelt0 < nelt)
6135 2340 : sel[3 * i + nelt0] = j0++;
6136 6906 : if (3 * i + nelt1 < nelt)
6137 2301 : sel[3 * i + nelt1] = nelt + j1++;
6138 6906 : if (3 * i + nelt2 < nelt)
6139 2265 : sel[3 * i + nelt2] = 0;
6140 : }
6141 2700 : indices.new_vector (sel, 2, nelt);
6142 2700 : if (!can_vec_perm_const_p (mode, mode, indices))
6143 : {
6144 66 : if (dump_enabled_p ())
6145 37 : dump_printf (MSG_MISSED_OPTIMIZATION,
6146 : "permutation op not supported by target.\n");
6147 : return false;
6148 : }
6149 :
6150 8964 : for (i = 0; i < nelt; i++)
6151 : {
6152 6330 : if (3 * i + nelt0 < nelt)
6153 2116 : sel[3 * i + nelt0] = 3 * i + nelt0;
6154 6330 : if (3 * i + nelt1 < nelt)
6155 2107 : sel[3 * i + nelt1] = 3 * i + nelt1;
6156 6330 : if (3 * i + nelt2 < nelt)
6157 2107 : sel[3 * i + nelt2] = nelt + j2++;
6158 : }
6159 2634 : indices.new_vector (sel, 2, nelt);
6160 2634 : if (!can_vec_perm_const_p (mode, mode, indices))
6161 : {
6162 9 : if (dump_enabled_p ())
6163 9 : dump_printf (MSG_MISSED_OPTIMIZATION,
6164 : "permutation op not supported by target.\n");
6165 : return false;
6166 : }
6167 : }
6168 : return true;
6169 950 : }
6170 : else
6171 : {
6172 : /* If length is not equal to 3 then only power of 2 is supported. */
6173 1281 : gcc_assert (pow2p_hwi (count));
6174 2562 : poly_uint64 nelt = GET_MODE_NUNITS (mode);
6175 :
6176 : /* The encoding has 2 interleaved stepped patterns. */
6177 2562 : if(!multiple_p (nelt, 2))
6178 1235 : return false;
6179 1281 : vec_perm_builder sel (nelt, 2, 3);
6180 1281 : sel.quick_grow (6);
6181 6405 : for (i = 0; i < 3; i++)
6182 : {
6183 3843 : sel[i * 2] = i;
6184 3843 : sel[i * 2 + 1] = i + nelt;
6185 : }
6186 1281 : vec_perm_indices indices (sel, 2, nelt);
6187 1281 : if (can_vec_perm_const_p (mode, mode, indices))
6188 : {
6189 8645 : for (i = 0; i < 6; i++)
6190 7410 : sel[i] += exact_div (nelt, 2);
6191 1235 : indices.new_vector (sel, 2, nelt);
6192 1235 : if (can_vec_perm_const_p (mode, mode, indices))
6193 1235 : return true;
6194 : }
6195 1281 : }
6196 : }
6197 :
6198 46 : if (dump_enabled_p ())
6199 3 : dump_printf (MSG_MISSED_OPTIMIZATION,
6200 : "permutation op not supported by target.\n");
6201 : return false;
6202 : }
6203 :
6204 : /* Return FN if vec_{mask_,mask_len_}store_lanes is available for COUNT vectors
6205 : of type VECTYPE. MASKED_P says whether the masked form is needed. */
6206 :
6207 : internal_fn
6208 41744 : vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
6209 : bool masked_p)
6210 : {
6211 41744 : if (vect_lanes_optab_supported_p ("vec_mask_len_store_lanes",
6212 : vec_mask_len_store_lanes_optab, vectype,
6213 : count))
6214 : return IFN_MASK_LEN_STORE_LANES;
6215 41744 : else if (masked_p)
6216 : {
6217 111 : if (vect_lanes_optab_supported_p ("vec_mask_store_lanes",
6218 : vec_mask_store_lanes_optab, vectype,
6219 : count))
6220 0 : return IFN_MASK_STORE_LANES;
6221 : }
6222 : else
6223 : {
6224 41633 : if (vect_lanes_optab_supported_p ("vec_store_lanes",
6225 : vec_store_lanes_optab, vectype, count))
6226 0 : return IFN_STORE_LANES;
6227 : }
6228 : return IFN_LAST;
6229 : }
6230 :
6231 :
6232 : /* Function vect_setup_realignment
6233 :
6234 : This function is called when vectorizing an unaligned load using
6235 : the dr_explicit_realign[_optimized] scheme.
6236 : This function generates the following code at the loop prolog:
6237 :
6238 : p = initial_addr;
6239 : x msq_init = *(floor(p)); # prolog load
6240 : realignment_token = call target_builtin;
6241 : loop:
6242 : x msq = phi (msq_init, ---)
6243 :
6244 : The stmts marked with x are generated only for the case of
6245 : dr_explicit_realign_optimized.
6246 :
6247 : The code above sets up a new (vector) pointer, pointing to the first
6248 : location accessed by STMT_INFO, and a "floor-aligned" load using that
6249 : pointer. It also generates code to compute the "realignment-token"
6250 : (if the relevant target hook was defined), and creates a phi-node at the
6251 : loop-header bb whose arguments are the result of the prolog-load (created
6252 : by this function) and the result of a load that takes place in the loop
6253 : (to be created by the caller to this function).
6254 :
6255 : For the case of dr_explicit_realign_optimized:
6256 : The caller to this function uses the phi-result (msq) to create the
6257 : realignment code inside the loop, and sets up the missing phi argument,
6258 : as follows:
6259 : loop:
6260 : msq = phi (msq_init, lsq)
6261 : lsq = *(floor(p')); # load in loop
6262 : result = realign_load (msq, lsq, realignment_token);
6263 :
6264 : For the case of dr_explicit_realign:
6265 : loop:
6266 : msq = *(floor(p)); # load in loop
6267 : p' = p + (VS-1);
6268 : lsq = *(floor(p')); # load in loop
6269 : result = realign_load (msq, lsq, realignment_token);
6270 :
6271 : Input:
6272 : STMT_INFO - (scalar) load stmt to be vectorized. This load accesses
6273 : a memory location that may be unaligned.
6274 : BSI - place where new code is to be inserted.
6275 : ALIGNMENT_SUPPORT_SCHEME - which of the two misalignment handling schemes
6276 : is used.
6277 :
6278 : Output:
6279 : REALIGNMENT_TOKEN - the result of a call to the builtin_mask_for_load
6280 : target hook, if defined.
6281 : Return value - the result of the loop-header phi node. */
6282 :
6283 : tree
6284 0 : vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
6285 : gimple_stmt_iterator *gsi, tree *realignment_token,
6286 : enum dr_alignment_support alignment_support_scheme,
6287 : tree init_addr,
6288 : class loop **at_loop)
6289 : {
6290 0 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6291 0 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
6292 0 : struct data_reference *dr = dr_info->dr;
6293 0 : class loop *loop = NULL;
6294 0 : edge pe = NULL;
6295 0 : tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
6296 0 : tree vec_dest;
6297 0 : gimple *inc;
6298 0 : tree ptr;
6299 0 : tree data_ref;
6300 0 : basic_block new_bb;
6301 0 : tree msq_init = NULL_TREE;
6302 0 : tree new_temp;
6303 0 : gphi *phi_stmt;
6304 0 : tree msq = NULL_TREE;
6305 0 : gimple_seq stmts = NULL;
6306 0 : bool compute_in_loop = false;
6307 0 : bool nested_in_vect_loop = false;
6308 0 : class loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
6309 0 : class loop *loop_for_initial_load = NULL;
6310 :
6311 0 : if (loop_vinfo)
6312 : {
6313 0 : loop = LOOP_VINFO_LOOP (loop_vinfo);
6314 0 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
6315 : }
6316 :
6317 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign
6318 : || alignment_support_scheme == dr_explicit_realign_optimized);
6319 :
6320 : /* We need to generate three things:
6321 : 1. the misalignment computation
6322 : 2. the extra vector load (for the optimized realignment scheme).
6323 : 3. the phi node for the two vectors from which the realignment is
6324 : done (for the optimized realignment scheme). */
6325 :
6326 : /* 1. Determine where to generate the misalignment computation.
6327 :
6328 : If INIT_ADDR is NULL_TREE, this indicates that the misalignment
6329 : calculation will be generated by this function, outside the loop (in the
6330 : preheader). Otherwise, INIT_ADDR had already been computed for us by the
6331 : caller, inside the loop.
6332 :
6333 : Background: If the misalignment remains fixed throughout the iterations of
6334 : the loop, then both realignment schemes are applicable, and also the
6335 : misalignment computation can be done outside LOOP. This is because we are
6336 : vectorizing LOOP, and so the memory accesses in LOOP advance in steps that
6337 : are a multiple of VS (the Vector Size), and therefore the misalignment in
6338 : different vectorized LOOP iterations is always the same.
6339 : The problem arises only if the memory access is in an inner-loop nested
6340 : inside LOOP, which is now being vectorized using outer-loop vectorization.
6341 : This is the only case when the misalignment of the memory access may not
6342 : remain fixed throughout the iterations of the inner-loop (as explained in
6343 : detail in vect_supportable_dr_alignment). In this case, not only is the
6344 : optimized realignment scheme not applicable, but also the misalignment
6345 : computation (and generation of the realignment token that is passed to
6346 : REALIGN_LOAD) have to be done inside the loop.
6347 :
6348 : In short, INIT_ADDR indicates whether we are in a COMPUTE_IN_LOOP mode
6349 : or not, which in turn determines if the misalignment is computed inside
6350 : the inner-loop, or outside LOOP. */
6351 :
6352 0 : if (init_addr != NULL_TREE || !loop_vinfo)
6353 : {
6354 0 : compute_in_loop = true;
6355 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign);
6356 : }
6357 :
6358 :
6359 : /* 2. Determine where to generate the extra vector load.
6360 :
6361 : For the optimized realignment scheme, instead of generating two vector
6362 : loads in each iteration, we generate a single extra vector load in the
6363 : preheader of the loop, and in each iteration reuse the result of the
6364 : vector load from the previous iteration. In case the memory access is in
6365 : an inner-loop nested inside LOOP, which is now being vectorized using
6366 : outer-loop vectorization, we need to determine whether this initial vector
6367 : load should be generated at the preheader of the inner-loop, or can be
6368 : generated at the preheader of LOOP. If the memory access has no evolution
6369 : in LOOP, it can be generated in the preheader of LOOP. Otherwise, it has
6370 : to be generated inside LOOP (in the preheader of the inner-loop). */
6371 :
6372 0 : if (nested_in_vect_loop)
6373 : {
6374 0 : tree outerloop_step = STMT_VINFO_DR_STEP (stmt_info);
6375 0 : bool invariant_in_outerloop =
6376 0 : (tree_int_cst_compare (outerloop_step, size_zero_node) == 0);
6377 0 : loop_for_initial_load = (invariant_in_outerloop ? loop : loop->inner);
6378 : }
6379 : else
6380 : loop_for_initial_load = loop;
6381 0 : if (at_loop)
6382 0 : *at_loop = loop_for_initial_load;
6383 :
6384 0 : tree vuse = NULL_TREE;
6385 0 : if (loop_for_initial_load)
6386 : {
6387 0 : pe = loop_preheader_edge (loop_for_initial_load);
6388 0 : if (gphi *vphi = get_virtual_phi (loop_for_initial_load->header))
6389 0 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
6390 : }
6391 0 : if (!vuse)
6392 0 : vuse = gimple_vuse (gsi_stmt (*gsi));
6393 :
6394 : /* 3. For the case of the optimized realignment, create the first vector
6395 : load at the loop preheader. */
6396 :
6397 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
6398 : {
6399 : /* Create msq_init = *(floor(p1)) in the loop preheader */
6400 0 : gassign *new_stmt;
6401 :
6402 0 : gcc_assert (!compute_in_loop);
6403 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6404 0 : ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
6405 : loop_for_initial_load, NULL_TREE,
6406 : &init_addr, NULL, &inc, true);
6407 0 : if (TREE_CODE (ptr) == SSA_NAME)
6408 0 : new_temp = copy_ssa_name (ptr);
6409 : else
6410 0 : new_temp = make_ssa_name (TREE_TYPE (ptr));
6411 0 : poly_uint64 align = DR_TARGET_ALIGNMENT (dr_info);
6412 0 : tree type = TREE_TYPE (ptr);
6413 0 : new_stmt = gimple_build_assign
6414 0 : (new_temp, BIT_AND_EXPR, ptr,
6415 0 : fold_build2 (MINUS_EXPR, type,
6416 : build_int_cst (type, 0),
6417 : build_int_cst (type, align)));
6418 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6419 0 : gcc_assert (!new_bb);
6420 0 : data_ref
6421 0 : = build2 (MEM_REF, TREE_TYPE (vec_dest), new_temp,
6422 : build_int_cst (reference_alias_ptr_type (DR_REF (dr)), 0));
6423 0 : vect_copy_ref_info (data_ref, DR_REF (dr));
6424 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
6425 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6426 0 : gimple_assign_set_lhs (new_stmt, new_temp);
6427 0 : gimple_set_vuse (new_stmt, vuse);
6428 0 : if (pe)
6429 : {
6430 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6431 0 : gcc_assert (!new_bb);
6432 : }
6433 : else
6434 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6435 :
6436 0 : msq_init = gimple_assign_lhs (new_stmt);
6437 : }
6438 :
6439 : /* 4. Create realignment token using a target builtin, if available.
6440 : It is done either inside the containing loop, or before LOOP (as
6441 : determined above). */
6442 :
6443 0 : if (targetm.vectorize.builtin_mask_for_load)
6444 : {
6445 0 : gcall *new_stmt;
6446 0 : tree builtin_decl;
6447 :
6448 : /* Compute INIT_ADDR - the initial addressed accessed by this memref. */
6449 0 : if (!init_addr)
6450 : {
6451 : /* Generate the INIT_ADDR computation outside LOOP. */
6452 0 : init_addr = vect_create_addr_base_for_vector_ref (vinfo,
6453 : stmt_info, &stmts,
6454 : NULL_TREE);
6455 0 : if (loop)
6456 : {
6457 0 : pe = loop_preheader_edge (loop);
6458 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
6459 0 : gcc_assert (!new_bb);
6460 : }
6461 : else
6462 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
6463 : }
6464 :
6465 0 : builtin_decl = targetm.vectorize.builtin_mask_for_load ();
6466 0 : new_stmt = gimple_build_call (builtin_decl, 1, init_addr);
6467 0 : vec_dest =
6468 0 : vect_create_destination_var (scalar_dest,
6469 : gimple_call_return_type (new_stmt));
6470 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6471 0 : gimple_call_set_lhs (new_stmt, new_temp);
6472 :
6473 0 : if (compute_in_loop)
6474 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6475 : else
6476 : {
6477 : /* Generate the misalignment computation outside LOOP. */
6478 0 : pe = loop_preheader_edge (loop);
6479 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6480 0 : gcc_assert (!new_bb);
6481 : }
6482 :
6483 0 : *realignment_token = gimple_call_lhs (new_stmt);
6484 :
6485 : /* The result of the CALL_EXPR to this builtin is determined from
6486 : the value of the parameter and no global variables are touched
6487 : which makes the builtin a "const" function. Requiring the
6488 : builtin to have the "const" attribute makes it unnecessary
6489 : to call mark_call_clobbered. */
6490 0 : gcc_assert (TREE_READONLY (builtin_decl));
6491 : }
6492 :
6493 0 : if (alignment_support_scheme == dr_explicit_realign)
6494 : return msq;
6495 :
6496 0 : gcc_assert (!compute_in_loop);
6497 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign_optimized);
6498 :
6499 :
6500 : /* 5. Create msq = phi <msq_init, lsq> in loop */
6501 :
6502 0 : pe = loop_preheader_edge (containing_loop);
6503 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6504 0 : msq = make_ssa_name (vec_dest);
6505 0 : phi_stmt = create_phi_node (msq, containing_loop->header);
6506 0 : add_phi_arg (phi_stmt, msq_init, pe, UNKNOWN_LOCATION);
6507 :
6508 0 : return msq;
6509 : }
6510 :
6511 :
6512 : /* Function vect_grouped_load_supported.
6513 :
6514 : COUNT is the size of the load group (the number of statements plus the
6515 : number of gaps). SINGLE_ELEMENT_P is true if there is actually
6516 : only one statement, with a gap of COUNT - 1.
6517 :
6518 : Returns true if a suitable permute exists. */
6519 :
6520 : bool
6521 2004 : vect_grouped_load_supported (tree vectype, bool single_element_p,
6522 : unsigned HOST_WIDE_INT count)
6523 : {
6524 2004 : machine_mode mode = TYPE_MODE (vectype);
6525 :
6526 : /* If this is single-element interleaving with an element distance
6527 : that leaves unused vector loads around punt - we at least create
6528 : very sub-optimal code in that case (and blow up memory,
6529 : see PR65518). */
6530 2004 : if (single_element_p && maybe_gt (count, TYPE_VECTOR_SUBPARTS (vectype)))
6531 : {
6532 42 : if (dump_enabled_p ())
6533 3 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6534 : "single-element interleaving not supported "
6535 : "for not adjacent vector loads\n");
6536 : return false;
6537 : }
6538 :
6539 : /* vect_permute_load_chain requires the group size to be equal to 3 or
6540 : be a power of two. */
6541 1962 : if (count != 3 && exact_log2 (count) == -1)
6542 : {
6543 220 : if (dump_enabled_p ())
6544 8 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6545 : "the size of the group of accesses"
6546 : " is not a power of 2 or not equal to 3\n");
6547 : return false;
6548 : }
6549 :
6550 : /* Check that the permutation is supported. */
6551 1742 : if (VECTOR_MODE_P (mode))
6552 : {
6553 1742 : unsigned int i, j;
6554 1742 : if (count == 3)
6555 : {
6556 838 : unsigned int nelt;
6557 1676 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6558 : {
6559 : if (dump_enabled_p ())
6560 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6561 : "cannot handle groups of 3 loads for"
6562 : " variable-length vectors\n");
6563 : return false;
6564 : }
6565 :
6566 838 : vec_perm_builder sel (nelt, nelt, 1);
6567 838 : sel.quick_grow (nelt);
6568 838 : vec_perm_indices indices;
6569 838 : unsigned int k;
6570 3295 : for (k = 0; k < 3; k++)
6571 : {
6572 8848 : for (i = 0; i < nelt; i++)
6573 6372 : if (3 * i + k < 2 * nelt)
6574 4256 : sel[i] = 3 * i + k;
6575 : else
6576 2116 : sel[i] = 0;
6577 2476 : indices.new_vector (sel, 2, nelt);
6578 2476 : if (!can_vec_perm_const_p (mode, mode, indices))
6579 : {
6580 19 : if (dump_enabled_p ())
6581 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6582 : "shuffle of 3 loads is not supported by"
6583 : " target\n");
6584 : return false;
6585 : }
6586 8649 : for (i = 0, j = 0; i < nelt; i++)
6587 6192 : if (3 * i + k < 2 * nelt)
6588 4128 : sel[i] = i;
6589 : else
6590 2064 : sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
6591 2457 : indices.new_vector (sel, 2, nelt);
6592 2457 : if (!can_vec_perm_const_p (mode, mode, indices))
6593 : {
6594 0 : if (dump_enabled_p ())
6595 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6596 : "shuffle of 3 loads is not supported by"
6597 : " target\n");
6598 : return false;
6599 : }
6600 : }
6601 : return true;
6602 838 : }
6603 : else
6604 : {
6605 : /* If length is not equal to 3 then only power of 2 is supported. */
6606 904 : gcc_assert (pow2p_hwi (count));
6607 1808 : poly_uint64 nelt = GET_MODE_NUNITS (mode);
6608 :
6609 : /* The encoding has a single stepped pattern. */
6610 904 : vec_perm_builder sel (nelt, 1, 3);
6611 904 : sel.quick_grow (3);
6612 4520 : for (i = 0; i < 3; i++)
6613 2712 : sel[i] = i * 2;
6614 904 : vec_perm_indices indices (sel, 2, nelt);
6615 904 : if (can_vec_perm_const_p (mode, mode, indices))
6616 : {
6617 3564 : for (i = 0; i < 3; i++)
6618 2673 : sel[i] = i * 2 + 1;
6619 891 : indices.new_vector (sel, 2, nelt);
6620 891 : if (can_vec_perm_const_p (mode, mode, indices))
6621 891 : return true;
6622 : }
6623 904 : }
6624 : }
6625 :
6626 13 : if (dump_enabled_p ())
6627 2 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6628 : "extract even/odd not supported by target\n");
6629 : return false;
6630 : }
6631 :
6632 : /* Return FN if vec_{masked_,mask_len_}load_lanes is available for COUNT vectors
6633 : of type VECTYPE. MASKED_P says whether the masked form is needed.
6634 : If it is available and ELSVALS is nonzero store the possible else values
6635 : in the vector it points to. */
6636 :
6637 : internal_fn
6638 145431 : vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
6639 : bool masked_p, vec<int> *elsvals)
6640 : {
6641 145431 : if (vect_lanes_optab_supported_p ("vec_mask_len_load_lanes",
6642 : vec_mask_len_load_lanes_optab, vectype,
6643 : count, elsvals))
6644 : return IFN_MASK_LEN_LOAD_LANES;
6645 145431 : else if (masked_p)
6646 : {
6647 30 : if (vect_lanes_optab_supported_p ("vec_mask_load_lanes",
6648 : vec_mask_load_lanes_optab, vectype,
6649 : count, elsvals))
6650 0 : return IFN_MASK_LOAD_LANES;
6651 : }
6652 : else
6653 : {
6654 145401 : if (vect_lanes_optab_supported_p ("vec_load_lanes", vec_load_lanes_optab,
6655 : vectype, count, elsvals))
6656 0 : return IFN_LOAD_LANES;
6657 : }
6658 : return IFN_LAST;
6659 : }
6660 :
6661 : /* Function vect_force_dr_alignment_p.
6662 :
6663 : Returns whether the alignment of a DECL can be forced to be aligned
6664 : on ALIGNMENT bit boundary. */
6665 :
6666 : bool
6667 751466 : vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
6668 : {
6669 751466 : if (!VAR_P (decl))
6670 : return false;
6671 :
6672 225921 : if (decl_in_symtab_p (decl)
6673 225921 : && (!symtab_node::get (decl)
6674 25137 : || !symtab_node::get (decl)->can_increase_alignment_p ()))
6675 : return false;
6676 :
6677 211499 : if (TREE_STATIC (decl))
6678 10715 : return (known_le (alignment,
6679 10715 : (unsigned HOST_WIDE_INT) MAX_OFILE_ALIGNMENT));
6680 : else
6681 200784 : return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
6682 : }
6683 :
6684 : /* Return whether the data reference DR_INFO is supported with respect to its
6685 : alignment.
6686 : If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
6687 : it is aligned, i.e., check if it is possible to vectorize it with different
6688 : alignment. If IS_GATHER_SCATTER is true we are dealing with a
6689 : gather/scatter. */
6690 :
6691 : enum dr_alignment_support
6692 2984593 : vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
6693 : tree vectype, int misalignment,
6694 : bool is_gather_scatter)
6695 : {
6696 2984593 : data_reference *dr = dr_info->dr;
6697 2984593 : stmt_vec_info stmt_info = dr_info->stmt;
6698 2984593 : machine_mode mode = TYPE_MODE (vectype);
6699 2984593 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6700 2984593 : class loop *vect_loop = NULL;
6701 2984593 : bool nested_in_vect_loop = false;
6702 :
6703 2984593 : if (misalignment == 0)
6704 : return dr_aligned;
6705 1827515 : else if (dr_safe_speculative_read_required (stmt_info))
6706 : return dr_unaligned_unsupported;
6707 :
6708 1404402 : if (loop_vinfo)
6709 : {
6710 983278 : vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
6711 983278 : nested_in_vect_loop = nested_in_vect_loop_p (vect_loop, stmt_info);
6712 : }
6713 :
6714 : /* Possibly unaligned access. */
6715 :
6716 : /* We can choose between using the implicit realignment scheme (generating
6717 : a misaligned_move stmt) and the explicit realignment scheme (generating
6718 : aligned loads with a REALIGN_LOAD). There are two variants to the
6719 : explicit realignment scheme: optimized, and unoptimized.
6720 : We can optimize the realignment only if the step between consecutive
6721 : vector loads is equal to the vector size. Since the vector memory
6722 : accesses advance in steps of VS (Vector Size) in the vectorized loop, it
6723 : is guaranteed that the misalignment amount remains the same throughout the
6724 : execution of the vectorized loop. Therefore, we can create the
6725 : "realignment token" (the permutation mask that is passed to REALIGN_LOAD)
6726 : at the loop preheader.
6727 :
6728 : However, in the case of outer-loop vectorization, when vectorizing a
6729 : memory access in the inner-loop nested within the LOOP that is now being
6730 : vectorized, while it is guaranteed that the misalignment of the
6731 : vectorized memory access will remain the same in different outer-loop
6732 : iterations, it is *not* guaranteed that is will remain the same throughout
6733 : the execution of the inner-loop. This is because the inner-loop advances
6734 : with the original scalar step (and not in steps of VS). If the inner-loop
6735 : step happens to be a multiple of VS, then the misalignment remains fixed
6736 : and we can use the optimized realignment scheme. For example:
6737 :
6738 : for (i=0; i<N; i++)
6739 : for (j=0; j<M; j++)
6740 : s += a[i+j];
6741 :
6742 : When vectorizing the i-loop in the above example, the step between
6743 : consecutive vector loads is 1, and so the misalignment does not remain
6744 : fixed across the execution of the inner-loop, and the realignment cannot
6745 : be optimized (as illustrated in the following pseudo vectorized loop):
6746 :
6747 : for (i=0; i<N; i+=4)
6748 : for (j=0; j<M; j++){
6749 : vs += vp[i+j]; // misalignment of &vp[i+j] is {0,1,2,3,0,1,2,3,...}
6750 : // when j is {0,1,2,3,4,5,6,7,...} respectively.
6751 : // (assuming that we start from an aligned address).
6752 : }
6753 :
6754 : We therefore have to use the unoptimized realignment scheme:
6755 :
6756 : for (i=0; i<N; i+=4)
6757 : for (j=k; j<M; j+=4)
6758 : vs += vp[i+j]; // misalignment of &vp[i+j] is always k (assuming
6759 : // that the misalignment of the initial address is
6760 : // 0).
6761 :
6762 : The loop can then be vectorized as follows:
6763 :
6764 : for (k=0; k<4; k++){
6765 : rt = get_realignment_token (&vp[k]);
6766 : for (i=0; i<N; i+=4){
6767 : v1 = vp[i+k];
6768 : for (j=k; j<M; j+=4){
6769 : v2 = vp[i+j+VS-1];
6770 : va = REALIGN_LOAD <v1,v2,rt>;
6771 : vs += va;
6772 : v1 = v2;
6773 : }
6774 : }
6775 : } */
6776 :
6777 1404402 : if (DR_IS_READ (dr) && !is_gather_scatter)
6778 : {
6779 636550 : if (can_implement_p (vec_realign_load_optab, mode)
6780 636550 : && (!targetm.vectorize.builtin_mask_for_load
6781 0 : || targetm.vectorize.builtin_mask_for_load ()))
6782 : {
6783 : /* If we are doing SLP then the accesses need not have the
6784 : same alignment, instead it depends on the SLP group size. */
6785 0 : if (loop_vinfo
6786 0 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
6787 0 : && !multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
6788 0 : * (DR_GROUP_SIZE
6789 0 : (DR_GROUP_FIRST_ELEMENT (stmt_info))),
6790 0 : TYPE_VECTOR_SUBPARTS (vectype)))
6791 : ;
6792 0 : else if (!loop_vinfo
6793 0 : || (nested_in_vect_loop
6794 0 : && maybe_ne (TREE_INT_CST_LOW (DR_STEP (dr)),
6795 0 : GET_MODE_SIZE (TYPE_MODE (vectype)))))
6796 0 : return dr_explicit_realign;
6797 : else
6798 : return dr_explicit_realign_optimized;
6799 : }
6800 : }
6801 :
6802 1404402 : bool is_packed = not_size_aligned (DR_REF (dr));
6803 1404402 : if (misalignment == DR_MISALIGNMENT_UNKNOWN
6804 1404402 : && is_gather_scatter)
6805 2995 : misalignment = (get_object_alignment (DR_REF (dr))
6806 2995 : % (GET_MODE_BITSIZE (GET_MODE_INNER (mode))))
6807 2995 : / BITS_PER_UNIT;
6808 1404402 : if (targetm.vectorize.support_vector_misalignment (mode, misalignment,
6809 : is_packed,
6810 : is_gather_scatter))
6811 1404207 : return dr_unaligned_supported;
6812 :
6813 : /* Unsupported. */
6814 : return dr_unaligned_unsupported;
6815 : }
|
