Branch data Line data Source code
1 : : /* Data References Analysis and Manipulation Utilities for Vectorization.
2 : : Copyright (C) 2003-2025 Free Software Foundation, Inc.
3 : : Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 : : and Ira Rosen <irar@il.ibm.com>
5 : :
6 : : This file is part of GCC.
7 : :
8 : : GCC is free software; you can redistribute it and/or modify it under
9 : : the terms of the GNU General Public License as published by the Free
10 : : Software Foundation; either version 3, or (at your option) any later
11 : : version.
12 : :
13 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : : for more details.
17 : :
18 : : You should have received a copy of the GNU General Public License
19 : : along with GCC; see the file COPYING3. If not see
20 : : <http://www.gnu.org/licenses/>. */
21 : :
22 : : #define INCLUDE_ALGORITHM
23 : : #include "config.h"
24 : : #include "system.h"
25 : : #include "coretypes.h"
26 : : #include "backend.h"
27 : : #include "target.h"
28 : : #include "rtl.h"
29 : : #include "tree.h"
30 : : #include "gimple.h"
31 : : #include "predict.h"
32 : : #include "memmodel.h"
33 : : #include "tm_p.h"
34 : : #include "ssa.h"
35 : : #include "optabs-tree.h"
36 : : #include "cgraph.h"
37 : : #include "dumpfile.h"
38 : : #include "pretty-print.h"
39 : : #include "alias.h"
40 : : #include "fold-const.h"
41 : : #include "stor-layout.h"
42 : : #include "tree-eh.h"
43 : : #include "gimplify.h"
44 : : #include "gimple-iterator.h"
45 : : #include "gimplify-me.h"
46 : : #include "tree-ssa-loop-ivopts.h"
47 : : #include "tree-ssa-loop-manip.h"
48 : : #include "tree-ssa-loop.h"
49 : : #include "cfgloop.h"
50 : : #include "tree-scalar-evolution.h"
51 : : #include "tree-vectorizer.h"
52 : : #include "expr.h"
53 : : #include "builtins.h"
54 : : #include "tree-cfg.h"
55 : : #include "tree-hash-traits.h"
56 : : #include "vec-perm-indices.h"
57 : : #include "internal-fn.h"
58 : : #include "gimple-fold.h"
59 : : #include "optabs-query.h"
60 : :
61 : : /* Return true if load- or store-lanes optab OPTAB is implemented for
62 : : COUNT vectors of type VECTYPE. NAME is the name of OPTAB.
63 : :
64 : : If it is implemented and ELSVALS is nonzero store the possible else
65 : : values in the vector it points to. */
66 : :
67 : : static bool
68 : 242094 : vect_lanes_optab_supported_p (const char *name, convert_optab optab,
69 : : tree vectype, unsigned HOST_WIDE_INT count,
70 : : vec<int> *elsvals = nullptr)
71 : : {
72 : 242094 : machine_mode mode, array_mode;
73 : 242094 : bool limit_p;
74 : :
75 : 242094 : mode = TYPE_MODE (vectype);
76 : 242094 : if (!targetm.array_mode (mode, count).exists (&array_mode))
77 : : {
78 : 484188 : poly_uint64 bits = count * GET_MODE_BITSIZE (mode);
79 : 242094 : limit_p = !targetm.array_mode_supported_p (mode, count);
80 : 242094 : if (!int_mode_for_size (bits, limit_p).exists (&array_mode))
81 : : {
82 : 221862 : if (dump_enabled_p ())
83 : 13710 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
84 : : "no array mode for %s[%wu]\n",
85 : 13710 : GET_MODE_NAME (mode), count);
86 : 221862 : return false;
87 : : }
88 : : }
89 : :
90 : 20232 : enum insn_code icode;
91 : 20232 : if ((icode = convert_optab_handler (optab, array_mode, mode))
92 : : == CODE_FOR_nothing)
93 : : {
94 : 20232 : if (dump_enabled_p ())
95 : 3838 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
96 : : "cannot use %s<%s><%s>\n", name,
97 : 3838 : GET_MODE_NAME (array_mode), GET_MODE_NAME (mode));
98 : 20232 : return false;
99 : : }
100 : :
101 : 0 : if (dump_enabled_p ())
102 : 0 : dump_printf_loc (MSG_NOTE, vect_location,
103 : 0 : "can use %s<%s><%s>\n", name, GET_MODE_NAME (array_mode),
104 : 0 : GET_MODE_NAME (mode));
105 : :
106 : 0 : if (elsvals)
107 : 0 : get_supported_else_vals (icode,
108 : 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD_LANES),
109 : : *elsvals);
110 : :
111 : : return true;
112 : : }
113 : :
114 : : /* Helper function to identify a simd clone call. If this is a call to a
115 : : function with simd clones then return the corresponding cgraph_node,
116 : : otherwise return NULL. */
117 : :
118 : : static cgraph_node*
119 : 811230 : simd_clone_call_p (gimple *stmt)
120 : : {
121 : 894043 : gcall *call = dyn_cast <gcall *> (stmt);
122 : 85126 : if (!call)
123 : : return NULL;
124 : :
125 : 85126 : tree fndecl = NULL_TREE;
126 : 85126 : if (gimple_call_internal_p (call, IFN_MASK_CALL))
127 : 319 : fndecl = TREE_OPERAND (gimple_call_arg (stmt, 0), 0);
128 : : else
129 : 84807 : fndecl = gimple_call_fndecl (stmt);
130 : :
131 : 85126 : if (fndecl == NULL_TREE)
132 : : return NULL;
133 : :
134 : 39569 : cgraph_node *node = cgraph_node::get (fndecl);
135 : 39569 : if (node && node->simd_clones != NULL)
136 : : return node;
137 : :
138 : : return NULL;
139 : : }
140 : :
141 : :
142 : :
143 : : /* Return the smallest scalar part of STMT_INFO.
144 : : This is used to determine the vectype of the stmt. We generally set the
145 : : vectype according to the type of the result (lhs). For stmts whose
146 : : result-type is different than the type of the arguments (e.g., demotion,
147 : : promotion), vectype will be reset appropriately (later). Note that we have
148 : : to visit the smallest datatype in this function, because that determines the
149 : : VF. If the smallest datatype in the loop is present only as the rhs of a
150 : : promotion operation - we'd miss it.
151 : : Such a case, where a variable of this datatype does not appear in the lhs
152 : : anywhere in the loop, can only occur if it's an invariant: e.g.:
153 : : 'int_x = (int) short_inv', which we'd expect to have been optimized away by
154 : : invariant motion. However, we cannot rely on invariant motion to always
155 : : take invariants out of the loop, and so in the case of promotion we also
156 : : have to check the rhs.
157 : : LHS_SIZE_UNIT and RHS_SIZE_UNIT contain the sizes of the corresponding
158 : : types. */
159 : :
160 : : tree
161 : 6351257 : vect_get_smallest_scalar_type (stmt_vec_info stmt_info, tree scalar_type)
162 : : {
163 : 6351257 : HOST_WIDE_INT lhs, rhs;
164 : :
165 : : /* During the analysis phase, this function is called on arbitrary
166 : : statements that might not have scalar results. */
167 : 6351257 : if (!tree_fits_uhwi_p (TYPE_SIZE_UNIT (scalar_type)))
168 : : return scalar_type;
169 : :
170 : 6351257 : lhs = rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
171 : :
172 : 6351257 : gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
173 : 6351257 : if (assign)
174 : : {
175 : 5540027 : scalar_type = TREE_TYPE (gimple_assign_lhs (assign));
176 : 5540027 : if (gimple_assign_cast_p (assign)
177 : 5049195 : || gimple_assign_rhs_code (assign) == DOT_PROD_EXPR
178 : 5048429 : || gimple_assign_rhs_code (assign) == WIDEN_SUM_EXPR
179 : 5048429 : || gimple_assign_rhs_code (assign) == SAD_EXPR
180 : 5048087 : || gimple_assign_rhs_code (assign) == WIDEN_MULT_EXPR
181 : 5042640 : || gimple_assign_rhs_code (assign) == WIDEN_MULT_PLUS_EXPR
182 : 5042640 : || gimple_assign_rhs_code (assign) == WIDEN_MULT_MINUS_EXPR
183 : 5042640 : || gimple_assign_rhs_code (assign) == WIDEN_LSHIFT_EXPR
184 : 10582667 : || gimple_assign_rhs_code (assign) == FLOAT_EXPR)
185 : : {
186 : 518464 : tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign));
187 : :
188 : 518464 : rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
189 : 518464 : if (rhs < lhs)
190 : 6351257 : scalar_type = rhs_type;
191 : : }
192 : : }
193 : 811230 : else if (cgraph_node *node = simd_clone_call_p (stmt_info->stmt))
194 : : {
195 : 2313 : auto clone = node->simd_clones->simdclone;
196 : 7394 : for (unsigned int i = 0; i < clone->nargs; ++i)
197 : : {
198 : 5081 : if (clone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR)
199 : : {
200 : 2995 : tree arg_scalar_type = TREE_TYPE (clone->args[i].vector_type);
201 : 2995 : rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (arg_scalar_type));
202 : 2995 : if (rhs < lhs)
203 : : {
204 : 5081 : scalar_type = arg_scalar_type;
205 : 5081 : lhs = rhs;
206 : : }
207 : : }
208 : : }
209 : : }
210 : 808917 : else if (gcall *call = dyn_cast <gcall *> (stmt_info->stmt))
211 : : {
212 : 82813 : unsigned int i = 0;
213 : 82813 : if (gimple_call_internal_p (call))
214 : : {
215 : 43166 : internal_fn ifn = gimple_call_internal_fn (call);
216 : 43166 : if (internal_load_fn_p (ifn))
217 : : /* For loads the LHS type does the trick. */
218 : : i = ~0U;
219 : 36861 : else if (internal_store_fn_p (ifn))
220 : : {
221 : : /* For stores use the tyep of the stored value. */
222 : 2949 : i = internal_fn_stored_value_index (ifn);
223 : 2949 : scalar_type = TREE_TYPE (gimple_call_arg (call, i));
224 : 2949 : i = ~0U;
225 : : }
226 : 33912 : else if (internal_fn_mask_index (ifn) == 0)
227 : 9123 : i = 1;
228 : : }
229 : 82813 : if (i < gimple_call_num_args (call))
230 : : {
231 : 68943 : tree rhs_type = TREE_TYPE (gimple_call_arg (call, i));
232 : 68943 : if (tree_fits_uhwi_p (TYPE_SIZE_UNIT (rhs_type)))
233 : : {
234 : 68943 : rhs = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (rhs_type));
235 : 68943 : if (rhs < lhs)
236 : 6351257 : scalar_type = rhs_type;
237 : : }
238 : : }
239 : : }
240 : :
241 : : return scalar_type;
242 : : }
243 : :
244 : :
245 : : /* Insert DDR into LOOP_VINFO list of ddrs that may alias and need to be
246 : : tested at run-time. Return TRUE if DDR was successfully inserted.
247 : : Return false if versioning is not supported. */
248 : :
249 : : static opt_result
250 : 89253 : vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
251 : : {
252 : 89253 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
253 : :
254 : 89253 : if ((unsigned) param_vect_max_version_for_alias_checks == 0)
255 : 54 : return opt_result::failure_at (vect_location,
256 : : "will not create alias checks, as"
257 : : " --param vect-max-version-for-alias-checks"
258 : : " == 0\n");
259 : :
260 : 89199 : opt_result res
261 : 89199 : = runtime_alias_check_p (ddr, loop,
262 : 89199 : optimize_loop_nest_for_speed_p (loop));
263 : 89199 : if (!res)
264 : 125 : return res;
265 : :
266 : 89074 : LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo).safe_push (ddr);
267 : 89074 : return opt_result::success ();
268 : : }
269 : :
270 : : /* Record that loop LOOP_VINFO needs to check that VALUE is nonzero. */
271 : :
272 : : static void
273 : 1214 : vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value)
274 : : {
275 : 1214 : const vec<tree> &checks = LOOP_VINFO_CHECK_NONZERO (loop_vinfo);
276 : 1865 : for (unsigned int i = 0; i < checks.length(); ++i)
277 : 657 : if (checks[i] == value)
278 : : return;
279 : :
280 : 1208 : if (dump_enabled_p ())
281 : 428 : dump_printf_loc (MSG_NOTE, vect_location,
282 : : "need run-time check that %T is nonzero\n",
283 : : value);
284 : 1208 : LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value);
285 : : }
286 : :
287 : : /* Return true if we know that the order of vectorized DR_INFO_A and
288 : : vectorized DR_INFO_B will be the same as the order of DR_INFO_A and
289 : : DR_INFO_B. At least one of the accesses is a write. */
290 : :
291 : : static bool
292 : 107800 : vect_preserves_scalar_order_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b)
293 : : {
294 : 107800 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
295 : 107800 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
296 : :
297 : : /* Single statements are always kept in their original order. */
298 : 107800 : if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
299 : 173370 : && !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
300 : : return true;
301 : :
302 : : /* If there is a loop invariant read involved we might vectorize it in
303 : : the prologue, breaking scalar oder with respect to the in-loop store. */
304 : 22239 : if ((DR_IS_READ (dr_info_a->dr) && integer_zerop (DR_STEP (dr_info_a->dr)))
305 : 66610 : || (DR_IS_READ (dr_info_b->dr) && integer_zerop (DR_STEP (dr_info_b->dr))))
306 : 1305 : return false;
307 : :
308 : : /* STMT_A and STMT_B belong to overlapping groups. All loads are
309 : : emitted at the position of the first scalar load.
310 : : Stores in a group are emitted at the position of the last scalar store.
311 : : Compute that position and check whether the resulting order matches
312 : : the current one. */
313 : 44059 : stmt_vec_info il_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
314 : 44059 : if (il_a)
315 : : {
316 : 41918 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_a)))
317 : 159800 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
318 : 138518 : s = DR_GROUP_NEXT_ELEMENT (s))
319 : 138518 : il_a = get_later_stmt (il_a, s);
320 : : else /* DR_IS_READ */
321 : 80377 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
322 : 59741 : s = DR_GROUP_NEXT_ELEMENT (s))
323 : 59741 : if (get_later_stmt (il_a, s) == il_a)
324 : 1854 : il_a = s;
325 : : }
326 : : else
327 : : il_a = stmtinfo_a;
328 : 44059 : stmt_vec_info il_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
329 : 44059 : if (il_b)
330 : : {
331 : 38539 : if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_b)))
332 : 204274 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
333 : 174174 : s = DR_GROUP_NEXT_ELEMENT (s))
334 : 174174 : il_b = get_later_stmt (il_b, s);
335 : : else /* DR_IS_READ */
336 : 39067 : for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
337 : 30628 : s = DR_GROUP_NEXT_ELEMENT (s))
338 : 30628 : if (get_later_stmt (il_b, s) == il_b)
339 : 207 : il_b = s;
340 : : }
341 : : else
342 : : il_b = stmtinfo_b;
343 : 44059 : bool a_after_b = (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a);
344 : 44059 : return (get_later_stmt (il_a, il_b) == il_a) == a_after_b;
345 : : }
346 : :
347 : : /* A subroutine of vect_analyze_data_ref_dependence. Handle
348 : : DDR_COULD_BE_INDEPENDENT_P ddr DDR that has a known set of dependence
349 : : distances. These distances are conservatively correct but they don't
350 : : reflect a guaranteed dependence.
351 : :
352 : : Return true if this function does all the work necessary to avoid
353 : : an alias or false if the caller should use the dependence distances
354 : : to limit the vectorization factor in the usual way. LOOP_DEPTH is
355 : : the depth of the loop described by LOOP_VINFO and the other arguments
356 : : are as for vect_analyze_data_ref_dependence. */
357 : :
358 : : static bool
359 : 7771 : vect_analyze_possibly_independent_ddr (data_dependence_relation *ddr,
360 : : loop_vec_info loop_vinfo,
361 : : int loop_depth, unsigned int *max_vf)
362 : : {
363 : 7771 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
364 : 31102 : for (lambda_vector &dist_v : DDR_DIST_VECTS (ddr))
365 : : {
366 : 15392 : int dist = dist_v[loop_depth];
367 : 15392 : if (dist != 0 && !(dist > 0 && DDR_REVERSED_P (ddr)))
368 : : {
369 : : /* If the user asserted safelen >= DIST consecutive iterations
370 : : can be executed concurrently, assume independence.
371 : :
372 : : ??? An alternative would be to add the alias check even
373 : : in this case, and vectorize the fallback loop with the
374 : : maximum VF set to safelen. However, if the user has
375 : : explicitly given a length, it's less likely that that
376 : : would be a win. */
377 : 7635 : if (loop->safelen >= 2 && abs_hwi (dist) <= loop->safelen)
378 : : {
379 : 32 : if ((unsigned int) loop->safelen < *max_vf)
380 : 2 : *max_vf = loop->safelen;
381 : 32 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
382 : 32 : continue;
383 : : }
384 : :
385 : : /* For dependence distances of 2 or more, we have the option
386 : : of limiting VF or checking for an alias at runtime.
387 : : Prefer to check at runtime if we can, to avoid limiting
388 : : the VF unnecessarily when the bases are in fact independent.
389 : :
390 : : Note that the alias checks will be removed if the VF ends up
391 : : being small enough. */
392 : 7603 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
393 : 7603 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
394 : 7603 : return (!STMT_VINFO_GATHER_SCATTER_P (dr_info_a->stmt)
395 : 7603 : && !STMT_VINFO_GATHER_SCATTER_P (dr_info_b->stmt)
396 : 15214 : && vect_mark_for_runtime_alias_test (ddr, loop_vinfo));
397 : : }
398 : : }
399 : : return true;
400 : : }
401 : :
402 : :
403 : : /* Function vect_analyze_data_ref_dependence.
404 : :
405 : : FIXME: I needed to change the sense of the returned flag.
406 : :
407 : : Return FALSE if there (might) exist a dependence between a memory-reference
408 : : DRA and a memory-reference DRB. When versioning for alias may check a
409 : : dependence at run-time, return TRUE. Adjust *MAX_VF according to
410 : : the data dependence. */
411 : :
412 : : static opt_result
413 : 922516 : vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
414 : : loop_vec_info loop_vinfo,
415 : : unsigned int *max_vf)
416 : : {
417 : 922516 : unsigned int i;
418 : 922516 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
419 : 922516 : struct data_reference *dra = DDR_A (ddr);
420 : 922516 : struct data_reference *drb = DDR_B (ddr);
421 : 922516 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (dra);
422 : 922516 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (drb);
423 : 922516 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
424 : 922516 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
425 : 922516 : lambda_vector dist_v;
426 : 922516 : unsigned int loop_depth;
427 : :
428 : : /* If user asserted safelen consecutive iterations can be
429 : : executed concurrently, assume independence. */
430 : 1019384 : auto apply_safelen = [&]()
431 : : {
432 : 96868 : if (loop->safelen >= 2)
433 : : {
434 : 7454 : if ((unsigned int) loop->safelen < *max_vf)
435 : 1893 : *max_vf = loop->safelen;
436 : 7454 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
437 : 7454 : return true;
438 : : }
439 : : return false;
440 : 922516 : };
441 : :
442 : : /* In loop analysis all data references should be vectorizable. */
443 : 922516 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
444 : 922516 : || !STMT_VINFO_VECTORIZABLE (stmtinfo_b))
445 : 0 : gcc_unreachable ();
446 : :
447 : : /* Independent data accesses. */
448 : 922516 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
449 : 763980 : return opt_result::success ();
450 : :
451 : 158536 : if (dra == drb
452 : 158536 : || (DR_IS_READ (dra) && DR_IS_READ (drb)))
453 : 0 : return opt_result::success ();
454 : :
455 : : /* We do not have to consider dependences between accesses that belong
456 : : to the same group, unless the stride could be smaller than the
457 : : group size. */
458 : 158536 : if (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
459 : 56046 : && (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
460 : 56046 : == DR_GROUP_FIRST_ELEMENT (stmtinfo_b))
461 : 163391 : && !STMT_VINFO_STRIDED_P (stmtinfo_a))
462 : 137 : return opt_result::success ();
463 : :
464 : : /* Even if we have an anti-dependence then, as the vectorized loop covers at
465 : : least two scalar iterations, there is always also a true dependence.
466 : : As the vectorizer does not re-order loads and stores we can ignore
467 : : the anti-dependence if TBAA can disambiguate both DRs similar to the
468 : : case with known negative distance anti-dependences (positive
469 : : distance anti-dependences would violate TBAA constraints). */
470 : 97899 : if (((DR_IS_READ (dra) && DR_IS_WRITE (drb))
471 : 60500 : || (DR_IS_WRITE (dra) && DR_IS_READ (drb)))
472 : 270569 : && !alias_sets_conflict_p (get_alias_set (DR_REF (dra)),
473 : : get_alias_set (DR_REF (drb))))
474 : 5357 : return opt_result::success ();
475 : :
476 : 153042 : if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
477 : 145731 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
478 : : {
479 : 9156 : if (apply_safelen ())
480 : 1392 : return opt_result::success ();
481 : :
482 : 7764 : return opt_result::failure_at
483 : 7764 : (stmtinfo_a->stmt,
484 : : "possible alias involving gather/scatter between %T and %T\n",
485 : : DR_REF (dra), DR_REF (drb));
486 : : }
487 : :
488 : : /* Unknown data dependence. */
489 : 143886 : if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
490 : : {
491 : 87213 : if (apply_safelen ())
492 : 6062 : return opt_result::success ();
493 : :
494 : 81151 : if (dump_enabled_p ())
495 : 7018 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
496 : : "versioning for alias required: "
497 : : "can't determine dependence between %T and %T\n",
498 : : DR_REF (dra), DR_REF (drb));
499 : :
500 : : /* Add to list of ddrs that need to be tested at run-time. */
501 : 81151 : return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
502 : : }
503 : :
504 : : /* Known data dependence. */
505 : 56673 : if (DDR_NUM_DIST_VECTS (ddr) == 0)
506 : : {
507 : 499 : if (apply_safelen ())
508 : 0 : return opt_result::success ();
509 : :
510 : 499 : if (dump_enabled_p ())
511 : 114 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
512 : : "versioning for alias required: "
513 : : "bad dist vector for %T and %T\n",
514 : : DR_REF (dra), DR_REF (drb));
515 : : /* Add to list of ddrs that need to be tested at run-time. */
516 : 499 : return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
517 : : }
518 : :
519 : 56174 : loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
520 : :
521 : 56174 : if (DDR_COULD_BE_INDEPENDENT_P (ddr)
522 : 56174 : && vect_analyze_possibly_independent_ddr (ddr, loop_vinfo,
523 : : loop_depth, max_vf))
524 : 7763 : return opt_result::success ();
525 : :
526 : 90907 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
527 : : {
528 : 48431 : int dist = dist_v[loop_depth];
529 : :
530 : 48431 : if (dump_enabled_p ())
531 : 3665 : dump_printf_loc (MSG_NOTE, vect_location,
532 : : "dependence distance = %d.\n", dist);
533 : :
534 : 48431 : if (dist == 0)
535 : : {
536 : 38892 : if (dump_enabled_p ())
537 : 3257 : dump_printf_loc (MSG_NOTE, vect_location,
538 : : "dependence distance == 0 between %T and %T\n",
539 : : DR_REF (dra), DR_REF (drb));
540 : :
541 : : /* When we perform grouped accesses and perform implicit CSE
542 : : by detecting equal accesses and doing disambiguation with
543 : : runtime alias tests like for
544 : : .. = a[i];
545 : : .. = a[i+1];
546 : : a[i] = ..;
547 : : a[i+1] = ..;
548 : : *p = ..;
549 : : .. = a[i];
550 : : .. = a[i+1];
551 : : where we will end up loading { a[i], a[i+1] } once, make
552 : : sure that inserting group loads before the first load and
553 : : stores after the last store will do the right thing.
554 : : Similar for groups like
555 : : a[i] = ...;
556 : : ... = a[i];
557 : : a[i+1] = ...;
558 : : where loads from the group interleave with the store. */
559 : 38892 : if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
560 : 0 : return opt_result::failure_at (stmtinfo_a->stmt,
561 : : "READ_WRITE dependence"
562 : : " in interleaving.\n");
563 : :
564 : 38892 : if (loop->safelen < 2)
565 : : {
566 : 35158 : tree indicator = dr_zero_step_indicator (dra);
567 : 35158 : if (!indicator || integer_zerop (indicator))
568 : 0 : return opt_result::failure_at (stmtinfo_a->stmt,
569 : : "access also has a zero step\n");
570 : 35158 : else if (TREE_CODE (indicator) != INTEGER_CST)
571 : 1214 : vect_check_nonzero_value (loop_vinfo, indicator);
572 : : }
573 : 38892 : continue;
574 : 38892 : }
575 : :
576 : 9539 : if (dist > 0 && DDR_REVERSED_P (ddr))
577 : : {
578 : : /* If DDR_REVERSED_P the order of the data-refs in DDR was
579 : : reversed (to make distance vector positive), and the actual
580 : : distance is negative. */
581 : 3191 : if (dump_enabled_p ())
582 : 105 : dump_printf_loc (MSG_NOTE, vect_location,
583 : : "dependence distance negative.\n");
584 : : /* When doing outer loop vectorization, we need to check if there is
585 : : a backward dependence at the inner loop level if the dependence
586 : : at the outer loop is reversed. See PR81740. */
587 : 3191 : if (nested_in_vect_loop_p (loop, stmtinfo_a)
588 : 3179 : || nested_in_vect_loop_p (loop, stmtinfo_b))
589 : : {
590 : 12 : unsigned inner_depth = index_in_loop_nest (loop->inner->num,
591 : 12 : DDR_LOOP_NEST (ddr));
592 : 12 : if (dist_v[inner_depth] < 0)
593 : 9 : return opt_result::failure_at (stmtinfo_a->stmt,
594 : : "not vectorized, dependence "
595 : : "between data-refs %T and %T\n",
596 : : DR_REF (dra), DR_REF (drb));
597 : : }
598 : : /* Record a negative dependence distance to later limit the
599 : : amount of stmt copying / unrolling we can perform.
600 : : Only need to handle read-after-write dependence. */
601 : 3182 : if (DR_IS_READ (drb)
602 : 78 : && (STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) == 0
603 : 12 : || STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) > (unsigned)dist))
604 : 78 : STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) = dist;
605 : 3182 : continue;
606 : 3182 : }
607 : :
608 : 6348 : unsigned int abs_dist = abs (dist);
609 : 6348 : if (abs_dist >= 2 && abs_dist < *max_vf)
610 : : {
611 : : /* The dependence distance requires reduction of the maximal
612 : : vectorization factor. */
613 : 313 : *max_vf = abs_dist;
614 : 313 : if (dump_enabled_p ())
615 : 30 : dump_printf_loc (MSG_NOTE, vect_location,
616 : : "adjusting maximal vectorization factor to %i\n",
617 : : *max_vf);
618 : : }
619 : :
620 : 6348 : if (abs_dist >= *max_vf)
621 : : {
622 : : /* Dependence distance does not create dependence, as far as
623 : : vectorization is concerned, in this case. */
624 : 422 : if (dump_enabled_p ())
625 : 57 : dump_printf_loc (MSG_NOTE, vect_location,
626 : : "dependence distance >= VF.\n");
627 : 422 : continue;
628 : : }
629 : :
630 : 5926 : return opt_result::failure_at (stmtinfo_a->stmt,
631 : : "not vectorized, possible dependence "
632 : : "between data-refs %T and %T\n",
633 : : DR_REF (dra), DR_REF (drb));
634 : : }
635 : :
636 : 42476 : return opt_result::success ();
637 : : }
638 : :
639 : : /* Function vect_analyze_early_break_dependences.
640 : :
641 : : Examine all the data references in the loop and make sure that if we have
642 : : multiple exits that we are able to safely move stores such that they become
643 : : safe for vectorization. The function also calculates the place where to move
644 : : the instructions to and computes what the new vUSE chain should be.
645 : :
646 : : This works in tandem with the CFG that will be produced by
647 : : slpeel_tree_duplicate_loop_to_edge_cfg later on.
648 : :
649 : : This function tries to validate whether an early break vectorization
650 : : is possible for the current instruction sequence. Returns True i
651 : : possible, otherwise False.
652 : :
653 : : Requirements:
654 : : - Any memory access must be to a fixed size buffer.
655 : : - There must not be any loads and stores to the same object.
656 : : - Multiple loads are allowed as long as they don't alias.
657 : :
658 : : NOTE:
659 : : This implementation is very conservative. Any overlapping loads/stores
660 : : that take place before the early break statement gets rejected aside from
661 : : WAR dependencies.
662 : :
663 : : i.e.:
664 : :
665 : : a[i] = 8
666 : : c = a[i]
667 : : if (b[i])
668 : : ...
669 : :
670 : : is not allowed, but
671 : :
672 : : c = a[i]
673 : : a[i] = 8
674 : : if (b[i])
675 : : ...
676 : :
677 : : is which is the common case. */
678 : :
679 : : static opt_result
680 : 113406 : vect_analyze_early_break_dependences (loop_vec_info loop_vinfo)
681 : : {
682 : 113406 : DUMP_VECT_SCOPE ("vect_analyze_early_break_dependences");
683 : :
684 : : /* List of all load data references found during traversal. */
685 : 113406 : auto_vec<data_reference *> bases;
686 : 113406 : basic_block dest_bb = NULL;
687 : :
688 : 113406 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
689 : 113406 : class loop *loop_nest = loop_outer (loop);
690 : :
691 : 113406 : if (dump_enabled_p ())
692 : 1336 : dump_printf_loc (MSG_NOTE, vect_location,
693 : : "loop contains multiple exits, analyzing"
694 : : " statement dependencies.\n");
695 : :
696 : 113406 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
697 : 5229 : if (dump_enabled_p ())
698 : 159 : dump_printf_loc (MSG_NOTE, vect_location,
699 : : "alternate exit has been chosen as main exit.\n");
700 : :
701 : : /* Since we don't support general control flow, the location we'll move the
702 : : side-effects to is always the latch connected exit. When we support
703 : : general control flow we can do better but for now this is fine. Move
704 : : side-effects to the in-loop destination of the last early exit. For the
705 : : PEELED case we move the side-effects to the latch block as this is
706 : : guaranteed to be the last block to be executed when a vector iteration
707 : : finished. */
708 : 113406 : if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
709 : 5229 : dest_bb = loop->latch;
710 : : else
711 : 108177 : dest_bb = single_pred (loop->latch);
712 : :
713 : : /* We start looking from dest_bb, for the non-PEELED case we don't want to
714 : : move any stores already present, but we do want to read and validate the
715 : : loads. */
716 : 113406 : basic_block bb = dest_bb;
717 : :
718 : : /* We move stores across all loads to the beginning of dest_bb, so
719 : : the first block processed below doesn't need dependence checking. */
720 : 113406 : bool check_deps = false;
721 : :
722 : 420858 : do
723 : : {
724 : 267132 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
725 : :
726 : : /* Now analyze all the remaining statements and try to determine which
727 : : instructions are allowed/needed to be moved. */
728 : 2038842 : while (!gsi_end_p (gsi))
729 : : {
730 : 1771857 : gimple *stmt = gsi_stmt (gsi);
731 : 1771857 : gsi_prev (&gsi);
732 : 1771857 : if (is_gimple_debug (stmt))
733 : 1568187 : continue;
734 : :
735 : 950184 : stmt_vec_info stmt_vinfo
736 : 950184 : = vect_stmt_to_vectorize (loop_vinfo->lookup_stmt (stmt));
737 : 950184 : auto dr_ref = STMT_VINFO_DATA_REF (stmt_vinfo);
738 : 950184 : if (!dr_ref)
739 : 735229 : continue;
740 : :
741 : : /* We know everything below dest_bb is safe since we know we
742 : : had a full vector iteration when reaching it. Either by
743 : : the loop entry / IV exit test being last or because this
744 : : is the loop latch itself. */
745 : 214955 : if (!check_deps)
746 : 11285 : continue;
747 : :
748 : : /* Check if vector accesses to the object will be within bounds.
749 : : must be a constant or assume loop will be versioned or niters
750 : : bounded by VF so accesses are within range. We only need to check
751 : : the reads since writes are moved to a safe place where if we get
752 : : there we know they are safe to perform. */
753 : 203670 : if (DR_IS_READ (dr_ref))
754 : : {
755 : 190249 : dr_set_safe_speculative_read_required (stmt_vinfo, true);
756 : 190249 : bool inbounds = ref_within_array_bound (stmt, DR_REF (dr_ref));
757 : 190249 : DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_vinfo)) = inbounds;
758 : :
759 : 190249 : if (dump_enabled_p ())
760 : 2196 : dump_printf_loc (MSG_NOTE, vect_location,
761 : : "marking DR (read) as possibly needing peeling "
762 : : "for alignment at %G", stmt);
763 : : }
764 : :
765 : 203670 : if (DR_IS_READ (dr_ref))
766 : 190249 : bases.safe_push (dr_ref);
767 : 13421 : else if (DR_IS_WRITE (dr_ref))
768 : : {
769 : : /* We are moving writes down in the CFG. To be sure that this
770 : : is valid after vectorization we have to check all the loads
771 : : we are sinking the stores past to see if any of them may
772 : : alias or are the same object.
773 : :
774 : : Same objects will not be an issue because unless the store
775 : : is marked volatile the value can be forwarded. If the
776 : : store is marked volatile we don't vectorize the loop
777 : : anyway.
778 : :
779 : : That leaves the check for aliasing. We don't really need
780 : : to care about the stores aliasing with each other since the
781 : : stores are moved in order so the effects are still observed
782 : : correctly. This leaves the check for WAR dependencies
783 : : which we would be introducing here if the DR can alias.
784 : : The check is quadratic in loads/stores but I have not found
785 : : a better API to do this. I believe all loads and stores
786 : : must be checked. We also must check them when we
787 : : encountered the store, since we don't care about loads past
788 : : the store. */
789 : :
790 : 45083 : for (auto dr_read : bases)
791 : 14775 : if (dr_may_alias_p (dr_ref, dr_read, loop_nest))
792 : : {
793 : 147 : if (dump_enabled_p ())
794 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION,
795 : : vect_location,
796 : : "early breaks not supported: "
797 : : "overlapping loads and stores "
798 : : "found before the break "
799 : : "statement.\n");
800 : :
801 : 147 : return opt_result::failure_at (stmt,
802 : : "can't safely apply code motion to dependencies"
803 : : " to vectorize the early exit. %G may alias with"
804 : : " %G\n", stmt, dr_read->stmt);
805 : : }
806 : : }
807 : :
808 : 407046 : if (gimple_vdef (stmt))
809 : : {
810 : 13274 : if (dump_enabled_p ())
811 : 251 : dump_printf_loc (MSG_NOTE, vect_location,
812 : : "==> recording stmt %G", stmt);
813 : :
814 : 13274 : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (stmt);
815 : : }
816 : 584021 : else if (gimple_vuse (stmt))
817 : : {
818 : 190249 : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).safe_insert (0, stmt);
819 : 190249 : if (dump_enabled_p ())
820 : 2196 : dump_printf_loc (MSG_NOTE, vect_location,
821 : : "marked statement for vUSE update: %G", stmt);
822 : : }
823 : : }
824 : :
825 : 266985 : if (!single_pred_p (bb))
826 : : {
827 : 113259 : gcc_assert (bb == loop->header);
828 : 113259 : break;
829 : : }
830 : :
831 : : /* If we possibly sink through a virtual PHI make sure to elide that. */
832 : 153726 : if (gphi *vphi = get_virtual_phi (bb))
833 : 41 : LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (vphi);
834 : :
835 : : /* All earlier blocks need dependence checking. */
836 : 153726 : check_deps = true;
837 : 153726 : bb = single_pred (bb);
838 : 153726 : }
839 : : while (1);
840 : :
841 : : /* We don't allow outer -> inner loop transitions which should have been
842 : : trapped already during loop form analysis. */
843 : 113259 : gcc_assert (dest_bb->loop_father == loop);
844 : :
845 : : /* Check that the destination block we picked has only one pred. To relax this we
846 : : have to take special care when moving the statements. We don't currently support
847 : : such control flow however this check is there to simplify how we handle
848 : : labels that may be present anywhere in the IL. This check is to ensure that the
849 : : labels aren't significant for the CFG. */
850 : 113259 : if (!single_pred (dest_bb))
851 : 0 : return opt_result::failure_at (vect_location,
852 : : "chosen loop exit block (BB %d) does not have a "
853 : : "single predecessor which is currently not "
854 : : "supported for early break vectorization.\n",
855 : : dest_bb->index);
856 : :
857 : 113259 : LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo) = dest_bb;
858 : :
859 : 113259 : if (!LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).is_empty ())
860 : : {
861 : : /* All uses shall be updated to that of the first load. Entries are
862 : : stored in reverse order. */
863 : 104669 : tree vuse = gimple_vuse (LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).last ());
864 : 294769 : for (auto g : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
865 : : {
866 : 190100 : if (dump_enabled_p ())
867 : 2196 : dump_printf_loc (MSG_NOTE, vect_location,
868 : : "will update use: %T, mem_ref: %G", vuse, g);
869 : : }
870 : : }
871 : :
872 : 113259 : if (dump_enabled_p ())
873 : 1336 : dump_printf_loc (MSG_NOTE, vect_location,
874 : : "recorded statements to be moved to BB %d\n",
875 : 1336 : LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo)->index);
876 : :
877 : 113259 : return opt_result::success ();
878 : 113406 : }
879 : :
880 : : /* Function vect_analyze_data_ref_dependences.
881 : :
882 : : Examine all the data references in the loop, and make sure there do not
883 : : exist any data dependences between them. Set *MAX_VF according to
884 : : the maximum vectorization factor the data dependences allow. */
885 : :
886 : : opt_result
887 : 300887 : vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
888 : : unsigned int *max_vf)
889 : : {
890 : 300887 : unsigned int i;
891 : 300887 : struct data_dependence_relation *ddr;
892 : :
893 : 300887 : DUMP_VECT_SCOPE ("vect_analyze_data_ref_dependences");
894 : :
895 : 300887 : if (!LOOP_VINFO_DDRS (loop_vinfo).exists ())
896 : : {
897 : 145133 : LOOP_VINFO_DDRS (loop_vinfo)
898 : 145133 : .create (LOOP_VINFO_DATAREFS (loop_vinfo).length ()
899 : 145133 : * LOOP_VINFO_DATAREFS (loop_vinfo).length ());
900 : : /* We do not need read-read dependences. */
901 : 290266 : bool res = compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo),
902 : : &LOOP_VINFO_DDRS (loop_vinfo),
903 : 145133 : LOOP_VINFO_LOOP_NEST (loop_vinfo),
904 : : false);
905 : 145133 : gcc_assert (res);
906 : : }
907 : :
908 : 300887 : LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = true;
909 : :
910 : : /* For epilogues we either have no aliases or alias versioning
911 : : was applied to original loop. Therefore we may just get max_vf
912 : : using VF of original loop. */
913 : 300887 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
914 : 25840 : *max_vf = LOOP_VINFO_ORIG_MAX_VECT_FACTOR (loop_vinfo);
915 : : else
916 : 1183693 : FOR_EACH_VEC_ELT (LOOP_VINFO_DDRS (loop_vinfo), i, ddr)
917 : : {
918 : 922516 : opt_result res
919 : 922516 : = vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf);
920 : 922516 : if (!res)
921 : 13870 : return res;
922 : : }
923 : :
924 : : /* If we have early break statements in the loop, check to see if they
925 : : are of a form we can vectorizer. */
926 : 287017 : if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
927 : 113406 : return vect_analyze_early_break_dependences (loop_vinfo);
928 : :
929 : 173611 : return opt_result::success ();
930 : : }
931 : :
932 : :
933 : : /* Function vect_slp_analyze_data_ref_dependence.
934 : :
935 : : Return TRUE if there (might) exist a dependence between a memory-reference
936 : : DRA and a memory-reference DRB for VINFO. When versioning for alias
937 : : may check a dependence at run-time, return FALSE. Adjust *MAX_VF
938 : : according to the data dependence. */
939 : :
940 : : static bool
941 : 6570272 : vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
942 : : struct data_dependence_relation *ddr)
943 : : {
944 : 6570272 : struct data_reference *dra = DDR_A (ddr);
945 : 6570272 : struct data_reference *drb = DDR_B (ddr);
946 : 6570272 : dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
947 : 6570272 : dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
948 : :
949 : : /* We need to check dependences of statements marked as unvectorizable
950 : : as well, they still can prohibit vectorization. */
951 : :
952 : : /* Independent data accesses. */
953 : 6570272 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
954 : : return false;
955 : :
956 : 1188844 : if (dra == drb)
957 : : return false;
958 : :
959 : : /* Read-read is OK. */
960 : 9414 : if (DR_IS_READ (dra) && DR_IS_READ (drb))
961 : : return false;
962 : :
963 : : /* If dra and drb are part of the same interleaving chain consider
964 : : them independent. */
965 : 9414 : if (STMT_VINFO_GROUPED_ACCESS (dr_info_a->stmt)
966 : 9414 : && (DR_GROUP_FIRST_ELEMENT (dr_info_a->stmt)
967 : 9414 : == DR_GROUP_FIRST_ELEMENT (dr_info_b->stmt)))
968 : : return false;
969 : :
970 : : /* Unknown data dependence. */
971 : 9414 : if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
972 : : {
973 : 9414 : if (dump_enabled_p ())
974 : 6 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
975 : : "can't determine dependence between %T and %T\n",
976 : : DR_REF (dra), DR_REF (drb));
977 : : }
978 : 0 : else if (dump_enabled_p ())
979 : 0 : dump_printf_loc (MSG_NOTE, vect_location,
980 : : "determined dependence between %T and %T\n",
981 : : DR_REF (dra), DR_REF (drb));
982 : :
983 : : return true;
984 : : }
985 : :
986 : :
987 : : /* Analyze dependences involved in the transform of a store SLP NODE. */
988 : :
989 : : static bool
990 : 735025 : vect_slp_analyze_store_dependences (vec_info *vinfo, slp_tree node)
991 : : {
992 : : /* This walks over all stmts involved in the SLP store done
993 : : in NODE verifying we can sink them up to the last stmt in the
994 : : group. */
995 : 735025 : stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
996 : 735025 : gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info)));
997 : :
998 : 2631410 : for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
999 : : {
1000 : 1905809 : stmt_vec_info access_info
1001 : 1905809 : = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
1002 : 1905809 : if (access_info == last_access_info)
1003 : 726379 : continue;
1004 : 1179430 : data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
1005 : 1179430 : ao_ref ref;
1006 : 1179430 : bool ref_initialized_p = false;
1007 : 1179430 : for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
1008 : 10794091 : gsi_stmt (gsi) != last_access_info->stmt; gsi_next (&gsi))
1009 : : {
1010 : 9624085 : gimple *stmt = gsi_stmt (gsi);
1011 : 16812288 : if (! gimple_vuse (stmt))
1012 : 2968952 : continue;
1013 : :
1014 : : /* If we couldn't record a (single) data reference for this
1015 : : stmt we have to resort to the alias oracle. */
1016 : 6655133 : stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
1017 : 6655133 : data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
1018 : 6655133 : if (!dr_b)
1019 : : {
1020 : : /* We are moving a store - this means
1021 : : we cannot use TBAA for disambiguation. */
1022 : 85139 : if (!ref_initialized_p)
1023 : 85139 : ao_ref_init (&ref, DR_REF (dr_a));
1024 : 85139 : if (stmt_may_clobber_ref_p_1 (stmt, &ref, false)
1025 : 85139 : || ref_maybe_used_by_stmt_p (stmt, &ref, false))
1026 : 9424 : return false;
1027 : 85090 : continue;
1028 : : }
1029 : :
1030 : 6569994 : gcc_assert (!gimple_visited_p (stmt));
1031 : :
1032 : 6569994 : ddr_p ddr = initialize_data_dependence_relation (dr_a,
1033 : : dr_b, vNULL);
1034 : 6569994 : bool dependent = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
1035 : 6569994 : free_dependence_relation (ddr);
1036 : 6569994 : if (dependent)
1037 : : return false;
1038 : : }
1039 : : }
1040 : : return true;
1041 : : }
1042 : :
1043 : : /* Analyze dependences involved in the transform of a load SLP NODE. STORES
1044 : : contain the vector of scalar stores of this instance if we are
1045 : : disambiguating the loads. */
1046 : :
1047 : : static bool
1048 : 164939 : vect_slp_analyze_load_dependences (vec_info *vinfo, slp_tree node,
1049 : : vec<stmt_vec_info> stores,
1050 : : stmt_vec_info last_store_info)
1051 : : {
1052 : : /* This walks over all stmts involved in the SLP load done
1053 : : in NODE verifying we can hoist them up to the first stmt in the
1054 : : group. */
1055 : 164939 : stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node);
1056 : 164939 : gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info)));
1057 : :
1058 : 577695 : for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (node).length (); ++k)
1059 : : {
1060 : 412823 : if (! SLP_TREE_SCALAR_STMTS (node)[k])
1061 : 170917 : continue;
1062 : 412823 : stmt_vec_info access_info
1063 : 412823 : = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (node)[k]);
1064 : 412823 : if (access_info == first_access_info)
1065 : 170917 : continue;
1066 : 241906 : data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
1067 : 241906 : ao_ref ref;
1068 : 241906 : bool ref_initialized_p = false;
1069 : 241906 : hash_set<stmt_vec_info> grp_visited;
1070 : 241906 : for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
1071 : 5029500 : gsi_stmt (gsi) != first_access_info->stmt; gsi_prev (&gsi))
1072 : : {
1073 : 2393864 : gimple *stmt = gsi_stmt (gsi);
1074 : 3865407 : if (! gimple_vdef (stmt))
1075 : 2306587 : continue;
1076 : :
1077 : 370063 : stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
1078 : :
1079 : : /* If we run into a store of this same instance (we've just
1080 : : marked those) then delay dependence checking until we run
1081 : : into the last store because this is where it will have
1082 : : been sunk to (and we verified that we can do that already). */
1083 : 370063 : if (gimple_visited_p (stmt))
1084 : : {
1085 : 282786 : if (stmt_info != last_store_info)
1086 : 282784 : continue;
1087 : :
1088 : 10 : for (stmt_vec_info &store_info : stores)
1089 : : {
1090 : 4 : data_reference *store_dr = STMT_VINFO_DATA_REF (store_info);
1091 : 4 : ddr_p ddr = initialize_data_dependence_relation
1092 : 4 : (dr_a, store_dr, vNULL);
1093 : 4 : bool dependent
1094 : 4 : = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
1095 : 4 : free_dependence_relation (ddr);
1096 : 4 : if (dependent)
1097 : 67 : return false;
1098 : : }
1099 : 2 : continue;
1100 : 2 : }
1101 : :
1102 : 178270 : auto check_hoist = [&] (stmt_vec_info stmt_info) -> bool
1103 : : {
1104 : : /* We are hoisting a load - this means we can use TBAA for
1105 : : disambiguation. */
1106 : 90993 : if (!ref_initialized_p)
1107 : 90993 : ao_ref_init (&ref, DR_REF (dr_a));
1108 : 90993 : if (stmt_may_clobber_ref_p_1 (stmt_info->stmt, &ref, true))
1109 : : {
1110 : : /* If we couldn't record a (single) data reference for this
1111 : : stmt we have to give up now. */
1112 : 302 : data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
1113 : 302 : if (!dr_b)
1114 : : return false;
1115 : 274 : ddr_p ddr = initialize_data_dependence_relation (dr_a,
1116 : : dr_b, vNULL);
1117 : 274 : bool dependent
1118 : 274 : = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
1119 : 274 : free_dependence_relation (ddr);
1120 : 274 : if (dependent)
1121 : : return false;
1122 : : }
1123 : : /* No dependence. */
1124 : : return true;
1125 : 87277 : };
1126 : 87277 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1127 : : {
1128 : : /* When we run into a store group we have to honor
1129 : : that earlier stores might be moved here. We don't
1130 : : know exactly which and where to since we lack a
1131 : : back-mapping from DR to SLP node, so assume all
1132 : : earlier stores are sunk here. It's enough to
1133 : : consider the last stmt of a group for this.
1134 : : ??? Both this and the fact that we disregard that
1135 : : the conflicting instance might be removed later
1136 : : is overly conservative. */
1137 : 67691 : if (!grp_visited.add (DR_GROUP_FIRST_ELEMENT (stmt_info)))
1138 : 12918 : for (auto store_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
1139 : 158709 : store_info != NULL;
1140 : 145791 : store_info = DR_GROUP_NEXT_ELEMENT (store_info))
1141 : 145830 : if ((store_info == stmt_info
1142 : 132921 : || get_later_stmt (store_info, stmt_info) == stmt_info)
1143 : 204328 : && !check_hoist (store_info))
1144 : : return false;
1145 : : }
1146 : : else
1147 : : {
1148 : 19586 : if (!check_hoist (stmt_info))
1149 : : return false;
1150 : : }
1151 : : }
1152 : 241906 : }
1153 : : return true;
1154 : : }
1155 : :
1156 : :
1157 : : /* Function vect_analyze_data_ref_dependences.
1158 : :
1159 : : Examine all the data references in the basic-block, and make sure there
1160 : : do not exist any data dependences between them. Set *MAX_VF according to
1161 : : the maximum vectorization factor the data dependences allow. */
1162 : :
1163 : : bool
1164 : 875506 : vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
1165 : : {
1166 : 875506 : DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
1167 : :
1168 : : /* The stores of this instance are at the root of the SLP tree. */
1169 : 875506 : slp_tree store = NULL;
1170 : 875506 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store)
1171 : 735025 : store = SLP_INSTANCE_TREE (instance);
1172 : :
1173 : : /* Verify we can sink stores to the vectorized stmt insert location. */
1174 : 735025 : stmt_vec_info last_store_info = NULL;
1175 : 735025 : if (store)
1176 : : {
1177 : 735025 : if (! vect_slp_analyze_store_dependences (vinfo, store))
1178 : : return false;
1179 : :
1180 : : /* Mark stores in this instance and remember the last one. */
1181 : 725601 : last_store_info = vect_find_last_scalar_stmt_in_slp (store);
1182 : 2621177 : for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
1183 : 1895576 : gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, true);
1184 : : }
1185 : :
1186 : 866082 : bool res = true;
1187 : :
1188 : : /* Verify we can sink loads to the vectorized stmt insert location,
1189 : : special-casing stores of this instance. */
1190 : 1290852 : for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
1191 : 164939 : if (! vect_slp_analyze_load_dependences (vinfo, load,
1192 : : store
1193 : : ? SLP_TREE_SCALAR_STMTS (store)
1194 : : : vNULL, last_store_info))
1195 : : {
1196 : : res = false;
1197 : : break;
1198 : : }
1199 : :
1200 : : /* Unset the visited flag. */
1201 : 866082 : if (store)
1202 : 2621177 : for (unsigned k = 0; k < SLP_TREE_SCALAR_STMTS (store).length (); ++k)
1203 : 1895576 : gimple_set_visited (SLP_TREE_SCALAR_STMTS (store)[k]->stmt, false);
1204 : :
1205 : : /* If this is a SLP instance with a store check if there's a dependent
1206 : : load that cannot be forwarded from a previous iteration of a loop
1207 : : both are in. This is to avoid situations like that in PR115777. */
1208 : 866082 : if (res && store)
1209 : : {
1210 : 725550 : stmt_vec_info store_info
1211 : 725550 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (store)[0]);
1212 : 725550 : class loop *store_loop = gimple_bb (store_info->stmt)->loop_father;
1213 : 725550 : if (! loop_outer (store_loop))
1214 : 613648 : return res;
1215 : 111902 : vec<loop_p> loop_nest;
1216 : 111902 : loop_nest.create (1);
1217 : 111902 : loop_nest.quick_push (store_loop);
1218 : 111902 : data_reference *drs = nullptr;
1219 : 196679 : for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
1220 : : {
1221 : 36820 : if (! STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (load)[0]))
1222 : 0 : continue;
1223 : 36820 : stmt_vec_info load_info
1224 : 36820 : = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (load)[0]);
1225 : 36820 : if (gimple_bb (load_info->stmt)->loop_father != store_loop)
1226 : 6482 : continue;
1227 : :
1228 : : /* For now concern ourselves with write-after-read as we also
1229 : : only look for re-use of the store within the same SLP instance.
1230 : : We can still get a RAW here when the instance contais a PHI
1231 : : with a backedge though, thus this test. */
1232 : 30338 : if (! vect_stmt_dominates_stmt_p (STMT_VINFO_STMT (load_info),
1233 : : STMT_VINFO_STMT (store_info)))
1234 : 10353 : continue;
1235 : :
1236 : 19985 : if (! drs)
1237 : : {
1238 : 18827 : drs = create_data_ref (loop_preheader_edge (store_loop),
1239 : : store_loop,
1240 : 18827 : DR_REF (STMT_VINFO_DATA_REF (store_info)),
1241 : : store_info->stmt, false, false);
1242 : 18827 : if (! DR_BASE_ADDRESS (drs)
1243 : 15651 : || TREE_CODE (DR_STEP (drs)) != INTEGER_CST)
1244 : : break;
1245 : : }
1246 : 16508 : data_reference *drl
1247 : 16508 : = create_data_ref (loop_preheader_edge (store_loop),
1248 : : store_loop,
1249 : 16508 : DR_REF (STMT_VINFO_DATA_REF (load_info)),
1250 : : load_info->stmt, true, false);
1251 : :
1252 : : /* See whether the DRs have a known constant distance throughout
1253 : : the containing loop iteration. */
1254 : 31459 : if (! DR_BASE_ADDRESS (drl)
1255 : 11955 : || ! operand_equal_p (DR_STEP (drs), DR_STEP (drl))
1256 : 8886 : || ! operand_equal_p (DR_BASE_ADDRESS (drs),
1257 : 8886 : DR_BASE_ADDRESS (drl))
1258 : 18069 : || ! operand_equal_p (DR_OFFSET (drs), DR_OFFSET (drl)))
1259 : : {
1260 : 14951 : free_data_ref (drl);
1261 : 14951 : continue;
1262 : : }
1263 : :
1264 : : /* If the next iteration load overlaps with a non-power-of-two offset
1265 : : we are surely failing any STLF attempt. */
1266 : 1557 : HOST_WIDE_INT step = TREE_INT_CST_LOW (DR_STEP (drl));
1267 : 1557 : unsigned HOST_WIDE_INT sizes
1268 : 1557 : = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drs))))
1269 : 1557 : * DR_GROUP_SIZE (store_info));
1270 : 1557 : unsigned HOST_WIDE_INT sizel
1271 : 1557 : = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drl))))
1272 : 1557 : * DR_GROUP_SIZE (load_info));
1273 : 1557 : if (ranges_overlap_p (TREE_INT_CST_LOW (DR_INIT (drl)) + step, sizel,
1274 : 1557 : TREE_INT_CST_LOW (DR_INIT (drs)), sizes))
1275 : : {
1276 : 869 : unsigned HOST_WIDE_INT dist
1277 : 869 : = absu_hwi (TREE_INT_CST_LOW (DR_INIT (drl)) + step
1278 : 869 : - TREE_INT_CST_LOW (DR_INIT (drs)));
1279 : 869 : poly_uint64 loadsz = tree_to_poly_uint64
1280 : 869 : (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (load)));
1281 : 869 : poly_uint64 storesz = tree_to_poly_uint64
1282 : 869 : (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (store)));
1283 : : /* When the overlap aligns with vector sizes used for the loads
1284 : : and the vector stores are larger or equal to the loads
1285 : : forwarding should work. */
1286 : 1738 : if (maybe_gt (loadsz, storesz) || ! multiple_p (dist, loadsz))
1287 : 74 : load->avoid_stlf_fail = true;
1288 : : }
1289 : 1557 : free_data_ref (drl);
1290 : : }
1291 : 111902 : if (drs)
1292 : 18827 : free_data_ref (drs);
1293 : 111902 : loop_nest.release ();
1294 : : }
1295 : :
1296 : : return res;
1297 : : }
1298 : :
1299 : : /* Return the misalignment of DR_INFO accessed in VECTYPE with OFFSET
1300 : : applied. */
1301 : :
1302 : : int
1303 : 7077219 : dr_misalignment (dr_vec_info *dr_info, tree vectype, poly_int64 offset)
1304 : : {
1305 : 7077219 : HOST_WIDE_INT diff = 0;
1306 : : /* Alignment is only analyzed for the first element of a DR group,
1307 : : use that but adjust misalignment by the offset of the access. */
1308 : 7077219 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
1309 : : {
1310 : 3461812 : dr_vec_info *first_dr
1311 : 3461812 : = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
1312 : : /* vect_analyze_data_ref_accesses guarantees that DR_INIT are
1313 : : INTEGER_CSTs and the first element in the group has the lowest
1314 : : address. */
1315 : 3461812 : diff = (TREE_INT_CST_LOW (DR_INIT (dr_info->dr))
1316 : 3461812 : - TREE_INT_CST_LOW (DR_INIT (first_dr->dr)));
1317 : 3461812 : gcc_assert (diff >= 0);
1318 : : dr_info = first_dr;
1319 : : }
1320 : :
1321 : 7077219 : int misalign = dr_info->misalignment;
1322 : 7077219 : gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED);
1323 : 7077219 : if (misalign == DR_MISALIGNMENT_UNKNOWN)
1324 : : return misalign;
1325 : :
1326 : : /* If the access is only aligned for a vector type with smaller alignment
1327 : : requirement the access has unknown misalignment. */
1328 : 4724871 : if (maybe_lt (dr_info->target_alignment * BITS_PER_UNIT,
1329 : 4724871 : targetm.vectorize.preferred_vector_alignment (vectype)))
1330 : : return DR_MISALIGNMENT_UNKNOWN;
1331 : :
1332 : : /* Apply the offset from the DR group start and the externally supplied
1333 : : offset which can for example result from a negative stride access. */
1334 : 4724866 : poly_int64 misalignment = misalign + diff + offset;
1335 : :
1336 : : /* Below we reject compile-time non-constant target alignments, but if
1337 : : our misalignment is zero, then we are known to already be aligned
1338 : : w.r.t. any such possible target alignment. */
1339 : 4724866 : if (known_eq (misalignment, 0))
1340 : : return 0;
1341 : :
1342 : 956772 : unsigned HOST_WIDE_INT target_alignment_c;
1343 : 956772 : if (!dr_info->target_alignment.is_constant (&target_alignment_c)
1344 : 956772 : || !known_misalignment (misalignment, target_alignment_c, &misalign))
1345 : : return DR_MISALIGNMENT_UNKNOWN;
1346 : 956772 : return misalign;
1347 : : }
1348 : :
1349 : : /* Record the base alignment guarantee given by DRB, which occurs
1350 : : in STMT_INFO. */
1351 : :
1352 : : static void
1353 : 4485690 : vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
1354 : : innermost_loop_behavior *drb)
1355 : : {
1356 : 4485690 : bool existed;
1357 : 4485690 : std::pair<stmt_vec_info, innermost_loop_behavior *> &entry
1358 : 4485690 : = vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
1359 : 4485690 : if (!existed || entry.second->base_alignment < drb->base_alignment)
1360 : : {
1361 : 1366812 : entry = std::make_pair (stmt_info, drb);
1362 : 1366812 : if (dump_enabled_p ())
1363 : 35252 : dump_printf_loc (MSG_NOTE, vect_location,
1364 : : "recording new base alignment for %T\n"
1365 : : " alignment: %d\n"
1366 : : " misalignment: %d\n"
1367 : : " based on: %G",
1368 : : drb->base_address,
1369 : : drb->base_alignment,
1370 : : drb->base_misalignment,
1371 : : stmt_info->stmt);
1372 : : }
1373 : 4485690 : }
1374 : :
1375 : : /* If the region we're going to vectorize is reached, all unconditional
1376 : : data references occur at least once. We can therefore pool the base
1377 : : alignment guarantees from each unconditional reference. Do this by
1378 : : going through all the data references in VINFO and checking whether
1379 : : the containing statement makes the reference unconditionally. If so,
1380 : : record the alignment of the base address in VINFO so that it can be
1381 : : used for all other references with the same base. */
1382 : :
1383 : : void
1384 : 994525 : vect_record_base_alignments (vec_info *vinfo)
1385 : : {
1386 : 994525 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
1387 : 360053 : class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
1388 : 14703323 : for (data_reference *dr : vinfo->shared->datarefs)
1389 : : {
1390 : 11817370 : dr_vec_info *dr_info = vinfo->lookup_dr (dr);
1391 : 11817370 : stmt_vec_info stmt_info = dr_info->stmt;
1392 : 11817370 : if (!DR_IS_CONDITIONAL_IN_STMT (dr)
1393 : 11810520 : && STMT_VINFO_VECTORIZABLE (stmt_info)
1394 : 4499505 : && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1395 : : {
1396 : 4484197 : vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
1397 : :
1398 : : /* If DR is nested in the loop that is being vectorized, we can also
1399 : : record the alignment of the base wrt the outer loop. */
1400 : 12615788 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
1401 : 1493 : vect_record_base_alignment
1402 : 1493 : (vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
1403 : : }
1404 : : }
1405 : 994525 : }
1406 : :
1407 : : /* Function vect_compute_data_ref_alignment
1408 : :
1409 : : Compute the misalignment of the data reference DR_INFO when vectorizing
1410 : : with VECTYPE.
1411 : :
1412 : : Output:
1413 : : 1. initialized misalignment info for DR_INFO
1414 : :
1415 : : FOR NOW: No analysis is actually performed. Misalignment is calculated
1416 : : only for trivial cases. TODO. */
1417 : :
1418 : : static void
1419 : 1574736 : vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
1420 : : tree vectype)
1421 : : {
1422 : 1574736 : stmt_vec_info stmt_info = dr_info->stmt;
1423 : 1574736 : vec_base_alignments *base_alignments = &vinfo->base_alignments;
1424 : 1574736 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
1425 : 1574736 : class loop *loop = NULL;
1426 : 1574736 : tree ref = DR_REF (dr_info->dr);
1427 : :
1428 : 1574736 : if (dump_enabled_p ())
1429 : 55722 : dump_printf_loc (MSG_NOTE, vect_location,
1430 : : "vect_compute_data_ref_alignment:\n");
1431 : :
1432 : 1574736 : if (loop_vinfo)
1433 : 709401 : loop = LOOP_VINFO_LOOP (loop_vinfo);
1434 : :
1435 : : /* Initialize misalignment to unknown. */
1436 : 1574736 : SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
1437 : :
1438 : 1574736 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
1439 : : return;
1440 : :
1441 : 1557625 : innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
1442 : 1557625 : bool step_preserves_misalignment_p;
1443 : :
1444 : 1557625 : poly_uint64 vector_alignment
1445 : 1557625 : = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
1446 : : BITS_PER_UNIT);
1447 : :
1448 : 1557625 : if (loop_vinfo
1449 : 1557625 : && dr_safe_speculative_read_required (stmt_info))
1450 : : {
1451 : 260489 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1452 : 260489 : auto vectype_size
1453 : 260489 : = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
1454 : 260489 : poly_uint64 new_alignment = vf * vectype_size;
1455 : : /* If we have a grouped access we require that the alignment be N * elem. */
1456 : 260489 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1457 : 61810 : new_alignment *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
1458 : :
1459 : 260489 : unsigned HOST_WIDE_INT target_alignment;
1460 : 260489 : if (new_alignment.is_constant (&target_alignment)
1461 : 520978 : && pow2p_hwi (target_alignment))
1462 : : {
1463 : 226085 : if (dump_enabled_p ())
1464 : : {
1465 : 3178 : dump_printf_loc (MSG_NOTE, vect_location,
1466 : : "alignment increased due to early break to ");
1467 : 3178 : dump_dec (MSG_NOTE, new_alignment);
1468 : 3178 : dump_printf (MSG_NOTE, " bytes.\n");
1469 : : }
1470 : : vector_alignment = target_alignment;
1471 : : }
1472 : : }
1473 : :
1474 : 1557625 : SET_DR_TARGET_ALIGNMENT (dr_info, vector_alignment);
1475 : :
1476 : : /* If the main loop has peeled for alignment we have no way of knowing
1477 : : whether the data accesses in the epilogues are aligned. We can't at
1478 : : compile time answer the question whether we have entered the main loop or
1479 : : not. Fixes PR 92351. */
1480 : 1557625 : if (loop_vinfo)
1481 : : {
1482 : 692290 : loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
1483 : 692290 : if (orig_loop_vinfo
1484 : 59518 : && LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo) != 0)
1485 : : return;
1486 : : }
1487 : :
1488 : 1557166 : unsigned HOST_WIDE_INT vect_align_c;
1489 : 1557166 : if (!vector_alignment.is_constant (&vect_align_c))
1490 : : return;
1491 : :
1492 : : /* No step for BB vectorization. */
1493 : 1557166 : if (!loop)
1494 : : {
1495 : 865335 : gcc_assert (integer_zerop (drb->step));
1496 : : step_preserves_misalignment_p = true;
1497 : : }
1498 : :
1499 : : else
1500 : : {
1501 : : /* We can only use base and misalignment information relative to
1502 : : an innermost loop if the misalignment stays the same throughout the
1503 : : execution of the loop. As above, this is the case if the stride of
1504 : : the dataref evenly divides by the alignment. */
1505 : 691831 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
1506 : 691831 : step_preserves_misalignment_p
1507 : 691831 : = multiple_p (drb->step_alignment * vf, vect_align_c);
1508 : :
1509 : 691831 : if (!step_preserves_misalignment_p && dump_enabled_p ())
1510 : 313 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1511 : : "step doesn't divide the vector alignment.\n");
1512 : :
1513 : : /* In case the dataref is in an inner-loop of the loop that is being
1514 : : vectorized (LOOP), we use the base and misalignment information
1515 : : relative to the outer-loop (LOOP). This is ok only if the
1516 : : misalignment stays the same throughout the execution of the
1517 : : inner-loop, which is why we have to check that the stride of the
1518 : : dataref in the inner-loop evenly divides by the vector alignment. */
1519 : 691831 : if (step_preserves_misalignment_p
1520 : 691831 : && nested_in_vect_loop_p (loop, stmt_info))
1521 : : {
1522 : 1492 : step_preserves_misalignment_p
1523 : 1492 : = (DR_STEP_ALIGNMENT (dr_info->dr) % vect_align_c) == 0;
1524 : :
1525 : 1492 : if (dump_enabled_p ())
1526 : : {
1527 : 517 : if (step_preserves_misalignment_p)
1528 : 371 : dump_printf_loc (MSG_NOTE, vect_location,
1529 : : "inner step divides the vector alignment.\n");
1530 : : else
1531 : 146 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1532 : : "inner step doesn't divide the vector"
1533 : : " alignment.\n");
1534 : : }
1535 : : }
1536 : : }
1537 : :
1538 : 1557166 : unsigned int base_alignment = drb->base_alignment;
1539 : 1557166 : unsigned int base_misalignment = drb->base_misalignment;
1540 : :
1541 : : /* Calculate the maximum of the pooled base address alignment and the
1542 : : alignment that we can compute for DR itself. */
1543 : 1557166 : std::pair<stmt_vec_info, innermost_loop_behavior *> *entry
1544 : 1557166 : = base_alignments->get (drb->base_address);
1545 : 1557166 : if (entry
1546 : 1553733 : && base_alignment < (*entry).second->base_alignment
1547 : 1559309 : && (loop_vinfo
1548 : 1033 : || (dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt_info->stmt),
1549 : 1033 : gimple_bb (entry->first->stmt))
1550 : 951 : && (gimple_bb (stmt_info->stmt) != gimple_bb (entry->first->stmt)
1551 : 678 : || (entry->first->dr_aux.group <= dr_info->group)))))
1552 : : {
1553 : 2044 : base_alignment = entry->second->base_alignment;
1554 : 2044 : base_misalignment = entry->second->base_misalignment;
1555 : : }
1556 : :
1557 : 1557166 : if (drb->offset_alignment < vect_align_c
1558 : 1484612 : || !step_preserves_misalignment_p
1559 : : /* We need to know whether the step wrt the vectorized loop is
1560 : : negative when computing the starting misalignment below. */
1561 : 1475194 : || TREE_CODE (drb->step) != INTEGER_CST)
1562 : : {
1563 : 110615 : if (dump_enabled_p ())
1564 : 4340 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1565 : : "Unknown alignment for access: %T\n", ref);
1566 : 110615 : return;
1567 : : }
1568 : :
1569 : 1446551 : if (base_alignment < vect_align_c)
1570 : : {
1571 : 664680 : unsigned int max_alignment;
1572 : 664680 : tree base = get_base_for_alignment (drb->base_address, &max_alignment);
1573 : 664680 : if (max_alignment < vect_align_c
1574 : 664680 : || !vect_can_force_dr_alignment_p (base,
1575 : 664516 : vect_align_c * BITS_PER_UNIT))
1576 : : {
1577 : 466127 : if (dump_enabled_p ())
1578 : 15786 : dump_printf_loc (MSG_NOTE, vect_location,
1579 : : "can't force alignment of ref: %T\n", ref);
1580 : 466127 : return;
1581 : : }
1582 : :
1583 : : /* Force the alignment of the decl.
1584 : : NOTE: This is the only change to the code we make during
1585 : : the analysis phase, before deciding to vectorize the loop. */
1586 : 198553 : if (dump_enabled_p ())
1587 : 8223 : dump_printf_loc (MSG_NOTE, vect_location,
1588 : : "force alignment of %T\n", ref);
1589 : :
1590 : 198553 : dr_info->base_decl = base;
1591 : 198553 : dr_info->base_misaligned = true;
1592 : 198553 : base_misalignment = 0;
1593 : : }
1594 : 980424 : poly_int64 misalignment
1595 : 980424 : = base_misalignment + wi::to_poly_offset (drb->init).force_shwi ();
1596 : :
1597 : 980424 : unsigned int const_misalignment;
1598 : 980424 : if (!known_misalignment (misalignment, vect_align_c, &const_misalignment))
1599 : : {
1600 : : if (dump_enabled_p ())
1601 : : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1602 : : "Non-constant misalignment for access: %T\n", ref);
1603 : : return;
1604 : : }
1605 : :
1606 : 980424 : SET_DR_MISALIGNMENT (dr_info, const_misalignment);
1607 : :
1608 : 980424 : if (dump_enabled_p ())
1609 : 34040 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1610 : : "misalign = %d bytes of ref %T\n",
1611 : : const_misalignment, ref);
1612 : :
1613 : : return;
1614 : : }
1615 : :
1616 : : /* Return whether DR_INFO, which is related to DR_PEEL_INFO in
1617 : : that it only differs in DR_INIT, is aligned if DR_PEEL_INFO
1618 : : is made aligned via peeling. */
1619 : :
1620 : : static bool
1621 : 1597024 : vect_dr_aligned_if_related_peeled_dr_is (dr_vec_info *dr_info,
1622 : : dr_vec_info *dr_peel_info)
1623 : : {
1624 : 1597024 : if (multiple_p (DR_TARGET_ALIGNMENT (dr_peel_info),
1625 : 1598101 : DR_TARGET_ALIGNMENT (dr_info)))
1626 : : {
1627 : 1595947 : poly_offset_int diff
1628 : 1595947 : = (wi::to_poly_offset (DR_INIT (dr_peel_info->dr))
1629 : 1595947 : - wi::to_poly_offset (DR_INIT (dr_info->dr)));
1630 : 1595947 : if (known_eq (diff, 0)
1631 : 1595947 : || multiple_p (diff, DR_TARGET_ALIGNMENT (dr_info)))
1632 : 552008 : return true;
1633 : : }
1634 : : return false;
1635 : : }
1636 : :
1637 : : /* Return whether DR_INFO is aligned if DR_PEEL_INFO is made
1638 : : aligned via peeling. */
1639 : :
1640 : : static bool
1641 : 157454 : vect_dr_aligned_if_peeled_dr_is (dr_vec_info *dr_info,
1642 : : dr_vec_info *dr_peel_info)
1643 : : {
1644 : 157454 : if (!operand_equal_p (DR_BASE_ADDRESS (dr_info->dr),
1645 : 157454 : DR_BASE_ADDRESS (dr_peel_info->dr), 0)
1646 : 47793 : || !operand_equal_p (DR_OFFSET (dr_info->dr),
1647 : 47793 : DR_OFFSET (dr_peel_info->dr), 0)
1648 : 204447 : || !operand_equal_p (DR_STEP (dr_info->dr),
1649 : 46993 : DR_STEP (dr_peel_info->dr), 0))
1650 : 110826 : return false;
1651 : :
1652 : 46628 : return vect_dr_aligned_if_related_peeled_dr_is (dr_info, dr_peel_info);
1653 : : }
1654 : :
1655 : : /* Compute the value for dr_info->misalign so that the access appears
1656 : : aligned. This is used by peeling to compensate for dr_misalignment
1657 : : applying the offset for negative step. */
1658 : :
1659 : : int
1660 : 16353 : vect_dr_misalign_for_aligned_access (dr_vec_info *dr_info)
1661 : : {
1662 : 16353 : if (tree_int_cst_sgn (DR_STEP (dr_info->dr)) >= 0)
1663 : : return 0;
1664 : :
1665 : 160 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
1666 : 160 : poly_int64 misalignment
1667 : 160 : = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
1668 : 160 : * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
1669 : :
1670 : 160 : unsigned HOST_WIDE_INT target_alignment_c;
1671 : 160 : int misalign;
1672 : 160 : if (!dr_info->target_alignment.is_constant (&target_alignment_c)
1673 : 160 : || !known_misalignment (misalignment, target_alignment_c, &misalign))
1674 : : return DR_MISALIGNMENT_UNKNOWN;
1675 : 160 : return misalign;
1676 : : }
1677 : :
1678 : : /* Function vect_update_misalignment_for_peel.
1679 : : Sets DR_INFO's misalignment
1680 : : - to 0 if it has the same alignment as DR_PEEL_INFO,
1681 : : - to the misalignment computed using NPEEL if DR_INFO's salignment is known,
1682 : : - to -1 (unknown) otherwise.
1683 : :
1684 : : DR_INFO - the data reference whose misalignment is to be adjusted.
1685 : : DR_PEEL_INFO - the data reference whose misalignment is being made
1686 : : zero in the vector loop by the peel.
1687 : : NPEEL - the number of iterations in the peel loop if the misalignment
1688 : : of DR_PEEL_INFO is known at compile time. */
1689 : :
1690 : : static void
1691 : 864 : vect_update_misalignment_for_peel (dr_vec_info *dr_info,
1692 : : dr_vec_info *dr_peel_info, int npeel)
1693 : : {
1694 : : /* If dr_info is aligned of dr_peel_info is, then mark it so. */
1695 : 864 : if (vect_dr_aligned_if_peeled_dr_is (dr_info, dr_peel_info))
1696 : : {
1697 : 426 : SET_DR_MISALIGNMENT (dr_info,
1698 : : vect_dr_misalign_for_aligned_access (dr_peel_info));
1699 : 426 : return;
1700 : : }
1701 : :
1702 : 438 : unsigned HOST_WIDE_INT alignment;
1703 : 438 : if (DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)
1704 : 438 : && known_alignment_for_access_p (dr_info,
1705 : 438 : STMT_VINFO_VECTYPE (dr_info->stmt))
1706 : 193 : && known_alignment_for_access_p (dr_peel_info,
1707 : 193 : STMT_VINFO_VECTYPE (dr_peel_info->stmt)))
1708 : : {
1709 : 185 : int misal = dr_info->misalignment;
1710 : 185 : misal += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
1711 : 185 : misal &= alignment - 1;
1712 : 185 : set_dr_misalignment (dr_info, misal);
1713 : 185 : return;
1714 : : }
1715 : :
1716 : 253 : if (dump_enabled_p ())
1717 : 23 : dump_printf_loc (MSG_NOTE, vect_location, "Setting misalignment " \
1718 : : "to unknown (-1).\n");
1719 : 253 : SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
1720 : : }
1721 : :
1722 : : /* Return true if alignment is relevant for DR_INFO. */
1723 : :
1724 : : static bool
1725 : 1454930 : vect_relevant_for_alignment_p (dr_vec_info *dr_info)
1726 : : {
1727 : 1454930 : stmt_vec_info stmt_info = dr_info->stmt;
1728 : :
1729 : 1454930 : if (!STMT_VINFO_RELEVANT_P (stmt_info))
1730 : : return false;
1731 : :
1732 : : /* For interleaving, only the alignment of the first access matters. */
1733 : 1453784 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
1734 : 1654346 : && DR_GROUP_FIRST_ELEMENT (stmt_info) != stmt_info)
1735 : : return false;
1736 : :
1737 : : /* Scatter-gather and invariant accesses continue to address individual
1738 : : scalars, so vector-level alignment is irrelevant. */
1739 : 1364856 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
1740 : 1364856 : || integer_zerop (DR_STEP (dr_info->dr)))
1741 : 43675 : return false;
1742 : :
1743 : : /* Strided accesses perform only component accesses, alignment is
1744 : : irrelevant for them. */
1745 : 1321181 : if (STMT_VINFO_STRIDED_P (stmt_info)
1746 : 1321181 : && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
1747 : : return false;
1748 : :
1749 : : return true;
1750 : : }
1751 : :
1752 : : /* Given an memory reference EXP return whether its alignment is less
1753 : : than its size. */
1754 : :
1755 : : static bool
1756 : 1167442 : not_size_aligned (tree exp)
1757 : : {
1758 : 1167442 : if (!tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (exp))))
1759 : : return true;
1760 : :
1761 : 1167442 : return (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (exp)))
1762 : 1167442 : > get_object_alignment (exp));
1763 : : }
1764 : :
1765 : : /* Function vector_alignment_reachable_p
1766 : :
1767 : : Return true if vector alignment for DR_INFO is reachable by peeling
1768 : : a few loop iterations. Return false otherwise. */
1769 : :
1770 : : static bool
1771 : 492400 : vector_alignment_reachable_p (dr_vec_info *dr_info, poly_uint64 vf)
1772 : : {
1773 : 492400 : stmt_vec_info stmt_info = dr_info->stmt;
1774 : 492400 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
1775 : 492400 : poly_uint64 nelements = TYPE_VECTOR_SUBPARTS (vectype);
1776 : 984800 : poly_uint64 vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
1777 : 492400 : unsigned elem_size = vector_element_size (vector_size, nelements);
1778 : 492400 : unsigned group_size = 1;
1779 : :
1780 : 492400 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
1781 : : {
1782 : : /* For interleaved access we peel only if number of iterations in
1783 : : the prolog loop ({VF - misalignment}), is a multiple of the
1784 : : number of the interleaved accesses. */
1785 : :
1786 : : /* FORNOW: handle only known alignment. */
1787 : 64371 : if (!known_alignment_for_access_p (dr_info, vectype))
1788 : 492400 : return false;
1789 : :
1790 : 35429 : unsigned mis_in_elements = dr_misalignment (dr_info, vectype) / elem_size;
1791 : 46741 : if (!multiple_p (nelements - mis_in_elements, DR_GROUP_SIZE (stmt_info)))
1792 : : return false;
1793 : :
1794 : 11312 : group_size = DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
1795 : : }
1796 : :
1797 : : /* If the vectorization factor does not guarantee DR advancement of
1798 : : a multiple of the target alignment no peeling will help. */
1799 : 439341 : if (!multiple_p (elem_size * group_size * vf, dr_target_alignment (dr_info)))
1800 : 76 : return false;
1801 : :
1802 : : /* If misalignment is known at the compile time then allow peeling
1803 : : only if natural alignment is reachable through peeling. */
1804 : 439265 : if (known_alignment_for_access_p (dr_info, vectype)
1805 : 692344 : && !aligned_access_p (dr_info, vectype))
1806 : : {
1807 : 12248 : HOST_WIDE_INT elmsize =
1808 : 12248 : int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
1809 : 12248 : if (dump_enabled_p ())
1810 : : {
1811 : 787 : dump_printf_loc (MSG_NOTE, vect_location,
1812 : : "data size = %wd. misalignment = %d.\n", elmsize,
1813 : 787 : dr_misalignment (dr_info, vectype));
1814 : : }
1815 : 12248 : if (dr_misalignment (dr_info, vectype) % elmsize)
1816 : : {
1817 : 53 : if (dump_enabled_p ())
1818 : 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1819 : : "data size does not divide the misalignment.\n");
1820 : 53 : return false;
1821 : : }
1822 : : }
1823 : :
1824 : 439212 : if (!known_alignment_for_access_p (dr_info, vectype))
1825 : : {
1826 : 186186 : tree type = TREE_TYPE (DR_REF (dr_info->dr));
1827 : 186186 : bool is_packed = not_size_aligned (DR_REF (dr_info->dr));
1828 : 186186 : if (dump_enabled_p ())
1829 : 13891 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
1830 : : "Unknown misalignment, %snaturally aligned\n",
1831 : : is_packed ? "not " : "");
1832 : 186186 : return targetm.vectorize.vector_alignment_reachable (type, is_packed);
1833 : : }
1834 : :
1835 : : return true;
1836 : : }
1837 : :
1838 : :
1839 : : /* Calculate the cost of the memory access represented by DR_INFO. */
1840 : :
1841 : : static void
1842 : 567313 : vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
1843 : : dr_alignment_support alignment_support_scheme,
1844 : : int misalignment,
1845 : : unsigned int *inside_cost,
1846 : : unsigned int *outside_cost,
1847 : : stmt_vector_for_cost *body_cost_vec,
1848 : : stmt_vector_for_cost *prologue_cost_vec)
1849 : : {
1850 : 567313 : stmt_vec_info stmt_info = dr_info->stmt;
1851 : 567313 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
1852 : 567313 : int ncopies;
1853 : :
1854 : 567313 : if (PURE_SLP_STMT (stmt_info))
1855 : : ncopies = 1;
1856 : : else
1857 : 15730 : ncopies = vect_get_num_copies (loop_vinfo, STMT_VINFO_VECTYPE (stmt_info));
1858 : :
1859 : 567313 : if (DR_IS_READ (dr_info->dr))
1860 : 404748 : vect_get_load_cost (vinfo, stmt_info, NULL, ncopies,
1861 : : alignment_support_scheme, misalignment, true,
1862 : : inside_cost, outside_cost, prologue_cost_vec,
1863 : : body_cost_vec, false);
1864 : : else
1865 : 162565 : vect_get_store_cost (vinfo,stmt_info, NULL, ncopies,
1866 : : alignment_support_scheme, misalignment, inside_cost,
1867 : : body_cost_vec);
1868 : :
1869 : 567313 : if (dump_enabled_p ())
1870 : 26449 : dump_printf_loc (MSG_NOTE, vect_location,
1871 : : "vect_get_data_access_cost: inside_cost = %d, "
1872 : : "outside_cost = %d.\n", *inside_cost, *outside_cost);
1873 : 567313 : }
1874 : :
1875 : :
1876 : : typedef struct _vect_peel_info
1877 : : {
1878 : : dr_vec_info *dr_info;
1879 : : int npeel;
1880 : : unsigned int count;
1881 : : } *vect_peel_info;
1882 : :
1883 : : typedef struct _vect_peel_extended_info
1884 : : {
1885 : : vec_info *vinfo;
1886 : : struct _vect_peel_info peel_info;
1887 : : unsigned int inside_cost;
1888 : : unsigned int outside_cost;
1889 : : } *vect_peel_extended_info;
1890 : :
1891 : :
1892 : : /* Peeling hashtable helpers. */
1893 : :
1894 : : struct peel_info_hasher : free_ptr_hash <_vect_peel_info>
1895 : : {
1896 : : static inline hashval_t hash (const _vect_peel_info *);
1897 : : static inline bool equal (const _vect_peel_info *, const _vect_peel_info *);
1898 : : };
1899 : :
1900 : : inline hashval_t
1901 : 706370 : peel_info_hasher::hash (const _vect_peel_info *peel_info)
1902 : : {
1903 : 706370 : return (hashval_t) peel_info->npeel;
1904 : : }
1905 : :
1906 : : inline bool
1907 : 390601 : peel_info_hasher::equal (const _vect_peel_info *a, const _vect_peel_info *b)
1908 : : {
1909 : 390601 : return (a->npeel == b->npeel);
1910 : : }
1911 : :
1912 : :
1913 : : /* Insert DR_INFO into peeling hash table with NPEEL as key. */
1914 : :
1915 : : static void
1916 : 316585 : vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
1917 : : loop_vec_info loop_vinfo, dr_vec_info *dr_info,
1918 : : int npeel, bool supportable_if_not_aligned)
1919 : : {
1920 : 316585 : struct _vect_peel_info elem, *slot;
1921 : 316585 : _vect_peel_info **new_slot;
1922 : :
1923 : 316585 : elem.npeel = npeel;
1924 : 316585 : slot = peeling_htab->find (&elem);
1925 : 316585 : if (slot)
1926 : 135331 : slot->count++;
1927 : : else
1928 : : {
1929 : 181254 : slot = XNEW (struct _vect_peel_info);
1930 : 181254 : slot->npeel = npeel;
1931 : 181254 : slot->dr_info = dr_info;
1932 : 181254 : slot->count = 1;
1933 : 181254 : new_slot = peeling_htab->find_slot (slot, INSERT);
1934 : 181254 : *new_slot = slot;
1935 : : }
1936 : :
1937 : : /* If this DR is not supported with unknown misalignment then bias
1938 : : this slot when the cost model is disabled. */
1939 : 316585 : if (!supportable_if_not_aligned
1940 : 316585 : && unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
1941 : 4880 : slot->count += VECT_MAX_COST;
1942 : 316585 : }
1943 : :
1944 : :
1945 : : /* Traverse peeling hash table to find peeling option that aligns maximum
1946 : : number of data accesses. */
1947 : :
1948 : : int
1949 : 38141 : vect_peeling_hash_get_most_frequent (_vect_peel_info **slot,
1950 : : _vect_peel_extended_info *max)
1951 : : {
1952 : 38141 : vect_peel_info elem = *slot;
1953 : :
1954 : 38141 : if (elem->count > max->peel_info.count
1955 : 23536 : || (elem->count == max->peel_info.count
1956 : 18613 : && max->peel_info.npeel > elem->npeel))
1957 : : {
1958 : 14619 : max->peel_info.npeel = elem->npeel;
1959 : 14619 : max->peel_info.count = elem->count;
1960 : 14619 : max->peel_info.dr_info = elem->dr_info;
1961 : : }
1962 : :
1963 : 38141 : return 1;
1964 : : }
1965 : :
1966 : : /* Get the costs of peeling NPEEL iterations for LOOP_VINFO, checking
1967 : : data access costs for all data refs. If UNKNOWN_MISALIGNMENT is true,
1968 : : npeel is computed at runtime but DR0_INFO's misalignment will be zero
1969 : : after peeling. */
1970 : :
1971 : : static void
1972 : 316966 : vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
1973 : : dr_vec_info *dr0_info,
1974 : : unsigned int *inside_cost,
1975 : : unsigned int *outside_cost,
1976 : : stmt_vector_for_cost *body_cost_vec,
1977 : : stmt_vector_for_cost *prologue_cost_vec,
1978 : : unsigned int npeel)
1979 : : {
1980 : 316966 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
1981 : :
1982 : 316966 : bool dr0_alignment_known_p
1983 : : = (dr0_info
1984 : 582649 : && known_alignment_for_access_p (dr0_info,
1985 : 265683 : STMT_VINFO_VECTYPE (dr0_info->stmt)));
1986 : :
1987 : 1553257 : for (data_reference *dr : datarefs)
1988 : : {
1989 : 602359 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
1990 : 602359 : if (!vect_relevant_for_alignment_p (dr_info))
1991 : 35046 : continue;
1992 : :
1993 : 567313 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
1994 : 567313 : dr_alignment_support alignment_support_scheme;
1995 : 567313 : int misalignment;
1996 : 567313 : unsigned HOST_WIDE_INT alignment;
1997 : :
1998 : 567313 : bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
1999 : 567313 : size_zero_node) < 0;
2000 : 567313 : poly_int64 off = 0;
2001 : 567313 : if (negative)
2002 : 20026 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
2003 : 20026 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
2004 : :
2005 : 567313 : if (npeel == 0)
2006 : 289429 : misalignment = dr_misalignment (dr_info, vectype, off);
2007 : 277884 : else if (dr_info == dr0_info
2008 : 277884 : || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
2009 : : misalignment = 0;
2010 : 89943 : else if (!dr0_alignment_known_p
2011 : 7000 : || !known_alignment_for_access_p (dr_info, vectype)
2012 : 96943 : || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
2013 : : misalignment = DR_MISALIGNMENT_UNKNOWN;
2014 : : else
2015 : : {
2016 : 6014 : misalignment = dr_misalignment (dr_info, vectype, off);
2017 : 6014 : misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
2018 : 6014 : misalignment &= alignment - 1;
2019 : : }
2020 : 567313 : alignment_support_scheme
2021 : 567313 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2022 : : misalignment);
2023 : :
2024 : 567313 : vect_get_data_access_cost (loop_vinfo, dr_info,
2025 : : alignment_support_scheme, misalignment,
2026 : : inside_cost, outside_cost,
2027 : : body_cost_vec, prologue_cost_vec);
2028 : : }
2029 : 316966 : }
2030 : :
2031 : : /* Traverse peeling hash table and calculate cost for each peeling option.
2032 : : Find the one with the lowest cost. */
2033 : :
2034 : : int
2035 : 123006 : vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
2036 : : _vect_peel_extended_info *min)
2037 : : {
2038 : 123006 : vect_peel_info elem = *slot;
2039 : 123006 : int dummy;
2040 : 123006 : unsigned int inside_cost = 0, outside_cost = 0;
2041 : 123006 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
2042 : 123006 : stmt_vector_for_cost prologue_cost_vec, body_cost_vec,
2043 : : epilogue_cost_vec;
2044 : :
2045 : 123006 : prologue_cost_vec.create (2);
2046 : 123006 : body_cost_vec.create (2);
2047 : 123006 : epilogue_cost_vec.create (2);
2048 : :
2049 : 123006 : vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost,
2050 : : &outside_cost, &body_cost_vec,
2051 : 123006 : &prologue_cost_vec, elem->npeel);
2052 : :
2053 : 123006 : body_cost_vec.release ();
2054 : :
2055 : 246012 : outside_cost += vect_get_known_peeling_cost
2056 : 123006 : (loop_vinfo, elem->npeel, &dummy,
2057 : : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
2058 : : &prologue_cost_vec, &epilogue_cost_vec);
2059 : :
2060 : : /* Prologue and epilogue costs are added to the target model later.
2061 : : These costs depend only on the scalar iteration cost, the
2062 : : number of peeling iterations finally chosen, and the number of
2063 : : misaligned statements. So discard the information found here. */
2064 : 123006 : prologue_cost_vec.release ();
2065 : 123006 : epilogue_cost_vec.release ();
2066 : :
2067 : 123006 : if (inside_cost < min->inside_cost
2068 : 1200 : || (inside_cost == min->inside_cost
2069 : 986 : && outside_cost < min->outside_cost))
2070 : : {
2071 : 121812 : min->inside_cost = inside_cost;
2072 : 121812 : min->outside_cost = outside_cost;
2073 : 121812 : min->peel_info.dr_info = elem->dr_info;
2074 : 121812 : min->peel_info.npeel = elem->npeel;
2075 : 121812 : min->peel_info.count = elem->count;
2076 : : }
2077 : :
2078 : 123006 : return 1;
2079 : : }
2080 : :
2081 : :
2082 : : /* Choose best peeling option by traversing peeling hash table and either
2083 : : choosing an option with the lowest cost (if cost model is enabled) or the
2084 : : option that aligns as many accesses as possible. */
2085 : :
2086 : : static struct _vect_peel_extended_info
2087 : 134928 : vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_htab,
2088 : : loop_vec_info loop_vinfo)
2089 : : {
2090 : 134928 : struct _vect_peel_extended_info res;
2091 : :
2092 : 134928 : res.peel_info.dr_info = NULL;
2093 : 134928 : res.vinfo = loop_vinfo;
2094 : :
2095 : 134928 : if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
2096 : : {
2097 : 120376 : res.inside_cost = INT_MAX;
2098 : 120376 : res.outside_cost = INT_MAX;
2099 : 120376 : peeling_htab->traverse <_vect_peel_extended_info *,
2100 : 243382 : vect_peeling_hash_get_lowest_cost> (&res);
2101 : : }
2102 : : else
2103 : : {
2104 : 14552 : res.peel_info.count = 0;
2105 : 14552 : peeling_htab->traverse <_vect_peel_extended_info *,
2106 : 52693 : vect_peeling_hash_get_most_frequent> (&res);
2107 : 14552 : res.inside_cost = 0;
2108 : 14552 : res.outside_cost = 0;
2109 : : }
2110 : :
2111 : 134928 : return res;
2112 : : }
2113 : :
2114 : : /* Return true if the new peeling NPEEL is supported. */
2115 : :
2116 : : static bool
2117 : 60334 : vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
2118 : : unsigned npeel)
2119 : : {
2120 : 60334 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
2121 : 60334 : enum dr_alignment_support supportable_dr_alignment;
2122 : :
2123 : 60334 : bool dr0_alignment_known_p
2124 : 120668 : = known_alignment_for_access_p (dr0_info,
2125 : 60334 : STMT_VINFO_VECTYPE (dr0_info->stmt));
2126 : :
2127 : : /* Ensure that all data refs can be vectorized after the peel. */
2128 : 237624 : for (data_reference *dr : datarefs)
2129 : : {
2130 : 83692 : if (dr == dr0_info->dr)
2131 : 50864 : continue;
2132 : :
2133 : 32828 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2134 : 32828 : if (!vect_relevant_for_alignment_p (dr_info)
2135 : 32828 : || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
2136 : 4994 : continue;
2137 : :
2138 : 27834 : tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
2139 : 27834 : int misalignment;
2140 : 27834 : unsigned HOST_WIDE_INT alignment;
2141 : 27834 : if (!dr0_alignment_known_p
2142 : 1829 : || !known_alignment_for_access_p (dr_info, vectype)
2143 : 29663 : || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
2144 : : misalignment = DR_MISALIGNMENT_UNKNOWN;
2145 : : else
2146 : : {
2147 : 1815 : misalignment = dr_misalignment (dr_info, vectype);
2148 : 1815 : misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
2149 : 1815 : misalignment &= alignment - 1;
2150 : : }
2151 : 27834 : supportable_dr_alignment
2152 : 27834 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2153 : : misalignment);
2154 : 27834 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2155 : 60334 : return false;
2156 : : }
2157 : :
2158 : : return true;
2159 : : }
2160 : :
2161 : : /* Compare two data-references DRA and DRB to group them into chunks
2162 : : with related alignment. */
2163 : :
2164 : : static int
2165 : 3671822 : dr_align_group_sort_cmp (const void *dra_, const void *drb_)
2166 : : {
2167 : 3671822 : data_reference_p dra = *(data_reference_p *)const_cast<void *>(dra_);
2168 : 3671822 : data_reference_p drb = *(data_reference_p *)const_cast<void *>(drb_);
2169 : 3671822 : int cmp;
2170 : :
2171 : : /* Stabilize sort. */
2172 : 3671822 : if (dra == drb)
2173 : : return 0;
2174 : :
2175 : : /* Ordering of DRs according to base. */
2176 : 3671822 : cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
2177 : : DR_BASE_ADDRESS (drb));
2178 : 3671822 : if (cmp != 0)
2179 : : return cmp;
2180 : :
2181 : : /* And according to DR_OFFSET. */
2182 : 1697589 : cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
2183 : 1697589 : if (cmp != 0)
2184 : : return cmp;
2185 : :
2186 : : /* And after step. */
2187 : 1684192 : cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
2188 : 1684192 : if (cmp != 0)
2189 : : return cmp;
2190 : :
2191 : : /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */
2192 : 1679673 : cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
2193 : 1679673 : if (cmp == 0)
2194 : 211972 : return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
2195 : : return cmp;
2196 : : }
2197 : :
2198 : : /* Function vect_enhance_data_refs_alignment
2199 : :
2200 : : This pass will use loop versioning and loop peeling in order to enhance
2201 : : the alignment of data references in the loop.
2202 : :
2203 : : FOR NOW: we assume that whatever versioning/peeling takes place, only the
2204 : : original loop is to be vectorized. Any other loops that are created by
2205 : : the transformations performed in this pass - are not supposed to be
2206 : : vectorized. This restriction will be relaxed.
2207 : :
2208 : : This pass will require a cost model to guide it whether to apply peeling
2209 : : or versioning or a combination of the two. For example, the scheme that
2210 : : intel uses when given a loop with several memory accesses, is as follows:
2211 : : choose one memory access ('p') which alignment you want to force by doing
2212 : : peeling. Then, either (1) generate a loop in which 'p' is aligned and all
2213 : : other accesses are not necessarily aligned, or (2) use loop versioning to
2214 : : generate one loop in which all accesses are aligned, and another loop in
2215 : : which only 'p' is necessarily aligned.
2216 : :
2217 : : ("Automatic Intra-Register Vectorization for the Intel Architecture",
2218 : : Aart J.C. Bik, Milind Girkar, Paul M. Grey and Ximmin Tian, International
2219 : : Journal of Parallel Programming, Vol. 30, No. 2, April 2002.)
2220 : :
2221 : : Devising a cost model is the most critical aspect of this work. It will
2222 : : guide us on which access to peel for, whether to use loop versioning, how
2223 : : many versions to create, etc. The cost model will probably consist of
2224 : : generic considerations as well as target specific considerations (on
2225 : : powerpc for example, misaligned stores are more painful than misaligned
2226 : : loads).
2227 : :
2228 : : Here are the general steps involved in alignment enhancements:
2229 : :
2230 : : -- original loop, before alignment analysis:
2231 : : for (i=0; i<N; i++){
2232 : : x = q[i]; # DR_MISALIGNMENT(q) = unknown
2233 : : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2234 : : }
2235 : :
2236 : : -- After vect_compute_data_refs_alignment:
2237 : : for (i=0; i<N; i++){
2238 : : x = q[i]; # DR_MISALIGNMENT(q) = 3
2239 : : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2240 : : }
2241 : :
2242 : : -- Possibility 1: we do loop versioning:
2243 : : if (p is aligned) {
2244 : : for (i=0; i<N; i++){ # loop 1A
2245 : : x = q[i]; # DR_MISALIGNMENT(q) = 3
2246 : : p[i] = y; # DR_MISALIGNMENT(p) = 0
2247 : : }
2248 : : }
2249 : : else {
2250 : : for (i=0; i<N; i++){ # loop 1B
2251 : : x = q[i]; # DR_MISALIGNMENT(q) = 3
2252 : : p[i] = y; # DR_MISALIGNMENT(p) = unaligned
2253 : : }
2254 : : }
2255 : :
2256 : : -- Possibility 2: we do loop peeling:
2257 : : for (i = 0; i < 3; i++){ # (scalar loop, not to be vectorized).
2258 : : x = q[i];
2259 : : p[i] = y;
2260 : : }
2261 : : for (i = 3; i < N; i++){ # loop 2A
2262 : : x = q[i]; # DR_MISALIGNMENT(q) = 0
2263 : : p[i] = y; # DR_MISALIGNMENT(p) = unknown
2264 : : }
2265 : :
2266 : : -- Possibility 3: combination of loop peeling and versioning:
2267 : : for (i = 0; i < 3; i++){ # (scalar loop, not to be vectorized).
2268 : : x = q[i];
2269 : : p[i] = y;
2270 : : }
2271 : : if (p is aligned) {
2272 : : for (i = 3; i<N; i++){ # loop 3A
2273 : : x = q[i]; # DR_MISALIGNMENT(q) = 0
2274 : : p[i] = y; # DR_MISALIGNMENT(p) = 0
2275 : : }
2276 : : }
2277 : : else {
2278 : : for (i = 3; i<N; i++){ # loop 3B
2279 : : x = q[i]; # DR_MISALIGNMENT(q) = 0
2280 : : p[i] = y; # DR_MISALIGNMENT(p) = unaligned
2281 : : }
2282 : : }
2283 : :
2284 : : These loops are later passed to loop_transform to be vectorized. The
2285 : : vectorizer will use the alignment information to guide the transformation
2286 : : (whether to generate regular loads/stores, or with special handling for
2287 : : misalignment). */
2288 : :
2289 : : opt_result
2290 : 318870 : vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
2291 : : {
2292 : 318870 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
2293 : 318870 : dr_vec_info *first_store = NULL;
2294 : 318870 : dr_vec_info *dr0_info = NULL;
2295 : 318870 : struct data_reference *dr;
2296 : 318870 : unsigned int i;
2297 : 318870 : bool do_peeling = false;
2298 : 318870 : bool do_versioning = false;
2299 : 318870 : unsigned int npeel = 0;
2300 : 318870 : bool one_misalignment_known = false;
2301 : 318870 : bool one_misalignment_unknown = false;
2302 : 318870 : bool one_dr_unsupportable = false;
2303 : 318870 : dr_vec_info *unsupportable_dr_info = NULL;
2304 : 318870 : unsigned int dr0_same_align_drs = 0, first_store_same_align_drs = 0;
2305 : 318870 : hash_table<peel_info_hasher> peeling_htab (1);
2306 : :
2307 : 318870 : DUMP_VECT_SCOPE ("vect_enhance_data_refs_alignment");
2308 : :
2309 : : /* Reset data so we can safely be called multiple times. */
2310 : 318870 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
2311 : 318870 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = 0;
2312 : :
2313 : 318870 : if (LOOP_VINFO_DATAREFS (loop_vinfo).is_empty ())
2314 : 13277 : return opt_result::success ();
2315 : :
2316 : : /* Sort the vector of datarefs so DRs that have the same or dependent
2317 : : alignment are next to each other. */
2318 : 305593 : auto_vec<data_reference_p> datarefs
2319 : 305593 : = LOOP_VINFO_DATAREFS (loop_vinfo).copy ();
2320 : 305593 : datarefs.qsort (dr_align_group_sort_cmp);
2321 : :
2322 : : /* Compute the number of DRs that become aligned when we peel
2323 : : a dataref so it becomes aligned. */
2324 : 611186 : auto_vec<unsigned> n_same_align_refs (datarefs.length ());
2325 : 305593 : n_same_align_refs.quick_grow_cleared (datarefs.length ());
2326 : 305593 : unsigned i0;
2327 : 627054 : for (i0 = 0; i0 < datarefs.length (); ++i0)
2328 : 318230 : if (DR_BASE_ADDRESS (datarefs[i0]))
2329 : : break;
2330 : 1963392 : for (i = i0 + 1; i <= datarefs.length (); ++i)
2331 : : {
2332 : 676103 : if (i == datarefs.length ()
2333 : 373741 : || !operand_equal_p (DR_BASE_ADDRESS (datarefs[i0]),
2334 : 373741 : DR_BASE_ADDRESS (datarefs[i]), 0)
2335 : 176103 : || !operand_equal_p (DR_OFFSET (datarefs[i0]),
2336 : 176103 : DR_OFFSET (datarefs[i]), 0)
2337 : 850988 : || !operand_equal_p (DR_STEP (datarefs[i0]),
2338 : 174885 : DR_STEP (datarefs[i]), 0))
2339 : : {
2340 : : /* The subgroup [i0, i-1] now only differs in DR_INIT and
2341 : : possibly DR_TARGET_ALIGNMENT. Still the whole subgroup
2342 : : will get known misalignment if we align one of the refs
2343 : : with the largest DR_TARGET_ALIGNMENT. */
2344 : 1177834 : for (unsigned j = i0; j < i; ++j)
2345 : : {
2346 : 676103 : dr_vec_info *dr_infoj = loop_vinfo->lookup_dr (datarefs[j]);
2347 : 2902602 : for (unsigned k = i0; k < i; ++k)
2348 : : {
2349 : 2226499 : if (k == j)
2350 : 676103 : continue;
2351 : 1550396 : dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]);
2352 : 1550396 : if (vect_dr_aligned_if_related_peeled_dr_is (dr_infok,
2353 : : dr_infoj))
2354 : 512769 : n_same_align_refs[j]++;
2355 : : }
2356 : : }
2357 : : i0 = i;
2358 : : }
2359 : : }
2360 : :
2361 : : /* While cost model enhancements are expected in the future, the high level
2362 : : view of the code at this time is as follows:
2363 : :
2364 : : A) If there is a misaligned access then see if peeling to align
2365 : : this access can make all data references satisfy
2366 : : vect_supportable_dr_alignment. If so, update data structures
2367 : : as needed and return true.
2368 : :
2369 : : B) If peeling wasn't possible and there is a data reference with an
2370 : : unknown misalignment that does not satisfy vect_supportable_dr_alignment
2371 : : then see if loop versioning checks can be used to make all data
2372 : : references satisfy vect_supportable_dr_alignment. If so, update
2373 : : data structures as needed and return true.
2374 : :
2375 : : C) If neither peeling nor versioning were successful then return false if
2376 : : any data reference does not satisfy vect_supportable_dr_alignment.
2377 : :
2378 : : D) Return true (all data references satisfy vect_supportable_dr_alignment).
2379 : :
2380 : : Note, Possibility 3 above (which is peeling and versioning together) is not
2381 : : being done at this time. */
2382 : :
2383 : : /* (1) Peeling to force alignment. */
2384 : :
2385 : : /* (1.1) Decide whether to perform peeling, and how many iterations to peel:
2386 : : Considerations:
2387 : : + How many accesses will become aligned due to the peeling
2388 : : - How many accesses will become unaligned due to the peeling,
2389 : : and the cost of misaligned accesses.
2390 : : - The cost of peeling (the extra runtime checks, the increase
2391 : : in code size). */
2392 : :
2393 : 857810 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2394 : : {
2395 : 591161 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2396 : 591161 : if (!vect_relevant_for_alignment_p (dr_info))
2397 : 98761 : continue;
2398 : :
2399 : 492400 : stmt_vec_info stmt_info = dr_info->stmt;
2400 : 492400 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2401 : 492400 : do_peeling
2402 : 492400 : = vector_alignment_reachable_p (dr_info,
2403 : : LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2404 : 492400 : if (do_peeling)
2405 : : {
2406 : 437247 : if (known_alignment_for_access_p (dr_info, vectype))
2407 : : {
2408 : 253026 : unsigned int npeel_tmp = 0;
2409 : 253026 : bool negative = tree_int_cst_compare (DR_STEP (dr),
2410 : 253026 : size_zero_node) < 0;
2411 : :
2412 : : /* If known_alignment_for_access_p then we have set
2413 : : DR_MISALIGNMENT which is only done if we know it at compiler
2414 : : time, so it is safe to assume target alignment is constant.
2415 : : */
2416 : 253026 : unsigned int target_align =
2417 : 253026 : DR_TARGET_ALIGNMENT (dr_info).to_constant ();
2418 : 253026 : unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (dr_info);
2419 : 253026 : poly_int64 off = 0;
2420 : 253026 : if (negative)
2421 : 2156 : off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
2422 : 253026 : unsigned int mis = dr_misalignment (dr_info, vectype, off);
2423 : 253026 : mis = negative ? mis : -mis;
2424 : 253026 : if (mis != 0)
2425 : 11411 : npeel_tmp = (mis & (target_align - 1)) / dr_size;
2426 : :
2427 : : /* For multiple types, it is possible that the bigger type access
2428 : : will have more than one peeling option. E.g., a loop with two
2429 : : types: one of size (vector size / 4), and the other one of
2430 : : size (vector size / 8). Vectorization factor will 8. If both
2431 : : accesses are misaligned by 3, the first one needs one scalar
2432 : : iteration to be aligned, and the second one needs 5. But the
2433 : : first one will be aligned also by peeling 5 scalar
2434 : : iterations, and in that case both accesses will be aligned.
2435 : : Hence, except for the immediate peeling amount, we also want
2436 : : to try to add full vector size, while we don't exceed
2437 : : vectorization factor.
2438 : : We do this automatically for cost model, since we calculate
2439 : : cost for every peeling option. */
2440 : 253026 : poly_uint64 nscalars = npeel_tmp;
2441 : 253026 : if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
2442 : : {
2443 : 41486 : poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2444 : 41486 : unsigned group_size = 1;
2445 : 41486 : if (STMT_SLP_TYPE (stmt_info)
2446 : 41486 : && STMT_VINFO_GROUPED_ACCESS (stmt_info))
2447 : 1899 : group_size = DR_GROUP_SIZE (stmt_info);
2448 : 41486 : nscalars = vf * group_size;
2449 : : }
2450 : :
2451 : : /* Save info about DR in the hash table. Also include peeling
2452 : : amounts according to the explanation above. Indicate
2453 : : the alignment status when the ref is not aligned.
2454 : : ??? Rather than using unknown alignment here we should
2455 : : prune all entries from the peeling hashtable which cause
2456 : : DRs to be not supported. */
2457 : 253026 : bool supportable_if_not_aligned
2458 : : = vect_supportable_dr_alignment
2459 : 253026 : (loop_vinfo, dr_info, vectype, DR_MISALIGNMENT_UNKNOWN);
2460 : 569611 : while (known_le (npeel_tmp, nscalars))
2461 : : {
2462 : 316585 : vect_peeling_hash_insert (&peeling_htab, loop_vinfo,
2463 : : dr_info, npeel_tmp,
2464 : : supportable_if_not_aligned);
2465 : 316585 : npeel_tmp += MAX (1, target_align / dr_size);
2466 : : }
2467 : :
2468 : 253026 : one_misalignment_known = true;
2469 : : }
2470 : : else
2471 : : {
2472 : : /* If we don't know any misalignment values, we prefer
2473 : : peeling for data-ref that has the maximum number of data-refs
2474 : : with the same alignment, unless the target prefers to align
2475 : : stores over load. */
2476 : 184221 : unsigned same_align_drs = n_same_align_refs[i];
2477 : 184221 : if (!dr0_info
2478 : 184221 : || dr0_same_align_drs < same_align_drs)
2479 : : {
2480 : : dr0_same_align_drs = same_align_drs;
2481 : : dr0_info = dr_info;
2482 : : }
2483 : : /* For data-refs with the same number of related
2484 : : accesses prefer the one where the misalign
2485 : : computation will be invariant in the outermost loop. */
2486 : 58881 : else if (dr0_same_align_drs == same_align_drs)
2487 : : {
2488 : 57920 : class loop *ivloop0, *ivloop;
2489 : 57920 : ivloop0 = outermost_invariant_loop_for_expr
2490 : 57920 : (loop, DR_BASE_ADDRESS (dr0_info->dr));
2491 : 57920 : ivloop = outermost_invariant_loop_for_expr
2492 : 57920 : (loop, DR_BASE_ADDRESS (dr));
2493 : 57920 : if ((ivloop && !ivloop0)
2494 : 57920 : || (ivloop && ivloop0
2495 : 57914 : && flow_loop_nested_p (ivloop, ivloop0)))
2496 : : dr0_info = dr_info;
2497 : : }
2498 : :
2499 : 184221 : one_misalignment_unknown = true;
2500 : :
2501 : : /* Check for data refs with unsupportable alignment that
2502 : : can be peeled. */
2503 : 184221 : enum dr_alignment_support supportable_dr_alignment
2504 : 184221 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2505 : : DR_MISALIGNMENT_UNKNOWN);
2506 : 184221 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2507 : : {
2508 : 76536 : one_dr_unsupportable = true;
2509 : 76536 : unsupportable_dr_info = dr_info;
2510 : : }
2511 : :
2512 : 184221 : if (!first_store && DR_IS_WRITE (dr))
2513 : : {
2514 : 45165 : first_store = dr_info;
2515 : 45165 : first_store_same_align_drs = same_align_drs;
2516 : : }
2517 : : }
2518 : : }
2519 : : else
2520 : : {
2521 : 55153 : if (!aligned_access_p (dr_info, vectype))
2522 : : {
2523 : 38944 : if (dump_enabled_p ())
2524 : 1963 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
2525 : : "vector alignment may not be reachable\n");
2526 : : break;
2527 : : }
2528 : : }
2529 : : }
2530 : :
2531 : : /* Check if we can possibly peel the loop. */
2532 : 305593 : if (!vect_can_advance_ivs_p (loop_vinfo)
2533 : 297222 : || !slpeel_can_duplicate_loop_p (loop, LOOP_VINFO_IV_EXIT (loop_vinfo),
2534 : 297222 : loop_preheader_edge (loop))
2535 : 297222 : || loop->inner
2536 : : /* We don't currently maintaing the LCSSA for prologue peeled inversed
2537 : : loops. */
2538 : 601195 : || LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
2539 : : do_peeling = false;
2540 : :
2541 : 305593 : struct _vect_peel_extended_info peel_for_known_alignment;
2542 : 305593 : struct _vect_peel_extended_info peel_for_unknown_alignment;
2543 : 305593 : struct _vect_peel_extended_info best_peel;
2544 : :
2545 : 305593 : peel_for_unknown_alignment.inside_cost = INT_MAX;
2546 : 305593 : peel_for_unknown_alignment.outside_cost = INT_MAX;
2547 : 305593 : peel_for_unknown_alignment.peel_info.count = 0;
2548 : :
2549 : 305593 : if (do_peeling
2550 : 305593 : && one_misalignment_unknown)
2551 : : {
2552 : : /* Check if the target requires to prefer stores over loads, i.e., if
2553 : : misaligned stores are more expensive than misaligned loads (taking
2554 : : drs with same alignment into account). */
2555 : 107649 : unsigned int load_inside_cost = 0;
2556 : 107649 : unsigned int load_outside_cost = 0;
2557 : 107649 : unsigned int store_inside_cost = 0;
2558 : 107649 : unsigned int store_outside_cost = 0;
2559 : 107649 : unsigned int estimated_npeels = vect_vf_for_cost (loop_vinfo) / 2;
2560 : :
2561 : 107649 : stmt_vector_for_cost dummy;
2562 : 107649 : dummy.create (2);
2563 : 107649 : vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info,
2564 : : &load_inside_cost,
2565 : : &load_outside_cost,
2566 : : &dummy, &dummy, estimated_npeels);
2567 : 107649 : dummy.release ();
2568 : :
2569 : 107649 : if (first_store)
2570 : : {
2571 : 35028 : dummy.create (2);
2572 : 35028 : vect_get_peeling_costs_all_drs (loop_vinfo, first_store,
2573 : : &store_inside_cost,
2574 : : &store_outside_cost,
2575 : : &dummy, &dummy,
2576 : : estimated_npeels);
2577 : 35028 : dummy.release ();
2578 : : }
2579 : : else
2580 : : {
2581 : 72621 : store_inside_cost = INT_MAX;
2582 : 72621 : store_outside_cost = INT_MAX;
2583 : : }
2584 : :
2585 : 107649 : if (load_inside_cost > store_inside_cost
2586 : 107649 : || (load_inside_cost == store_inside_cost
2587 : 34841 : && load_outside_cost > store_outside_cost))
2588 : : {
2589 : 107649 : dr0_info = first_store;
2590 : 107649 : dr0_same_align_drs = first_store_same_align_drs;
2591 : 107649 : peel_for_unknown_alignment.inside_cost = store_inside_cost;
2592 : 107649 : peel_for_unknown_alignment.outside_cost = store_outside_cost;
2593 : : }
2594 : : else
2595 : : {
2596 : 107649 : peel_for_unknown_alignment.inside_cost = load_inside_cost;
2597 : 107649 : peel_for_unknown_alignment.outside_cost = load_outside_cost;
2598 : : }
2599 : :
2600 : 107649 : stmt_vector_for_cost prologue_cost_vec, epilogue_cost_vec;
2601 : 107649 : prologue_cost_vec.create (2);
2602 : 107649 : epilogue_cost_vec.create (2);
2603 : :
2604 : 107649 : int dummy2;
2605 : 215298 : peel_for_unknown_alignment.outside_cost += vect_get_known_peeling_cost
2606 : 107649 : (loop_vinfo, estimated_npeels, &dummy2,
2607 : : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
2608 : : &prologue_cost_vec, &epilogue_cost_vec);
2609 : :
2610 : 107649 : prologue_cost_vec.release ();
2611 : 107649 : epilogue_cost_vec.release ();
2612 : :
2613 : 107649 : peel_for_unknown_alignment.peel_info.count = dr0_same_align_drs + 1;
2614 : : }
2615 : :
2616 : 305593 : peel_for_unknown_alignment.peel_info.npeel = 0;
2617 : 305593 : peel_for_unknown_alignment.peel_info.dr_info = dr0_info;
2618 : :
2619 : 305593 : best_peel = peel_for_unknown_alignment;
2620 : :
2621 : 305593 : peel_for_known_alignment.inside_cost = INT_MAX;
2622 : 305593 : peel_for_known_alignment.outside_cost = INT_MAX;
2623 : 305593 : peel_for_known_alignment.peel_info.count = 0;
2624 : 305593 : peel_for_known_alignment.peel_info.dr_info = NULL;
2625 : :
2626 : 305593 : if (do_peeling && one_misalignment_known)
2627 : : {
2628 : : /* Peeling is possible, but there is no data access that is not supported
2629 : : unless aligned. So we try to choose the best possible peeling from
2630 : : the hash table. */
2631 : 134928 : peel_for_known_alignment = vect_peeling_hash_choose_best_peeling
2632 : 134928 : (&peeling_htab, loop_vinfo);
2633 : : }
2634 : :
2635 : : /* Compare costs of peeling for known and unknown alignment. */
2636 : 305593 : if (peel_for_known_alignment.peel_info.dr_info != NULL
2637 : 134928 : && peel_for_unknown_alignment.inside_cost
2638 : : >= peel_for_known_alignment.inside_cost)
2639 : : {
2640 : 122217 : best_peel = peel_for_known_alignment;
2641 : :
2642 : : /* If the best peeling for known alignment has NPEEL == 0, perform no
2643 : : peeling at all except if there is an unsupportable dr that we can
2644 : : align. */
2645 : 122217 : if (best_peel.peel_info.npeel == 0 && !one_dr_unsupportable)
2646 : : do_peeling = false;
2647 : : }
2648 : :
2649 : : /* If there is an unsupportable data ref, prefer this over all choices so far
2650 : : since we'd have to discard a chosen peeling except when it accidentally
2651 : : aligned the unsupportable data ref. */
2652 : 189725 : if (one_dr_unsupportable)
2653 : : dr0_info = unsupportable_dr_info;
2654 : 244019 : else if (do_peeling)
2655 : : {
2656 : : /* Calculate the penalty for no peeling, i.e. leaving everything as-is.
2657 : : TODO: Use nopeel_outside_cost or get rid of it? */
2658 : 51283 : unsigned nopeel_inside_cost = 0;
2659 : 51283 : unsigned nopeel_outside_cost = 0;
2660 : :
2661 : 51283 : stmt_vector_for_cost dummy;
2662 : 51283 : dummy.create (2);
2663 : 51283 : vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
2664 : : &nopeel_outside_cost, &dummy, &dummy, 0);
2665 : 51283 : dummy.release ();
2666 : :
2667 : : /* Add epilogue costs. As we do not peel for alignment here, no prologue
2668 : : costs will be recorded. */
2669 : 51283 : stmt_vector_for_cost prologue_cost_vec, epilogue_cost_vec;
2670 : 51283 : prologue_cost_vec.create (2);
2671 : 51283 : epilogue_cost_vec.create (2);
2672 : :
2673 : 51283 : int dummy2;
2674 : 102566 : nopeel_outside_cost += vect_get_known_peeling_cost
2675 : 51283 : (loop_vinfo, 0, &dummy2,
2676 : : &LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo),
2677 : : &prologue_cost_vec, &epilogue_cost_vec);
2678 : :
2679 : 51283 : prologue_cost_vec.release ();
2680 : 51283 : epilogue_cost_vec.release ();
2681 : :
2682 : 51283 : npeel = best_peel.peel_info.npeel;
2683 : 51283 : dr0_info = best_peel.peel_info.dr_info;
2684 : :
2685 : : /* If no peeling is not more expensive than the best peeling we
2686 : : have so far, don't perform any peeling. */
2687 : 51283 : if (nopeel_inside_cost <= best_peel.inside_cost)
2688 : 45906 : do_peeling = false;
2689 : : }
2690 : :
2691 : 112857 : if (do_peeling)
2692 : : {
2693 : 60334 : stmt_vec_info stmt_info = dr0_info->stmt;
2694 : 60334 : if (known_alignment_for_access_p (dr0_info,
2695 : : STMT_VINFO_VECTYPE (stmt_info)))
2696 : : {
2697 : 5356 : bool negative = tree_int_cst_compare (DR_STEP (dr0_info->dr),
2698 : 5356 : size_zero_node) < 0;
2699 : 5356 : if (!npeel)
2700 : : {
2701 : : /* Since it's known at compile time, compute the number of
2702 : : iterations in the peeled loop (the peeling factor) for use in
2703 : : updating DR_MISALIGNMENT values. The peeling factor is the
2704 : : vectorization factor minus the misalignment as an element
2705 : : count. */
2706 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2707 : 0 : poly_int64 off = 0;
2708 : 0 : if (negative)
2709 : 0 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
2710 : 0 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
2711 : 0 : unsigned int mis
2712 : 0 : = dr_misalignment (dr0_info, vectype, off);
2713 : 0 : mis = negative ? mis : -mis;
2714 : : /* If known_alignment_for_access_p then we have set
2715 : : DR_MISALIGNMENT which is only done if we know it at compiler
2716 : : time, so it is safe to assume target alignment is constant.
2717 : : */
2718 : 0 : unsigned int target_align =
2719 : 0 : DR_TARGET_ALIGNMENT (dr0_info).to_constant ();
2720 : 0 : npeel = ((mis & (target_align - 1))
2721 : 0 : / vect_get_scalar_dr_size (dr0_info));
2722 : : }
2723 : :
2724 : : /* For interleaved data access every iteration accesses all the
2725 : : members of the group, therefore we divide the number of iterations
2726 : : by the group size. */
2727 : 5356 : if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2728 : 223 : npeel /= DR_GROUP_SIZE (stmt_info);
2729 : :
2730 : 5356 : if (dump_enabled_p ())
2731 : 264 : dump_printf_loc (MSG_NOTE, vect_location,
2732 : : "Try peeling by %d\n", npeel);
2733 : : }
2734 : :
2735 : : /* Ensure that all datarefs can be vectorized after the peel. */
2736 : 60334 : if (!vect_peeling_supportable (loop_vinfo, dr0_info, npeel))
2737 : : do_peeling = false;
2738 : :
2739 : : /* Check if all datarefs are supportable and log. */
2740 : 60334 : if (do_peeling
2741 : 60334 : && npeel == 0
2742 : 60334 : && known_alignment_for_access_p (dr0_info,
2743 : : STMT_VINFO_VECTYPE (stmt_info)))
2744 : 3 : return opt_result::success ();
2745 : :
2746 : : /* Cost model #1 - honor --param vect-max-peeling-for-alignment. */
2747 : 60331 : if (do_peeling)
2748 : : {
2749 : 33261 : unsigned max_allowed_peel
2750 : 33261 : = param_vect_max_peeling_for_alignment;
2751 : 33261 : if (loop_cost_model (loop) <= VECT_COST_MODEL_CHEAP)
2752 : : max_allowed_peel = 0;
2753 : 6545 : if (max_allowed_peel != (unsigned)-1)
2754 : : {
2755 : 26719 : unsigned max_peel = npeel;
2756 : 26719 : if (max_peel == 0)
2757 : : {
2758 : 24138 : poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr0_info);
2759 : 24138 : unsigned HOST_WIDE_INT target_align_c;
2760 : 24138 : if (target_align.is_constant (&target_align_c))
2761 : 48276 : max_peel =
2762 : 24138 : target_align_c / vect_get_scalar_dr_size (dr0_info) - 1;
2763 : : else
2764 : : {
2765 : : do_peeling = false;
2766 : : if (dump_enabled_p ())
2767 : : dump_printf_loc (MSG_NOTE, vect_location,
2768 : : "Disable peeling, max peels set and vector"
2769 : : " alignment unknown\n");
2770 : : }
2771 : : }
2772 : 26719 : if (max_peel > max_allowed_peel)
2773 : : {
2774 : 26719 : do_peeling = false;
2775 : 26719 : if (dump_enabled_p ())
2776 : 51 : dump_printf_loc (MSG_NOTE, vect_location,
2777 : : "Disable peeling, max peels reached: %d\n", max_peel);
2778 : : }
2779 : : }
2780 : : }
2781 : :
2782 : : /* Cost model #2 - if peeling may result in a remaining loop not
2783 : : iterating enough to be vectorized then do not peel. Since this
2784 : : is a cost heuristic rather than a correctness decision, use the
2785 : : most likely runtime value for variable vectorization factors. */
2786 : 51 : if (do_peeling
2787 : 6542 : && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
2788 : : {
2789 : 2526 : unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
2790 : 2526 : unsigned int max_peel = npeel == 0 ? assumed_vf - 1 : npeel;
2791 : 2526 : if ((unsigned HOST_WIDE_INT) LOOP_VINFO_INT_NITERS (loop_vinfo)
2792 : 2526 : < assumed_vf + max_peel)
2793 : : do_peeling = false;
2794 : : }
2795 : :
2796 : : if (do_peeling)
2797 : : {
2798 : : /* (1.2) Update the DR_MISALIGNMENT of each data reference DR_i.
2799 : : If the misalignment of DR_i is identical to that of dr0 then set
2800 : : DR_MISALIGNMENT (DR_i) to zero. If the misalignment of DR_i and
2801 : : dr0 are known at compile time then increment DR_MISALIGNMENT (DR_i)
2802 : : by the peeling factor times the element size of DR_i (MOD the
2803 : : vectorization factor times the size). Otherwise, the
2804 : : misalignment of DR_i must be set to unknown. */
2805 : 13308 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2806 : 7242 : if (dr != dr0_info->dr)
2807 : : {
2808 : 1176 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2809 : 1176 : if (!vect_relevant_for_alignment_p (dr_info))
2810 : 312 : continue;
2811 : :
2812 : 864 : vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
2813 : : }
2814 : :
2815 : 6066 : LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
2816 : 6066 : if (npeel)
2817 : 1173 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = npeel;
2818 : : else
2819 : 4893 : LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = -1;
2820 : 6066 : SET_DR_MISALIGNMENT (dr0_info,
2821 : : vect_dr_misalign_for_aligned_access (dr0_info));
2822 : 6066 : if (dump_enabled_p ())
2823 : : {
2824 : 286 : dump_printf_loc (MSG_NOTE, vect_location,
2825 : : "Alignment of access forced using peeling.\n");
2826 : 286 : dump_printf_loc (MSG_NOTE, vect_location,
2827 : : "Peeling for alignment will be applied.\n");
2828 : : }
2829 : :
2830 : : /* The inside-loop cost will be accounted for in vectorizable_load
2831 : : and vectorizable_store correctly with adjusted alignments.
2832 : : Drop the body_cst_vec on the floor here. */
2833 : 6066 : return opt_result::success ();
2834 : : }
2835 : : }
2836 : :
2837 : : /* (2) Versioning to force alignment. */
2838 : :
2839 : : /* Try versioning if:
2840 : : 1) optimize loop for speed and the cost-model is not cheap
2841 : : 2) there is at least one unsupported misaligned data ref with an unknown
2842 : : misalignment, and
2843 : : 3) all misaligned data refs with a known misalignment are supported, and
2844 : : 4) the number of runtime alignment checks is within reason. */
2845 : :
2846 : 299524 : do_versioning
2847 : 299524 : = (optimize_loop_nest_for_speed_p (loop)
2848 : 299081 : && !loop->inner /* FORNOW */
2849 : 596985 : && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP);
2850 : :
2851 : : if (do_versioning)
2852 : : {
2853 : 300234 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2854 : : {
2855 : 227406 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2856 : 227406 : if (!vect_relevant_for_alignment_p (dr_info))
2857 : 166360 : continue;
2858 : :
2859 : 157356 : stmt_vec_info stmt_info = dr_info->stmt;
2860 : 157356 : if (STMT_VINFO_STRIDED_P (stmt_info))
2861 : : {
2862 : : do_versioning = false;
2863 : 5281 : break;
2864 : : }
2865 : :
2866 : 156519 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
2867 : 156519 : bool negative = tree_int_cst_compare (DR_STEP (dr),
2868 : 156519 : size_zero_node) < 0;
2869 : 156519 : poly_int64 off = 0;
2870 : 156519 : if (negative)
2871 : 2844 : off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
2872 : 2844 : * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
2873 : 156519 : int misalignment;
2874 : 156519 : if ((misalignment = dr_misalignment (dr_info, vectype, off)) == 0)
2875 : 96310 : continue;
2876 : :
2877 : 60209 : enum dr_alignment_support supportable_dr_alignment
2878 : 60209 : = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
2879 : : misalignment);
2880 : 60209 : if (supportable_dr_alignment == dr_unaligned_unsupported)
2881 : : {
2882 : 17709 : if (misalignment != DR_MISALIGNMENT_UNKNOWN
2883 : 17709 : || (LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
2884 : 13819 : >= (unsigned) param_vect_max_version_for_alignment_checks))
2885 : : {
2886 : : do_versioning = false;
2887 : 5281 : break;
2888 : : }
2889 : :
2890 : : /* At present we don't support versioning for alignment
2891 : : with variable VF, since there's no guarantee that the
2892 : : VF is a power of two. We could relax this if we added
2893 : : a way of enforcing a power-of-two size. */
2894 : 13265 : unsigned HOST_WIDE_INT size;
2895 : 26530 : if (!GET_MODE_SIZE (TYPE_MODE (vectype)).is_constant (&size))
2896 : : {
2897 : : do_versioning = false;
2898 : : break;
2899 : : }
2900 : :
2901 : : /* Forcing alignment in the first iteration is no good if
2902 : : we don't keep it across iterations. For now, just disable
2903 : : versioning in this case.
2904 : : ?? We could actually unroll the loop to achieve the required
2905 : : overall step alignment, and forcing the alignment could be
2906 : : done by doing some iterations of the non-vectorized loop. */
2907 : 13265 : if (!multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
2908 : 13265 : * DR_STEP_ALIGNMENT (dr),
2909 : 13265 : DR_TARGET_ALIGNMENT (dr_info)))
2910 : : {
2911 : : do_versioning = false;
2912 : : break;
2913 : : }
2914 : :
2915 : : /* The rightmost bits of an aligned address must be zeros.
2916 : : Construct the mask needed for this test. For example,
2917 : : GET_MODE_SIZE for the vector mode V4SI is 16 bytes so the
2918 : : mask must be 15 = 0xf. */
2919 : 13265 : int mask = size - 1;
2920 : :
2921 : : /* FORNOW: use the same mask to test all potentially unaligned
2922 : : references in the loop. */
2923 : 13265 : if (LOOP_VINFO_PTR_MASK (loop_vinfo)
2924 : 10023 : && LOOP_VINFO_PTR_MASK (loop_vinfo) != mask)
2925 : : {
2926 : : do_versioning = false;
2927 : : break;
2928 : : }
2929 : :
2930 : 13265 : LOOP_VINFO_PTR_MASK (loop_vinfo) = mask;
2931 : 13265 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).safe_push (stmt_info);
2932 : : }
2933 : : }
2934 : :
2935 : : /* Versioning requires at least one misaligned data reference. */
2936 : 78109 : if (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
2937 : : do_versioning = false;
2938 : 6455 : else if (!do_versioning)
2939 : 582 : LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
2940 : : }
2941 : :
2942 : 582 : if (do_versioning)
2943 : : {
2944 : : const vec<stmt_vec_info> &may_misalign_stmts
2945 : : = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
2946 : : stmt_vec_info stmt_info;
2947 : :
2948 : : /* It can now be assumed that the data references in the statements
2949 : : in LOOP_VINFO_MAY_MISALIGN_STMTS will be aligned in the version
2950 : : of the loop being vectorized. */
2951 : 15734 : FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
2952 : : {
2953 : 9861 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
2954 : 9861 : SET_DR_MISALIGNMENT (dr_info,
2955 : : vect_dr_misalign_for_aligned_access (dr_info));
2956 : 9861 : if (dump_enabled_p ())
2957 : 145 : dump_printf_loc (MSG_NOTE, vect_location,
2958 : : "Alignment of access forced using versioning.\n");
2959 : : }
2960 : :
2961 : 5873 : if (dump_enabled_p ())
2962 : 86 : dump_printf_loc (MSG_NOTE, vect_location,
2963 : : "Versioning for alignment will be applied.\n");
2964 : :
2965 : : /* Peeling and versioning can't be done together at this time. */
2966 : 5873 : gcc_assert (! (do_peeling && do_versioning));
2967 : :
2968 : 5873 : return opt_result::success ();
2969 : : }
2970 : :
2971 : : /* This point is reached if neither peeling nor versioning is being done. */
2972 : 293651 : gcc_assert (! (do_peeling || do_versioning));
2973 : :
2974 : 293651 : return opt_result::success ();
2975 : 624463 : }
2976 : :
2977 : :
2978 : : /* Function vect_analyze_data_refs_alignment
2979 : :
2980 : : Analyze the alignment of the data-references in the loop.
2981 : : Return FALSE if a data reference is found that cannot be vectorized. */
2982 : :
2983 : : opt_result
2984 : 360053 : vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
2985 : : {
2986 : 360053 : DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment");
2987 : :
2988 : 360053 : vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
2989 : 360053 : struct data_reference *dr;
2990 : 360053 : unsigned int i;
2991 : :
2992 : 360053 : vect_record_base_alignments (loop_vinfo);
2993 : 1179378 : FOR_EACH_VEC_ELT (datarefs, i, dr)
2994 : : {
2995 : 819325 : dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
2996 : 819325 : if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
2997 : : {
2998 : 819325 : if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)
2999 : 1051534 : && DR_GROUP_FIRST_ELEMENT (dr_info->stmt) != dr_info->stmt)
3000 : 109924 : continue;
3001 : :
3002 : 709401 : vect_compute_data_ref_alignment (loop_vinfo, dr_info,
3003 : : STMT_VINFO_VECTYPE (dr_info->stmt));
3004 : : }
3005 : : }
3006 : :
3007 : 360053 : return opt_result::success ();
3008 : : }
3009 : :
3010 : :
3011 : : /* Analyze alignment of DRs of stmts in NODE. */
3012 : :
3013 : : static bool
3014 : 903843 : vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
3015 : : {
3016 : : /* Alignment is maintained in the first element of the group. */
3017 : 903843 : stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
3018 : 903843 : first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
3019 : 903843 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
3020 : 903843 : tree vectype = SLP_TREE_VECTYPE (node);
3021 : 903843 : poly_uint64 vector_alignment
3022 : 903843 : = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
3023 : : BITS_PER_UNIT);
3024 : 903843 : if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
3025 : 865252 : vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
3026 : : /* Re-analyze alignment when we're facing a vectorization with a bigger
3027 : : alignment requirement. */
3028 : 38591 : else if (known_lt (dr_info->target_alignment, vector_alignment))
3029 : : {
3030 : 83 : poly_uint64 old_target_alignment = dr_info->target_alignment;
3031 : 83 : int old_misalignment = dr_info->misalignment;
3032 : 83 : vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
3033 : : /* But keep knowledge about a smaller alignment. */
3034 : 83 : if (old_misalignment != DR_MISALIGNMENT_UNKNOWN
3035 : 50 : && dr_info->misalignment == DR_MISALIGNMENT_UNKNOWN)
3036 : : {
3037 : 1 : dr_info->target_alignment = old_target_alignment;
3038 : 1 : dr_info->misalignment = old_misalignment;
3039 : : }
3040 : : }
3041 : : /* When we ever face unordered target alignments the first one wins in terms
3042 : : of analyzing and the other will become unknown in dr_misalignment. */
3043 : 903843 : return true;
3044 : : }
3045 : :
3046 : : /* Function vect_slp_analyze_instance_alignment
3047 : :
3048 : : Analyze the alignment of the data-references in the SLP instance.
3049 : : Return FALSE if a data reference is found that cannot be vectorized. */
3050 : :
3051 : : bool
3052 : 875506 : vect_slp_analyze_instance_alignment (vec_info *vinfo,
3053 : : slp_instance instance)
3054 : : {
3055 : 875506 : DUMP_VECT_SCOPE ("vect_slp_analyze_instance_alignment");
3056 : :
3057 : 875506 : slp_tree node;
3058 : 875506 : unsigned i;
3059 : 1044324 : FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
3060 : 168818 : if (! vect_slp_analyze_node_alignment (vinfo, node))
3061 : : return false;
3062 : :
3063 : 875506 : if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
3064 : 875506 : && ! vect_slp_analyze_node_alignment
3065 : 735025 : (vinfo, SLP_INSTANCE_TREE (instance)))
3066 : : return false;
3067 : :
3068 : : return true;
3069 : : }
3070 : :
3071 : :
3072 : : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
3073 : : accesses of legal size, step, etc. Detect gaps, single element
3074 : : interleaving, and other special cases. Set grouped access info.
3075 : : Collect groups of strided stores for further use in SLP analysis.
3076 : : Worker for vect_analyze_group_access. */
3077 : :
3078 : : static bool
3079 : 12584002 : vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
3080 : : {
3081 : 12584002 : data_reference *dr = dr_info->dr;
3082 : 12584002 : tree step = DR_STEP (dr);
3083 : 12584002 : tree scalar_type = TREE_TYPE (DR_REF (dr));
3084 : 12584002 : HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
3085 : 12584002 : stmt_vec_info stmt_info = dr_info->stmt;
3086 : 12584002 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3087 : 12584002 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
3088 : 12584002 : HOST_WIDE_INT dr_step = -1;
3089 : 12584002 : HOST_WIDE_INT groupsize, last_accessed_element = 1;
3090 : 12584002 : bool slp_impossible = false;
3091 : :
3092 : : /* For interleaving, GROUPSIZE is STEP counted in elements, i.e., the
3093 : : size of the interleaving group (including gaps). */
3094 : 12584002 : if (tree_fits_shwi_p (step))
3095 : : {
3096 : 12575255 : dr_step = tree_to_shwi (step);
3097 : : /* Check that STEP is a multiple of type size. Otherwise there is
3098 : : a non-element-sized gap at the end of the group which we
3099 : : cannot represent in DR_GROUP_GAP or DR_GROUP_SIZE.
3100 : : ??? As we can handle non-constant step fine here we should
3101 : : simply remove uses of DR_GROUP_GAP between the last and first
3102 : : element and instead rely on DR_STEP. DR_GROUP_SIZE then would
3103 : : simply not include that gap. */
3104 : 12575255 : if ((dr_step % type_size) != 0)
3105 : : {
3106 : 71 : if (dump_enabled_p ())
3107 : 27 : dump_printf_loc (MSG_NOTE, vect_location,
3108 : : "Step %T is not a multiple of the element size"
3109 : : " for %T\n",
3110 : : step, DR_REF (dr));
3111 : 71 : return false;
3112 : : }
3113 : 12575184 : groupsize = absu_hwi (dr_step) / type_size;
3114 : : }
3115 : : else
3116 : : groupsize = 0;
3117 : :
3118 : : /* Not consecutive access is possible only if it is a part of interleaving. */
3119 : 12583931 : if (!DR_GROUP_FIRST_ELEMENT (stmt_info))
3120 : : {
3121 : : /* Check if it this DR is a part of interleaving, and is a single
3122 : : element of the group that is accessed in the loop. */
3123 : :
3124 : : /* Gaps are supported only for loads. STEP must be a multiple of the type
3125 : : size. */
3126 : 8483387 : if (DR_IS_READ (dr)
3127 : 5070922 : && (dr_step % type_size) == 0
3128 : : && groupsize > 0
3129 : : /* This could be UINT_MAX but as we are generating code in a very
3130 : : inefficient way we have to cap earlier.
3131 : : See PR91403 for example. */
3132 : 5070922 : && groupsize <= 4096)
3133 : : {
3134 : 55131 : DR_GROUP_FIRST_ELEMENT (stmt_info) = stmt_info;
3135 : 55131 : DR_GROUP_SIZE (stmt_info) = groupsize;
3136 : 55131 : DR_GROUP_GAP (stmt_info) = groupsize - 1;
3137 : 55131 : if (dump_enabled_p ())
3138 : 1327 : dump_printf_loc (MSG_NOTE, vect_location,
3139 : : "Detected single element interleaving %T"
3140 : : " step %T\n",
3141 : : DR_REF (dr), step);
3142 : :
3143 : 55131 : return true;
3144 : : }
3145 : :
3146 : 8428256 : if (dump_enabled_p ())
3147 : 3117 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3148 : : "not consecutive access %G", stmt_info->stmt);
3149 : :
3150 : 8428256 : if (bb_vinfo)
3151 : : {
3152 : : /* Mark the statement as unvectorizable. */
3153 : 8412390 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
3154 : 8412390 : return true;
3155 : : }
3156 : :
3157 : 15866 : if (dump_enabled_p ())
3158 : 334 : dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
3159 : 15866 : STMT_VINFO_STRIDED_P (stmt_info) = true;
3160 : 15866 : return true;
3161 : : }
3162 : :
3163 : 4100544 : if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
3164 : : {
3165 : : /* First stmt in the interleaving chain. Check the chain. */
3166 : 1493603 : stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3167 : 1493603 : struct data_reference *data_ref = dr;
3168 : 1493603 : unsigned int count = 1;
3169 : 1493603 : tree prev_init = DR_INIT (data_ref);
3170 : 1493603 : HOST_WIDE_INT diff, gaps = 0;
3171 : :
3172 : : /* By construction, all group members have INTEGER_CST DR_INITs. */
3173 : 4100553 : while (next)
3174 : : {
3175 : : /* We never have the same DR multiple times. */
3176 : 2607012 : gcc_assert (tree_int_cst_compare (DR_INIT (data_ref),
3177 : : DR_INIT (STMT_VINFO_DATA_REF (next))) != 0);
3178 : :
3179 : 2607012 : data_ref = STMT_VINFO_DATA_REF (next);
3180 : :
3181 : : /* All group members have the same STEP by construction. */
3182 : 2607012 : gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0));
3183 : :
3184 : : /* Check that the distance between two accesses is equal to the type
3185 : : size. Otherwise, we have gaps. */
3186 : 2607012 : diff = (TREE_INT_CST_LOW (DR_INIT (data_ref))
3187 : 2607012 : - TREE_INT_CST_LOW (prev_init)) / type_size;
3188 : 2607012 : if (diff < 1 || diff > UINT_MAX)
3189 : : {
3190 : : /* For artificial testcases with array accesses with large
3191 : : constant indices we can run into overflow issues which
3192 : : can end up fooling the groupsize constraint below so
3193 : : check the individual gaps (which are represented as
3194 : : unsigned int) as well. */
3195 : 0 : if (dump_enabled_p ())
3196 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3197 : : "interleaved access with gap larger "
3198 : : "than representable\n");
3199 : 0 : return false;
3200 : : }
3201 : 2607012 : if (diff != 1)
3202 : : {
3203 : : /* FORNOW: SLP of accesses with gaps is not supported. */
3204 : 102924 : slp_impossible = true;
3205 : 102924 : if (DR_IS_WRITE (data_ref))
3206 : : {
3207 : 62 : if (dump_enabled_p ())
3208 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3209 : : "interleaved store with gaps\n");
3210 : 62 : return false;
3211 : : }
3212 : :
3213 : 102862 : gaps += diff - 1;
3214 : : }
3215 : :
3216 : 2606950 : last_accessed_element += diff;
3217 : :
3218 : : /* Store the gap from the previous member of the group. If there is no
3219 : : gap in the access, DR_GROUP_GAP is always 1. */
3220 : 2606950 : DR_GROUP_GAP (next) = diff;
3221 : :
3222 : 2606950 : prev_init = DR_INIT (data_ref);
3223 : 2606950 : next = DR_GROUP_NEXT_ELEMENT (next);
3224 : : /* Count the number of data-refs in the chain. */
3225 : 2606950 : count++;
3226 : : }
3227 : :
3228 : 1493541 : if (groupsize == 0)
3229 : 1441162 : groupsize = count + gaps;
3230 : :
3231 : : /* This could be UINT_MAX but as we are generating code in a very
3232 : : inefficient way we have to cap earlier. See PR78699 for example. */
3233 : 1493541 : if (groupsize > 4096)
3234 : : {
3235 : 1 : if (dump_enabled_p ())
3236 : 1 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3237 : : "group is too large\n");
3238 : 1 : return false;
3239 : : }
3240 : :
3241 : : /* Check that the size of the interleaving is equal to count for stores,
3242 : : i.e., that there are no gaps. */
3243 : 1493540 : if (groupsize != count
3244 : 99288 : && !DR_IS_READ (dr))
3245 : : {
3246 : 4172 : groupsize = count;
3247 : 4172 : STMT_VINFO_STRIDED_P (stmt_info) = true;
3248 : : }
3249 : :
3250 : : /* If there is a gap after the last load in the group it is the
3251 : : difference between the groupsize and the last accessed
3252 : : element.
3253 : : When there is no gap, this difference should be 0. */
3254 : 1493540 : DR_GROUP_GAP (stmt_info) = groupsize - last_accessed_element;
3255 : :
3256 : 1493540 : DR_GROUP_SIZE (stmt_info) = groupsize;
3257 : 1493540 : if (dump_enabled_p ())
3258 : : {
3259 : 7998 : dump_printf_loc (MSG_NOTE, vect_location,
3260 : : "Detected interleaving ");
3261 : 7998 : if (DR_IS_READ (dr))
3262 : 4237 : dump_printf (MSG_NOTE, "load ");
3263 : 3761 : else if (STMT_VINFO_STRIDED_P (stmt_info))
3264 : 574 : dump_printf (MSG_NOTE, "strided store ");
3265 : : else
3266 : 3187 : dump_printf (MSG_NOTE, "store ");
3267 : 7998 : dump_printf (MSG_NOTE, "of size %u\n",
3268 : : (unsigned)groupsize);
3269 : 7998 : dump_printf_loc (MSG_NOTE, vect_location, "\t%G", stmt_info->stmt);
3270 : 7998 : next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3271 : 38419 : while (next)
3272 : : {
3273 : 30421 : if (DR_GROUP_GAP (next) != 1)
3274 : 286 : dump_printf_loc (MSG_NOTE, vect_location,
3275 : : "\t<gap of %d elements>\n",
3276 : 286 : DR_GROUP_GAP (next) - 1);
3277 : 30421 : dump_printf_loc (MSG_NOTE, vect_location, "\t%G", next->stmt);
3278 : 30421 : next = DR_GROUP_NEXT_ELEMENT (next);
3279 : : }
3280 : 7998 : if (DR_GROUP_GAP (stmt_info) != 0)
3281 : 372 : dump_printf_loc (MSG_NOTE, vect_location,
3282 : : "\t<gap of %d elements>\n",
3283 : 372 : DR_GROUP_GAP (stmt_info));
3284 : : }
3285 : :
3286 : : /* SLP: create an SLP data structure for every interleaving group of
3287 : : stores for further analysis in vect_analyse_slp. */
3288 : 1493540 : if (DR_IS_WRITE (dr) && !slp_impossible)
3289 : : {
3290 : 902930 : if (loop_vinfo)
3291 : 18268 : LOOP_VINFO_GROUPED_STORES (loop_vinfo).safe_push (stmt_info);
3292 : 902930 : if (bb_vinfo)
3293 : 884662 : BB_VINFO_GROUPED_STORES (bb_vinfo).safe_push (stmt_info);
3294 : : }
3295 : : }
3296 : :
3297 : : return true;
3298 : : }
3299 : :
3300 : : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
3301 : : accesses of legal size, step, etc. Detect gaps, single element
3302 : : interleaving, and other special cases. Set grouped access info.
3303 : : Collect groups of strided stores for further use in SLP analysis. */
3304 : :
3305 : : static bool
3306 : 12584002 : vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
3307 : : {
3308 : 12584002 : if (!vect_analyze_group_access_1 (vinfo, dr_info))
3309 : : {
3310 : : /* Dissolve the group if present. */
3311 : 134 : stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
3312 : 365 : while (stmt_info)
3313 : : {
3314 : 231 : stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
3315 : 231 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3316 : 231 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3317 : 231 : stmt_info = next;
3318 : : }
3319 : : return false;
3320 : : }
3321 : : return true;
3322 : : }
3323 : :
3324 : : /* Analyze the access pattern of the data-reference DR_INFO.
3325 : : In case of non-consecutive accesses call vect_analyze_group_access() to
3326 : : analyze groups of accesses. */
3327 : :
3328 : : static bool
3329 : 13113319 : vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
3330 : : {
3331 : 13113319 : data_reference *dr = dr_info->dr;
3332 : 13113319 : tree step = DR_STEP (dr);
3333 : 13113319 : tree scalar_type = TREE_TYPE (DR_REF (dr));
3334 : 13113319 : stmt_vec_info stmt_info = dr_info->stmt;
3335 : 13113319 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
3336 : 13113319 : class loop *loop = NULL;
3337 : :
3338 : 13113319 : if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
3339 : : return true;
3340 : :
3341 : 13080501 : if (loop_vinfo)
3342 : 738824 : loop = LOOP_VINFO_LOOP (loop_vinfo);
3343 : :
3344 : 13080501 : if (loop_vinfo && !step)
3345 : : {
3346 : 0 : if (dump_enabled_p ())
3347 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3348 : : "bad data-ref access in loop\n");
3349 : 0 : return false;
3350 : : }
3351 : :
3352 : : /* Allow loads with zero step in inner-loop vectorization. */
3353 : 13080501 : if (loop_vinfo && integer_zerop (step))
3354 : : {
3355 : 12891 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3356 : 12891 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3357 : 12891 : if (!nested_in_vect_loop_p (loop, stmt_info))
3358 : 12624 : return DR_IS_READ (dr);
3359 : : /* Allow references with zero step for outer loops marked
3360 : : with pragma omp simd only - it guarantees absence of
3361 : : loop-carried dependencies between inner loop iterations. */
3362 : 267 : if (loop->safelen < 2)
3363 : : {
3364 : 231 : if (dump_enabled_p ())
3365 : 5 : dump_printf_loc (MSG_NOTE, vect_location,
3366 : : "zero step in inner loop of nest\n");
3367 : 231 : return false;
3368 : : }
3369 : : }
3370 : :
3371 : 13067610 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
3372 : : {
3373 : : /* Interleaved accesses are not yet supported within outer-loop
3374 : : vectorization for references in the inner-loop. */
3375 : 5196 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3376 : 5196 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3377 : :
3378 : : /* For the rest of the analysis we use the outer-loop step. */
3379 : 5196 : step = STMT_VINFO_DR_STEP (stmt_info);
3380 : 5196 : if (integer_zerop (step))
3381 : : {
3382 : 1195 : if (dump_enabled_p ())
3383 : 228 : dump_printf_loc (MSG_NOTE, vect_location,
3384 : : "zero step in outer loop.\n");
3385 : 1195 : return DR_IS_READ (dr);
3386 : : }
3387 : : }
3388 : :
3389 : : /* Consecutive? */
3390 : 13066451 : if (TREE_CODE (step) == INTEGER_CST)
3391 : : {
3392 : 13029222 : HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
3393 : 13029222 : if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type))
3394 : 13029222 : || (dr_step < 0
3395 : 22958 : && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step)))
3396 : : {
3397 : : /* Mark that it is not interleaving. */
3398 : 451486 : DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
3399 : 451486 : DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
3400 : 451486 : return true;
3401 : : }
3402 : : }
3403 : :
3404 : 12614965 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
3405 : : {
3406 : 2892 : if (dump_enabled_p ())
3407 : 144 : dump_printf_loc (MSG_NOTE, vect_location,
3408 : : "grouped access in outer loop.\n");
3409 : 2892 : return false;
3410 : : }
3411 : :
3412 : :
3413 : : /* Assume this is a DR handled by non-constant strided load case. */
3414 : 12612073 : if (TREE_CODE (step) != INTEGER_CST)
3415 : 36818 : return (STMT_VINFO_STRIDED_P (stmt_info)
3416 : 36818 : && (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
3417 : 8747 : || vect_analyze_group_access (vinfo, dr_info)));
3418 : :
3419 : : /* Not consecutive access - check if it's a part of interleaving group. */
3420 : 12575255 : return vect_analyze_group_access (vinfo, dr_info);
3421 : : }
3422 : :
3423 : : /* Compare two data-references DRA and DRB to group them into chunks
3424 : : suitable for grouping. */
3425 : :
3426 : : static int
3427 : 338788643 : dr_group_sort_cmp (const void *dra_, const void *drb_)
3428 : : {
3429 : 338788643 : dr_vec_info *dra_info = *(dr_vec_info **)const_cast<void *>(dra_);
3430 : 338788643 : dr_vec_info *drb_info = *(dr_vec_info **)const_cast<void *>(drb_);
3431 : 338788643 : data_reference_p dra = dra_info->dr;
3432 : 338788643 : data_reference_p drb = drb_info->dr;
3433 : 338788643 : int cmp;
3434 : :
3435 : : /* Stabilize sort. */
3436 : 338788643 : if (dra == drb)
3437 : : return 0;
3438 : :
3439 : : /* Different group IDs lead never belong to the same group. */
3440 : 338788643 : if (dra_info->group != drb_info->group)
3441 : 369814551 : return dra_info->group < drb_info->group ? -1 : 1;
3442 : :
3443 : : /* Ordering of DRs according to base. */
3444 : 95543594 : cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
3445 : : DR_BASE_ADDRESS (drb));
3446 : 95543594 : if (cmp != 0)
3447 : : return cmp;
3448 : :
3449 : : /* And according to DR_OFFSET. */
3450 : 51268686 : cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
3451 : 51268686 : if (cmp != 0)
3452 : : return cmp;
3453 : :
3454 : : /* Put reads before writes. */
3455 : 50929656 : if (DR_IS_READ (dra) != DR_IS_READ (drb))
3456 : 4189053 : return DR_IS_READ (dra) ? -1 : 1;
3457 : :
3458 : : /* Then sort after access size. */
3459 : 48056514 : cmp = data_ref_compare_tree (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra))),
3460 : 48056514 : TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb))));
3461 : 48056514 : if (cmp != 0)
3462 : : return cmp;
3463 : :
3464 : : /* And after step. */
3465 : 41813218 : cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
3466 : 41813218 : if (cmp != 0)
3467 : : return cmp;
3468 : :
3469 : : /* Then sort after DR_INIT. In case of identical DRs sort after stmt UID. */
3470 : 41806738 : cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
3471 : 41806738 : if (cmp == 0)
3472 : 329432 : return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
3473 : : return cmp;
3474 : : }
3475 : :
3476 : : /* If OP is the result of a conversion, return the unconverted value,
3477 : : otherwise return null. */
3478 : :
3479 : : static tree
3480 : 180 : strip_conversion (tree op)
3481 : : {
3482 : 180 : if (TREE_CODE (op) != SSA_NAME)
3483 : : return NULL_TREE;
3484 : 180 : gimple *stmt = SSA_NAME_DEF_STMT (op);
3485 : 180 : if (!is_gimple_assign (stmt)
3486 : 180 : || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)))
3487 : : return NULL_TREE;
3488 : 68 : return gimple_assign_rhs1 (stmt);
3489 : : }
3490 : :
3491 : : /* Return true if vectorizable_* routines can handle statements STMT1_INFO
3492 : : and STMT2_INFO being in a single group. When ALLOW_SLP_P, masked loads can
3493 : : be grouped in SLP mode. */
3494 : :
3495 : : static bool
3496 : 6950532 : can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info,
3497 : : bool allow_slp_p)
3498 : : {
3499 : 6950532 : if (gimple_assign_single_p (stmt1_info->stmt))
3500 : 6950099 : return gimple_assign_single_p (stmt2_info->stmt);
3501 : :
3502 : 433 : gcall *call1 = dyn_cast <gcall *> (stmt1_info->stmt);
3503 : 433 : if (call1 && gimple_call_internal_p (call1))
3504 : : {
3505 : : /* Check for two masked loads or two masked stores. */
3506 : 575 : gcall *call2 = dyn_cast <gcall *> (stmt2_info->stmt);
3507 : 422 : if (!call2 || !gimple_call_internal_p (call2))
3508 : : return false;
3509 : 422 : internal_fn ifn = gimple_call_internal_fn (call1);
3510 : 422 : if (ifn != IFN_MASK_LOAD && ifn != IFN_MASK_STORE)
3511 : : return false;
3512 : 422 : if (ifn != gimple_call_internal_fn (call2))
3513 : : return false;
3514 : :
3515 : : /* Check that the masks are the same. Cope with casts of masks,
3516 : : like those created by build_mask_conversion. */
3517 : 422 : tree mask1 = gimple_call_arg (call1, 2);
3518 : 422 : tree mask2 = gimple_call_arg (call2, 2);
3519 : 422 : if (!operand_equal_p (mask1, mask2, 0) && !allow_slp_p)
3520 : : {
3521 : 146 : mask1 = strip_conversion (mask1);
3522 : 146 : if (!mask1)
3523 : : return false;
3524 : 34 : mask2 = strip_conversion (mask2);
3525 : 34 : if (!mask2)
3526 : : return false;
3527 : 34 : if (!operand_equal_p (mask1, mask2, 0))
3528 : : return false;
3529 : : }
3530 : 280 : return true;
3531 : : }
3532 : :
3533 : : return false;
3534 : : }
3535 : :
3536 : : /* Function vect_analyze_data_ref_accesses.
3537 : :
3538 : : Analyze the access pattern of all the data references in the loop.
3539 : :
3540 : : FORNOW: the only access pattern that is considered vectorizable is a
3541 : : simple step 1 (consecutive) access.
3542 : :
3543 : : FORNOW: handle only arrays and pointer accesses. */
3544 : :
3545 : : opt_result
3546 : 2754718 : vect_analyze_data_ref_accesses (vec_info *vinfo,
3547 : : vec<int> *dataref_groups)
3548 : : {
3549 : 2754718 : unsigned int i;
3550 : 2754718 : vec<data_reference_p> datarefs = vinfo->shared->datarefs;
3551 : :
3552 : 2754718 : DUMP_VECT_SCOPE ("vect_analyze_data_ref_accesses");
3553 : :
3554 : 2754718 : if (datarefs.is_empty ())
3555 : 1215533 : return opt_result::success ();
3556 : :
3557 : : /* Sort the array of datarefs to make building the interleaving chains
3558 : : linear. Don't modify the original vector's order, it is needed for
3559 : : determining what dependencies are reversed. */
3560 : 1539185 : vec<dr_vec_info *> datarefs_copy;
3561 : 1539185 : datarefs_copy.create (datarefs.length ());
3562 : 16363525 : for (unsigned i = 0; i < datarefs.length (); i++)
3563 : : {
3564 : 14824340 : dr_vec_info *dr_info = vinfo->lookup_dr (datarefs[i]);
3565 : : /* If the caller computed DR grouping use that, otherwise group by
3566 : : basic blocks. */
3567 : 14824340 : if (dataref_groups)
3568 : 14040837 : dr_info->group = (*dataref_groups)[i];
3569 : : else
3570 : 783503 : dr_info->group = gimple_bb (DR_STMT (datarefs[i]))->index;
3571 : 14824340 : datarefs_copy.quick_push (dr_info);
3572 : : }
3573 : 1539185 : datarefs_copy.qsort (dr_group_sort_cmp);
3574 : 1539185 : hash_set<stmt_vec_info> to_fixup;
3575 : :
3576 : : /* Build the interleaving chains. */
3577 : 13974585 : for (i = 0; i < datarefs_copy.length () - 1;)
3578 : : {
3579 : 10896215 : dr_vec_info *dr_info_a = datarefs_copy[i];
3580 : 10896215 : data_reference_p dra = dr_info_a->dr;
3581 : 10896215 : int dra_group_id = dr_info_a->group;
3582 : 10896215 : stmt_vec_info stmtinfo_a = dr_info_a->stmt;
3583 : 10896215 : stmt_vec_info lastinfo = NULL;
3584 : 10896215 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
3585 : 9332400 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a))
3586 : : {
3587 : 1590298 : ++i;
3588 : 1590298 : continue;
3589 : : }
3590 : 24381173 : for (i = i + 1; i < datarefs_copy.length (); ++i)
3591 : : {
3592 : 11694857 : dr_vec_info *dr_info_b = datarefs_copy[i];
3593 : 11694857 : data_reference_p drb = dr_info_b->dr;
3594 : 11694857 : int drb_group_id = dr_info_b->group;
3595 : 11694857 : stmt_vec_info stmtinfo_b = dr_info_b->stmt;
3596 : 11694857 : if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b)
3597 : 11389109 : || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
3598 : : break;
3599 : :
3600 : : /* ??? Imperfect sorting (non-compatible types, non-modulo
3601 : : accesses, same accesses) can lead to a group to be artificially
3602 : : split here as we don't just skip over those. If it really
3603 : : matters we can push those to a worklist and re-iterate
3604 : : over them. The we can just skip ahead to the next DR here. */
3605 : :
3606 : : /* DRs in a different DR group should not be put into the same
3607 : : interleaving group. */
3608 : 11386157 : if (dra_group_id != drb_group_id)
3609 : : break;
3610 : :
3611 : : /* Check that the data-refs have same first location (except init)
3612 : : and they are both either store or load (not load and store,
3613 : : not masked loads or stores). */
3614 : 7197139 : if (DR_IS_READ (dra) != DR_IS_READ (drb)
3615 : 5915443 : || data_ref_compare_tree (DR_BASE_ADDRESS (dra),
3616 : : DR_BASE_ADDRESS (drb)) != 0
3617 : 4331065 : || data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)) != 0
3618 : 11507845 : || !can_group_stmts_p (stmtinfo_a, stmtinfo_b, true))
3619 : : break;
3620 : :
3621 : : /* Check that the data-refs have the same constant size. */
3622 : 4310689 : tree sza = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra)));
3623 : 4310689 : tree szb = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb)));
3624 : 4310689 : if (!tree_fits_uhwi_p (sza)
3625 : 4310689 : || !tree_fits_uhwi_p (szb)
3626 : 8621378 : || !tree_int_cst_equal (sza, szb))
3627 : : break;
3628 : :
3629 : : /* Check that the data-refs have the same step. */
3630 : 3971839 : if (data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb)) != 0)
3631 : : break;
3632 : :
3633 : : /* Check the types are compatible.
3634 : : ??? We don't distinguish this during sorting. */
3635 : 3971185 : if (!types_compatible_p (TREE_TYPE (DR_REF (dra)),
3636 : 3971185 : TREE_TYPE (DR_REF (drb))))
3637 : : break;
3638 : :
3639 : : /* Check that the DR_INITs are compile-time constants. */
3640 : 2836837 : if (!tree_fits_shwi_p (DR_INIT (dra))
3641 : 2836837 : || !tree_fits_shwi_p (DR_INIT (drb)))
3642 : : break;
3643 : :
3644 : : /* Different .GOMP_SIMD_LANE calls still give the same lane,
3645 : : just hold extra information. */
3646 : 2836837 : if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
3647 : 1240 : && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
3648 : 2838077 : && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
3649 : : break;
3650 : :
3651 : : /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb). */
3652 : 2835597 : HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
3653 : 2835597 : HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
3654 : 2835597 : HOST_WIDE_INT init_prev
3655 : 2835597 : = TREE_INT_CST_LOW (DR_INIT (datarefs_copy[i-1]->dr));
3656 : 2835597 : gcc_assert (init_a <= init_b
3657 : : && init_a <= init_prev
3658 : : && init_prev <= init_b);
3659 : :
3660 : : /* Do not place the same access in the interleaving chain twice. */
3661 : 2835597 : if (init_b == init_prev)
3662 : : {
3663 : 29195 : gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1]->dr))
3664 : : < gimple_uid (DR_STMT (drb)));
3665 : : /* Simply link in duplicates and fix up the chain below. */
3666 : : }
3667 : : else
3668 : : {
3669 : : /* If init_b == init_a + the size of the type * k, we have an
3670 : : interleaving, and DRA is accessed before DRB. */
3671 : 2806402 : unsigned HOST_WIDE_INT type_size_a = tree_to_uhwi (sza);
3672 : 2806402 : if (type_size_a == 0
3673 : 2806402 : || (((unsigned HOST_WIDE_INT)init_b - init_a)
3674 : 2806402 : % type_size_a != 0))
3675 : : break;
3676 : :
3677 : : /* If we have a store, the accesses are adjacent. This splits
3678 : : groups into chunks we support (we don't support vectorization
3679 : : of stores with gaps). */
3680 : 2805339 : if (!DR_IS_READ (dra)
3681 : 1830663 : && (((unsigned HOST_WIDE_INT)init_b - init_prev)
3682 : : != type_size_a))
3683 : : break;
3684 : :
3685 : : /* For datarefs with big gap, it's better to split them into different
3686 : : groups.
3687 : : .i.e a[0], a[1], a[2], .. a[7], a[100], a[101],..., a[107] */
3688 : 2633053 : if ((unsigned HOST_WIDE_INT)(init_b - init_prev)
3689 : : > MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT)
3690 : : break;
3691 : :
3692 : : /* If the step (if not zero or non-constant) is smaller than the
3693 : : difference between data-refs' inits this splits groups into
3694 : : suitable sizes. */
3695 : 2625026 : if (tree_fits_shwi_p (DR_STEP (dra)))
3696 : : {
3697 : 2619559 : unsigned HOST_WIDE_INT step
3698 : 2619559 : = absu_hwi (tree_to_shwi (DR_STEP (dra)));
3699 : 2619559 : if (step != 0
3700 : 127653 : && step <= ((unsigned HOST_WIDE_INT)init_b - init_a))
3701 : : break;
3702 : : }
3703 : : }
3704 : :
3705 : 2639856 : if (dump_enabled_p ())
3706 : 31204 : dump_printf_loc (MSG_NOTE, vect_location,
3707 : 31204 : DR_IS_READ (dra)
3708 : : ? "Detected interleaving load %T and %T\n"
3709 : : : "Detected interleaving store %T and %T\n",
3710 : : DR_REF (dra), DR_REF (drb));
3711 : :
3712 : : /* Link the found element into the group list. */
3713 : 2639856 : if (!DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
3714 : : {
3715 : 1471901 : DR_GROUP_FIRST_ELEMENT (stmtinfo_a) = stmtinfo_a;
3716 : 1471901 : lastinfo = stmtinfo_a;
3717 : : }
3718 : 2639856 : DR_GROUP_FIRST_ELEMENT (stmtinfo_b) = stmtinfo_a;
3719 : 2639856 : DR_GROUP_NEXT_ELEMENT (lastinfo) = stmtinfo_b;
3720 : 2639856 : lastinfo = stmtinfo_b;
3721 : :
3722 : 2639856 : if (! STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
3723 : : {
3724 : 2639826 : STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a)
3725 : 2639826 : = !can_group_stmts_p (stmtinfo_a, stmtinfo_b, false);
3726 : :
3727 : 2639826 : if (dump_enabled_p () && STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
3728 : 73 : dump_printf_loc (MSG_NOTE, vect_location,
3729 : : "Load suitable for SLP vectorization only.\n");
3730 : : }
3731 : :
3732 : 2639856 : if (init_b == init_prev
3733 : 29195 : && !to_fixup.add (DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
3734 : 2657989 : && dump_enabled_p ())
3735 : 246 : dump_printf_loc (MSG_NOTE, vect_location,
3736 : : "Queuing group with duplicate access for fixup\n");
3737 : : }
3738 : : }
3739 : :
3740 : : /* Fixup groups with duplicate entries by splitting it. */
3741 : 1583328 : while (1)
3742 : : {
3743 : 1583328 : hash_set<stmt_vec_info>::iterator it = to_fixup.begin ();
3744 : 1583328 : if (!(it != to_fixup.end ()))
3745 : : break;
3746 : 44143 : stmt_vec_info grp = *it;
3747 : 44143 : to_fixup.remove (grp);
3748 : :
3749 : : /* Find the earliest duplicate group member. */
3750 : 44143 : unsigned first_duplicate = -1u;
3751 : 44143 : stmt_vec_info next, g = grp;
3752 : 214773 : while ((next = DR_GROUP_NEXT_ELEMENT (g)))
3753 : : {
3754 : 126487 : if (tree_int_cst_equal (DR_INIT (STMT_VINFO_DR_INFO (next)->dr),
3755 : 126487 : DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
3756 : 126487 : && gimple_uid (STMT_VINFO_STMT (next)) < first_duplicate)
3757 : : first_duplicate = gimple_uid (STMT_VINFO_STMT (next));
3758 : : g = next;
3759 : : }
3760 : 44143 : if (first_duplicate == -1U)
3761 : 18133 : continue;
3762 : :
3763 : : /* Then move all stmts after the first duplicate to a new group.
3764 : : Note this is a heuristic but one with the property that *it
3765 : : is fixed up completely. */
3766 : 26010 : g = grp;
3767 : 26010 : stmt_vec_info newgroup = NULL, ng = grp;
3768 : 176543 : while ((next = DR_GROUP_NEXT_ELEMENT (g)))
3769 : : {
3770 : 124523 : if (gimple_uid (STMT_VINFO_STMT (next)) >= first_duplicate)
3771 : : {
3772 : 118402 : DR_GROUP_NEXT_ELEMENT (g) = DR_GROUP_NEXT_ELEMENT (next);
3773 : 118402 : if (!newgroup)
3774 : : {
3775 : 26010 : newgroup = next;
3776 : 26010 : STMT_VINFO_SLP_VECT_ONLY (newgroup)
3777 : 26010 : = STMT_VINFO_SLP_VECT_ONLY (grp);
3778 : : }
3779 : : else
3780 : 92392 : DR_GROUP_NEXT_ELEMENT (ng) = next;
3781 : 118402 : ng = next;
3782 : 118402 : DR_GROUP_FIRST_ELEMENT (ng) = newgroup;
3783 : : }
3784 : : else
3785 : : g = DR_GROUP_NEXT_ELEMENT (g);
3786 : : }
3787 : 26010 : DR_GROUP_NEXT_ELEMENT (ng) = NULL;
3788 : :
3789 : : /* Fixup the new group which still may contain duplicates. */
3790 : 26010 : to_fixup.add (newgroup);
3791 : : }
3792 : :
3793 : 1539185 : dr_vec_info *dr_info;
3794 : 16345491 : FOR_EACH_VEC_ELT (datarefs_copy, i, dr_info)
3795 : : {
3796 : 14812479 : if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
3797 : 14812479 : && !vect_analyze_data_ref_access (vinfo, dr_info))
3798 : : {
3799 : 6227 : if (dump_enabled_p ())
3800 : 260 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
3801 : : "not vectorized: complicated access pattern.\n");
3802 : :
3803 : 6227 : if (is_a <bb_vec_info> (vinfo))
3804 : : {
3805 : : /* Mark the statement as not vectorizable. */
3806 : 54 : STMT_VINFO_VECTORIZABLE (dr_info->stmt) = false;
3807 : 54 : continue;
3808 : : }
3809 : : else
3810 : : {
3811 : 6173 : datarefs_copy.release ();
3812 : 6173 : return opt_result::failure_at (dr_info->stmt->stmt,
3813 : : "not vectorized:"
3814 : : " complicated access pattern.\n");
3815 : : }
3816 : : }
3817 : : }
3818 : :
3819 : 1533012 : datarefs_copy.release ();
3820 : 1533012 : return opt_result::success ();
3821 : 1539185 : }
3822 : :
3823 : : /* Function vect_vfa_segment_size.
3824 : :
3825 : : Input:
3826 : : DR_INFO: The data reference.
3827 : : LENGTH_FACTOR: segment length to consider.
3828 : :
3829 : : Return a value suitable for the dr_with_seg_len::seg_len field.
3830 : : This is the "distance travelled" by the pointer from the first
3831 : : iteration in the segment to the last. Note that it does not include
3832 : : the size of the access; in effect it only describes the first byte. */
3833 : :
3834 : : static tree
3835 : 116732 : vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
3836 : : {
3837 : 116732 : length_factor = size_binop (MINUS_EXPR,
3838 : : fold_convert (sizetype, length_factor),
3839 : : size_one_node);
3840 : 116732 : return size_binop (MULT_EXPR, fold_convert (sizetype, DR_STEP (dr_info->dr)),
3841 : : length_factor);
3842 : : }
3843 : :
3844 : : /* Return a value that, when added to abs (vect_vfa_segment_size (DR_INFO)),
3845 : : gives the worst-case number of bytes covered by the segment. */
3846 : :
3847 : : static unsigned HOST_WIDE_INT
3848 : 117214 : vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
3849 : : {
3850 : 117214 : stmt_vec_info stmt_vinfo = dr_info->stmt;
3851 : 117214 : tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
3852 : 117214 : unsigned HOST_WIDE_INT ref_size = tree_to_uhwi (TYPE_SIZE_UNIT (ref_type));
3853 : 117214 : unsigned HOST_WIDE_INT access_size = ref_size;
3854 : 117214 : if (DR_GROUP_FIRST_ELEMENT (stmt_vinfo))
3855 : : {
3856 : 36834 : gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == stmt_vinfo);
3857 : 36834 : access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
3858 : : }
3859 : 117214 : tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
3860 : 117214 : int misalignment;
3861 : 117214 : if (STMT_VINFO_VEC_STMTS (stmt_vinfo).exists ()
3862 : 117214 : && ((misalignment = dr_misalignment (dr_info, vectype)), true)
3863 : 117214 : && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
3864 : : == dr_explicit_realign_optimized))
3865 : : {
3866 : : /* We might access a full vector's worth. */
3867 : 0 : access_size += tree_to_uhwi (TYPE_SIZE_UNIT (vectype)) - ref_size;
3868 : : }
3869 : 117214 : return access_size;
3870 : : }
3871 : :
3872 : : /* Get the minimum alignment for all the scalar accesses that DR_INFO
3873 : : describes. */
3874 : :
3875 : : static unsigned int
3876 : 117214 : vect_vfa_align (dr_vec_info *dr_info)
3877 : : {
3878 : 0 : return dr_alignment (dr_info->dr);
3879 : : }
3880 : :
3881 : : /* Function vect_no_alias_p.
3882 : :
3883 : : Given data references A and B with equal base and offset, see whether
3884 : : the alias relation can be decided at compilation time. Return 1 if
3885 : : it can and the references alias, 0 if it can and the references do
3886 : : not alias, and -1 if we cannot decide at compile time. SEGMENT_LENGTH_A,
3887 : : SEGMENT_LENGTH_B, ACCESS_SIZE_A and ACCESS_SIZE_B are the equivalent
3888 : : of dr_with_seg_len::{seg_len,access_size} for A and B. */
3889 : :
3890 : : static int
3891 : 2066 : vect_compile_time_alias (dr_vec_info *a, dr_vec_info *b,
3892 : : tree segment_length_a, tree segment_length_b,
3893 : : unsigned HOST_WIDE_INT access_size_a,
3894 : : unsigned HOST_WIDE_INT access_size_b)
3895 : : {
3896 : 2066 : poly_offset_int offset_a = wi::to_poly_offset (DR_INIT (a->dr));
3897 : 2066 : poly_offset_int offset_b = wi::to_poly_offset (DR_INIT (b->dr));
3898 : 2066 : poly_uint64 const_length_a;
3899 : 2066 : poly_uint64 const_length_b;
3900 : :
3901 : : /* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
3902 : : bytes, e.g., int a[3] -> a[1] range is [a+4, a+16) instead of
3903 : : [a, a+12) */
3904 : 2066 : if (tree_int_cst_compare (DR_STEP (a->dr), size_zero_node) < 0)
3905 : : {
3906 : 277 : const_length_a = (-wi::to_poly_wide (segment_length_a)).force_uhwi ();
3907 : 277 : offset_a -= const_length_a;
3908 : : }
3909 : : else
3910 : 1789 : const_length_a = tree_to_poly_uint64 (segment_length_a);
3911 : 2066 : if (tree_int_cst_compare (DR_STEP (b->dr), size_zero_node) < 0)
3912 : : {
3913 : 431 : const_length_b = (-wi::to_poly_wide (segment_length_b)).force_uhwi ();
3914 : 431 : offset_b -= const_length_b;
3915 : : }
3916 : : else
3917 : 1635 : const_length_b = tree_to_poly_uint64 (segment_length_b);
3918 : :
3919 : 2066 : const_length_a += access_size_a;
3920 : 2066 : const_length_b += access_size_b;
3921 : :
3922 : 2066 : if (ranges_known_overlap_p (offset_a, const_length_a,
3923 : : offset_b, const_length_b))
3924 : : return 1;
3925 : :
3926 : 461 : if (!ranges_maybe_overlap_p (offset_a, const_length_a,
3927 : : offset_b, const_length_b))
3928 : 461 : return 0;
3929 : :
3930 : : return -1;
3931 : : }
3932 : :
3933 : : /* Return true if the minimum nonzero dependence distance for loop LOOP_DEPTH
3934 : : in DDR is >= VF. */
3935 : :
3936 : : static bool
3937 : 68993 : dependence_distance_ge_vf (data_dependence_relation *ddr,
3938 : : unsigned int loop_depth, poly_uint64 vf)
3939 : : {
3940 : 68993 : if (DDR_ARE_DEPENDENT (ddr) != NULL_TREE
3941 : 73690 : || DDR_NUM_DIST_VECTS (ddr) == 0)
3942 : : return false;
3943 : :
3944 : : /* If the dependence is exact, we should have limited the VF instead. */
3945 : 4736 : gcc_checking_assert (DDR_COULD_BE_INDEPENDENT_P (ddr));
3946 : :
3947 : : unsigned int i;
3948 : : lambda_vector dist_v;
3949 : 9507 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
3950 : : {
3951 : 9468 : HOST_WIDE_INT dist = dist_v[loop_depth];
3952 : 9468 : if (dist != 0
3953 : 4736 : && !(dist > 0 && DDR_REVERSED_P (ddr))
3954 : 14204 : && maybe_lt ((unsigned HOST_WIDE_INT) abs_hwi (dist), vf))
3955 : : return false;
3956 : : }
3957 : :
3958 : 39 : if (dump_enabled_p ())
3959 : 2 : dump_printf_loc (MSG_NOTE, vect_location,
3960 : : "dependence distance between %T and %T is >= VF\n",
3961 : 2 : DR_REF (DDR_A (ddr)), DR_REF (DDR_B (ddr)));
3962 : :
3963 : : return true;
3964 : : }
3965 : :
3966 : : /* Dump LOWER_BOUND using flags DUMP_KIND. Dumps are known to be enabled. */
3967 : :
3968 : : static void
3969 : 439 : dump_lower_bound (dump_flags_t dump_kind, const vec_lower_bound &lower_bound)
3970 : : {
3971 : 439 : dump_printf (dump_kind, "%s (%T) >= ",
3972 : 439 : lower_bound.unsigned_p ? "unsigned" : "abs",
3973 : 439 : lower_bound.expr);
3974 : 439 : dump_dec (dump_kind, lower_bound.min_value);
3975 : 439 : }
3976 : :
3977 : : /* Record that the vectorized loop requires the vec_lower_bound described
3978 : : by EXPR, UNSIGNED_P and MIN_VALUE. */
3979 : :
3980 : : static void
3981 : 6053 : vect_check_lower_bound (loop_vec_info loop_vinfo, tree expr, bool unsigned_p,
3982 : : poly_uint64 min_value)
3983 : : {
3984 : 6053 : vec<vec_lower_bound> &lower_bounds
3985 : : = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo);
3986 : 7025 : for (unsigned int i = 0; i < lower_bounds.length (); ++i)
3987 : 5581 : if (operand_equal_p (lower_bounds[i].expr, expr, 0))
3988 : : {
3989 : 4609 : unsigned_p &= lower_bounds[i].unsigned_p;
3990 : 4609 : min_value = upper_bound (lower_bounds[i].min_value, min_value);
3991 : 4609 : if (lower_bounds[i].unsigned_p != unsigned_p
3992 : 4609 : || maybe_lt (lower_bounds[i].min_value, min_value))
3993 : : {
3994 : 732 : lower_bounds[i].unsigned_p = unsigned_p;
3995 : 732 : lower_bounds[i].min_value = min_value;
3996 : 732 : if (dump_enabled_p ())
3997 : : {
3998 : 246 : dump_printf_loc (MSG_NOTE, vect_location,
3999 : : "updating run-time check to ");
4000 : 246 : dump_lower_bound (MSG_NOTE, lower_bounds[i]);
4001 : 246 : dump_printf (MSG_NOTE, "\n");
4002 : : }
4003 : : }
4004 : 4609 : return;
4005 : : }
4006 : :
4007 : 1444 : vec_lower_bound lower_bound (expr, unsigned_p, min_value);
4008 : 1444 : if (dump_enabled_p ())
4009 : : {
4010 : 193 : dump_printf_loc (MSG_NOTE, vect_location, "need a run-time check that ");
4011 : 193 : dump_lower_bound (MSG_NOTE, lower_bound);
4012 : 193 : dump_printf (MSG_NOTE, "\n");
4013 : : }
4014 : 1444 : LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).safe_push (lower_bound);
4015 : : }
4016 : :
4017 : : /* Return true if it's unlikely that the step of the vectorized form of DR_INFO
4018 : : will span fewer than GAP bytes. */
4019 : :
4020 : : static bool
4021 : 4835 : vect_small_gap_p (loop_vec_info loop_vinfo, dr_vec_info *dr_info,
4022 : : poly_int64 gap)
4023 : : {
4024 : 4835 : stmt_vec_info stmt_info = dr_info->stmt;
4025 : 4835 : HOST_WIDE_INT count
4026 : 4835 : = estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
4027 : 4835 : if (DR_GROUP_FIRST_ELEMENT (stmt_info))
4028 : 4123 : count *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
4029 : 4835 : return (estimated_poly_value (gap)
4030 : 4835 : <= count * vect_get_scalar_dr_size (dr_info));
4031 : : }
4032 : :
4033 : : /* Return true if we know that there is no alias between DR_INFO_A and
4034 : : DR_INFO_B when abs (DR_STEP (DR_INFO_A->dr)) >= N for some N.
4035 : : When returning true, set *LOWER_BOUND_OUT to this N. */
4036 : :
4037 : : static bool
4038 : 18378 : vectorizable_with_step_bound_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b,
4039 : : poly_uint64 *lower_bound_out)
4040 : : {
4041 : : /* Check that there is a constant gap of known sign between DR_A
4042 : : and DR_B. */
4043 : 18378 : data_reference *dr_a = dr_info_a->dr;
4044 : 18378 : data_reference *dr_b = dr_info_b->dr;
4045 : 18378 : poly_int64 init_a, init_b;
4046 : 18378 : if (!operand_equal_p (DR_BASE_ADDRESS (dr_a), DR_BASE_ADDRESS (dr_b), 0)
4047 : 7911 : || !operand_equal_p (DR_OFFSET (dr_a), DR_OFFSET (dr_b), 0)
4048 : 6891 : || !operand_equal_p (DR_STEP (dr_a), DR_STEP (dr_b), 0)
4049 : 6881 : || !poly_int_tree_p (DR_INIT (dr_a), &init_a)
4050 : 6881 : || !poly_int_tree_p (DR_INIT (dr_b), &init_b)
4051 : 18378 : || !ordered_p (init_a, init_b))
4052 : 11497 : return false;
4053 : :
4054 : : /* Sort DR_A and DR_B by the address they access. */
4055 : 6881 : if (maybe_lt (init_b, init_a))
4056 : : {
4057 : 128 : std::swap (init_a, init_b);
4058 : 128 : std::swap (dr_info_a, dr_info_b);
4059 : 128 : std::swap (dr_a, dr_b);
4060 : : }
4061 : :
4062 : : /* If the two accesses could be dependent within a scalar iteration,
4063 : : make sure that we'd retain their order. */
4064 : 6881 : if (maybe_gt (init_a + vect_get_scalar_dr_size (dr_info_a), init_b)
4065 : 6881 : && !vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
4066 : : return false;
4067 : :
4068 : : /* There is no alias if abs (DR_STEP) is greater than or equal to
4069 : : the bytes spanned by the combination of the two accesses. */
4070 : 6881 : *lower_bound_out = init_b + vect_get_scalar_dr_size (dr_info_b) - init_a;
4071 : 6881 : return true;
4072 : : }
4073 : :
4074 : : /* Function vect_prune_runtime_alias_test_list.
4075 : :
4076 : : Prune a list of ddrs to be tested at run-time by versioning for alias.
4077 : : Merge several alias checks into one if possible.
4078 : : Return FALSE if resulting list of ddrs is longer then allowed by
4079 : : PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE. */
4080 : :
4081 : : opt_result
4082 : 360053 : vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
4083 : : {
4084 : 360053 : typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
4085 : 360053 : hash_set <tree_pair_hash> compared_objects;
4086 : :
4087 : 360053 : const vec<ddr_p> &may_alias_ddrs = LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo);
4088 : 360053 : vec<dr_with_seg_len_pair_t> &comp_alias_ddrs
4089 : : = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo);
4090 : 360053 : const vec<vec_object_pair> &check_unequal_addrs
4091 : : = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo);
4092 : 360053 : poly_uint64 vect_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
4093 : 360053 : tree scalar_loop_iters = LOOP_VINFO_NITERS (loop_vinfo);
4094 : :
4095 : 360053 : ddr_p ddr;
4096 : 360053 : unsigned int i;
4097 : 360053 : tree length_factor;
4098 : :
4099 : 360053 : DUMP_VECT_SCOPE ("vect_prune_runtime_alias_test_list");
4100 : :
4101 : : /* Step values are irrelevant for aliasing if the number of vector
4102 : : iterations is equal to the number of scalar iterations (which can
4103 : : happen for fully-SLP loops). */
4104 : 360053 : bool vf_one_p = known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U);
4105 : :
4106 : 360053 : if (!vf_one_p)
4107 : : {
4108 : : /* Convert the checks for nonzero steps into bound tests. */
4109 : : tree value;
4110 : 357989 : FOR_EACH_VEC_ELT (LOOP_VINFO_CHECK_NONZERO (loop_vinfo), i, value)
4111 : 1527 : vect_check_lower_bound (loop_vinfo, value, true, 1);
4112 : : }
4113 : :
4114 : 360053 : if (may_alias_ddrs.is_empty ())
4115 : 340202 : return opt_result::success ();
4116 : :
4117 : 19851 : comp_alias_ddrs.create (may_alias_ddrs.length ());
4118 : :
4119 : 19851 : unsigned int loop_depth
4120 : 19851 : = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
4121 : 19851 : LOOP_VINFO_LOOP_NEST (loop_vinfo));
4122 : :
4123 : : /* First, we collect all data ref pairs for aliasing checks. */
4124 : 87239 : FOR_EACH_VEC_ELT (may_alias_ddrs, i, ddr)
4125 : : {
4126 : 68993 : poly_uint64 lower_bound;
4127 : 68993 : tree segment_length_a, segment_length_b;
4128 : 68993 : unsigned HOST_WIDE_INT access_size_a, access_size_b;
4129 : 68993 : unsigned HOST_WIDE_INT align_a, align_b;
4130 : :
4131 : : /* Ignore the alias if the VF we chose ended up being no greater
4132 : : than the dependence distance. */
4133 : 68993 : if (dependence_distance_ge_vf (ddr, loop_depth, vect_factor))
4134 : 10847 : continue;
4135 : :
4136 : 68954 : if (DDR_OBJECT_A (ddr))
4137 : : {
4138 : 46 : vec_object_pair new_pair (DDR_OBJECT_A (ddr), DDR_OBJECT_B (ddr));
4139 : 46 : if (!compared_objects.add (new_pair))
4140 : : {
4141 : 14 : if (dump_enabled_p ())
4142 : 8 : dump_printf_loc (MSG_NOTE, vect_location,
4143 : : "checking that %T and %T"
4144 : : " have different addresses\n",
4145 : : new_pair.first, new_pair.second);
4146 : 14 : LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).safe_push (new_pair);
4147 : : }
4148 : 46 : continue;
4149 : 46 : }
4150 : :
4151 : 68908 : dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
4152 : 68908 : stmt_vec_info stmt_info_a = dr_info_a->stmt;
4153 : :
4154 : 68908 : dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
4155 : 68908 : stmt_vec_info stmt_info_b = dr_info_b->stmt;
4156 : :
4157 : 68908 : bool preserves_scalar_order_p
4158 : 68908 : = vect_preserves_scalar_order_p (dr_info_a, dr_info_b);
4159 : 68908 : bool ignore_step_p
4160 : : = (vf_one_p
4161 : 68908 : && (preserves_scalar_order_p
4162 : 2818 : || operand_equal_p (DR_STEP (dr_info_a->dr),
4163 : 2818 : DR_STEP (dr_info_b->dr))));
4164 : :
4165 : : /* Skip the pair if inter-iteration dependencies are irrelevant
4166 : : and intra-iteration dependencies are guaranteed to be honored. */
4167 : 11791 : if (ignore_step_p
4168 : 6016 : && (preserves_scalar_order_p
4169 : 2556 : || vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
4170 : : &lower_bound)))
4171 : : {
4172 : 5775 : if (dump_enabled_p ())
4173 : 2352 : dump_printf_loc (MSG_NOTE, vect_location,
4174 : : "no need for alias check between "
4175 : : "%T and %T when VF is 1\n",
4176 : 2352 : DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
4177 : 5775 : continue;
4178 : : }
4179 : :
4180 : : /* See whether we can handle the alias using a bounds check on
4181 : : the step, and whether that's likely to be the best approach.
4182 : : (It might not be, for example, if the minimum step is much larger
4183 : : than the number of bytes handled by one vector iteration.) */
4184 : 63133 : if (!ignore_step_p
4185 : 62892 : && TREE_CODE (DR_STEP (dr_info_a->dr)) != INTEGER_CST
4186 : 15822 : && vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
4187 : : &lower_bound)
4188 : 67699 : && (vect_small_gap_p (loop_vinfo, dr_info_a, lower_bound)
4189 : 269 : || vect_small_gap_p (loop_vinfo, dr_info_b, lower_bound)))
4190 : : {
4191 : 4526 : bool unsigned_p = dr_known_forward_stride_p (dr_info_a->dr);
4192 : 4526 : if (dump_enabled_p ())
4193 : : {
4194 : 3312 : dump_printf_loc (MSG_NOTE, vect_location, "no alias between "
4195 : : "%T and %T when the step %T is outside ",
4196 : : DR_REF (dr_info_a->dr),
4197 : 1656 : DR_REF (dr_info_b->dr),
4198 : 1656 : DR_STEP (dr_info_a->dr));
4199 : 1656 : if (unsigned_p)
4200 : 504 : dump_printf (MSG_NOTE, "[0");
4201 : : else
4202 : : {
4203 : 1152 : dump_printf (MSG_NOTE, "(");
4204 : 1152 : dump_dec (MSG_NOTE, poly_int64 (-lower_bound));
4205 : : }
4206 : 1656 : dump_printf (MSG_NOTE, ", ");
4207 : 1656 : dump_dec (MSG_NOTE, lower_bound);
4208 : 1656 : dump_printf (MSG_NOTE, ")\n");
4209 : : }
4210 : 4526 : vect_check_lower_bound (loop_vinfo, DR_STEP (dr_info_a->dr),
4211 : : unsigned_p, lower_bound);
4212 : 4526 : continue;
4213 : 4526 : }
4214 : :
4215 : 58607 : stmt_vec_info dr_group_first_a = DR_GROUP_FIRST_ELEMENT (stmt_info_a);
4216 : 58607 : if (dr_group_first_a)
4217 : : {
4218 : 18978 : stmt_info_a = dr_group_first_a;
4219 : 18978 : dr_info_a = STMT_VINFO_DR_INFO (stmt_info_a);
4220 : : }
4221 : :
4222 : 58607 : stmt_vec_info dr_group_first_b = DR_GROUP_FIRST_ELEMENT (stmt_info_b);
4223 : 58607 : if (dr_group_first_b)
4224 : : {
4225 : 17856 : stmt_info_b = dr_group_first_b;
4226 : 17856 : dr_info_b = STMT_VINFO_DR_INFO (stmt_info_b);
4227 : : }
4228 : :
4229 : 58607 : if (ignore_step_p)
4230 : : {
4231 : 241 : segment_length_a = size_zero_node;
4232 : 241 : segment_length_b = size_zero_node;
4233 : : }
4234 : : else
4235 : : {
4236 : 58366 : if (!operand_equal_p (DR_STEP (dr_info_a->dr),
4237 : 58366 : DR_STEP (dr_info_b->dr), 0))
4238 : : length_factor = scalar_loop_iters;
4239 : : else
4240 : 45666 : length_factor = size_int (vect_factor);
4241 : 58366 : segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
4242 : 58366 : segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
4243 : : }
4244 : 58607 : access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
4245 : 58607 : access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
4246 : 58607 : align_a = vect_vfa_align (dr_info_a);
4247 : 58607 : align_b = vect_vfa_align (dr_info_b);
4248 : :
4249 : : /* See whether the alias is known at compilation time. */
4250 : 58607 : if (operand_equal_p (DR_BASE_ADDRESS (dr_info_a->dr),
4251 : 58607 : DR_BASE_ADDRESS (dr_info_b->dr), 0)
4252 : 4021 : && operand_equal_p (DR_OFFSET (dr_info_a->dr),
4253 : 4021 : DR_OFFSET (dr_info_b->dr), 0)
4254 : 2186 : && TREE_CODE (DR_STEP (dr_info_a->dr)) == INTEGER_CST
4255 : 2136 : && TREE_CODE (DR_STEP (dr_info_b->dr)) == INTEGER_CST
4256 : 2126 : && poly_int_tree_p (segment_length_a)
4257 : 60684 : && poly_int_tree_p (segment_length_b))
4258 : : {
4259 : 2066 : int res = vect_compile_time_alias (dr_info_a, dr_info_b,
4260 : : segment_length_a,
4261 : : segment_length_b,
4262 : : access_size_a,
4263 : : access_size_b);
4264 : 2066 : if (res >= 0 && dump_enabled_p ())
4265 : : {
4266 : 232 : dump_printf_loc (MSG_NOTE, vect_location,
4267 : : "can tell at compile time that %T and %T",
4268 : 116 : DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
4269 : 116 : if (res == 0)
4270 : 57 : dump_printf (MSG_NOTE, " do not alias\n");
4271 : : else
4272 : 59 : dump_printf (MSG_NOTE, " alias\n");
4273 : : }
4274 : :
4275 : 2066 : if (res == 0)
4276 : 461 : continue;
4277 : :
4278 : 1605 : if (res == 1)
4279 : 1605 : return opt_result::failure_at (stmt_info_b->stmt,
4280 : : "not vectorized:"
4281 : : " compilation time alias: %G%G",
4282 : : stmt_info_a->stmt,
4283 : : stmt_info_b->stmt);
4284 : : }
4285 : :
4286 : : /* dr_with_seg_len requires the alignment to apply to the segment length
4287 : : and access size, not just the start address. The access size can be
4288 : : smaller than the pointer alignment for grouped accesses and bitfield
4289 : : references; see PR115192 and PR116125 respectively. */
4290 : 56541 : align_a = std::min (align_a, least_bit_hwi (access_size_a));
4291 : 56541 : align_b = std::min (align_b, least_bit_hwi (access_size_b));
4292 : :
4293 : 56541 : dr_with_seg_len dr_a (dr_info_a->dr, segment_length_a,
4294 : 56541 : access_size_a, align_a);
4295 : 56541 : dr_with_seg_len dr_b (dr_info_b->dr, segment_length_b,
4296 : 56541 : access_size_b, align_b);
4297 : : /* Canonicalize the order to be the one that's needed for accurate
4298 : : RAW, WAR and WAW flags, in cases where the data references are
4299 : : well-ordered. The order doesn't really matter otherwise,
4300 : : but we might as well be consistent. */
4301 : 56541 : if (get_later_stmt (stmt_info_a, stmt_info_b) == stmt_info_a)
4302 : 4419 : std::swap (dr_a, dr_b);
4303 : :
4304 : 56541 : dr_with_seg_len_pair_t dr_with_seg_len_pair
4305 : : (dr_a, dr_b, (preserves_scalar_order_p
4306 : : ? dr_with_seg_len_pair_t::WELL_ORDERED
4307 : 62326 : : dr_with_seg_len_pair_t::REORDERED));
4308 : :
4309 : 56541 : comp_alias_ddrs.safe_push (dr_with_seg_len_pair);
4310 : : }
4311 : :
4312 : 18246 : prune_runtime_alias_test_list (&comp_alias_ddrs, vect_factor);
4313 : :
4314 : 36492 : unsigned int count = (comp_alias_ddrs.length ()
4315 : 18246 : + check_unequal_addrs.length ());
4316 : :
4317 : 18246 : if (count
4318 : 18246 : && (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo))
4319 : : == VECT_COST_MODEL_VERY_CHEAP))
4320 : 12508 : return opt_result::failure_at
4321 : 12508 : (vect_location, "would need a runtime alias check\n");
4322 : :
4323 : 5738 : if (dump_enabled_p ())
4324 : 1906 : dump_printf_loc (MSG_NOTE, vect_location,
4325 : : "improved number of alias checks from %d to %d\n",
4326 : : may_alias_ddrs.length (), count);
4327 : 5738 : unsigned limit = param_vect_max_version_for_alias_checks;
4328 : 5738 : if (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo)) == VECT_COST_MODEL_CHEAP)
4329 : 766 : limit = param_vect_max_version_for_alias_checks * 6 / 10;
4330 : 5738 : if (count > limit)
4331 : 166 : return opt_result::failure_at
4332 : 166 : (vect_location,
4333 : : "number of versioning for alias run-time tests exceeds %d "
4334 : : "(--param vect-max-version-for-alias-checks)\n", limit);
4335 : :
4336 : 5572 : return opt_result::success ();
4337 : 360053 : }
4338 : :
4339 : : /* Check whether we can use an internal function for a gather load
4340 : : or scatter store. READ_P is true for loads and false for stores.
4341 : : MASKED_P is true if the load or store is conditional. MEMORY_TYPE is
4342 : : the type of the memory elements being loaded or stored. OFFSET_TYPE
4343 : : is the type of the offset that is being applied to the invariant
4344 : : base address. SCALE is the amount by which the offset should
4345 : : be multiplied *after* it has been converted to address width.
4346 : :
4347 : : Return true if the function is supported, storing the function id in
4348 : : *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.
4349 : :
4350 : : If we can use gather and store the possible else values in ELSVALS. */
4351 : :
4352 : : bool
4353 : 63010 : vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
4354 : : tree vectype, tree memory_type, tree offset_type,
4355 : : int scale, internal_fn *ifn_out,
4356 : : tree *offset_vectype_out, vec<int> *elsvals)
4357 : : {
4358 : 63010 : unsigned int memory_bits = tree_to_uhwi (TYPE_SIZE (memory_type));
4359 : 63010 : unsigned int element_bits = vector_element_bits (vectype);
4360 : 63010 : if (element_bits != memory_bits)
4361 : : /* For now the vector elements must be the same width as the
4362 : : memory elements. */
4363 : : return false;
4364 : :
4365 : : /* Work out which function we need. */
4366 : 63010 : internal_fn ifn, alt_ifn, alt_ifn2;
4367 : 63010 : if (read_p)
4368 : : {
4369 : 39016 : ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
4370 : : alt_ifn = IFN_MASK_GATHER_LOAD;
4371 : : /* When target supports MASK_LEN_GATHER_LOAD, we always
4372 : : use MASK_LEN_GATHER_LOAD regardless whether len and
4373 : : mask are valid or not. */
4374 : : alt_ifn2 = IFN_MASK_LEN_GATHER_LOAD;
4375 : : }
4376 : : else
4377 : : {
4378 : 23994 : ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
4379 : 63010 : alt_ifn = IFN_MASK_SCATTER_STORE;
4380 : : /* When target supports MASK_LEN_SCATTER_STORE, we always
4381 : : use MASK_LEN_SCATTER_STORE regardless whether len and
4382 : : mask are valid or not. */
4383 : 63010 : alt_ifn2 = IFN_MASK_LEN_SCATTER_STORE;
4384 : : }
4385 : :
4386 : 488532 : for (;;)
4387 : : {
4388 : 275771 : tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
4389 : 275771 : if (!offset_vectype)
4390 : : return false;
4391 : :
4392 : : /* Test whether the target supports this combination. */
4393 : 271825 : if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
4394 : : offset_vectype, scale,
4395 : : elsvals))
4396 : : {
4397 : 0 : *ifn_out = ifn;
4398 : 0 : *offset_vectype_out = offset_vectype;
4399 : 0 : return true;
4400 : : }
4401 : 271825 : else if (!masked_p
4402 : 271825 : && internal_gather_scatter_fn_supported_p (alt_ifn, vectype,
4403 : : memory_type,
4404 : : offset_vectype,
4405 : : scale, elsvals))
4406 : : {
4407 : 0 : *ifn_out = alt_ifn;
4408 : 0 : *offset_vectype_out = offset_vectype;
4409 : 0 : return true;
4410 : : }
4411 : 271825 : else if (internal_gather_scatter_fn_supported_p (alt_ifn2, vectype,
4412 : : memory_type,
4413 : : offset_vectype, scale,
4414 : : elsvals))
4415 : : {
4416 : 0 : *ifn_out = alt_ifn2;
4417 : 0 : *offset_vectype_out = offset_vectype;
4418 : 0 : return true;
4419 : : }
4420 : :
4421 : 271825 : if (TYPE_PRECISION (offset_type) >= POINTER_SIZE
4422 : 271825 : && TYPE_PRECISION (offset_type) >= element_bits)
4423 : : return false;
4424 : :
4425 : 212761 : offset_type = build_nonstandard_integer_type
4426 : 212761 : (TYPE_PRECISION (offset_type) * 2, TYPE_UNSIGNED (offset_type));
4427 : 212761 : }
4428 : : }
4429 : :
4430 : : /* STMT_INFO is a call to an internal gather load or scatter store function.
4431 : : Describe the operation in INFO. */
4432 : :
4433 : : static void
4434 : 0 : vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
4435 : : gather_scatter_info *info)
4436 : : {
4437 : 0 : gcall *call = as_a <gcall *> (stmt_info->stmt);
4438 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4439 : 0 : data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
4440 : :
4441 : 0 : info->ifn = gimple_call_internal_fn (call);
4442 : 0 : info->decl = NULL_TREE;
4443 : 0 : info->base = gimple_call_arg (call, 0);
4444 : 0 : info->offset = gimple_call_arg (call, 1);
4445 : 0 : info->offset_dt = vect_unknown_def_type;
4446 : 0 : info->offset_vectype = NULL_TREE;
4447 : 0 : info->scale = TREE_INT_CST_LOW (gimple_call_arg (call, 2));
4448 : 0 : info->element_type = TREE_TYPE (vectype);
4449 : 0 : info->memory_type = TREE_TYPE (DR_REF (dr));
4450 : 0 : }
4451 : :
4452 : : /* Return true if a non-affine read or write in STMT_INFO is suitable for a
4453 : : gather load or scatter store. Describe the operation in *INFO if so.
4454 : : If it is suitable and ELSVALS is nonzero store the supported else values
4455 : : in the vector it points to. */
4456 : :
4457 : : bool
4458 : 191763 : vect_check_gather_scatter (stmt_vec_info stmt_info, loop_vec_info loop_vinfo,
4459 : : gather_scatter_info *info, vec<int> *elsvals)
4460 : : {
4461 : 191763 : HOST_WIDE_INT scale = 1;
4462 : 191763 : poly_int64 pbitpos, pbitsize;
4463 : 191763 : class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
4464 : 191763 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
4465 : 191763 : tree offtype = NULL_TREE;
4466 : 191763 : tree decl = NULL_TREE, base, off;
4467 : 191763 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
4468 : 191763 : tree memory_type = TREE_TYPE (DR_REF (dr));
4469 : 191763 : machine_mode pmode;
4470 : 191763 : int punsignedp, reversep, pvolatilep = 0;
4471 : 191763 : internal_fn ifn;
4472 : 191763 : tree offset_vectype;
4473 : 191763 : bool masked_p = false;
4474 : :
4475 : : /* See whether this is already a call to a gather/scatter internal function.
4476 : : If not, see whether it's a masked load or store. */
4477 : 191763 : gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
4478 : 7419 : if (call && gimple_call_internal_p (call))
4479 : : {
4480 : 7419 : ifn = gimple_call_internal_fn (call);
4481 : 7419 : if (internal_gather_scatter_fn_p (ifn))
4482 : : {
4483 : 0 : vect_describe_gather_scatter_call (stmt_info, info);
4484 : :
4485 : : /* In pattern recog we simply used a ZERO else value that
4486 : : we need to correct here. To that end just re-use the
4487 : : (already succesful) check if we support a gather IFN
4488 : : and have it populate the else values. */
4489 : 0 : if (DR_IS_READ (dr) && internal_fn_mask_index (ifn) >= 0 && elsvals)
4490 : 0 : supports_vec_gather_load_p (TYPE_MODE (vectype), elsvals);
4491 : 0 : return true;
4492 : : }
4493 : 7419 : masked_p = (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE);
4494 : : }
4495 : :
4496 : : /* ??? For epilogues we adjust DR_REF to make the following stmt-based
4497 : : analysis work, but this adjustment doesn't work for epilogues of
4498 : : epilogues during transform, so disable gather/scatter in that case. */
4499 : 191763 : if (LOOP_VINFO_EPILOGUE_P (loop_vinfo)
4500 : 9771 : && LOOP_VINFO_EPILOGUE_P (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo)))
4501 : : return false;
4502 : :
4503 : : /* True if we should aim to use internal functions rather than
4504 : : built-in functions. */
4505 : 191739 : bool use_ifn_p = (DR_IS_READ (dr)
4506 : 191739 : ? supports_vec_gather_load_p (TYPE_MODE (vectype),
4507 : : elsvals)
4508 : 69182 : : supports_vec_scatter_store_p (TYPE_MODE (vectype)));
4509 : :
4510 : 191739 : base = DR_REF (dr);
4511 : : /* For masked loads/stores, DR_REF (dr) is an artificial MEM_REF,
4512 : : see if we can use the def stmt of the address. */
4513 : 191739 : if (masked_p
4514 : 7419 : && TREE_CODE (base) == MEM_REF
4515 : 7419 : && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
4516 : 7419 : && integer_zerop (TREE_OPERAND (base, 1))
4517 : 199158 : && !expr_invariant_in_loop_p (loop, TREE_OPERAND (base, 0)))
4518 : : {
4519 : 7419 : gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base, 0));
4520 : 7419 : if (is_gimple_assign (def_stmt)
4521 : 7419 : && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
4522 : 761 : base = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
4523 : : }
4524 : :
4525 : : /* The gather and scatter builtins need address of the form
4526 : : loop_invariant + vector * {1, 2, 4, 8}
4527 : : or
4528 : : loop_invariant + sign_extend (vector) * { 1, 2, 4, 8 }.
4529 : : Unfortunately DR_BASE_ADDRESS/DR_OFFSET can be a mixture
4530 : : of loop invariants/SSA_NAMEs defined in the loop, with casts,
4531 : : multiplications and additions in it. To get a vector, we need
4532 : : a single SSA_NAME that will be defined in the loop and will
4533 : : contain everything that is not loop invariant and that can be
4534 : : vectorized. The following code attempts to find such a preexistng
4535 : : SSA_NAME OFF and put the loop invariants into a tree BASE
4536 : : that can be gimplified before the loop. */
4537 : 191739 : base = get_inner_reference (base, &pbitsize, &pbitpos, &off, &pmode,
4538 : : &punsignedp, &reversep, &pvolatilep);
4539 : 191739 : if (reversep)
4540 : : return false;
4541 : :
4542 : : /* PR 107346. Packed structs can have fields at offsets that are not
4543 : : multiples of BITS_PER_UNIT. Do not use gather/scatters in such cases. */
4544 : 191739 : if (!multiple_p (pbitpos, BITS_PER_UNIT))
4545 : : return false;
4546 : :
4547 : : /* We need to be able to form an address to the base which for example
4548 : : isn't possible for hard registers. */
4549 : 191739 : if (may_be_nonaddressable_p (base))
4550 : : return false;
4551 : :
4552 : 191731 : poly_int64 pbytepos = exact_div (pbitpos, BITS_PER_UNIT);
4553 : :
4554 : 191731 : if (TREE_CODE (base) == MEM_REF)
4555 : : {
4556 : 130425 : if (!integer_zerop (TREE_OPERAND (base, 1)))
4557 : : {
4558 : 3392 : if (off == NULL_TREE)
4559 : 3207 : off = wide_int_to_tree (sizetype, mem_ref_offset (base));
4560 : : else
4561 : 185 : off = size_binop (PLUS_EXPR, off,
4562 : : fold_convert (sizetype, TREE_OPERAND (base, 1)));
4563 : : }
4564 : 130425 : base = TREE_OPERAND (base, 0);
4565 : : }
4566 : : else
4567 : 61306 : base = build_fold_addr_expr (base);
4568 : :
4569 : 191731 : if (off == NULL_TREE)
4570 : 109560 : off = size_zero_node;
4571 : :
4572 : : /* If base is not loop invariant, either off is 0, then we start with just
4573 : : the constant offset in the loop invariant BASE and continue with base
4574 : : as OFF, otherwise give up.
4575 : : We could handle that case by gimplifying the addition of base + off
4576 : : into some SSA_NAME and use that as off, but for now punt. */
4577 : 191731 : if (!expr_invariant_in_loop_p (loop, base))
4578 : : {
4579 : 113284 : if (!integer_zerop (off))
4580 : : return false;
4581 : 109556 : off = base;
4582 : 109556 : base = size_int (pbytepos);
4583 : : }
4584 : : /* Otherwise put base + constant offset into the loop invariant BASE
4585 : : and continue with OFF. */
4586 : : else
4587 : : {
4588 : 78447 : base = fold_convert (sizetype, base);
4589 : 78447 : base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
4590 : : }
4591 : :
4592 : : /* OFF at this point may be either a SSA_NAME or some tree expression
4593 : : from get_inner_reference. Try to peel off loop invariants from it
4594 : : into BASE as long as possible. */
4595 : 188003 : STRIP_NOPS (off);
4596 : 622155 : while (offtype == NULL_TREE)
4597 : : {
4598 : 544204 : enum tree_code code;
4599 : 544204 : tree op0, op1, add = NULL_TREE;
4600 : :
4601 : 544204 : if (TREE_CODE (off) == SSA_NAME)
4602 : : {
4603 : 372220 : gimple *def_stmt = SSA_NAME_DEF_STMT (off);
4604 : :
4605 : 372220 : if (expr_invariant_in_loop_p (loop, off))
4606 : 16 : return false;
4607 : :
4608 : 372204 : if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
4609 : : break;
4610 : :
4611 : 343903 : op0 = gimple_assign_rhs1 (def_stmt);
4612 : 343903 : code = gimple_assign_rhs_code (def_stmt);
4613 : 343903 : op1 = gimple_assign_rhs2 (def_stmt);
4614 : : }
4615 : : else
4616 : : {
4617 : 171984 : if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS)
4618 : : return false;
4619 : 171984 : code = TREE_CODE (off);
4620 : 171984 : extract_ops_from_tree (off, &code, &op0, &op1);
4621 : : }
4622 : 515887 : switch (code)
4623 : : {
4624 : 150135 : case POINTER_PLUS_EXPR:
4625 : 150135 : case PLUS_EXPR:
4626 : 150135 : if (expr_invariant_in_loop_p (loop, op0))
4627 : : {
4628 : 90468 : add = op0;
4629 : 90468 : off = op1;
4630 : 138948 : do_add:
4631 : 138948 : add = fold_convert (sizetype, add);
4632 : 138948 : if (scale != 1)
4633 : 50914 : add = size_binop (MULT_EXPR, add, size_int (scale));
4634 : 138948 : base = size_binop (PLUS_EXPR, base, add);
4635 : 434152 : continue;
4636 : : }
4637 : 59667 : if (expr_invariant_in_loop_p (loop, op1))
4638 : : {
4639 : 48182 : add = op1;
4640 : 48182 : off = op0;
4641 : 48182 : goto do_add;
4642 : : }
4643 : : break;
4644 : 466 : case MINUS_EXPR:
4645 : 466 : if (expr_invariant_in_loop_p (loop, op1))
4646 : : {
4647 : 298 : add = fold_convert (sizetype, op1);
4648 : 298 : add = size_binop (MINUS_EXPR, size_zero_node, add);
4649 : 298 : off = op0;
4650 : 298 : goto do_add;
4651 : : }
4652 : : break;
4653 : 169712 : case MULT_EXPR:
4654 : 169712 : if (scale == 1 && tree_fits_shwi_p (op1))
4655 : : {
4656 : 136414 : int new_scale = tree_to_shwi (op1);
4657 : : /* Only treat this as a scaling operation if the target
4658 : : supports it for at least some offset type. */
4659 : 136414 : if (use_ifn_p
4660 : 0 : && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4661 : : masked_p, vectype, memory_type,
4662 : : signed_char_type_node,
4663 : : new_scale, &ifn,
4664 : : &offset_vectype,
4665 : : elsvals)
4666 : 136414 : && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4667 : : masked_p, vectype, memory_type,
4668 : : unsigned_char_type_node,
4669 : : new_scale, &ifn,
4670 : : &offset_vectype,
4671 : : elsvals))
4672 : : break;
4673 : 136414 : scale = new_scale;
4674 : 136414 : off = op0;
4675 : 136414 : continue;
4676 : 136414 : }
4677 : : break;
4678 : 0 : case SSA_NAME:
4679 : 0 : off = op0;
4680 : 0 : continue;
4681 : 163754 : CASE_CONVERT:
4682 : 327508 : if (!POINTER_TYPE_P (TREE_TYPE (op0))
4683 : 327508 : && !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
4684 : : break;
4685 : :
4686 : : /* Don't include the conversion if the target is happy with
4687 : : the current offset type. */
4688 : 163754 : if (use_ifn_p
4689 : 0 : && TREE_CODE (off) == SSA_NAME
4690 : 0 : && !POINTER_TYPE_P (TREE_TYPE (off))
4691 : 163754 : && vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
4692 : : masked_p, vectype, memory_type,
4693 : 0 : TREE_TYPE (off), scale, &ifn,
4694 : : &offset_vectype, elsvals))
4695 : : break;
4696 : :
4697 : 163754 : if (TYPE_PRECISION (TREE_TYPE (op0))
4698 : 163754 : == TYPE_PRECISION (TREE_TYPE (off)))
4699 : : {
4700 : 80839 : off = op0;
4701 : 80839 : continue;
4702 : : }
4703 : :
4704 : : /* Include the conversion if it is widening and we're using
4705 : : the IFN path or the target can handle the converted from
4706 : : offset or the current size is not already the same as the
4707 : : data vector element size. */
4708 : 82915 : if ((TYPE_PRECISION (TREE_TYPE (op0))
4709 : 82915 : < TYPE_PRECISION (TREE_TYPE (off)))
4710 : 82915 : && (use_ifn_p
4711 : 82865 : || (DR_IS_READ (dr)
4712 : 50890 : ? (targetm.vectorize.builtin_gather
4713 : 50890 : && targetm.vectorize.builtin_gather (vectype,
4714 : 50890 : TREE_TYPE (op0),
4715 : : scale))
4716 : 31975 : : (targetm.vectorize.builtin_scatter
4717 : 31975 : && targetm.vectorize.builtin_scatter (vectype,
4718 : 31975 : TREE_TYPE (op0),
4719 : : scale)))
4720 : 81634 : || !operand_equal_p (TYPE_SIZE (TREE_TYPE (off)),
4721 : 81634 : TYPE_SIZE (TREE_TYPE (vectype)), 0)))
4722 : : {
4723 : 77951 : off = op0;
4724 : 77951 : offtype = TREE_TYPE (off);
4725 : 77951 : STRIP_NOPS (off);
4726 : 77951 : continue;
4727 : : }
4728 : : break;
4729 : : default:
4730 : : break;
4731 : 0 : }
4732 : : break;
4733 : : }
4734 : :
4735 : : /* If at the end OFF still isn't a SSA_NAME or isn't
4736 : : defined in the loop, punt. */
4737 : 187987 : if (TREE_CODE (off) != SSA_NAME
4738 : 187987 : || expr_invariant_in_loop_p (loop, off))
4739 : 5344 : return false;
4740 : :
4741 : 182643 : if (offtype == NULL_TREE)
4742 : 104914 : offtype = TREE_TYPE (off);
4743 : :
4744 : 182643 : if (use_ifn_p)
4745 : : {
4746 : 0 : if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
4747 : : vectype, memory_type, offtype, scale,
4748 : : &ifn, &offset_vectype, elsvals))
4749 : 0 : ifn = IFN_LAST;
4750 : : decl = NULL_TREE;
4751 : : }
4752 : : else
4753 : : {
4754 : 182643 : if (DR_IS_READ (dr))
4755 : : {
4756 : 116352 : if (targetm.vectorize.builtin_gather)
4757 : 116352 : decl = targetm.vectorize.builtin_gather (vectype, offtype, scale);
4758 : : }
4759 : : else
4760 : : {
4761 : 66291 : if (targetm.vectorize.builtin_scatter)
4762 : 66291 : decl = targetm.vectorize.builtin_scatter (vectype, offtype, scale);
4763 : : }
4764 : 182643 : ifn = IFN_LAST;
4765 : : /* The offset vector type will be read from DECL when needed. */
4766 : 182643 : offset_vectype = NULL_TREE;
4767 : : }
4768 : :
4769 : 182643 : info->ifn = ifn;
4770 : 182643 : info->decl = decl;
4771 : 182643 : info->base = base;
4772 : 182643 : info->offset = off;
4773 : 182643 : info->offset_dt = vect_unknown_def_type;
4774 : 182643 : info->offset_vectype = offset_vectype;
4775 : 182643 : info->scale = scale;
4776 : 182643 : info->element_type = TREE_TYPE (vectype);
4777 : 182643 : info->memory_type = memory_type;
4778 : 182643 : return true;
4779 : : }
4780 : :
4781 : : /* Find the data references in STMT, analyze them with respect to LOOP and
4782 : : append them to DATAREFS. Return false if datarefs in this stmt cannot
4783 : : be handled. */
4784 : :
4785 : : opt_result
4786 : 32189106 : vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
4787 : : vec<data_reference_p> *datarefs,
4788 : : vec<int> *dataref_groups, int group_id)
4789 : : {
4790 : : /* We can ignore clobbers for dataref analysis - they are removed during
4791 : : loop vectorization and BB vectorization checks dependences with a
4792 : : stmt walk. */
4793 : 32189106 : if (gimple_clobber_p (stmt))
4794 : 1396235 : return opt_result::success ();
4795 : :
4796 : 57300647 : if (gimple_has_volatile_ops (stmt))
4797 : 317198 : return opt_result::failure_at (stmt, "not vectorized: volatile type: %G",
4798 : : stmt);
4799 : :
4800 : 30475673 : if (stmt_can_throw_internal (cfun, stmt))
4801 : 800639 : return opt_result::failure_at (stmt,
4802 : : "not vectorized:"
4803 : : " statement can throw an exception: %G",
4804 : : stmt);
4805 : :
4806 : 29675034 : auto_vec<data_reference_p, 2> refs;
4807 : 29675034 : opt_result res = find_data_references_in_stmt (loop, stmt, &refs);
4808 : 29675034 : if (!res)
4809 : 3581135 : return res;
4810 : :
4811 : 26093899 : if (refs.is_empty ())
4812 : 14821788 : return opt_result::success ();
4813 : :
4814 : 11272111 : if (refs.length () > 1)
4815 : : {
4816 : 1461220 : while (!refs.is_empty ())
4817 : 974449 : free_data_ref (refs.pop ());
4818 : 486771 : return opt_result::failure_at (stmt,
4819 : : "not vectorized: more than one "
4820 : : "data ref in stmt: %G", stmt);
4821 : : }
4822 : :
4823 : 10785340 : data_reference_p dr = refs.pop ();
4824 : 10785340 : if (gcall *call = dyn_cast <gcall *> (stmt))
4825 : 5493 : if (!gimple_call_internal_p (call)
4826 : 5493 : || (gimple_call_internal_fn (call) != IFN_MASK_LOAD
4827 : 1260 : && gimple_call_internal_fn (call) != IFN_MASK_STORE))
4828 : : {
4829 : 2662 : free_data_ref (dr);
4830 : 2662 : return opt_result::failure_at (stmt,
4831 : : "not vectorized: dr in a call %G", stmt);
4832 : : }
4833 : :
4834 : 10782678 : if (TREE_CODE (DR_REF (dr)) == COMPONENT_REF
4835 : 10782678 : && DECL_BIT_FIELD (TREE_OPERAND (DR_REF (dr), 1)))
4836 : : {
4837 : 78864 : free_data_ref (dr);
4838 : 78864 : return opt_result::failure_at (stmt,
4839 : : "not vectorized:"
4840 : : " statement is an unsupported"
4841 : : " bitfield access %G", stmt);
4842 : : }
4843 : :
4844 : 10703814 : if (DR_BASE_ADDRESS (dr)
4845 : 10667996 : && TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
4846 : : {
4847 : 953 : free_data_ref (dr);
4848 : 953 : return opt_result::failure_at (stmt,
4849 : : "not vectorized:"
4850 : : " base addr of dr is a constant\n");
4851 : : }
4852 : :
4853 : : /* Check whether this may be a SIMD lane access and adjust the
4854 : : DR to make it easier for us to handle it. */
4855 : 10702861 : if (loop
4856 : 468720 : && loop->simduid
4857 : 10702 : && (!DR_BASE_ADDRESS (dr)
4858 : 2955 : || !DR_OFFSET (dr)
4859 : 2955 : || !DR_INIT (dr)
4860 : 2955 : || !DR_STEP (dr)))
4861 : : {
4862 : 7747 : struct data_reference *newdr
4863 : 7747 : = create_data_ref (NULL, loop_containing_stmt (stmt), DR_REF (dr), stmt,
4864 : 7747 : DR_IS_READ (dr), DR_IS_CONDITIONAL_IN_STMT (dr));
4865 : 7747 : if (DR_BASE_ADDRESS (newdr)
4866 : 7747 : && DR_OFFSET (newdr)
4867 : 7747 : && DR_INIT (newdr)
4868 : 7747 : && DR_STEP (newdr)
4869 : 7747 : && TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
4870 : 15494 : && integer_zerop (DR_STEP (newdr)))
4871 : : {
4872 : 7747 : tree base_address = DR_BASE_ADDRESS (newdr);
4873 : 7747 : tree off = DR_OFFSET (newdr);
4874 : 7747 : tree step = ssize_int (1);
4875 : 7747 : if (integer_zerop (off)
4876 : 7747 : && TREE_CODE (base_address) == POINTER_PLUS_EXPR)
4877 : : {
4878 : 89 : off = TREE_OPERAND (base_address, 1);
4879 : 89 : base_address = TREE_OPERAND (base_address, 0);
4880 : : }
4881 : 7747 : STRIP_NOPS (off);
4882 : 7747 : if (TREE_CODE (off) == MULT_EXPR
4883 : 7747 : && tree_fits_uhwi_p (TREE_OPERAND (off, 1)))
4884 : : {
4885 : 7497 : step = TREE_OPERAND (off, 1);
4886 : 7497 : off = TREE_OPERAND (off, 0);
4887 : 7497 : STRIP_NOPS (off);
4888 : : }
4889 : 544 : if (CONVERT_EXPR_P (off)
4890 : 7747 : && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
4891 : 7203 : < TYPE_PRECISION (TREE_TYPE (off))))
4892 : 7203 : off = TREE_OPERAND (off, 0);
4893 : 7747 : if (TREE_CODE (off) == SSA_NAME)
4894 : : {
4895 : 7219 : gimple *def = SSA_NAME_DEF_STMT (off);
4896 : : /* Look through widening conversion. */
4897 : 7219 : if (is_gimple_assign (def)
4898 : 7219 : && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
4899 : : {
4900 : 0 : tree rhs1 = gimple_assign_rhs1 (def);
4901 : 0 : if (TREE_CODE (rhs1) == SSA_NAME
4902 : 0 : && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
4903 : 0 : && (TYPE_PRECISION (TREE_TYPE (off))
4904 : 0 : > TYPE_PRECISION (TREE_TYPE (rhs1))))
4905 : 0 : def = SSA_NAME_DEF_STMT (rhs1);
4906 : : }
4907 : 7219 : if (is_gimple_call (def)
4908 : 7080 : && gimple_call_internal_p (def)
4909 : 14299 : && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
4910 : : {
4911 : 7080 : tree arg = gimple_call_arg (def, 0);
4912 : 7080 : tree reft = TREE_TYPE (DR_REF (newdr));
4913 : 7080 : gcc_assert (TREE_CODE (arg) == SSA_NAME);
4914 : 7080 : arg = SSA_NAME_VAR (arg);
4915 : 7080 : if (arg == loop->simduid
4916 : : /* For now. */
4917 : 7080 : && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
4918 : : {
4919 : 7055 : DR_BASE_ADDRESS (newdr) = base_address;
4920 : 7055 : DR_OFFSET (newdr) = ssize_int (0);
4921 : 7055 : DR_STEP (newdr) = step;
4922 : 7055 : DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
4923 : 7055 : DR_STEP_ALIGNMENT (newdr) = highest_pow2_factor (step);
4924 : : /* Mark as simd-lane access. */
4925 : 7055 : tree arg2 = gimple_call_arg (def, 1);
4926 : 7055 : newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
4927 : 7055 : free_data_ref (dr);
4928 : 7055 : datarefs->safe_push (newdr);
4929 : 7055 : if (dataref_groups)
4930 : 0 : dataref_groups->safe_push (group_id);
4931 : 7055 : return opt_result::success ();
4932 : : }
4933 : : }
4934 : : }
4935 : : }
4936 : 692 : free_data_ref (newdr);
4937 : : }
4938 : :
4939 : 10695806 : datarefs->safe_push (dr);
4940 : 10695806 : if (dataref_groups)
4941 : 10234141 : dataref_groups->safe_push (group_id);
4942 : 10695806 : return opt_result::success ();
4943 : 29675034 : }
4944 : :
4945 : : /* Function vect_analyze_data_refs.
4946 : :
4947 : : Find all the data references in the loop or basic block.
4948 : :
4949 : : The general structure of the analysis of data refs in the vectorizer is as
4950 : : follows:
4951 : : 1- vect_analyze_data_refs(loop/bb): call
4952 : : compute_data_dependences_for_loop/bb to find and analyze all data-refs
4953 : : in the loop/bb and their dependences.
4954 : : 2- vect_analyze_dependences(): apply dependence testing using ddrs.
4955 : : 3- vect_analyze_drs_alignment(): check that ref_stmt.alignment is ok.
4956 : : 4- vect_analyze_drs_access(): check that ref_stmt.step is ok.
4957 : :
4958 : : */
4959 : :
4960 : : opt_result
4961 : 2807554 : vect_analyze_data_refs (vec_info *vinfo, poly_uint64 *min_vf, bool *fatal)
4962 : : {
4963 : 2807554 : class loop *loop = NULL;
4964 : 2807554 : unsigned int i;
4965 : 2807554 : struct data_reference *dr;
4966 : 2807554 : tree scalar_type;
4967 : :
4968 : 2807554 : DUMP_VECT_SCOPE ("vect_analyze_data_refs");
4969 : :
4970 : 2807554 : if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
4971 : 372637 : loop = LOOP_VINFO_LOOP (loop_vinfo);
4972 : :
4973 : : /* Go through the data-refs, check that the analysis succeeded. Update
4974 : : pointer from stmt_vec_info struct to DR and vectype. */
4975 : :
4976 : 2807554 : vec<data_reference_p> datarefs = vinfo->shared->datarefs;
4977 : 17643460 : FOR_EACH_VEC_ELT (datarefs, i, dr)
4978 : : {
4979 : 14888742 : enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
4980 : 14888742 : poly_uint64 vf;
4981 : :
4982 : 14888742 : gcc_assert (DR_REF (dr));
4983 : 14888742 : stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
4984 : 14888742 : gcc_assert (!stmt_info->dr_aux.dr);
4985 : 14888742 : stmt_info->dr_aux.dr = dr;
4986 : 14888742 : stmt_info->dr_aux.stmt = stmt_info;
4987 : :
4988 : : /* Check that analysis of the data-ref succeeded. */
4989 : 14888742 : if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
4990 : 14843621 : || !DR_STEP (dr))
4991 : : {
4992 : 90242 : bool maybe_gather
4993 : 45121 : = DR_IS_READ (dr)
4994 : 45121 : && !TREE_THIS_VOLATILE (DR_REF (dr));
4995 : 90242 : bool maybe_scatter
4996 : : = DR_IS_WRITE (dr)
4997 : 45121 : && !TREE_THIS_VOLATILE (DR_REF (dr));
4998 : :
4999 : : /* If target supports vector gather loads or scatter stores,
5000 : : see if they can't be used. */
5001 : 45121 : if (is_a <loop_vec_info> (vinfo)
5002 : 45121 : && !nested_in_vect_loop_p (loop, stmt_info))
5003 : : {
5004 : 41484 : if (maybe_gather || maybe_scatter)
5005 : : {
5006 : 41484 : if (maybe_gather)
5007 : : gatherscatter = GATHER;
5008 : : else
5009 : 12720 : gatherscatter = SCATTER;
5010 : : }
5011 : : }
5012 : :
5013 : 12720 : if (gatherscatter == SG_NONE)
5014 : : {
5015 : 3637 : if (dump_enabled_p ())
5016 : 5 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5017 : : "not vectorized: data ref analysis "
5018 : : "failed %G", stmt_info->stmt);
5019 : 3637 : if (is_a <bb_vec_info> (vinfo))
5020 : : {
5021 : : /* In BB vectorization the ref can still participate
5022 : : in dependence analysis, we just can't vectorize it. */
5023 : 3023 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5024 : 1699160 : continue;
5025 : : }
5026 : 614 : return opt_result::failure_at (stmt_info->stmt,
5027 : : "not vectorized:"
5028 : : " data ref analysis failed: %G",
5029 : : stmt_info->stmt);
5030 : : }
5031 : : }
5032 : :
5033 : : /* See if this was detected as SIMD lane access. */
5034 : 14885105 : if (dr->aux == (void *)-1
5035 : 14885105 : || dr->aux == (void *)-2
5036 : 14876166 : || dr->aux == (void *)-3
5037 : 14875326 : || dr->aux == (void *)-4)
5038 : : {
5039 : 10579 : if (nested_in_vect_loop_p (loop, stmt_info))
5040 : 0 : return opt_result::failure_at (stmt_info->stmt,
5041 : : "not vectorized:"
5042 : : " data ref analysis failed: %G",
5043 : : stmt_info->stmt);
5044 : 10579 : STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
5045 : 10579 : = -(uintptr_t) dr->aux;
5046 : : }
5047 : :
5048 : 14885105 : tree base = get_base_address (DR_REF (dr));
5049 : 14885105 : if (base && VAR_P (base) && DECL_NONALIASED (base))
5050 : : {
5051 : 8060 : if (dump_enabled_p ())
5052 : 186 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5053 : : "not vectorized: base object not addressable "
5054 : : "for stmt: %G", stmt_info->stmt);
5055 : 8060 : if (is_a <bb_vec_info> (vinfo))
5056 : : {
5057 : : /* In BB vectorization the ref can still participate
5058 : : in dependence analysis, we just can't vectorize it. */
5059 : 8060 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5060 : 8060 : continue;
5061 : : }
5062 : 0 : return opt_result::failure_at (stmt_info->stmt,
5063 : : "not vectorized: base object not"
5064 : : " addressable for stmt: %G",
5065 : : stmt_info->stmt);
5066 : : }
5067 : :
5068 : 14877045 : if (is_a <loop_vec_info> (vinfo)
5069 : 847291 : && DR_STEP (dr)
5070 : 15682852 : && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
5071 : : {
5072 : 42140 : if (nested_in_vect_loop_p (loop, stmt_info))
5073 : 381 : return opt_result::failure_at (stmt_info->stmt,
5074 : : "not vectorized: "
5075 : : "not suitable for strided load %G",
5076 : : stmt_info->stmt);
5077 : 41759 : STMT_VINFO_STRIDED_P (stmt_info) = true;
5078 : : }
5079 : :
5080 : : /* Update DR field in stmt_vec_info struct. */
5081 : :
5082 : : /* If the dataref is in an inner-loop of the loop that is considered for
5083 : : for vectorization, we also want to analyze the access relative to
5084 : : the outer-loop (DR contains information only relative to the
5085 : : inner-most enclosing loop). We do that by building a reference to the
5086 : : first location accessed by the inner-loop, and analyze it relative to
5087 : : the outer-loop. */
5088 : 14876664 : if (loop && nested_in_vect_loop_p (loop, stmt_info))
5089 : : {
5090 : : /* Build a reference to the first location accessed by the
5091 : : inner loop: *(BASE + INIT + OFFSET). By construction,
5092 : : this address must be invariant in the inner loop, so we
5093 : : can consider it as being used in the outer loop. */
5094 : 11138 : tree base = unshare_expr (DR_BASE_ADDRESS (dr));
5095 : 11138 : tree offset = unshare_expr (DR_OFFSET (dr));
5096 : 11138 : tree init = unshare_expr (DR_INIT (dr));
5097 : 11138 : tree init_offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset),
5098 : : init, offset);
5099 : 11138 : tree init_addr = fold_build_pointer_plus (base, init_offset);
5100 : 11138 : tree init_ref = build_fold_indirect_ref (init_addr);
5101 : :
5102 : 11138 : if (dump_enabled_p ())
5103 : 1155 : dump_printf_loc (MSG_NOTE, vect_location,
5104 : : "analyze in outer loop: %T\n", init_ref);
5105 : :
5106 : 11138 : opt_result res
5107 : 11138 : = dr_analyze_innermost (&STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info),
5108 : 11138 : init_ref, loop, stmt_info->stmt);
5109 : 11138 : if (!res)
5110 : : /* dr_analyze_innermost already explained the failure. */
5111 : 156 : return res;
5112 : :
5113 : 10982 : if (dump_enabled_p ())
5114 : 1155 : dump_printf_loc (MSG_NOTE, vect_location,
5115 : : "\touter base_address: %T\n"
5116 : : "\touter offset from base address: %T\n"
5117 : : "\touter constant offset from base address: %T\n"
5118 : : "\touter step: %T\n"
5119 : : "\touter base alignment: %d\n\n"
5120 : : "\touter base misalignment: %d\n"
5121 : : "\touter offset alignment: %d\n"
5122 : : "\touter step alignment: %d\n",
5123 : : STMT_VINFO_DR_BASE_ADDRESS (stmt_info),
5124 : : STMT_VINFO_DR_OFFSET (stmt_info),
5125 : : STMT_VINFO_DR_INIT (stmt_info),
5126 : : STMT_VINFO_DR_STEP (stmt_info),
5127 : : STMT_VINFO_DR_BASE_ALIGNMENT (stmt_info),
5128 : : STMT_VINFO_DR_BASE_MISALIGNMENT (stmt_info),
5129 : : STMT_VINFO_DR_OFFSET_ALIGNMENT (stmt_info),
5130 : : STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
5131 : : }
5132 : :
5133 : : /* Set vectype for STMT. */
5134 : 14876508 : scalar_type = TREE_TYPE (DR_REF (dr));
5135 : 14876508 : tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
5136 : 14876508 : if (!vectype)
5137 : : {
5138 : 1734112 : if (dump_enabled_p ())
5139 : : {
5140 : 1774 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5141 : : "not vectorized: no vectype for stmt: %G",
5142 : : stmt_info->stmt);
5143 : 1774 : dump_printf (MSG_MISSED_OPTIMIZATION, " scalar_type: ");
5144 : 1774 : dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_DETAILS,
5145 : : scalar_type);
5146 : 1774 : dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
5147 : : }
5148 : :
5149 : 1734112 : if (is_a <bb_vec_info> (vinfo))
5150 : : {
5151 : : /* No vector type is fine, the ref can still participate
5152 : : in dependence analysis, we just can't vectorize it. */
5153 : 1688077 : STMT_VINFO_VECTORIZABLE (stmt_info) = false;
5154 : 1688077 : continue;
5155 : : }
5156 : 46035 : if (fatal)
5157 : 46035 : *fatal = false;
5158 : 46035 : return opt_result::failure_at (stmt_info->stmt,
5159 : : "not vectorized:"
5160 : : " no vectype for stmt: %G"
5161 : : " scalar_type: %T\n",
5162 : : stmt_info->stmt, scalar_type);
5163 : : }
5164 : : else
5165 : : {
5166 : 13142396 : if (dump_enabled_p ())
5167 : 82766 : dump_printf_loc (MSG_NOTE, vect_location,
5168 : : "got vectype for stmt: %G%T\n",
5169 : : stmt_info->stmt, vectype);
5170 : : }
5171 : :
5172 : : /* Adjust the minimal vectorization factor according to the
5173 : : vector type. */
5174 : 13142396 : vf = TYPE_VECTOR_SUBPARTS (vectype);
5175 : 13142396 : *min_vf = upper_bound (*min_vf, vf);
5176 : :
5177 : : /* Leave the BB vectorizer to pick the vector type later, based on
5178 : : the final dataref group size and SLP node size. */
5179 : 13142396 : if (is_a <loop_vec_info> (vinfo))
5180 : 800719 : STMT_VINFO_VECTYPE (stmt_info) = vectype;
5181 : :
5182 : 13142396 : if (gatherscatter != SG_NONE)
5183 : : {
5184 : 39021 : gather_scatter_info gs_info;
5185 : 39021 : if (!vect_check_gather_scatter (stmt_info,
5186 : : as_a <loop_vec_info> (vinfo),
5187 : : &gs_info)
5188 : 73090 : || !get_vectype_for_scalar_type (vinfo,
5189 : 34069 : TREE_TYPE (gs_info.offset)))
5190 : : {
5191 : 5650 : if (fatal)
5192 : 5650 : *fatal = false;
5193 : 5650 : return opt_result::failure_at
5194 : 6312 : (stmt_info->stmt,
5195 : : (gatherscatter == GATHER)
5196 : : ? "not vectorized: not suitable for gather load %G"
5197 : : : "not vectorized: not suitable for scatter store %G",
5198 : : stmt_info->stmt);
5199 : : }
5200 : 33371 : STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
5201 : : }
5202 : : }
5203 : :
5204 : : /* We used to stop processing and prune the list here. Verify we no
5205 : : longer need to. */
5206 : 4293903 : gcc_assert (i == datarefs.length ());
5207 : :
5208 : 2754718 : return opt_result::success ();
5209 : : }
5210 : :
5211 : :
5212 : : /* Function vect_get_new_vect_var.
5213 : :
5214 : : Returns a name for a new variable. The current naming scheme appends the
5215 : : prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to
5216 : : the name of vectorizer generated variables, and appends that to NAME if
5217 : : provided. */
5218 : :
5219 : : tree
5220 : 1830776 : vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)
5221 : : {
5222 : 1830776 : const char *prefix;
5223 : 1830776 : tree new_vect_var;
5224 : :
5225 : 1830776 : switch (var_kind)
5226 : : {
5227 : : case vect_simple_var:
5228 : : prefix = "vect";
5229 : : break;
5230 : 22015 : case vect_scalar_var:
5231 : 22015 : prefix = "stmp";
5232 : 22015 : break;
5233 : 13599 : case vect_mask_var:
5234 : 13599 : prefix = "mask";
5235 : 13599 : break;
5236 : 1362442 : case vect_pointer_var:
5237 : 1362442 : prefix = "vectp";
5238 : 1362442 : break;
5239 : 0 : default:
5240 : 0 : gcc_unreachable ();
5241 : : }
5242 : :
5243 : 1830776 : if (name)
5244 : : {
5245 : 1006542 : char* tmp = concat (prefix, "_", name, NULL);
5246 : 1006542 : new_vect_var = create_tmp_reg (type, tmp);
5247 : 1006542 : free (tmp);
5248 : : }
5249 : : else
5250 : 824234 : new_vect_var = create_tmp_reg (type, prefix);
5251 : :
5252 : 1830776 : return new_vect_var;
5253 : : }
5254 : :
5255 : : /* Like vect_get_new_vect_var but return an SSA name. */
5256 : :
5257 : : tree
5258 : 6611 : vect_get_new_ssa_name (tree type, enum vect_var_kind var_kind, const char *name)
5259 : : {
5260 : 6611 : const char *prefix;
5261 : 6611 : tree new_vect_var;
5262 : :
5263 : 6611 : switch (var_kind)
5264 : : {
5265 : : case vect_simple_var:
5266 : : prefix = "vect";
5267 : : break;
5268 : 313 : case vect_scalar_var:
5269 : 313 : prefix = "stmp";
5270 : 313 : break;
5271 : 0 : case vect_pointer_var:
5272 : 0 : prefix = "vectp";
5273 : 0 : break;
5274 : 0 : default:
5275 : 0 : gcc_unreachable ();
5276 : : }
5277 : :
5278 : 6611 : if (name)
5279 : : {
5280 : 6125 : char* tmp = concat (prefix, "_", name, NULL);
5281 : 6125 : new_vect_var = make_temp_ssa_name (type, NULL, tmp);
5282 : 6125 : free (tmp);
5283 : : }
5284 : : else
5285 : 486 : new_vect_var = make_temp_ssa_name (type, NULL, prefix);
5286 : :
5287 : 6611 : return new_vect_var;
5288 : : }
5289 : :
5290 : : /* Duplicate points-to info on NAME from DR_INFO. */
5291 : :
5292 : : static void
5293 : 266327 : vect_duplicate_ssa_name_ptr_info (tree name, dr_vec_info *dr_info)
5294 : : {
5295 : 266327 : duplicate_ssa_name_ptr_info (name, DR_PTR_INFO (dr_info->dr));
5296 : : /* DR_PTR_INFO is for a base SSA name, not including constant or
5297 : : variable offsets in the ref so its alignment info does not apply. */
5298 : 266327 : mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (name));
5299 : 266327 : }
5300 : :
5301 : : /* Function vect_create_addr_base_for_vector_ref.
5302 : :
5303 : : Create an expression that computes the address of the first memory location
5304 : : that will be accessed for a data reference.
5305 : :
5306 : : Input:
5307 : : STMT_INFO: The statement containing the data reference.
5308 : : NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
5309 : : OFFSET: Optional. If supplied, it is be added to the initial address.
5310 : : LOOP: Specify relative to which loop-nest should the address be computed.
5311 : : For example, when the dataref is in an inner-loop nested in an
5312 : : outer-loop that is now being vectorized, LOOP can be either the
5313 : : outer-loop, or the inner-loop. The first memory location accessed
5314 : : by the following dataref ('in' points to short):
5315 : :
5316 : : for (i=0; i<N; i++)
5317 : : for (j=0; j<M; j++)
5318 : : s += in[i+j]
5319 : :
5320 : : is as follows:
5321 : : if LOOP=i_loop: &in (relative to i_loop)
5322 : : if LOOP=j_loop: &in+i*2B (relative to j_loop)
5323 : :
5324 : : Output:
5325 : : 1. Return an SSA_NAME whose value is the address of the memory location of
5326 : : the first vector of the data reference.
5327 : : 2. If new_stmt_list is not NULL_TREE after return then the caller must insert
5328 : : these statement(s) which define the returned SSA_NAME.
5329 : :
5330 : : FORNOW: We are only handling array accesses with step 1. */
5331 : :
5332 : : tree
5333 : 681303 : vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
5334 : : gimple_seq *new_stmt_list,
5335 : : tree offset)
5336 : : {
5337 : 681303 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
5338 : 681303 : struct data_reference *dr = dr_info->dr;
5339 : 681303 : const char *base_name;
5340 : 681303 : tree addr_base;
5341 : 681303 : tree dest;
5342 : 681303 : gimple_seq seq = NULL;
5343 : 681303 : tree vect_ptr_type;
5344 : 681303 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5345 : 681303 : innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
5346 : :
5347 : 681303 : tree data_ref_base = unshare_expr (drb->base_address);
5348 : 681303 : tree base_offset = unshare_expr (get_dr_vinfo_offset (vinfo, dr_info, true));
5349 : 681303 : tree init = unshare_expr (drb->init);
5350 : :
5351 : 681303 : if (loop_vinfo)
5352 : 119154 : base_name = get_name (data_ref_base);
5353 : : else
5354 : : {
5355 : 562149 : base_offset = ssize_int (0);
5356 : 562149 : init = ssize_int (0);
5357 : 562149 : base_name = get_name (DR_REF (dr));
5358 : : }
5359 : :
5360 : : /* Create base_offset */
5361 : 681303 : base_offset = size_binop (PLUS_EXPR,
5362 : : fold_convert (sizetype, base_offset),
5363 : : fold_convert (sizetype, init));
5364 : :
5365 : 681303 : if (offset)
5366 : : {
5367 : 2972 : offset = fold_convert (sizetype, offset);
5368 : 2972 : base_offset = fold_build2 (PLUS_EXPR, sizetype,
5369 : : base_offset, offset);
5370 : : }
5371 : :
5372 : : /* base + base_offset */
5373 : 681303 : if (loop_vinfo)
5374 : 119154 : addr_base = fold_build_pointer_plus (data_ref_base, base_offset);
5375 : : else
5376 : 1124298 : addr_base = build1 (ADDR_EXPR,
5377 : 562149 : build_pointer_type (TREE_TYPE (DR_REF (dr))),
5378 : : /* Strip zero offset components since we don't need
5379 : : them and they can confuse late diagnostics if
5380 : : we CSE them wrongly. See PR106904 for example. */
5381 : : unshare_expr (strip_zero_offset_components
5382 : : (DR_REF (dr))));
5383 : :
5384 : 681303 : vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
5385 : 681303 : dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
5386 : 681303 : addr_base = force_gimple_operand (addr_base, &seq, true, dest);
5387 : 681303 : gimple_seq_add_seq (new_stmt_list, seq);
5388 : :
5389 : 681303 : if (DR_PTR_INFO (dr)
5390 : 163884 : && TREE_CODE (addr_base) == SSA_NAME
5391 : : /* We should only duplicate pointer info to newly created SSA names. */
5392 : 844556 : && SSA_NAME_VAR (addr_base) == dest)
5393 : : {
5394 : 136651 : gcc_assert (!SSA_NAME_PTR_INFO (addr_base));
5395 : 136651 : vect_duplicate_ssa_name_ptr_info (addr_base, dr_info);
5396 : : }
5397 : :
5398 : 681303 : if (dump_enabled_p ())
5399 : 23751 : dump_printf_loc (MSG_NOTE, vect_location, "created %T\n", addr_base);
5400 : :
5401 : 681303 : return addr_base;
5402 : : }
5403 : :
5404 : :
5405 : : /* Function vect_create_data_ref_ptr.
5406 : :
5407 : : Create a new pointer-to-AGGR_TYPE variable (ap), that points to the first
5408 : : location accessed in the loop by STMT_INFO, along with the def-use update
5409 : : chain to appropriately advance the pointer through the loop iterations.
5410 : : Also set aliasing information for the pointer. This pointer is used by
5411 : : the callers to this function to create a memory reference expression for
5412 : : vector load/store access.
5413 : :
5414 : : Input:
5415 : : 1. STMT_INFO: a stmt that references memory. Expected to be of the form
5416 : : GIMPLE_ASSIGN <name, data-ref> or
5417 : : GIMPLE_ASSIGN <data-ref, name>.
5418 : : 2. AGGR_TYPE: the type of the reference, which should be either a vector
5419 : : or an array.
5420 : : 3. AT_LOOP: the loop where the vector memref is to be created.
5421 : : 4. OFFSET (optional): a byte offset to be added to the initial address
5422 : : accessed by the data-ref in STMT_INFO.
5423 : : 5. BSI: location where the new stmts are to be placed if there is no loop
5424 : : 6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain
5425 : : pointing to the initial address.
5426 : : 8. IV_STEP (optional, defaults to NULL): the amount that should be added
5427 : : to the IV during each iteration of the loop. NULL says to move
5428 : : by one copy of AGGR_TYPE up or down, depending on the step of the
5429 : : data reference.
5430 : :
5431 : : Output:
5432 : : 1. Declare a new ptr to vector_type, and have it point to the base of the
5433 : : data reference (initial addressed accessed by the data reference).
5434 : : For example, for vector of type V8HI, the following code is generated:
5435 : :
5436 : : v8hi *ap;
5437 : : ap = (v8hi *)initial_address;
5438 : :
5439 : : if OFFSET is not supplied:
5440 : : initial_address = &a[init];
5441 : : if OFFSET is supplied:
5442 : : initial_address = &a[init] + OFFSET;
5443 : : if BYTE_OFFSET is supplied:
5444 : : initial_address = &a[init] + BYTE_OFFSET;
5445 : :
5446 : : Return the initial_address in INITIAL_ADDRESS.
5447 : :
5448 : : 2. If ONLY_INIT is true, just return the initial pointer. Otherwise, also
5449 : : update the pointer in each iteration of the loop.
5450 : :
5451 : : Return the increment stmt that updates the pointer in PTR_INCR.
5452 : :
5453 : : 3. Return the pointer. */
5454 : :
5455 : : tree
5456 : 681139 : vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
5457 : : tree aggr_type, class loop *at_loop, tree offset,
5458 : : tree *initial_address, gimple_stmt_iterator *gsi,
5459 : : gimple **ptr_incr, bool only_init,
5460 : : tree iv_step)
5461 : : {
5462 : 681139 : const char *base_name;
5463 : 681139 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
5464 : 681139 : class loop *loop = NULL;
5465 : 681139 : bool nested_in_vect_loop = false;
5466 : 681139 : class loop *containing_loop = NULL;
5467 : 681139 : tree aggr_ptr_type;
5468 : 681139 : tree aggr_ptr;
5469 : 681139 : tree new_temp;
5470 : 681139 : gimple_seq new_stmt_list = NULL;
5471 : 681139 : edge pe = NULL;
5472 : 681139 : basic_block new_bb;
5473 : 681139 : tree aggr_ptr_init;
5474 : 681139 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
5475 : 681139 : struct data_reference *dr = dr_info->dr;
5476 : 681139 : tree aptr;
5477 : 681139 : gimple_stmt_iterator incr_gsi;
5478 : 681139 : bool insert_after;
5479 : 681139 : tree indx_before_incr, indx_after_incr;
5480 : 681139 : gimple *incr;
5481 : 681139 : bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
5482 : :
5483 : 681139 : gcc_assert (iv_step != NULL_TREE
5484 : : || TREE_CODE (aggr_type) == ARRAY_TYPE
5485 : : || TREE_CODE (aggr_type) == VECTOR_TYPE);
5486 : :
5487 : 681139 : if (loop_vinfo)
5488 : : {
5489 : 118990 : loop = LOOP_VINFO_LOOP (loop_vinfo);
5490 : 118990 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
5491 : 118990 : containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
5492 : 118990 : pe = loop_preheader_edge (loop);
5493 : : }
5494 : : else
5495 : : {
5496 : 562149 : gcc_assert (bb_vinfo);
5497 : 562149 : only_init = true;
5498 : 562149 : *ptr_incr = NULL;
5499 : : }
5500 : :
5501 : : /* Create an expression for the first address accessed by this load
5502 : : in LOOP. */
5503 : 681139 : base_name = get_name (DR_BASE_ADDRESS (dr));
5504 : :
5505 : 681139 : if (dump_enabled_p ())
5506 : : {
5507 : 23716 : tree dr_base_type = TREE_TYPE (DR_BASE_OBJECT (dr));
5508 : 23716 : dump_printf_loc (MSG_NOTE, vect_location,
5509 : : "create %s-pointer variable to type: %T",
5510 : 23716 : get_tree_code_name (TREE_CODE (aggr_type)),
5511 : : aggr_type);
5512 : 23716 : if (TREE_CODE (dr_base_type) == ARRAY_TYPE)
5513 : 13016 : dump_printf (MSG_NOTE, " vectorizing an array ref: ");
5514 : 10700 : else if (TREE_CODE (dr_base_type) == VECTOR_TYPE)
5515 : 0 : dump_printf (MSG_NOTE, " vectorizing a vector ref: ");
5516 : 10700 : else if (TREE_CODE (dr_base_type) == RECORD_TYPE)
5517 : 1550 : dump_printf (MSG_NOTE, " vectorizing a record based array ref: ");
5518 : : else
5519 : 9150 : dump_printf (MSG_NOTE, " vectorizing a pointer ref: ");
5520 : 23716 : dump_printf (MSG_NOTE, "%T\n", DR_BASE_OBJECT (dr));
5521 : : }
5522 : :
5523 : : /* (1) Create the new aggregate-pointer variable.
5524 : : Vector and array types inherit the alias set of their component
5525 : : type by default so we need to use a ref-all pointer if the data
5526 : : reference does not conflict with the created aggregated data
5527 : : reference because it is not addressable. */
5528 : 681139 : bool need_ref_all = false;
5529 : 681139 : if (!alias_sets_conflict_p (get_alias_set (aggr_type),
5530 : : get_alias_set (DR_REF (dr))))
5531 : : need_ref_all = true;
5532 : : /* Likewise for any of the data references in the stmt group. */
5533 : 565046 : else if (DR_GROUP_SIZE (stmt_info) > 1)
5534 : : {
5535 : 459776 : stmt_vec_info sinfo = DR_GROUP_FIRST_ELEMENT (stmt_info);
5536 : 1263550 : do
5537 : : {
5538 : 1263550 : struct data_reference *sdr = STMT_VINFO_DATA_REF (sinfo);
5539 : 1263550 : if (!alias_sets_conflict_p (get_alias_set (aggr_type),
5540 : : get_alias_set (DR_REF (sdr))))
5541 : : {
5542 : : need_ref_all = true;
5543 : : break;
5544 : : }
5545 : 1261701 : sinfo = DR_GROUP_NEXT_ELEMENT (sinfo);
5546 : : }
5547 : 1261701 : while (sinfo);
5548 : : }
5549 : 681139 : aggr_ptr_type = build_pointer_type_for_mode (aggr_type, VOIDmode,
5550 : : need_ref_all);
5551 : 681139 : aggr_ptr = vect_get_new_vect_var (aggr_ptr_type, vect_pointer_var, base_name);
5552 : :
5553 : :
5554 : : /* Note: If the dataref is in an inner-loop nested in LOOP, and we are
5555 : : vectorizing LOOP (i.e., outer-loop vectorization), we need to create two
5556 : : def-use update cycles for the pointer: one relative to the outer-loop
5557 : : (LOOP), which is what steps (3) and (4) below do. The other is relative
5558 : : to the inner-loop (which is the inner-most loop containing the dataref),
5559 : : and this is done be step (5) below.
5560 : :
5561 : : When vectorizing inner-most loops, the vectorized loop (LOOP) is also the
5562 : : inner-most loop, and so steps (3),(4) work the same, and step (5) is
5563 : : redundant. Steps (3),(4) create the following:
5564 : :
5565 : : vp0 = &base_addr;
5566 : : LOOP: vp1 = phi(vp0,vp2)
5567 : : ...
5568 : : ...
5569 : : vp2 = vp1 + step
5570 : : goto LOOP
5571 : :
5572 : : If there is an inner-loop nested in loop, then step (5) will also be
5573 : : applied, and an additional update in the inner-loop will be created:
5574 : :
5575 : : vp0 = &base_addr;
5576 : : LOOP: vp1 = phi(vp0,vp2)
5577 : : ...
5578 : : inner: vp3 = phi(vp1,vp4)
5579 : : vp4 = vp3 + inner_step
5580 : : if () goto inner
5581 : : ...
5582 : : vp2 = vp1 + step
5583 : : if () goto LOOP */
5584 : :
5585 : : /* (2) Calculate the initial address of the aggregate-pointer, and set
5586 : : the aggregate-pointer to point to it before the loop. */
5587 : :
5588 : : /* Create: (&(base[init_val]+offset) in the loop preheader. */
5589 : :
5590 : 681139 : new_temp = vect_create_addr_base_for_vector_ref (vinfo,
5591 : : stmt_info, &new_stmt_list,
5592 : : offset);
5593 : 681139 : if (new_stmt_list)
5594 : : {
5595 : 158997 : if (pe)
5596 : : {
5597 : 52168 : new_bb = gsi_insert_seq_on_edge_immediate (pe, new_stmt_list);
5598 : 52168 : gcc_assert (!new_bb);
5599 : : }
5600 : : else
5601 : 106829 : gsi_insert_seq_before (gsi, new_stmt_list, GSI_SAME_STMT);
5602 : : }
5603 : :
5604 : 681139 : *initial_address = new_temp;
5605 : 681139 : aggr_ptr_init = new_temp;
5606 : :
5607 : : /* (3) Handle the updating of the aggregate-pointer inside the loop.
5608 : : This is needed when ONLY_INIT is false, and also when AT_LOOP is the
5609 : : inner-loop nested in LOOP (during outer-loop vectorization). */
5610 : :
5611 : : /* No update in loop is required. */
5612 : 681139 : if (only_init && (!loop_vinfo || at_loop == loop))
5613 : : aptr = aggr_ptr_init;
5614 : : else
5615 : : {
5616 : : /* Accesses to invariant addresses should be handled specially
5617 : : by the caller. */
5618 : 118982 : tree step = vect_dr_behavior (vinfo, dr_info)->step;
5619 : 118982 : gcc_assert (!integer_zerop (step));
5620 : :
5621 : 118982 : if (iv_step == NULL_TREE)
5622 : : {
5623 : : /* The step of the aggregate pointer is the type size,
5624 : : negated for downward accesses. */
5625 : 0 : iv_step = TYPE_SIZE_UNIT (aggr_type);
5626 : 0 : if (tree_int_cst_sgn (step) == -1)
5627 : 0 : iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
5628 : : }
5629 : :
5630 : 118982 : standard_iv_increment_position (loop, &incr_gsi, &insert_after);
5631 : :
5632 : 118982 : create_iv (aggr_ptr_init, PLUS_EXPR,
5633 : : fold_convert (aggr_ptr_type, iv_step),
5634 : : aggr_ptr, loop, &incr_gsi, insert_after,
5635 : : &indx_before_incr, &indx_after_incr);
5636 : 118982 : incr = gsi_stmt (incr_gsi);
5637 : :
5638 : : /* Copy the points-to information if it exists. */
5639 : 118982 : if (DR_PTR_INFO (dr))
5640 : : {
5641 : 64763 : vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
5642 : 64763 : vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
5643 : : }
5644 : 118982 : if (ptr_incr)
5645 : 118982 : *ptr_incr = incr;
5646 : :
5647 : 118982 : aptr = indx_before_incr;
5648 : : }
5649 : :
5650 : 681139 : if (!nested_in_vect_loop || only_init)
5651 : : return aptr;
5652 : :
5653 : :
5654 : : /* (4) Handle the updating of the aggregate-pointer inside the inner-loop
5655 : : nested in LOOP, if exists. */
5656 : :
5657 : 339 : gcc_assert (nested_in_vect_loop);
5658 : 339 : if (!only_init)
5659 : : {
5660 : 339 : standard_iv_increment_position (containing_loop, &incr_gsi,
5661 : : &insert_after);
5662 : 339 : create_iv (aptr, PLUS_EXPR, fold_convert (aggr_ptr_type, DR_STEP (dr)),
5663 : : aggr_ptr, containing_loop, &incr_gsi, insert_after,
5664 : : &indx_before_incr, &indx_after_incr);
5665 : 339 : incr = gsi_stmt (incr_gsi);
5666 : :
5667 : : /* Copy the points-to information if it exists. */
5668 : 339 : if (DR_PTR_INFO (dr))
5669 : : {
5670 : 75 : vect_duplicate_ssa_name_ptr_info (indx_before_incr, dr_info);
5671 : 75 : vect_duplicate_ssa_name_ptr_info (indx_after_incr, dr_info);
5672 : : }
5673 : 339 : if (ptr_incr)
5674 : 339 : *ptr_incr = incr;
5675 : :
5676 : 339 : return indx_before_incr;
5677 : : }
5678 : : else
5679 : : gcc_unreachable ();
5680 : : }
5681 : :
5682 : :
5683 : : /* Function bump_vector_ptr
5684 : :
5685 : : Increment a pointer (to a vector type) by vector-size. If requested,
5686 : : i.e. if PTR-INCR is given, then also connect the new increment stmt
5687 : : to the existing def-use update-chain of the pointer, by modifying
5688 : : the PTR_INCR as illustrated below:
5689 : :
5690 : : The pointer def-use update-chain before this function:
5691 : : DATAREF_PTR = phi (p_0, p_2)
5692 : : ....
5693 : : PTR_INCR: p_2 = DATAREF_PTR + step
5694 : :
5695 : : The pointer def-use update-chain after this function:
5696 : : DATAREF_PTR = phi (p_0, p_2)
5697 : : ....
5698 : : NEW_DATAREF_PTR = DATAREF_PTR + BUMP
5699 : : ....
5700 : : PTR_INCR: p_2 = NEW_DATAREF_PTR + step
5701 : :
5702 : : Input:
5703 : : DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated
5704 : : in the loop.
5705 : : PTR_INCR - optional. The stmt that updates the pointer in each iteration of
5706 : : the loop. The increment amount across iterations is expected
5707 : : to be vector_size.
5708 : : BSI - location where the new update stmt is to be placed.
5709 : : STMT_INFO - the original scalar memory-access stmt that is being vectorized.
5710 : : BUMP - optional. The offset by which to bump the pointer. If not given,
5711 : : the offset is assumed to be vector_size.
5712 : :
5713 : : Output: Return NEW_DATAREF_PTR as illustrated above.
5714 : :
5715 : : */
5716 : :
5717 : : tree
5718 : 200247 : bump_vector_ptr (vec_info *vinfo,
5719 : : tree dataref_ptr, gimple *ptr_incr, gimple_stmt_iterator *gsi,
5720 : : stmt_vec_info stmt_info, tree bump)
5721 : : {
5722 : 200247 : struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
5723 : 200247 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
5724 : 200247 : tree update = TYPE_SIZE_UNIT (vectype);
5725 : 200247 : gimple *incr_stmt;
5726 : 200247 : ssa_op_iter iter;
5727 : 200247 : use_operand_p use_p;
5728 : 200247 : tree new_dataref_ptr;
5729 : :
5730 : 200247 : if (bump)
5731 : 200247 : update = bump;
5732 : :
5733 : 200247 : if (TREE_CODE (dataref_ptr) == SSA_NAME)
5734 : 79612 : new_dataref_ptr = copy_ssa_name (dataref_ptr);
5735 : 120635 : else if (is_gimple_min_invariant (dataref_ptr))
5736 : : /* When possible avoid emitting a separate increment stmt that will
5737 : : force the addressed object addressable. */
5738 : 241270 : return build1 (ADDR_EXPR, TREE_TYPE (dataref_ptr),
5739 : 120635 : fold_build2 (MEM_REF,
5740 : : TREE_TYPE (TREE_TYPE (dataref_ptr)),
5741 : : dataref_ptr,
5742 : 120635 : fold_convert (ptr_type_node, update)));
5743 : : else
5744 : 0 : new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
5745 : 79612 : incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
5746 : : dataref_ptr, update);
5747 : 79612 : vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
5748 : : /* Fold the increment, avoiding excessive chains use-def chains of
5749 : : those, leading to compile-time issues for passes until the next
5750 : : forwprop pass which would do this as well. */
5751 : 79612 : gimple_stmt_iterator fold_gsi = gsi_for_stmt (incr_stmt);
5752 : 79612 : if (fold_stmt (&fold_gsi, follow_all_ssa_edges))
5753 : : {
5754 : 43382 : incr_stmt = gsi_stmt (fold_gsi);
5755 : 43382 : update_stmt (incr_stmt);
5756 : : }
5757 : :
5758 : : /* Copy the points-to information if it exists. */
5759 : 79612 : if (DR_PTR_INFO (dr))
5760 : : {
5761 : 53906 : duplicate_ssa_name_ptr_info (new_dataref_ptr, DR_PTR_INFO (dr));
5762 : 53906 : mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (new_dataref_ptr));
5763 : : }
5764 : :
5765 : 79612 : if (!ptr_incr)
5766 : : return new_dataref_ptr;
5767 : :
5768 : : /* Update the vector-pointer's cross-iteration increment. */
5769 : 76690 : FOR_EACH_SSA_USE_OPERAND (use_p, ptr_incr, iter, SSA_OP_USE)
5770 : : {
5771 : 38345 : tree use = USE_FROM_PTR (use_p);
5772 : :
5773 : 38345 : if (use == dataref_ptr)
5774 : 38345 : SET_USE (use_p, new_dataref_ptr);
5775 : : else
5776 : 0 : gcc_assert (operand_equal_p (use, update, 0));
5777 : : }
5778 : :
5779 : : return new_dataref_ptr;
5780 : : }
5781 : :
5782 : :
5783 : : /* Copy memory reference info such as base/clique from the SRC reference
5784 : : to the DEST MEM_REF. */
5785 : :
5786 : : void
5787 : 892627 : vect_copy_ref_info (tree dest, tree src)
5788 : : {
5789 : 892627 : if (TREE_CODE (dest) != MEM_REF)
5790 : : return;
5791 : :
5792 : : tree src_base = src;
5793 : 1828073 : while (handled_component_p (src_base))
5794 : 935971 : src_base = TREE_OPERAND (src_base, 0);
5795 : 892102 : if (TREE_CODE (src_base) != MEM_REF
5796 : 892102 : && TREE_CODE (src_base) != TARGET_MEM_REF)
5797 : : return;
5798 : :
5799 : 469552 : MR_DEPENDENCE_CLIQUE (dest) = MR_DEPENDENCE_CLIQUE (src_base);
5800 : 469552 : MR_DEPENDENCE_BASE (dest) = MR_DEPENDENCE_BASE (src_base);
5801 : : }
5802 : :
5803 : :
5804 : : /* Function vect_create_destination_var.
5805 : :
5806 : : Create a new temporary of type VECTYPE. */
5807 : :
5808 : : tree
5809 : 451892 : vect_create_destination_var (tree scalar_dest, tree vectype)
5810 : : {
5811 : 451892 : tree vec_dest;
5812 : 451892 : const char *name;
5813 : 451892 : char *new_name;
5814 : 451892 : tree type;
5815 : 451892 : enum vect_var_kind kind;
5816 : :
5817 : 925799 : kind = vectype
5818 : 881769 : ? VECTOR_BOOLEAN_TYPE_P (vectype)
5819 : 429877 : ? vect_mask_var
5820 : : : vect_simple_var
5821 : : : vect_scalar_var;
5822 : 22015 : type = vectype ? vectype : TREE_TYPE (scalar_dest);
5823 : :
5824 : 451892 : gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);
5825 : :
5826 : 451892 : name = get_name (scalar_dest);
5827 : 451892 : if (name)
5828 : 164013 : new_name = xasprintf ("%s_%u", name, SSA_NAME_VERSION (scalar_dest));
5829 : : else
5830 : 287879 : new_name = xasprintf ("_%u", SSA_NAME_VERSION (scalar_dest));
5831 : 451892 : vec_dest = vect_get_new_vect_var (type, kind, new_name);
5832 : 451892 : free (new_name);
5833 : :
5834 : 451892 : return vec_dest;
5835 : : }
5836 : :
5837 : : /* Function vect_grouped_store_supported.
5838 : :
5839 : : Returns TRUE if interleave high and interleave low permutations
5840 : : are supported, and FALSE otherwise. */
5841 : :
5842 : : bool
5843 : 2754 : vect_grouped_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
5844 : : {
5845 : 2754 : machine_mode mode = TYPE_MODE (vectype);
5846 : :
5847 : : /* vect_permute_store_chain requires the group size to be equal to 3 or
5848 : : be a power of two. */
5849 : 2754 : if (count != 3 && exact_log2 (count) == -1)
5850 : : {
5851 : 567 : if (dump_enabled_p ())
5852 : 9 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5853 : : "the size of the group of accesses"
5854 : : " is not a power of 2 or not eqaul to 3\n");
5855 : 567 : return false;
5856 : : }
5857 : :
5858 : : /* Check that the permutation is supported. */
5859 : 2187 : if (VECTOR_MODE_P (mode))
5860 : : {
5861 : 2187 : unsigned int i;
5862 : 2187 : if (count == 3)
5863 : : {
5864 : 987 : unsigned int j0 = 0, j1 = 0, j2 = 0;
5865 : 987 : unsigned int i, j;
5866 : :
5867 : 987 : unsigned int nelt;
5868 : 1974 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5869 : : {
5870 : : if (dump_enabled_p ())
5871 : : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
5872 : : "cannot handle groups of 3 stores for"
5873 : : " variable-length vectors\n");
5874 : : return false;
5875 : : }
5876 : :
5877 : 987 : vec_perm_builder sel (nelt, nelt, 1);
5878 : 987 : sel.quick_grow (nelt);
5879 : 987 : vec_perm_indices indices;
5880 : 3693 : for (j = 0; j < 3; j++)
5881 : : {
5882 : 2791 : int nelt0 = ((3 - j) * nelt) % 3;
5883 : 2791 : int nelt1 = ((3 - j) * nelt + 1) % 3;
5884 : 2791 : int nelt2 = ((3 - j) * nelt + 2) % 3;
5885 : 10179 : for (i = 0; i < nelt; i++)
5886 : : {
5887 : 7388 : if (3 * i + nelt0 < nelt)
5888 : 2504 : sel[3 * i + nelt0] = j0++;
5889 : 7388 : if (3 * i + nelt1 < nelt)
5890 : 2465 : sel[3 * i + nelt1] = nelt + j1++;
5891 : 7388 : if (3 * i + nelt2 < nelt)
5892 : 2419 : sel[3 * i + nelt2] = 0;
5893 : : }
5894 : 2791 : indices.new_vector (sel, 2, nelt);
5895 : 2791 : if (!can_vec_perm_const_p (mode, mode, indices))
5896 : : {
5897 : 76 : if (dump_enabled_p ())
5898 : 47 : dump_printf (MSG_MISSED_OPTIMIZATION,
5899 : : "permutation op not supported by target.\n");
5900 : 76 : return false;
5901 : : }
5902 : :
5903 : 9447 : for (i = 0; i < nelt; i++)
5904 : : {
5905 : 6732 : if (3 * i + nelt0 < nelt)
5906 : 2250 : sel[3 * i + nelt0] = 3 * i + nelt0;
5907 : 6732 : if (3 * i + nelt1 < nelt)
5908 : 2241 : sel[3 * i + nelt1] = 3 * i + nelt1;
5909 : 6732 : if (3 * i + nelt2 < nelt)
5910 : 2241 : sel[3 * i + nelt2] = nelt + j2++;
5911 : : }
5912 : 2715 : indices.new_vector (sel, 2, nelt);
5913 : 2715 : if (!can_vec_perm_const_p (mode, mode, indices))
5914 : : {
5915 : 9 : if (dump_enabled_p ())
5916 : 9 : dump_printf (MSG_MISSED_OPTIMIZATION,
5917 : : "permutation op not supported by target.\n");
5918 : 9 : return false;
5919 : : }
5920 : : }
5921 : : return true;
5922 : 987 : }
5923 : : else
5924 : : {
5925 : : /* If length is not equal to 3 then only power of 2 is supported. */
5926 : 1200 : gcc_assert (pow2p_hwi (count));
5927 : 2400 : poly_uint64 nelt = GET_MODE_NUNITS (mode);
5928 : :
5929 : : /* The encoding has 2 interleaved stepped patterns. */
5930 : 2400 : if(!multiple_p (nelt, 2))
5931 : 1134 : return false;
5932 : 1200 : vec_perm_builder sel (nelt, 2, 3);
5933 : 1200 : sel.quick_grow (6);
5934 : 6000 : for (i = 0; i < 3; i++)
5935 : : {
5936 : 3600 : sel[i * 2] = i;
5937 : 3600 : sel[i * 2 + 1] = i + nelt;
5938 : : }
5939 : 1200 : vec_perm_indices indices (sel, 2, nelt);
5940 : 1200 : if (can_vec_perm_const_p (mode, mode, indices))
5941 : : {
5942 : 7938 : for (i = 0; i < 6; i++)
5943 : 6804 : sel[i] += exact_div (nelt, 2);
5944 : 1134 : indices.new_vector (sel, 2, nelt);
5945 : 1134 : if (can_vec_perm_const_p (mode, mode, indices))
5946 : 1134 : return true;
5947 : : }
5948 : 1200 : }
5949 : : }
5950 : :
5951 : 66 : if (dump_enabled_p ())
5952 : 5 : dump_printf (MSG_MISSED_OPTIMIZATION,
5953 : : "permutation op not supported by target.\n");
5954 : : return false;
5955 : : }
5956 : :
5957 : : /* Return FN if vec_{mask_,mask_len_}store_lanes is available for COUNT vectors
5958 : : of type VECTYPE. MASKED_P says whether the masked form is needed. */
5959 : :
5960 : : internal_fn
5961 : 25502 : vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
5962 : : bool masked_p)
5963 : : {
5964 : 25502 : if (vect_lanes_optab_supported_p ("vec_mask_len_store_lanes",
5965 : : vec_mask_len_store_lanes_optab, vectype,
5966 : : count))
5967 : : return IFN_MASK_LEN_STORE_LANES;
5968 : 25502 : else if (masked_p)
5969 : : {
5970 : 160 : if (vect_lanes_optab_supported_p ("vec_mask_store_lanes",
5971 : : vec_mask_store_lanes_optab, vectype,
5972 : : count))
5973 : : return IFN_MASK_STORE_LANES;
5974 : : }
5975 : : else
5976 : : {
5977 : 25342 : if (vect_lanes_optab_supported_p ("vec_store_lanes",
5978 : : vec_store_lanes_optab, vectype, count))
5979 : : return IFN_STORE_LANES;
5980 : : }
5981 : : return IFN_LAST;
5982 : : }
5983 : :
5984 : :
5985 : : /* Function vect_permute_store_chain.
5986 : :
5987 : : Given a chain of interleaved stores in DR_CHAIN of LENGTH that must be
5988 : : a power of 2 or equal to 3, generate interleave_high/low stmts to reorder
5989 : : the data correctly for the stores. Return the final references for stores
5990 : : in RESULT_CHAIN.
5991 : :
5992 : : E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
5993 : : The input is 4 vectors each containing 8 elements. We assign a number to
5994 : : each element, the input sequence is:
5995 : :
5996 : : 1st vec: 0 1 2 3 4 5 6 7
5997 : : 2nd vec: 8 9 10 11 12 13 14 15
5998 : : 3rd vec: 16 17 18 19 20 21 22 23
5999 : : 4th vec: 24 25 26 27 28 29 30 31
6000 : :
6001 : : The output sequence should be:
6002 : :
6003 : : 1st vec: 0 8 16 24 1 9 17 25
6004 : : 2nd vec: 2 10 18 26 3 11 19 27
6005 : : 3rd vec: 4 12 20 28 5 13 21 30
6006 : : 4th vec: 6 14 22 30 7 15 23 31
6007 : :
6008 : : i.e., we interleave the contents of the four vectors in their order.
6009 : :
6010 : : We use interleave_high/low instructions to create such output. The input of
6011 : : each interleave_high/low operation is two vectors:
6012 : : 1st vec 2nd vec
6013 : : 0 1 2 3 4 5 6 7
6014 : : the even elements of the result vector are obtained left-to-right from the
6015 : : high/low elements of the first vector. The odd elements of the result are
6016 : : obtained left-to-right from the high/low elements of the second vector.
6017 : : The output of interleave_high will be: 0 4 1 5
6018 : : and of interleave_low: 2 6 3 7
6019 : :
6020 : :
6021 : : The permutation is done in log LENGTH stages. In each stage interleave_high
6022 : : and interleave_low stmts are created for each pair of vectors in DR_CHAIN,
6023 : : where the first argument is taken from the first half of DR_CHAIN and the
6024 : : second argument from it's second half.
6025 : : In our example,
6026 : :
6027 : : I1: interleave_high (1st vec, 3rd vec)
6028 : : I2: interleave_low (1st vec, 3rd vec)
6029 : : I3: interleave_high (2nd vec, 4th vec)
6030 : : I4: interleave_low (2nd vec, 4th vec)
6031 : :
6032 : : The output for the first stage is:
6033 : :
6034 : : I1: 0 16 1 17 2 18 3 19
6035 : : I2: 4 20 5 21 6 22 7 23
6036 : : I3: 8 24 9 25 10 26 11 27
6037 : : I4: 12 28 13 29 14 30 15 31
6038 : :
6039 : : The output of the second stage, i.e. the final result is:
6040 : :
6041 : : I1: 0 8 16 24 1 9 17 25
6042 : : I2: 2 10 18 26 3 11 19 27
6043 : : I3: 4 12 20 28 5 13 21 30
6044 : : I4: 6 14 22 30 7 15 23 31. */
6045 : :
6046 : : void
6047 : 0 : vect_permute_store_chain (vec_info *vinfo, vec<tree> &dr_chain,
6048 : : unsigned int length,
6049 : : stmt_vec_info stmt_info,
6050 : : gimple_stmt_iterator *gsi,
6051 : : vec<tree> *result_chain)
6052 : : {
6053 : 0 : tree vect1, vect2, high, low;
6054 : 0 : gimple *perm_stmt;
6055 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6056 : 0 : tree perm_mask_low, perm_mask_high;
6057 : 0 : tree data_ref;
6058 : 0 : tree perm3_mask_low, perm3_mask_high;
6059 : 0 : unsigned int i, j, n, log_length = exact_log2 (length);
6060 : :
6061 : 0 : result_chain->quick_grow (length);
6062 : 0 : memcpy (result_chain->address (), dr_chain.address (),
6063 : : length * sizeof (tree));
6064 : :
6065 : 0 : if (length == 3)
6066 : : {
6067 : : /* vect_grouped_store_supported ensures that this is constant. */
6068 : 0 : unsigned int nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
6069 : 0 : unsigned int j0 = 0, j1 = 0, j2 = 0;
6070 : :
6071 : 0 : vec_perm_builder sel (nelt, nelt, 1);
6072 : 0 : sel.quick_grow (nelt);
6073 : 0 : vec_perm_indices indices;
6074 : 0 : for (j = 0; j < 3; j++)
6075 : : {
6076 : 0 : int nelt0 = ((3 - j) * nelt) % 3;
6077 : 0 : int nelt1 = ((3 - j) * nelt + 1) % 3;
6078 : 0 : int nelt2 = ((3 - j) * nelt + 2) % 3;
6079 : :
6080 : 0 : for (i = 0; i < nelt; i++)
6081 : : {
6082 : 0 : if (3 * i + nelt0 < nelt)
6083 : 0 : sel[3 * i + nelt0] = j0++;
6084 : 0 : if (3 * i + nelt1 < nelt)
6085 : 0 : sel[3 * i + nelt1] = nelt + j1++;
6086 : 0 : if (3 * i + nelt2 < nelt)
6087 : 0 : sel[3 * i + nelt2] = 0;
6088 : : }
6089 : 0 : indices.new_vector (sel, 2, nelt);
6090 : 0 : perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
6091 : :
6092 : 0 : for (i = 0; i < nelt; i++)
6093 : : {
6094 : 0 : if (3 * i + nelt0 < nelt)
6095 : 0 : sel[3 * i + nelt0] = 3 * i + nelt0;
6096 : 0 : if (3 * i + nelt1 < nelt)
6097 : 0 : sel[3 * i + nelt1] = 3 * i + nelt1;
6098 : 0 : if (3 * i + nelt2 < nelt)
6099 : 0 : sel[3 * i + nelt2] = nelt + j2++;
6100 : : }
6101 : 0 : indices.new_vector (sel, 2, nelt);
6102 : 0 : perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
6103 : :
6104 : 0 : vect1 = dr_chain[0];
6105 : 0 : vect2 = dr_chain[1];
6106 : :
6107 : : /* Create interleaving stmt:
6108 : : low = VEC_PERM_EXPR <vect1, vect2,
6109 : : {j, nelt, *, j + 1, nelt + j + 1, *,
6110 : : j + 2, nelt + j + 2, *, ...}> */
6111 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
6112 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
6113 : : vect2, perm3_mask_low);
6114 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6115 : :
6116 : 0 : vect1 = data_ref;
6117 : 0 : vect2 = dr_chain[2];
6118 : : /* Create interleaving stmt:
6119 : : low = VEC_PERM_EXPR <vect1, vect2,
6120 : : {0, 1, nelt + j, 3, 4, nelt + j + 1,
6121 : : 6, 7, nelt + j + 2, ...}> */
6122 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
6123 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect1,
6124 : : vect2, perm3_mask_high);
6125 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6126 : 0 : (*result_chain)[j] = data_ref;
6127 : : }
6128 : 0 : }
6129 : : else
6130 : : {
6131 : : /* If length is not equal to 3 then only power of 2 is supported. */
6132 : 0 : gcc_assert (pow2p_hwi (length));
6133 : :
6134 : : /* The encoding has 2 interleaved stepped patterns. */
6135 : 0 : poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
6136 : 0 : vec_perm_builder sel (nelt, 2, 3);
6137 : 0 : sel.quick_grow (6);
6138 : 0 : for (i = 0; i < 3; i++)
6139 : : {
6140 : 0 : sel[i * 2] = i;
6141 : 0 : sel[i * 2 + 1] = i + nelt;
6142 : : }
6143 : 0 : vec_perm_indices indices (sel, 2, nelt);
6144 : 0 : perm_mask_high = vect_gen_perm_mask_checked (vectype, indices);
6145 : :
6146 : 0 : for (i = 0; i < 6; i++)
6147 : 0 : sel[i] += exact_div (nelt, 2);
6148 : 0 : indices.new_vector (sel, 2, nelt);
6149 : 0 : perm_mask_low = vect_gen_perm_mask_checked (vectype, indices);
6150 : :
6151 : 0 : for (i = 0, n = log_length; i < n; i++)
6152 : : {
6153 : 0 : for (j = 0; j < length/2; j++)
6154 : : {
6155 : 0 : vect1 = dr_chain[j];
6156 : 0 : vect2 = dr_chain[j+length/2];
6157 : :
6158 : : /* Create interleaving stmt:
6159 : : high = VEC_PERM_EXPR <vect1, vect2, {0, nelt, 1, nelt+1,
6160 : : ...}> */
6161 : 0 : high = make_temp_ssa_name (vectype, NULL, "vect_inter_high");
6162 : 0 : perm_stmt = gimple_build_assign (high, VEC_PERM_EXPR, vect1,
6163 : : vect2, perm_mask_high);
6164 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6165 : 0 : (*result_chain)[2*j] = high;
6166 : :
6167 : : /* Create interleaving stmt:
6168 : : low = VEC_PERM_EXPR <vect1, vect2,
6169 : : {nelt/2, nelt*3/2, nelt/2+1, nelt*3/2+1,
6170 : : ...}> */
6171 : 0 : low = make_temp_ssa_name (vectype, NULL, "vect_inter_low");
6172 : 0 : perm_stmt = gimple_build_assign (low, VEC_PERM_EXPR, vect1,
6173 : : vect2, perm_mask_low);
6174 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6175 : 0 : (*result_chain)[2*j+1] = low;
6176 : : }
6177 : 0 : memcpy (dr_chain.address (), result_chain->address (),
6178 : : length * sizeof (tree));
6179 : : }
6180 : 0 : }
6181 : 0 : }
6182 : :
6183 : : /* Function vect_setup_realignment
6184 : :
6185 : : This function is called when vectorizing an unaligned load using
6186 : : the dr_explicit_realign[_optimized] scheme.
6187 : : This function generates the following code at the loop prolog:
6188 : :
6189 : : p = initial_addr;
6190 : : x msq_init = *(floor(p)); # prolog load
6191 : : realignment_token = call target_builtin;
6192 : : loop:
6193 : : x msq = phi (msq_init, ---)
6194 : :
6195 : : The stmts marked with x are generated only for the case of
6196 : : dr_explicit_realign_optimized.
6197 : :
6198 : : The code above sets up a new (vector) pointer, pointing to the first
6199 : : location accessed by STMT_INFO, and a "floor-aligned" load using that
6200 : : pointer. It also generates code to compute the "realignment-token"
6201 : : (if the relevant target hook was defined), and creates a phi-node at the
6202 : : loop-header bb whose arguments are the result of the prolog-load (created
6203 : : by this function) and the result of a load that takes place in the loop
6204 : : (to be created by the caller to this function).
6205 : :
6206 : : For the case of dr_explicit_realign_optimized:
6207 : : The caller to this function uses the phi-result (msq) to create the
6208 : : realignment code inside the loop, and sets up the missing phi argument,
6209 : : as follows:
6210 : : loop:
6211 : : msq = phi (msq_init, lsq)
6212 : : lsq = *(floor(p')); # load in loop
6213 : : result = realign_load (msq, lsq, realignment_token);
6214 : :
6215 : : For the case of dr_explicit_realign:
6216 : : loop:
6217 : : msq = *(floor(p)); # load in loop
6218 : : p' = p + (VS-1);
6219 : : lsq = *(floor(p')); # load in loop
6220 : : result = realign_load (msq, lsq, realignment_token);
6221 : :
6222 : : Input:
6223 : : STMT_INFO - (scalar) load stmt to be vectorized. This load accesses
6224 : : a memory location that may be unaligned.
6225 : : BSI - place where new code is to be inserted.
6226 : : ALIGNMENT_SUPPORT_SCHEME - which of the two misalignment handling schemes
6227 : : is used.
6228 : :
6229 : : Output:
6230 : : REALIGNMENT_TOKEN - the result of a call to the builtin_mask_for_load
6231 : : target hook, if defined.
6232 : : Return value - the result of the loop-header phi node. */
6233 : :
6234 : : tree
6235 : 0 : vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info,
6236 : : gimple_stmt_iterator *gsi, tree *realignment_token,
6237 : : enum dr_alignment_support alignment_support_scheme,
6238 : : tree init_addr,
6239 : : class loop **at_loop)
6240 : : {
6241 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6242 : 0 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6243 : 0 : dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
6244 : 0 : struct data_reference *dr = dr_info->dr;
6245 : 0 : class loop *loop = NULL;
6246 : 0 : edge pe = NULL;
6247 : 0 : tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
6248 : 0 : tree vec_dest;
6249 : 0 : gimple *inc;
6250 : 0 : tree ptr;
6251 : 0 : tree data_ref;
6252 : 0 : basic_block new_bb;
6253 : 0 : tree msq_init = NULL_TREE;
6254 : 0 : tree new_temp;
6255 : 0 : gphi *phi_stmt;
6256 : 0 : tree msq = NULL_TREE;
6257 : 0 : gimple_seq stmts = NULL;
6258 : 0 : bool compute_in_loop = false;
6259 : 0 : bool nested_in_vect_loop = false;
6260 : 0 : class loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
6261 : 0 : class loop *loop_for_initial_load = NULL;
6262 : :
6263 : 0 : if (loop_vinfo)
6264 : : {
6265 : 0 : loop = LOOP_VINFO_LOOP (loop_vinfo);
6266 : 0 : nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
6267 : : }
6268 : :
6269 : 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign
6270 : : || alignment_support_scheme == dr_explicit_realign_optimized);
6271 : :
6272 : : /* We need to generate three things:
6273 : : 1. the misalignment computation
6274 : : 2. the extra vector load (for the optimized realignment scheme).
6275 : : 3. the phi node for the two vectors from which the realignment is
6276 : : done (for the optimized realignment scheme). */
6277 : :
6278 : : /* 1. Determine where to generate the misalignment computation.
6279 : :
6280 : : If INIT_ADDR is NULL_TREE, this indicates that the misalignment
6281 : : calculation will be generated by this function, outside the loop (in the
6282 : : preheader). Otherwise, INIT_ADDR had already been computed for us by the
6283 : : caller, inside the loop.
6284 : :
6285 : : Background: If the misalignment remains fixed throughout the iterations of
6286 : : the loop, then both realignment schemes are applicable, and also the
6287 : : misalignment computation can be done outside LOOP. This is because we are
6288 : : vectorizing LOOP, and so the memory accesses in LOOP advance in steps that
6289 : : are a multiple of VS (the Vector Size), and therefore the misalignment in
6290 : : different vectorized LOOP iterations is always the same.
6291 : : The problem arises only if the memory access is in an inner-loop nested
6292 : : inside LOOP, which is now being vectorized using outer-loop vectorization.
6293 : : This is the only case when the misalignment of the memory access may not
6294 : : remain fixed throughout the iterations of the inner-loop (as explained in
6295 : : detail in vect_supportable_dr_alignment). In this case, not only is the
6296 : : optimized realignment scheme not applicable, but also the misalignment
6297 : : computation (and generation of the realignment token that is passed to
6298 : : REALIGN_LOAD) have to be done inside the loop.
6299 : :
6300 : : In short, INIT_ADDR indicates whether we are in a COMPUTE_IN_LOOP mode
6301 : : or not, which in turn determines if the misalignment is computed inside
6302 : : the inner-loop, or outside LOOP. */
6303 : :
6304 : 0 : if (init_addr != NULL_TREE || !loop_vinfo)
6305 : : {
6306 : 0 : compute_in_loop = true;
6307 : 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign);
6308 : : }
6309 : :
6310 : :
6311 : : /* 2. Determine where to generate the extra vector load.
6312 : :
6313 : : For the optimized realignment scheme, instead of generating two vector
6314 : : loads in each iteration, we generate a single extra vector load in the
6315 : : preheader of the loop, and in each iteration reuse the result of the
6316 : : vector load from the previous iteration. In case the memory access is in
6317 : : an inner-loop nested inside LOOP, which is now being vectorized using
6318 : : outer-loop vectorization, we need to determine whether this initial vector
6319 : : load should be generated at the preheader of the inner-loop, or can be
6320 : : generated at the preheader of LOOP. If the memory access has no evolution
6321 : : in LOOP, it can be generated in the preheader of LOOP. Otherwise, it has
6322 : : to be generated inside LOOP (in the preheader of the inner-loop). */
6323 : :
6324 : 0 : if (nested_in_vect_loop)
6325 : : {
6326 : 0 : tree outerloop_step = STMT_VINFO_DR_STEP (stmt_info);
6327 : 0 : bool invariant_in_outerloop =
6328 : 0 : (tree_int_cst_compare (outerloop_step, size_zero_node) == 0);
6329 : 0 : loop_for_initial_load = (invariant_in_outerloop ? loop : loop->inner);
6330 : : }
6331 : : else
6332 : : loop_for_initial_load = loop;
6333 : 0 : if (at_loop)
6334 : 0 : *at_loop = loop_for_initial_load;
6335 : :
6336 : 0 : tree vuse = NULL_TREE;
6337 : 0 : if (loop_for_initial_load)
6338 : : {
6339 : 0 : pe = loop_preheader_edge (loop_for_initial_load);
6340 : 0 : if (gphi *vphi = get_virtual_phi (loop_for_initial_load->header))
6341 : 0 : vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
6342 : : }
6343 : 0 : if (!vuse)
6344 : 0 : vuse = gimple_vuse (gsi_stmt (*gsi));
6345 : :
6346 : : /* 3. For the case of the optimized realignment, create the first vector
6347 : : load at the loop preheader. */
6348 : :
6349 : 0 : if (alignment_support_scheme == dr_explicit_realign_optimized)
6350 : : {
6351 : : /* Create msq_init = *(floor(p1)) in the loop preheader */
6352 : 0 : gassign *new_stmt;
6353 : :
6354 : 0 : gcc_assert (!compute_in_loop);
6355 : 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6356 : 0 : ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
6357 : : loop_for_initial_load, NULL_TREE,
6358 : : &init_addr, NULL, &inc, true);
6359 : 0 : if (TREE_CODE (ptr) == SSA_NAME)
6360 : 0 : new_temp = copy_ssa_name (ptr);
6361 : : else
6362 : 0 : new_temp = make_ssa_name (TREE_TYPE (ptr));
6363 : 0 : poly_uint64 align = DR_TARGET_ALIGNMENT (dr_info);
6364 : 0 : tree type = TREE_TYPE (ptr);
6365 : 0 : new_stmt = gimple_build_assign
6366 : 0 : (new_temp, BIT_AND_EXPR, ptr,
6367 : 0 : fold_build2 (MINUS_EXPR, type,
6368 : : build_int_cst (type, 0),
6369 : : build_int_cst (type, align)));
6370 : 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6371 : 0 : gcc_assert (!new_bb);
6372 : 0 : data_ref
6373 : 0 : = build2 (MEM_REF, TREE_TYPE (vec_dest), new_temp,
6374 : 0 : build_int_cst (reference_alias_ptr_type (DR_REF (dr)), 0));
6375 : 0 : vect_copy_ref_info (data_ref, DR_REF (dr));
6376 : 0 : new_stmt = gimple_build_assign (vec_dest, data_ref);
6377 : 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6378 : 0 : gimple_assign_set_lhs (new_stmt, new_temp);
6379 : 0 : gimple_set_vuse (new_stmt, vuse);
6380 : 0 : if (pe)
6381 : : {
6382 : 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6383 : 0 : gcc_assert (!new_bb);
6384 : : }
6385 : : else
6386 : 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6387 : :
6388 : 0 : msq_init = gimple_assign_lhs (new_stmt);
6389 : : }
6390 : :
6391 : : /* 4. Create realignment token using a target builtin, if available.
6392 : : It is done either inside the containing loop, or before LOOP (as
6393 : : determined above). */
6394 : :
6395 : 0 : if (targetm.vectorize.builtin_mask_for_load)
6396 : : {
6397 : 0 : gcall *new_stmt;
6398 : 0 : tree builtin_decl;
6399 : :
6400 : : /* Compute INIT_ADDR - the initial addressed accessed by this memref. */
6401 : 0 : if (!init_addr)
6402 : : {
6403 : : /* Generate the INIT_ADDR computation outside LOOP. */
6404 : 0 : init_addr = vect_create_addr_base_for_vector_ref (vinfo,
6405 : : stmt_info, &stmts,
6406 : : NULL_TREE);
6407 : 0 : if (loop)
6408 : : {
6409 : 0 : pe = loop_preheader_edge (loop);
6410 : 0 : new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
6411 : 0 : gcc_assert (!new_bb);
6412 : : }
6413 : : else
6414 : 0 : gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
6415 : : }
6416 : :
6417 : 0 : builtin_decl = targetm.vectorize.builtin_mask_for_load ();
6418 : 0 : new_stmt = gimple_build_call (builtin_decl, 1, init_addr);
6419 : 0 : vec_dest =
6420 : 0 : vect_create_destination_var (scalar_dest,
6421 : : gimple_call_return_type (new_stmt));
6422 : 0 : new_temp = make_ssa_name (vec_dest, new_stmt);
6423 : 0 : gimple_call_set_lhs (new_stmt, new_temp);
6424 : :
6425 : 0 : if (compute_in_loop)
6426 : 0 : gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
6427 : : else
6428 : : {
6429 : : /* Generate the misalignment computation outside LOOP. */
6430 : 0 : pe = loop_preheader_edge (loop);
6431 : 0 : new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
6432 : 0 : gcc_assert (!new_bb);
6433 : : }
6434 : :
6435 : 0 : *realignment_token = gimple_call_lhs (new_stmt);
6436 : :
6437 : : /* The result of the CALL_EXPR to this builtin is determined from
6438 : : the value of the parameter and no global variables are touched
6439 : : which makes the builtin a "const" function. Requiring the
6440 : : builtin to have the "const" attribute makes it unnecessary
6441 : : to call mark_call_clobbered. */
6442 : 0 : gcc_assert (TREE_READONLY (builtin_decl));
6443 : : }
6444 : :
6445 : 0 : if (alignment_support_scheme == dr_explicit_realign)
6446 : : return msq;
6447 : :
6448 : 0 : gcc_assert (!compute_in_loop);
6449 : 0 : gcc_assert (alignment_support_scheme == dr_explicit_realign_optimized);
6450 : :
6451 : :
6452 : : /* 5. Create msq = phi <msq_init, lsq> in loop */
6453 : :
6454 : 0 : pe = loop_preheader_edge (containing_loop);
6455 : 0 : vec_dest = vect_create_destination_var (scalar_dest, vectype);
6456 : 0 : msq = make_ssa_name (vec_dest);
6457 : 0 : phi_stmt = create_phi_node (msq, containing_loop->header);
6458 : 0 : add_phi_arg (phi_stmt, msq_init, pe, UNKNOWN_LOCATION);
6459 : :
6460 : 0 : return msq;
6461 : : }
6462 : :
6463 : :
6464 : : /* Function vect_grouped_load_supported.
6465 : :
6466 : : COUNT is the size of the load group (the number of statements plus the
6467 : : number of gaps). SINGLE_ELEMENT_P is true if there is actually
6468 : : only one statement, with a gap of COUNT - 1.
6469 : :
6470 : : Returns true if a suitable permute exists. */
6471 : :
6472 : : bool
6473 : 2097 : vect_grouped_load_supported (tree vectype, bool single_element_p,
6474 : : unsigned HOST_WIDE_INT count)
6475 : : {
6476 : 2097 : machine_mode mode = TYPE_MODE (vectype);
6477 : :
6478 : : /* If this is single-element interleaving with an element distance
6479 : : that leaves unused vector loads around punt - we at least create
6480 : : very sub-optimal code in that case (and blow up memory,
6481 : : see PR65518). */
6482 : 2097 : if (single_element_p && maybe_gt (count, TYPE_VECTOR_SUBPARTS (vectype)))
6483 : : {
6484 : 41 : if (dump_enabled_p ())
6485 : 13 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6486 : : "single-element interleaving not supported "
6487 : : "for not adjacent vector loads\n");
6488 : 41 : return false;
6489 : : }
6490 : :
6491 : : /* vect_permute_load_chain requires the group size to be equal to 3 or
6492 : : be a power of two. */
6493 : 2056 : if (count != 3 && exact_log2 (count) == -1)
6494 : : {
6495 : 441 : if (dump_enabled_p ())
6496 : 14 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6497 : : "the size of the group of accesses"
6498 : : " is not a power of 2 or not equal to 3\n");
6499 : 441 : return false;
6500 : : }
6501 : :
6502 : : /* Check that the permutation is supported. */
6503 : 1615 : if (VECTOR_MODE_P (mode))
6504 : : {
6505 : 1615 : unsigned int i, j;
6506 : 1615 : if (count == 3)
6507 : : {
6508 : 760 : unsigned int nelt;
6509 : 1520 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6510 : : {
6511 : : if (dump_enabled_p ())
6512 : : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6513 : : "cannot handle groups of 3 loads for"
6514 : : " variable-length vectors\n");
6515 : : return false;
6516 : : }
6517 : :
6518 : 760 : vec_perm_builder sel (nelt, nelt, 1);
6519 : 760 : sel.quick_grow (nelt);
6520 : 760 : vec_perm_indices indices;
6521 : 760 : unsigned int k;
6522 : 2992 : for (k = 0; k < 3; k++)
6523 : : {
6524 : 8653 : for (i = 0; i < nelt; i++)
6525 : 6405 : if (3 * i + k < 2 * nelt)
6526 : 4277 : sel[i] = 3 * i + k;
6527 : : else
6528 : 2128 : sel[i] = 0;
6529 : 2248 : indices.new_vector (sel, 2, nelt);
6530 : 2248 : if (!can_vec_perm_const_p (mode, mode, indices))
6531 : : {
6532 : 16 : if (dump_enabled_p ())
6533 : 7 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6534 : : "shuffle of 3 loads is not supported by"
6535 : : " target\n");
6536 : 16 : return false;
6537 : : }
6538 : 8472 : for (i = 0, j = 0; i < nelt; i++)
6539 : 6240 : if (3 * i + k < 2 * nelt)
6540 : 4160 : sel[i] = i;
6541 : : else
6542 : 2080 : sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
6543 : 2232 : indices.new_vector (sel, 2, nelt);
6544 : 2232 : if (!can_vec_perm_const_p (mode, mode, indices))
6545 : : {
6546 : 0 : if (dump_enabled_p ())
6547 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6548 : : "shuffle of 3 loads is not supported by"
6549 : : " target\n");
6550 : 0 : return false;
6551 : : }
6552 : : }
6553 : : return true;
6554 : 760 : }
6555 : : else
6556 : : {
6557 : : /* If length is not equal to 3 then only power of 2 is supported. */
6558 : 855 : gcc_assert (pow2p_hwi (count));
6559 : 1710 : poly_uint64 nelt = GET_MODE_NUNITS (mode);
6560 : :
6561 : : /* The encoding has a single stepped pattern. */
6562 : 855 : vec_perm_builder sel (nelt, 1, 3);
6563 : 855 : sel.quick_grow (3);
6564 : 4275 : for (i = 0; i < 3; i++)
6565 : 2565 : sel[i] = i * 2;
6566 : 855 : vec_perm_indices indices (sel, 2, nelt);
6567 : 855 : if (can_vec_perm_const_p (mode, mode, indices))
6568 : : {
6569 : 3400 : for (i = 0; i < 3; i++)
6570 : 2550 : sel[i] = i * 2 + 1;
6571 : 850 : indices.new_vector (sel, 2, nelt);
6572 : 850 : if (can_vec_perm_const_p (mode, mode, indices))
6573 : 850 : return true;
6574 : : }
6575 : 855 : }
6576 : : }
6577 : :
6578 : 5 : if (dump_enabled_p ())
6579 : 4 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6580 : : "extract even/odd not supported by target\n");
6581 : : return false;
6582 : : }
6583 : :
6584 : : /* Return FN if vec_{masked_,mask_len_}load_lanes is available for COUNT vectors
6585 : : of type VECTYPE. MASKED_P says whether the masked form is needed.
6586 : : If it is available and ELSVALS is nonzero store the possible else values
6587 : : in the vector it points to. */
6588 : :
6589 : : internal_fn
6590 : 95545 : vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
6591 : : bool masked_p, vec<int> *elsvals)
6592 : : {
6593 : 95545 : if (vect_lanes_optab_supported_p ("vec_mask_len_load_lanes",
6594 : : vec_mask_len_load_lanes_optab, vectype,
6595 : : count, elsvals))
6596 : : return IFN_MASK_LEN_LOAD_LANES;
6597 : 95545 : else if (masked_p)
6598 : : {
6599 : 0 : if (vect_lanes_optab_supported_p ("vec_mask_load_lanes",
6600 : : vec_mask_load_lanes_optab, vectype,
6601 : : count, elsvals))
6602 : : return IFN_MASK_LOAD_LANES;
6603 : : }
6604 : : else
6605 : : {
6606 : 95545 : if (vect_lanes_optab_supported_p ("vec_load_lanes", vec_load_lanes_optab,
6607 : : vectype, count, elsvals))
6608 : : return IFN_LOAD_LANES;
6609 : : }
6610 : : return IFN_LAST;
6611 : : }
6612 : :
6613 : : /* Function vect_permute_load_chain.
6614 : :
6615 : : Given a chain of interleaved loads in DR_CHAIN of LENGTH that must be
6616 : : a power of 2 or equal to 3, generate extract_even/odd stmts to reorder
6617 : : the input data correctly. Return the final references for loads in
6618 : : RESULT_CHAIN.
6619 : :
6620 : : E.g., LENGTH is 4 and the scalar type is short, i.e., VF is 8.
6621 : : The input is 4 vectors each containing 8 elements. We assign a number to each
6622 : : element, the input sequence is:
6623 : :
6624 : : 1st vec: 0 1 2 3 4 5 6 7
6625 : : 2nd vec: 8 9 10 11 12 13 14 15
6626 : : 3rd vec: 16 17 18 19 20 21 22 23
6627 : : 4th vec: 24 25 26 27 28 29 30 31
6628 : :
6629 : : The output sequence should be:
6630 : :
6631 : : 1st vec: 0 4 8 12 16 20 24 28
6632 : : 2nd vec: 1 5 9 13 17 21 25 29
6633 : : 3rd vec: 2 6 10 14 18 22 26 30
6634 : : 4th vec: 3 7 11 15 19 23 27 31
6635 : :
6636 : : i.e., the first output vector should contain the first elements of each
6637 : : interleaving group, etc.
6638 : :
6639 : : We use extract_even/odd instructions to create such output. The input of
6640 : : each extract_even/odd operation is two vectors
6641 : : 1st vec 2nd vec
6642 : : 0 1 2 3 4 5 6 7
6643 : :
6644 : : and the output is the vector of extracted even/odd elements. The output of
6645 : : extract_even will be: 0 2 4 6
6646 : : and of extract_odd: 1 3 5 7
6647 : :
6648 : :
6649 : : The permutation is done in log LENGTH stages. In each stage extract_even
6650 : : and extract_odd stmts are created for each pair of vectors in DR_CHAIN in
6651 : : their order. In our example,
6652 : :
6653 : : E1: extract_even (1st vec, 2nd vec)
6654 : : E2: extract_odd (1st vec, 2nd vec)
6655 : : E3: extract_even (3rd vec, 4th vec)
6656 : : E4: extract_odd (3rd vec, 4th vec)
6657 : :
6658 : : The output for the first stage will be:
6659 : :
6660 : : E1: 0 2 4 6 8 10 12 14
6661 : : E2: 1 3 5 7 9 11 13 15
6662 : : E3: 16 18 20 22 24 26 28 30
6663 : : E4: 17 19 21 23 25 27 29 31
6664 : :
6665 : : In order to proceed and create the correct sequence for the next stage (or
6666 : : for the correct output, if the second stage is the last one, as in our
6667 : : example), we first put the output of extract_even operation and then the
6668 : : output of extract_odd in RESULT_CHAIN (which is then copied to DR_CHAIN).
6669 : : The input for the second stage is:
6670 : :
6671 : : 1st vec (E1): 0 2 4 6 8 10 12 14
6672 : : 2nd vec (E3): 16 18 20 22 24 26 28 30
6673 : : 3rd vec (E2): 1 3 5 7 9 11 13 15
6674 : : 4th vec (E4): 17 19 21 23 25 27 29 31
6675 : :
6676 : : The output of the second stage:
6677 : :
6678 : : E1: 0 4 8 12 16 20 24 28
6679 : : E2: 2 6 10 14 18 22 26 30
6680 : : E3: 1 5 9 13 17 21 25 29
6681 : : E4: 3 7 11 15 19 23 27 31
6682 : :
6683 : : And RESULT_CHAIN after reordering:
6684 : :
6685 : : 1st vec (E1): 0 4 8 12 16 20 24 28
6686 : : 2nd vec (E3): 1 5 9 13 17 21 25 29
6687 : : 3rd vec (E2): 2 6 10 14 18 22 26 30
6688 : : 4th vec (E4): 3 7 11 15 19 23 27 31. */
6689 : :
6690 : : static void
6691 : 0 : vect_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
6692 : : unsigned int length,
6693 : : stmt_vec_info stmt_info,
6694 : : gimple_stmt_iterator *gsi,
6695 : : vec<tree> *result_chain)
6696 : : {
6697 : 0 : tree data_ref, first_vect, second_vect;
6698 : 0 : tree perm_mask_even, perm_mask_odd;
6699 : 0 : tree perm3_mask_low, perm3_mask_high;
6700 : 0 : gimple *perm_stmt;
6701 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6702 : 0 : unsigned int i, j, log_length = exact_log2 (length);
6703 : :
6704 : 0 : result_chain->quick_grow (length);
6705 : 0 : memcpy (result_chain->address (), dr_chain.address (),
6706 : : length * sizeof (tree));
6707 : :
6708 : 0 : if (length == 3)
6709 : : {
6710 : : /* vect_grouped_load_supported ensures that this is constant. */
6711 : 0 : unsigned nelt = TYPE_VECTOR_SUBPARTS (vectype).to_constant ();
6712 : 0 : unsigned int k;
6713 : :
6714 : 0 : vec_perm_builder sel (nelt, nelt, 1);
6715 : 0 : sel.quick_grow (nelt);
6716 : 0 : vec_perm_indices indices;
6717 : 0 : for (k = 0; k < 3; k++)
6718 : : {
6719 : 0 : for (i = 0; i < nelt; i++)
6720 : 0 : if (3 * i + k < 2 * nelt)
6721 : 0 : sel[i] = 3 * i + k;
6722 : : else
6723 : 0 : sel[i] = 0;
6724 : 0 : indices.new_vector (sel, 2, nelt);
6725 : 0 : perm3_mask_low = vect_gen_perm_mask_checked (vectype, indices);
6726 : :
6727 : 0 : for (i = 0, j = 0; i < nelt; i++)
6728 : 0 : if (3 * i + k < 2 * nelt)
6729 : 0 : sel[i] = i;
6730 : : else
6731 : 0 : sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
6732 : 0 : indices.new_vector (sel, 2, nelt);
6733 : 0 : perm3_mask_high = vect_gen_perm_mask_checked (vectype, indices);
6734 : :
6735 : 0 : first_vect = dr_chain[0];
6736 : 0 : second_vect = dr_chain[1];
6737 : :
6738 : : /* Create interleaving stmt (low part of):
6739 : : low = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
6740 : : ...}> */
6741 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_low");
6742 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
6743 : : second_vect, perm3_mask_low);
6744 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6745 : :
6746 : : /* Create interleaving stmt (high part of):
6747 : : high = VEC_PERM_EXPR <first_vect, second_vect2, {k, 3 + k, 6 + k,
6748 : : ...}> */
6749 : 0 : first_vect = data_ref;
6750 : 0 : second_vect = dr_chain[2];
6751 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3_high");
6752 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, first_vect,
6753 : : second_vect, perm3_mask_high);
6754 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6755 : 0 : (*result_chain)[k] = data_ref;
6756 : : }
6757 : 0 : }
6758 : : else
6759 : : {
6760 : : /* If length is not equal to 3 then only power of 2 is supported. */
6761 : 0 : gcc_assert (pow2p_hwi (length));
6762 : :
6763 : : /* The encoding has a single stepped pattern. */
6764 : 0 : poly_uint64 nelt = TYPE_VECTOR_SUBPARTS (vectype);
6765 : 0 : vec_perm_builder sel (nelt, 1, 3);
6766 : 0 : sel.quick_grow (3);
6767 : 0 : for (i = 0; i < 3; ++i)
6768 : 0 : sel[i] = i * 2;
6769 : 0 : vec_perm_indices indices (sel, 2, nelt);
6770 : 0 : perm_mask_even = vect_gen_perm_mask_checked (vectype, indices);
6771 : :
6772 : 0 : for (i = 0; i < 3; ++i)
6773 : 0 : sel[i] = i * 2 + 1;
6774 : 0 : indices.new_vector (sel, 2, nelt);
6775 : 0 : perm_mask_odd = vect_gen_perm_mask_checked (vectype, indices);
6776 : :
6777 : 0 : for (i = 0; i < log_length; i++)
6778 : : {
6779 : 0 : for (j = 0; j < length; j += 2)
6780 : : {
6781 : 0 : first_vect = dr_chain[j];
6782 : 0 : second_vect = dr_chain[j+1];
6783 : :
6784 : : /* data_ref = permute_even (first_data_ref, second_data_ref); */
6785 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_even");
6786 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
6787 : : first_vect, second_vect,
6788 : : perm_mask_even);
6789 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6790 : 0 : (*result_chain)[j/2] = data_ref;
6791 : :
6792 : : /* data_ref = permute_odd (first_data_ref, second_data_ref); */
6793 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_perm_odd");
6794 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
6795 : : first_vect, second_vect,
6796 : : perm_mask_odd);
6797 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6798 : 0 : (*result_chain)[j/2+length/2] = data_ref;
6799 : : }
6800 : 0 : memcpy (dr_chain.address (), result_chain->address (),
6801 : : length * sizeof (tree));
6802 : : }
6803 : 0 : }
6804 : 0 : }
6805 : :
6806 : : /* Function vect_shift_permute_load_chain.
6807 : :
6808 : : Given a chain of loads in DR_CHAIN of LENGTH 2 or 3, generate
6809 : : sequence of stmts to reorder the input data accordingly.
6810 : : Return the final references for loads in RESULT_CHAIN.
6811 : : Return true if successed, false otherwise.
6812 : :
6813 : : E.g., LENGTH is 3 and the scalar type is short, i.e., VF is 8.
6814 : : The input is 3 vectors each containing 8 elements. We assign a
6815 : : number to each element, the input sequence is:
6816 : :
6817 : : 1st vec: 0 1 2 3 4 5 6 7
6818 : : 2nd vec: 8 9 10 11 12 13 14 15
6819 : : 3rd vec: 16 17 18 19 20 21 22 23
6820 : :
6821 : : The output sequence should be:
6822 : :
6823 : : 1st vec: 0 3 6 9 12 15 18 21
6824 : : 2nd vec: 1 4 7 10 13 16 19 22
6825 : : 3rd vec: 2 5 8 11 14 17 20 23
6826 : :
6827 : : We use 3 shuffle instructions and 3 * 3 - 1 shifts to create such output.
6828 : :
6829 : : First we shuffle all 3 vectors to get correct elements order:
6830 : :
6831 : : 1st vec: ( 0 3 6) ( 1 4 7) ( 2 5)
6832 : : 2nd vec: ( 8 11 14) ( 9 12 15) (10 13)
6833 : : 3rd vec: (16 19 22) (17 20 23) (18 21)
6834 : :
6835 : : Next we unite and shift vector 3 times:
6836 : :
6837 : : 1st step:
6838 : : shift right by 6 the concatenation of:
6839 : : "1st vec" and "2nd vec"
6840 : : ( 0 3 6) ( 1 4 7) |( 2 5) _ ( 8 11 14) ( 9 12 15)| (10 13)
6841 : : "2nd vec" and "3rd vec"
6842 : : ( 8 11 14) ( 9 12 15) |(10 13) _ (16 19 22) (17 20 23)| (18 21)
6843 : : "3rd vec" and "1st vec"
6844 : : (16 19 22) (17 20 23) |(18 21) _ ( 0 3 6) ( 1 4 7)| ( 2 5)
6845 : : | New vectors |
6846 : :
6847 : : So that now new vectors are:
6848 : :
6849 : : 1st vec: ( 2 5) ( 8 11 14) ( 9 12 15)
6850 : : 2nd vec: (10 13) (16 19 22) (17 20 23)
6851 : : 3rd vec: (18 21) ( 0 3 6) ( 1 4 7)
6852 : :
6853 : : 2nd step:
6854 : : shift right by 5 the concatenation of:
6855 : : "1st vec" and "3rd vec"
6856 : : ( 2 5) ( 8 11 14) |( 9 12 15) _ (18 21) ( 0 3 6)| ( 1 4 7)
6857 : : "2nd vec" and "1st vec"
6858 : : (10 13) (16 19 22) |(17 20 23) _ ( 2 5) ( 8 11 14)| ( 9 12 15)
6859 : : "3rd vec" and "2nd vec"
6860 : : (18 21) ( 0 3 6) |( 1 4 7) _ (10 13) (16 19 22)| (17 20 23)
6861 : : | New vectors |
6862 : :
6863 : : So that now new vectors are:
6864 : :
6865 : : 1st vec: ( 9 12 15) (18 21) ( 0 3 6)
6866 : : 2nd vec: (17 20 23) ( 2 5) ( 8 11 14)
6867 : : 3rd vec: ( 1 4 7) (10 13) (16 19 22) READY
6868 : :
6869 : : 3rd step:
6870 : : shift right by 5 the concatenation of:
6871 : : "1st vec" and "1st vec"
6872 : : ( 9 12 15) (18 21) |( 0 3 6) _ ( 9 12 15) (18 21)| ( 0 3 6)
6873 : : shift right by 3 the concatenation of:
6874 : : "2nd vec" and "2nd vec"
6875 : : (17 20 23) |( 2 5) ( 8 11 14) _ (17 20 23)| ( 2 5) ( 8 11 14)
6876 : : | New vectors |
6877 : :
6878 : : So that now all vectors are READY:
6879 : : 1st vec: ( 0 3 6) ( 9 12 15) (18 21)
6880 : : 2nd vec: ( 2 5) ( 8 11 14) (17 20 23)
6881 : : 3rd vec: ( 1 4 7) (10 13) (16 19 22)
6882 : :
6883 : : This algorithm is faster than one in vect_permute_load_chain if:
6884 : : 1. "shift of a concatination" is faster than general permutation.
6885 : : This is usually so.
6886 : : 2. The TARGET machine can't execute vector instructions in parallel.
6887 : : This is because each step of the algorithm depends on previous.
6888 : : The algorithm in vect_permute_load_chain is much more parallel.
6889 : :
6890 : : The algorithm is applicable only for LOAD CHAIN LENGTH less than VF.
6891 : : */
6892 : :
6893 : : static bool
6894 : 0 : vect_shift_permute_load_chain (vec_info *vinfo, vec<tree> dr_chain,
6895 : : unsigned int length,
6896 : : stmt_vec_info stmt_info,
6897 : : gimple_stmt_iterator *gsi,
6898 : : vec<tree> *result_chain)
6899 : : {
6900 : 0 : tree vect[3], vect_shift[3], data_ref, first_vect, second_vect;
6901 : 0 : tree perm2_mask1, perm2_mask2, perm3_mask;
6902 : 0 : tree select_mask, shift1_mask, shift2_mask, shift3_mask, shift4_mask;
6903 : 0 : gimple *perm_stmt;
6904 : :
6905 : 0 : tree vectype = STMT_VINFO_VECTYPE (stmt_info);
6906 : 0 : machine_mode vmode = TYPE_MODE (vectype);
6907 : 0 : unsigned int i;
6908 : 0 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
6909 : :
6910 : 0 : unsigned HOST_WIDE_INT nelt, vf;
6911 : 0 : if (!TYPE_VECTOR_SUBPARTS (vectype).is_constant (&nelt)
6912 : 0 : || !LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
6913 : : /* Not supported for variable-length vectors. */
6914 : : return false;
6915 : :
6916 : 0 : vec_perm_builder sel (nelt, nelt, 1);
6917 : 0 : sel.quick_grow (nelt);
6918 : :
6919 : 0 : result_chain->quick_grow (length);
6920 : 0 : memcpy (result_chain->address (), dr_chain.address (),
6921 : 0 : length * sizeof (tree));
6922 : :
6923 : 0 : if (pow2p_hwi (length) && vf > 4)
6924 : : {
6925 : 0 : unsigned int j, log_length = exact_log2 (length);
6926 : 0 : for (i = 0; i < nelt / 2; ++i)
6927 : 0 : sel[i] = i * 2;
6928 : 0 : for (i = 0; i < nelt / 2; ++i)
6929 : 0 : sel[nelt / 2 + i] = i * 2 + 1;
6930 : 0 : vec_perm_indices indices (sel, 2, nelt);
6931 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
6932 : : {
6933 : 0 : if (dump_enabled_p ())
6934 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6935 : : "shuffle of 2 fields structure is not \
6936 : : supported by target\n");
6937 : 0 : return false;
6938 : : }
6939 : 0 : perm2_mask1 = vect_gen_perm_mask_checked (vectype, indices);
6940 : :
6941 : 0 : for (i = 0; i < nelt / 2; ++i)
6942 : 0 : sel[i] = i * 2 + 1;
6943 : 0 : for (i = 0; i < nelt / 2; ++i)
6944 : 0 : sel[nelt / 2 + i] = i * 2;
6945 : 0 : indices.new_vector (sel, 2, nelt);
6946 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
6947 : : {
6948 : 0 : if (dump_enabled_p ())
6949 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6950 : : "shuffle of 2 fields structure is not \
6951 : : supported by target\n");
6952 : 0 : return false;
6953 : : }
6954 : 0 : perm2_mask2 = vect_gen_perm_mask_checked (vectype, indices);
6955 : :
6956 : : /* Generating permutation constant to shift all elements.
6957 : : For vector length 8 it is {4 5 6 7 8 9 10 11}. */
6958 : 0 : for (i = 0; i < nelt; i++)
6959 : 0 : sel[i] = nelt / 2 + i;
6960 : 0 : indices.new_vector (sel, 2, nelt);
6961 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
6962 : : {
6963 : 0 : if (dump_enabled_p ())
6964 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6965 : : "shift permutation is not supported by target\n");
6966 : 0 : return false;
6967 : : }
6968 : 0 : shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
6969 : :
6970 : : /* Generating permutation constant to select vector from 2.
6971 : : For vector length 8 it is {0 1 2 3 12 13 14 15}. */
6972 : 0 : for (i = 0; i < nelt / 2; i++)
6973 : 0 : sel[i] = i;
6974 : 0 : for (i = nelt / 2; i < nelt; i++)
6975 : 0 : sel[i] = nelt + i;
6976 : 0 : indices.new_vector (sel, 2, nelt);
6977 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
6978 : : {
6979 : 0 : if (dump_enabled_p ())
6980 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
6981 : : "select is not supported by target\n");
6982 : 0 : return false;
6983 : : }
6984 : 0 : select_mask = vect_gen_perm_mask_checked (vectype, indices);
6985 : :
6986 : 0 : for (i = 0; i < log_length; i++)
6987 : : {
6988 : 0 : for (j = 0; j < length; j += 2)
6989 : : {
6990 : 0 : first_vect = dr_chain[j];
6991 : 0 : second_vect = dr_chain[j + 1];
6992 : :
6993 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
6994 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
6995 : : first_vect, first_vect,
6996 : : perm2_mask1);
6997 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
6998 : 0 : vect[0] = data_ref;
6999 : :
7000 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle2");
7001 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7002 : : second_vect, second_vect,
7003 : : perm2_mask2);
7004 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7005 : 0 : vect[1] = data_ref;
7006 : :
7007 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift");
7008 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7009 : : vect[0], vect[1], shift1_mask);
7010 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7011 : 0 : (*result_chain)[j/2 + length/2] = data_ref;
7012 : :
7013 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_select");
7014 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7015 : : vect[0], vect[1], select_mask);
7016 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7017 : 0 : (*result_chain)[j/2] = data_ref;
7018 : : }
7019 : 0 : memcpy (dr_chain.address (), result_chain->address (),
7020 : : length * sizeof (tree));
7021 : : }
7022 : : return true;
7023 : 0 : }
7024 : 0 : if (length == 3 && vf > 2)
7025 : : {
7026 : : unsigned int k = 0, l = 0;
7027 : :
7028 : : /* Generating permutation constant to get all elements in rigth order.
7029 : : For vector length 8 it is {0 3 6 1 4 7 2 5}. */
7030 : 0 : for (i = 0; i < nelt; i++)
7031 : : {
7032 : 0 : if (3 * k + (l % 3) >= nelt)
7033 : : {
7034 : 0 : k = 0;
7035 : 0 : l += (3 - (nelt % 3));
7036 : : }
7037 : 0 : sel[i] = 3 * k + (l % 3);
7038 : 0 : k++;
7039 : : }
7040 : 0 : vec_perm_indices indices (sel, 2, nelt);
7041 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
7042 : : {
7043 : 0 : if (dump_enabled_p ())
7044 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7045 : : "shuffle of 3 fields structure is not \
7046 : : supported by target\n");
7047 : 0 : return false;
7048 : : }
7049 : 0 : perm3_mask = vect_gen_perm_mask_checked (vectype, indices);
7050 : :
7051 : : /* Generating permutation constant to shift all elements.
7052 : : For vector length 8 it is {6 7 8 9 10 11 12 13}. */
7053 : 0 : for (i = 0; i < nelt; i++)
7054 : 0 : sel[i] = 2 * (nelt / 3) + (nelt % 3) + i;
7055 : 0 : indices.new_vector (sel, 2, nelt);
7056 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
7057 : : {
7058 : 0 : if (dump_enabled_p ())
7059 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7060 : : "shift permutation is not supported by target\n");
7061 : 0 : return false;
7062 : : }
7063 : 0 : shift1_mask = vect_gen_perm_mask_checked (vectype, indices);
7064 : :
7065 : : /* Generating permutation constant to shift all elements.
7066 : : For vector length 8 it is {5 6 7 8 9 10 11 12}. */
7067 : 0 : for (i = 0; i < nelt; i++)
7068 : 0 : sel[i] = 2 * (nelt / 3) + 1 + i;
7069 : 0 : indices.new_vector (sel, 2, nelt);
7070 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
7071 : : {
7072 : 0 : if (dump_enabled_p ())
7073 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7074 : : "shift permutation is not supported by target\n");
7075 : 0 : return false;
7076 : : }
7077 : 0 : shift2_mask = vect_gen_perm_mask_checked (vectype, indices);
7078 : :
7079 : : /* Generating permutation constant to shift all elements.
7080 : : For vector length 8 it is {3 4 5 6 7 8 9 10}. */
7081 : 0 : for (i = 0; i < nelt; i++)
7082 : 0 : sel[i] = (nelt / 3) + (nelt % 3) / 2 + i;
7083 : 0 : indices.new_vector (sel, 2, nelt);
7084 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
7085 : : {
7086 : 0 : if (dump_enabled_p ())
7087 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7088 : : "shift permutation is not supported by target\n");
7089 : 0 : return false;
7090 : : }
7091 : 0 : shift3_mask = vect_gen_perm_mask_checked (vectype, indices);
7092 : :
7093 : : /* Generating permutation constant to shift all elements.
7094 : : For vector length 8 it is {5 6 7 8 9 10 11 12}. */
7095 : 0 : for (i = 0; i < nelt; i++)
7096 : 0 : sel[i] = 2 * (nelt / 3) + (nelt % 3) / 2 + i;
7097 : 0 : indices.new_vector (sel, 2, nelt);
7098 : 0 : if (!can_vec_perm_const_p (vmode, vmode, indices))
7099 : : {
7100 : 0 : if (dump_enabled_p ())
7101 : 0 : dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
7102 : : "shift permutation is not supported by target\n");
7103 : 0 : return false;
7104 : : }
7105 : 0 : shift4_mask = vect_gen_perm_mask_checked (vectype, indices);
7106 : :
7107 : 0 : for (k = 0; k < 3; k++)
7108 : : {
7109 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shuffle3");
7110 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7111 : 0 : dr_chain[k], dr_chain[k],
7112 : : perm3_mask);
7113 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7114 : 0 : vect[k] = data_ref;
7115 : : }
7116 : :
7117 : 0 : for (k = 0; k < 3; k++)
7118 : : {
7119 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift1");
7120 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7121 : 0 : vect[k % 3], vect[(k + 1) % 3],
7122 : : shift1_mask);
7123 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7124 : 0 : vect_shift[k] = data_ref;
7125 : : }
7126 : :
7127 : 0 : for (k = 0; k < 3; k++)
7128 : : {
7129 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift2");
7130 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR,
7131 : 0 : vect_shift[(4 - k) % 3],
7132 : 0 : vect_shift[(3 - k) % 3],
7133 : : shift2_mask);
7134 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7135 : 0 : vect[k] = data_ref;
7136 : : }
7137 : :
7138 : 0 : (*result_chain)[3 - (nelt % 3)] = vect[2];
7139 : :
7140 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift3");
7141 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[0],
7142 : : vect[0], shift3_mask);
7143 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7144 : 0 : (*result_chain)[nelt % 3] = data_ref;
7145 : :
7146 : 0 : data_ref = make_temp_ssa_name (vectype, NULL, "vect_shift4");
7147 : 0 : perm_stmt = gimple_build_assign (data_ref, VEC_PERM_EXPR, vect[1],
7148 : : vect[1], shift4_mask);
7149 : 0 : vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, gsi);
7150 : 0 : (*result_chain)[0] = data_ref;
7151 : 0 : return true;
7152 : 0 : }
7153 : : return false;
7154 : 0 : }
7155 : :
7156 : : /* Function vect_transform_grouped_load.
7157 : :
7158 : : Given a chain of input interleaved data-refs (in DR_CHAIN), build statements
7159 : : to perform their permutation and ascribe the result vectorized statements to
7160 : : the scalar statements.
7161 : : */
7162 : :
7163 : : void
7164 : 0 : vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info,
7165 : : vec<tree> dr_chain,
7166 : : int size, gimple_stmt_iterator *gsi)
7167 : : {
7168 : 0 : machine_mode mode;
7169 : 0 : vec<tree> result_chain = vNULL;
7170 : :
7171 : : /* DR_CHAIN contains input data-refs that are a part of the interleaving.
7172 : : RESULT_CHAIN is the output of vect_permute_load_chain, it contains permuted
7173 : : vectors, that are ready for vector computation. */
7174 : 0 : result_chain.create (size);
7175 : :
7176 : : /* If reassociation width for vector type is 2 or greater target machine can
7177 : : execute 2 or more vector instructions in parallel. Otherwise try to
7178 : : get chain for loads group using vect_shift_permute_load_chain. */
7179 : 0 : mode = TYPE_MODE (STMT_VINFO_VECTYPE (stmt_info));
7180 : 0 : if (targetm.sched.reassociation_width (VEC_PERM_EXPR, mode) > 1
7181 : 0 : || pow2p_hwi (size)
7182 : 0 : || !vect_shift_permute_load_chain (vinfo, dr_chain, size, stmt_info,
7183 : : gsi, &result_chain))
7184 : 0 : vect_permute_load_chain (vinfo, dr_chain,
7185 : : size, stmt_info, gsi, &result_chain);
7186 : 0 : vect_record_grouped_load_vectors (vinfo, stmt_info, result_chain);
7187 : 0 : result_chain.release ();
7188 : 0 : }
7189 : :
7190 : : /* RESULT_CHAIN contains the output of a group of grouped loads that were
7191 : : generated as part of the vectorization of STMT_INFO. Assign the statement
7192 : : for each vector to the associated scalar statement. */
7193 : :
7194 : : void
7195 : 0 : vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info,
7196 : : vec<tree> result_chain)
7197 : : {
7198 : 0 : stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
7199 : 0 : unsigned int i, gap_count;
7200 : 0 : tree tmp_data_ref;
7201 : :
7202 : : /* Put a permuted data-ref in the VECTORIZED_STMT field.
7203 : : Since we scan the chain starting from it's first node, their order
7204 : : corresponds the order of data-refs in RESULT_CHAIN. */
7205 : 0 : stmt_vec_info next_stmt_info = first_stmt_info;
7206 : 0 : gap_count = 1;
7207 : 0 : FOR_EACH_VEC_ELT (result_chain, i, tmp_data_ref)
7208 : : {
7209 : 0 : if (!next_stmt_info)
7210 : : break;
7211 : :
7212 : : /* Skip the gaps. Loads created for the gaps will be removed by dead
7213 : : code elimination pass later. No need to check for the first stmt in
7214 : : the group, since it always exists.
7215 : : DR_GROUP_GAP is the number of steps in elements from the previous
7216 : : access (if there is no gap DR_GROUP_GAP is 1). We skip loads that
7217 : : correspond to the gaps. */
7218 : 0 : if (next_stmt_info != first_stmt_info
7219 : 0 : && gap_count < DR_GROUP_GAP (next_stmt_info))
7220 : : {
7221 : 0 : gap_count++;
7222 : 0 : continue;
7223 : : }
7224 : :
7225 : : /* ??? The following needs cleanup after the removal of
7226 : : DR_GROUP_SAME_DR_STMT. */
7227 : 0 : if (next_stmt_info)
7228 : : {
7229 : 0 : gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref);
7230 : : /* We assume that if VEC_STMT is not NULL, this is a case of multiple
7231 : : copies, and we put the new vector statement last. */
7232 : 0 : STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt);
7233 : :
7234 : 0 : next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info);
7235 : 0 : gap_count = 1;
7236 : : }
7237 : : }
7238 : 0 : }
7239 : :
7240 : : /* Function vect_force_dr_alignment_p.
7241 : :
7242 : : Returns whether the alignment of a DECL can be forced to be aligned
7243 : : on ALIGNMENT bit boundary. */
7244 : :
7245 : : bool
7246 : 664516 : vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
7247 : : {
7248 : 664516 : if (!VAR_P (decl))
7249 : : return false;
7250 : :
7251 : 207123 : if (decl_in_symtab_p (decl)
7252 : 207123 : && (!symtab_node::get (decl)
7253 : 16758 : || !symtab_node::get (decl)->can_increase_alignment_p ()))
7254 : 8570 : return false;
7255 : :
7256 : 198553 : if (TREE_STATIC (decl))
7257 : 8188 : return (known_le (alignment,
7258 : 8188 : (unsigned HOST_WIDE_INT) MAX_OFILE_ALIGNMENT));
7259 : : else
7260 : 190365 : return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
7261 : : }
7262 : :
7263 : : /* Return whether the data reference DR_INFO is supported with respect to its
7264 : : alignment.
7265 : : If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
7266 : : it is aligned, i.e., check if it is possible to vectorize it with different
7267 : : alignment. */
7268 : :
7269 : : enum dr_alignment_support
7270 : 3101000 : vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
7271 : : tree vectype, int misalignment)
7272 : : {
7273 : 3101000 : data_reference *dr = dr_info->dr;
7274 : 3101000 : stmt_vec_info stmt_info = dr_info->stmt;
7275 : 3101000 : machine_mode mode = TYPE_MODE (vectype);
7276 : 3101000 : loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
7277 : 3101000 : class loop *vect_loop = NULL;
7278 : 3101000 : bool nested_in_vect_loop = false;
7279 : :
7280 : 3101000 : if (misalignment == 0)
7281 : : return dr_aligned;
7282 : 1721584 : else if (dr_safe_speculative_read_required (stmt_info))
7283 : : return dr_unaligned_unsupported;
7284 : :
7285 : : /* For now assume all conditional loads/stores support unaligned
7286 : : access without any special code. */
7287 : 1376598 : if (gcall *stmt = dyn_cast <gcall *> (stmt_info->stmt))
7288 : 7216 : if (gimple_call_internal_p (stmt)
7289 : 7216 : && (gimple_call_internal_fn (stmt) == IFN_MASK_LOAD
7290 : 2534 : || gimple_call_internal_fn (stmt) == IFN_MASK_STORE))
7291 : : return dr_unaligned_supported;
7292 : :
7293 : 1369382 : if (loop_vinfo)
7294 : : {
7295 : 672606 : vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
7296 : 672606 : nested_in_vect_loop = nested_in_vect_loop_p (vect_loop, stmt_info);
7297 : : }
7298 : :
7299 : : /* Possibly unaligned access. */
7300 : :
7301 : : /* We can choose between using the implicit realignment scheme (generating
7302 : : a misaligned_move stmt) and the explicit realignment scheme (generating
7303 : : aligned loads with a REALIGN_LOAD). There are two variants to the
7304 : : explicit realignment scheme: optimized, and unoptimized.
7305 : : We can optimize the realignment only if the step between consecutive
7306 : : vector loads is equal to the vector size. Since the vector memory
7307 : : accesses advance in steps of VS (Vector Size) in the vectorized loop, it
7308 : : is guaranteed that the misalignment amount remains the same throughout the
7309 : : execution of the vectorized loop. Therefore, we can create the
7310 : : "realignment token" (the permutation mask that is passed to REALIGN_LOAD)
7311 : : at the loop preheader.
7312 : :
7313 : : However, in the case of outer-loop vectorization, when vectorizing a
7314 : : memory access in the inner-loop nested within the LOOP that is now being
7315 : : vectorized, while it is guaranteed that the misalignment of the
7316 : : vectorized memory access will remain the same in different outer-loop
7317 : : iterations, it is *not* guaranteed that is will remain the same throughout
7318 : : the execution of the inner-loop. This is because the inner-loop advances
7319 : : with the original scalar step (and not in steps of VS). If the inner-loop
7320 : : step happens to be a multiple of VS, then the misalignment remains fixed
7321 : : and we can use the optimized realignment scheme. For example:
7322 : :
7323 : : for (i=0; i<N; i++)
7324 : : for (j=0; j<M; j++)
7325 : : s += a[i+j];
7326 : :
7327 : : When vectorizing the i-loop in the above example, the step between
7328 : : consecutive vector loads is 1, and so the misalignment does not remain
7329 : : fixed across the execution of the inner-loop, and the realignment cannot
7330 : : be optimized (as illustrated in the following pseudo vectorized loop):
7331 : :
7332 : : for (i=0; i<N; i+=4)
7333 : : for (j=0; j<M; j++){
7334 : : vs += vp[i+j]; // misalignment of &vp[i+j] is {0,1,2,3,0,1,2,3,...}
7335 : : // when j is {0,1,2,3,4,5,6,7,...} respectively.
7336 : : // (assuming that we start from an aligned address).
7337 : : }
7338 : :
7339 : : We therefore have to use the unoptimized realignment scheme:
7340 : :
7341 : : for (i=0; i<N; i+=4)
7342 : : for (j=k; j<M; j+=4)
7343 : : vs += vp[i+j]; // misalignment of &vp[i+j] is always k (assuming
7344 : : // that the misalignment of the initial address is
7345 : : // 0).
7346 : :
7347 : : The loop can then be vectorized as follows:
7348 : :
7349 : : for (k=0; k<4; k++){
7350 : : rt = get_realignment_token (&vp[k]);
7351 : : for (i=0; i<N; i+=4){
7352 : : v1 = vp[i+k];
7353 : : for (j=k; j<M; j+=4){
7354 : : v2 = vp[i+j+VS-1];
7355 : : va = REALIGN_LOAD <v1,v2,rt>;
7356 : : vs += va;
7357 : : v1 = v2;
7358 : : }
7359 : : }
7360 : : } */
7361 : :
7362 : 1369382 : if (DR_IS_READ (dr))
7363 : : {
7364 : 521010 : if (can_implement_p (vec_realign_load_optab, mode)
7365 : 521010 : && (!targetm.vectorize.builtin_mask_for_load
7366 : 0 : || targetm.vectorize.builtin_mask_for_load ()))
7367 : : {
7368 : : /* If we are doing SLP then the accesses need not have the
7369 : : same alignment, instead it depends on the SLP group size. */
7370 : 0 : if (loop_vinfo
7371 : 0 : && STMT_SLP_TYPE (stmt_info)
7372 : 0 : && STMT_VINFO_GROUPED_ACCESS (stmt_info)
7373 : 0 : && !multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
7374 : 0 : * (DR_GROUP_SIZE
7375 : 0 : (DR_GROUP_FIRST_ELEMENT (stmt_info))),
7376 : 0 : TYPE_VECTOR_SUBPARTS (vectype)))
7377 : : ;
7378 : 0 : else if (!loop_vinfo
7379 : 0 : || (nested_in_vect_loop
7380 : 0 : && maybe_ne (TREE_INT_CST_LOW (DR_STEP (dr)),
7381 : 0 : GET_MODE_SIZE (TYPE_MODE (vectype)))))
7382 : 0 : return dr_explicit_realign;
7383 : : else
7384 : 0 : return dr_explicit_realign_optimized;
7385 : : }
7386 : : }
7387 : :
7388 : 1369382 : bool is_packed = false;
7389 : 1369382 : tree type = TREE_TYPE (DR_REF (dr));
7390 : 1369382 : if (misalignment == DR_MISALIGNMENT_UNKNOWN)
7391 : 981256 : is_packed = not_size_aligned (DR_REF (dr));
7392 : 1369382 : if (targetm.vectorize.support_vector_misalignment (mode, type, misalignment,
7393 : : is_packed))
7394 : : return dr_unaligned_supported;
7395 : :
7396 : : /* Unsupported. */
7397 : : return dr_unaligned_unsupported;
7398 : : }
|
