Line data Source code
1 : /* Data references and dependences detectors.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Sebastian Pop <pop@cri.ensmp.fr>
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* This pass walks a given loop structure searching for array
22 : references. The information about the array accesses is recorded
23 : in DATA_REFERENCE structures.
24 :
25 : The basic test for determining the dependences is:
26 : given two access functions chrec1 and chrec2 to a same array, and
27 : x and y two vectors from the iteration domain, the same element of
28 : the array is accessed twice at iterations x and y if and only if:
29 : | chrec1 (x) == chrec2 (y).
30 :
31 : The goals of this analysis are:
32 :
33 : - to determine the independence: the relation between two
34 : independent accesses is qualified with the chrec_known (this
35 : information allows a loop parallelization),
36 :
37 : - when two data references access the same data, to qualify the
38 : dependence relation with classic dependence representations:
39 :
40 : - distance vectors
41 : - direction vectors
42 : - loop carried level dependence
43 : - polyhedron dependence
44 : or with the chains of recurrences based representation,
45 :
46 : - to define a knowledge base for storing the data dependence
47 : information,
48 :
49 : - to define an interface to access this data.
50 :
51 :
52 : Definitions:
53 :
54 : - subscript: given two array accesses a subscript is the tuple
55 : composed of the access functions for a given dimension. Example:
56 : Given A[f1][f2][f3] and B[g1][g2][g3], there are three subscripts:
57 : (f1, g1), (f2, g2), (f3, g3).
58 :
59 : - Diophantine equation: an equation whose coefficients and
60 : solutions are integer constants, for example the equation
61 : | 3*x + 2*y = 1
62 : has an integer solution x = 1 and y = -1.
63 :
64 : References:
65 :
66 : - "Advanced Compilation for High Performance Computing" by Randy
67 : Allen and Ken Kennedy.
68 : http://citeseer.ist.psu.edu/goff91practical.html
69 :
70 : - "Loop Transformations for Restructuring Compilers - The Foundations"
71 : by Utpal Banerjee.
72 :
73 :
74 : */
75 :
76 : #define INCLUDE_ALGORITHM
77 : #include "config.h"
78 : #include "system.h"
79 : #include "coretypes.h"
80 : #include "backend.h"
81 : #include "rtl.h"
82 : #include "tree.h"
83 : #include "gimple.h"
84 : #include "gimple-pretty-print.h"
85 : #include "alias.h"
86 : #include "fold-const.h"
87 : #include "expr.h"
88 : #include "gimple-iterator.h"
89 : #include "tree-ssa-loop-niter.h"
90 : #include "tree-ssa-loop.h"
91 : #include "tree-ssa.h"
92 : #include "cfgloop.h"
93 : #include "tree-data-ref.h"
94 : #include "tree-scalar-evolution.h"
95 : #include "dumpfile.h"
96 : #include "tree-affine.h"
97 : #include "builtins.h"
98 : #include "tree-eh.h"
99 : #include "ssa.h"
100 : #include "internal-fn.h"
101 : #include "vr-values.h"
102 : #include "range-op.h"
103 : #include "tree-ssa-loop-ivopts.h"
104 : #include "calls.h"
105 :
106 : static struct datadep_stats
107 : {
108 : int num_dependence_tests;
109 : int num_dependence_dependent;
110 : int num_dependence_independent;
111 : int num_dependence_undetermined;
112 :
113 : int num_subscript_tests;
114 : int num_subscript_undetermined;
115 : int num_same_subscript_function;
116 :
117 : int num_ziv;
118 : int num_ziv_independent;
119 : int num_ziv_dependent;
120 : int num_ziv_unimplemented;
121 :
122 : int num_siv;
123 : int num_siv_independent;
124 : int num_siv_dependent;
125 : int num_siv_unimplemented;
126 :
127 : int num_miv;
128 : int num_miv_independent;
129 : int num_miv_dependent;
130 : int num_miv_unimplemented;
131 : } dependence_stats;
132 :
133 : static bool subscript_dependence_tester_1 (struct data_dependence_relation *,
134 : unsigned int, unsigned int,
135 : class loop *);
136 : /* Returns true iff A divides B. */
137 :
138 : static inline bool
139 2075 : tree_fold_divides_p (const_tree a, const_tree b)
140 : {
141 2075 : gcc_assert (TREE_CODE (a) == INTEGER_CST);
142 2075 : gcc_assert (TREE_CODE (b) == INTEGER_CST);
143 2075 : return integer_zerop (int_const_binop (TRUNC_MOD_EXPR, b, a));
144 : }
145 :
146 : /* Returns true iff A divides B. */
147 :
148 : static inline bool
149 1689847 : int_divides_p (lambda_int a, lambda_int b)
150 : {
151 1689847 : return ((b % a) == 0);
152 : }
153 :
154 : /* Return true if reference REF contains a union access. */
155 :
156 : static bool
157 484021 : ref_contains_union_access_p (tree ref)
158 : {
159 530839 : while (handled_component_p (ref))
160 : {
161 105503 : ref = TREE_OPERAND (ref, 0);
162 211006 : if (TREE_CODE (TREE_TYPE (ref)) == UNION_TYPE
163 105503 : || TREE_CODE (TREE_TYPE (ref)) == QUAL_UNION_TYPE)
164 : return true;
165 : }
166 : return false;
167 : }
168 :
169 :
170 :
171 : /* Dump into FILE all the data references from DATAREFS. */
172 :
173 : static void
174 0 : dump_data_references (FILE *file, vec<data_reference_p> datarefs)
175 : {
176 0 : for (data_reference *dr : datarefs)
177 0 : dump_data_reference (file, dr);
178 0 : }
179 :
180 : /* Unified dump into FILE all the data references from DATAREFS. */
181 :
182 : DEBUG_FUNCTION void
183 0 : debug (vec<data_reference_p> &ref)
184 : {
185 0 : dump_data_references (stderr, ref);
186 0 : }
187 :
188 : DEBUG_FUNCTION void
189 0 : debug (vec<data_reference_p> *ptr)
190 : {
191 0 : if (ptr)
192 0 : debug (*ptr);
193 : else
194 0 : fprintf (stderr, "<nil>\n");
195 0 : }
196 :
197 :
198 : /* Dump into STDERR all the data references from DATAREFS. */
199 :
200 : DEBUG_FUNCTION void
201 0 : debug_data_references (vec<data_reference_p> datarefs)
202 : {
203 0 : dump_data_references (stderr, datarefs);
204 0 : }
205 :
206 : /* Print to STDERR the data_reference DR. */
207 :
208 : DEBUG_FUNCTION void
209 0 : debug_data_reference (struct data_reference *dr)
210 : {
211 0 : dump_data_reference (stderr, dr);
212 0 : }
213 :
214 : /* Dump function for a DATA_REFERENCE structure. */
215 :
216 : void
217 3486 : dump_data_reference (FILE *outf,
218 : struct data_reference *dr)
219 : {
220 3486 : unsigned int i;
221 :
222 3486 : fprintf (outf, "#(Data Ref: \n");
223 3486 : fprintf (outf, "# bb: %d \n", gimple_bb (DR_STMT (dr))->index);
224 3486 : fprintf (outf, "# stmt: ");
225 3486 : print_gimple_stmt (outf, DR_STMT (dr), 0);
226 3486 : fprintf (outf, "# ref: ");
227 3486 : print_generic_stmt (outf, DR_REF (dr));
228 3486 : fprintf (outf, "# base_object: ");
229 3486 : print_generic_stmt (outf, DR_BASE_OBJECT (dr));
230 :
231 10810 : for (i = 0; i < DR_NUM_DIMENSIONS (dr); i++)
232 : {
233 3838 : fprintf (outf, "# Access function %d: ", i);
234 3838 : print_generic_stmt (outf, DR_ACCESS_FN (dr, i));
235 : }
236 3486 : fprintf (outf, "#)\n");
237 3486 : }
238 :
239 : /* Unified dump function for a DATA_REFERENCE structure. */
240 :
241 : DEBUG_FUNCTION void
242 0 : debug (data_reference &ref)
243 : {
244 0 : dump_data_reference (stderr, &ref);
245 0 : }
246 :
247 : DEBUG_FUNCTION void
248 0 : debug (data_reference *ptr)
249 : {
250 0 : if (ptr)
251 0 : debug (*ptr);
252 : else
253 0 : fprintf (stderr, "<nil>\n");
254 0 : }
255 :
256 :
257 : /* Dumps the affine function described by FN to the file OUTF. */
258 :
259 : DEBUG_FUNCTION void
260 32888 : dump_affine_function (FILE *outf, affine_fn fn)
261 : {
262 32888 : unsigned i;
263 32888 : tree coef;
264 :
265 32888 : print_generic_expr (outf, fn[0], TDF_SLIM);
266 69504 : for (i = 1; fn.iterate (i, &coef); i++)
267 : {
268 3728 : fprintf (outf, " + ");
269 3728 : print_generic_expr (outf, coef, TDF_SLIM);
270 3728 : fprintf (outf, " * x_%u", i);
271 : }
272 32888 : }
273 :
274 : /* Dumps the conflict function CF to the file OUTF. */
275 :
276 : DEBUG_FUNCTION void
277 164472 : dump_conflict_function (FILE *outf, conflict_function *cf)
278 : {
279 164472 : unsigned i;
280 :
281 164472 : if (cf->n == NO_DEPENDENCE)
282 125446 : fprintf (outf, "no dependence");
283 39026 : else if (cf->n == NOT_KNOWN)
284 6138 : fprintf (outf, "not known");
285 : else
286 : {
287 65776 : for (i = 0; i < cf->n; i++)
288 : {
289 32888 : if (i != 0)
290 0 : fprintf (outf, " ");
291 32888 : fprintf (outf, "[");
292 32888 : dump_affine_function (outf, cf->fns[i]);
293 32888 : fprintf (outf, "]");
294 : }
295 : }
296 164472 : }
297 :
298 : /* Dump function for a SUBSCRIPT structure. */
299 :
300 : DEBUG_FUNCTION void
301 844 : dump_subscript (FILE *outf, struct subscript *subscript)
302 : {
303 844 : conflict_function *cf = SUB_CONFLICTS_IN_A (subscript);
304 :
305 844 : fprintf (outf, "\n (subscript \n");
306 844 : fprintf (outf, " iterations_that_access_an_element_twice_in_A: ");
307 844 : dump_conflict_function (outf, cf);
308 844 : if (CF_NONTRIVIAL_P (cf))
309 : {
310 844 : tree last_iteration = SUB_LAST_CONFLICT (subscript);
311 844 : fprintf (outf, "\n last_conflict: ");
312 844 : print_generic_expr (outf, last_iteration);
313 : }
314 :
315 844 : cf = SUB_CONFLICTS_IN_B (subscript);
316 844 : fprintf (outf, "\n iterations_that_access_an_element_twice_in_B: ");
317 844 : dump_conflict_function (outf, cf);
318 844 : if (CF_NONTRIVIAL_P (cf))
319 : {
320 844 : tree last_iteration = SUB_LAST_CONFLICT (subscript);
321 844 : fprintf (outf, "\n last_conflict: ");
322 844 : print_generic_expr (outf, last_iteration);
323 : }
324 :
325 844 : fprintf (outf, "\n (Subscript distance: ");
326 844 : print_generic_expr (outf, SUB_DISTANCE (subscript));
327 844 : fprintf (outf, " ))\n");
328 844 : }
329 :
330 : /* Print the classic direction vector DIRV to OUTF. */
331 :
332 : DEBUG_FUNCTION void
333 784 : print_direction_vector (FILE *outf,
334 : lambda_vector dirv,
335 : int length)
336 : {
337 784 : int eq;
338 :
339 1704 : for (eq = 0; eq < length; eq++)
340 : {
341 920 : enum data_dependence_direction dir = ((enum data_dependence_direction)
342 920 : dirv[eq]);
343 :
344 920 : switch (dir)
345 : {
346 146 : case dir_positive:
347 146 : fprintf (outf, " +");
348 146 : break;
349 8 : case dir_negative:
350 8 : fprintf (outf, " -");
351 8 : break;
352 766 : case dir_equal:
353 766 : fprintf (outf, " =");
354 766 : break;
355 0 : case dir_positive_or_equal:
356 0 : fprintf (outf, " +=");
357 0 : break;
358 0 : case dir_positive_or_negative:
359 0 : fprintf (outf, " +-");
360 0 : break;
361 0 : case dir_negative_or_equal:
362 0 : fprintf (outf, " -=");
363 0 : break;
364 0 : case dir_star:
365 0 : fprintf (outf, " *");
366 0 : break;
367 0 : default:
368 0 : fprintf (outf, "indep");
369 0 : break;
370 : }
371 : }
372 784 : fprintf (outf, "\n");
373 784 : }
374 :
375 : /* Print a vector of direction vectors. */
376 :
377 : DEBUG_FUNCTION void
378 0 : print_dir_vectors (FILE *outf, vec<lambda_vector> dir_vects,
379 : int length)
380 : {
381 0 : for (lambda_vector v : dir_vects)
382 0 : print_direction_vector (outf, v, length);
383 0 : }
384 :
385 : /* Print out a vector VEC of length N to OUTFILE. */
386 :
387 : DEBUG_FUNCTION void
388 4917 : print_lambda_vector (FILE * outfile, lambda_vector vector, int n)
389 : {
390 4917 : int i;
391 :
392 10255 : for (i = 0; i < n; i++)
393 5338 : fprintf (outfile, HOST_WIDE_INT_PRINT_DEC " ", vector[i]);
394 4917 : fprintf (outfile, "\n");
395 4917 : }
396 :
397 : /* Print a vector of distance vectors. */
398 :
399 : DEBUG_FUNCTION void
400 0 : print_dist_vectors (FILE *outf, vec<lambda_vector> dist_vects,
401 : int length)
402 : {
403 0 : for (lambda_vector v : dist_vects)
404 0 : print_lambda_vector (outf, v, length);
405 0 : }
406 :
407 : /* Dump function for a DATA_DEPENDENCE_RELATION structure. */
408 :
409 : DEBUG_FUNCTION void
410 1585 : dump_data_dependence_relation (FILE *outf, const data_dependence_relation *ddr)
411 : {
412 1585 : struct data_reference *dra, *drb;
413 :
414 1585 : fprintf (outf, "(Data Dep: \n");
415 :
416 1585 : if (!ddr || DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
417 : {
418 399 : if (ddr)
419 : {
420 399 : dra = DDR_A (ddr);
421 399 : drb = DDR_B (ddr);
422 399 : if (dra)
423 399 : dump_data_reference (outf, dra);
424 : else
425 0 : fprintf (outf, " (nil)\n");
426 399 : if (drb)
427 399 : dump_data_reference (outf, drb);
428 : else
429 0 : fprintf (outf, " (nil)\n");
430 : }
431 399 : fprintf (outf, " (don't know)\n)\n");
432 399 : return;
433 : }
434 :
435 1186 : dra = DDR_A (ddr);
436 1186 : drb = DDR_B (ddr);
437 1186 : dump_data_reference (outf, dra);
438 1186 : dump_data_reference (outf, drb);
439 :
440 1186 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
441 426 : fprintf (outf, " (no dependence)\n");
442 :
443 760 : else if (DDR_ARE_DEPENDENT (ddr) == NULL_TREE)
444 : {
445 : unsigned int i;
446 : class loop *loopi;
447 :
448 : subscript *sub;
449 1604 : FOR_EACH_VEC_ELT (DDR_SUBSCRIPTS (ddr), i, sub)
450 : {
451 844 : fprintf (outf, " access_fn_A: ");
452 844 : print_generic_stmt (outf, SUB_ACCESS_FN (sub, 0));
453 844 : fprintf (outf, " access_fn_B: ");
454 844 : print_generic_stmt (outf, SUB_ACCESS_FN (sub, 1));
455 844 : dump_subscript (outf, sub);
456 : }
457 :
458 760 : fprintf (outf, " loop nest: (");
459 2386 : FOR_EACH_VEC_ELT (DDR_LOOP_NEST (ddr), i, loopi)
460 866 : fprintf (outf, "%d ", loopi->num);
461 760 : fprintf (outf, ")\n");
462 :
463 3843 : for (i = 0; i < DDR_NUM_DIST_VECTS (ddr); i++)
464 : {
465 784 : fprintf (outf, " distance_vector: ");
466 784 : print_lambda_vector (outf, DDR_DIST_VECT (ddr, i),
467 1568 : DDR_NB_LOOPS (ddr));
468 : }
469 :
470 1544 : for (i = 0; i < DDR_NUM_DIR_VECTS (ddr); i++)
471 : {
472 784 : fprintf (outf, " direction_vector: ");
473 784 : print_direction_vector (outf, DDR_DIR_VECT (ddr, i),
474 1568 : DDR_NB_LOOPS (ddr));
475 : }
476 : }
477 :
478 1186 : fprintf (outf, ")\n");
479 : }
480 :
481 : /* Debug version. */
482 :
483 : DEBUG_FUNCTION void
484 0 : debug_data_dependence_relation (const struct data_dependence_relation *ddr)
485 : {
486 0 : dump_data_dependence_relation (stderr, ddr);
487 0 : }
488 :
489 : /* Dump into FILE all the dependence relations from DDRS. */
490 :
491 : DEBUG_FUNCTION void
492 308 : dump_data_dependence_relations (FILE *file, const vec<ddr_p> &ddrs)
493 : {
494 2479 : for (auto ddr : ddrs)
495 1585 : dump_data_dependence_relation (file, ddr);
496 308 : }
497 :
498 : DEBUG_FUNCTION void
499 0 : debug (vec<ddr_p> &ref)
500 : {
501 0 : dump_data_dependence_relations (stderr, ref);
502 0 : }
503 :
504 : DEBUG_FUNCTION void
505 0 : debug (vec<ddr_p> *ptr)
506 : {
507 0 : if (ptr)
508 0 : debug (*ptr);
509 : else
510 0 : fprintf (stderr, "<nil>\n");
511 0 : }
512 :
513 :
514 : /* Dump to STDERR all the dependence relations from DDRS. */
515 :
516 : DEBUG_FUNCTION void
517 0 : debug_data_dependence_relations (vec<ddr_p> ddrs)
518 : {
519 0 : dump_data_dependence_relations (stderr, ddrs);
520 0 : }
521 :
522 : /* Dumps the distance and direction vectors in FILE. DDRS contains
523 : the dependence relations, and VECT_SIZE is the size of the
524 : dependence vectors, or in other words the number of loops in the
525 : considered nest. */
526 :
527 : DEBUG_FUNCTION void
528 0 : dump_dist_dir_vectors (FILE *file, vec<ddr_p> ddrs)
529 : {
530 0 : for (data_dependence_relation *ddr : ddrs)
531 0 : if (DDR_ARE_DEPENDENT (ddr) == NULL_TREE && DDR_AFFINE_P (ddr))
532 : {
533 0 : for (lambda_vector v : DDR_DIST_VECTS (ddr))
534 : {
535 0 : fprintf (file, "DISTANCE_V (");
536 0 : print_lambda_vector (file, v, DDR_NB_LOOPS (ddr));
537 0 : fprintf (file, ")\n");
538 : }
539 :
540 0 : for (lambda_vector v : DDR_DIR_VECTS (ddr))
541 : {
542 0 : fprintf (file, "DIRECTION_V (");
543 0 : print_direction_vector (file, v, DDR_NB_LOOPS (ddr));
544 0 : fprintf (file, ")\n");
545 : }
546 : }
547 :
548 0 : fprintf (file, "\n\n");
549 0 : }
550 :
551 : /* Dumps the data dependence relations DDRS in FILE. */
552 :
553 : DEBUG_FUNCTION void
554 0 : dump_ddrs (FILE *file, vec<ddr_p> ddrs)
555 : {
556 0 : for (data_dependence_relation *ddr : ddrs)
557 0 : dump_data_dependence_relation (file, ddr);
558 :
559 0 : fprintf (file, "\n\n");
560 0 : }
561 :
562 : DEBUG_FUNCTION void
563 0 : debug_ddrs (vec<ddr_p> ddrs)
564 : {
565 0 : dump_ddrs (stderr, ddrs);
566 0 : }
567 :
568 : /* If RESULT_RANGE is nonnull, set *RESULT_RANGE to the range of
569 : OP0 CODE OP1, where:
570 :
571 : - OP0 CODE OP1 has integral type TYPE
572 : - the range of OP0 is given by OP0_RANGE and
573 : - the range of OP1 is given by OP1_RANGE.
574 :
575 : Independently of RESULT_RANGE, try to compute:
576 :
577 : DELTA = ((sizetype) OP0 CODE (sizetype) OP1)
578 : - (sizetype) (OP0 CODE OP1)
579 :
580 : as a constant and subtract DELTA from the ssizetype constant in *OFF.
581 : Return true on success, or false if DELTA is not known at compile time.
582 :
583 : Truncation and sign changes are known to distribute over CODE, i.e.
584 :
585 : (itype) (A CODE B) == (itype) A CODE (itype) B
586 :
587 : for any integral type ITYPE whose precision is no greater than the
588 : precision of A and B. */
589 :
590 : static bool
591 4654814 : compute_distributive_range (tree type, irange &op0_range,
592 : tree_code code, irange &op1_range,
593 : tree *off, irange *result_range)
594 : {
595 4654814 : gcc_assert (INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type));
596 4654814 : if (result_range)
597 : {
598 1076973 : range_op_handler op (code);
599 1076973 : if (!op.fold_range (*result_range, type, op0_range, op1_range))
600 0 : result_range->set_varying (type);
601 : }
602 :
603 : /* The distributive property guarantees that if TYPE is no narrower
604 : than SIZETYPE,
605 :
606 : (sizetype) (OP0 CODE OP1) == (sizetype) OP0 CODE (sizetype) OP1
607 :
608 : and so we can treat DELTA as zero. */
609 4654814 : if (TYPE_PRECISION (type) >= TYPE_PRECISION (sizetype))
610 : return true;
611 :
612 : /* If overflow is undefined, we can assume that:
613 :
614 : X == (ssizetype) OP0 CODE (ssizetype) OP1
615 :
616 : is within the range of TYPE, i.e.:
617 :
618 : X == (ssizetype) (TYPE) X
619 :
620 : Distributing the (TYPE) truncation over X gives:
621 :
622 : X == (ssizetype) (OP0 CODE OP1)
623 :
624 : Casting both sides to sizetype and distributing the sizetype cast
625 : over X gives:
626 :
627 : (sizetype) OP0 CODE (sizetype) OP1 == (sizetype) (OP0 CODE OP1)
628 :
629 : and so we can treat DELTA as zero. */
630 286881 : if (TYPE_OVERFLOW_UNDEFINED (type))
631 : return true;
632 :
633 : /* Compute the range of:
634 :
635 : (ssizetype) OP0 CODE (ssizetype) OP1
636 :
637 : The distributive property guarantees that this has the same bitpattern as:
638 :
639 : (sizetype) OP0 CODE (sizetype) OP1
640 :
641 : but its range is more conducive to analysis. */
642 113093 : range_cast (op0_range, ssizetype);
643 113093 : range_cast (op1_range, ssizetype);
644 113093 : int_range_max wide_range;
645 113093 : range_op_handler op (code);
646 113093 : bool saved_flag_wrapv = flag_wrapv;
647 113093 : flag_wrapv = 1;
648 113093 : if (!op.fold_range (wide_range, ssizetype, op0_range, op1_range))
649 0 : wide_range.set_varying (ssizetype);;
650 113093 : flag_wrapv = saved_flag_wrapv;
651 113093 : if (wide_range.num_pairs () != 1
652 113093 : || wide_range.varying_p () || wide_range.undefined_p ())
653 : return false;
654 :
655 93048 : wide_int lb = wide_range.lower_bound ();
656 93048 : wide_int ub = wide_range.upper_bound ();
657 :
658 : /* Calculate the number of times that each end of the range overflows or
659 : underflows TYPE. We can only calculate DELTA if the numbers match. */
660 93048 : unsigned int precision = TYPE_PRECISION (type);
661 93048 : if (!TYPE_UNSIGNED (type))
662 : {
663 242 : wide_int type_min = wi::mask (precision - 1, true, lb.get_precision ());
664 242 : lb -= type_min;
665 242 : ub -= type_min;
666 242 : }
667 93048 : wide_int upper_bits = wi::mask (precision, true, lb.get_precision ());
668 93048 : lb &= upper_bits;
669 93048 : ub &= upper_bits;
670 93048 : if (lb != ub)
671 : return false;
672 :
673 : /* OP0 CODE OP1 overflows exactly arshift (LB, PRECISION) times, with
674 : negative values indicating underflow. The low PRECISION bits of LB
675 : are clear, so DELTA is therefore LB (== UB). */
676 29065 : *off = wide_int_to_tree (ssizetype, wi::to_wide (*off) - lb);
677 29065 : return true;
678 113093 : }
679 :
680 : /* Return true if (sizetype) OP == (sizetype) (TO_TYPE) OP,
681 : given that OP has type FROM_TYPE and range RANGE. Both TO_TYPE and
682 : FROM_TYPE are integral types. */
683 :
684 : static bool
685 2645718 : nop_conversion_for_offset_p (tree to_type, tree from_type, irange &range)
686 : {
687 2645718 : gcc_assert (INTEGRAL_TYPE_P (to_type)
688 : && INTEGRAL_TYPE_P (from_type)
689 : && !TYPE_OVERFLOW_TRAPS (to_type)
690 : && !TYPE_OVERFLOW_TRAPS (from_type));
691 :
692 : /* Converting to something no narrower than sizetype and then to sizetype
693 : is equivalent to converting directly to sizetype. */
694 2645718 : if (TYPE_PRECISION (to_type) >= TYPE_PRECISION (sizetype))
695 : return true;
696 :
697 : /* Check whether TO_TYPE can represent all values that FROM_TYPE can. */
698 88476 : if (TYPE_PRECISION (from_type) < TYPE_PRECISION (to_type)
699 88476 : && (TYPE_UNSIGNED (from_type) || !TYPE_UNSIGNED (to_type)))
700 : return true;
701 :
702 : /* For narrowing conversions, we could in principle test whether
703 : the bits in FROM_TYPE but not in TO_TYPE have a fixed value
704 : and apply a constant adjustment.
705 :
706 : For other conversions (which involve a sign change) we could
707 : check that the signs are always equal, and apply a constant
708 : adjustment if the signs are negative.
709 :
710 : However, both cases should be rare. */
711 73290 : return range_fits_type_p (&range, TYPE_PRECISION (to_type),
712 146580 : TYPE_SIGN (to_type));
713 : }
714 :
715 : static void
716 : split_constant_offset (tree type, tree *var, tree *off,
717 : irange *result_range,
718 : hash_map<tree, std::pair<tree, tree> > &cache,
719 : unsigned *limit);
720 :
721 : /* Helper function for split_constant_offset. If TYPE is a pointer type,
722 : try to express OP0 CODE OP1 as:
723 :
724 : POINTER_PLUS <*VAR, (sizetype) *OFF>
725 :
726 : where:
727 :
728 : - *VAR has type TYPE
729 : - *OFF is a constant of type ssizetype.
730 :
731 : If TYPE is an integral type, try to express (sizetype) (OP0 CODE OP1) as:
732 :
733 : *VAR + (sizetype) *OFF
734 :
735 : where:
736 :
737 : - *VAR has type sizetype
738 : - *OFF is a constant of type ssizetype.
739 :
740 : In both cases, OP0 CODE OP1 has type TYPE.
741 :
742 : Return true on success. A false return value indicates that we can't
743 : do better than set *OFF to zero.
744 :
745 : When returning true, set RESULT_RANGE to the range of OP0 CODE OP1,
746 : if RESULT_RANGE is nonnull and if we can do better than assume VR_VARYING.
747 :
748 : CACHE caches {*VAR, *OFF} pairs for SSA names that we've previously
749 : visited. LIMIT counts down the number of SSA names that we are
750 : allowed to process before giving up. */
751 :
752 : static bool
753 59884463 : split_constant_offset_1 (tree type, tree op0, enum tree_code code, tree op1,
754 : tree *var, tree *off, irange *result_range,
755 : hash_map<tree, std::pair<tree, tree> > &cache,
756 : unsigned *limit)
757 : {
758 59884463 : tree var0, var1;
759 59884463 : tree off0, off1;
760 59884463 : int_range_max op0_range, op1_range;
761 :
762 59884463 : *var = NULL_TREE;
763 59884463 : *off = NULL_TREE;
764 :
765 59884463 : if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type))
766 : return false;
767 :
768 59883841 : if (TREE_CODE (op0) == SSA_NAME
769 59883841 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op0))
770 : return false;
771 59883452 : if (op1
772 7672589 : && TREE_CODE (op1) == SSA_NAME
773 62288439 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op1))
774 : return false;
775 :
776 59883452 : switch (code)
777 : {
778 17887218 : case INTEGER_CST:
779 17887218 : *var = size_int (0);
780 17887218 : *off = fold_convert (ssizetype, op0);
781 17887218 : if (result_range)
782 : {
783 1448247 : wide_int w = wi::to_wide (op0);
784 1448247 : result_range->set (TREE_TYPE (op0), w, w);
785 1448247 : }
786 : return true;
787 :
788 2207354 : case POINTER_PLUS_EXPR:
789 2207354 : split_constant_offset (op0, &var0, &off0, nullptr, cache, limit);
790 2207354 : split_constant_offset (op1, &var1, &off1, nullptr, cache, limit);
791 2207354 : *var = fold_build2 (POINTER_PLUS_EXPR, type, var0, var1);
792 2207354 : *off = size_binop (PLUS_EXPR, off0, off1);
793 2207354 : return true;
794 :
795 2473063 : case PLUS_EXPR:
796 2473063 : case MINUS_EXPR:
797 2473063 : split_constant_offset (op0, &var0, &off0, &op0_range, cache, limit);
798 2473063 : split_constant_offset (op1, &var1, &off1, &op1_range, cache, limit);
799 2473063 : *off = size_binop (code, off0, off1);
800 2473063 : if (!compute_distributive_range (type, op0_range, code, op1_range,
801 : off, result_range))
802 : return false;
803 2409053 : *var = fold_build2 (code, sizetype, var0, var1);
804 2409053 : return true;
805 :
806 2644448 : case MULT_EXPR:
807 2644448 : if (TREE_CODE (op1) != INTEGER_CST)
808 : return false;
809 :
810 2181751 : split_constant_offset (op0, &var0, &off0, &op0_range, cache, limit);
811 2181751 : op1_range.set (TREE_TYPE (op1), wi::to_wide (op1), wi::to_wide (op1));
812 2181751 : *off = size_binop (MULT_EXPR, off0, fold_convert (ssizetype, op1));
813 2181751 : if (!compute_distributive_range (type, op0_range, code, op1_range,
814 : off, result_range))
815 : return false;
816 2161733 : *var = fold_build2 (MULT_EXPR, sizetype, var0,
817 : fold_convert (sizetype, op1));
818 2161733 : return true;
819 :
820 10547970 : case ADDR_EXPR:
821 10547970 : {
822 10547970 : tree base, poffset;
823 10547970 : poly_int64 pbitsize, pbitpos, pbytepos;
824 10547970 : machine_mode pmode;
825 10547970 : int punsignedp, preversep, pvolatilep;
826 :
827 10547970 : op0 = TREE_OPERAND (op0, 0);
828 10547970 : base
829 10547970 : = get_inner_reference (op0, &pbitsize, &pbitpos, &poffset, &pmode,
830 : &punsignedp, &preversep, &pvolatilep);
831 :
832 21095940 : if (!multiple_p (pbitpos, BITS_PER_UNIT, &pbytepos))
833 : return false;
834 10547970 : base = build_fold_addr_expr (base);
835 10547970 : off0 = ssize_int (pbytepos);
836 :
837 10547970 : if (poffset)
838 : {
839 1531 : split_constant_offset (poffset, &poffset, &off1, nullptr,
840 : cache, limit);
841 1531 : off0 = size_binop (PLUS_EXPR, off0, off1);
842 1531 : base = fold_build_pointer_plus (base, poffset);
843 : }
844 :
845 10547970 : var0 = fold_convert (type, base);
846 :
847 : /* If variable length types are involved, punt, otherwise casts
848 : might be converted into ARRAY_REFs in gimplify_conversion.
849 : To compute that ARRAY_REF's element size TYPE_SIZE_UNIT, which
850 : possibly no longer appears in current GIMPLE, might resurface.
851 : This perhaps could run
852 : if (CONVERT_EXPR_P (var0))
853 : {
854 : gimplify_conversion (&var0);
855 : // Attempt to fill in any within var0 found ARRAY_REF's
856 : // element size from corresponding op embedded ARRAY_REF,
857 : // if unsuccessful, just punt.
858 : } */
859 21510940 : while (POINTER_TYPE_P (type))
860 10962970 : type = TREE_TYPE (type);
861 10547970 : if (int_size_in_bytes (type) < 0)
862 : return false;
863 :
864 10521652 : *var = var0;
865 10521652 : *off = off0;
866 10521652 : return true;
867 : }
868 :
869 16211477 : case SSA_NAME:
870 16211477 : {
871 16211477 : gimple *def_stmt = SSA_NAME_DEF_STMT (op0);
872 16211477 : enum tree_code subcode;
873 :
874 16211477 : if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
875 : return false;
876 :
877 8862615 : subcode = gimple_assign_rhs_code (def_stmt);
878 :
879 : /* We are using a cache to avoid un-CSEing large amounts of code. */
880 8862615 : bool use_cache = false;
881 8862615 : if (!has_single_use (op0)
882 8862615 : && (subcode == POINTER_PLUS_EXPR
883 4579845 : || subcode == PLUS_EXPR
884 : || subcode == MINUS_EXPR
885 : || subcode == MULT_EXPR
886 : || subcode == ADDR_EXPR
887 : || CONVERT_EXPR_CODE_P (subcode)))
888 : {
889 2216733 : use_cache = true;
890 2216733 : bool existed;
891 2216733 : std::pair<tree, tree> &e = cache.get_or_insert (op0, &existed);
892 2216733 : if (existed)
893 : {
894 31894 : if (integer_zerop (e.second))
895 31894 : return false;
896 1197 : *var = e.first;
897 1197 : *off = e.second;
898 : /* The caller sets the range in this case. */
899 1197 : return true;
900 : }
901 2184839 : e = std::make_pair (op0, ssize_int (0));
902 : }
903 :
904 8830721 : if (*limit == 0)
905 : return false;
906 8829685 : --*limit;
907 :
908 8829685 : var0 = gimple_assign_rhs1 (def_stmt);
909 8829685 : var1 = gimple_assign_rhs2 (def_stmt);
910 :
911 8829685 : bool res = split_constant_offset_1 (type, var0, subcode, var1,
912 : var, off, nullptr, cache, limit);
913 8829685 : if (res && use_cache)
914 1965339 : *cache.get (op0) = std::make_pair (*var, *off);
915 : /* The caller sets the range in this case. */
916 : return res;
917 : }
918 4412783 : CASE_CONVERT:
919 4412783 : {
920 : /* We can only handle the following conversions:
921 :
922 : - Conversions from one pointer type to another pointer type.
923 :
924 : - Conversions from one non-trapping integral type to another
925 : non-trapping integral type. In this case, the recursive
926 : call makes sure that:
927 :
928 : (sizetype) OP0
929 :
930 : can be expressed as a sizetype operation involving VAR and OFF,
931 : and all we need to do is check whether:
932 :
933 : (sizetype) OP0 == (sizetype) (TYPE) OP0
934 :
935 : - Conversions from a non-trapping sizetype-size integral type to
936 : a like-sized pointer type. In this case, the recursive call
937 : makes sure that:
938 :
939 : (sizetype) OP0 == *VAR + (sizetype) *OFF
940 :
941 : and we can convert that to:
942 :
943 : POINTER_PLUS <(TYPE) *VAR, (sizetype) *OFF>
944 :
945 : - Conversions from a sizetype-sized pointer type to a like-sized
946 : non-trapping integral type. In this case, the recursive call
947 : makes sure that:
948 :
949 : OP0 == POINTER_PLUS <*VAR, (sizetype) *OFF>
950 :
951 : where the POINTER_PLUS and *VAR have the same precision as
952 : TYPE (and the same precision as sizetype). Then:
953 :
954 : (sizetype) (TYPE) OP0 == (sizetype) *VAR + (sizetype) *OFF. */
955 4412783 : tree itype = TREE_TYPE (op0);
956 4412783 : if ((POINTER_TYPE_P (itype)
957 3217344 : || (INTEGRAL_TYPE_P (itype) && !TYPE_OVERFLOW_TRAPS (itype)))
958 4412366 : && (POINTER_TYPE_P (type)
959 3163202 : || (INTEGRAL_TYPE_P (type) && !TYPE_OVERFLOW_TRAPS (type)))
960 8825149 : && (POINTER_TYPE_P (type) == POINTER_TYPE_P (itype)
961 1088693 : || (TYPE_PRECISION (type) == TYPE_PRECISION (sizetype)
962 1088693 : && TYPE_PRECISION (itype) == TYPE_PRECISION (sizetype))))
963 : {
964 4412359 : if (POINTER_TYPE_P (type))
965 : {
966 1249157 : split_constant_offset (op0, var, off, nullptr, cache, limit);
967 1249157 : *var = fold_convert (type, *var);
968 : }
969 3163202 : else if (POINTER_TYPE_P (itype))
970 : {
971 517484 : split_constant_offset (op0, var, off, nullptr, cache, limit);
972 517484 : *var = fold_convert (sizetype, *var);
973 : }
974 : else
975 : {
976 2645718 : split_constant_offset (op0, var, off, &op0_range,
977 : cache, limit);
978 2645718 : if (!nop_conversion_for_offset_p (type, itype, op0_range))
979 : return false;
980 2593220 : if (result_range)
981 : {
982 1328435 : *result_range = op0_range;
983 1328435 : range_cast (*result_range, type);
984 : }
985 : }
986 : return true;
987 : }
988 : return false;
989 : }
990 :
991 : default:
992 : return false;
993 : }
994 59884463 : }
995 :
996 : /* If EXP has pointer type, try to express it as:
997 :
998 : POINTER_PLUS <*VAR, (sizetype) *OFF>
999 :
1000 : where:
1001 :
1002 : - *VAR has the same type as EXP
1003 : - *OFF is a constant of type ssizetype.
1004 :
1005 : If EXP has an integral type, try to express (sizetype) EXP as:
1006 :
1007 : *VAR + (sizetype) *OFF
1008 :
1009 : where:
1010 :
1011 : - *VAR has type sizetype
1012 : - *OFF is a constant of type ssizetype.
1013 :
1014 : If EXP_RANGE is nonnull, set it to the range of EXP.
1015 :
1016 : CACHE caches {*VAR, *OFF} pairs for SSA names that we've previously
1017 : visited. LIMIT counts down the number of SSA names that we are
1018 : allowed to process before giving up. */
1019 :
1020 : static void
1021 51054799 : split_constant_offset (tree exp, tree *var, tree *off, irange *exp_range,
1022 : hash_map<tree, std::pair<tree, tree> > &cache,
1023 : unsigned *limit)
1024 : {
1025 51054799 : tree type = TREE_TYPE (exp), op0, op1;
1026 51054799 : enum tree_code code;
1027 :
1028 51054799 : code = TREE_CODE (exp);
1029 51054799 : if (exp_range)
1030 : {
1031 9773595 : exp_range->set_varying (type);
1032 9773595 : if (code == SSA_NAME)
1033 : {
1034 5404010 : int_range_max vr;
1035 10808020 : get_range_query (cfun)->range_of_expr (vr, exp);
1036 5404010 : if (vr.undefined_p ())
1037 4937 : vr.set_varying (TREE_TYPE (exp));
1038 5404010 : tree vr_min, vr_max;
1039 5404010 : value_range_kind vr_kind = get_legacy_range (vr, vr_min, vr_max);
1040 5404010 : wide_int var_min = wi::to_wide (vr_min);
1041 5404010 : wide_int var_max = wi::to_wide (vr_max);
1042 5404010 : wide_int var_nonzero = get_nonzero_bits (exp);
1043 16212030 : vr_kind = intersect_range_with_nonzero_bits (vr_kind,
1044 : &var_min, &var_max,
1045 : var_nonzero,
1046 5404010 : TYPE_SIGN (type));
1047 : /* This check for VR_VARYING is here because the old code
1048 : using get_range_info would return VR_RANGE for the entire
1049 : domain, instead of VR_VARYING. The new code normalizes
1050 : full-domain ranges to VR_VARYING. */
1051 5404010 : if (vr_kind == VR_RANGE || vr_kind == VR_VARYING)
1052 5259830 : exp_range->set (type, var_min, var_max);
1053 5404010 : }
1054 : }
1055 :
1056 51054799 : if (!tree_is_chrec (exp)
1057 51054793 : && get_gimple_rhs_class (TREE_CODE (exp)) != GIMPLE_TERNARY_RHS)
1058 : {
1059 51054778 : extract_ops_from_tree (exp, &code, &op0, &op1);
1060 51054778 : if (split_constant_offset_1 (type, op0, code, op1, var, off,
1061 : exp_range, cache, limit))
1062 39548068 : return;
1063 : }
1064 :
1065 11506731 : *var = exp;
1066 11506731 : if (INTEGRAL_TYPE_P (type))
1067 3507959 : *var = fold_convert (sizetype, *var);
1068 11506731 : *off = ssize_int (0);
1069 :
1070 11506731 : int_range_max r;
1071 3193100 : if (exp_range && code != SSA_NAME
1072 117862 : && get_range_query (cfun)->range_of_expr (r, exp)
1073 11565662 : && !r.undefined_p ())
1074 58931 : *exp_range = r;
1075 11506731 : }
1076 :
1077 : /* Expresses EXP as VAR + OFF, where OFF is a constant. VAR has the same
1078 : type as EXP while OFF has type ssizetype. */
1079 :
1080 : void
1081 35098324 : split_constant_offset (tree exp, tree *var, tree *off)
1082 : {
1083 35098324 : unsigned limit = param_ssa_name_def_chain_limit;
1084 35098324 : static hash_map<tree, std::pair<tree, tree> > *cache;
1085 35098324 : if (!cache)
1086 82443 : cache = new hash_map<tree, std::pair<tree, tree> > (37);
1087 35098324 : split_constant_offset (exp, var, off, nullptr, *cache, &limit);
1088 35098324 : *var = fold_convert (TREE_TYPE (exp), *var);
1089 35098324 : cache->empty ();
1090 35098324 : }
1091 :
1092 : /* Returns the address ADDR of an object in a canonical shape (without nop
1093 : casts, and with type of pointer to the object). */
1094 :
1095 : static tree
1096 16512891 : canonicalize_base_object_address (tree addr)
1097 : {
1098 16512891 : tree orig = addr;
1099 :
1100 16512891 : STRIP_NOPS (addr);
1101 :
1102 : /* The base address may be obtained by casting from integer, in that case
1103 : keep the cast. */
1104 16512891 : if (!POINTER_TYPE_P (TREE_TYPE (addr)))
1105 : return orig;
1106 :
1107 16438753 : if (TREE_CODE (addr) != ADDR_EXPR)
1108 : return addr;
1109 :
1110 9803752 : return build_fold_addr_expr (TREE_OPERAND (addr, 0));
1111 : }
1112 :
1113 : /* Analyze the behavior of memory reference REF within STMT.
1114 : There are two modes:
1115 :
1116 : - BB analysis. In this case we simply split the address into base,
1117 : init and offset components, without reference to any containing loop.
1118 : The resulting base and offset are general expressions and they can
1119 : vary arbitrarily from one iteration of the containing loop to the next.
1120 : The step is always zero.
1121 :
1122 : - loop analysis. In this case we analyze the reference both wrt LOOP
1123 : and on the basis that the reference occurs (is "used") in LOOP;
1124 : see the comment above analyze_scalar_evolution_in_loop for more
1125 : information about this distinction. The base, init, offset and
1126 : step fields are all invariant in LOOP.
1127 :
1128 : Perform BB analysis if LOOP is null, or if LOOP is the function's
1129 : dummy outermost loop. In other cases perform loop analysis.
1130 :
1131 : Return true if the analysis succeeded and store the results in DRB if so.
1132 : BB analysis can only fail for bitfield or reversed-storage accesses.
1133 :
1134 : If NOWRAP_BOUND is nonzero, allow simple_iv to record no-wrap bounds.
1135 : Then, store non-trivial bounds in NOWRAP_BOUND. */
1136 :
1137 : opt_result
1138 17078637 : dr_analyze_innermost (innermost_loop_behavior *drb, tree ref,
1139 : class loop *loop, const gimple *stmt,
1140 : tree *nowrap_bound)
1141 : {
1142 17078637 : poly_int64 pbitsize, pbitpos;
1143 17078637 : tree base, poffset;
1144 17078637 : machine_mode pmode;
1145 17078637 : int punsignedp, preversep, pvolatilep;
1146 17078637 : affine_iv base_iv, offset_iv;
1147 17078637 : tree init, dinit, step;
1148 17078637 : bool in_loop = (loop && loop->num);
1149 :
1150 17078637 : if (dump_file && (dump_flags & TDF_DETAILS))
1151 70330 : fprintf (dump_file, "analyze_innermost: ");
1152 :
1153 17078637 : base = get_inner_reference (ref, &pbitsize, &pbitpos, &poffset, &pmode,
1154 : &punsignedp, &preversep, &pvolatilep);
1155 17078637 : gcc_assert (base != NULL_TREE);
1156 :
1157 17078637 : poly_int64 pbytepos;
1158 17078637 : if (!multiple_p (pbitpos, BITS_PER_UNIT, &pbytepos))
1159 39567 : return opt_result::failure_at (stmt,
1160 : "failed: bit offset alignment.\n");
1161 :
1162 17039070 : if (preversep)
1163 752 : return opt_result::failure_at (stmt,
1164 : "failed: reverse storage order.\n");
1165 :
1166 : /* Calculate the alignment and misalignment for the inner reference. */
1167 17038318 : unsigned int HOST_WIDE_INT bit_base_misalignment;
1168 17038318 : unsigned int bit_base_alignment;
1169 17038318 : get_object_alignment_1 (base, &bit_base_alignment, &bit_base_misalignment);
1170 :
1171 : /* There are no bitfield references remaining in BASE, so the values
1172 : we got back must be whole bytes. */
1173 17038318 : gcc_assert (bit_base_alignment % BITS_PER_UNIT == 0
1174 : && bit_base_misalignment % BITS_PER_UNIT == 0);
1175 17038318 : unsigned int base_alignment = bit_base_alignment / BITS_PER_UNIT;
1176 17038318 : poly_int64 base_misalignment = bit_base_misalignment / BITS_PER_UNIT;
1177 :
1178 17038318 : if (TREE_CODE (base) == MEM_REF)
1179 : {
1180 7379016 : if (!integer_zerop (TREE_OPERAND (base, 1)))
1181 : {
1182 : /* Subtract MOFF from the base and add it to POFFSET instead.
1183 : Adjust the misalignment to reflect the amount we subtracted. */
1184 1300314 : poly_offset_int moff = mem_ref_offset (base);
1185 1300314 : base_misalignment -= moff.force_shwi ();
1186 1300314 : tree mofft = wide_int_to_tree (sizetype, moff);
1187 1300314 : if (!poffset)
1188 : poffset = mofft;
1189 : else
1190 10310 : poffset = size_binop (PLUS_EXPR, poffset, mofft);
1191 : }
1192 7379016 : base = TREE_OPERAND (base, 0);
1193 : }
1194 : else
1195 : {
1196 9659302 : if (may_be_nonaddressable_p (base))
1197 2102 : return opt_result::failure_at (stmt,
1198 : "failed: base not addressable.\n");
1199 9657200 : base = build_fold_addr_expr (base);
1200 : }
1201 :
1202 17036216 : bool record_nowrap_assumptions_p = (nowrap_bound != nullptr);
1203 :
1204 17036216 : if (in_loop)
1205 : {
1206 3266985 : if (!simple_iv (loop, loop, base, &base_iv, true,
1207 : record_nowrap_assumptions_p))
1208 440679 : return opt_result::failure_at
1209 440679 : (stmt, "failed: evolution of base is not affine.\n");
1210 : }
1211 : else
1212 : {
1213 13769231 : base_iv.base = base;
1214 13769231 : base_iv.step = ssize_int (0);
1215 13769231 : base_iv.no_overflow = true;
1216 13769231 : base_iv.nowrap_bound = NULL_TREE;
1217 : }
1218 :
1219 16595537 : if (!poffset)
1220 : {
1221 13744896 : offset_iv.base = ssize_int (0);
1222 13744896 : offset_iv.step = ssize_int (0);
1223 13744896 : offset_iv.nowrap_bound = NULL_TREE;
1224 : }
1225 : else
1226 : {
1227 2850641 : if (!in_loop)
1228 : {
1229 1522668 : offset_iv.base = poffset;
1230 1522668 : offset_iv.step = ssize_int (0);
1231 1522668 : offset_iv.nowrap_bound = NULL_TREE;
1232 : }
1233 1327973 : else if (!simple_iv (loop, loop, poffset, &offset_iv, true,
1234 : record_nowrap_assumptions_p))
1235 82646 : return opt_result::failure_at
1236 82646 : (stmt, "failed: evolution of offset is not affine.\n");
1237 : }
1238 :
1239 : /* simple_iv might store bounds for BASE_IV and OFFSET_IV. Combine them
1240 : and store the result in NOWRAP_BOUND. */
1241 16512891 : if (nowrap_bound)
1242 : {
1243 11931556 : *nowrap_bound = base_iv.nowrap_bound;
1244 11931556 : scev_combine_nowrap_bounds (nowrap_bound, offset_iv.nowrap_bound);
1245 : }
1246 :
1247 16512891 : init = ssize_int (pbytepos);
1248 :
1249 : /* Subtract any constant component from the base and add it to INIT instead.
1250 : Adjust the misalignment to reflect the amount we subtracted. */
1251 16512891 : split_constant_offset (base_iv.base, &base_iv.base, &dinit);
1252 16512891 : init = size_binop (PLUS_EXPR, init, dinit);
1253 16512891 : base_misalignment -= TREE_INT_CST_LOW (dinit);
1254 :
1255 16512891 : split_constant_offset (offset_iv.base, &offset_iv.base, &dinit);
1256 16512891 : init = size_binop (PLUS_EXPR, init, dinit);
1257 :
1258 16512891 : step = size_binop (PLUS_EXPR,
1259 : fold_convert (ssizetype, base_iv.step),
1260 : fold_convert (ssizetype, offset_iv.step));
1261 :
1262 16512891 : base = canonicalize_base_object_address (base_iv.base);
1263 :
1264 : /* See if get_pointer_alignment can guarantee a higher alignment than
1265 : the one we calculated above. */
1266 16512891 : unsigned int HOST_WIDE_INT alt_misalignment;
1267 16512891 : unsigned int alt_alignment;
1268 16512891 : get_pointer_alignment_1 (base, &alt_alignment, &alt_misalignment);
1269 :
1270 : /* As above, these values must be whole bytes. */
1271 16512891 : gcc_assert (alt_alignment % BITS_PER_UNIT == 0
1272 : && alt_misalignment % BITS_PER_UNIT == 0);
1273 16512891 : alt_alignment /= BITS_PER_UNIT;
1274 16512891 : alt_misalignment /= BITS_PER_UNIT;
1275 :
1276 16512891 : if (base_alignment < alt_alignment)
1277 : {
1278 153326 : base_alignment = alt_alignment;
1279 153326 : base_misalignment = alt_misalignment;
1280 : }
1281 :
1282 16512891 : drb->base_address = base;
1283 16512891 : drb->offset = fold_convert (ssizetype, offset_iv.base);
1284 16512891 : drb->init = init;
1285 16512891 : drb->step = step;
1286 16512891 : if (known_misalignment (base_misalignment, base_alignment,
1287 : &drb->base_misalignment))
1288 16512891 : drb->base_alignment = base_alignment;
1289 : else
1290 : {
1291 : drb->base_alignment = known_alignment (base_misalignment);
1292 : drb->base_misalignment = 0;
1293 : }
1294 16512891 : drb->offset_alignment = highest_pow2_factor (offset_iv.base);
1295 16512891 : drb->step_alignment = highest_pow2_factor (step);
1296 :
1297 16512891 : if (dump_file && (dump_flags & TDF_DETAILS))
1298 66897 : fprintf (dump_file, "success.\n");
1299 :
1300 16512891 : return opt_result::success ();
1301 : }
1302 :
1303 : /* Return true if OP is a valid component reference for a DR access
1304 : function. This accepts a subset of what handled_component_p accepts. */
1305 :
1306 : static bool
1307 5821988 : access_fn_component_p (tree op)
1308 : {
1309 5821988 : switch (TREE_CODE (op))
1310 : {
1311 : case REALPART_EXPR:
1312 : case IMAGPART_EXPR:
1313 : case ARRAY_REF:
1314 : return true;
1315 :
1316 2004160 : case COMPONENT_REF:
1317 2004160 : return (TREE_CODE (TREE_TYPE (TREE_OPERAND (op, 0))) == RECORD_TYPE
1318 2004160 : || (!AGGREGATE_TYPE_P (TREE_TYPE (op))
1319 1545 : && TREE_CODE (TREE_TYPE (op)) != COMPLEX_TYPE));
1320 :
1321 : default:
1322 : return false;
1323 : }
1324 : }
1325 :
1326 : /* Returns whether BASE can have a access_fn_component_p with BASE
1327 : as base. */
1328 :
1329 : static bool
1330 1730775 : base_supports_access_fn_components_p (tree base)
1331 : {
1332 1730775 : switch (TREE_CODE (TREE_TYPE (base)))
1333 : {
1334 : case COMPLEX_TYPE:
1335 : case ARRAY_TYPE:
1336 : case RECORD_TYPE:
1337 : return true;
1338 1723981 : default:
1339 1723981 : return false;
1340 : }
1341 : }
1342 :
1343 : /* Determines the base object and the list of indices of memory reference
1344 : DR, analyzed in LOOP and instantiated before NEST.
1345 :
1346 : If NOWRAP_BOUND is nonzero, pass it to scev which will then store
1347 : the bound under which we do not wrap. */
1348 :
1349 : static void
1350 17180579 : dr_analyze_indices (struct indices *dri, tree ref, edge nest, loop_p loop,
1351 : tree *nowrap_bound = nullptr)
1352 : {
1353 : /* If analyzing a basic-block there are no indices to analyze
1354 : and thus no access functions. */
1355 17180579 : if (!nest)
1356 : {
1357 13810932 : dri->base_object = ref;
1358 13810932 : dri->access_fns.create (0);
1359 13810932 : return;
1360 : }
1361 :
1362 3369647 : vec<tree> access_fns = vNULL;
1363 :
1364 : /* REALPART_EXPR and IMAGPART_EXPR can be handled like accesses
1365 : into a two element array with a constant index. The base is
1366 : then just the immediate underlying object. */
1367 3369647 : if (TREE_CODE (ref) == REALPART_EXPR)
1368 : {
1369 42381 : ref = TREE_OPERAND (ref, 0);
1370 42381 : access_fns.safe_push (integer_zero_node);
1371 : }
1372 3327266 : else if (TREE_CODE (ref) == IMAGPART_EXPR)
1373 : {
1374 40535 : ref = TREE_OPERAND (ref, 0);
1375 40535 : access_fns.safe_push (integer_one_node);
1376 : }
1377 :
1378 : /* Analyze access functions of dimensions we know to be independent.
1379 : The list of component references handled here should be kept in
1380 : sync with access_fn_component_p. */
1381 5969662 : while (handled_component_p (ref))
1382 : {
1383 2744868 : if (TREE_CODE (ref) == ARRAY_REF)
1384 : {
1385 1335409 : tree op = TREE_OPERAND (ref, 1);
1386 1335409 : tree access_fn = analyze_scalar_evolution (loop, op, nowrap_bound);
1387 1335409 : access_fn = instantiate_scev (nest, loop, access_fn);
1388 1335409 : access_fns.safe_push (access_fn);
1389 : }
1390 1409459 : else if (TREE_CODE (ref) == COMPONENT_REF
1391 1409459 : && (TREE_CODE (TREE_TYPE (TREE_OPERAND (ref, 0))) == RECORD_TYPE
1392 99082 : || (!AGGREGATE_TYPE_P (TREE_TYPE (ref))
1393 14638 : && TREE_CODE (TREE_TYPE (ref)) != COMPLEX_TYPE)))
1394 : {
1395 : /* For COMPONENT_REFs of records (but not unions!) use the
1396 : FIELD_DECL offset as constant access function so we can
1397 : disambiguate a[i].f1 and a[i].f2. For unions and accesses
1398 : we do not create further access functions for just use
1399 : zero. */
1400 1264606 : tree off;
1401 1264606 : if (TREE_CODE (TREE_TYPE (TREE_OPERAND (ref, 0))) == RECORD_TYPE)
1402 : {
1403 1249968 : off = component_ref_field_offset (ref);
1404 1249968 : off = size_binop (PLUS_EXPR,
1405 : size_binop (MULT_EXPR,
1406 : fold_convert (bitsizetype, off),
1407 : bitsize_int (BITS_PER_UNIT)),
1408 : DECL_FIELD_BIT_OFFSET (TREE_OPERAND (ref, 1)));
1409 : }
1410 : else
1411 14638 : off = bitsize_zero_node;
1412 1264606 : access_fns.safe_push (off);
1413 : }
1414 : else
1415 : /* If we have an unhandled component we could not translate
1416 : to an access function stop analyzing. We have determined
1417 : our base object in this case. */
1418 : break;
1419 :
1420 2600015 : ref = TREE_OPERAND (ref, 0);
1421 : }
1422 :
1423 : /* If the address operand of a MEM_REF base has an evolution in the
1424 : analyzed nest, add it as an additional independent access-function. */
1425 3369647 : if (TREE_CODE (ref) == MEM_REF)
1426 : {
1427 2363725 : tree op = TREE_OPERAND (ref, 0);
1428 2363725 : tree access_fn = analyze_scalar_evolution (loop, op, nowrap_bound);
1429 2363725 : access_fn = instantiate_scev (nest, loop, access_fn);
1430 2363725 : STRIP_NOPS (access_fn);
1431 2363725 : if (TREE_CODE (access_fn) == POLYNOMIAL_CHREC)
1432 : {
1433 1180276 : tree memoff = TREE_OPERAND (ref, 1);
1434 1180276 : tree base = initial_condition (access_fn);
1435 1180276 : tree orig_type = TREE_TYPE (base);
1436 1180276 : STRIP_USELESS_TYPE_CONVERSION (base);
1437 1180276 : tree off;
1438 1180276 : split_constant_offset (base, &base, &off);
1439 1180276 : STRIP_USELESS_TYPE_CONVERSION (base);
1440 : /* Fold the MEM_REF offset into the evolutions initial
1441 : value to make more bases comparable. */
1442 1180276 : if (!integer_zerop (memoff))
1443 : {
1444 127664 : off = size_binop (PLUS_EXPR, off,
1445 : fold_convert (ssizetype, memoff));
1446 127664 : memoff = build_int_cst (TREE_TYPE (memoff), 0);
1447 : }
1448 : /* Adjust the offset so it is a multiple of the access type
1449 : size and thus we separate bases that can possibly be used
1450 : to produce partial overlaps (which the access_fn machinery
1451 : cannot handle). */
1452 1180276 : wide_int rem;
1453 1180276 : if (TYPE_SIZE_UNIT (TREE_TYPE (ref))
1454 1180140 : && TREE_CODE (TYPE_SIZE_UNIT (TREE_TYPE (ref))) == INTEGER_CST
1455 2360099 : && !integer_zerop (TYPE_SIZE_UNIT (TREE_TYPE (ref))))
1456 1179823 : rem = wi::mod_trunc
1457 1179823 : (wi::to_wide (off),
1458 2359646 : wi::to_wide (TYPE_SIZE_UNIT (TREE_TYPE (ref))),
1459 1179823 : SIGNED);
1460 : else
1461 : /* If we can't compute the remainder simply force the initial
1462 : condition to zero. */
1463 453 : rem = wi::to_wide (off);
1464 1180276 : off = wide_int_to_tree (ssizetype, wi::to_wide (off) - rem);
1465 1180276 : memoff = wide_int_to_tree (TREE_TYPE (memoff), rem);
1466 : /* And finally replace the initial condition. */
1467 2360552 : access_fn = chrec_replace_initial_condition
1468 1180276 : (access_fn, fold_convert (orig_type, off));
1469 : /* ??? This is still not a suitable base object for
1470 : dr_may_alias_p - the base object needs to be an
1471 : access that covers the object as whole. With
1472 : an evolution in the pointer this cannot be
1473 : guaranteed.
1474 : As a band-aid, mark the access so we can special-case
1475 : it in dr_may_alias_p. */
1476 1180276 : tree old = ref;
1477 1180276 : ref = fold_build2_loc (EXPR_LOCATION (ref),
1478 1180276 : MEM_REF, TREE_TYPE (ref),
1479 : base, memoff);
1480 1180276 : MR_DEPENDENCE_CLIQUE (ref) = MR_DEPENDENCE_CLIQUE (old);
1481 1180276 : MR_DEPENDENCE_BASE (ref) = MR_DEPENDENCE_BASE (old);
1482 1180276 : dri->unconstrained_base = true;
1483 1180276 : access_fns.safe_push (access_fn);
1484 1180276 : }
1485 : }
1486 1005922 : else if (DECL_P (ref))
1487 : {
1488 : /* Canonicalize DR_BASE_OBJECT to MEM_REF form. */
1489 861069 : ref = build2 (MEM_REF, TREE_TYPE (ref),
1490 : build_fold_addr_expr (ref),
1491 : build_int_cst (reference_alias_ptr_type (ref), 0));
1492 : }
1493 :
1494 3369647 : dri->base_object = ref;
1495 3369647 : dri->access_fns = access_fns;
1496 : }
1497 :
1498 : /* Extracts the alias analysis information from the memory reference DR. */
1499 :
1500 : static void
1501 17066280 : dr_analyze_alias (struct data_reference *dr)
1502 : {
1503 17066280 : tree ref = DR_REF (dr);
1504 17066280 : tree base = get_base_address (ref), addr;
1505 :
1506 17066280 : if (INDIRECT_REF_P (base)
1507 17066280 : || TREE_CODE (base) == MEM_REF)
1508 : {
1509 7374276 : addr = TREE_OPERAND (base, 0);
1510 7374276 : if (TREE_CODE (addr) == SSA_NAME)
1511 14745806 : DR_PTR_INFO (dr) = ssa_name_pt_solution (addr);
1512 : }
1513 17066280 : }
1514 :
1515 : /* Frees data reference DR. */
1516 :
1517 : void
1518 17559179 : free_data_ref (data_reference_p dr)
1519 : {
1520 17559179 : DR_ACCESS_FNS (dr).release ();
1521 17559179 : if (dr->alt_indices.base_object)
1522 114299 : dr->alt_indices.access_fns.release ();
1523 17559179 : free (dr);
1524 17559179 : }
1525 :
1526 : /* Analyze memory reference MEMREF, which is accessed in STMT.
1527 : The reference is a read if IS_READ is true, otherwise it is a write.
1528 : IS_CONDITIONAL_IN_STMT indicates that the reference is conditional
1529 : within STMT, i.e. that it might not occur even if STMT is executed
1530 : and runs to completion.
1531 :
1532 : Return the data_reference description of MEMREF. NEST is the outermost
1533 : loop in which the reference should be instantiated, LOOP is the loop
1534 : in which the data reference should be analyzed.
1535 :
1536 : If RECORD_NOWRAP_ASSUMPTIONS is true, pass an empty no-wrap bound to
1537 : scev and have it store its assumptions there. If we got a nontrivial
1538 : assumption, save it in DR's DR_NOWRAP_BOUND. */
1539 :
1540 : struct data_reference *
1541 17066280 : create_data_ref (edge nest, loop_p loop, tree memref, gimple *stmt,
1542 : bool is_read, bool is_conditional_in_stmt,
1543 : bool record_nowrap_assumptions)
1544 : {
1545 17066280 : struct data_reference *dr;
1546 :
1547 17066280 : if (dump_file && (dump_flags & TDF_DETAILS))
1548 : {
1549 69102 : fprintf (dump_file, "Creating dr for ");
1550 69102 : print_generic_expr (dump_file, memref, TDF_SLIM);
1551 69102 : fprintf (dump_file, "\n");
1552 : }
1553 :
1554 17066280 : dr = XCNEW (struct data_reference);
1555 17066280 : DR_STMT (dr) = stmt;
1556 17066280 : DR_REF (dr) = memref;
1557 17066280 : DR_IS_READ (dr) = is_read;
1558 17066280 : DR_IS_CONDITIONAL_IN_STMT (dr) = is_conditional_in_stmt;
1559 :
1560 17066280 : tree nowrap_bound = NULL_TREE;
1561 35894654 : dr_analyze_innermost (&DR_INNERMOST (dr), memref,
1562 : nest != NULL ? loop : NULL, stmt,
1563 : record_nowrap_assumptions ? &nowrap_bound : nullptr);
1564 17066280 : dr_analyze_indices (&dr->indices, DR_REF (dr), nest, loop,
1565 : record_nowrap_assumptions ? &nowrap_bound : nullptr);
1566 17066280 : dr_analyze_alias (dr);
1567 :
1568 17066280 : if (nowrap_bound)
1569 : {
1570 : /* If there is a bound but it is zero, the assumption we build with it
1571 : would never hold and we would always fall back to the scalar loop.
1572 : Reset the DR so we can still get gather/scatter. */
1573 6397 : if (integer_zerop (nowrap_bound))
1574 96 : dr_reset (dr);
1575 : else
1576 6301 : DR_NOWRAP_BOUND (dr) = nowrap_bound;
1577 : }
1578 :
1579 17066280 : if (dump_file && (dump_flags & TDF_DETAILS))
1580 : {
1581 69102 : unsigned i;
1582 69102 : fprintf (dump_file, "\tbase_address: ");
1583 69102 : print_generic_expr (dump_file, DR_BASE_ADDRESS (dr), TDF_SLIM);
1584 69102 : fprintf (dump_file, "\n\toffset from base address: ");
1585 69102 : print_generic_expr (dump_file, DR_OFFSET (dr), TDF_SLIM);
1586 69102 : fprintf (dump_file, "\n\tconstant offset from base address: ");
1587 69102 : print_generic_expr (dump_file, DR_INIT (dr), TDF_SLIM);
1588 69102 : fprintf (dump_file, "\n\tstep: ");
1589 69102 : print_generic_expr (dump_file, DR_STEP (dr), TDF_SLIM);
1590 69102 : fprintf (dump_file, "\n\tbase alignment: %d", DR_BASE_ALIGNMENT (dr));
1591 69102 : fprintf (dump_file, "\n\tbase misalignment: %d",
1592 : DR_BASE_MISALIGNMENT (dr));
1593 69102 : fprintf (dump_file, "\n\toffset alignment: %d",
1594 : DR_OFFSET_ALIGNMENT (dr));
1595 69102 : fprintf (dump_file, "\n\tstep alignment: %d", DR_STEP_ALIGNMENT (dr));
1596 69102 : fprintf (dump_file, "\n\tbase_object: ");
1597 69102 : print_generic_expr (dump_file, DR_BASE_OBJECT (dr), TDF_SLIM);
1598 69102 : if (DR_NOWRAP_BOUND (dr))
1599 : {
1600 93 : fprintf (dump_file, "\n\tnowrap bound: ");
1601 93 : print_generic_expr (dump_file, DR_NOWRAP_BOUND (dr), TDF_SLIM);
1602 : }
1603 69102 : fprintf (dump_file, "\n");
1604 198030 : for (i = 0; i < DR_NUM_DIMENSIONS (dr); i++)
1605 : {
1606 59826 : fprintf (dump_file, "\tAccess function %d: ", i);
1607 59826 : print_generic_stmt (dump_file, DR_ACCESS_FN (dr, i), TDF_SLIM);
1608 : }
1609 : }
1610 :
1611 17066280 : return dr;
1612 : }
1613 :
1614 : /* A helper function computes order between two tree expressions T1 and T2.
1615 : This is used in comparator functions sorting objects based on the order
1616 : of tree expressions. The function returns -1, 0, or 1. */
1617 :
1618 : int
1619 446339277 : data_ref_compare_tree (tree t1, tree t2)
1620 : {
1621 446339277 : int i, cmp;
1622 446339277 : enum tree_code code;
1623 446339277 : char tclass;
1624 :
1625 446339277 : if (t1 == t2)
1626 : return 0;
1627 203709302 : if (t1 == NULL)
1628 : return -1;
1629 203549892 : if (t2 == NULL)
1630 : return 1;
1631 :
1632 203439368 : STRIP_USELESS_TYPE_CONVERSION (t1);
1633 203439368 : STRIP_USELESS_TYPE_CONVERSION (t2);
1634 203439368 : if (t1 == t2)
1635 : return 0;
1636 :
1637 202861054 : if (TREE_CODE (t1) != TREE_CODE (t2)
1638 14295442 : && ! (CONVERT_EXPR_P (t1) && CONVERT_EXPR_P (t2)))
1639 14295442 : return TREE_CODE (t1) < TREE_CODE (t2) ? -1 : 1;
1640 :
1641 188565612 : code = TREE_CODE (t1);
1642 188565612 : switch (code)
1643 : {
1644 55112951 : case INTEGER_CST:
1645 55112951 : return tree_int_cst_compare (t1, t2);
1646 :
1647 16 : case STRING_CST:
1648 16 : if (TREE_STRING_LENGTH (t1) != TREE_STRING_LENGTH (t2))
1649 16 : return TREE_STRING_LENGTH (t1) < TREE_STRING_LENGTH (t2) ? -1 : 1;
1650 0 : return memcmp (TREE_STRING_POINTER (t1), TREE_STRING_POINTER (t2),
1651 0 : TREE_STRING_LENGTH (t1));
1652 :
1653 18363396 : case SSA_NAME:
1654 18363396 : if (SSA_NAME_VERSION (t1) != SSA_NAME_VERSION (t2))
1655 18363396 : return SSA_NAME_VERSION (t1) < SSA_NAME_VERSION (t2) ? -1 : 1;
1656 : break;
1657 :
1658 115089249 : default:
1659 115089249 : if (POLY_INT_CST_P (t1))
1660 : return compare_sizes_for_sort (wi::to_poly_widest (t1),
1661 : wi::to_poly_widest (t2));
1662 :
1663 115089249 : tclass = TREE_CODE_CLASS (code);
1664 :
1665 : /* For decls, compare their UIDs. */
1666 115089249 : if (tclass == tcc_declaration)
1667 : {
1668 21655104 : if (DECL_UID (t1) != DECL_UID (t2))
1669 21654617 : return DECL_UID (t1) < DECL_UID (t2) ? -1 : 1;
1670 : break;
1671 : }
1672 : /* For expressions, compare their operands recursively. */
1673 93434145 : else if (IS_EXPR_CODE_CLASS (tclass))
1674 : {
1675 167098262 : for (i = TREE_OPERAND_LENGTH (t1) - 1; i >= 0; --i)
1676 : {
1677 108287380 : cmp = data_ref_compare_tree (TREE_OPERAND (t1, i),
1678 108287380 : TREE_OPERAND (t2, i));
1679 108287380 : if (cmp != 0)
1680 : return cmp;
1681 : }
1682 : }
1683 : else
1684 0 : gcc_unreachable ();
1685 : }
1686 :
1687 : return 0;
1688 : }
1689 :
1690 : /* Return TRUE it's possible to resolve data dependence DDR by runtime alias
1691 : check. */
1692 :
1693 : opt_result
1694 236151 : runtime_alias_check_p (ddr_p ddr, class loop *loop, bool speed_p)
1695 : {
1696 236151 : if (dump_enabled_p ())
1697 7848 : dump_printf (MSG_NOTE,
1698 : "consider run-time aliasing test between %T and %T\n",
1699 7848 : DR_REF (DDR_A (ddr)), DR_REF (DDR_B (ddr)));
1700 :
1701 236151 : if (!speed_p)
1702 0 : return opt_result::failure_at (DR_STMT (DDR_A (ddr)),
1703 : "runtime alias check not supported when"
1704 : " optimizing for size.\n");
1705 :
1706 : /* FORNOW: We don't support versioning with outer-loop in either
1707 : vectorization or loop distribution. */
1708 236151 : if (loop != NULL && loop->inner != NULL)
1709 149 : return opt_result::failure_at (DR_STMT (DDR_A (ddr)),
1710 : "runtime alias check not supported for"
1711 : " outer loop.\n");
1712 :
1713 : /* FORNOW: We don't support handling different address spaces. */
1714 236002 : if (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (DR_BASE_ADDRESS (DDR_A (ddr)))))
1715 236002 : != TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (DR_BASE_ADDRESS (DDR_B (ddr))))))
1716 1 : return opt_result::failure_at (DR_STMT (DDR_A (ddr)),
1717 : "runtime alias check between different "
1718 : "address spaces not supported.\n");
1719 :
1720 236001 : return opt_result::success ();
1721 : }
1722 :
1723 : /* Operator == between two dr_with_seg_len objects.
1724 :
1725 : This equality operator is used to make sure two data refs
1726 : are the same one so that we will consider to combine the
1727 : aliasing checks of those two pairs of data dependent data
1728 : refs. */
1729 :
1730 : static bool
1731 147939 : operator == (const dr_with_seg_len& d1,
1732 : const dr_with_seg_len& d2)
1733 : {
1734 147939 : return (operand_equal_p (DR_BASE_ADDRESS (d1.dr),
1735 147939 : DR_BASE_ADDRESS (d2.dr), 0)
1736 112761 : && data_ref_compare_tree (DR_OFFSET (d1.dr), DR_OFFSET (d2.dr)) == 0
1737 111861 : && data_ref_compare_tree (DR_INIT (d1.dr), DR_INIT (d2.dr)) == 0
1738 102428 : && data_ref_compare_tree (d1.seg_len, d2.seg_len) == 0
1739 101536 : && known_eq (d1.access_size, d2.access_size)
1740 246099 : && d1.align == d2.align);
1741 : }
1742 :
1743 : /* Comparison function for sorting objects of dr_with_seg_len_pair_t
1744 : so that we can combine aliasing checks in one scan. */
1745 :
1746 : static int
1747 1202296 : comp_dr_with_seg_len_pair (const void *pa_, const void *pb_)
1748 : {
1749 1202296 : const dr_with_seg_len_pair_t* pa = (const dr_with_seg_len_pair_t *) pa_;
1750 1202296 : const dr_with_seg_len_pair_t* pb = (const dr_with_seg_len_pair_t *) pb_;
1751 1202296 : const dr_with_seg_len &a1 = pa->first, &a2 = pa->second;
1752 1202296 : const dr_with_seg_len &b1 = pb->first, &b2 = pb->second;
1753 :
1754 : /* For DR pairs (a, b) and (c, d), we only consider to merge the alias checks
1755 : if a and c have the same basic address snd step, and b and d have the same
1756 : address and step. Therefore, if any a&c or b&d don't have the same address
1757 : and step, we don't care the order of those two pairs after sorting. */
1758 1202296 : int comp_res;
1759 :
1760 1202296 : if ((comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (a1.dr),
1761 1202296 : DR_BASE_ADDRESS (b1.dr))) != 0)
1762 : return comp_res;
1763 630081 : if ((comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (a2.dr),
1764 630081 : DR_BASE_ADDRESS (b2.dr))) != 0)
1765 : return comp_res;
1766 426423 : if ((comp_res = data_ref_compare_tree (DR_STEP (a1.dr),
1767 426423 : DR_STEP (b1.dr))) != 0)
1768 : return comp_res;
1769 425803 : if ((comp_res = data_ref_compare_tree (DR_STEP (a2.dr),
1770 425803 : DR_STEP (b2.dr))) != 0)
1771 : return comp_res;
1772 418208 : if ((comp_res = data_ref_compare_tree (DR_OFFSET (a1.dr),
1773 418208 : DR_OFFSET (b1.dr))) != 0)
1774 : return comp_res;
1775 402129 : if ((comp_res = data_ref_compare_tree (DR_INIT (a1.dr),
1776 402129 : DR_INIT (b1.dr))) != 0)
1777 : return comp_res;
1778 301347 : if ((comp_res = data_ref_compare_tree (DR_OFFSET (a2.dr),
1779 301347 : DR_OFFSET (b2.dr))) != 0)
1780 : return comp_res;
1781 286044 : if ((comp_res = data_ref_compare_tree (DR_INIT (a2.dr),
1782 286044 : DR_INIT (b2.dr))) != 0)
1783 : return comp_res;
1784 :
1785 : return 0;
1786 : }
1787 :
1788 : /* Dump information about ALIAS_PAIR, indenting each line by INDENT. */
1789 :
1790 : static void
1791 1155 : dump_alias_pair (dr_with_seg_len_pair_t *alias_pair, const char *indent)
1792 : {
1793 2310 : dump_printf (MSG_NOTE, "%sreference: %T vs. %T\n", indent,
1794 1155 : DR_REF (alias_pair->first.dr),
1795 1155 : DR_REF (alias_pair->second.dr));
1796 :
1797 1155 : dump_printf (MSG_NOTE, "%ssegment length: %T", indent,
1798 : alias_pair->first.seg_len);
1799 1155 : if (!operand_equal_p (alias_pair->first.seg_len,
1800 1155 : alias_pair->second.seg_len, 0))
1801 267 : dump_printf (MSG_NOTE, " vs. %T", alias_pair->second.seg_len);
1802 :
1803 1155 : dump_printf (MSG_NOTE, "\n%saccess size: ", indent);
1804 1155 : dump_dec (MSG_NOTE, alias_pair->first.access_size);
1805 1155 : if (maybe_ne (alias_pair->first.access_size, alias_pair->second.access_size))
1806 : {
1807 250 : dump_printf (MSG_NOTE, " vs. ");
1808 250 : dump_dec (MSG_NOTE, alias_pair->second.access_size);
1809 : }
1810 :
1811 1155 : dump_printf (MSG_NOTE, "\n%salignment: %d", indent,
1812 : alias_pair->first.align);
1813 1155 : if (alias_pair->first.align != alias_pair->second.align)
1814 75 : dump_printf (MSG_NOTE, " vs. %d", alias_pair->second.align);
1815 :
1816 1155 : dump_printf (MSG_NOTE, "\n%sflags: ", indent);
1817 1155 : if (alias_pair->flags & DR_ALIAS_RAW)
1818 171 : dump_printf (MSG_NOTE, " RAW");
1819 1155 : if (alias_pair->flags & DR_ALIAS_WAR)
1820 922 : dump_printf (MSG_NOTE, " WAR");
1821 1155 : if (alias_pair->flags & DR_ALIAS_WAW)
1822 194 : dump_printf (MSG_NOTE, " WAW");
1823 1155 : if (alias_pair->flags & DR_ALIAS_ARBITRARY)
1824 226 : dump_printf (MSG_NOTE, " ARBITRARY");
1825 1155 : if (alias_pair->flags & DR_ALIAS_SWAPPED)
1826 0 : dump_printf (MSG_NOTE, " SWAPPED");
1827 1155 : if (alias_pair->flags & DR_ALIAS_UNSWAPPED)
1828 0 : dump_printf (MSG_NOTE, " UNSWAPPED");
1829 1155 : if (alias_pair->flags & DR_ALIAS_MIXED_STEPS)
1830 0 : dump_printf (MSG_NOTE, " MIXED_STEPS");
1831 1155 : if (alias_pair->flags == 0)
1832 0 : dump_printf (MSG_NOTE, " <none>");
1833 1155 : dump_printf (MSG_NOTE, "\n");
1834 1155 : }
1835 :
1836 : /* Merge alias checks recorded in ALIAS_PAIRS and remove redundant ones.
1837 : FACTOR is number of iterations that each data reference is accessed.
1838 :
1839 : Basically, for each pair of dependent data refs store_ptr_0 & load_ptr_0,
1840 : we create an expression:
1841 :
1842 : ((store_ptr_0 + store_segment_length_0) <= load_ptr_0)
1843 : || (load_ptr_0 + load_segment_length_0) <= store_ptr_0))
1844 :
1845 : for aliasing checks. However, in some cases we can decrease the number
1846 : of checks by combining two checks into one. For example, suppose we have
1847 : another pair of data refs store_ptr_0 & load_ptr_1, and if the following
1848 : condition is satisfied:
1849 :
1850 : load_ptr_0 < load_ptr_1 &&
1851 : load_ptr_1 - load_ptr_0 - load_segment_length_0 < store_segment_length_0
1852 :
1853 : (this condition means, in each iteration of vectorized loop, the accessed
1854 : memory of store_ptr_0 cannot be between the memory of load_ptr_0 and
1855 : load_ptr_1.)
1856 :
1857 : we then can use only the following expression to finish the aliasing checks
1858 : between store_ptr_0 & load_ptr_0 and store_ptr_0 & load_ptr_1:
1859 :
1860 : ((store_ptr_0 + store_segment_length_0) <= load_ptr_0)
1861 : || (load_ptr_1 + load_segment_length_1 <= store_ptr_0))
1862 :
1863 : Note that we only consider that load_ptr_0 and load_ptr_1 have the same
1864 : basic address. */
1865 :
1866 : void
1867 23846 : prune_runtime_alias_test_list (vec<dr_with_seg_len_pair_t> *alias_pairs,
1868 : poly_uint64)
1869 : {
1870 23846 : if (alias_pairs->is_empty ())
1871 23846 : return;
1872 :
1873 : /* Canonicalize each pair so that the base components are ordered wrt
1874 : data_ref_compare_tree. This allows the loop below to merge more
1875 : cases. */
1876 : unsigned int i;
1877 : dr_with_seg_len_pair_t *alias_pair;
1878 95323 : FOR_EACH_VEC_ELT (*alias_pairs, i, alias_pair)
1879 : {
1880 72362 : data_reference_p dr_a = alias_pair->first.dr;
1881 72362 : data_reference_p dr_b = alias_pair->second.dr;
1882 72362 : int comp_res = data_ref_compare_tree (DR_BASE_ADDRESS (dr_a),
1883 : DR_BASE_ADDRESS (dr_b));
1884 72362 : if (comp_res == 0)
1885 1852 : comp_res = data_ref_compare_tree (DR_OFFSET (dr_a), DR_OFFSET (dr_b));
1886 1852 : if (comp_res == 0)
1887 144 : comp_res = data_ref_compare_tree (DR_INIT (dr_a), DR_INIT (dr_b));
1888 72362 : if (comp_res > 0)
1889 : {
1890 25945 : std::swap (alias_pair->first, alias_pair->second);
1891 25945 : alias_pair->flags |= DR_ALIAS_SWAPPED;
1892 : }
1893 : else
1894 46417 : alias_pair->flags |= DR_ALIAS_UNSWAPPED;
1895 : }
1896 :
1897 : /* Sort the collected data ref pairs so that we can scan them once to
1898 : combine all possible aliasing checks. */
1899 22961 : alias_pairs->qsort (comp_dr_with_seg_len_pair);
1900 :
1901 : /* Scan the sorted dr pairs and check if we can combine alias checks
1902 : of two neighboring dr pairs. */
1903 22961 : unsigned int last = 0;
1904 72362 : for (i = 1; i < alias_pairs->length (); ++i)
1905 : {
1906 : /* Deal with two ddrs (dr_a1, dr_b1) and (dr_a2, dr_b2). */
1907 49401 : dr_with_seg_len_pair_t *alias_pair1 = &(*alias_pairs)[last];
1908 49401 : dr_with_seg_len_pair_t *alias_pair2 = &(*alias_pairs)[i];
1909 :
1910 49401 : dr_with_seg_len *dr_a1 = &alias_pair1->first;
1911 49401 : dr_with_seg_len *dr_b1 = &alias_pair1->second;
1912 49401 : dr_with_seg_len *dr_a2 = &alias_pair2->first;
1913 49401 : dr_with_seg_len *dr_b2 = &alias_pair2->second;
1914 :
1915 : /* Remove duplicate data ref pairs. */
1916 49401 : if (*dr_a1 == *dr_a2 && *dr_b1 == *dr_b2)
1917 : {
1918 22752 : if (dump_enabled_p ())
1919 1696 : dump_printf (MSG_NOTE, "found equal ranges %T, %T and %T, %T\n",
1920 1696 : DR_REF (dr_a1->dr), DR_REF (dr_b1->dr),
1921 1696 : DR_REF (dr_a2->dr), DR_REF (dr_b2->dr));
1922 22752 : alias_pair1->flags |= alias_pair2->flags;
1923 22752 : continue;
1924 : }
1925 :
1926 : /* Assume that we won't be able to merge the pairs, then correct
1927 : if we do. */
1928 26649 : last += 1;
1929 26649 : if (last != i)
1930 7308 : (*alias_pairs)[last] = (*alias_pairs)[i];
1931 :
1932 26649 : if (*dr_a1 == *dr_a2 || *dr_b1 == *dr_b2)
1933 : {
1934 : /* We consider the case that DR_B1 and DR_B2 are same memrefs,
1935 : and DR_A1 and DR_A2 are two consecutive memrefs. */
1936 22488 : if (*dr_a1 == *dr_a2)
1937 : {
1938 15084 : std::swap (dr_a1, dr_b1);
1939 15084 : std::swap (dr_a2, dr_b2);
1940 : }
1941 :
1942 22488 : poly_int64 init_a1, init_a2;
1943 : /* Only consider cases in which the distance between the initial
1944 : DR_A1 and the initial DR_A2 is known at compile time. */
1945 40599 : if (!operand_equal_p (DR_BASE_ADDRESS (dr_a1->dr),
1946 22488 : DR_BASE_ADDRESS (dr_a2->dr), 0)
1947 4874 : || !operand_equal_p (DR_OFFSET (dr_a1->dr),
1948 4874 : DR_OFFSET (dr_a2->dr), 0)
1949 4377 : || !poly_int_tree_p (DR_INIT (dr_a1->dr), &init_a1)
1950 26865 : || !poly_int_tree_p (DR_INIT (dr_a2->dr), &init_a2))
1951 18138 : continue;
1952 :
1953 : /* Don't combine if we can't tell which one comes first. */
1954 4377 : if (!ordered_p (init_a1, init_a2))
1955 : continue;
1956 :
1957 : /* Work out what the segment length would be if we did combine
1958 : DR_A1 and DR_A2:
1959 :
1960 : - If DR_A1 and DR_A2 have equal lengths, that length is
1961 : also the combined length.
1962 :
1963 : - If DR_A1 and DR_A2 both have negative "lengths", the combined
1964 : length is the lower bound on those lengths.
1965 :
1966 : - If DR_A1 and DR_A2 both have positive lengths, the combined
1967 : length is the upper bound on those lengths.
1968 :
1969 : Other cases are unlikely to give a useful combination.
1970 :
1971 : The lengths both have sizetype, so the sign is taken from
1972 : the step instead. */
1973 4377 : poly_uint64 new_seg_len = 0;
1974 4377 : bool new_seg_len_p = !operand_equal_p (dr_a1->seg_len,
1975 4377 : dr_a2->seg_len, 0);
1976 4377 : if (new_seg_len_p)
1977 : {
1978 27 : poly_uint64 seg_len_a1, seg_len_a2;
1979 27 : if (!poly_int_tree_p (dr_a1->seg_len, &seg_len_a1)
1980 27 : || !poly_int_tree_p (dr_a2->seg_len, &seg_len_a2))
1981 27 : continue;
1982 :
1983 0 : tree indicator_a = dr_direction_indicator (dr_a1->dr);
1984 0 : if (TREE_CODE (indicator_a) != INTEGER_CST)
1985 0 : continue;
1986 :
1987 0 : tree indicator_b = dr_direction_indicator (dr_a2->dr);
1988 0 : if (TREE_CODE (indicator_b) != INTEGER_CST)
1989 0 : continue;
1990 :
1991 0 : int sign_a = tree_int_cst_sgn (indicator_a);
1992 0 : int sign_b = tree_int_cst_sgn (indicator_b);
1993 :
1994 0 : if (sign_a <= 0 && sign_b <= 0)
1995 0 : new_seg_len = lower_bound (seg_len_a1, seg_len_a2);
1996 0 : else if (sign_a >= 0 && sign_b >= 0)
1997 0 : new_seg_len = upper_bound (seg_len_a1, seg_len_a2);
1998 : else
1999 0 : continue;
2000 : }
2001 : /* At this point we're committed to merging the refs. */
2002 :
2003 : /* Make sure dr_a1 starts left of dr_a2. */
2004 4350 : if (maybe_gt (init_a1, init_a2))
2005 : {
2006 0 : std::swap (*dr_a1, *dr_a2);
2007 0 : std::swap (init_a1, init_a2);
2008 : }
2009 :
2010 : /* The DR_Bs are equal, so only the DR_As can introduce
2011 : mixed steps. */
2012 4350 : if (!operand_equal_p (DR_STEP (dr_a1->dr), DR_STEP (dr_a2->dr), 0))
2013 0 : alias_pair1->flags |= DR_ALIAS_MIXED_STEPS;
2014 :
2015 4350 : if (new_seg_len_p)
2016 : {
2017 0 : dr_a1->seg_len = build_int_cst (TREE_TYPE (dr_a1->seg_len),
2018 0 : new_seg_len);
2019 0 : dr_a1->align = MIN (dr_a1->align, known_alignment (new_seg_len));
2020 : }
2021 :
2022 : /* This is always positive due to the swap above. */
2023 4350 : poly_uint64 diff = init_a2 - init_a1;
2024 :
2025 : /* The new check will start at DR_A1. Make sure that its access
2026 : size encompasses the initial DR_A2. */
2027 4350 : if (maybe_lt (dr_a1->access_size, diff + dr_a2->access_size))
2028 : {
2029 1463 : dr_a1->access_size = upper_bound (dr_a1->access_size,
2030 1463 : diff + dr_a2->access_size);
2031 1463 : unsigned int new_align = known_alignment (dr_a1->access_size);
2032 1463 : dr_a1->align = MIN (dr_a1->align, new_align);
2033 : }
2034 4350 : if (dump_enabled_p ())
2035 1027 : dump_printf (MSG_NOTE, "merging ranges for %T, %T and %T, %T\n",
2036 1027 : DR_REF (dr_a1->dr), DR_REF (dr_b1->dr),
2037 1027 : DR_REF (dr_a2->dr), DR_REF (dr_b2->dr));
2038 4350 : alias_pair1->flags |= alias_pair2->flags;
2039 4350 : last -= 1;
2040 : }
2041 : }
2042 22961 : alias_pairs->truncate (last + 1);
2043 :
2044 : /* Try to restore the original dr_with_seg_len order within each
2045 : dr_with_seg_len_pair_t. If we ended up combining swapped and
2046 : unswapped pairs into the same check, we have to invalidate any
2047 : RAW, WAR and WAW information for it. */
2048 22961 : if (dump_enabled_p ())
2049 845 : dump_printf (MSG_NOTE, "merged alias checks:\n");
2050 68221 : FOR_EACH_VEC_ELT (*alias_pairs, i, alias_pair)
2051 : {
2052 45260 : unsigned int swap_mask = (DR_ALIAS_SWAPPED | DR_ALIAS_UNSWAPPED);
2053 45260 : unsigned int swapped = (alias_pair->flags & swap_mask);
2054 45260 : if (swapped == DR_ALIAS_SWAPPED)
2055 14084 : std::swap (alias_pair->first, alias_pair->second);
2056 31176 : else if (swapped != DR_ALIAS_UNSWAPPED)
2057 3337 : alias_pair->flags |= DR_ALIAS_ARBITRARY;
2058 45260 : alias_pair->flags &= ~swap_mask;
2059 45260 : if (dump_enabled_p ())
2060 1155 : dump_alias_pair (alias_pair, " ");
2061 : }
2062 : }
2063 :
2064 : /* A subroutine of create_intersect_range_checks, with a subset of the
2065 : same arguments. Try to use IFN_CHECK_RAW_PTRS and IFN_CHECK_WAR_PTRS
2066 : to optimize cases in which the references form a simple RAW, WAR or
2067 : WAR dependence. */
2068 :
2069 : static bool
2070 4856 : create_ifn_alias_checks (tree *cond_expr,
2071 : const dr_with_seg_len_pair_t &alias_pair)
2072 : {
2073 4856 : const dr_with_seg_len& dr_a = alias_pair.first;
2074 4856 : const dr_with_seg_len& dr_b = alias_pair.second;
2075 :
2076 : /* Check for cases in which:
2077 :
2078 : (a) we have a known RAW, WAR or WAR dependence
2079 : (b) the accesses are well-ordered in both the original and new code
2080 : (see the comment above the DR_ALIAS_* flags for details); and
2081 : (c) the DR_STEPs describe all access pairs covered by ALIAS_PAIR. */
2082 4856 : if (alias_pair.flags & ~(DR_ALIAS_RAW | DR_ALIAS_WAR | DR_ALIAS_WAW))
2083 : return false;
2084 :
2085 : /* Make sure that both DRs access the same pattern of bytes,
2086 : with a constant length and step. */
2087 3169 : poly_uint64 seg_len;
2088 3169 : if (!operand_equal_p (dr_a.seg_len, dr_b.seg_len, 0)
2089 2757 : || !poly_int_tree_p (dr_a.seg_len, &seg_len)
2090 2750 : || maybe_ne (dr_a.access_size, dr_b.access_size)
2091 2709 : || !operand_equal_p (DR_STEP (dr_a.dr), DR_STEP (dr_b.dr), 0)
2092 5878 : || !tree_fits_uhwi_p (DR_STEP (dr_a.dr)))
2093 : return false;
2094 :
2095 2694 : unsigned HOST_WIDE_INT bytes = tree_to_uhwi (DR_STEP (dr_a.dr));
2096 2694 : tree addr_a = DR_BASE_ADDRESS (dr_a.dr);
2097 2694 : tree addr_b = DR_BASE_ADDRESS (dr_b.dr);
2098 :
2099 : /* See whether the target supports what we want to do. WAW checks are
2100 : equivalent to WAR checks here. */
2101 2654 : internal_fn ifn = (alias_pair.flags & DR_ALIAS_RAW
2102 2694 : ? IFN_CHECK_RAW_PTRS
2103 : : IFN_CHECK_WAR_PTRS);
2104 2694 : unsigned int align = MIN (dr_a.align, dr_b.align);
2105 2694 : poly_uint64 full_length = seg_len + bytes;
2106 2694 : if (!internal_check_ptrs_fn_supported_p (ifn, TREE_TYPE (addr_a),
2107 : full_length, align))
2108 : {
2109 2694 : full_length = seg_len + dr_a.access_size;
2110 2694 : if (!internal_check_ptrs_fn_supported_p (ifn, TREE_TYPE (addr_a),
2111 : full_length, align))
2112 : return false;
2113 : }
2114 :
2115 : /* Commit to using this form of test. */
2116 0 : addr_a = fold_build_pointer_plus (addr_a, DR_OFFSET (dr_a.dr));
2117 0 : addr_a = fold_build_pointer_plus (addr_a, DR_INIT (dr_a.dr));
2118 :
2119 0 : addr_b = fold_build_pointer_plus (addr_b, DR_OFFSET (dr_b.dr));
2120 0 : addr_b = fold_build_pointer_plus (addr_b, DR_INIT (dr_b.dr));
2121 :
2122 0 : *cond_expr = build_call_expr_internal_loc (UNKNOWN_LOCATION,
2123 : ifn, boolean_type_node,
2124 : 4, addr_a, addr_b,
2125 0 : size_int (full_length),
2126 0 : size_int (align));
2127 :
2128 0 : if (dump_enabled_p ())
2129 : {
2130 0 : if (ifn == IFN_CHECK_RAW_PTRS)
2131 0 : dump_printf (MSG_NOTE, "using an IFN_CHECK_RAW_PTRS test\n");
2132 : else
2133 0 : dump_printf (MSG_NOTE, "using an IFN_CHECK_WAR_PTRS test\n");
2134 : }
2135 : return true;
2136 : }
2137 :
2138 : /* Try to generate a runtime condition that is true if ALIAS_PAIR is
2139 : free of aliases, using a condition based on index values instead
2140 : of a condition based on addresses. Return true on success,
2141 : storing the condition in *COND_EXPR.
2142 :
2143 : This can only be done if the two data references in ALIAS_PAIR access
2144 : the same array object and the index is the only difference. For example,
2145 : if the two data references are DR_A and DR_B:
2146 :
2147 : DR_A DR_B
2148 : data-ref arr[i] arr[j]
2149 : base_object arr arr
2150 : index {i_0, +, 1}_loop {j_0, +, 1}_loop
2151 :
2152 : The addresses and their index are like:
2153 :
2154 : |<- ADDR_A ->| |<- ADDR_B ->|
2155 : ------------------------------------------------------->
2156 : | | | | | | | | | |
2157 : ------------------------------------------------------->
2158 : i_0 ... i_0+4 j_0 ... j_0+4
2159 :
2160 : We can create expression based on index rather than address:
2161 :
2162 : (unsigned) (i_0 - j_0 + 3) <= 6
2163 :
2164 : i.e. the indices are less than 4 apart.
2165 :
2166 : Note evolution step of index needs to be considered in comparison. */
2167 :
2168 : static bool
2169 5007 : create_intersect_range_checks_index (class loop *loop, tree *cond_expr,
2170 : const dr_with_seg_len_pair_t &alias_pair)
2171 : {
2172 5007 : const dr_with_seg_len &dr_a = alias_pair.first;
2173 5007 : const dr_with_seg_len &dr_b = alias_pair.second;
2174 5007 : if ((alias_pair.flags & DR_ALIAS_MIXED_STEPS)
2175 5007 : || integer_zerop (DR_STEP (dr_a.dr))
2176 4745 : || integer_zerop (DR_STEP (dr_b.dr))
2177 19083 : || DR_NUM_DIMENSIONS (dr_a.dr) != DR_NUM_DIMENSIONS (dr_b.dr))
2178 : return false;
2179 :
2180 4626 : poly_uint64 seg_len1, seg_len2;
2181 4626 : if (!poly_int_tree_p (dr_a.seg_len, &seg_len1)
2182 4626 : || !poly_int_tree_p (dr_b.seg_len, &seg_len2))
2183 : return false;
2184 :
2185 4349 : if (!tree_fits_shwi_p (DR_STEP (dr_a.dr)))
2186 : return false;
2187 :
2188 4349 : if (!operand_equal_p (DR_BASE_OBJECT (dr_a.dr), DR_BASE_OBJECT (dr_b.dr), 0))
2189 : return false;
2190 :
2191 152 : if (!operand_equal_p (DR_STEP (dr_a.dr), DR_STEP (dr_b.dr), 0))
2192 : return false;
2193 :
2194 152 : gcc_assert (TREE_CODE (DR_STEP (dr_a.dr)) == INTEGER_CST);
2195 :
2196 152 : bool neg_step = tree_int_cst_compare (DR_STEP (dr_a.dr), size_zero_node) < 0;
2197 152 : unsigned HOST_WIDE_INT abs_step = tree_to_shwi (DR_STEP (dr_a.dr));
2198 152 : if (neg_step)
2199 : {
2200 30 : abs_step = -abs_step;
2201 30 : seg_len1 = (-wi::to_poly_wide (dr_a.seg_len)).force_uhwi ();
2202 30 : seg_len2 = (-wi::to_poly_wide (dr_b.seg_len)).force_uhwi ();
2203 : }
2204 :
2205 : /* Infer the number of iterations with which the memory segment is accessed
2206 : by DR. In other words, alias is checked if memory segment accessed by
2207 : DR_A in some iterations intersect with memory segment accessed by DR_B
2208 : in the same amount iterations.
2209 : Note segnment length is a linear function of number of iterations with
2210 : DR_STEP as the coefficient. */
2211 152 : poly_uint64 niter_len1, niter_len2;
2212 152 : if (!can_div_trunc_p (seg_len1 + abs_step - 1, abs_step, &niter_len1)
2213 152 : || !can_div_trunc_p (seg_len2 + abs_step - 1, abs_step, &niter_len2))
2214 : return false;
2215 :
2216 : /* Divide each access size by the byte step, rounding up. */
2217 152 : poly_uint64 niter_access1, niter_access2;
2218 152 : if (!can_div_trunc_p (dr_a.access_size + abs_step - 1,
2219 : abs_step, &niter_access1)
2220 152 : || !can_div_trunc_p (dr_b.access_size + abs_step - 1,
2221 : abs_step, &niter_access2))
2222 : return false;
2223 :
2224 152 : bool waw_or_war_p = (alias_pair.flags & ~(DR_ALIAS_WAR | DR_ALIAS_WAW)) == 0;
2225 :
2226 152 : int found = -1;
2227 311 : for (unsigned int i = 0; i < DR_NUM_DIMENSIONS (dr_a.dr); i++)
2228 : {
2229 160 : tree access1 = DR_ACCESS_FN (dr_a.dr, i);
2230 160 : tree access2 = DR_ACCESS_FN (dr_b.dr, i);
2231 : /* Two indices must be the same if they are not scev, or not scev wrto
2232 : current loop being vecorized. */
2233 160 : if (TREE_CODE (access1) != POLYNOMIAL_CHREC
2234 152 : || TREE_CODE (access2) != POLYNOMIAL_CHREC
2235 152 : || CHREC_VARIABLE (access1) != (unsigned)loop->num
2236 312 : || CHREC_VARIABLE (access2) != (unsigned)loop->num)
2237 : {
2238 8 : if (operand_equal_p (access1, access2, 0))
2239 7 : continue;
2240 :
2241 : return false;
2242 : }
2243 152 : if (found >= 0)
2244 : return false;
2245 152 : found = i;
2246 : }
2247 :
2248 : /* Ought not to happen in practice, since if all accesses are equal then the
2249 : alias should be decidable at compile time. */
2250 151 : if (found < 0)
2251 : return false;
2252 :
2253 : /* The two indices must have the same step. */
2254 151 : tree access1 = DR_ACCESS_FN (dr_a.dr, found);
2255 151 : tree access2 = DR_ACCESS_FN (dr_b.dr, found);
2256 151 : if (!operand_equal_p (CHREC_RIGHT (access1), CHREC_RIGHT (access2), 0))
2257 : return false;
2258 :
2259 151 : tree idx_step = CHREC_RIGHT (access1);
2260 : /* Index must have const step, otherwise DR_STEP won't be constant. */
2261 151 : gcc_assert (TREE_CODE (idx_step) == INTEGER_CST);
2262 : /* Index must evaluate in the same direction as DR. */
2263 151 : gcc_assert (!neg_step || tree_int_cst_sign_bit (idx_step) == 1);
2264 :
2265 151 : tree min1 = CHREC_LEFT (access1);
2266 151 : tree min2 = CHREC_LEFT (access2);
2267 151 : if (!types_compatible_p (TREE_TYPE (min1), TREE_TYPE (min2)))
2268 : return false;
2269 :
2270 : /* Ideally, alias can be checked against loop's control IV, but we
2271 : need to prove linear mapping between control IV and reference
2272 : index. Although that should be true, we check against (array)
2273 : index of data reference. Like segment length, index length is
2274 : linear function of the number of iterations with index_step as
2275 : the coefficient, i.e, niter_len * idx_step. */
2276 151 : offset_int abs_idx_step = offset_int::from (wi::to_wide (idx_step),
2277 : SIGNED);
2278 151 : if (neg_step)
2279 30 : abs_idx_step = -abs_idx_step;
2280 151 : poly_offset_int idx_len1 = abs_idx_step * niter_len1;
2281 151 : poly_offset_int idx_len2 = abs_idx_step * niter_len2;
2282 151 : poly_offset_int idx_access1 = abs_idx_step * niter_access1;
2283 151 : poly_offset_int idx_access2 = abs_idx_step * niter_access2;
2284 :
2285 151 : gcc_assert (known_ge (idx_len1, 0)
2286 : && known_ge (idx_len2, 0)
2287 : && known_ge (idx_access1, 0)
2288 : && known_ge (idx_access2, 0));
2289 :
2290 : /* Each access has the following pattern, with lengths measured
2291 : in units of INDEX:
2292 :
2293 : <-- idx_len -->
2294 : <--- A: -ve step --->
2295 : +-----+-------+-----+-------+-----+
2296 : | n-1 | ..... | 0 | ..... | n-1 |
2297 : +-----+-------+-----+-------+-----+
2298 : <--- B: +ve step --->
2299 : <-- idx_len -->
2300 : |
2301 : min
2302 :
2303 : where "n" is the number of scalar iterations covered by the segment
2304 : and where each access spans idx_access units.
2305 :
2306 : A is the range of bytes accessed when the step is negative,
2307 : B is the range when the step is positive.
2308 :
2309 : When checking for general overlap, we need to test whether
2310 : the range:
2311 :
2312 : [min1 + low_offset1, min1 + high_offset1 + idx_access1 - 1]
2313 :
2314 : overlaps:
2315 :
2316 : [min2 + low_offset2, min2 + high_offset2 + idx_access2 - 1]
2317 :
2318 : where:
2319 :
2320 : low_offsetN = +ve step ? 0 : -idx_lenN;
2321 : high_offsetN = +ve step ? idx_lenN : 0;
2322 :
2323 : This is equivalent to testing whether:
2324 :
2325 : min1 + low_offset1 <= min2 + high_offset2 + idx_access2 - 1
2326 : && min2 + low_offset2 <= min1 + high_offset1 + idx_access1 - 1
2327 :
2328 : Converting this into a single test, there is an overlap if:
2329 :
2330 : 0 <= min2 - min1 + bias <= limit
2331 :
2332 : where bias = high_offset2 + idx_access2 - 1 - low_offset1
2333 : limit = (high_offset1 - low_offset1 + idx_access1 - 1)
2334 : + (high_offset2 - low_offset2 + idx_access2 - 1)
2335 : i.e. limit = idx_len1 + idx_access1 - 1 + idx_len2 + idx_access2 - 1
2336 :
2337 : Combining the tests requires limit to be computable in an unsigned
2338 : form of the index type; if it isn't, we fall back to the usual
2339 : pointer-based checks.
2340 :
2341 : We can do better if DR_B is a write and if DR_A and DR_B are
2342 : well-ordered in both the original and the new code (see the
2343 : comment above the DR_ALIAS_* flags for details). In this case
2344 : we know that for each i in [0, n-1], the write performed by
2345 : access i of DR_B occurs after access numbers j<=i of DR_A in
2346 : both the original and the new code. Any write or anti
2347 : dependencies wrt those DR_A accesses are therefore maintained.
2348 :
2349 : We just need to make sure that each individual write in DR_B does not
2350 : overlap any higher-indexed access in DR_A; such DR_A accesses happen
2351 : after the DR_B access in the original code but happen before it in
2352 : the new code.
2353 :
2354 : We know the steps for both accesses are equal, so by induction, we
2355 : just need to test whether the first write of DR_B overlaps a later
2356 : access of DR_A. In other words, we need to move min1 along by
2357 : one iteration:
2358 :
2359 : min1' = min1 + idx_step
2360 :
2361 : and use the ranges:
2362 :
2363 : [min1' + low_offset1', min1' + high_offset1' + idx_access1 - 1]
2364 :
2365 : and:
2366 :
2367 : [min2, min2 + idx_access2 - 1]
2368 :
2369 : where:
2370 :
2371 : low_offset1' = +ve step ? 0 : -(idx_len1 - |idx_step|)
2372 : high_offset1' = +ve_step ? idx_len1 - |idx_step| : 0. */
2373 151 : if (waw_or_war_p)
2374 120 : idx_len1 -= abs_idx_step;
2375 :
2376 151 : poly_offset_int limit = idx_len1 + idx_access1 - 1 + idx_access2 - 1;
2377 151 : if (!waw_or_war_p)
2378 151 : limit += idx_len2;
2379 :
2380 151 : tree utype = unsigned_type_for (TREE_TYPE (min1));
2381 151 : if (!wi::fits_to_tree_p (limit, utype))
2382 : return false;
2383 :
2384 151 : poly_offset_int low_offset1 = neg_step ? -idx_len1 : 0;
2385 151 : poly_offset_int high_offset2 = neg_step || waw_or_war_p ? 0 : idx_len2;
2386 151 : poly_offset_int bias = high_offset2 + idx_access2 - 1 - low_offset1;
2387 : /* Equivalent to adding IDX_STEP to MIN1. */
2388 151 : if (waw_or_war_p)
2389 120 : bias -= wi::to_offset (idx_step);
2390 :
2391 151 : tree subject = fold_build2 (MINUS_EXPR, utype,
2392 : fold_convert (utype, min2),
2393 : fold_convert (utype, min1));
2394 151 : subject = fold_build2 (PLUS_EXPR, utype, subject,
2395 : wide_int_to_tree (utype, bias));
2396 151 : tree part_cond_expr = fold_build2 (GT_EXPR, boolean_type_node, subject,
2397 : wide_int_to_tree (utype, limit));
2398 151 : if (*cond_expr)
2399 0 : *cond_expr = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
2400 : *cond_expr, part_cond_expr);
2401 : else
2402 : *cond_expr = part_cond_expr;
2403 151 : if (dump_enabled_p ())
2404 : {
2405 133 : if (waw_or_war_p)
2406 103 : dump_printf (MSG_NOTE, "using an index-based WAR/WAW test\n");
2407 : else
2408 30 : dump_printf (MSG_NOTE, "using an index-based overlap test\n");
2409 : }
2410 : return true;
2411 : }
2412 :
2413 : /* A subroutine of create_intersect_range_checks, with a subset of the
2414 : same arguments. Try to optimize cases in which the second access
2415 : is a write and in which some overlap is valid. */
2416 :
2417 : static bool
2418 4856 : create_waw_or_war_checks (tree *cond_expr,
2419 : const dr_with_seg_len_pair_t &alias_pair)
2420 : {
2421 4856 : const dr_with_seg_len& dr_a = alias_pair.first;
2422 4856 : const dr_with_seg_len& dr_b = alias_pair.second;
2423 :
2424 : /* Check for cases in which:
2425 :
2426 : (a) DR_B is always a write;
2427 : (b) the accesses are well-ordered in both the original and new code
2428 : (see the comment above the DR_ALIAS_* flags for details); and
2429 : (c) the DR_STEPs describe all access pairs covered by ALIAS_PAIR. */
2430 4856 : if (alias_pair.flags & ~(DR_ALIAS_WAR | DR_ALIAS_WAW))
2431 : return false;
2432 :
2433 : /* Check for equal (but possibly variable) steps. */
2434 3122 : tree step = DR_STEP (dr_a.dr);
2435 3122 : if (!operand_equal_p (step, DR_STEP (dr_b.dr)))
2436 : return false;
2437 :
2438 : /* Make sure that we can operate on sizetype without loss of precision. */
2439 2717 : tree addr_type = TREE_TYPE (DR_BASE_ADDRESS (dr_a.dr));
2440 2717 : if (TYPE_PRECISION (addr_type) != TYPE_PRECISION (sizetype))
2441 : return false;
2442 :
2443 : /* All addresses involved are known to have a common alignment ALIGN.
2444 : We can therefore subtract ALIGN from an exclusive endpoint to get
2445 : an inclusive endpoint. In the best (and common) case, ALIGN is the
2446 : same as the access sizes of both DRs, and so subtracting ALIGN
2447 : cancels out the addition of an access size. */
2448 2717 : unsigned int align = MIN (dr_a.align, dr_b.align);
2449 2717 : poly_uint64 last_chunk_a = dr_a.access_size - align;
2450 2717 : poly_uint64 last_chunk_b = dr_b.access_size - align;
2451 :
2452 : /* Get a boolean expression that is true when the step is negative. */
2453 2717 : tree indicator = dr_direction_indicator (dr_a.dr);
2454 2717 : tree neg_step = fold_build2 (LT_EXPR, boolean_type_node,
2455 : fold_convert (ssizetype, indicator),
2456 : ssize_int (0));
2457 :
2458 : /* Get lengths in sizetype. */
2459 2717 : tree seg_len_a
2460 2717 : = fold_convert (sizetype, rewrite_to_non_trapping_overflow (dr_a.seg_len));
2461 2717 : step = fold_convert (sizetype, rewrite_to_non_trapping_overflow (step));
2462 :
2463 : /* Each access has the following pattern:
2464 :
2465 : <- |seg_len| ->
2466 : <--- A: -ve step --->
2467 : +-----+-------+-----+-------+-----+
2468 : | n-1 | ..... | 0 | ..... | n-1 |
2469 : +-----+-------+-----+-------+-----+
2470 : <--- B: +ve step --->
2471 : <- |seg_len| ->
2472 : |
2473 : base address
2474 :
2475 : where "n" is the number of scalar iterations covered by the segment.
2476 :
2477 : A is the range of bytes accessed when the step is negative,
2478 : B is the range when the step is positive.
2479 :
2480 : We know that DR_B is a write. We also know (from checking that
2481 : DR_A and DR_B are well-ordered) that for each i in [0, n-1],
2482 : the write performed by access i of DR_B occurs after access numbers
2483 : j<=i of DR_A in both the original and the new code. Any write or
2484 : anti dependencies wrt those DR_A accesses are therefore maintained.
2485 :
2486 : We just need to make sure that each individual write in DR_B does not
2487 : overlap any higher-indexed access in DR_A; such DR_A accesses happen
2488 : after the DR_B access in the original code but happen before it in
2489 : the new code.
2490 :
2491 : We know the steps for both accesses are equal, so by induction, we
2492 : just need to test whether the first write of DR_B overlaps a later
2493 : access of DR_A. In other words, we need to move addr_a along by
2494 : one iteration:
2495 :
2496 : addr_a' = addr_a + step
2497 :
2498 : and check whether:
2499 :
2500 : [addr_b, addr_b + last_chunk_b]
2501 :
2502 : overlaps:
2503 :
2504 : [addr_a' + low_offset_a, addr_a' + high_offset_a + last_chunk_a]
2505 :
2506 : where [low_offset_a, high_offset_a] spans accesses [1, n-1]. I.e.:
2507 :
2508 : low_offset_a = +ve step ? 0 : seg_len_a - step
2509 : high_offset_a = +ve step ? seg_len_a - step : 0
2510 :
2511 : This is equivalent to testing whether:
2512 :
2513 : addr_a' + low_offset_a <= addr_b + last_chunk_b
2514 : && addr_b <= addr_a' + high_offset_a + last_chunk_a
2515 :
2516 : Converting this into a single test, there is an overlap if:
2517 :
2518 : 0 <= addr_b + last_chunk_b - addr_a' - low_offset_a <= limit
2519 :
2520 : where limit = high_offset_a - low_offset_a + last_chunk_a + last_chunk_b
2521 :
2522 : If DR_A is performed, limit + |step| - last_chunk_b is known to be
2523 : less than the size of the object underlying DR_A. We also know
2524 : that last_chunk_b <= |step|; this is checked elsewhere if it isn't
2525 : guaranteed at compile time. There can therefore be no overflow if
2526 : "limit" is calculated in an unsigned type with pointer precision. */
2527 2717 : tree addr_a = fold_build_pointer_plus (DR_BASE_ADDRESS (dr_a.dr),
2528 : DR_OFFSET (dr_a.dr));
2529 2717 : addr_a = fold_build_pointer_plus (addr_a, DR_INIT (dr_a.dr));
2530 :
2531 2717 : tree addr_b = fold_build_pointer_plus (DR_BASE_ADDRESS (dr_b.dr),
2532 : DR_OFFSET (dr_b.dr));
2533 2717 : addr_b = fold_build_pointer_plus (addr_b, DR_INIT (dr_b.dr));
2534 :
2535 : /* Advance ADDR_A by one iteration and adjust the length to compensate. */
2536 2717 : addr_a = fold_build_pointer_plus (addr_a, step);
2537 2717 : tree seg_len_a_minus_step = fold_build2 (MINUS_EXPR, sizetype,
2538 : seg_len_a, step);
2539 2717 : if (!CONSTANT_CLASS_P (seg_len_a_minus_step))
2540 3 : seg_len_a_minus_step = build1 (SAVE_EXPR, sizetype, seg_len_a_minus_step);
2541 :
2542 2717 : tree low_offset_a = fold_build3 (COND_EXPR, sizetype, neg_step,
2543 : seg_len_a_minus_step, size_zero_node);
2544 2717 : if (!CONSTANT_CLASS_P (low_offset_a))
2545 3 : low_offset_a = build1 (SAVE_EXPR, sizetype, low_offset_a);
2546 :
2547 : /* We could use COND_EXPR <neg_step, size_zero_node, seg_len_a_minus_step>,
2548 : but it's usually more efficient to reuse the LOW_OFFSET_A result. */
2549 2717 : tree high_offset_a = fold_build2 (MINUS_EXPR, sizetype, seg_len_a_minus_step,
2550 : low_offset_a);
2551 :
2552 : /* The amount added to addr_b - addr_a'. */
2553 2717 : tree bias = fold_build2 (MINUS_EXPR, sizetype,
2554 : size_int (last_chunk_b), low_offset_a);
2555 :
2556 2717 : tree limit = fold_build2 (MINUS_EXPR, sizetype, high_offset_a, low_offset_a);
2557 2717 : limit = fold_build2 (PLUS_EXPR, sizetype, limit,
2558 : size_int (last_chunk_a + last_chunk_b));
2559 :
2560 2717 : tree subject = fold_build2 (MINUS_EXPR, sizetype,
2561 : fold_convert (sizetype, addr_b),
2562 : fold_convert (sizetype, addr_a));
2563 2717 : subject = fold_build2 (PLUS_EXPR, sizetype, subject, bias);
2564 :
2565 2717 : *cond_expr = fold_build2 (GT_EXPR, boolean_type_node, subject, limit);
2566 2717 : if (dump_enabled_p ())
2567 368 : dump_printf (MSG_NOTE, "using an address-based WAR/WAW test\n");
2568 : return true;
2569 : }
2570 :
2571 : /* If ALIGN is nonzero, set up *SEQ_MIN_OUT and *SEQ_MAX_OUT so that for
2572 : every address ADDR accessed by D:
2573 :
2574 : *SEQ_MIN_OUT <= ADDR (== ADDR & -ALIGN) <= *SEQ_MAX_OUT
2575 :
2576 : In this case, every element accessed by D is aligned to at least
2577 : ALIGN bytes.
2578 :
2579 : If ALIGN is zero then instead set *SEG_MAX_OUT so that:
2580 :
2581 : *SEQ_MIN_OUT <= ADDR < *SEQ_MAX_OUT. */
2582 :
2583 : static void
2584 4278 : get_segment_min_max (const dr_with_seg_len &d, tree *seg_min_out,
2585 : tree *seg_max_out, HOST_WIDE_INT align)
2586 : {
2587 : /* Each access has the following pattern:
2588 :
2589 : <- |seg_len| ->
2590 : <--- A: -ve step --->
2591 : +-----+-------+-----+-------+-----+
2592 : | n-1 | ,.... | 0 | ..... | n-1 |
2593 : +-----+-------+-----+-------+-----+
2594 : <--- B: +ve step --->
2595 : <- |seg_len| ->
2596 : |
2597 : base address
2598 :
2599 : where "n" is the number of scalar iterations covered by the segment.
2600 : (This should be VF for a particular pair if we know that both steps
2601 : are the same, otherwise it will be the full number of scalar loop
2602 : iterations.)
2603 :
2604 : A is the range of bytes accessed when the step is negative,
2605 : B is the range when the step is positive.
2606 :
2607 : If the access size is "access_size" bytes, the lowest addressed byte is:
2608 :
2609 : base + (step < 0 ? seg_len : 0) [LB]
2610 :
2611 : and the highest addressed byte is always below:
2612 :
2613 : base + (step < 0 ? 0 : seg_len) + access_size [UB]
2614 :
2615 : Thus:
2616 :
2617 : LB <= ADDR < UB
2618 :
2619 : If ALIGN is nonzero, all three values are aligned to at least ALIGN
2620 : bytes, so:
2621 :
2622 : LB <= ADDR <= UB - ALIGN
2623 :
2624 : where "- ALIGN" folds naturally with the "+ access_size" and often
2625 : cancels it out.
2626 :
2627 : We don't try to simplify LB and UB beyond this (e.g. by using
2628 : MIN and MAX based on whether seg_len rather than the stride is
2629 : negative) because it is possible for the absolute size of the
2630 : segment to overflow the range of a ssize_t.
2631 :
2632 : Keeping the pointer_plus outside of the cond_expr should allow
2633 : the cond_exprs to be shared with other alias checks. */
2634 4278 : tree indicator = dr_direction_indicator (d.dr);
2635 4278 : tree neg_step = fold_build2 (LT_EXPR, boolean_type_node,
2636 : fold_convert (ssizetype, indicator),
2637 : ssize_int (0));
2638 4278 : tree addr_base = fold_build_pointer_plus (DR_BASE_ADDRESS (d.dr),
2639 : DR_OFFSET (d.dr));
2640 4278 : addr_base = fold_build_pointer_plus (addr_base, DR_INIT (d.dr));
2641 4278 : tree seg_len
2642 4278 : = fold_convert (sizetype, rewrite_to_non_trapping_overflow (d.seg_len));
2643 :
2644 4278 : tree min_reach = fold_build3 (COND_EXPR, sizetype, neg_step,
2645 : seg_len, size_zero_node);
2646 4278 : tree max_reach = fold_build3 (COND_EXPR, sizetype, neg_step,
2647 : size_zero_node, seg_len);
2648 4278 : max_reach = fold_build2 (PLUS_EXPR, sizetype, max_reach,
2649 : size_int (d.access_size - align));
2650 :
2651 4278 : *seg_min_out = fold_build_pointer_plus (addr_base, min_reach);
2652 4278 : *seg_max_out = fold_build_pointer_plus (addr_base, max_reach);
2653 4278 : }
2654 :
2655 : /* Generate a runtime condition that is true if ALIAS_PAIR is free of aliases,
2656 : storing the condition in *COND_EXPR. The fallback is to generate a
2657 : a test that the two accesses do not overlap:
2658 :
2659 : end_a <= start_b || end_b <= start_a. */
2660 :
2661 : static void
2662 5007 : create_intersect_range_checks (class loop *loop, tree *cond_expr,
2663 : const dr_with_seg_len_pair_t &alias_pair)
2664 : {
2665 5007 : const dr_with_seg_len& dr_a = alias_pair.first;
2666 5007 : const dr_with_seg_len& dr_b = alias_pair.second;
2667 5007 : *cond_expr = NULL_TREE;
2668 5007 : if (create_intersect_range_checks_index (loop, cond_expr, alias_pair))
2669 2868 : return;
2670 :
2671 4856 : if (create_ifn_alias_checks (cond_expr, alias_pair))
2672 : return;
2673 :
2674 4856 : if (create_waw_or_war_checks (cond_expr, alias_pair))
2675 : return;
2676 :
2677 2139 : unsigned HOST_WIDE_INT min_align;
2678 2139 : tree_code cmp_code;
2679 : /* We don't have to check DR_ALIAS_MIXED_STEPS here, since both versions
2680 : are equivalent. This is just an optimization heuristic. */
2681 2139 : if (TREE_CODE (DR_STEP (dr_a.dr)) == INTEGER_CST
2682 2047 : && TREE_CODE (DR_STEP (dr_b.dr)) == INTEGER_CST)
2683 : {
2684 : /* In this case adding access_size to seg_len is likely to give
2685 : a simple X * step, where X is either the number of scalar
2686 : iterations or the vectorization factor. We're better off
2687 : keeping that, rather than subtracting an alignment from it.
2688 :
2689 : In this case the maximum values are exclusive and so there is
2690 : no alias if the maximum of one segment equals the minimum
2691 : of another. */
2692 : min_align = 0;
2693 : cmp_code = LE_EXPR;
2694 : }
2695 : else
2696 : {
2697 : /* Calculate the minimum alignment shared by all four pointers,
2698 : then arrange for this alignment to be subtracted from the
2699 : exclusive maximum values to get inclusive maximum values.
2700 : This "- min_align" is cumulative with a "+ access_size"
2701 : in the calculation of the maximum values. In the best
2702 : (and common) case, the two cancel each other out, leaving
2703 : us with an inclusive bound based only on seg_len. In the
2704 : worst case we're simply adding a smaller number than before.
2705 :
2706 : Because the maximum values are inclusive, there is an alias
2707 : if the maximum value of one segment is equal to the minimum
2708 : value of the other. */
2709 200 : min_align = std::min (dr_a.align, dr_b.align);
2710 200 : cmp_code = LT_EXPR;
2711 : }
2712 :
2713 2139 : tree seg_a_min, seg_a_max, seg_b_min, seg_b_max;
2714 2139 : get_segment_min_max (dr_a, &seg_a_min, &seg_a_max, min_align);
2715 2139 : get_segment_min_max (dr_b, &seg_b_min, &seg_b_max, min_align);
2716 :
2717 2139 : *cond_expr
2718 2139 : = fold_build2 (TRUTH_OR_EXPR, boolean_type_node,
2719 : fold_build2 (cmp_code, boolean_type_node, seg_a_max, seg_b_min),
2720 : fold_build2 (cmp_code, boolean_type_node, seg_b_max, seg_a_min));
2721 2139 : if (dump_enabled_p ())
2722 307 : dump_printf (MSG_NOTE, "using an address-based overlap test\n");
2723 : }
2724 :
2725 : /* Create a conditional expression that represents the run-time checks for
2726 : overlapping of address ranges represented by a list of data references
2727 : pairs passed in ALIAS_PAIRS. Data references are in LOOP. The returned
2728 : COND_EXPR is the conditional expression to be used in the if statement
2729 : that controls which version of the loop gets executed at runtime. */
2730 :
2731 : void
2732 3338 : create_runtime_alias_checks (class loop *loop,
2733 : const vec<dr_with_seg_len_pair_t> *alias_pairs,
2734 : tree * cond_expr)
2735 : {
2736 3338 : tree part_cond_expr;
2737 :
2738 15021 : for (const dr_with_seg_len_pair_t &alias_pair : alias_pairs)
2739 : {
2740 5007 : gcc_assert (alias_pair.flags);
2741 5007 : if (dump_enabled_p ())
2742 808 : dump_printf (MSG_NOTE,
2743 : "create runtime check for data references %T and %T\n",
2744 808 : DR_REF (alias_pair.first.dr),
2745 808 : DR_REF (alias_pair.second.dr));
2746 :
2747 : /* Create condition expression for each pair data references. */
2748 5007 : create_intersect_range_checks (loop, &part_cond_expr, alias_pair);
2749 5007 : if (*cond_expr)
2750 4922 : *cond_expr = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
2751 : *cond_expr, part_cond_expr);
2752 : else
2753 85 : *cond_expr = part_cond_expr;
2754 : }
2755 3338 : }
2756 :
2757 : /* Check if OFFSET1 and OFFSET2 (DR_OFFSETs of some data-refs) are identical
2758 : expressions. */
2759 : static bool
2760 0 : dr_equal_offsets_p1 (tree offset1, tree offset2)
2761 : {
2762 0 : bool res;
2763 :
2764 0 : STRIP_NOPS (offset1);
2765 0 : STRIP_NOPS (offset2);
2766 :
2767 0 : if (offset1 == offset2)
2768 : return true;
2769 :
2770 0 : if (TREE_CODE (offset1) != TREE_CODE (offset2)
2771 0 : || (!BINARY_CLASS_P (offset1) && !UNARY_CLASS_P (offset1)))
2772 : return false;
2773 :
2774 0 : res = dr_equal_offsets_p1 (TREE_OPERAND (offset1, 0),
2775 0 : TREE_OPERAND (offset2, 0));
2776 :
2777 0 : if (!res || !BINARY_CLASS_P (offset1))
2778 : return res;
2779 :
2780 0 : res = dr_equal_offsets_p1 (TREE_OPERAND (offset1, 1),
2781 0 : TREE_OPERAND (offset2, 1));
2782 :
2783 0 : return res;
2784 : }
2785 :
2786 : /* Check if DRA and DRB have equal offsets. */
2787 : bool
2788 0 : dr_equal_offsets_p (struct data_reference *dra,
2789 : struct data_reference *drb)
2790 : {
2791 0 : tree offset1, offset2;
2792 :
2793 0 : offset1 = DR_OFFSET (dra);
2794 0 : offset2 = DR_OFFSET (drb);
2795 :
2796 0 : return dr_equal_offsets_p1 (offset1, offset2);
2797 : }
2798 :
2799 : /* Returns true if FNA == FNB. */
2800 :
2801 : static bool
2802 0 : affine_function_equal_p (affine_fn fna, affine_fn fnb)
2803 : {
2804 0 : unsigned i, n = fna.length ();
2805 :
2806 0 : if (n != fnb.length ())
2807 : return false;
2808 :
2809 0 : for (i = 0; i < n; i++)
2810 0 : if (!operand_equal_p (fna[i], fnb[i], 0))
2811 : return false;
2812 :
2813 : return true;
2814 : }
2815 :
2816 : /* If all the functions in CF are the same, returns one of them,
2817 : otherwise returns NULL. */
2818 :
2819 : static affine_fn
2820 2321700 : common_affine_function (conflict_function *cf)
2821 : {
2822 2321700 : unsigned i;
2823 2321700 : affine_fn comm;
2824 :
2825 2321700 : if (!CF_NONTRIVIAL_P (cf))
2826 0 : return affine_fn ();
2827 :
2828 2321700 : comm = cf->fns[0];
2829 :
2830 2321700 : for (i = 1; i < cf->n; i++)
2831 0 : if (!affine_function_equal_p (comm, cf->fns[i]))
2832 0 : return affine_fn ();
2833 :
2834 2321700 : return comm;
2835 : }
2836 :
2837 : /* Returns the base of the affine function FN. */
2838 :
2839 : static tree
2840 1338076 : affine_function_base (affine_fn fn)
2841 : {
2842 0 : return fn[0];
2843 : }
2844 :
2845 : /* Returns true if FN is a constant. */
2846 :
2847 : static bool
2848 1338385 : affine_function_constant_p (affine_fn fn)
2849 : {
2850 1338385 : unsigned i;
2851 1338385 : tree coef;
2852 :
2853 1396409 : for (i = 1; fn.iterate (i, &coef); i++)
2854 58333 : if (!integer_zerop (coef))
2855 : return false;
2856 :
2857 : return true;
2858 : }
2859 :
2860 : /* Returns true if FN is the zero constant function. */
2861 :
2862 : static bool
2863 177535 : affine_function_zero_p (affine_fn fn)
2864 : {
2865 177535 : return (integer_zerop (affine_function_base (fn))
2866 177535 : && affine_function_constant_p (fn));
2867 : }
2868 :
2869 : /* Returns a signed integer type with the largest precision from TA
2870 : and TB. */
2871 :
2872 : static tree
2873 1757126 : signed_type_for_types (tree ta, tree tb)
2874 : {
2875 1757126 : if (TYPE_PRECISION (ta) > TYPE_PRECISION (tb))
2876 565 : return signed_type_for (ta);
2877 : else
2878 1756561 : return signed_type_for (tb);
2879 : }
2880 :
2881 : /* Applies operation OP on affine functions FNA and FNB, and returns the
2882 : result. */
2883 :
2884 : static affine_fn
2885 1160850 : affine_fn_op (enum tree_code op, affine_fn fna, affine_fn fnb)
2886 : {
2887 1160850 : unsigned i, n, m;
2888 1160850 : affine_fn ret;
2889 1160850 : tree coef;
2890 :
2891 3482550 : if (fnb.length () > fna.length ())
2892 : {
2893 0 : n = fna.length ();
2894 0 : m = fnb.length ();
2895 : }
2896 : else
2897 : {
2898 1160850 : n = fnb.length ();
2899 1160850 : m = fna.length ();
2900 : }
2901 :
2902 1160850 : ret.create (m);
2903 3540883 : for (i = 0; i < n; i++)
2904 : {
2905 2438366 : tree type = signed_type_for_types (TREE_TYPE (fna[i]),
2906 1219183 : TREE_TYPE (fnb[i]));
2907 1219183 : ret.quick_push (fold_build2 (op, type, fna[i], fnb[i]));
2908 : }
2909 :
2910 1160850 : for (; fna.iterate (i, &coef); i++)
2911 0 : ret.quick_push (fold_build2 (op, signed_type_for (TREE_TYPE (coef)),
2912 : coef, integer_zero_node));
2913 1160850 : for (; fnb.iterate (i, &coef); i++)
2914 0 : ret.quick_push (fold_build2 (op, signed_type_for (TREE_TYPE (coef)),
2915 : integer_zero_node, coef));
2916 :
2917 1160850 : return ret;
2918 : }
2919 :
2920 : /* Returns the sum of affine functions FNA and FNB. */
2921 :
2922 : static affine_fn
2923 0 : affine_fn_plus (affine_fn fna, affine_fn fnb)
2924 : {
2925 0 : return affine_fn_op (PLUS_EXPR, fna, fnb);
2926 : }
2927 :
2928 : /* Returns the difference of affine functions FNA and FNB. */
2929 :
2930 : static affine_fn
2931 1160850 : affine_fn_minus (affine_fn fna, affine_fn fnb)
2932 : {
2933 0 : return affine_fn_op (MINUS_EXPR, fna, fnb);
2934 : }
2935 :
2936 : /* Frees affine function FN. */
2937 :
2938 : static void
2939 3684456 : affine_fn_free (affine_fn fn)
2940 : {
2941 0 : fn.release ();
2942 0 : }
2943 :
2944 : /* Determine for each subscript in the data dependence relation DDR
2945 : the distance. */
2946 :
2947 : static void
2948 3115003 : compute_subscript_distance (struct data_dependence_relation *ddr)
2949 : {
2950 3115003 : conflict_function *cf_a, *cf_b;
2951 3115003 : affine_fn fn_a, fn_b, diff;
2952 :
2953 3115003 : if (DDR_ARE_DEPENDENT (ddr) == NULL_TREE)
2954 : {
2955 : unsigned int i;
2956 :
2957 4275853 : for (i = 0; i < DDR_NUM_SUBSCRIPTS (ddr); i++)
2958 : {
2959 1160850 : struct subscript *subscript;
2960 :
2961 1160850 : subscript = DDR_SUBSCRIPT (ddr, i);
2962 1160850 : cf_a = SUB_CONFLICTS_IN_A (subscript);
2963 1160850 : cf_b = SUB_CONFLICTS_IN_B (subscript);
2964 :
2965 1160850 : fn_a = common_affine_function (cf_a);
2966 1160850 : fn_b = common_affine_function (cf_b);
2967 1160850 : if (!fn_a.exists () || !fn_b.exists ())
2968 : {
2969 0 : SUB_DISTANCE (subscript) = chrec_dont_know;
2970 0 : return;
2971 : }
2972 1160850 : diff = affine_fn_minus (fn_a, fn_b);
2973 :
2974 1160850 : if (affine_function_constant_p (diff))
2975 1160541 : SUB_DISTANCE (subscript) = affine_function_base (diff);
2976 : else
2977 309 : SUB_DISTANCE (subscript) = chrec_dont_know;
2978 :
2979 1160850 : affine_fn_free (diff);
2980 : }
2981 : }
2982 : }
2983 :
2984 : /* Returns the conflict function for "unknown". */
2985 :
2986 : static conflict_function *
2987 8073346 : conflict_fn_not_known (void)
2988 : {
2989 0 : conflict_function *fn = XCNEW (conflict_function);
2990 8073346 : fn->n = NOT_KNOWN;
2991 :
2992 8073346 : return fn;
2993 : }
2994 :
2995 : /* Returns the conflict function for "independent". */
2996 :
2997 : static conflict_function *
2998 4319394 : conflict_fn_no_dependence (void)
2999 : {
3000 0 : conflict_function *fn = XCNEW (conflict_function);
3001 4319394 : fn->n = NO_DEPENDENCE;
3002 :
3003 4319394 : return fn;
3004 : }
3005 :
3006 : /* Returns true if the address of OBJ is invariant in LOOP. */
3007 :
3008 : static bool
3009 3316649 : object_address_invariant_in_loop_p (const class loop *loop, const_tree obj)
3010 : {
3011 3482845 : while (handled_component_p (obj))
3012 : {
3013 171602 : if (TREE_CODE (obj) == ARRAY_REF)
3014 : {
3015 9714 : for (int i = 1; i < 4; ++i)
3016 8637 : if (chrec_contains_symbols_defined_in_loop (TREE_OPERAND (obj, i),
3017 8637 : loop->num))
3018 : return false;
3019 : }
3020 165119 : else if (TREE_CODE (obj) == COMPONENT_REF)
3021 : {
3022 143766 : if (chrec_contains_symbols_defined_in_loop (TREE_OPERAND (obj, 2),
3023 143766 : loop->num))
3024 : return false;
3025 : }
3026 166196 : obj = TREE_OPERAND (obj, 0);
3027 : }
3028 :
3029 3311243 : if (!INDIRECT_REF_P (obj)
3030 3311243 : && TREE_CODE (obj) != MEM_REF)
3031 : return true;
3032 :
3033 3286551 : return !chrec_contains_symbols_defined_in_loop (TREE_OPERAND (obj, 0),
3034 6573102 : loop->num);
3035 : }
3036 :
3037 : /* Helper for contains_ssa_ref_p. */
3038 :
3039 : static bool
3040 101043 : contains_ssa_ref_p_1 (tree, tree *idx, void *data)
3041 : {
3042 101043 : if (TREE_CODE (*idx) == SSA_NAME)
3043 : {
3044 94430 : *(bool *)data = true;
3045 94430 : return false;
3046 : }
3047 : return true;
3048 : }
3049 :
3050 : /* Returns true if the reference REF contains a SSA index. */
3051 :
3052 : static bool
3053 259058 : contains_ssa_ref_p (tree ref)
3054 : {
3055 259058 : bool res = false;
3056 0 : for_each_index (&ref, contains_ssa_ref_p_1, &res);
3057 259058 : return res;
3058 : }
3059 :
3060 : /* Returns false if we can prove that data references A and B do not alias,
3061 : true otherwise. If LOOP_NEST is false no cross-iteration aliases are
3062 : considered. */
3063 :
3064 : bool
3065 14735510 : dr_may_alias_p (const struct data_reference *a, const struct data_reference *b,
3066 : class loop *loop_nest)
3067 : {
3068 14735510 : tree addr_a = DR_BASE_OBJECT (a);
3069 14735510 : tree addr_b = DR_BASE_OBJECT (b);
3070 :
3071 : /* If we are not processing a loop nest but scalar code we
3072 : do not need to care about possible cross-iteration dependences
3073 : and thus can process the full original reference. Do so,
3074 : similar to how loop invariant motion applies extra offset-based
3075 : disambiguation. */
3076 14735510 : if (!loop_nest)
3077 : {
3078 8203218 : tree tree_size_a = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (a)));
3079 8203218 : tree tree_size_b = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (b)));
3080 :
3081 8203218 : if (DR_BASE_ADDRESS (a)
3082 8194502 : && DR_BASE_ADDRESS (b)
3083 8194155 : && operand_equal_p (DR_BASE_ADDRESS (a), DR_BASE_ADDRESS (b))
3084 7339828 : && operand_equal_p (DR_OFFSET (a), DR_OFFSET (b))
3085 7250106 : && tree_size_a
3086 7250106 : && tree_size_b
3087 7250097 : && poly_int_tree_p (tree_size_a)
3088 7250071 : && poly_int_tree_p (tree_size_b)
3089 15453289 : && !ranges_maybe_overlap_p (wi::to_poly_widest (DR_INIT (a)),
3090 7250071 : wi::to_poly_widest (tree_size_a),
3091 7250071 : wi::to_poly_widest (DR_INIT (b)),
3092 7250071 : wi::to_poly_widest (tree_size_b)))
3093 : {
3094 5404708 : gcc_assert (integer_zerop (DR_STEP (a))
3095 : && integer_zerop (DR_STEP (b)));
3096 5404741 : return false;
3097 : }
3098 :
3099 5597020 : aff_tree off1, off2;
3100 2798510 : poly_widest_int size1, size2;
3101 2798510 : get_inner_reference_aff (DR_REF (a), &off1, &size1);
3102 2798510 : get_inner_reference_aff (DR_REF (b), &off2, &size2);
3103 2798510 : aff_combination_scale (&off1, -1);
3104 2798510 : aff_combination_add (&off2, &off1);
3105 2798510 : if (aff_comb_cannot_overlap_p (&off2, size1, size2))
3106 33 : return false;
3107 2798510 : }
3108 :
3109 : /* Try the points-to information recorded for the base pointers the
3110 : references were originally analyzed from. DR_BASE_OBJECT can be less
3111 : precise, rooting at another SSA name or at one created after points-to
3112 : information was computed and thus without points-to info. A
3113 : recorded solution is not revalidated. It covers every dynamic value
3114 : of its SSA name, so it holds for cross-iteration queries as well. */
3115 9330769 : const pt_solution *pt_a = DR_PTR_INFO (a);
3116 9330769 : const pt_solution *pt_b = DR_PTR_INFO (b);
3117 9330769 : if (pt_a && pt_b && !pt_solutions_intersect (pt_a, pt_b))
3118 : return false;
3119 :
3120 8916690 : if ((TREE_CODE (addr_a) == MEM_REF || TREE_CODE (addr_a) == TARGET_MEM_REF)
3121 6552047 : && (TREE_CODE (addr_b) == MEM_REF || TREE_CODE (addr_b) == TARGET_MEM_REF)
3122 : /* For cross-iteration dependences the cliques must be valid for the
3123 : whole loop, not just individual iterations. */
3124 6295337 : && (!loop_nest
3125 5984299 : || MR_DEPENDENCE_CLIQUE (addr_a) == 1
3126 5313720 : || MR_DEPENDENCE_CLIQUE (addr_a) == loop_nest->owned_clique)
3127 6074052 : && MR_DEPENDENCE_CLIQUE (addr_a) == MR_DEPENDENCE_CLIQUE (addr_b)
3128 14798067 : && MR_DEPENDENCE_BASE (addr_a) != MR_DEPENDENCE_BASE (addr_b))
3129 : return false;
3130 :
3131 : /* If we had an evolution in a pointer-based MEM_REF BASE_OBJECT we
3132 : do not know the size of the base-object. So we cannot do any
3133 : offset/overlap based analysis but have to rely on points-to
3134 : information only. */
3135 8893755 : if (TREE_CODE (addr_a) == MEM_REF
3136 8893755 : && (DR_UNCONSTRAINED_BASE (a)
3137 4170200 : || TREE_CODE (TREE_OPERAND (addr_a, 0)) == SSA_NAME))
3138 : {
3139 : /* For true dependences we can apply TBAA. */
3140 4037169 : if (flag_strict_aliasing
3141 3862083 : && DR_IS_WRITE (a) && DR_IS_READ (b)
3142 4192900 : && !alias_sets_conflict_p (get_alias_set (DR_REF (a)),
3143 155731 : get_alias_set (DR_REF (b))))
3144 : return false;
3145 4007284 : if (TREE_CODE (addr_b) == MEM_REF)
3146 3901268 : return ptr_derefs_may_alias_p (TREE_OPERAND (addr_a, 0),
3147 7802536 : TREE_OPERAND (addr_b, 0));
3148 : else
3149 106016 : return ptr_derefs_may_alias_p (TREE_OPERAND (addr_a, 0),
3150 106016 : build_fold_addr_expr (addr_b));
3151 : }
3152 4856586 : else if (TREE_CODE (addr_b) == MEM_REF
3153 4856586 : && (DR_UNCONSTRAINED_BASE (b)
3154 2553151 : || TREE_CODE (TREE_OPERAND (addr_b, 0)) == SSA_NAME))
3155 : {
3156 : /* For true dependences we can apply TBAA. */
3157 327811 : if (flag_strict_aliasing
3158 269773 : && DR_IS_WRITE (a) && DR_IS_READ (b)
3159 404648 : && !alias_sets_conflict_p (get_alias_set (DR_REF (a)),
3160 76837 : get_alias_set (DR_REF (b))))
3161 : return false;
3162 312450 : if (TREE_CODE (addr_a) == MEM_REF)
3163 183879 : return ptr_derefs_may_alias_p (TREE_OPERAND (addr_a, 0),
3164 367758 : TREE_OPERAND (addr_b, 0));
3165 : else
3166 128571 : return ptr_derefs_may_alias_p (build_fold_addr_expr (addr_a),
3167 257142 : TREE_OPERAND (addr_b, 0));
3168 : }
3169 : /* If dr_analyze_innermost failed to handle a component we are
3170 : possibly left with a non-base in which case we didn't analyze
3171 : a possible evolution of the base when analyzing a loop. */
3172 4528775 : else if (loop_nest
3173 6695651 : && ((handled_component_p (addr_a) && contains_ssa_ref_p (addr_a))
3174 84751 : || (handled_component_p (addr_b) && contains_ssa_ref_p (addr_b))))
3175 : {
3176 : /* For true dependences we can apply TBAA. */
3177 94430 : if (flag_strict_aliasing
3178 93797 : && DR_IS_WRITE (a) && DR_IS_READ (b)
3179 103888 : && !alias_sets_conflict_p (get_alias_set (DR_REF (a)),
3180 9458 : get_alias_set (DR_REF (b))))
3181 : return false;
3182 90291 : if (TREE_CODE (addr_a) == MEM_REF)
3183 3845 : return ptr_derefs_may_alias_p (TREE_OPERAND (addr_a, 0),
3184 3845 : build_fold_addr_expr (addr_b));
3185 86446 : else if (TREE_CODE (addr_b) == MEM_REF)
3186 6382 : return ptr_derefs_may_alias_p (build_fold_addr_expr (addr_a),
3187 12764 : TREE_OPERAND (addr_b, 0));
3188 : else
3189 80064 : return ptr_derefs_may_alias_p (build_fold_addr_expr (addr_a),
3190 80064 : build_fold_addr_expr (addr_b));
3191 : }
3192 :
3193 : /* Otherwise DR_BASE_OBJECT is an access that covers the whole object
3194 : that is being subsetted in the loop nest. */
3195 4434345 : if (DR_IS_WRITE (a) && DR_IS_WRITE (b))
3196 2995414 : return refs_output_dependent_p (addr_a, addr_b);
3197 1438931 : else if (DR_IS_READ (a) && DR_IS_WRITE (b))
3198 409569 : return refs_anti_dependent_p (addr_a, addr_b);
3199 1029362 : return refs_may_alias_p (addr_a, addr_b);
3200 : }
3201 :
3202 : /* REF_A and REF_B both satisfy access_fn_component_p. Return true
3203 : if it is meaningful to compare their associated access functions
3204 : when checking for dependencies. */
3205 :
3206 : static bool
3207 2910994 : access_fn_components_comparable_p (tree ref_a, tree ref_b)
3208 : {
3209 : /* Allow pairs of component refs from the following sets:
3210 :
3211 : { REALPART_EXPR, IMAGPART_EXPR }
3212 : { COMPONENT_REF }
3213 : { ARRAY_REF }. */
3214 2910994 : tree_code code_a = TREE_CODE (ref_a);
3215 2910994 : tree_code code_b = TREE_CODE (ref_b);
3216 2910994 : if (code_a == IMAGPART_EXPR)
3217 36565 : code_a = REALPART_EXPR;
3218 2910994 : if (code_b == IMAGPART_EXPR)
3219 42883 : code_b = REALPART_EXPR;
3220 2910994 : if (code_a != code_b)
3221 : return false;
3222 :
3223 2886328 : if (TREE_CODE (ref_a) == COMPONENT_REF)
3224 : /* ??? We cannot simply use the type of operand #0 of the refs here as
3225 : the Fortran compiler smuggles type punning into COMPONENT_REFs.
3226 : Use the DECL_CONTEXT of the FIELD_DECLs instead. */
3227 989789 : return (DECL_CONTEXT (TREE_OPERAND (ref_a, 1))
3228 989789 : == DECL_CONTEXT (TREE_OPERAND (ref_b, 1)));
3229 :
3230 1896539 : return types_compatible_p (TREE_TYPE (TREE_OPERAND (ref_a, 0)),
3231 3793078 : TREE_TYPE (TREE_OPERAND (ref_b, 0)));
3232 : }
3233 :
3234 : /* Initialize a data dependence relation RES in LOOP_NEST. USE_ALT_INDICES
3235 : is true when the main indices of A and B were not comparable so we try again
3236 : with alternate indices computed on an indirect reference. */
3237 :
3238 : struct data_dependence_relation *
3239 6283304 : initialize_data_dependence_relation (struct data_dependence_relation *res,
3240 : vec<loop_p> loop_nest,
3241 : bool use_alt_indices)
3242 : {
3243 6653344 : struct data_reference *a = DDR_A (res);
3244 6653344 : struct data_reference *b = DDR_B (res);
3245 6653344 : unsigned int i;
3246 :
3247 6653344 : struct indices *indices_a = &a->indices;
3248 6653344 : struct indices *indices_b = &b->indices;
3249 6653344 : if (use_alt_indices)
3250 : {
3251 370040 : if (TREE_CODE (DR_REF (a)) != MEM_REF)
3252 231701 : indices_a = &a->alt_indices;
3253 370040 : if (TREE_CODE (DR_REF (b)) != MEM_REF)
3254 262434 : indices_b = &b->alt_indices;
3255 : }
3256 6653344 : unsigned int num_dimensions_a = indices_a->access_fns.length ();
3257 6653344 : unsigned int num_dimensions_b = indices_b->access_fns.length ();
3258 6653344 : if (num_dimensions_a == 0 || num_dimensions_b == 0)
3259 : {
3260 2265340 : DDR_ARE_DEPENDENT (res) = chrec_dont_know;
3261 2265340 : return res;
3262 : }
3263 :
3264 : /* For unconstrained bases, the root (highest-indexed) subscript
3265 : describes a variation in the base of the original DR_REF rather
3266 : than a component access. We have no type that accurately describes
3267 : the new DR_BASE_OBJECT (whose TREE_TYPE describes the type *after*
3268 : applying this subscript) so limit the search to the last real
3269 : component access.
3270 :
3271 : E.g. for:
3272 :
3273 : void
3274 : f (int a[][8], int b[][8])
3275 : {
3276 : for (int i = 0; i < 8; ++i)
3277 : a[i * 2][0] = b[i][0];
3278 : }
3279 :
3280 : the a and b accesses have a single ARRAY_REF component reference [0]
3281 : but have two subscripts. */
3282 4388004 : if (indices_a->unconstrained_base)
3283 2498673 : num_dimensions_a -= 1;
3284 4388004 : if (indices_b->unconstrained_base)
3285 2452314 : num_dimensions_b -= 1;
3286 :
3287 : /* These structures describe sequences of component references in
3288 : DR_REF (A) and DR_REF (B). Each component reference is tied to a
3289 : specific access function. */
3290 4388004 : struct {
3291 : /* The sequence starts at DR_ACCESS_FN (A, START_A) of A and
3292 : DR_ACCESS_FN (B, START_B) of B (inclusive) and extends to higher
3293 : indices. In C notation, these are the indices of the rightmost
3294 : component references; e.g. for a sequence .b.c.d, the start
3295 : index is for .d. */
3296 : unsigned int start_a;
3297 : unsigned int start_b;
3298 :
3299 : /* The sequence contains LENGTH consecutive access functions from
3300 : each DR. */
3301 : unsigned int length;
3302 :
3303 : /* The enclosing objects for the A and B sequences respectively,
3304 : i.e. the objects to which DR_ACCESS_FN (A, START_A + LENGTH - 1)
3305 : and DR_ACCESS_FN (B, START_B + LENGTH - 1) are applied. */
3306 : tree object_a;
3307 : tree object_b;
3308 4388004 : } full_seq = {}, struct_seq = {};
3309 :
3310 : /* Before each iteration of the loop:
3311 :
3312 : - REF_A is what you get after applying DR_ACCESS_FN (A, INDEX_A) and
3313 : - REF_B is what you get after applying DR_ACCESS_FN (B, INDEX_B). */
3314 4388004 : unsigned int index_a = 0;
3315 4388004 : unsigned int index_b = 0;
3316 4388004 : tree ref_a = DR_REF (a);
3317 4388004 : tree ref_b = DR_REF (b);
3318 :
3319 : /* Now walk the component references from the final DR_REFs back up to
3320 : the enclosing base objects. Each component reference corresponds
3321 : to one access function in the DR, with access function 0 being for
3322 : the final DR_REF and the highest-indexed access function being the
3323 : one that is applied to the base of the DR.
3324 :
3325 : Look for a sequence of component references whose access functions
3326 : are comparable (see access_fn_components_comparable_p). If more
3327 : than one such sequence exists, pick the one nearest the base
3328 : (which is the leftmost sequence in C notation). Store this sequence
3329 : in FULL_SEQ.
3330 :
3331 : For example, if we have:
3332 :
3333 : struct foo { struct bar s; ... } (*a)[10], (*b)[10];
3334 :
3335 : A: a[0][i].s.c.d
3336 : B: __real b[0][i].s.e[i].f
3337 :
3338 : (where d is the same type as the real component of f) then the access
3339 : functions would be:
3340 :
3341 : 0 1 2 3
3342 : A: .d .c .s [i]
3343 :
3344 : 0 1 2 3 4 5
3345 : B: __real .f [i] .e .s [i]
3346 :
3347 : The A0/B2 column isn't comparable, since .d is a COMPONENT_REF
3348 : and [i] is an ARRAY_REF. However, the A1/B3 column contains two
3349 : COMPONENT_REF accesses for struct bar, so is comparable. Likewise
3350 : the A2/B4 column contains two COMPONENT_REF accesses for struct foo,
3351 : so is comparable. The A3/B5 column contains two ARRAY_REFs that
3352 : index foo[10] arrays, so is again comparable. The sequence is
3353 : therefore:
3354 :
3355 : A: [1, 3] (i.e. [i].s.c)
3356 : B: [3, 5] (i.e. [i].s.e)
3357 :
3358 : Also look for sequences of component references whose access
3359 : functions are comparable and whose enclosing objects have the same
3360 : RECORD_TYPE. Store this sequence in STRUCT_SEQ. In the above
3361 : example, STRUCT_SEQ would be:
3362 :
3363 : A: [1, 2] (i.e. s.c)
3364 : B: [3, 4] (i.e. s.e) */
3365 7286084 : while (index_a < num_dimensions_a && index_b < num_dimensions_b)
3366 : {
3367 : /* The alternate indices form always has a single dimension
3368 : with unconstrained base. */
3369 2910994 : gcc_assert (!use_alt_indices);
3370 :
3371 : /* REF_A and REF_B must be one of the component access types
3372 : allowed by dr_analyze_indices. */
3373 2910994 : gcc_checking_assert (access_fn_component_p (ref_a));
3374 2910994 : gcc_checking_assert (access_fn_component_p (ref_b));
3375 :
3376 : /* Get the immediately-enclosing objects for REF_A and REF_B,
3377 : i.e. the references *before* applying DR_ACCESS_FN (A, INDEX_A)
3378 : and DR_ACCESS_FN (B, INDEX_B). */
3379 2910994 : tree object_a = TREE_OPERAND (ref_a, 0);
3380 2910994 : tree object_b = TREE_OPERAND (ref_b, 0);
3381 :
3382 2910994 : tree type_a = TREE_TYPE (object_a);
3383 2910994 : tree type_b = TREE_TYPE (object_b);
3384 2910994 : if (access_fn_components_comparable_p (ref_a, ref_b))
3385 : {
3386 : /* This pair of component accesses is comparable for dependence
3387 : analysis, so we can include DR_ACCESS_FN (A, INDEX_A) and
3388 : DR_ACCESS_FN (B, INDEX_B) in the sequence. */
3389 2658176 : if (full_seq.start_a + full_seq.length != index_a
3390 2601237 : || full_seq.start_b + full_seq.length != index_b)
3391 : {
3392 : /* The accesses don't extend the current sequence,
3393 : so start a new one here. */
3394 64667 : full_seq.start_a = index_a;
3395 64667 : full_seq.start_b = index_b;
3396 64667 : full_seq.length = 0;
3397 : }
3398 :
3399 : /* Add this pair of references to the sequence. */
3400 2658176 : full_seq.length += 1;
3401 2658176 : full_seq.object_a = object_a;
3402 2658176 : full_seq.object_b = object_b;
3403 :
3404 : /* If the enclosing objects are structures (and thus have the
3405 : same RECORD_TYPE), record the new sequence in STRUCT_SEQ. */
3406 2658176 : if (TREE_CODE (type_a) == RECORD_TYPE)
3407 778018 : struct_seq = full_seq;
3408 :
3409 : /* Move to the next containing reference for both A and B. */
3410 2658176 : ref_a = object_a;
3411 2658176 : ref_b = object_b;
3412 2658176 : index_a += 1;
3413 2658176 : index_b += 1;
3414 2658176 : continue;
3415 : }
3416 :
3417 : /* Try to approach equal type sizes. */
3418 252818 : if (!COMPLETE_TYPE_P (type_a)
3419 249775 : || !COMPLETE_TYPE_P (type_b)
3420 241779 : || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type_a))
3421 493001 : || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type_b)))
3422 : break;
3423 :
3424 239904 : unsigned HOST_WIDE_INT size_a = tree_to_uhwi (TYPE_SIZE_UNIT (type_a));
3425 239904 : unsigned HOST_WIDE_INT size_b = tree_to_uhwi (TYPE_SIZE_UNIT (type_b));
3426 239904 : if (size_a <= size_b)
3427 : {
3428 145202 : index_a += 1;
3429 145202 : ref_a = object_a;
3430 : }
3431 239904 : if (size_b <= size_a)
3432 : {
3433 109410 : index_b += 1;
3434 109410 : ref_b = object_b;
3435 : }
3436 : }
3437 :
3438 : /* See whether FULL_SEQ ends at the base and whether the two bases
3439 : are equal. We do not care about TBAA or alignment info so we can
3440 : use OEP_ADDRESS_OF to avoid false negatives. */
3441 4388004 : tree base_a = indices_a->base_object;
3442 4388004 : tree base_b = indices_b->base_object;
3443 4388004 : bool same_base_p = (full_seq.start_a + full_seq.length == num_dimensions_a
3444 4183467 : && full_seq.start_b + full_seq.length == num_dimensions_b
3445 4032076 : && (indices_a->unconstrained_base
3446 4032076 : == indices_b->unconstrained_base)
3447 4026900 : && operand_equal_p (base_a, base_b, OEP_ADDRESS_OF)
3448 3543289 : && (types_compatible_p (TREE_TYPE (base_a),
3449 3543289 : TREE_TYPE (base_b))
3450 867991 : || (!base_supports_access_fn_components_p (base_a)
3451 862784 : && !base_supports_access_fn_components_p (base_b)
3452 861197 : && operand_equal_p
3453 861197 : (TYPE_SIZE (TREE_TYPE (base_a)),
3454 861197 : TYPE_SIZE (TREE_TYPE (base_b)), 0)))
3455 7473320 : && (!loop_nest.exists ()
3456 3085316 : || (object_address_invariant_in_loop_p
3457 3085316 : (loop_nest[0], base_a))));
3458 :
3459 : /* If the bases are the same, we can include the base variation too.
3460 : E.g. the b accesses in:
3461 :
3462 : for (int i = 0; i < n; ++i)
3463 : b[i + 4][0] = b[i][0];
3464 :
3465 : have a definite dependence distance of 4, while for:
3466 :
3467 : for (int i = 0; i < n; ++i)
3468 : a[i + 4][0] = b[i][0];
3469 :
3470 : the dependence distance depends on the gap between a and b.
3471 :
3472 : If the bases are different then we can only rely on the sequence
3473 : rooted at a structure access, since arrays are allowed to overlap
3474 : arbitrarily and change shape arbitrarily. E.g. we treat this as
3475 : valid code:
3476 :
3477 : int a[256];
3478 : ...
3479 : ((int (*)[4][3]) &a[1])[i][0] += ((int (*)[4][3]) &a[2])[i][0];
3480 :
3481 : where two lvalues with the same int[4][3] type overlap, and where
3482 : both lvalues are distinct from the object's declared type. */
3483 : if (same_base_p)
3484 : {
3485 2958825 : if (indices_a->unconstrained_base)
3486 1496370 : full_seq.length += 1;
3487 : }
3488 : else
3489 : full_seq = struct_seq;
3490 :
3491 : /* Punt if we didn't find a suitable sequence. */
3492 4388004 : if (full_seq.length == 0)
3493 : {
3494 1146161 : if (use_alt_indices
3495 1024510 : || (TREE_CODE (DR_REF (a)) == MEM_REF
3496 792013 : && TREE_CODE (DR_REF (b)) == MEM_REF)
3497 372216 : || may_be_nonaddressable_p (DR_REF (a))
3498 1517968 : || may_be_nonaddressable_p (DR_REF (b)))
3499 : {
3500 : /* Fully exhausted possibilities. */
3501 776121 : DDR_ARE_DEPENDENT (res) = chrec_dont_know;
3502 776121 : return res;
3503 : }
3504 :
3505 : /* Try evaluating both DRs as dereferences of pointers. */
3506 370040 : if (!a->alt_indices.base_object
3507 172647 : && TREE_CODE (DR_REF (a)) != MEM_REF)
3508 : {
3509 34308 : tree alt_ref = build2 (MEM_REF, TREE_TYPE (DR_REF (a)),
3510 : build1 (ADDR_EXPR, ptr_type_node, DR_REF (a)),
3511 : build_int_cst
3512 : (reference_alias_ptr_type (DR_REF (a)), 0));
3513 102924 : dr_analyze_indices (&a->alt_indices, alt_ref,
3514 34308 : loop_preheader_edge (loop_nest[0]),
3515 : loop_containing_stmt (DR_STMT (a)));
3516 : }
3517 370040 : if (!b->alt_indices.base_object
3518 187597 : && TREE_CODE (DR_REF (b)) != MEM_REF)
3519 : {
3520 79991 : tree alt_ref = build2 (MEM_REF, TREE_TYPE (DR_REF (b)),
3521 : build1 (ADDR_EXPR, ptr_type_node, DR_REF (b)),
3522 : build_int_cst
3523 : (reference_alias_ptr_type (DR_REF (b)), 0));
3524 239973 : dr_analyze_indices (&b->alt_indices, alt_ref,
3525 79991 : loop_preheader_edge (loop_nest[0]),
3526 : loop_containing_stmt (DR_STMT (b)));
3527 : }
3528 370040 : return initialize_data_dependence_relation (res, loop_nest, true);
3529 : }
3530 :
3531 3241843 : if (!same_base_p)
3532 : {
3533 : /* Partial overlap is possible for different bases when strict aliasing
3534 : is not in effect. It's also possible if either base involves a union
3535 : access; e.g. for:
3536 :
3537 : struct s1 { int a[2]; };
3538 : struct s2 { struct s1 b; int c; };
3539 : struct s3 { int d; struct s1 e; };
3540 : union u { struct s2 f; struct s3 g; } *p, *q;
3541 :
3542 : the s1 at "p->f.b" (base "p->f") partially overlaps the s1 at
3543 : "p->g.e" (base "p->g") and might partially overlap the s1 at
3544 : "q->g.e" (base "q->g"). */
3545 283018 : if (!flag_strict_aliasing
3546 271340 : || ref_contains_union_access_p (full_seq.object_a)
3547 495699 : || ref_contains_union_access_p (full_seq.object_b))
3548 : {
3549 70363 : DDR_ARE_DEPENDENT (res) = chrec_dont_know;
3550 70363 : return res;
3551 : }
3552 :
3553 212655 : DDR_COULD_BE_INDEPENDENT_P (res) = true;
3554 212655 : if (!loop_nest.exists ()
3555 425310 : || (object_address_invariant_in_loop_p (loop_nest[0],
3556 212655 : full_seq.object_a)
3557 18678 : && object_address_invariant_in_loop_p (loop_nest[0],
3558 18678 : full_seq.object_b)))
3559 : {
3560 9273 : DDR_OBJECT_A (res) = full_seq.object_a;
3561 9273 : DDR_OBJECT_B (res) = full_seq.object_b;
3562 : }
3563 : }
3564 :
3565 3171480 : DDR_AFFINE_P (res) = true;
3566 3171480 : DDR_ARE_DEPENDENT (res) = NULL_TREE;
3567 3171480 : DDR_SUBSCRIPTS (res).create (full_seq.length);
3568 3171480 : DDR_LOOP_NEST (res) = loop_nest;
3569 3171480 : DDR_SELF_REFERENCE (res) = false;
3570 :
3571 7178551 : for (i = 0; i < full_seq.length; ++i)
3572 : {
3573 4007071 : struct subscript *subscript;
3574 :
3575 4007071 : subscript = XNEW (struct subscript);
3576 4007071 : SUB_ACCESS_FN (subscript, 0) = indices_a->access_fns[full_seq.start_a + i];
3577 4007071 : SUB_ACCESS_FN (subscript, 1) = indices_b->access_fns[full_seq.start_b + i];
3578 4007071 : SUB_CONFLICTS_IN_A (subscript) = conflict_fn_not_known ();
3579 4007071 : SUB_CONFLICTS_IN_B (subscript) = conflict_fn_not_known ();
3580 4007071 : SUB_LAST_CONFLICT (subscript) = chrec_dont_know;
3581 4007071 : SUB_DISTANCE (subscript) = chrec_dont_know;
3582 4007071 : DDR_SUBSCRIPTS (res).safe_push (subscript);
3583 : }
3584 :
3585 : return res;
3586 : }
3587 :
3588 : /* Initialize a data dependence relation between data accesses A and
3589 : B. NB_LOOPS is the number of loops surrounding the references: the
3590 : size of the classic distance/direction vectors. */
3591 :
3592 : struct data_dependence_relation *
3593 13484885 : initialize_data_dependence_relation (struct data_reference *a,
3594 : struct data_reference *b,
3595 : vec<loop_p> loop_nest)
3596 : {
3597 13484885 : data_dependence_relation *res = XCNEW (struct data_dependence_relation);
3598 13484885 : DDR_A (res) = a;
3599 13484885 : DDR_B (res) = b;
3600 13484885 : DDR_LOOP_NEST (res).create (0);
3601 13484885 : DDR_SUBSCRIPTS (res).create (0);
3602 13484885 : DDR_DIR_VECTS (res).create (0);
3603 13484885 : DDR_DIST_VECTS (res).create (0);
3604 :
3605 13484885 : if (a == NULL || b == NULL)
3606 : {
3607 0 : DDR_ARE_DEPENDENT (res) = chrec_dont_know;
3608 0 : return res;
3609 : }
3610 :
3611 : /* If the data references do not alias, then they are independent. */
3612 19998530 : if (!dr_may_alias_p (a, b, loop_nest.exists () ? loop_nest[0] : NULL))
3613 : {
3614 7201581 : DDR_ARE_DEPENDENT (res) = chrec_known;
3615 7201581 : return res;
3616 : }
3617 :
3618 6283304 : return initialize_data_dependence_relation (res, loop_nest, false);
3619 : }
3620 :
3621 :
3622 : /* Frees memory used by the conflict function F. */
3623 :
3624 : static void
3625 14916346 : free_conflict_function (conflict_function *f)
3626 : {
3627 14916346 : unsigned i;
3628 :
3629 14916346 : if (CF_NONTRIVIAL_P (f))
3630 : {
3631 5047212 : for (i = 0; i < f->n; i++)
3632 2523606 : affine_fn_free (f->fns[i]);
3633 : }
3634 14916346 : free (f);
3635 14916346 : }
3636 :
3637 : /* Frees memory used by SUBSCRIPTS. */
3638 :
3639 : static void
3640 3171480 : free_subscripts (vec<subscript_p> subscripts)
3641 : {
3642 13521511 : for (subscript_p s : subscripts)
3643 : {
3644 4007071 : free_conflict_function (s->conflicting_iterations_in_a);
3645 4007071 : free_conflict_function (s->conflicting_iterations_in_b);
3646 4007071 : free (s);
3647 : }
3648 3171480 : subscripts.release ();
3649 3171480 : }
3650 :
3651 : /* Set DDR_ARE_DEPENDENT to CHREC and finalize the subscript overlap
3652 : description. */
3653 :
3654 : static inline void
3655 2252349 : finalize_ddr_dependent (struct data_dependence_relation *ddr,
3656 : tree chrec)
3657 : {
3658 2252349 : DDR_ARE_DEPENDENT (ddr) = chrec;
3659 2252349 : free_subscripts (DDR_SUBSCRIPTS (ddr));
3660 2252349 : DDR_SUBSCRIPTS (ddr).create (0);
3661 : }
3662 :
3663 : /* The dependence relation DDR cannot be represented by a distance
3664 : vector. */
3665 :
3666 : static inline void
3667 2184 : non_affine_dependence_relation (struct data_dependence_relation *ddr)
3668 : {
3669 2184 : if (dump_file && (dump_flags & TDF_DETAILS))
3670 92 : fprintf (dump_file, "(Dependence relation cannot be represented by distance vector.) \n");
3671 :
3672 2184 : DDR_AFFINE_P (ddr) = false;
3673 2184 : }
3674 :
3675 :
3676 :
3677 : /* This section contains the classic Banerjee tests. */
3678 :
3679 : /* Returns true iff CHREC_A and CHREC_B are not dependent on any index
3680 : variables, i.e., if the ZIV (Zero Index Variable) test is true. */
3681 :
3682 : static inline bool
3683 2247693 : ziv_subscript_p (const_tree chrec_a, const_tree chrec_b)
3684 : {
3685 2247693 : return (evolution_function_is_constant_p (chrec_a)
3686 2757923 : && evolution_function_is_constant_p (chrec_b));
3687 : }
3688 :
3689 : /* Returns true iff CHREC_A and CHREC_B are dependent on an index
3690 : variable, i.e., if the SIV (Single Index Variable) test is true. */
3691 :
3692 : static bool
3693 1739250 : siv_subscript_p (const_tree chrec_a, const_tree chrec_b)
3694 : {
3695 3476716 : if ((evolution_function_is_constant_p (chrec_a)
3696 1787 : && evolution_function_is_univariate_p (chrec_b))
3697 3476716 : || (evolution_function_is_constant_p (chrec_b)
3698 1270 : && evolution_function_is_univariate_p (chrec_a)))
3699 : return true;
3700 :
3701 1736199 : if (evolution_function_is_univariate_p (chrec_a)
3702 1736199 : && evolution_function_is_univariate_p (chrec_b))
3703 : {
3704 1709832 : switch (TREE_CODE (chrec_a))
3705 : {
3706 1709832 : case POLYNOMIAL_CHREC:
3707 1709832 : switch (TREE_CODE (chrec_b))
3708 : {
3709 1709832 : case POLYNOMIAL_CHREC:
3710 1709832 : if (CHREC_VARIABLE (chrec_a) != CHREC_VARIABLE (chrec_b))
3711 : return false;
3712 : /* FALLTHRU */
3713 :
3714 1709750 : default:
3715 1709750 : return true;
3716 : }
3717 :
3718 : default:
3719 : return true;
3720 : }
3721 : }
3722 :
3723 : return false;
3724 : }
3725 :
3726 : /* Creates a conflict function with N dimensions. The affine functions
3727 : in each dimension follow. */
3728 :
3729 : static conflict_function *
3730 2523606 : conflict_fn (unsigned n, ...)
3731 : {
3732 2523606 : unsigned i;
3733 2523606 : conflict_function *ret = XCNEW (conflict_function);
3734 2523606 : va_list ap;
3735 :
3736 2523606 : gcc_assert (n > 0 && n <= MAX_DIM);
3737 2523606 : va_start (ap, n);
3738 :
3739 2523606 : ret->n = n;
3740 5047212 : for (i = 0; i < n; i++)
3741 2523606 : ret->fns[i] = va_arg (ap, affine_fn);
3742 2523606 : va_end (ap);
3743 :
3744 2523606 : return ret;
3745 : }
3746 :
3747 : /* Returns constant affine function with value CST. */
3748 :
3749 : static affine_fn
3750 2406472 : affine_fn_cst (tree cst)
3751 : {
3752 2406472 : affine_fn fn;
3753 2406472 : fn.create (1);
3754 2406472 : fn.quick_push (cst);
3755 2406472 : return fn;
3756 : }
3757 :
3758 : /* Returns affine function with single variable, CST + COEF * x_DIM. */
3759 :
3760 : static affine_fn
3761 117134 : affine_fn_univar (tree cst, unsigned dim, tree coef)
3762 : {
3763 117134 : affine_fn fn;
3764 117134 : fn.create (dim + 1);
3765 117134 : unsigned i;
3766 :
3767 117134 : gcc_assert (dim > 0);
3768 117134 : fn.quick_push (cst);
3769 234268 : for (i = 1; i < dim; i++)
3770 0 : fn.quick_push (integer_zero_node);
3771 117134 : fn.quick_push (coef);
3772 117134 : return fn;
3773 : }
3774 :
3775 : /* Analyze a ZIV (Zero Index Variable) subscript. *OVERLAPS_A and
3776 : *OVERLAPS_B are initialized to the functions that describe the
3777 : relation between the elements accessed twice by CHREC_A and
3778 : CHREC_B. For k >= 0, the following property is verified:
3779 :
3780 : CHREC_A (*OVERLAPS_A (k)) = CHREC_B (*OVERLAPS_B (k)). */
3781 :
3782 : static void
3783 508443 : analyze_ziv_subscript (tree chrec_a,
3784 : tree chrec_b,
3785 : conflict_function **overlaps_a,
3786 : conflict_function **overlaps_b,
3787 : tree *last_conflicts)
3788 : {
3789 508443 : tree type, difference;
3790 508443 : dependence_stats.num_ziv++;
3791 :
3792 508443 : if (dump_file && (dump_flags & TDF_DETAILS))
3793 22441 : fprintf (dump_file, "(analyze_ziv_subscript \n");
3794 :
3795 508443 : type = signed_type_for_types (TREE_TYPE (chrec_a), TREE_TYPE (chrec_b));
3796 508443 : chrec_a = chrec_convert (type, chrec_a, NULL);
3797 508443 : chrec_b = chrec_convert (type, chrec_b, NULL);
3798 508443 : difference = chrec_fold_minus (type, chrec_a, chrec_b);
3799 :
3800 508443 : switch (TREE_CODE (difference))
3801 : {
3802 508443 : case INTEGER_CST:
3803 508443 : if (integer_zerop (difference))
3804 : {
3805 : /* The difference is equal to zero: the accessed index
3806 : overlaps for each iteration in the loop. */
3807 0 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
3808 0 : *overlaps_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
3809 0 : *last_conflicts = chrec_dont_know;
3810 0 : dependence_stats.num_ziv_dependent++;
3811 : }
3812 : else
3813 : {
3814 : /* The accesses do not overlap. */
3815 508443 : *overlaps_a = conflict_fn_no_dependence ();
3816 508443 : *overlaps_b = conflict_fn_no_dependence ();
3817 508443 : *last_conflicts = integer_zero_node;
3818 508443 : dependence_stats.num_ziv_independent++;
3819 : }
3820 : break;
3821 :
3822 0 : default:
3823 : /* We're not sure whether the indexes overlap. For the moment,
3824 : conservatively answer "don't know". */
3825 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3826 0 : fprintf (dump_file, "ziv test failed: difference is non-integer.\n");
3827 :
3828 0 : *overlaps_a = conflict_fn_not_known ();
3829 0 : *overlaps_b = conflict_fn_not_known ();
3830 0 : *last_conflicts = chrec_dont_know;
3831 0 : dependence_stats.num_ziv_unimplemented++;
3832 0 : break;
3833 : }
3834 :
3835 508443 : if (dump_file && (dump_flags & TDF_DETAILS))
3836 22441 : fprintf (dump_file, ")\n");
3837 508443 : }
3838 :
3839 : /* Similar to max_stmt_executions_int, but returns the bound as a tree,
3840 : and only if it fits to the int type. If this is not the case, or the
3841 : bound on the number of iterations of LOOP could not be derived, returns
3842 : chrec_dont_know. */
3843 :
3844 : static tree
3845 0 : max_stmt_executions_tree (class loop *loop)
3846 : {
3847 0 : widest_int nit;
3848 :
3849 0 : if (!max_stmt_executions (loop, &nit))
3850 0 : return chrec_dont_know;
3851 :
3852 0 : if (!wi::fits_to_tree_p (nit, unsigned_type_node))
3853 0 : return chrec_dont_know;
3854 :
3855 0 : return wide_int_to_tree (unsigned_type_node, nit);
3856 0 : }
3857 :
3858 : /* Determine whether the CHREC is always positive/negative. If the expression
3859 : cannot be statically analyzed, return false, otherwise set the answer into
3860 : VALUE. */
3861 :
3862 : static bool
3863 4642 : chrec_is_positive (tree chrec, bool *value)
3864 : {
3865 4642 : bool value0, value1, value2;
3866 4642 : tree end_value, nb_iter;
3867 :
3868 4642 : switch (TREE_CODE (chrec))
3869 : {
3870 0 : case POLYNOMIAL_CHREC:
3871 0 : if (!chrec_is_positive (CHREC_LEFT (chrec), &value0)
3872 0 : || !chrec_is_positive (CHREC_RIGHT (chrec), &value1))
3873 : return false;
3874 :
3875 : /* FIXME -- overflows. */
3876 0 : if (value0 == value1)
3877 : {
3878 0 : *value = value0;
3879 0 : return true;
3880 : }
3881 :
3882 : /* Otherwise the chrec is under the form: "{-197, +, 2}_1",
3883 : and the proof consists in showing that the sign never
3884 : changes during the execution of the loop, from 0 to
3885 : loop->nb_iterations. */
3886 0 : if (!evolution_function_is_affine_p (chrec))
3887 : return false;
3888 :
3889 0 : nb_iter = number_of_latch_executions (get_chrec_loop (chrec));
3890 0 : if (chrec_contains_undetermined (nb_iter))
3891 : return false;
3892 :
3893 : #if 0
3894 : /* TODO -- If the test is after the exit, we may decrease the number of
3895 : iterations by one. */
3896 : if (after_exit)
3897 : nb_iter = chrec_fold_minus (type, nb_iter, build_int_cst (type, 1));
3898 : #endif
3899 :
3900 0 : end_value = chrec_apply (CHREC_VARIABLE (chrec), chrec, nb_iter);
3901 :
3902 0 : if (!chrec_is_positive (end_value, &value2))
3903 : return false;
3904 :
3905 0 : *value = value0;
3906 0 : return value0 == value1;
3907 :
3908 4642 : case INTEGER_CST:
3909 4642 : switch (tree_int_cst_sgn (chrec))
3910 : {
3911 2079 : case -1:
3912 2079 : *value = false;
3913 2079 : break;
3914 2563 : case 1:
3915 2563 : *value = true;
3916 2563 : break;
3917 : default:
3918 : return false;
3919 : }
3920 : return true;
3921 :
3922 : default:
3923 : return false;
3924 : }
3925 : }
3926 :
3927 :
3928 : /* Analyze a SIV (Single Index Variable) subscript where CHREC_A is a
3929 : constant, and CHREC_B is an affine function. *OVERLAPS_A and
3930 : *OVERLAPS_B are initialized to the functions that describe the
3931 : relation between the elements accessed twice by CHREC_A and
3932 : CHREC_B. For k >= 0, the following property is verified:
3933 :
3934 : CHREC_A (*OVERLAPS_A (k)) = CHREC_B (*OVERLAPS_B (k)). */
3935 :
3936 : static void
3937 3051 : analyze_siv_subscript_cst_affine (tree chrec_a,
3938 : tree chrec_b,
3939 : conflict_function **overlaps_a,
3940 : conflict_function **overlaps_b,
3941 : tree *last_conflicts)
3942 : {
3943 3051 : bool value0, value1, value2;
3944 3051 : tree type, difference, tmp;
3945 :
3946 3051 : type = signed_type_for_types (TREE_TYPE (chrec_a), TREE_TYPE (chrec_b));
3947 3051 : chrec_a = chrec_convert (type, chrec_a, NULL);
3948 3051 : chrec_b = chrec_convert (type, chrec_b, NULL);
3949 3051 : difference = chrec_fold_minus (type, initial_condition (chrec_b), chrec_a);
3950 :
3951 : /* Special case overlap in the first iteration. */
3952 3051 : if (integer_zerop (difference))
3953 : {
3954 728 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
3955 728 : *overlaps_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
3956 728 : *last_conflicts = integer_one_node;
3957 728 : return;
3958 : }
3959 :
3960 2323 : if (!chrec_is_positive (initial_condition (difference), &value0))
3961 : {
3962 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3963 0 : fprintf (dump_file, "siv test failed: chrec is not positive.\n");
3964 :
3965 0 : dependence_stats.num_siv_unimplemented++;
3966 0 : *overlaps_a = conflict_fn_not_known ();
3967 0 : *overlaps_b = conflict_fn_not_known ();
3968 0 : *last_conflicts = chrec_dont_know;
3969 0 : return;
3970 : }
3971 : else
3972 : {
3973 2323 : if (value0 == false)
3974 : {
3975 1865 : if (TREE_CODE (chrec_b) != POLYNOMIAL_CHREC
3976 1865 : || !chrec_is_positive (CHREC_RIGHT (chrec_b), &value1))
3977 : {
3978 4 : if (dump_file && (dump_flags & TDF_DETAILS))
3979 0 : fprintf (dump_file, "siv test failed: chrec not positive.\n");
3980 :
3981 4 : *overlaps_a = conflict_fn_not_known ();
3982 4 : *overlaps_b = conflict_fn_not_known ();
3983 4 : *last_conflicts = chrec_dont_know;
3984 4 : dependence_stats.num_siv_unimplemented++;
3985 4 : return;
3986 : }
3987 : else
3988 : {
3989 1861 : if (value1 == true)
3990 : {
3991 : /* Example:
3992 : chrec_a = 12
3993 : chrec_b = {10, +, 1}
3994 : */
3995 :
3996 1861 : if (tree_fold_divides_p (CHREC_RIGHT (chrec_b), difference))
3997 : {
3998 1563 : HOST_WIDE_INT numiter;
3999 1563 : class loop *loop = get_chrec_loop (chrec_b);
4000 :
4001 1563 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
4002 1563 : tmp = fold_build2 (EXACT_DIV_EXPR, type,
4003 : fold_build1 (ABS_EXPR, type, difference),
4004 : CHREC_RIGHT (chrec_b));
4005 1563 : *overlaps_b = conflict_fn (1, affine_fn_cst (tmp));
4006 1563 : *last_conflicts = integer_one_node;
4007 :
4008 :
4009 : /* Perform weak-zero siv test to see if overlap is
4010 : outside the loop bounds. */
4011 1563 : numiter = max_stmt_executions_int (loop);
4012 :
4013 1563 : if (numiter >= 0
4014 1563 : && compare_tree_int (tmp, numiter) > 0)
4015 : {
4016 0 : free_conflict_function (*overlaps_a);
4017 0 : free_conflict_function (*overlaps_b);
4018 0 : *overlaps_a = conflict_fn_no_dependence ();
4019 0 : *overlaps_b = conflict_fn_no_dependence ();
4020 0 : *last_conflicts = integer_zero_node;
4021 0 : dependence_stats.num_siv_independent++;
4022 0 : return;
4023 : }
4024 1563 : dependence_stats.num_siv_dependent++;
4025 1563 : return;
4026 : }
4027 :
4028 : /* When the step does not divide the difference, there are
4029 : no overlaps. */
4030 : else
4031 : {
4032 298 : *overlaps_a = conflict_fn_no_dependence ();
4033 298 : *overlaps_b = conflict_fn_no_dependence ();
4034 298 : *last_conflicts = integer_zero_node;
4035 298 : dependence_stats.num_siv_independent++;
4036 298 : return;
4037 : }
4038 : }
4039 :
4040 : else
4041 : {
4042 : /* Example:
4043 : chrec_a = 12
4044 : chrec_b = {10, +, -1}
4045 :
4046 : In this case, chrec_a will not overlap with chrec_b. */
4047 0 : *overlaps_a = conflict_fn_no_dependence ();
4048 0 : *overlaps_b = conflict_fn_no_dependence ();
4049 0 : *last_conflicts = integer_zero_node;
4050 0 : dependence_stats.num_siv_independent++;
4051 0 : return;
4052 : }
4053 : }
4054 : }
4055 : else
4056 : {
4057 458 : if (TREE_CODE (chrec_b) != POLYNOMIAL_CHREC
4058 458 : || !chrec_is_positive (CHREC_RIGHT (chrec_b), &value2))
4059 : {
4060 0 : if (dump_file && (dump_flags & TDF_DETAILS))
4061 0 : fprintf (dump_file, "siv test failed: chrec not positive.\n");
4062 :
4063 0 : *overlaps_a = conflict_fn_not_known ();
4064 0 : *overlaps_b = conflict_fn_not_known ();
4065 0 : *last_conflicts = chrec_dont_know;
4066 0 : dependence_stats.num_siv_unimplemented++;
4067 0 : return;
4068 : }
4069 : else
4070 : {
4071 458 : if (value2 == false)
4072 : {
4073 : /* Example:
4074 : chrec_a = 3
4075 : chrec_b = {10, +, -1}
4076 : */
4077 214 : if (tree_fold_divides_p (CHREC_RIGHT (chrec_b), difference))
4078 : {
4079 109 : HOST_WIDE_INT numiter;
4080 109 : class loop *loop = get_chrec_loop (chrec_b);
4081 :
4082 109 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
4083 109 : tmp = fold_build2 (EXACT_DIV_EXPR, type, difference,
4084 : CHREC_RIGHT (chrec_b));
4085 109 : *overlaps_b = conflict_fn (1, affine_fn_cst (tmp));
4086 109 : *last_conflicts = integer_one_node;
4087 :
4088 : /* Perform weak-zero siv test to see if overlap is
4089 : outside the loop bounds. */
4090 109 : numiter = max_stmt_executions_int (loop);
4091 :
4092 109 : if (numiter >= 0
4093 109 : && compare_tree_int (tmp, numiter) > 0)
4094 : {
4095 0 : free_conflict_function (*overlaps_a);
4096 0 : free_conflict_function (*overlaps_b);
4097 0 : *overlaps_a = conflict_fn_no_dependence ();
4098 0 : *overlaps_b = conflict_fn_no_dependence ();
4099 0 : *last_conflicts = integer_zero_node;
4100 0 : dependence_stats.num_siv_independent++;
4101 0 : return;
4102 : }
4103 109 : dependence_stats.num_siv_dependent++;
4104 109 : return;
4105 : }
4106 :
4107 : /* When the step does not divide the difference, there
4108 : are no overlaps. */
4109 : else
4110 : {
4111 105 : *overlaps_a = conflict_fn_no_dependence ();
4112 105 : *overlaps_b = conflict_fn_no_dependence ();
4113 105 : *last_conflicts = integer_zero_node;
4114 105 : dependence_stats.num_siv_independent++;
4115 105 : return;
4116 : }
4117 : }
4118 : else
4119 : {
4120 : /* Example:
4121 : chrec_a = 3
4122 : chrec_b = {4, +, 1}
4123 :
4124 : In this case, chrec_a will not overlap with chrec_b. */
4125 244 : *overlaps_a = conflict_fn_no_dependence ();
4126 244 : *overlaps_b = conflict_fn_no_dependence ();
4127 244 : *last_conflicts = integer_zero_node;
4128 244 : dependence_stats.num_siv_independent++;
4129 244 : return;
4130 : }
4131 : }
4132 : }
4133 : }
4134 : }
4135 :
4136 : /* Helper recursive function for initializing the matrix A. Returns
4137 : the initial value of CHREC. */
4138 :
4139 : static tree
4140 3379846 : initialize_matrix_A (lambda_matrix A, tree chrec, unsigned index, int mult)
4141 : {
4142 6759684 : gcc_assert (chrec);
4143 :
4144 6759684 : switch (TREE_CODE (chrec))
4145 : {
4146 3379846 : case POLYNOMIAL_CHREC:
4147 3379846 : HOST_WIDE_INT chrec_right;
4148 3379846 : if (!cst_and_fits_in_hwi (CHREC_RIGHT (chrec)))
4149 8 : return chrec_dont_know;
4150 3379838 : chrec_right = int_cst_value (CHREC_RIGHT (chrec));
4151 : /* We want to be able to negate without overflow. */
4152 3379838 : if (chrec_right == HOST_WIDE_INT_MIN)
4153 0 : return chrec_dont_know;
4154 3379838 : A[index][0] = mult * chrec_right;
4155 3379838 : return initialize_matrix_A (A, CHREC_LEFT (chrec), index + 1, mult);
4156 :
4157 0 : case PLUS_EXPR:
4158 0 : case MULT_EXPR:
4159 0 : case MINUS_EXPR:
4160 0 : {
4161 0 : tree op0 = initialize_matrix_A (A, TREE_OPERAND (chrec, 0), index, mult);
4162 0 : tree op1 = initialize_matrix_A (A, TREE_OPERAND (chrec, 1), index, mult);
4163 :
4164 0 : return chrec_fold_op (TREE_CODE (chrec), chrec_type (chrec), op0, op1);
4165 : }
4166 :
4167 0 : CASE_CONVERT:
4168 0 : {
4169 0 : tree op = initialize_matrix_A (A, TREE_OPERAND (chrec, 0), index, mult);
4170 0 : return chrec_convert (chrec_type (chrec), op, NULL);
4171 : }
4172 :
4173 0 : case BIT_NOT_EXPR:
4174 0 : {
4175 : /* Handle ~X as -1 - X. */
4176 0 : tree op = initialize_matrix_A (A, TREE_OPERAND (chrec, 0), index, mult);
4177 0 : return chrec_fold_op (MINUS_EXPR, chrec_type (chrec),
4178 0 : build_int_cst (TREE_TYPE (chrec), -1), op);
4179 : }
4180 :
4181 3379838 : case INTEGER_CST:
4182 3379838 : return cst_and_fits_in_hwi (chrec) ? chrec : chrec_dont_know;
4183 :
4184 0 : default:
4185 0 : gcc_unreachable ();
4186 : return NULL_TREE;
4187 : }
4188 : }
4189 :
4190 : #define FLOOR_DIV(x,y) ((x) / (y))
4191 :
4192 : /* Solves the special case of the Diophantine equation:
4193 : | {0, +, STEP_A}_x (OVERLAPS_A) = {0, +, STEP_B}_y (OVERLAPS_B)
4194 :
4195 : Computes the descriptions OVERLAPS_A and OVERLAPS_B. NITER is the
4196 : number of iterations that loops X and Y run. The overlaps will be
4197 : constructed as evolutions in dimension DIM. */
4198 :
4199 : static void
4200 64 : compute_overlap_steps_for_affine_univar (HOST_WIDE_INT niter,
4201 : HOST_WIDE_INT step_a,
4202 : HOST_WIDE_INT step_b,
4203 : affine_fn *overlaps_a,
4204 : affine_fn *overlaps_b,
4205 : tree *last_conflicts, int dim)
4206 : {
4207 64 : if (((step_a > 0 && step_b > 0)
4208 8 : || (step_a < 0 && step_b < 0)))
4209 : {
4210 60 : HOST_WIDE_INT step_overlaps_a, step_overlaps_b;
4211 60 : HOST_WIDE_INT gcd_steps_a_b, last_conflict, tau2;
4212 :
4213 60 : gcd_steps_a_b = gcd (step_a, step_b);
4214 60 : step_overlaps_a = step_b / gcd_steps_a_b;
4215 60 : step_overlaps_b = step_a / gcd_steps_a_b;
4216 :
4217 60 : if (niter > 0)
4218 : {
4219 60 : tau2 = FLOOR_DIV (niter, step_overlaps_a);
4220 60 : tau2 = MIN (tau2, FLOOR_DIV (niter, step_overlaps_b));
4221 60 : last_conflict = tau2;
4222 60 : *last_conflicts = build_int_cst (integer_type_node, last_conflict);
4223 : }
4224 : else
4225 0 : *last_conflicts = chrec_dont_know;
4226 :
4227 60 : *overlaps_a = affine_fn_univar (integer_zero_node, dim,
4228 : build_int_cst (integer_type_node,
4229 60 : step_overlaps_a));
4230 60 : *overlaps_b = affine_fn_univar (integer_zero_node, dim,
4231 : build_int_cst (integer_type_node,
4232 60 : step_overlaps_b));
4233 60 : }
4234 :
4235 : else
4236 : {
4237 4 : *overlaps_a = affine_fn_cst (integer_zero_node);
4238 4 : *overlaps_b = affine_fn_cst (integer_zero_node);
4239 4 : *last_conflicts = integer_zero_node;
4240 : }
4241 64 : }
4242 :
4243 : /* Solves the special case of a Diophantine equation where CHREC_A is
4244 : an affine bivariate function, and CHREC_B is an affine univariate
4245 : function. For example,
4246 :
4247 : | {{0, +, 1}_x, +, 1335}_y = {0, +, 1336}_z
4248 :
4249 : has the following overlapping functions:
4250 :
4251 : | x (t, u, v) = {{0, +, 1336}_t, +, 1}_v
4252 : | y (t, u, v) = {{0, +, 1336}_u, +, 1}_v
4253 : | z (t, u, v) = {{{0, +, 1}_t, +, 1335}_u, +, 1}_v
4254 :
4255 : FORNOW: This is a specialized implementation for a case occurring in
4256 : a common benchmark. Implement the general algorithm. */
4257 :
4258 : static void
4259 0 : compute_overlap_steps_for_affine_1_2 (tree chrec_a, tree chrec_b,
4260 : conflict_function **overlaps_a,
4261 : conflict_function **overlaps_b,
4262 : tree *last_conflicts)
4263 : {
4264 0 : bool xz_p, yz_p, xyz_p;
4265 0 : HOST_WIDE_INT step_x, step_y, step_z;
4266 0 : HOST_WIDE_INT niter_x, niter_y, niter_z, niter;
4267 0 : affine_fn overlaps_a_xz, overlaps_b_xz;
4268 0 : affine_fn overlaps_a_yz, overlaps_b_yz;
4269 0 : affine_fn overlaps_a_xyz, overlaps_b_xyz;
4270 0 : affine_fn ova1, ova2, ovb;
4271 0 : tree last_conflicts_xz, last_conflicts_yz, last_conflicts_xyz;
4272 :
4273 0 : step_x = int_cst_value (CHREC_RIGHT (CHREC_LEFT (chrec_a)));
4274 0 : step_y = int_cst_value (CHREC_RIGHT (chrec_a));
4275 0 : step_z = int_cst_value (CHREC_RIGHT (chrec_b));
4276 :
4277 0 : niter_x = max_stmt_executions_int (get_chrec_loop (CHREC_LEFT (chrec_a)));
4278 0 : niter_y = max_stmt_executions_int (get_chrec_loop (chrec_a));
4279 0 : niter_z = max_stmt_executions_int (get_chrec_loop (chrec_b));
4280 :
4281 0 : if (niter_x < 0 || niter_y < 0 || niter_z < 0)
4282 : {
4283 0 : if (dump_file && (dump_flags & TDF_DETAILS))
4284 0 : fprintf (dump_file, "overlap steps test failed: no iteration counts.\n");
4285 :
4286 0 : *overlaps_a = conflict_fn_not_known ();
4287 0 : *overlaps_b = conflict_fn_not_known ();
4288 0 : *last_conflicts = chrec_dont_know;
4289 0 : return;
4290 : }
4291 :
4292 0 : niter = MIN (niter_x, niter_z);
4293 0 : compute_overlap_steps_for_affine_univar (niter, step_x, step_z,
4294 : &overlaps_a_xz,
4295 : &overlaps_b_xz,
4296 : &last_conflicts_xz, 1);
4297 0 : niter = MIN (niter_y, niter_z);
4298 0 : compute_overlap_steps_for_affine_univar (niter, step_y, step_z,
4299 : &overlaps_a_yz,
4300 : &overlaps_b_yz,
4301 : &last_conflicts_yz, 2);
4302 0 : niter = MIN (niter_x, niter_z);
4303 0 : niter = MIN (niter_y, niter);
4304 0 : compute_overlap_steps_for_affine_univar (niter, step_x + step_y, step_z,
4305 : &overlaps_a_xyz,
4306 : &overlaps_b_xyz,
4307 : &last_conflicts_xyz, 3);
4308 :
4309 0 : xz_p = !integer_zerop (last_conflicts_xz);
4310 0 : yz_p = !integer_zerop (last_conflicts_yz);
4311 0 : xyz_p = !integer_zerop (last_conflicts_xyz);
4312 :
4313 0 : if (xz_p || yz_p || xyz_p)
4314 : {
4315 0 : ova1 = affine_fn_cst (integer_zero_node);
4316 0 : ova2 = affine_fn_cst (integer_zero_node);
4317 0 : ovb = affine_fn_cst (integer_zero_node);
4318 0 : if (xz_p)
4319 : {
4320 0 : affine_fn t0 = ova1;
4321 0 : affine_fn t2 = ovb;
4322 :
4323 0 : ova1 = affine_fn_plus (ova1, overlaps_a_xz);
4324 0 : ovb = affine_fn_plus (ovb, overlaps_b_xz);
4325 0 : affine_fn_free (t0);
4326 0 : affine_fn_free (t2);
4327 0 : *last_conflicts = last_conflicts_xz;
4328 : }
4329 0 : if (yz_p)
4330 : {
4331 0 : affine_fn t0 = ova2;
4332 0 : affine_fn t2 = ovb;
4333 :
4334 0 : ova2 = affine_fn_plus (ova2, overlaps_a_yz);
4335 0 : ovb = affine_fn_plus (ovb, overlaps_b_yz);
4336 0 : affine_fn_free (t0);
4337 0 : affine_fn_free (t2);
4338 0 : *last_conflicts = last_conflicts_yz;
4339 : }
4340 0 : if (xyz_p)
4341 : {
4342 0 : affine_fn t0 = ova1;
4343 0 : affine_fn t2 = ova2;
4344 0 : affine_fn t4 = ovb;
4345 :
4346 0 : ova1 = affine_fn_plus (ova1, overlaps_a_xyz);
4347 0 : ova2 = affine_fn_plus (ova2, overlaps_a_xyz);
4348 0 : ovb = affine_fn_plus (ovb, overlaps_b_xyz);
4349 0 : affine_fn_free (t0);
4350 0 : affine_fn_free (t2);
4351 0 : affine_fn_free (t4);
4352 0 : *last_conflicts = last_conflicts_xyz;
4353 : }
4354 0 : *overlaps_a = conflict_fn (2, ova1, ova2);
4355 0 : *overlaps_b = conflict_fn (1, ovb);
4356 0 : }
4357 : else
4358 : {
4359 0 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
4360 0 : *overlaps_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
4361 0 : *last_conflicts = integer_zero_node;
4362 : }
4363 :
4364 0 : affine_fn_free (overlaps_a_xz);
4365 0 : affine_fn_free (overlaps_b_xz);
4366 0 : affine_fn_free (overlaps_a_yz);
4367 0 : affine_fn_free (overlaps_b_yz);
4368 0 : affine_fn_free (overlaps_a_xyz);
4369 0 : affine_fn_free (overlaps_b_xyz);
4370 : }
4371 :
4372 : /* Copy the elements of vector VEC1 with length SIZE to VEC2. */
4373 :
4374 : static void
4375 3425872 : lambda_vector_copy (lambda_vector vec1, lambda_vector vec2,
4376 : int size)
4377 : {
4378 3425872 : memcpy (vec2, vec1, size * sizeof (*vec1));
4379 0 : }
4380 :
4381 : /* Copy the elements of M x N matrix MAT1 to MAT2. */
4382 :
4383 : static void
4384 1689847 : lambda_matrix_copy (lambda_matrix mat1, lambda_matrix mat2,
4385 : int m, int n)
4386 : {
4387 1689847 : int i;
4388 :
4389 5069541 : for (i = 0; i < m; i++)
4390 3379694 : lambda_vector_copy (mat1[i], mat2[i], n);
4391 1689847 : }
4392 :
4393 : /* Store the N x N identity matrix in MAT. */
4394 :
4395 : static void
4396 1689847 : lambda_matrix_id (lambda_matrix mat, int size)
4397 : {
4398 1689847 : int i, j;
4399 :
4400 5069541 : for (i = 0; i < size; i++)
4401 10139082 : for (j = 0; j < size; j++)
4402 10139082 : mat[i][j] = (i == j) ? 1 : 0;
4403 1689847 : }
4404 :
4405 : /* Return the index of the first nonzero element of vector VEC1 between
4406 : START and N. We must have START <= N.
4407 : Returns N if VEC1 is the zero vector. */
4408 :
4409 : static int
4410 1689847 : lambda_vector_first_nz (lambda_vector vec1, int n, int start)
4411 : {
4412 1689847 : int j = start;
4413 1689847 : while (j < n && vec1[j] == 0)
4414 0 : j++;
4415 1689847 : return j;
4416 : }
4417 :
4418 : /* Add a multiple of row R1 of matrix MAT with N columns to row R2:
4419 : R2 = R2 + CONST1 * R1. */
4420 :
4421 : static bool
4422 3379968 : lambda_matrix_row_add (lambda_matrix mat, int n, int r1, int r2,
4423 : lambda_int const1)
4424 : {
4425 3379968 : int i;
4426 :
4427 3379968 : if (const1 == 0)
4428 : return true;
4429 :
4430 8449325 : for (i = 0; i < n; i++)
4431 : {
4432 5069595 : bool ovf;
4433 5069595 : lambda_int tem = mul_hwi (mat[r1][i], const1, &ovf);
4434 5069595 : if (ovf)
4435 3379968 : return false;
4436 5069595 : lambda_int tem2 = add_hwi (mat[r2][i], tem, &ovf);
4437 5069595 : if (ovf || tem2 == HOST_WIDE_INT_MIN)
4438 : return false;
4439 5069595 : mat[r2][i] = tem2;
4440 : }
4441 :
4442 : return true;
4443 : }
4444 :
4445 : /* Multiply vector VEC1 of length SIZE by a constant CONST1,
4446 : and store the result in VEC2. */
4447 :
4448 : static void
4449 1679261 : lambda_vector_mult_const (lambda_vector vec1, lambda_vector vec2,
4450 : int size, lambda_int const1)
4451 : {
4452 1679261 : int i;
4453 :
4454 1679261 : if (const1 == 0)
4455 0 : lambda_vector_clear (vec2, size);
4456 : else
4457 5037783 : for (i = 0; i < size; i++)
4458 3358522 : vec2[i] = const1 * vec1[i];
4459 1679261 : }
4460 :
4461 : /* Negate vector VEC1 with length SIZE and store it in VEC2. */
4462 :
4463 : static void
4464 1679261 : lambda_vector_negate (lambda_vector vec1, lambda_vector vec2,
4465 : int size)
4466 : {
4467 0 : lambda_vector_mult_const (vec1, vec2, size, -1);
4468 0 : }
4469 :
4470 : /* Negate row R1 of matrix MAT which has N columns. */
4471 :
4472 : static void
4473 1679261 : lambda_matrix_row_negate (lambda_matrix mat, int n, int r1)
4474 : {
4475 0 : lambda_vector_negate (mat[r1], mat[r1], n);
4476 0 : }
4477 :
4478 : /* Return true if two vectors are equal. */
4479 :
4480 : static bool
4481 373706 : lambda_vector_equal (lambda_vector vec1, lambda_vector vec2, int size)
4482 : {
4483 373706 : int i;
4484 380274 : for (i = 0; i < size; i++)
4485 378537 : if (vec1[i] != vec2[i])
4486 : return false;
4487 : return true;
4488 : }
4489 :
4490 : /* Given an M x N integer matrix A, this function determines an M x
4491 : M unimodular matrix U, and an M x N echelon matrix S such that
4492 : "U.A = S". This decomposition is also known as "right Hermite".
4493 :
4494 : Ref: Algorithm 2.1 page 33 in "Loop Transformations for
4495 : Restructuring Compilers" Utpal Banerjee. */
4496 :
4497 : static bool
4498 1689847 : lambda_matrix_right_hermite (lambda_matrix A, int m, int n,
4499 : lambda_matrix S, lambda_matrix U)
4500 : {
4501 1689847 : int i, j, i0 = 0;
4502 :
4503 1689847 : lambda_matrix_copy (A, S, m, n);
4504 1689847 : lambda_matrix_id (U, m);
4505 :
4506 5069541 : for (j = 0; j < n; j++)
4507 : {
4508 3379694 : if (lambda_vector_first_nz (S[j], m, i0) < m)
4509 : {
4510 1689847 : ++i0;
4511 3379694 : for (i = m - 1; i >= i0; i--)
4512 : {
4513 3379831 : while (S[i][j] != 0)
4514 : {
4515 1689984 : lambda_int factor, a, b;
4516 :
4517 1689984 : a = S[i-1][j];
4518 1689984 : b = S[i][j];
4519 1689984 : gcc_assert (a != HOST_WIDE_INT_MIN);
4520 1689984 : factor = a / b;
4521 :
4522 1689984 : if (!lambda_matrix_row_add (S, n, i, i-1, -factor))
4523 : return false;
4524 1689984 : std::swap (S[i], S[i-1]);
4525 :
4526 1689984 : if (!lambda_matrix_row_add (U, m, i, i-1, -factor))
4527 : return false;
4528 1689984 : std::swap (U[i], U[i-1]);
4529 : }
4530 : }
4531 : }
4532 : }
4533 :
4534 : return true;
4535 : }
4536 :
4537 : /* Determines the overlapping elements due to accesses CHREC_A and
4538 : CHREC_B, that are affine functions. This function cannot handle
4539 : symbolic evolution functions, ie. when initial conditions are
4540 : parameters, because it uses lambda matrices of integers. */
4541 :
4542 : static void
4543 1689923 : analyze_subscript_affine_affine (tree chrec_a,
4544 : tree chrec_b,
4545 : conflict_function **overlaps_a,
4546 : conflict_function **overlaps_b,
4547 : tree *last_conflicts)
4548 : {
4549 1689923 : unsigned nb_vars_a, nb_vars_b, dim;
4550 1689923 : lambda_int gamma, gcd_alpha_beta;
4551 1689923 : lambda_matrix A, U, S;
4552 1689923 : struct obstack scratch_obstack;
4553 :
4554 1689923 : if (eq_evolutions_p (chrec_a, chrec_b))
4555 : {
4556 : /* The accessed index overlaps for each iteration in the
4557 : loop. */
4558 0 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
4559 0 : *overlaps_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
4560 0 : *last_conflicts = chrec_dont_know;
4561 0 : return;
4562 : }
4563 1689923 : if (dump_file && (dump_flags & TDF_DETAILS))
4564 21033 : fprintf (dump_file, "(analyze_subscript_affine_affine \n");
4565 :
4566 : /* For determining the initial intersection, we have to solve a
4567 : Diophantine equation. This is the most time consuming part.
4568 :
4569 : For answering to the question: "Is there a dependence?" we have
4570 : to prove that there exists a solution to the Diophantine
4571 : equation, and that the solution is in the iteration domain,
4572 : i.e. the solution is positive or zero, and that the solution
4573 : happens before the upper bound loop.nb_iterations. Otherwise
4574 : there is no dependence. This function outputs a description of
4575 : the iterations that hold the intersections. */
4576 :
4577 1689923 : nb_vars_a = nb_vars_in_chrec (chrec_a);
4578 1689923 : nb_vars_b = nb_vars_in_chrec (chrec_b);
4579 :
4580 1689923 : gcc_obstack_init (&scratch_obstack);
4581 :
4582 1689923 : dim = nb_vars_a + nb_vars_b;
4583 1689923 : U = lambda_matrix_new (dim, dim, &scratch_obstack);
4584 1689923 : A = lambda_matrix_new (dim, 1, &scratch_obstack);
4585 1689923 : S = lambda_matrix_new (dim, 1, &scratch_obstack);
4586 :
4587 1689923 : tree init_a = initialize_matrix_A (A, chrec_a, 0, 1);
4588 1689923 : tree init_b = initialize_matrix_A (A, chrec_b, nb_vars_a, -1);
4589 1689923 : if (init_a == chrec_dont_know
4590 1689911 : || init_b == chrec_dont_know)
4591 : {
4592 12 : if (dump_file && (dump_flags & TDF_DETAILS))
4593 0 : fprintf (dump_file, "affine-affine test failed: "
4594 : "representation issue.\n");
4595 12 : *overlaps_a = conflict_fn_not_known ();
4596 12 : *overlaps_b = conflict_fn_not_known ();
4597 12 : *last_conflicts = chrec_dont_know;
4598 12 : goto end_analyze_subs_aa;
4599 : }
4600 1689911 : gamma = int_cst_value (init_b) - int_cst_value (init_a);
4601 :
4602 : /* Don't do all the hard work of solving the Diophantine equation
4603 : when we already know the solution: for example,
4604 : | {3, +, 1}_1
4605 : | {3, +, 4}_2
4606 : | gamma = 3 - 3 = 0.
4607 : Then the first overlap occurs during the first iterations:
4608 : | {3, +, 1}_1 ({0, +, 4}_x) = {3, +, 4}_2 ({0, +, 1}_x)
4609 : */
4610 1689911 : if (gamma == 0)
4611 : {
4612 64 : if (nb_vars_a == 1 && nb_vars_b == 1)
4613 : {
4614 64 : HOST_WIDE_INT step_a, step_b;
4615 64 : HOST_WIDE_INT niter, niter_a, niter_b;
4616 64 : affine_fn ova, ovb;
4617 :
4618 64 : niter_a = max_stmt_executions_int (get_chrec_loop (chrec_a));
4619 64 : niter_b = max_stmt_executions_int (get_chrec_loop (chrec_b));
4620 64 : niter = MIN (niter_a, niter_b);
4621 64 : step_a = int_cst_value (CHREC_RIGHT (chrec_a));
4622 64 : step_b = int_cst_value (CHREC_RIGHT (chrec_b));
4623 :
4624 64 : compute_overlap_steps_for_affine_univar (niter, step_a, step_b,
4625 : &ova, &ovb,
4626 : last_conflicts, 1);
4627 64 : *overlaps_a = conflict_fn (1, ova);
4628 64 : *overlaps_b = conflict_fn (1, ovb);
4629 : }
4630 :
4631 0 : else if (nb_vars_a == 2 && nb_vars_b == 1)
4632 0 : compute_overlap_steps_for_affine_1_2
4633 0 : (chrec_a, chrec_b, overlaps_a, overlaps_b, last_conflicts);
4634 :
4635 0 : else if (nb_vars_a == 1 && nb_vars_b == 2)
4636 0 : compute_overlap_steps_for_affine_1_2
4637 0 : (chrec_b, chrec_a, overlaps_b, overlaps_a, last_conflicts);
4638 :
4639 : else
4640 : {
4641 0 : if (dump_file && (dump_flags & TDF_DETAILS))
4642 0 : fprintf (dump_file, "affine-affine test failed: too many variables.\n");
4643 0 : *overlaps_a = conflict_fn_not_known ();
4644 0 : *overlaps_b = conflict_fn_not_known ();
4645 0 : *last_conflicts = chrec_dont_know;
4646 : }
4647 64 : goto end_analyze_subs_aa;
4648 : }
4649 :
4650 : /* U.A = S */
4651 1689847 : if (!lambda_matrix_right_hermite (A, dim, 1, S, U))
4652 : {
4653 0 : *overlaps_a = conflict_fn_not_known ();
4654 0 : *overlaps_b = conflict_fn_not_known ();
4655 0 : *last_conflicts = chrec_dont_know;
4656 0 : goto end_analyze_subs_aa;
4657 : }
4658 :
4659 1689847 : if (S[0][0] < 0)
4660 : {
4661 1679261 : S[0][0] *= -1;
4662 1679261 : lambda_matrix_row_negate (U, dim, 0);
4663 : }
4664 1689847 : gcd_alpha_beta = S[0][0];
4665 :
4666 : /* Something went wrong: for example in {1, +, 0}_5 vs. {0, +, 0}_5,
4667 : but that is a quite strange case. Instead of ICEing, answer
4668 : don't know. */
4669 1689847 : if (gcd_alpha_beta == 0)
4670 : {
4671 0 : *overlaps_a = conflict_fn_not_known ();
4672 0 : *overlaps_b = conflict_fn_not_known ();
4673 0 : *last_conflicts = chrec_dont_know;
4674 0 : goto end_analyze_subs_aa;
4675 : }
4676 :
4677 : /* The classic "gcd-test". */
4678 1689847 : if (!int_divides_p (gcd_alpha_beta, gamma))
4679 : {
4680 : /* The "gcd-test" has determined that there is no integer
4681 : solution, i.e. there is no dependence. */
4682 1575655 : *overlaps_a = conflict_fn_no_dependence ();
4683 1575655 : *overlaps_b = conflict_fn_no_dependence ();
4684 1575655 : *last_conflicts = integer_zero_node;
4685 : }
4686 :
4687 : /* Both access functions are univariate. This includes SIV and MIV cases. */
4688 114192 : else if (nb_vars_a == 1 && nb_vars_b == 1)
4689 : {
4690 : /* Both functions should have the same evolution sign. */
4691 114192 : if (((A[0][0] > 0 && -A[1][0] > 0)
4692 5846 : || (A[0][0] < 0 && -A[1][0] < 0)))
4693 : {
4694 : /* The solutions are given by:
4695 : |
4696 : | [GAMMA/GCD_ALPHA_BETA t].[u11 u12] = [x0]
4697 : | [u21 u22] [y0]
4698 :
4699 : For a given integer t. Using the following variables,
4700 :
4701 : | i0 = u11 * gamma / gcd_alpha_beta
4702 : | j0 = u12 * gamma / gcd_alpha_beta
4703 : | i1 = u21
4704 : | j1 = u22
4705 :
4706 : the solutions are:
4707 :
4708 : | x0 = i0 + i1 * t,
4709 : | y0 = j0 + j1 * t. */
4710 113789 : HOST_WIDE_INT i0, j0, i1, j1;
4711 :
4712 113789 : i0 = U[0][0] * gamma / gcd_alpha_beta;
4713 113789 : j0 = U[0][1] * gamma / gcd_alpha_beta;
4714 113789 : i1 = U[1][0];
4715 113789 : j1 = U[1][1];
4716 :
4717 113789 : if ((i1 == 0 && i0 < 0)
4718 113789 : || (j1 == 0 && j0 < 0))
4719 : {
4720 : /* There is no solution.
4721 : FIXME: The case "i0 > nb_iterations, j0 > nb_iterations"
4722 : falls in here, but for the moment we don't look at the
4723 : upper bound of the iteration domain. */
4724 0 : *overlaps_a = conflict_fn_no_dependence ();
4725 0 : *overlaps_b = conflict_fn_no_dependence ();
4726 0 : *last_conflicts = integer_zero_node;
4727 55282 : goto end_analyze_subs_aa;
4728 : }
4729 :
4730 113789 : if (i1 > 0 && j1 > 0)
4731 : {
4732 113789 : HOST_WIDE_INT niter_a
4733 113789 : = max_stmt_executions_int (get_chrec_loop (chrec_a));
4734 113789 : HOST_WIDE_INT niter_b
4735 113789 : = max_stmt_executions_int (get_chrec_loop (chrec_b));
4736 113789 : HOST_WIDE_INT niter = MIN (niter_a, niter_b);
4737 :
4738 : /* (X0, Y0) is a solution of the Diophantine equation:
4739 : "chrec_a (X0) = chrec_b (Y0)". */
4740 113789 : HOST_WIDE_INT tau1 = MAX (CEIL (-i0, i1),
4741 : CEIL (-j0, j1));
4742 113789 : HOST_WIDE_INT x0 = i1 * tau1 + i0;
4743 113789 : HOST_WIDE_INT y0 = j1 * tau1 + j0;
4744 :
4745 : /* (X1, Y1) is the smallest positive solution of the eq
4746 : "chrec_a (X1) = chrec_b (Y1)", i.e. this is where the
4747 : first conflict occurs. */
4748 113789 : HOST_WIDE_INT min_multiple = MIN (x0 / i1, y0 / j1);
4749 113789 : HOST_WIDE_INT x1 = x0 - i1 * min_multiple;
4750 113789 : HOST_WIDE_INT y1 = y0 - j1 * min_multiple;
4751 :
4752 113789 : if (niter > 0)
4753 : {
4754 : /* If the overlap occurs outside of the bounds of the
4755 : loop, there is no dependence. */
4756 107212 : if (x1 >= niter_a || y1 >= niter_b)
4757 : {
4758 55282 : *overlaps_a = conflict_fn_no_dependence ();
4759 55282 : *overlaps_b = conflict_fn_no_dependence ();
4760 55282 : *last_conflicts = integer_zero_node;
4761 55282 : goto end_analyze_subs_aa;
4762 : }
4763 :
4764 : /* max stmt executions can get quite large, avoid
4765 : overflows by using wide ints here. */
4766 51930 : widest_int tau2
4767 103860 : = wi::smin (wi::sdiv_floor (wi::sub (niter_a, i0), i1),
4768 155790 : wi::sdiv_floor (wi::sub (niter_b, j0), j1));
4769 51930 : widest_int last_conflict = wi::sub (tau2, (x1 - i0)/i1);
4770 51930 : if (wi::min_precision (last_conflict, SIGNED)
4771 51930 : <= TYPE_PRECISION (integer_type_node))
4772 46996 : *last_conflicts
4773 46996 : = build_int_cst (integer_type_node,
4774 46996 : last_conflict.to_shwi ());
4775 : else
4776 4934 : *last_conflicts = chrec_dont_know;
4777 51930 : }
4778 : else
4779 6577 : *last_conflicts = chrec_dont_know;
4780 :
4781 58507 : *overlaps_a
4782 58507 : = conflict_fn (1,
4783 58507 : affine_fn_univar (build_int_cst (integer_type_node, x1),
4784 : 1,
4785 58507 : build_int_cst (integer_type_node, i1)));
4786 58507 : *overlaps_b
4787 58507 : = conflict_fn (1,
4788 58507 : affine_fn_univar (build_int_cst (integer_type_node, y1),
4789 : 1,
4790 58507 : build_int_cst (integer_type_node, j1)));
4791 58507 : }
4792 : else
4793 : {
4794 : /* FIXME: For the moment, the upper bound of the
4795 : iteration domain for i and j is not checked. */
4796 0 : if (dump_file && (dump_flags & TDF_DETAILS))
4797 0 : fprintf (dump_file, "affine-affine test failed: unimplemented.\n");
4798 0 : *overlaps_a = conflict_fn_not_known ();
4799 0 : *overlaps_b = conflict_fn_not_known ();
4800 0 : *last_conflicts = chrec_dont_know;
4801 : }
4802 58507 : }
4803 : else
4804 : {
4805 403 : if (dump_file && (dump_flags & TDF_DETAILS))
4806 19 : fprintf (dump_file, "affine-affine test failed: unimplemented.\n");
4807 403 : *overlaps_a = conflict_fn_not_known ();
4808 403 : *overlaps_b = conflict_fn_not_known ();
4809 403 : *last_conflicts = chrec_dont_know;
4810 : }
4811 : }
4812 : else
4813 : {
4814 0 : if (dump_file && (dump_flags & TDF_DETAILS))
4815 0 : fprintf (dump_file, "affine-affine test failed: unimplemented.\n");
4816 0 : *overlaps_a = conflict_fn_not_known ();
4817 0 : *overlaps_b = conflict_fn_not_known ();
4818 0 : *last_conflicts = chrec_dont_know;
4819 : }
4820 :
4821 1689923 : end_analyze_subs_aa:
4822 1689923 : obstack_free (&scratch_obstack, NULL);
4823 1689923 : if (dump_file && (dump_flags & TDF_DETAILS))
4824 : {
4825 21033 : fprintf (dump_file, " (overlaps_a = ");
4826 21033 : dump_conflict_function (dump_file, *overlaps_a);
4827 21033 : fprintf (dump_file, ")\n (overlaps_b = ");
4828 21033 : dump_conflict_function (dump_file, *overlaps_b);
4829 21033 : fprintf (dump_file, "))\n");
4830 : }
4831 : }
4832 :
4833 : /* Returns true when analyze_subscript_affine_affine can be used for
4834 : determining the dependence relation between chrec_a and chrec_b,
4835 : that contain symbols. This function modifies chrec_a and chrec_b
4836 : such that the analysis result is the same, and such that they don't
4837 : contain symbols, and then can safely be passed to the analyzer.
4838 :
4839 : Example: The analysis of the following tuples of evolutions produce
4840 : the same results: {x+1, +, 1}_1 vs. {x+3, +, 1}_1, and {-2, +, 1}_1
4841 : vs. {0, +, 1}_1
4842 :
4843 : {x+1, +, 1}_1 ({2, +, 1}_1) = {x+3, +, 1}_1 ({0, +, 1}_1)
4844 : {-2, +, 1}_1 ({2, +, 1}_1) = {0, +, 1}_1 ({0, +, 1}_1)
4845 : */
4846 :
4847 : static bool
4848 43567 : can_use_analyze_subscript_affine_affine (tree *chrec_a, tree *chrec_b)
4849 : {
4850 43567 : tree diff, type, left_a, left_b, right_b;
4851 :
4852 43567 : if (chrec_contains_symbols (CHREC_RIGHT (*chrec_a))
4853 43567 : || chrec_contains_symbols (CHREC_RIGHT (*chrec_b)))
4854 : /* FIXME: For the moment not handled. Might be refined later. */
4855 : return false;
4856 :
4857 28997 : type = chrec_type (*chrec_a);
4858 28997 : left_a = CHREC_LEFT (*chrec_a);
4859 28997 : left_b = chrec_convert (type, CHREC_LEFT (*chrec_b), NULL);
4860 28997 : diff = chrec_fold_minus (type, left_a, left_b);
4861 :
4862 57994 : if (!evolution_function_is_constant_p (diff))
4863 : return false;
4864 :
4865 23660 : if (dump_file && (dump_flags & TDF_DETAILS))
4866 105 : fprintf (dump_file, "can_use_subscript_aff_aff_for_symbolic \n");
4867 :
4868 23660 : *chrec_a = build_polynomial_chrec (CHREC_VARIABLE (*chrec_a),
4869 23660 : diff, CHREC_RIGHT (*chrec_a));
4870 23660 : right_b = chrec_convert (type, CHREC_RIGHT (*chrec_b), NULL);
4871 23660 : *chrec_b = build_polynomial_chrec (CHREC_VARIABLE (*chrec_b),
4872 : build_int_cst (type, 0),
4873 : right_b);
4874 23660 : return true;
4875 : }
4876 :
4877 : /* Analyze a SIV (Single Index Variable) subscript. *OVERLAPS_A and
4878 : *OVERLAPS_B are initialized to the functions that describe the
4879 : relation between the elements accessed twice by CHREC_A and
4880 : CHREC_B. For k >= 0, the following property is verified:
4881 :
4882 : CHREC_A (*OVERLAPS_A (k)) = CHREC_B (*OVERLAPS_B (k)). */
4883 :
4884 : static void
4885 1712801 : analyze_siv_subscript (tree chrec_a,
4886 : tree chrec_b,
4887 : conflict_function **overlaps_a,
4888 : conflict_function **overlaps_b,
4889 : tree *last_conflicts,
4890 : int loop_nest_num)
4891 : {
4892 1712801 : dependence_stats.num_siv++;
4893 :
4894 1712801 : if (dump_file && (dump_flags & TDF_DETAILS))
4895 24164 : fprintf (dump_file, "(analyze_siv_subscript \n");
4896 :
4897 1712801 : if (evolution_function_is_constant_p (chrec_a)
4898 1712801 : && evolution_function_is_affine_in_loop (chrec_b, loop_nest_num))
4899 1784 : analyze_siv_subscript_cst_affine (chrec_a, chrec_b,
4900 : overlaps_a, overlaps_b, last_conflicts);
4901 :
4902 1711017 : else if (evolution_function_is_affine_in_loop (chrec_a, loop_nest_num)
4903 3422034 : && evolution_function_is_constant_p (chrec_b))
4904 1267 : analyze_siv_subscript_cst_affine (chrec_b, chrec_a,
4905 : overlaps_b, overlaps_a, last_conflicts);
4906 :
4907 1709750 : else if (evolution_function_is_affine_in_loop (chrec_a, loop_nest_num)
4908 1709750 : && evolution_function_is_affine_in_loop (chrec_b, loop_nest_num))
4909 : {
4910 1709750 : if (!chrec_contains_symbols (chrec_a)
4911 1709750 : && !chrec_contains_symbols (chrec_b))
4912 : {
4913 1666183 : analyze_subscript_affine_affine (chrec_a, chrec_b,
4914 : overlaps_a, overlaps_b,
4915 : last_conflicts);
4916 :
4917 1666183 : if (CF_NOT_KNOWN_P (*overlaps_a)
4918 1665788 : || CF_NOT_KNOWN_P (*overlaps_b))
4919 395 : dependence_stats.num_siv_unimplemented++;
4920 1665788 : else if (CF_NO_DEPENDENCE_P (*overlaps_a)
4921 57675 : || CF_NO_DEPENDENCE_P (*overlaps_b))
4922 1608113 : dependence_stats.num_siv_independent++;
4923 : else
4924 57675 : dependence_stats.num_siv_dependent++;
4925 : }
4926 43567 : else if (can_use_analyze_subscript_affine_affine (&chrec_a,
4927 : &chrec_b))
4928 : {
4929 23660 : analyze_subscript_affine_affine (chrec_a, chrec_b,
4930 : overlaps_a, overlaps_b,
4931 : last_conflicts);
4932 :
4933 23660 : if (CF_NOT_KNOWN_P (*overlaps_a)
4934 23644 : || CF_NOT_KNOWN_P (*overlaps_b))
4935 16 : dependence_stats.num_siv_unimplemented++;
4936 23644 : else if (CF_NO_DEPENDENCE_P (*overlaps_a)
4937 834 : || CF_NO_DEPENDENCE_P (*overlaps_b))
4938 22810 : dependence_stats.num_siv_independent++;
4939 : else
4940 834 : dependence_stats.num_siv_dependent++;
4941 : }
4942 : else
4943 19907 : goto siv_subscript_dontknow;
4944 : }
4945 :
4946 : else
4947 : {
4948 19907 : siv_subscript_dontknow:;
4949 19907 : if (dump_file && (dump_flags & TDF_DETAILS))
4950 2946 : fprintf (dump_file, " siv test failed: unimplemented");
4951 19907 : *overlaps_a = conflict_fn_not_known ();
4952 19907 : *overlaps_b = conflict_fn_not_known ();
4953 19907 : *last_conflicts = chrec_dont_know;
4954 19907 : dependence_stats.num_siv_unimplemented++;
4955 : }
4956 :
4957 1712801 : if (dump_file && (dump_flags & TDF_DETAILS))
4958 24164 : fprintf (dump_file, ")\n");
4959 1712801 : }
4960 :
4961 : /* Returns false if we can prove that the greatest common divisor of the steps
4962 : of CHREC does not divide CST, false otherwise. */
4963 :
4964 : static bool
4965 20662 : gcd_of_steps_may_divide_p (const_tree chrec, const_tree cst)
4966 : {
4967 20662 : HOST_WIDE_INT cd = 0, val;
4968 20662 : tree step;
4969 :
4970 20662 : if (!tree_fits_shwi_p (cst))
4971 : return true;
4972 20662 : val = tree_to_shwi (cst);
4973 :
4974 61838 : while (TREE_CODE (chrec) == POLYNOMIAL_CHREC)
4975 : {
4976 41322 : step = CHREC_RIGHT (chrec);
4977 41322 : if (!tree_fits_shwi_p (step))
4978 : return true;
4979 41176 : cd = gcd (cd, tree_to_shwi (step));
4980 41176 : chrec = CHREC_LEFT (chrec);
4981 : }
4982 :
4983 20516 : return val % cd == 0;
4984 : }
4985 :
4986 : /* Analyze a MIV (Multiple Index Variable) subscript with respect to
4987 : LOOP_NEST. *OVERLAPS_A and *OVERLAPS_B are initialized to the
4988 : functions that describe the relation between the elements accessed
4989 : twice by CHREC_A and CHREC_B. For k >= 0, the following property
4990 : is verified:
4991 :
4992 : CHREC_A (*OVERLAPS_A (k)) = CHREC_B (*OVERLAPS_B (k)). */
4993 :
4994 : static void
4995 26449 : analyze_miv_subscript (tree chrec_a,
4996 : tree chrec_b,
4997 : conflict_function **overlaps_a,
4998 : conflict_function **overlaps_b,
4999 : tree *last_conflicts,
5000 : class loop *loop_nest)
5001 : {
5002 26449 : tree type, difference;
5003 :
5004 26449 : dependence_stats.num_miv++;
5005 26449 : if (dump_file && (dump_flags & TDF_DETAILS))
5006 27 : fprintf (dump_file, "(analyze_miv_subscript \n");
5007 :
5008 26449 : type = signed_type_for_types (TREE_TYPE (chrec_a), TREE_TYPE (chrec_b));
5009 26449 : chrec_a = chrec_convert (type, chrec_a, NULL);
5010 26449 : chrec_b = chrec_convert (type, chrec_b, NULL);
5011 26449 : difference = chrec_fold_minus (type, chrec_a, chrec_b);
5012 :
5013 26449 : if (eq_evolutions_p (chrec_a, chrec_b))
5014 : {
5015 : /* Access functions are the same: all the elements are accessed
5016 : in the same order. */
5017 0 : *overlaps_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
5018 0 : *overlaps_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
5019 0 : *last_conflicts = max_stmt_executions_tree (get_chrec_loop (chrec_a));
5020 0 : dependence_stats.num_miv_dependent++;
5021 : }
5022 :
5023 26449 : else if (evolution_function_is_constant_p (difference)
5024 20692 : && evolution_function_is_affine_multivariate_p (chrec_a,
5025 : loop_nest->num)
5026 47111 : && !gcd_of_steps_may_divide_p (chrec_a, difference))
5027 : {
5028 : /* testsuite/.../ssa-chrec-33.c
5029 : {{21, +, 2}_1, +, -2}_2 vs. {{20, +, 2}_1, +, -2}_2
5030 :
5031 : The difference is 1, and all the evolution steps are multiples
5032 : of 2, consequently there are no overlapping elements. */
5033 19670 : *overlaps_a = conflict_fn_no_dependence ();
5034 19670 : *overlaps_b = conflict_fn_no_dependence ();
5035 19670 : *last_conflicts = integer_zero_node;
5036 19670 : dependence_stats.num_miv_independent++;
5037 : }
5038 :
5039 6779 : else if (evolution_function_is_affine_in_loop (chrec_a, loop_nest->num)
5040 122 : && !chrec_contains_symbols (chrec_a, loop_nest)
5041 110 : && evolution_function_is_affine_in_loop (chrec_b, loop_nest->num)
5042 6859 : && !chrec_contains_symbols (chrec_b, loop_nest))
5043 : {
5044 : /* testsuite/.../ssa-chrec-35.c
5045 : {0, +, 1}_2 vs. {0, +, 1}_3
5046 : the overlapping elements are respectively located at iterations:
5047 : {0, +, 1}_x and {0, +, 1}_x,
5048 : in other words, we have the equality:
5049 : {0, +, 1}_2 ({0, +, 1}_x) = {0, +, 1}_3 ({0, +, 1}_x)
5050 :
5051 : Other examples:
5052 : {{0, +, 1}_1, +, 2}_2 ({0, +, 1}_x, {0, +, 1}_y) =
5053 : {0, +, 1}_1 ({{0, +, 1}_x, +, 2}_y)
5054 :
5055 : {{0, +, 2}_1, +, 3}_2 ({0, +, 1}_y, {0, +, 1}_x) =
5056 : {{0, +, 3}_1, +, 2}_2 ({0, +, 1}_x, {0, +, 1}_y)
5057 : */
5058 80 : analyze_subscript_affine_affine (chrec_a, chrec_b,
5059 : overlaps_a, overlaps_b, last_conflicts);
5060 :
5061 80 : if (CF_NOT_KNOWN_P (*overlaps_a)
5062 76 : || CF_NOT_KNOWN_P (*overlaps_b))
5063 4 : dependence_stats.num_miv_unimplemented++;
5064 76 : else if (CF_NO_DEPENDENCE_P (*overlaps_a)
5065 62 : || CF_NO_DEPENDENCE_P (*overlaps_b))
5066 14 : dependence_stats.num_miv_independent++;
5067 : else
5068 62 : dependence_stats.num_miv_dependent++;
5069 : }
5070 :
5071 : else
5072 : {
5073 : /* When the analysis is too difficult, answer "don't know". */
5074 6699 : if (dump_file && (dump_flags & TDF_DETAILS))
5075 23 : fprintf (dump_file, "analyze_miv_subscript test failed: unimplemented.\n");
5076 :
5077 6699 : *overlaps_a = conflict_fn_not_known ();
5078 6699 : *overlaps_b = conflict_fn_not_known ();
5079 6699 : *last_conflicts = chrec_dont_know;
5080 6699 : dependence_stats.num_miv_unimplemented++;
5081 : }
5082 :
5083 26449 : if (dump_file && (dump_flags & TDF_DETAILS))
5084 27 : fprintf (dump_file, ")\n");
5085 26449 : }
5086 :
5087 : /* Determines the iterations for which CHREC_A is equal to CHREC_B in
5088 : with respect to LOOP_NEST. OVERLAP_ITERATIONS_A and
5089 : OVERLAP_ITERATIONS_B are initialized with two functions that
5090 : describe the iterations that contain conflicting elements.
5091 :
5092 : Remark: For an integer k >= 0, the following equality is true:
5093 :
5094 : CHREC_A (OVERLAP_ITERATIONS_A (k)) == CHREC_B (OVERLAP_ITERATIONS_B (k)).
5095 : */
5096 :
5097 : static void
5098 3451102 : analyze_overlapping_iterations (tree chrec_a,
5099 : tree chrec_b,
5100 : conflict_function **overlap_iterations_a,
5101 : conflict_function **overlap_iterations_b,
5102 : tree *last_conflicts, class loop *loop_nest)
5103 : {
5104 3451102 : unsigned int lnn = loop_nest->num;
5105 :
5106 3451102 : dependence_stats.num_subscript_tests++;
5107 :
5108 3451102 : if (dump_file && (dump_flags & TDF_DETAILS))
5109 : {
5110 60359 : fprintf (dump_file, "(analyze_overlapping_iterations \n");
5111 60359 : fprintf (dump_file, " (chrec_a = ");
5112 60359 : print_generic_expr (dump_file, chrec_a);
5113 60359 : fprintf (dump_file, ")\n (chrec_b = ");
5114 60359 : print_generic_expr (dump_file, chrec_b);
5115 60359 : fprintf (dump_file, ")\n");
5116 : }
5117 :
5118 3451102 : if (chrec_a == NULL_TREE
5119 3451102 : || chrec_b == NULL_TREE
5120 3451102 : || chrec_contains_undetermined (chrec_a)
5121 6902204 : || chrec_contains_undetermined (chrec_b))
5122 : {
5123 0 : dependence_stats.num_subscript_undetermined++;
5124 :
5125 0 : *overlap_iterations_a = conflict_fn_not_known ();
5126 0 : *overlap_iterations_b = conflict_fn_not_known ();
5127 : }
5128 :
5129 : /* If they are the same chrec, and are affine, they overlap
5130 : on every iteration. */
5131 3451102 : else if (eq_evolutions_p (chrec_a, chrec_b)
5132 3451102 : && (evolution_function_is_affine_multivariate_p (chrec_a, lnn)
5133 492800 : || operand_equal_p (chrec_a, chrec_b, 0)))
5134 : {
5135 1200832 : dependence_stats.num_same_subscript_function++;
5136 1200832 : *overlap_iterations_a = conflict_fn (1, affine_fn_cst (integer_zero_node));
5137 1200832 : *overlap_iterations_b = conflict_fn (1, affine_fn_cst (integer_zero_node));
5138 1200832 : *last_conflicts = chrec_dont_know;
5139 : }
5140 :
5141 : /* If they aren't the same, and aren't affine, we can't do anything
5142 : yet. */
5143 2250270 : else if ((chrec_contains_symbols (chrec_a)
5144 2198900 : || chrec_contains_symbols (chrec_b))
5145 2251107 : && (!evolution_function_is_affine_multivariate_p (chrec_a, lnn)
5146 49928 : || !evolution_function_is_affine_multivariate_p (chrec_b, lnn)))
5147 : {
5148 2577 : dependence_stats.num_subscript_undetermined++;
5149 2577 : *overlap_iterations_a = conflict_fn_not_known ();
5150 2577 : *overlap_iterations_b = conflict_fn_not_known ();
5151 : }
5152 :
5153 2247693 : else if (ziv_subscript_p (chrec_a, chrec_b))
5154 508443 : analyze_ziv_subscript (chrec_a, chrec_b,
5155 : overlap_iterations_a, overlap_iterations_b,
5156 : last_conflicts);
5157 :
5158 1739250 : else if (siv_subscript_p (chrec_a, chrec_b))
5159 1712801 : analyze_siv_subscript (chrec_a, chrec_b,
5160 : overlap_iterations_a, overlap_iterations_b,
5161 : last_conflicts, lnn);
5162 :
5163 : else
5164 26449 : analyze_miv_subscript (chrec_a, chrec_b,
5165 : overlap_iterations_a, overlap_iterations_b,
5166 : last_conflicts, loop_nest);
5167 :
5168 3451102 : if (dump_file && (dump_flags & TDF_DETAILS))
5169 : {
5170 60359 : fprintf (dump_file, " (overlap_iterations_a = ");
5171 60359 : dump_conflict_function (dump_file, *overlap_iterations_a);
5172 60359 : fprintf (dump_file, ")\n (overlap_iterations_b = ");
5173 60359 : dump_conflict_function (dump_file, *overlap_iterations_b);
5174 60359 : fprintf (dump_file, "))\n");
5175 : }
5176 3451102 : }
5177 :
5178 : /* Helper function for uniquely inserting distance vectors. */
5179 :
5180 : static void
5181 1102221 : save_dist_v (struct data_dependence_relation *ddr, lambda_vector dist_v)
5182 : {
5183 1660139 : for (lambda_vector v : DDR_DIST_VECTS (ddr))
5184 567321 : if (lambda_vector_equal (v, dist_v, DDR_NB_LOOPS (ddr)))
5185 : return;
5186 :
5187 1100484 : DDR_DIST_VECTS (ddr).safe_push (dist_v);
5188 : }
5189 :
5190 : /* Helper function for uniquely inserting direction vectors. */
5191 :
5192 : static void
5193 1097565 : save_dir_v (struct data_dependence_relation *ddr, lambda_vector dir_v)
5194 : {
5195 1648962 : for (lambda_vector v : DDR_DIR_VECTS (ddr))
5196 553797 : if (lambda_vector_equal (v, dir_v, DDR_NB_LOOPS (ddr)))
5197 : return;
5198 :
5199 1097565 : DDR_DIR_VECTS (ddr).safe_push (dir_v);
5200 : }
5201 :
5202 : /* Add a distance of 1 on all the loops outer than INDEX. If we
5203 : haven't yet determined a distance for this outer loop, push a new
5204 : distance vector composed of the previous distance, and a distance
5205 : of 1 for this outer loop. Example:
5206 :
5207 : | loop_1
5208 : | loop_2
5209 : | A[10]
5210 : | endloop_2
5211 : | endloop_1
5212 :
5213 : Saved vectors are of the form (dist_in_1, dist_in_2). First, we
5214 : save (0, 1), then we have to save (1, 0). */
5215 :
5216 : static void
5217 16946 : add_outer_distances (struct data_dependence_relation *ddr,
5218 : lambda_vector dist_v, int index)
5219 : {
5220 : /* For each outer loop where init_v is not set, the accesses are
5221 : in dependence of distance 1 in the loop. */
5222 20412 : while (--index >= 0)
5223 : {
5224 6932 : lambda_vector save_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5225 3466 : lambda_vector_copy (dist_v, save_v, DDR_NB_LOOPS (ddr));
5226 3466 : save_v[index] = 1;
5227 3466 : save_dist_v (ddr, save_v);
5228 : }
5229 16946 : }
5230 :
5231 : /* Return false when fail to represent the data dependence as a
5232 : distance vector. A_INDEX is the index of the first reference
5233 : (0 for DDR_A, 1 for DDR_B) and B_INDEX is the index of the
5234 : second reference. INIT_B is set to true when a component has been
5235 : added to the distance vector DIST_V. INDEX_CARRY is then set to
5236 : the index in DIST_V that carries the dependence. */
5237 :
5238 : static bool
5239 60099 : build_classic_dist_vector_1 (struct data_dependence_relation *ddr,
5240 : unsigned int a_index, unsigned int b_index,
5241 : lambda_vector dist_v, bool *init_b,
5242 : int *index_carry)
5243 : {
5244 60099 : unsigned i;
5245 120198 : lambda_vector init_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5246 60099 : class loop *loop = DDR_LOOP_NEST (ddr)[0];
5247 :
5248 135238 : for (i = 0; i < DDR_NUM_SUBSCRIPTS (ddr); i++)
5249 : {
5250 77325 : tree access_fn_a, access_fn_b;
5251 77325 : struct subscript *subscript = DDR_SUBSCRIPT (ddr, i);
5252 :
5253 77325 : if (chrec_contains_undetermined (SUB_DISTANCE (subscript)))
5254 : {
5255 309 : non_affine_dependence_relation (ddr);
5256 309 : return false;
5257 : }
5258 :
5259 77016 : access_fn_a = SUB_ACCESS_FN (subscript, a_index);
5260 77016 : access_fn_b = SUB_ACCESS_FN (subscript, b_index);
5261 :
5262 77016 : if (TREE_CODE (access_fn_a) == POLYNOMIAL_CHREC
5263 58733 : && TREE_CODE (access_fn_b) == POLYNOMIAL_CHREC)
5264 : {
5265 58107 : HOST_WIDE_INT dist;
5266 58107 : int index;
5267 58107 : int var_a = CHREC_VARIABLE (access_fn_a);
5268 58107 : int var_b = CHREC_VARIABLE (access_fn_b);
5269 :
5270 58107 : if (var_a != var_b
5271 58107 : || chrec_contains_undetermined (SUB_DISTANCE (subscript)))
5272 : {
5273 34 : non_affine_dependence_relation (ddr);
5274 34 : return false;
5275 : }
5276 :
5277 : /* When data references are collected in a loop while data
5278 : dependences are analyzed in loop nest nested in the loop, we
5279 : would have more number of access functions than number of
5280 : loops. Skip access functions of loops not in the loop nest.
5281 :
5282 : See PR89725 for more information. */
5283 58073 : if (flow_loop_nested_p (get_loop (cfun, var_a), loop))
5284 2 : continue;
5285 :
5286 58071 : dist = int_cst_value (SUB_DISTANCE (subscript));
5287 58071 : index = index_in_loop_nest (var_a, DDR_LOOP_NEST (ddr));
5288 58071 : *index_carry = MIN (index, *index_carry);
5289 :
5290 : /* This is the subscript coupling test. If we have already
5291 : recorded a distance for this loop (a distance coming from
5292 : another subscript), it should be the same. For example,
5293 : in the following code, there is no dependence:
5294 :
5295 : | loop i = 0, N, 1
5296 : | T[i+1][i] = ...
5297 : | ... = T[i][i]
5298 : | endloop
5299 : */
5300 58071 : if (init_v[index] != 0 && dist_v[index] != dist)
5301 : {
5302 2 : finalize_ddr_dependent (ddr, chrec_known);
5303 2 : return false;
5304 : }
5305 :
5306 58069 : dist_v[index] = dist;
5307 58069 : init_v[index] = 1;
5308 58069 : *init_b = true;
5309 58069 : }
5310 18909 : else if (!operand_equal_p (access_fn_a, access_fn_b, 0))
5311 : {
5312 : /* This can be for example an affine vs. constant dependence
5313 : (T[i] vs. T[3]) that is not an affine dependence and is
5314 : not representable as a distance vector. */
5315 1841 : non_affine_dependence_relation (ddr);
5316 1841 : return false;
5317 : }
5318 : }
5319 :
5320 : return true;
5321 : }
5322 :
5323 : /* Return true when the DDR contains only invariant access functions wrto. loop
5324 : number LNUM. */
5325 :
5326 : static bool
5327 866694 : invariant_access_functions (const struct data_dependence_relation *ddr,
5328 : int lnum)
5329 : {
5330 2925111 : for (subscript *sub : DDR_SUBSCRIPTS (ddr))
5331 1014188 : if (!evolution_function_is_invariant_p (SUB_ACCESS_FN (sub, 0), lnum)
5332 1014188 : || !evolution_function_is_invariant_p (SUB_ACCESS_FN (sub, 1), lnum))
5333 : return false;
5334 :
5335 : return true;
5336 : }
5337 :
5338 : /* Helper function for the case where DDR_A and DDR_B are the same
5339 : multivariate access function with a constant step. For an example
5340 : see pr34635-1.c. */
5341 :
5342 : static void
5343 4594 : add_multivariate_self_dist (struct data_dependence_relation *ddr, tree c_2)
5344 : {
5345 4594 : int x_1, x_2;
5346 4594 : tree c_1 = CHREC_LEFT (c_2);
5347 4594 : tree c_0 = CHREC_LEFT (c_1);
5348 4594 : lambda_vector dist_v;
5349 4594 : HOST_WIDE_INT v1, v2, cd;
5350 :
5351 : /* Polynomials with more than 2 variables are not handled yet. When
5352 : the evolution steps are parameters, it is not possible to
5353 : represent the dependence using classical distance vectors. */
5354 4594 : if (TREE_CODE (c_0) != INTEGER_CST
5355 3085 : || TREE_CODE (CHREC_RIGHT (c_1)) != INTEGER_CST
5356 7002 : || TREE_CODE (CHREC_RIGHT (c_2)) != INTEGER_CST)
5357 : {
5358 2194 : DDR_AFFINE_P (ddr) = false;
5359 2194 : DDR_ARE_DEPENDENT (ddr) = chrec_dont_know;
5360 2194 : DDR_DIST_VECTS (ddr).release ();
5361 2194 : return;
5362 : }
5363 :
5364 2400 : x_2 = index_in_loop_nest (CHREC_VARIABLE (c_2), DDR_LOOP_NEST (ddr));
5365 2400 : x_1 = index_in_loop_nest (CHREC_VARIABLE (c_1), DDR_LOOP_NEST (ddr));
5366 :
5367 : /* For "{{0, +, 2}_1, +, 3}_2" the distance vector is (3, -2). */
5368 4800 : dist_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5369 2400 : v1 = int_cst_value (CHREC_RIGHT (c_1));
5370 2400 : v2 = int_cst_value (CHREC_RIGHT (c_2));
5371 2400 : cd = gcd (v1, v2);
5372 2400 : v1 /= cd;
5373 2400 : v2 /= cd;
5374 :
5375 2400 : if (v2 < 0)
5376 : {
5377 19 : v2 = -v2;
5378 19 : v1 = -v1;
5379 : }
5380 :
5381 2400 : dist_v[x_1] = v2;
5382 2400 : dist_v[x_2] = -v1;
5383 2400 : save_dist_v (ddr, dist_v);
5384 :
5385 2400 : add_outer_distances (ddr, dist_v, x_1);
5386 : }
5387 :
5388 : /* Helper function for the case where DDR_A and DDR_B are the same
5389 : access functions. */
5390 :
5391 : static void
5392 19279 : add_other_self_distances (struct data_dependence_relation *ddr)
5393 : {
5394 19279 : lambda_vector dist_v;
5395 19279 : unsigned i;
5396 19279 : int index_carry = DDR_NB_LOOPS (ddr);
5397 19279 : subscript *sub;
5398 19279 : class loop *loop = DDR_LOOP_NEST (ddr)[0];
5399 :
5400 41023 : FOR_EACH_VEC_ELT (DDR_SUBSCRIPTS (ddr), i, sub)
5401 : {
5402 26855 : tree access_fun = SUB_ACCESS_FN (sub, 0);
5403 :
5404 26855 : if (TREE_CODE (access_fun) == POLYNOMIAL_CHREC)
5405 : {
5406 19376 : if (!evolution_function_is_univariate_p (access_fun, loop->num))
5407 : {
5408 5111 : if (DDR_NUM_SUBSCRIPTS (ddr) != 1)
5409 : {
5410 517 : DDR_ARE_DEPENDENT (ddr) = chrec_dont_know;
5411 517 : DDR_DIST_VECTS (ddr).release ();
5412 517 : return;
5413 : }
5414 :
5415 4594 : access_fun = SUB_ACCESS_FN (DDR_SUBSCRIPT (ddr, 0), 0);
5416 :
5417 4594 : if (TREE_CODE (CHREC_LEFT (access_fun)) == POLYNOMIAL_CHREC)
5418 4594 : add_multivariate_self_dist (ddr, access_fun);
5419 : else
5420 : {
5421 : /* The evolution step is not constant: it varies in
5422 : the outer loop, so this cannot be represented by a
5423 : distance vector. For example in pr34635.c the
5424 : evolution is {0, +, {0, +, 4}_1}_2. */
5425 0 : DDR_AFFINE_P (ddr) = false;
5426 0 : DDR_ARE_DEPENDENT (ddr) = chrec_dont_know;
5427 0 : DDR_DIST_VECTS (ddr).release ();
5428 : }
5429 :
5430 : return;
5431 : }
5432 :
5433 : /* When data references are collected in a loop while data
5434 : dependences are analyzed in loop nest nested in the loop, we
5435 : would have more number of access functions than number of
5436 : loops. Skip access functions of loops not in the loop nest.
5437 :
5438 : See PR89725 for more information. */
5439 14265 : if (flow_loop_nested_p (get_loop (cfun, CHREC_VARIABLE (access_fun)),
5440 : loop))
5441 0 : continue;
5442 :
5443 21904 : index_carry = MIN (index_carry,
5444 : index_in_loop_nest (CHREC_VARIABLE (access_fun),
5445 : DDR_LOOP_NEST (ddr)));
5446 : }
5447 : }
5448 :
5449 28336 : dist_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5450 14168 : add_outer_distances (ddr, dist_v, index_carry);
5451 : }
5452 :
5453 : static void
5454 177535 : insert_innermost_unit_dist_vector (struct data_dependence_relation *ddr)
5455 : {
5456 355070 : lambda_vector dist_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5457 :
5458 177535 : dist_v[0] = 1;
5459 177535 : save_dist_v (ddr, dist_v);
5460 177535 : }
5461 :
5462 : /* Adds a unit distance vector to DDR when there is a 0 overlap. This
5463 : is the case for example when access functions are the same and
5464 : equal to a constant, as in:
5465 :
5466 : | loop_1
5467 : | A[3] = ...
5468 : | ... = A[3]
5469 : | endloop_1
5470 :
5471 : in which case the distance vectors are (0) and (1). */
5472 :
5473 : static void
5474 177535 : add_distance_for_zero_overlaps (struct data_dependence_relation *ddr)
5475 : {
5476 177535 : unsigned i, j;
5477 :
5478 177535 : for (i = 0; i < DDR_NUM_SUBSCRIPTS (ddr); i++)
5479 : {
5480 177535 : subscript_p sub = DDR_SUBSCRIPT (ddr, i);
5481 177535 : conflict_function *ca = SUB_CONFLICTS_IN_A (sub);
5482 177535 : conflict_function *cb = SUB_CONFLICTS_IN_B (sub);
5483 :
5484 177535 : for (j = 0; j < ca->n; j++)
5485 177535 : if (affine_function_zero_p (ca->fns[j]))
5486 : {
5487 177535 : insert_innermost_unit_dist_vector (ddr);
5488 177535 : return;
5489 : }
5490 :
5491 0 : for (j = 0; j < cb->n; j++)
5492 0 : if (affine_function_zero_p (cb->fns[j]))
5493 : {
5494 0 : insert_innermost_unit_dist_vector (ddr);
5495 0 : return;
5496 : }
5497 : }
5498 : }
5499 :
5500 : /* Return true when the DDR contains two data references that have the
5501 : same access functions. */
5502 :
5503 : static inline bool
5504 919137 : same_access_functions (const struct data_dependence_relation *ddr)
5505 : {
5506 3847368 : for (subscript *sub : DDR_SUBSCRIPTS (ddr))
5507 1142400 : if (!eq_evolutions_p (SUB_ACCESS_FN (sub, 0),
5508 1142400 : SUB_ACCESS_FN (sub, 1)))
5509 : return false;
5510 :
5511 : return true;
5512 : }
5513 :
5514 : /* Compute the classic per loop distance vector. DDR is the data
5515 : dependence relation to build a vector from. Return false when fail
5516 : to represent the data dependence as a distance vector. */
5517 :
5518 : static bool
5519 3107347 : build_classic_dist_vector (struct data_dependence_relation *ddr,
5520 : class loop *loop_nest)
5521 : {
5522 3107347 : bool init_b = false;
5523 3107347 : int index_carry = DDR_NB_LOOPS (ddr);
5524 3107347 : lambda_vector dist_v;
5525 :
5526 3107347 : if (DDR_ARE_DEPENDENT (ddr) != NULL_TREE)
5527 : return false;
5528 :
5529 919137 : if (same_access_functions (ddr))
5530 : {
5531 : /* Save the 0 vector. */
5532 1733388 : dist_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5533 866694 : save_dist_v (ddr, dist_v);
5534 :
5535 866694 : if (invariant_access_functions (ddr, loop_nest->num))
5536 177535 : add_distance_for_zero_overlaps (ddr);
5537 :
5538 866694 : if (DDR_NB_LOOPS (ddr) > 1)
5539 19279 : add_other_self_distances (ddr);
5540 :
5541 : return true;
5542 : }
5543 :
5544 104886 : dist_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5545 52443 : if (!build_classic_dist_vector_1 (ddr, 0, 1, dist_v, &init_b, &index_carry))
5546 : return false;
5547 :
5548 : /* Save the distance vector if we initialized one. */
5549 50257 : if (init_b)
5550 : {
5551 : /* Verify a basic constraint: classic distance vectors should
5552 : always be lexicographically positive.
5553 :
5554 : Data references are collected in the order of execution of
5555 : the program, thus for the following loop
5556 :
5557 : | for (i = 1; i < 100; i++)
5558 : | for (j = 1; j < 100; j++)
5559 : | {
5560 : | t = T[j+1][i-1]; // A
5561 : | T[j][i] = t + 2; // B
5562 : | }
5563 :
5564 : references are collected following the direction of the wind:
5565 : A then B. The data dependence tests are performed also
5566 : following this order, such that we're looking at the distance
5567 : separating the elements accessed by A from the elements later
5568 : accessed by B. But in this example, the distance returned by
5569 : test_dep (A, B) is lexicographically negative (-1, 1), that
5570 : means that the access A occurs later than B with respect to
5571 : the outer loop, ie. we're actually looking upwind. In this
5572 : case we solve test_dep (B, A) looking downwind to the
5573 : lexicographically positive solution, that returns the
5574 : distance vector (1, -1). */
5575 100514 : if (!lambda_vector_lexico_pos (dist_v, DDR_NB_LOOPS (ddr)))
5576 : {
5577 7545 : lambda_vector save_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5578 7545 : if (!subscript_dependence_tester_1 (ddr, 1, 0, loop_nest))
5579 : return false;
5580 7541 : compute_subscript_distance (ddr);
5581 7541 : if (!build_classic_dist_vector_1 (ddr, 1, 0, save_v, &init_b,
5582 : &index_carry))
5583 : return false;
5584 7541 : save_dist_v (ddr, save_v);
5585 7541 : DDR_REVERSED_P (ddr) = true;
5586 :
5587 : /* In this case there is a dependence forward for all the
5588 : outer loops:
5589 :
5590 : | for (k = 1; k < 100; k++)
5591 : | for (i = 1; i < 100; i++)
5592 : | for (j = 1; j < 100; j++)
5593 : | {
5594 : | t = T[j+1][i-1]; // A
5595 : | T[j][i] = t + 2; // B
5596 : | }
5597 :
5598 : the vectors are:
5599 : (0, 1, -1)
5600 : (1, 1, -1)
5601 : (1, -1, 1)
5602 : */
5603 7541 : if (DDR_NB_LOOPS (ddr) > 1)
5604 : {
5605 74 : add_outer_distances (ddr, save_v, index_carry);
5606 74 : add_outer_distances (ddr, dist_v, index_carry);
5607 : }
5608 : }
5609 : else
5610 : {
5611 42712 : lambda_vector save_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5612 42712 : lambda_vector_copy (dist_v, save_v, DDR_NB_LOOPS (ddr));
5613 :
5614 42712 : if (DDR_NB_LOOPS (ddr) > 1)
5615 : {
5616 115 : lambda_vector opposite_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5617 :
5618 115 : if (!subscript_dependence_tester_1 (ddr, 1, 0, loop_nest))
5619 : return false;
5620 115 : compute_subscript_distance (ddr);
5621 115 : if (!build_classic_dist_vector_1 (ddr, 1, 0, opposite_v, &init_b,
5622 : &index_carry))
5623 : return false;
5624 :
5625 115 : save_dist_v (ddr, save_v);
5626 115 : add_outer_distances (ddr, dist_v, index_carry);
5627 115 : add_outer_distances (ddr, opposite_v, index_carry);
5628 : }
5629 : else
5630 42597 : save_dist_v (ddr, save_v);
5631 : }
5632 : }
5633 : else
5634 : {
5635 : /* There is a distance of 1 on all the outer loops: Example:
5636 : there is a dependence of distance 1 on loop_1 for the array A.
5637 :
5638 : | loop_1
5639 : | A[5] = ...
5640 : | endloop
5641 : */
5642 0 : add_outer_distances (ddr, dist_v,
5643 : lambda_vector_first_nz (dist_v,
5644 0 : DDR_NB_LOOPS (ddr), 0));
5645 : }
5646 :
5647 : return true;
5648 : }
5649 :
5650 : /* Return the direction for a given distance.
5651 : FIXME: Computing dir this way is suboptimal, since dir can catch
5652 : cases that dist is unable to represent. */
5653 :
5654 : static inline enum data_dependence_direction
5655 1120619 : dir_from_dist (int dist)
5656 : {
5657 1120619 : if (dist > 0)
5658 : return dir_positive;
5659 886697 : else if (dist < 0)
5660 : return dir_negative;
5661 : else
5662 884066 : return dir_equal;
5663 : }
5664 :
5665 : /* Compute the classic per loop direction vector. DDR is the data
5666 : dependence relation to build a vector from. */
5667 :
5668 : static void
5669 916873 : build_classic_dir_vector (struct data_dependence_relation *ddr)
5670 : {
5671 916873 : unsigned i, j;
5672 916873 : lambda_vector dist_v;
5673 :
5674 2014438 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
5675 : {
5676 2195130 : lambda_vector dir_v = lambda_vector_new (DDR_NB_LOOPS (ddr));
5677 :
5678 3315749 : for (j = 0; j < DDR_NB_LOOPS (ddr); j++)
5679 2007316 : dir_v[j] = dir_from_dist (dist_v[j]);
5680 :
5681 1097565 : save_dir_v (ddr, dir_v);
5682 : }
5683 916873 : }
5684 :
5685 : /* Helper function. Returns true when there is a dependence between the
5686 : data references. A_INDEX is the index of the first reference (0 for
5687 : DDR_A, 1 for DDR_B) and B_INDEX is the index of the second reference. */
5688 :
5689 : static bool
5690 3115007 : subscript_dependence_tester_1 (struct data_dependence_relation *ddr,
5691 : unsigned int a_index, unsigned int b_index,
5692 : class loop *loop_nest)
5693 : {
5694 3115007 : unsigned int i;
5695 3115007 : tree last_conflicts;
5696 3115007 : struct subscript *subscript;
5697 3115007 : tree res = NULL_TREE;
5698 :
5699 4406412 : for (i = 0; DDR_SUBSCRIPTS (ddr).iterate (i, &subscript); i++)
5700 : {
5701 3451102 : conflict_function *overlaps_a, *overlaps_b;
5702 :
5703 3451102 : analyze_overlapping_iterations (SUB_ACCESS_FN (subscript, a_index),
5704 : SUB_ACCESS_FN (subscript, b_index),
5705 : &overlaps_a, &overlaps_b,
5706 : &last_conflicts, loop_nest);
5707 :
5708 3451102 : if (SUB_CONFLICTS_IN_A (subscript))
5709 3451102 : free_conflict_function (SUB_CONFLICTS_IN_A (subscript));
5710 3451102 : if (SUB_CONFLICTS_IN_B (subscript))
5711 3451102 : free_conflict_function (SUB_CONFLICTS_IN_B (subscript));
5712 :
5713 3451102 : SUB_CONFLICTS_IN_A (subscript) = overlaps_a;
5714 3451102 : SUB_CONFLICTS_IN_B (subscript) = overlaps_b;
5715 3451102 : SUB_LAST_CONFLICT (subscript) = last_conflicts;
5716 :
5717 : /* If there is any undetermined conflict function we have to
5718 : give a conservative answer in case we cannot prove that
5719 : no dependence exists when analyzing another subscript. */
5720 3451102 : if (CF_NOT_KNOWN_P (overlaps_a)
5721 3421500 : || CF_NOT_KNOWN_P (overlaps_b))
5722 : {
5723 29602 : res = chrec_dont_know;
5724 29602 : continue;
5725 : }
5726 :
5727 : /* When there is a subscript with no dependence we can stop. */
5728 3421500 : else if (CF_NO_DEPENDENCE_P (overlaps_a)
5729 1261803 : || CF_NO_DEPENDENCE_P (overlaps_b))
5730 : {
5731 2159697 : res = chrec_known;
5732 2159697 : break;
5733 : }
5734 : }
5735 :
5736 3115007 : if (res == NULL_TREE)
5737 : return true;
5738 :
5739 2188214 : if (res == chrec_known)
5740 2159697 : dependence_stats.num_dependence_independent++;
5741 : else
5742 28517 : dependence_stats.num_dependence_undetermined++;
5743 2188214 : finalize_ddr_dependent (ddr, res);
5744 2188214 : return false;
5745 : }
5746 :
5747 : /* Return true if every distance vector of DDR has, in the loops marked in
5748 : VARIES, the distances of the first vector or all of them negated. */
5749 :
5750 : static bool
5751 3332 : dist_vectors_agree_p (const struct data_dependence_relation *ddr,
5752 : const vec<bool> &varies)
5753 : {
5754 3332 : if (DDR_NUM_DIST_VECTS (ddr) == 0)
5755 : return true;
5756 3323 : lambda_vector first = DDR_DIST_VECT (ddr, 0);
5757 10110 : for (lambda_vector v : DDR_DIST_VECTS (ddr))
5758 : {
5759 6791 : bool same = true, opposite = true;
5760 20485 : for (unsigned i = 0; i < DDR_NB_LOOPS (ddr); i++)
5761 13694 : if (varies[i])
5762 : {
5763 5932 : same &= v[i] == first[i];
5764 5932 : opposite &= v[i] == -first[i];
5765 : }
5766 6791 : if (!same && !opposite)
5767 : return false;
5768 : }
5769 : return true;
5770 : }
5771 :
5772 : /* Set VARIES[I] for each loop I of the nest of DDR in which CHREC has an
5773 : evolution part, as evolution_part_in_loop_num finds it, walking CHREC
5774 : once. The nest is a chain of loops at consecutive depths, as
5775 : find_loop_nest builds it, so loop I is at the depth of loop 0 plus I. */
5776 :
5777 : static void
5778 54362 : mark_evolving_loops (const struct data_dependence_relation *ddr, tree chrec,
5779 : vec<bool> *varies)
5780 : {
5781 54362 : unsigned n = DDR_NB_LOOPS (ddr);
5782 54362 : if (automatically_generated_chrec_p (chrec))
5783 0 : for (unsigned i = 0; i < n; i++)
5784 0 : (*varies)[i] = true;
5785 54362 : unsigned depth0 = loop_depth (DDR_LOOP_NEST (ddr)[0]);
5786 103910 : for (; TREE_CODE (chrec) == POLYNOMIAL_CHREC; chrec = CHREC_LEFT (chrec))
5787 : {
5788 49548 : class loop *loop = get_chrec_loop (chrec);
5789 49548 : unsigned i = loop_depth (loop) - depth0;
5790 49548 : if (i < n && DDR_LOOP_NEST (ddr)[i] == loop)
5791 49514 : (*varies)[i] = true;
5792 : }
5793 54362 : }
5794 :
5795 : /* What the distance vectors of a dependence miss in its free loops. */
5796 :
5797 : enum free_loop_distances
5798 : {
5799 : /* They miss nothing. */
5800 : FREE_LOOP_DISTANCES_KEPT,
5801 : /* They miss distances, and agree in the loops the subscripts constrain,
5802 : so they can be completed. */
5803 : FREE_LOOP_DISTANCES_LOST,
5804 : /* They miss distances and do not agree there. */
5805 : FREE_LOOP_DISTANCES_DISAGREE
5806 : };
5807 :
5808 : /* Return whether the distance vectors of DDR miss distances because of a
5809 : free loop, a loop of the nest that contains both references and in
5810 : which no compared access function evolves, so that the dependence can
5811 : have any distance in it. The vectors give a free loop a distance of
5812 : its own only when it is the only free loop and lies outside every loop
5813 : the subscripts constrain, and the vectors agree in the loops the
5814 : subscripts constrain. Otherwise its distance is taken as zero, which
5815 : misses the distances of opposite signs in it and in another free loop
5816 : or in a loop with a nonzero distance. Set FREE_LOOPS[I] when loop I of
5817 : the nest is free and VARIES[I] when a compared access function evolves
5818 : in it; both have room for DDR_NB_LOOPS (DDR) elements. */
5819 :
5820 : static enum free_loop_distances
5821 916947 : check_free_loop_distances (const struct data_dependence_relation *ddr,
5822 : vec<bool> *free_loops, vec<bool> *varies)
5823 : {
5824 916947 : unsigned n = DDR_NB_LOOPS (ddr);
5825 916947 : if (n < 2)
5826 : return FREE_LOOP_DISTANCES_KEPT;
5827 19468 : free_loops->quick_grow_cleared (n);
5828 :
5829 19468 : varies->quick_grow_cleared (n);
5830 85585 : for (subscript *sub : DDR_SUBSCRIPTS (ddr))
5831 81543 : for (unsigned k = 0; k < 2; k++)
5832 54362 : mark_evolving_loops (ddr, SUB_ACCESS_FN (sub, k), varies);
5833 :
5834 19468 : basic_block bb_a = gimple_bb (DR_STMT (DDR_A (ddr)));
5835 19468 : basic_block bb_b = gimple_bb (DR_STMT (DDR_B (ddr)));
5836 19468 : unsigned first_varying = n, nfree = 0, free_idx = 0;
5837 58492 : for (unsigned i = 0; i < n; i++)
5838 39024 : if ((*varies)[i])
5839 44609 : first_varying = MIN (first_varying, i);
5840 14273 : else if (flow_bb_inside_loop_p (DDR_LOOP_NEST (ddr)[i], bb_a)
5841 14273 : && flow_bb_inside_loop_p (DDR_LOOP_NEST (ddr)[i], bb_b))
5842 : {
5843 3577 : nfree++;
5844 3577 : free_idx = i;
5845 3577 : (*free_loops)[i] = true;
5846 : }
5847 19468 : if (nfree == 0)
5848 : return FREE_LOOP_DISTANCES_KEPT;
5849 3332 : if (!dist_vectors_agree_p (ddr, *varies))
5850 : return FREE_LOOP_DISTANCES_DISAGREE;
5851 3328 : if (nfree == 1 && free_idx < first_varying)
5852 : return FREE_LOOP_DISTANCES_KEPT;
5853 443 : if (nfree > 1)
5854 : return FREE_LOOP_DISTANCES_LOST;
5855 :
5856 : unsigned j;
5857 : lambda_vector v;
5858 553 : FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), j, v)
5859 536 : for (unsigned i = 0; i < n; i++)
5860 379 : if ((*varies)[i] && v[i] != 0)
5861 : return FREE_LOOP_DISTANCES_LOST;
5862 : return FREE_LOOP_DISTANCES_KEPT;
5863 : }
5864 :
5865 : /* Add the distance vectors of DDR that its free loops FREE_LOOPS miss,
5866 : when its vectors agree in the loops the subscripts constrain: every
5867 : lexicographically positive vector with the distances of the first
5868 : vector, or their opposite, in the other loops and -1, 0 or 1 in each
5869 : free loop. That is not possible when the bases differ and the objects
5870 : vary in the nest; return false then, when there is no vector to start
5871 : from, or as soon as more than 32 vectors would be needed. */
5872 :
5873 : static bool
5874 286 : add_free_loop_distances (struct data_dependence_relation *ddr,
5875 : const vec<bool> &free_loops)
5876 : {
5877 66 : if ((DDR_COULD_BE_INDEPENDENT_P (ddr) && !DDR_OBJECT_A (ddr))
5878 286 : || DDR_NUM_DIST_VECTS (ddr) == 0)
5879 : return false;
5880 216 : unsigned n = DDR_NB_LOOPS (ddr), nfree = 0, ncomb = 1;
5881 1115 : for (bool f : free_loops)
5882 467 : if (f)
5883 : {
5884 391 : nfree++;
5885 391 : ncomb *= 3;
5886 : }
5887 216 : if (nfree > 3)
5888 : return false;
5889 :
5890 216 : lambda_vector first = DDR_DIST_VECT (ddr, 0);
5891 648 : for (int sign = 1; sign >= -1; sign -= 2)
5892 3828 : for (unsigned c = 0; c < ncomb; c++)
5893 : {
5894 3396 : lambda_vector v = lambda_vector_new (n);
5895 14094 : for (unsigned i = 0, rest = c; i < n; i++)
5896 7302 : if (free_loops[i])
5897 : {
5898 6546 : v[i] = (int) (rest % 3) - 1;
5899 6546 : rest /= 3;
5900 : }
5901 : else
5902 756 : v[i] = sign * first[i];
5903 5269 : if (lambda_vector_lexico_pos (v, n))
5904 : {
5905 1873 : save_dist_v (ddr, v);
5906 3396 : if (DDR_NUM_DIST_VECTS (ddr) > 32)
5907 : return false;
5908 : }
5909 : }
5910 : return true;
5911 : }
5912 :
5913 : /* Computes the conflicting iterations in LOOP_NEST, and initialize DDR. */
5914 :
5915 : static void
5916 3107347 : subscript_dependence_tester (struct data_dependence_relation *ddr,
5917 : class loop *loop_nest)
5918 : {
5919 3107347 : if (subscript_dependence_tester_1 (ddr, 0, 1, loop_nest))
5920 919137 : dependence_stats.num_dependence_dependent++;
5921 :
5922 3107347 : compute_subscript_distance (ddr);
5923 3107347 : if (build_classic_dist_vector (ddr, loop_nest))
5924 : {
5925 916947 : unsigned n = DDR_NB_LOOPS (ddr);
5926 916947 : auto_vec<bool, 8> free_loops (n), varies (n);
5927 916947 : enum free_loop_distances lost
5928 916947 : = check_free_loop_distances (ddr, &free_loops, &varies);
5929 916947 : if (lost == FREE_LOOP_DISTANCES_DISAGREE
5930 916947 : || (lost == FREE_LOOP_DISTANCES_LOST
5931 286 : && !add_free_loop_distances (ddr, free_loops)))
5932 : {
5933 74 : if (dump_file && (dump_flags & TDF_DETAILS))
5934 0 : fprintf (dump_file, "(distance vectors miss a free loop)\n");
5935 74 : DDR_ARE_DEPENDENT (ddr) = chrec_dont_know;
5936 74 : DDR_DIST_VECTS (ddr).release ();
5937 74 : return;
5938 : }
5939 :
5940 916873 : if (dump_file && (dump_flags & TDF_DETAILS))
5941 : {
5942 4060 : unsigned i;
5943 :
5944 4060 : fprintf (dump_file, "(build_classic_dist_vector\n");
5945 12253 : for (i = 0; i < DDR_NUM_DIST_VECTS (ddr); i++)
5946 : {
5947 4133 : fprintf (dump_file, " dist_vector = (");
5948 4133 : print_lambda_vector (dump_file, DDR_DIST_VECT (ddr, i),
5949 8266 : DDR_NB_LOOPS (ddr));
5950 4133 : fprintf (dump_file, " )\n");
5951 : }
5952 4060 : fprintf (dump_file, ")\n");
5953 : }
5954 :
5955 916873 : build_classic_dir_vector (ddr);
5956 916947 : }
5957 : }
5958 :
5959 : /* Returns true when all the access functions of A are affine or
5960 : constant with respect to LOOP_NEST. */
5961 :
5962 : static bool
5963 6281883 : access_functions_are_affine_or_constant_p (const struct data_reference *a,
5964 : const class loop *loop_nest)
5965 : {
5966 6281883 : vec<tree> fns = DR_ACCESS_FNS (a);
5967 27279617 : for (tree t : fns)
5968 8498101 : if (!evolution_function_is_invariant_p (t, loop_nest->num)
5969 8498101 : && !evolution_function_is_affine_multivariate_p (t, loop_nest->num))
5970 : return false;
5971 :
5972 : return true;
5973 : }
5974 :
5975 : /* This computes the affine dependence relation between A and B with
5976 : respect to LOOP_NEST. CHREC_KNOWN is used for representing the
5977 : independence between two accesses, while CHREC_DONT_KNOW is used
5978 : for representing the unknown relation.
5979 :
5980 : Note that it is possible to stop the computation of the dependence
5981 : relation the first time we detect a CHREC_KNOWN element for a given
5982 : subscript. */
5983 :
5984 : void
5985 6513645 : compute_affine_dependence (struct data_dependence_relation *ddr,
5986 : class loop *loop_nest)
5987 : {
5988 6513645 : struct data_reference *dra = DDR_A (ddr);
5989 6513645 : struct data_reference *drb = DDR_B (ddr);
5990 :
5991 6513645 : if (dump_file && (dump_flags & TDF_DETAILS))
5992 : {
5993 135897 : fprintf (dump_file, "(compute_affine_dependence\n");
5994 135897 : fprintf (dump_file, " ref_a: ");
5995 135897 : print_generic_expr (dump_file, DR_REF (dra));
5996 135897 : fprintf (dump_file, ", stmt_a: ");
5997 135897 : print_gimple_stmt (dump_file, DR_STMT (dra), 0, TDF_SLIM);
5998 135897 : fprintf (dump_file, " ref_b: ");
5999 135897 : print_generic_expr (dump_file, DR_REF (drb));
6000 135897 : fprintf (dump_file, ", stmt_b: ");
6001 135897 : print_gimple_stmt (dump_file, DR_STMT (drb), 0, TDF_SLIM);
6002 : }
6003 :
6004 : /* Analyze only when the dependence relation is not yet known. */
6005 6513645 : if (DDR_ARE_DEPENDENT (ddr) == NULL_TREE)
6006 : {
6007 3171480 : dependence_stats.num_dependence_tests++;
6008 :
6009 3171480 : if (access_functions_are_affine_or_constant_p (dra, loop_nest)
6010 3171480 : && access_functions_are_affine_or_constant_p (drb, loop_nest))
6011 3107347 : subscript_dependence_tester (ddr, loop_nest);
6012 :
6013 : /* As a last case, if the dependence cannot be determined, or if
6014 : the dependence is considered too difficult to determine, answer
6015 : "don't know". */
6016 : else
6017 : {
6018 64133 : dependence_stats.num_dependence_undetermined++;
6019 :
6020 64133 : if (dump_file && (dump_flags & TDF_DETAILS))
6021 : {
6022 158 : fprintf (dump_file, "Data ref a:\n");
6023 158 : dump_data_reference (dump_file, dra);
6024 158 : fprintf (dump_file, "Data ref b:\n");
6025 158 : dump_data_reference (dump_file, drb);
6026 158 : fprintf (dump_file, "affine dependence test not usable: access function not affine or constant.\n");
6027 : }
6028 64133 : finalize_ddr_dependent (ddr, chrec_dont_know);
6029 : }
6030 : }
6031 :
6032 6513645 : if (dump_file && (dump_flags & TDF_DETAILS))
6033 : {
6034 135897 : if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
6035 120354 : fprintf (dump_file, ") -> no dependence\n");
6036 15543 : else if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
6037 11397 : fprintf (dump_file, ") -> dependence analysis failed\n");
6038 : else
6039 4146 : fprintf (dump_file, ")\n");
6040 : }
6041 6513645 : }
6042 :
6043 : /* Compute in DEPENDENCE_RELATIONS the data dependence graph for all
6044 : the data references in DATAREFS, in the LOOP_NEST. When
6045 : COMPUTE_SELF_AND_RR is FALSE, don't compute read-read and self
6046 : relations. Return true when successful, i.e. data references number
6047 : is small enough to be handled. */
6048 :
6049 : bool
6050 442791 : compute_all_dependences (const vec<data_reference_p> &datarefs,
6051 : vec<ddr_p> *dependence_relations,
6052 : const vec<loop_p> &loop_nest,
6053 : bool compute_self_and_rr)
6054 : {
6055 442791 : struct data_dependence_relation *ddr;
6056 442791 : struct data_reference *a, *b;
6057 442791 : unsigned int i, j;
6058 :
6059 442791 : if ((int) datarefs.length ()
6060 442791 : > param_loop_max_datarefs_for_datadeps)
6061 : {
6062 0 : struct data_dependence_relation *ddr;
6063 :
6064 : /* Insert a single relation into dependence_relations:
6065 : chrec_dont_know. */
6066 0 : ddr = initialize_data_dependence_relation (NULL, NULL, loop_nest);
6067 0 : dependence_relations->safe_push (ddr);
6068 0 : return false;
6069 : }
6070 :
6071 1631216 : FOR_EACH_VEC_ELT (datarefs, i, a)
6072 7319134 : for (j = i + 1; datarefs.iterate (j, &b); j++)
6073 4942284 : if (DR_IS_WRITE (a) || DR_IS_WRITE (b) || compute_self_and_rr)
6074 : {
6075 4567727 : ddr = initialize_data_dependence_relation (a, b, loop_nest);
6076 4567727 : dependence_relations->safe_push (ddr);
6077 4567727 : if (loop_nest.exists ())
6078 4545673 : compute_affine_dependence (ddr, loop_nest[0]);
6079 : }
6080 :
6081 442791 : if (compute_self_and_rr)
6082 1041704 : FOR_EACH_VEC_ELT (datarefs, i, a)
6083 : {
6084 772368 : ddr = initialize_data_dependence_relation (a, a, loop_nest);
6085 772368 : dependence_relations->safe_push (ddr);
6086 772368 : if (loop_nest.exists ())
6087 772368 : compute_affine_dependence (ddr, loop_nest[0]);
6088 : }
6089 :
6090 : return true;
6091 : }
6092 :
6093 : /* Describes a location of a memory reference. */
6094 :
6095 : struct data_ref_loc
6096 : {
6097 : /* The memory reference. */
6098 : tree ref;
6099 :
6100 : /* True if the memory reference is read. */
6101 : bool is_read;
6102 :
6103 : /* True if the data reference is conditional within the containing
6104 : statement, i.e. if it might not occur even when the statement
6105 : is executed and runs to completion. */
6106 : bool is_conditional_in_stmt;
6107 : };
6108 :
6109 :
6110 : /* Stores the locations of memory references in STMT to REFERENCES. Returns
6111 : true if STMT clobbers memory, false otherwise. */
6112 :
6113 : static bool
6114 51252616 : get_references_in_stmt (gimple *stmt, vec<data_ref_loc, va_heap> *references)
6115 : {
6116 51252616 : bool clobbers_memory = false;
6117 51252616 : data_ref_loc ref;
6118 51252616 : tree op0, op1;
6119 51252616 : enum gimple_code stmt_code = gimple_code (stmt);
6120 :
6121 : /* ASM_EXPR and CALL_EXPR may embed arbitrary side effects.
6122 : As we cannot model data-references to not spelled out
6123 : accesses give up if they may occur. */
6124 51252616 : if (stmt_code == GIMPLE_CALL
6125 51252616 : && !(gimple_call_flags (stmt) & ECF_CONST))
6126 : {
6127 : /* Allow IFN_GOMP_SIMD_LANE in their own loops. */
6128 4231184 : if (gimple_call_internal_p (stmt))
6129 62162 : switch (gimple_call_internal_fn (stmt))
6130 : {
6131 5616 : case IFN_GOMP_SIMD_LANE:
6132 5616 : {
6133 5616 : class loop *loop = gimple_bb (stmt)->loop_father;
6134 5616 : tree uid = gimple_call_arg (stmt, 0);
6135 5616 : gcc_assert (TREE_CODE (uid) == SSA_NAME);
6136 5616 : if (loop == NULL
6137 5616 : || loop->simduid != SSA_NAME_VAR (uid))
6138 : clobbers_memory = true;
6139 : break;
6140 : }
6141 : case IFN_MASK_LOAD:
6142 : case IFN_MASK_STORE:
6143 : break;
6144 1007 : case IFN_MASK_CALL:
6145 1007 : {
6146 1007 : tree orig_fndecl
6147 1007 : = gimple_call_addr_fndecl (gimple_call_arg (stmt, 0));
6148 1007 : if (!orig_fndecl
6149 1007 : || (flags_from_decl_or_type (orig_fndecl) & ECF_CONST) == 0)
6150 : clobbers_memory = true;
6151 : }
6152 : break;
6153 : default:
6154 4267294 : clobbers_memory = true;
6155 : break;
6156 : }
6157 4169022 : else if (gimple_call_builtin_p (stmt, BUILT_IN_PREFETCH))
6158 : clobbers_memory = false;
6159 : else
6160 4267294 : clobbers_memory = true;
6161 : }
6162 47021432 : else if (stmt_code == GIMPLE_ASM
6163 47021432 : && (gimple_asm_volatile_p (as_a <gasm *> (stmt))
6164 8544 : || gimple_vuse (stmt)))
6165 : clobbers_memory = true;
6166 :
6167 86696655 : if (!gimple_vuse (stmt))
6168 : return clobbers_memory;
6169 :
6170 19634572 : if (stmt_code == GIMPLE_ASSIGN)
6171 : {
6172 14466621 : tree base;
6173 14466621 : op0 = gimple_assign_lhs (stmt);
6174 14466621 : op1 = gimple_assign_rhs1 (stmt);
6175 :
6176 14466621 : if (DECL_P (op1)
6177 14466621 : || (REFERENCE_CLASS_P (op1)
6178 6921917 : && (base = get_base_address (op1))
6179 6921917 : && TREE_CODE (base) != SSA_NAME
6180 6921871 : && !is_gimple_min_invariant (base)))
6181 : {
6182 7815004 : ref.ref = op1;
6183 7815004 : ref.is_read = true;
6184 7815004 : ref.is_conditional_in_stmt = false;
6185 7815004 : references->safe_push (ref);
6186 : }
6187 : }
6188 5167951 : else if (stmt_code == GIMPLE_CALL)
6189 : {
6190 4246416 : unsigned i = 0, n;
6191 4246416 : tree ptr, type;
6192 4246416 : unsigned int align;
6193 :
6194 4246416 : ref.is_read = false;
6195 4246416 : if (gimple_call_internal_p (stmt))
6196 76946 : switch (gimple_call_internal_fn (stmt))
6197 : {
6198 2074 : case IFN_MASK_LOAD:
6199 2074 : if (gimple_call_lhs (stmt) == NULL_TREE)
6200 : break;
6201 2074 : ref.is_read = true;
6202 : /* FALLTHRU */
6203 3915 : case IFN_MASK_STORE:
6204 3915 : ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), 0);
6205 3915 : align = tree_to_shwi (gimple_call_arg (stmt, 1));
6206 3915 : if (ref.is_read)
6207 2074 : type = TREE_TYPE (gimple_call_lhs (stmt));
6208 : else
6209 1841 : type = TREE_TYPE (gimple_call_arg (stmt, 3));
6210 3915 : if (TYPE_ALIGN (type) != align)
6211 1544 : type = build_aligned_type (type, align);
6212 3915 : ref.is_conditional_in_stmt = true;
6213 3915 : ref.ref = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0),
6214 : ptr);
6215 3915 : references->safe_push (ref);
6216 3915 : return false;
6217 : case IFN_MASK_CALL:
6218 4242501 : i = 1;
6219 : gcc_fallthrough ();
6220 : default:
6221 : break;
6222 : }
6223 :
6224 4242501 : op0 = gimple_call_lhs (stmt);
6225 4242501 : n = gimple_call_num_args (stmt);
6226 17192913 : for (; i < n; i++)
6227 : {
6228 8707911 : op1 = gimple_call_arg (stmt, i);
6229 :
6230 8707911 : if (DECL_P (op1)
6231 8707911 : || (REFERENCE_CLASS_P (op1) && get_base_address (op1)))
6232 : {
6233 518074 : ref.ref = op1;
6234 518074 : ref.is_read = true;
6235 518074 : ref.is_conditional_in_stmt = false;
6236 518074 : references->safe_push (ref);
6237 : }
6238 : }
6239 : }
6240 : else
6241 : return clobbers_memory;
6242 :
6243 18709122 : if (op0
6244 18709122 : && (DECL_P (op0)
6245 15185227 : || (REFERENCE_CLASS_P (op0) && get_base_address (op0))))
6246 : {
6247 7495349 : ref.ref = op0;
6248 7495349 : ref.is_read = false;
6249 7495349 : ref.is_conditional_in_stmt = false;
6250 7495349 : references->safe_push (ref);
6251 : }
6252 : return clobbers_memory;
6253 : }
6254 :
6255 :
6256 : /* Returns true if the loop-nest has any data reference. */
6257 :
6258 : bool
6259 761 : loop_nest_has_data_refs (loop_p loop)
6260 : {
6261 761 : basic_block *bbs = get_loop_body (loop);
6262 761 : auto_vec<data_ref_loc, 3> references;
6263 :
6264 1010 : for (unsigned i = 0; i < loop->num_nodes; i++)
6265 : {
6266 940 : basic_block bb = bbs[i];
6267 940 : gimple_stmt_iterator bsi;
6268 :
6269 3243 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
6270 : {
6271 2054 : gimple *stmt = gsi_stmt (bsi);
6272 2054 : get_references_in_stmt (stmt, &references);
6273 2054 : if (references.length ())
6274 : {
6275 691 : free (bbs);
6276 691 : return true;
6277 : }
6278 : }
6279 : }
6280 70 : free (bbs);
6281 70 : return false;
6282 761 : }
6283 :
6284 : /* Stores the data references in STMT to DATAREFS. If there is an unanalyzable
6285 : reference, returns false, otherwise returns true. NEST is the outermost
6286 : loop of the loop nest in which the references should be analyzed.
6287 : RECORD_NOWRAP_ASSUMPTIONS determines whether or not no-wrap assumptions
6288 : are allowed and recorded. */
6289 :
6290 : opt_result
6291 51236199 : find_data_references_in_stmt (class loop *nest, gimple *stmt,
6292 : vec<data_reference_p> *datarefs,
6293 : bool record_nowrap_assumptions)
6294 : {
6295 51236199 : auto_vec<data_ref_loc, 2> references;
6296 51236199 : data_reference_p dr;
6297 :
6298 51236199 : if (get_references_in_stmt (stmt, &references))
6299 4267290 : return opt_result::failure_at (stmt, "statement clobbers memory: %G",
6300 : stmt);
6301 :
6302 155954708 : for (const data_ref_loc &ref : references)
6303 : {
6304 15047981 : dr = create_data_ref (nest ? loop_preheader_edge (nest) : NULL,
6305 15047981 : loop_containing_stmt (stmt), ref.ref,
6306 15047981 : stmt, ref.is_read, ref.is_conditional_in_stmt,
6307 : record_nowrap_assumptions);
6308 15047981 : gcc_assert (dr != NULL);
6309 15047981 : datarefs->safe_push (dr);
6310 : }
6311 :
6312 46968909 : return opt_result::success ();
6313 51236199 : }
6314 :
6315 : /* Stores the data references in STMT to DATAREFS. If there is an
6316 : unanalyzable reference, returns false, otherwise returns true.
6317 : NEST is the outermost loop of the loop nest in which the references
6318 : should be instantiated, LOOP is the loop in which the references
6319 : should be analyzed. */
6320 :
6321 : bool
6322 14363 : graphite_find_data_references_in_stmt (edge nest, loop_p loop, gimple *stmt,
6323 : vec<data_reference_p> *datarefs)
6324 : {
6325 14363 : auto_vec<data_ref_loc, 2> references;
6326 14363 : bool ret = true;
6327 14363 : data_reference_p dr;
6328 :
6329 14363 : if (get_references_in_stmt (stmt, &references))
6330 : return false;
6331 :
6332 45986 : for (const data_ref_loc &ref : references)
6333 : {
6334 5818 : dr = create_data_ref (nest, loop, ref.ref, stmt, ref.is_read,
6335 2909 : ref.is_conditional_in_stmt);
6336 2909 : gcc_assert (dr != NULL);
6337 2909 : datarefs->safe_push (dr);
6338 : }
6339 :
6340 : return ret;
6341 14363 : }
6342 :
6343 : /* Search the data references in LOOP, and record the information into
6344 : DATAREFS. Returns chrec_dont_know when failing to analyze a
6345 : difficult case, returns NULL_TREE otherwise. */
6346 :
6347 : tree
6348 2734467 : find_data_references_in_bb (class loop *loop, basic_block bb,
6349 : vec<data_reference_p> *datarefs)
6350 : {
6351 2734467 : gimple_stmt_iterator bsi;
6352 :
6353 23243514 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
6354 : {
6355 18269586 : gimple *stmt = gsi_stmt (bsi);
6356 :
6357 18269586 : if (!find_data_references_in_stmt (loop, stmt, datarefs))
6358 : {
6359 495006 : struct data_reference *res;
6360 495006 : res = XCNEW (struct data_reference);
6361 495006 : datarefs->safe_push (res);
6362 :
6363 495006 : return chrec_dont_know;
6364 : }
6365 : }
6366 :
6367 : return NULL_TREE;
6368 : }
6369 :
6370 : /* Search the data references in LOOP, and record the information into
6371 : DATAREFS. Returns chrec_dont_know when failing to analyze a
6372 : difficult case, returns NULL_TREE otherwise.
6373 :
6374 : TODO: This function should be made smarter so that it can handle address
6375 : arithmetic as if they were array accesses, etc. */
6376 :
6377 : tree
6378 827286 : find_data_references_in_loop (class loop *loop,
6379 : vec<data_reference_p> *datarefs)
6380 : {
6381 827286 : basic_block bb, *bbs;
6382 827286 : unsigned int i;
6383 :
6384 827286 : bbs = get_loop_body_in_dom_order (loop);
6385 :
6386 3677766 : for (i = 0; i < loop->num_nodes; i++)
6387 : {
6388 2320503 : bb = bbs[i];
6389 :
6390 2320503 : if (find_data_references_in_bb (loop, bb, datarefs) == chrec_dont_know)
6391 : {
6392 297309 : free (bbs);
6393 297309 : return chrec_dont_know;
6394 : }
6395 : }
6396 529977 : free (bbs);
6397 :
6398 529977 : return NULL_TREE;
6399 : }
6400 :
6401 : /* Return the alignment in bytes that DRB is guaranteed to have at all
6402 : times. */
6403 :
6404 : unsigned int
6405 481757 : dr_alignment (innermost_loop_behavior *drb)
6406 : {
6407 : /* Get the alignment of BASE_ADDRESS + INIT. */
6408 481757 : unsigned int alignment = drb->base_alignment;
6409 481757 : unsigned int misalignment = (drb->base_misalignment
6410 481757 : + TREE_INT_CST_LOW (drb->init));
6411 481757 : if (misalignment != 0)
6412 211572 : alignment = MIN (alignment, misalignment & -misalignment);
6413 :
6414 : /* Cap it to the alignment of OFFSET. */
6415 481757 : if (!integer_zerop (drb->offset))
6416 36374 : alignment = MIN (alignment, drb->offset_alignment);
6417 :
6418 : /* Cap it to the alignment of STEP. */
6419 481757 : if (!integer_zerop (drb->step))
6420 296189 : alignment = MIN (alignment, drb->step_alignment);
6421 :
6422 481757 : return alignment;
6423 : }
6424 :
6425 : /* If BASE is a pointer-typed SSA name, try to find the object that it
6426 : is based on. Return this object X on success and store the alignment
6427 : in bytes of BASE - &X in *ALIGNMENT_OUT. */
6428 :
6429 : static tree
6430 791220 : get_base_for_alignment_1 (tree base, unsigned int *alignment_out)
6431 : {
6432 791220 : if (TREE_CODE (base) != SSA_NAME || !POINTER_TYPE_P (TREE_TYPE (base)))
6433 : return NULL_TREE;
6434 :
6435 391379 : gimple *def = SSA_NAME_DEF_STMT (base);
6436 391379 : base = analyze_scalar_evolution (loop_containing_stmt (def), base);
6437 :
6438 : /* Peel chrecs and record the minimum alignment preserved by
6439 : all steps. */
6440 391379 : unsigned int alignment = MAX_OFILE_ALIGNMENT / BITS_PER_UNIT;
6441 795922 : while (TREE_CODE (base) == POLYNOMIAL_CHREC)
6442 : {
6443 13164 : unsigned int step_alignment = highest_pow2_factor (CHREC_RIGHT (base));
6444 13164 : alignment = MIN (alignment, step_alignment);
6445 13164 : base = CHREC_LEFT (base);
6446 : }
6447 :
6448 : /* Punt if the expression is too complicated to handle. */
6449 391379 : if (tree_contains_chrecs (base, NULL) || !POINTER_TYPE_P (TREE_TYPE (base)))
6450 : return NULL_TREE;
6451 :
6452 : /* The only useful cases are those for which a dereference folds to something
6453 : other than an INDIRECT_REF. */
6454 391337 : tree ref_type = TREE_TYPE (TREE_TYPE (base));
6455 391337 : tree ref = fold_indirect_ref_1 (UNKNOWN_LOCATION, ref_type, base);
6456 391337 : if (!ref)
6457 : return NULL_TREE;
6458 :
6459 : /* Analyze the base to which the steps we peeled were applied. */
6460 2658 : poly_int64 bitsize, bitpos, bytepos;
6461 2658 : machine_mode mode;
6462 2658 : int unsignedp, reversep, volatilep;
6463 2658 : tree offset;
6464 2658 : base = get_inner_reference (ref, &bitsize, &bitpos, &offset, &mode,
6465 : &unsignedp, &reversep, &volatilep);
6466 2658 : if (!base || !multiple_p (bitpos, BITS_PER_UNIT, &bytepos))
6467 : return NULL_TREE;
6468 :
6469 : /* Restrict the alignment to that guaranteed by the offsets. */
6470 2658 : unsigned int bytepos_alignment = known_alignment (bytepos);
6471 2658 : if (bytepos_alignment != 0)
6472 2499 : alignment = MIN (alignment, bytepos_alignment);
6473 2658 : if (offset)
6474 : {
6475 0 : unsigned int offset_alignment = highest_pow2_factor (offset);
6476 0 : alignment = MIN (alignment, offset_alignment);
6477 : }
6478 :
6479 2658 : *alignment_out = alignment;
6480 2658 : return base;
6481 : }
6482 :
6483 : /* Return the object whose alignment would need to be changed in order
6484 : to increase the alignment of ADDR. Store the maximum achievable
6485 : alignment in *MAX_ALIGNMENT. */
6486 :
6487 : tree
6488 791220 : get_base_for_alignment (tree addr, unsigned int *max_alignment)
6489 : {
6490 791220 : tree base = get_base_for_alignment_1 (addr, max_alignment);
6491 791220 : if (base)
6492 : return base;
6493 :
6494 788562 : if (TREE_CODE (addr) == ADDR_EXPR)
6495 293819 : addr = TREE_OPERAND (addr, 0);
6496 788562 : *max_alignment = MAX_OFILE_ALIGNMENT / BITS_PER_UNIT;
6497 788562 : return addr;
6498 : }
6499 :
6500 : /* Recursive helper function. */
6501 :
6502 : static bool
6503 144106 : find_loop_nest_1 (class loop *loop, vec<loop_p> *loop_nest)
6504 : {
6505 : /* Inner loops of the nest should not contain siblings. Example:
6506 : when there are two consecutive loops,
6507 :
6508 : | loop_0
6509 : | loop_1
6510 : | A[{0, +, 1}_1]
6511 : | endloop_1
6512 : | loop_2
6513 : | A[{0, +, 1}_2]
6514 : | endloop_2
6515 : | endloop_0
6516 :
6517 : the dependence relation cannot be captured by the distance
6518 : abstraction. */
6519 144106 : if (loop->next)
6520 : return false;
6521 :
6522 120958 : loop_nest->safe_push (loop);
6523 120958 : if (loop->inner)
6524 42875 : return find_loop_nest_1 (loop->inner, loop_nest);
6525 : return true;
6526 : }
6527 :
6528 : /* Return false when the LOOP is not well nested. Otherwise return
6529 : true and insert in LOOP_NEST the loops of the nest. LOOP_NEST will
6530 : contain the loops from the outermost to the innermost, as they will
6531 : appear in the classic distance vector. */
6532 :
6533 : bool
6534 1042371 : find_loop_nest (class loop *loop, vec<loop_p> *loop_nest)
6535 : {
6536 1042371 : loop_nest->safe_push (loop);
6537 1042371 : if (loop->inner)
6538 101231 : return find_loop_nest_1 (loop->inner, loop_nest);
6539 : return true;
6540 : }
6541 :
6542 : /* Returns true when the data dependences have been computed, false otherwise.
6543 : Given a loop nest LOOP, the following vectors are returned:
6544 : DATAREFS is initialized to all the array elements contained in this loop,
6545 : DEPENDENCE_RELATIONS contains the relations between the data references.
6546 : Compute read-read and self relations if
6547 : COMPUTE_SELF_AND_READ_READ_DEPENDENCES is TRUE. */
6548 :
6549 : bool
6550 415354 : compute_data_dependences_for_loop (class loop *loop,
6551 : bool compute_self_and_read_read_dependences,
6552 : vec<loop_p> *loop_nest,
6553 : vec<data_reference_p> *datarefs,
6554 : vec<ddr_p> *dependence_relations)
6555 : {
6556 415354 : bool res = true;
6557 :
6558 415354 : memset (&dependence_stats, 0, sizeof (dependence_stats));
6559 :
6560 : /* If the loop nest is not well formed, or one of the data references
6561 : is not computable, give up without spending time to compute other
6562 : dependences. */
6563 415354 : if (!loop
6564 415354 : || !find_loop_nest (loop, loop_nest)
6565 415352 : || find_data_references_in_loop (loop, datarefs) == chrec_dont_know
6566 684622 : || !compute_all_dependences (*datarefs, dependence_relations, *loop_nest,
6567 : compute_self_and_read_read_dependences))
6568 : res = false;
6569 :
6570 415354 : if (dump_file && (dump_flags & TDF_STATS))
6571 : {
6572 157 : fprintf (dump_file, "Dependence tester statistics:\n");
6573 :
6574 157 : fprintf (dump_file, "Number of dependence tests: %d\n",
6575 : dependence_stats.num_dependence_tests);
6576 157 : fprintf (dump_file, "Number of dependence tests classified dependent: %d\n",
6577 : dependence_stats.num_dependence_dependent);
6578 157 : fprintf (dump_file, "Number of dependence tests classified independent: %d\n",
6579 : dependence_stats.num_dependence_independent);
6580 157 : fprintf (dump_file, "Number of undetermined dependence tests: %d\n",
6581 : dependence_stats.num_dependence_undetermined);
6582 :
6583 157 : fprintf (dump_file, "Number of subscript tests: %d\n",
6584 : dependence_stats.num_subscript_tests);
6585 157 : fprintf (dump_file, "Number of undetermined subscript tests: %d\n",
6586 : dependence_stats.num_subscript_undetermined);
6587 157 : fprintf (dump_file, "Number of same subscript function: %d\n",
6588 : dependence_stats.num_same_subscript_function);
6589 :
6590 157 : fprintf (dump_file, "Number of ziv tests: %d\n",
6591 : dependence_stats.num_ziv);
6592 157 : fprintf (dump_file, "Number of ziv tests returning dependent: %d\n",
6593 : dependence_stats.num_ziv_dependent);
6594 157 : fprintf (dump_file, "Number of ziv tests returning independent: %d\n",
6595 : dependence_stats.num_ziv_independent);
6596 157 : fprintf (dump_file, "Number of ziv tests unimplemented: %d\n",
6597 : dependence_stats.num_ziv_unimplemented);
6598 :
6599 157 : fprintf (dump_file, "Number of siv tests: %d\n",
6600 : dependence_stats.num_siv);
6601 157 : fprintf (dump_file, "Number of siv tests returning dependent: %d\n",
6602 : dependence_stats.num_siv_dependent);
6603 157 : fprintf (dump_file, "Number of siv tests returning independent: %d\n",
6604 : dependence_stats.num_siv_independent);
6605 157 : fprintf (dump_file, "Number of siv tests unimplemented: %d\n",
6606 : dependence_stats.num_siv_unimplemented);
6607 :
6608 157 : fprintf (dump_file, "Number of miv tests: %d\n",
6609 : dependence_stats.num_miv);
6610 157 : fprintf (dump_file, "Number of miv tests returning dependent: %d\n",
6611 : dependence_stats.num_miv_dependent);
6612 157 : fprintf (dump_file, "Number of miv tests returning independent: %d\n",
6613 : dependence_stats.num_miv_independent);
6614 157 : fprintf (dump_file, "Number of miv tests unimplemented: %d\n",
6615 : dependence_stats.num_miv_unimplemented);
6616 : }
6617 :
6618 415354 : return res;
6619 : }
6620 :
6621 : /* Free the memory used by a data dependence relation DDR. */
6622 :
6623 : void
6624 13484885 : free_dependence_relation (struct data_dependence_relation *ddr)
6625 : {
6626 13484885 : if (ddr == NULL)
6627 : return;
6628 :
6629 13484885 : if (DDR_SUBSCRIPTS (ddr).exists ())
6630 919131 : free_subscripts (DDR_SUBSCRIPTS (ddr));
6631 13484885 : DDR_DIST_VECTS (ddr).release ();
6632 13484885 : DDR_DIR_VECTS (ddr).release ();
6633 :
6634 13484885 : free (ddr);
6635 : }
6636 :
6637 : /* Free the memory used by the data dependence relations from
6638 : DEPENDENCE_RELATIONS. */
6639 :
6640 : void
6641 2933765 : free_dependence_relations (vec<ddr_p>& dependence_relations)
6642 : {
6643 9473402 : for (data_dependence_relation *ddr : dependence_relations)
6644 5343447 : if (ddr)
6645 5343447 : free_dependence_relation (ddr);
6646 :
6647 2933765 : dependence_relations.release ();
6648 2933765 : }
6649 :
6650 : /* Free the memory used by the data references from DATAREFS. */
6651 :
6652 : void
6653 3600580 : free_data_refs (vec<data_reference_p>& datarefs)
6654 : {
6655 21515826 : for (data_reference *dr : datarefs)
6656 13450918 : free_data_ref (dr);
6657 3600580 : datarefs.release ();
6658 3600580 : }
6659 :
6660 : /* Reset DR_INNERMOST, DR_NOWRAP_BOUND, and access functions of DR. */
6661 :
6662 : void
6663 96 : dr_reset (data_reference *dr)
6664 : {
6665 96 : DR_INNERMOST (dr) = innermost_loop_behavior ();
6666 96 : DR_NOWRAP_BOUND (dr) = NULL_TREE;
6667 96 : dr->indices.access_fns.release ();
6668 96 : }
6669 :
6670 : /* Common routine implementing both dr_direction_indicator and
6671 : dr_zero_step_indicator. Return USEFUL_MIN if the indicator is known
6672 : to be >= USEFUL_MIN and -1 if the indicator is known to be negative.
6673 : Return the step as the indicator otherwise. */
6674 :
6675 : static tree
6676 67482 : dr_step_indicator (struct data_reference *dr, int useful_min)
6677 : {
6678 67482 : tree step = DR_STEP (dr);
6679 67482 : if (!step)
6680 : return NULL_TREE;
6681 67482 : STRIP_NOPS (step);
6682 : /* Look for cases where the step is scaled by a positive constant
6683 : integer, which will often be the access size. If the multiplication
6684 : doesn't change the sign (due to overflow effects) then we can
6685 : test the unscaled value instead. */
6686 67482 : if (TREE_CODE (step) == MULT_EXPR
6687 5508 : && TREE_CODE (TREE_OPERAND (step, 1)) == INTEGER_CST
6688 72934 : && tree_int_cst_sgn (TREE_OPERAND (step, 1)) > 0)
6689 : {
6690 5452 : tree factor = TREE_OPERAND (step, 1);
6691 5452 : step = TREE_OPERAND (step, 0);
6692 :
6693 : /* Strip widening and truncating conversions as well as nops. */
6694 1226 : if (CONVERT_EXPR_P (step)
6695 5452 : && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (step, 0))))
6696 4226 : step = TREE_OPERAND (step, 0);
6697 5452 : tree type = TREE_TYPE (step);
6698 :
6699 : /* Get the range of step values that would not cause overflow. */
6700 10904 : widest_int minv = (wi::to_widest (TYPE_MIN_VALUE (ssizetype))
6701 5452 : / wi::to_widest (factor));
6702 10904 : widest_int maxv = (wi::to_widest (TYPE_MAX_VALUE (ssizetype))
6703 5452 : / wi::to_widest (factor));
6704 :
6705 : /* Get the range of values that the unconverted step actually has. */
6706 5452 : wide_int step_min, step_max;
6707 5452 : int_range_max vr;
6708 5452 : if (TREE_CODE (step) != SSA_NAME
6709 10796 : || !get_range_query (cfun)->range_of_expr (vr, step)
6710 10850 : || vr.undefined_p ())
6711 : {
6712 54 : step_min = wi::to_wide (TYPE_MIN_VALUE (type));
6713 54 : step_max = wi::to_wide (TYPE_MAX_VALUE (type));
6714 : }
6715 : else
6716 : {
6717 5398 : step_min = vr.lower_bound ();
6718 5398 : step_max = vr.upper_bound ();
6719 : }
6720 :
6721 : /* Check whether the unconverted step has an acceptable range. */
6722 5452 : signop sgn = TYPE_SIGN (type);
6723 10904 : if (wi::les_p (minv, widest_int::from (step_min, sgn))
6724 14028 : && wi::ges_p (maxv, widest_int::from (step_max, sgn)))
6725 : {
6726 1553 : if (wi::ge_p (step_min, useful_min, sgn))
6727 440 : return ssize_int (useful_min);
6728 1113 : else if (wi::lt_p (step_max, 0, sgn))
6729 0 : return ssize_int (-1);
6730 : else
6731 1113 : return fold_convert (ssizetype, step);
6732 : }
6733 5452 : }
6734 65929 : return DR_STEP (dr);
6735 : }
6736 :
6737 : /* Return a value that is negative iff DR has a negative step. */
6738 :
6739 : tree
6740 12078 : dr_direction_indicator (struct data_reference *dr)
6741 : {
6742 12078 : return dr_step_indicator (dr, 0);
6743 : }
6744 :
6745 : /* Return a value that is zero iff DR has a zero step. */
6746 :
6747 : tree
6748 55404 : dr_zero_step_indicator (struct data_reference *dr)
6749 : {
6750 55404 : return dr_step_indicator (dr, 1);
6751 : }
6752 :
6753 : /* Return true if DR is known to have a nonnegative (but possibly zero)
6754 : step. */
6755 :
6756 : bool
6757 5083 : dr_known_forward_stride_p (struct data_reference *dr)
6758 : {
6759 5083 : tree indicator = dr_direction_indicator (dr);
6760 5083 : tree neg_step_val = fold_binary (LT_EXPR, boolean_type_node,
6761 : fold_convert (ssizetype, indicator),
6762 : ssize_int (0));
6763 5083 : return neg_step_val && integer_zerop (neg_step_val);
6764 : }
|