Line data Source code
1 : /* Induction variable optimizations.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : /* This pass tries to find the optimal set of induction variables for the loop.
21 : It optimizes just the basic linear induction variables (although adding
22 : support for other types should not be too hard). It includes the
23 : optimizations commonly known as strength reduction, induction variable
24 : coalescing and induction variable elimination. It does it in the
25 : following steps:
26 :
27 : 1) The interesting uses of induction variables are found. This includes
28 :
29 : -- uses of induction variables in non-linear expressions
30 : -- addresses of arrays
31 : -- comparisons of induction variables
32 :
33 : Note the interesting uses are categorized and handled in group.
34 : Generally, address type uses are grouped together if their iv bases
35 : are different in constant offset.
36 :
37 : 2) Candidates for the induction variables are found. This includes
38 :
39 : -- old induction variables
40 : -- the variables defined by expressions derived from the "interesting
41 : groups/uses" above
42 :
43 : 3) The optimal (w.r. to a cost function) set of variables is chosen. The
44 : cost function assigns a cost to sets of induction variables and consists
45 : of three parts:
46 :
47 : -- The group/use costs. Each of the interesting groups/uses chooses
48 : the best induction variable in the set and adds its cost to the sum.
49 : The cost reflects the time spent on modifying the induction variables
50 : value to be usable for the given purpose (adding base and offset for
51 : arrays, etc.).
52 : -- The variable costs. Each of the variables has a cost assigned that
53 : reflects the costs associated with incrementing the value of the
54 : variable. The original variables are somewhat preferred.
55 : -- The set cost. Depending on the size of the set, extra cost may be
56 : added to reflect register pressure.
57 :
58 : All the costs are defined in a machine-specific way, using the target
59 : hooks and machine descriptions to determine them.
60 :
61 : 4) The trees are transformed to use the new variables, the dead code is
62 : removed.
63 :
64 : All of this is done loop by loop. Doing it globally is theoretically
65 : possible, it might give a better performance and it might enable us
66 : to decide costs more precisely, but getting all the interactions right
67 : would be complicated.
68 :
69 : For the targets supporting low-overhead loops, IVOPTs has to take care of
70 : the loops which will probably be transformed in RTL doloop optimization,
71 : to try to make selected IV candidate set optimal. The process of doloop
72 : support includes:
73 :
74 : 1) Analyze the current loop will be transformed to doloop or not, find and
75 : mark its compare type IV use as doloop use (iv_group field doloop_p), and
76 : set flag doloop_use_p of ivopts_data to notify subsequent processings on
77 : doloop. See analyze_and_mark_doloop_use and its callees for the details.
78 : The target hook predict_doloop_p can be used for target specific checks.
79 :
80 : 2) Add one doloop dedicated IV cand {(may_be_zero ? 1 : (niter + 1)), +, -1},
81 : set flag doloop_p of iv_cand, step cost is set as zero and no extra cost
82 : like biv. For cost determination between doloop IV cand and IV use, the
83 : target hooks doloop_cost_for_generic and doloop_cost_for_address are
84 : provided to add on extra costs for generic type and address type IV use.
85 : Zero cost is assigned to the pair between doloop IV cand and doloop IV
86 : use, and bound zero is set for IV elimination.
87 :
88 : 3) With the cost setting in step 2), the current cost model based IV
89 : selection algorithm will process as usual, pick up doloop dedicated IV if
90 : profitable. */
91 :
92 : #include "config.h"
93 : #include "system.h"
94 : #include "coretypes.h"
95 : #include "backend.h"
96 : #include "rtl.h"
97 : #include "tree.h"
98 : #include "gimple.h"
99 : #include "cfghooks.h"
100 : #include "tree-pass.h"
101 : #include "memmodel.h"
102 : #include "tm_p.h"
103 : #include "ssa.h"
104 : #include "expmed.h"
105 : #include "insn-config.h"
106 : #include "emit-rtl.h"
107 : #include "recog.h"
108 : #include "cgraph.h"
109 : #include "gimple-pretty-print.h"
110 : #include "alias.h"
111 : #include "fold-const.h"
112 : #include "stor-layout.h"
113 : #include "tree-eh.h"
114 : #include "gimplify.h"
115 : #include "gimple-iterator.h"
116 : #include "gimplify-me.h"
117 : #include "tree-cfg.h"
118 : #include "tree-ssa-loop-ivopts.h"
119 : #include "tree-ssa-loop-manip.h"
120 : #include "tree-ssa-loop-niter.h"
121 : #include "tree-ssa-loop.h"
122 : #include "explow.h"
123 : #include "expr.h"
124 : #include "tree-dfa.h"
125 : #include "tree-ssa.h"
126 : #include "cfgloop.h"
127 : #include "tree-scalar-evolution.h"
128 : #include "tree-affine.h"
129 : #include "tree-ssa-propagate.h"
130 : #include "tree-ssa-address.h"
131 : #include "builtins.h"
132 : #include "tree-vectorizer.h"
133 : #include "dbgcnt.h"
134 : #include "cfganal.h"
135 : #include "gimple-fold.h"
136 : #include "gimple-range.h"
137 :
138 : /* For lang_hooks.types.type_for_mode. */
139 : #include "langhooks.h"
140 :
141 : /* FIXME: Expressions are expanded to RTL in this pass to determine the
142 : cost of different addressing modes. This should be moved to a TBD
143 : interface between the GIMPLE and RTL worlds. */
144 :
145 : /* The infinite cost. */
146 : #define INFTY 1000000000
147 :
148 : /* Returns the expected number of loop iterations for LOOP.
149 : The average trip count is computed from profile data if it
150 : exists. */
151 :
152 : static inline unsigned HOST_WIDE_INT
153 8834400 : avg_loop_niter (class loop *loop)
154 : {
155 8834400 : HOST_WIDE_INT niter = estimated_stmt_executions_int (loop);
156 8834400 : if (niter == -1)
157 : {
158 4965428 : niter = likely_max_stmt_executions_int (loop);
159 :
160 4965428 : if (niter == -1 || niter > param_avg_loop_niter)
161 4174794 : return param_avg_loop_niter;
162 : }
163 :
164 4659606 : return niter;
165 : }
166 :
167 : struct iv_use;
168 :
169 : /* Representation of the induction variable. */
170 : struct iv
171 : {
172 : tree base; /* Initial value of the iv. */
173 : tree base_object; /* A memory object to that the induction variable points. */
174 : tree step; /* Step of the iv (constant only). */
175 : tree ssa_name; /* The ssa name with the value. */
176 : struct iv_use *nonlin_use; /* The identifier in the use if it is the case. */
177 : bool biv_p; /* Is it a biv? */
178 : bool no_overflow; /* True if the iv doesn't overflow. */
179 : bool have_address_use;/* For biv, indicate if it's used in any address
180 : type use. */
181 : };
182 :
183 : /* Per-ssa version information (induction variable descriptions, etc.). */
184 : struct version_info
185 : {
186 : tree name; /* The ssa name. */
187 : struct iv *iv; /* Induction variable description. */
188 : bool has_nonlin_use; /* For a loop-level invariant, whether it is used in
189 : an expression that is not an induction variable. */
190 : bool preserve_biv; /* For the original biv, whether to preserve it. */
191 : unsigned inv_id; /* Id of an invariant. */
192 : };
193 :
194 : /* Types of uses. */
195 : enum use_type
196 : {
197 : USE_NONLINEAR_EXPR, /* Use in a nonlinear expression. */
198 : USE_REF_ADDRESS, /* Use is an address for an explicit memory
199 : reference. */
200 : USE_PTR_ADDRESS, /* Use is a pointer argument to a function in
201 : cases where the expansion of the function
202 : will turn the argument into a normal address. */
203 : USE_COMPARE /* Use is a compare. */
204 : };
205 :
206 : /* Cost of a computation. */
207 : class comp_cost
208 : {
209 : public:
210 132467538 : comp_cost (): cost (0), complexity (0), scratch (0)
211 : {}
212 :
213 25310885 : comp_cost (int64_t cost, unsigned complexity, int64_t scratch = 0)
214 15441248 : : cost (cost), complexity (complexity), scratch (scratch)
215 : {}
216 :
217 : /* Returns true if COST is infinite. */
218 : bool infinite_cost_p ();
219 :
220 : /* Adds costs COST1 and COST2. */
221 : friend comp_cost operator+ (comp_cost cost1, comp_cost cost2);
222 :
223 : /* Adds COST to the comp_cost. */
224 : comp_cost operator+= (comp_cost cost);
225 :
226 : /* Adds constant C to this comp_cost. */
227 : comp_cost operator+= (HOST_WIDE_INT c);
228 :
229 : /* Subtracts constant C to this comp_cost. */
230 : comp_cost operator-= (HOST_WIDE_INT c);
231 :
232 : /* Divide the comp_cost by constant C. */
233 : comp_cost operator/= (HOST_WIDE_INT c);
234 :
235 : /* Multiply the comp_cost by constant C. */
236 : comp_cost operator*= (HOST_WIDE_INT c);
237 :
238 : /* Subtracts costs COST1 and COST2. */
239 : friend comp_cost operator- (comp_cost cost1, comp_cost cost2);
240 :
241 : /* Subtracts COST from this comp_cost. */
242 : comp_cost operator-= (comp_cost cost);
243 :
244 : /* Returns true if COST1 is smaller than COST2. */
245 : friend bool operator< (comp_cost cost1, comp_cost cost2);
246 :
247 : /* Returns true if COST1 and COST2 are equal. */
248 : friend bool operator== (comp_cost cost1, comp_cost cost2);
249 :
250 : /* Returns true if COST1 is smaller or equal than COST2. */
251 : friend bool operator<= (comp_cost cost1, comp_cost cost2);
252 :
253 : int64_t cost; /* The runtime cost. */
254 : unsigned complexity; /* The estimate of the complexity of the code for
255 : the computation (in no concrete units --
256 : complexity field should be larger for more
257 : complex expressions and addressing modes). */
258 : int64_t scratch; /* Scratch used during cost computation. */
259 : };
260 :
261 : static const comp_cost no_cost;
262 : static const comp_cost infinite_cost (INFTY, 0, INFTY);
263 :
264 : bool
265 1865455872 : comp_cost::infinite_cost_p ()
266 : {
267 1865455872 : return cost == INFTY;
268 : }
269 :
270 : comp_cost
271 248829589 : operator+ (comp_cost cost1, comp_cost cost2)
272 : {
273 248829589 : if (cost1.infinite_cost_p () || cost2.infinite_cost_p ())
274 1886250 : return infinite_cost;
275 :
276 246943339 : gcc_assert (cost1.cost + cost2.cost < infinite_cost.cost);
277 246943339 : cost1.cost += cost2.cost;
278 246943339 : cost1.complexity += cost2.complexity;
279 :
280 246943339 : return cost1;
281 : }
282 :
283 : comp_cost
284 213573458 : operator- (comp_cost cost1, comp_cost cost2)
285 : {
286 213573458 : if (cost1.infinite_cost_p ())
287 0 : return infinite_cost;
288 :
289 213573458 : gcc_assert (!cost2.infinite_cost_p ());
290 213573458 : gcc_assert (cost1.cost - cost2.cost < infinite_cost.cost);
291 :
292 213573458 : cost1.cost -= cost2.cost;
293 213573458 : cost1.complexity -= cost2.complexity;
294 :
295 213573458 : return cost1;
296 : }
297 :
298 : comp_cost
299 248829589 : comp_cost::operator+= (comp_cost cost)
300 : {
301 248829589 : *this = *this + cost;
302 248829589 : return *this;
303 : }
304 :
305 : comp_cost
306 881875824 : comp_cost::operator+= (HOST_WIDE_INT c)
307 : {
308 881875824 : if (c >= INFTY)
309 0 : this->cost = INFTY;
310 :
311 881875824 : if (infinite_cost_p ())
312 0 : return *this;
313 :
314 881875824 : gcc_assert (this->cost + c < infinite_cost.cost);
315 881875824 : this->cost += c;
316 :
317 881875824 : return *this;
318 : }
319 :
320 : comp_cost
321 543097 : comp_cost::operator-= (HOST_WIDE_INT c)
322 : {
323 543097 : if (infinite_cost_p ())
324 0 : return *this;
325 :
326 543097 : gcc_assert (this->cost - c < infinite_cost.cost);
327 543097 : this->cost -= c;
328 :
329 543097 : return *this;
330 : }
331 :
332 : comp_cost
333 0 : comp_cost::operator/= (HOST_WIDE_INT c)
334 : {
335 0 : gcc_assert (c != 0);
336 0 : if (infinite_cost_p ())
337 0 : return *this;
338 :
339 0 : this->cost /= c;
340 :
341 0 : return *this;
342 : }
343 :
344 : comp_cost
345 0 : comp_cost::operator*= (HOST_WIDE_INT c)
346 : {
347 0 : if (infinite_cost_p ())
348 0 : return *this;
349 :
350 0 : gcc_assert (this->cost * c < infinite_cost.cost);
351 0 : this->cost *= c;
352 :
353 0 : return *this;
354 : }
355 :
356 : comp_cost
357 213573458 : comp_cost::operator-= (comp_cost cost)
358 : {
359 213573458 : *this = *this - cost;
360 213573458 : return *this;
361 : }
362 :
363 : bool
364 190852174 : operator< (comp_cost cost1, comp_cost cost2)
365 : {
366 190852174 : if (cost1.cost == cost2.cost)
367 83081356 : return cost1.complexity < cost2.complexity;
368 :
369 107770818 : return cost1.cost < cost2.cost;
370 : }
371 :
372 : bool
373 3919813 : operator== (comp_cost cost1, comp_cost cost2)
374 : {
375 3919813 : return cost1.cost == cost2.cost
376 3919813 : && cost1.complexity == cost2.complexity;
377 : }
378 :
379 : bool
380 6487327 : operator<= (comp_cost cost1, comp_cost cost2)
381 : {
382 6487327 : return cost1 < cost2 || cost1 == cost2;
383 : }
384 :
385 : struct iv_inv_expr_ent;
386 :
387 : /* The candidate - cost pair. */
388 : class cost_pair
389 : {
390 : public:
391 : struct iv_cand *cand; /* The candidate. */
392 : comp_cost cost; /* The cost. */
393 : enum tree_code comp; /* For iv elimination, the comparison. */
394 : bitmap inv_vars; /* The list of invariant ssa_vars that have to be
395 : preserved when representing iv_use with iv_cand. */
396 : bitmap inv_exprs; /* The list of newly created invariant expressions
397 : when representing iv_use with iv_cand. */
398 : tree value; /* For final value elimination, the expression for
399 : the final value of the iv. For iv elimination,
400 : the new bound to compare with. */
401 : };
402 :
403 : /* Use. */
404 : struct iv_use
405 : {
406 : unsigned id; /* The id of the use. */
407 : unsigned group_id; /* The group id the use belongs to. */
408 : enum use_type type; /* Type of the use. */
409 : tree mem_type; /* The memory type to use when testing whether an
410 : address is legitimate, and what the address's
411 : cost is. */
412 : struct iv *iv; /* The induction variable it is based on. */
413 : gimple *stmt; /* Statement in that it occurs. */
414 : tree *op_p; /* The place where it occurs. */
415 :
416 : tree addr_base; /* Base address with const offset stripped. */
417 : poly_uint64 addr_offset;
418 : /* Const offset stripped from base address. */
419 : };
420 :
421 : /* Group of uses. */
422 : struct iv_group
423 : {
424 : /* The id of the group. */
425 : unsigned id;
426 : /* Uses of the group are of the same type. */
427 : enum use_type type;
428 : /* The set of "related" IV candidates, plus the important ones. */
429 : bitmap related_cands;
430 : /* Number of IV candidates in the cost_map. */
431 : unsigned n_map_members;
432 : /* The costs wrto the iv candidates. */
433 : class cost_pair *cost_map;
434 : /* The selected candidate for the group. */
435 : struct iv_cand *selected;
436 : /* To indicate this is a doloop use group. */
437 : bool doloop_p;
438 : /* Uses in the group. */
439 : vec<struct iv_use *> vuses;
440 : };
441 :
442 : /* The position where the iv is computed. */
443 : enum iv_position
444 : {
445 : IP_NORMAL, /* At the end, just before the exit condition. */
446 : IP_END, /* At the end of the latch block. */
447 : IP_BEFORE_USE, /* Immediately before a specific use. */
448 : IP_AFTER_USE, /* Immediately after a specific use. */
449 : IP_ORIGINAL /* The original biv. */
450 : };
451 :
452 : /* The induction variable candidate. */
453 : struct iv_cand
454 : {
455 : unsigned id; /* The number of the candidate. */
456 : bool important; /* Whether this is an "important" candidate, i.e. such
457 : that it should be considered by all uses. */
458 : bool involves_undefs; /* Whether the IV involves undefined values. */
459 : enum iv_position pos : 8;/* Where it is computed. */
460 : gimple *incremented_at;/* For original biv, the statement where it is
461 : incremented. */
462 : tree var_before; /* The variable used for it before increment. */
463 : tree var_after; /* The variable used for it after increment. */
464 : struct iv *iv; /* The value of the candidate. NULL for
465 : "pseudocandidate" used to indicate the possibility
466 : to replace the final value of an iv by direct
467 : computation of the value. */
468 : unsigned cost; /* Cost of the candidate. */
469 : unsigned cost_step; /* Cost of the candidate's increment operation. */
470 : struct iv_use *ainc_use; /* For IP_{BEFORE,AFTER}_USE candidates, the place
471 : where it is incremented. */
472 : bitmap inv_vars; /* The list of invariant ssa_vars used in step of the
473 : iv_cand. */
474 : bitmap inv_exprs; /* If step is more complicated than a single ssa_var,
475 : handle it as a new invariant expression which will
476 : be hoisted out of loop. */
477 : struct iv *orig_iv; /* The original iv if this cand is added from biv with
478 : smaller type. */
479 : bool doloop_p; /* Whether this is a doloop candidate. */
480 : };
481 :
482 : /* Hashtable entry for common candidate derived from iv uses. */
483 2641902 : class iv_common_cand
484 : {
485 : public:
486 : tree base;
487 : tree step;
488 : /* IV uses from which this common candidate is derived. */
489 : auto_vec<struct iv_use *> uses;
490 : hashval_t hash;
491 : };
492 :
493 : /* Hashtable helpers. */
494 :
495 : struct iv_common_cand_hasher : delete_ptr_hash <iv_common_cand>
496 : {
497 : static inline hashval_t hash (const iv_common_cand *);
498 : static inline bool equal (const iv_common_cand *, const iv_common_cand *);
499 : };
500 :
501 : /* Hash function for possible common candidates. */
502 :
503 : inline hashval_t
504 9704069 : iv_common_cand_hasher::hash (const iv_common_cand *ccand)
505 : {
506 9704069 : return ccand->hash;
507 : }
508 :
509 : /* Hash table equality function for common candidates. */
510 :
511 : inline bool
512 10966455 : iv_common_cand_hasher::equal (const iv_common_cand *ccand1,
513 : const iv_common_cand *ccand2)
514 : {
515 10966455 : return (ccand1->hash == ccand2->hash
516 1631424 : && operand_equal_p (ccand1->base, ccand2->base, 0)
517 1612788 : && operand_equal_p (ccand1->step, ccand2->step, 0)
518 12574404 : && (TYPE_PRECISION (TREE_TYPE (ccand1->base))
519 1607949 : == TYPE_PRECISION (TREE_TYPE (ccand2->base))));
520 : }
521 :
522 : /* Loop invariant expression hashtable entry. */
523 :
524 : struct iv_inv_expr_ent
525 : {
526 : /* Tree expression of the entry. */
527 : tree expr;
528 : /* Unique identifier. */
529 : int id;
530 : /* Hash value. */
531 : hashval_t hash;
532 : };
533 :
534 : /* Sort iv_inv_expr_ent pair A and B by id field. */
535 :
536 : static int
537 5813 : sort_iv_inv_expr_ent (const void *a, const void *b)
538 : {
539 5813 : const iv_inv_expr_ent * const *e1 = (const iv_inv_expr_ent * const *) (a);
540 5813 : const iv_inv_expr_ent * const *e2 = (const iv_inv_expr_ent * const *) (b);
541 :
542 5813 : unsigned id1 = (*e1)->id;
543 5813 : unsigned id2 = (*e2)->id;
544 :
545 5813 : if (id1 < id2)
546 : return -1;
547 2700 : else if (id1 > id2)
548 : return 1;
549 : else
550 0 : return 0;
551 : }
552 :
553 : /* Hashtable helpers. */
554 :
555 : struct iv_inv_expr_hasher : free_ptr_hash <iv_inv_expr_ent>
556 : {
557 : static inline hashval_t hash (const iv_inv_expr_ent *);
558 : static inline bool equal (const iv_inv_expr_ent *, const iv_inv_expr_ent *);
559 : };
560 :
561 : /* Return true if uses of type TYPE represent some form of address. */
562 :
563 : inline bool
564 9060698 : address_p (use_type type)
565 : {
566 9060698 : return type == USE_REF_ADDRESS || type == USE_PTR_ADDRESS;
567 : }
568 :
569 : /* Hash function for loop invariant expressions. */
570 :
571 : inline hashval_t
572 6497448 : iv_inv_expr_hasher::hash (const iv_inv_expr_ent *expr)
573 : {
574 6497448 : return expr->hash;
575 : }
576 :
577 : /* Hash table equality function for expressions. */
578 :
579 : inline bool
580 7800031 : iv_inv_expr_hasher::equal (const iv_inv_expr_ent *expr1,
581 : const iv_inv_expr_ent *expr2)
582 : {
583 7800031 : return expr1->hash == expr2->hash
584 7800031 : && operand_equal_p (expr1->expr, expr2->expr, 0);
585 : }
586 :
587 : struct ivopts_data
588 : {
589 : /* The currently optimized loop. */
590 : class loop *current_loop;
591 : location_t loop_loc;
592 :
593 : /* Numbers of iterations for all exits of the current loop. */
594 : hash_map<edge, tree_niter_desc *> *niters;
595 :
596 : /* Number of registers used in it. */
597 : unsigned regs_used;
598 :
599 : /* The size of version_info array allocated. */
600 : unsigned version_info_size;
601 :
602 : /* The array of information for the ssa names. */
603 : struct version_info *version_info;
604 :
605 : /* The hashtable of loop invariant expressions created
606 : by ivopt. */
607 : hash_table<iv_inv_expr_hasher> *inv_expr_tab;
608 :
609 : /* The bitmap of indices in version_info whose value was changed. */
610 : bitmap relevant;
611 :
612 : /* The uses of induction variables. */
613 : vec<iv_group *> vgroups;
614 :
615 : /* The candidates. */
616 : vec<iv_cand *> vcands;
617 :
618 : /* A bitmap of important candidates. */
619 : bitmap important_candidates;
620 :
621 : /* Cache used by tree_to_aff_combination_expand. */
622 : hash_map<tree, name_expansion *> *name_expansion_cache;
623 :
624 : /* The hashtable of common candidates derived from iv uses. */
625 : hash_table<iv_common_cand_hasher> *iv_common_cand_tab;
626 :
627 : /* The common candidates. */
628 : vec<iv_common_cand *> iv_common_cands;
629 :
630 : /* Hash map recording base object information of tree exp. */
631 : hash_map<tree, tree> *base_object_map;
632 :
633 : /* The maximum invariant variable id. */
634 : unsigned max_inv_var_id;
635 :
636 : /* The maximum invariant expression id. */
637 : unsigned max_inv_expr_id;
638 :
639 : /* Number of no_overflow BIVs which are not used in memory address. */
640 : unsigned bivs_not_used_in_addr;
641 :
642 : /* Obstack for iv structure. */
643 : struct obstack iv_obstack;
644 :
645 : /* Whether to consider just related and important candidates when replacing a
646 : use. */
647 : bool consider_all_candidates;
648 :
649 : /* Are we optimizing for speed? */
650 : bool speed;
651 :
652 : /* Whether the loop body includes any function calls. */
653 : bool body_includes_call;
654 :
655 : /* Whether the loop body can only be exited via single exit. */
656 : bool loop_single_exit_p;
657 :
658 : /* Whether the loop has doloop comparison use. */
659 : bool doloop_use_p;
660 : };
661 :
662 : /* An assignment of iv candidates to uses. */
663 :
664 : class iv_ca
665 : {
666 : public:
667 : /* The number of uses covered by the assignment. */
668 : unsigned upto;
669 :
670 : /* Number of uses that cannot be expressed by the candidates in the set. */
671 : unsigned bad_groups;
672 :
673 : /* Candidate assigned to a use, together with the related costs. */
674 : class cost_pair **cand_for_group;
675 :
676 : /* Number of times each candidate is used. */
677 : unsigned *n_cand_uses;
678 :
679 : /* The candidates used. */
680 : bitmap cands;
681 :
682 : /* The number of candidates in the set. */
683 : unsigned n_cands;
684 :
685 : /* The number of invariants needed, including both invariant variants and
686 : invariant expressions. */
687 : unsigned n_invs;
688 :
689 : /* Total cost of expressing uses. */
690 : comp_cost cand_use_cost;
691 :
692 : /* Total cost of candidates. */
693 : int64_t cand_cost;
694 :
695 : /* Number of times each invariant variable is used. */
696 : unsigned *n_inv_var_uses;
697 :
698 : /* Number of times each invariant expression is used. */
699 : unsigned *n_inv_expr_uses;
700 :
701 : /* Total cost of the assignment. */
702 : comp_cost cost;
703 : };
704 :
705 : /* Difference of two iv candidate assignments. */
706 :
707 : struct iv_ca_delta
708 : {
709 : /* Changed group. */
710 : struct iv_group *group;
711 :
712 : /* An old assignment (for rollback purposes). */
713 : class cost_pair *old_cp;
714 :
715 : /* A new assignment. */
716 : class cost_pair *new_cp;
717 :
718 : /* Next change in the list. */
719 : struct iv_ca_delta *next;
720 : };
721 :
722 : /* Bound on number of candidates below that all candidates are considered. */
723 :
724 : #define CONSIDER_ALL_CANDIDATES_BOUND \
725 : ((unsigned) param_iv_consider_all_candidates_bound)
726 :
727 : /* If there are more iv occurrences, we just give up (it is quite unlikely that
728 : optimizing such a loop would help, and it would take ages). */
729 :
730 : #define MAX_CONSIDERED_GROUPS \
731 : ((unsigned) param_iv_max_considered_uses)
732 :
733 : /* If there are at most this number of ivs in the set, try removing unnecessary
734 : ivs from the set always. */
735 :
736 : #define ALWAYS_PRUNE_CAND_SET_BOUND \
737 : ((unsigned) param_iv_always_prune_cand_set_bound)
738 :
739 : /* The list of trees for that the decl_rtl field must be reset is stored
740 : here. */
741 :
742 : static vec<tree> decl_rtl_to_reset;
743 :
744 : static comp_cost force_expr_to_var_cost (tree, bool);
745 :
746 : /* The single loop exit if it dominates the latch, NULL otherwise. */
747 :
748 : edge
749 713154 : single_dom_exit (class loop *loop)
750 : {
751 713154 : edge exit = single_exit (loop);
752 :
753 713154 : if (!exit)
754 : return NULL;
755 :
756 471370 : if (!just_once_each_iteration_p (loop, exit->src))
757 4976 : return NULL;
758 :
759 : return exit;
760 : }
761 :
762 : /* Dumps information about the induction variable IV to FILE. Don't dump
763 : variable's name if DUMP_NAME is FALSE. The information is dumped with
764 : preceding spaces indicated by INDENT_LEVEL. */
765 :
766 : void
767 1649 : dump_iv (FILE *file, struct iv *iv, bool dump_name, unsigned indent_level)
768 : {
769 1649 : const char *p;
770 1649 : const char spaces[9] = {' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '\0'};
771 :
772 1649 : if (indent_level > 4)
773 : indent_level = 4;
774 1649 : p = spaces + 8 - (indent_level << 1);
775 :
776 1649 : fprintf (file, "%sIV struct:\n", p);
777 1649 : if (iv->ssa_name && dump_name)
778 : {
779 562 : fprintf (file, "%s SSA_NAME:\t", p);
780 562 : print_generic_expr (file, iv->ssa_name, TDF_SLIM);
781 562 : fprintf (file, "\n");
782 : }
783 :
784 1649 : fprintf (file, "%s Type:\t", p);
785 1649 : print_generic_expr (file, TREE_TYPE (iv->base), TDF_SLIM);
786 1649 : fprintf (file, "\n");
787 :
788 1649 : fprintf (file, "%s Base:\t", p);
789 1649 : print_generic_expr (file, iv->base, TDF_SLIM);
790 1649 : fprintf (file, "\n");
791 :
792 1649 : fprintf (file, "%s Step:\t", p);
793 1649 : print_generic_expr (file, iv->step, TDF_SLIM);
794 1649 : fprintf (file, "\n");
795 :
796 1649 : if (iv->base_object)
797 : {
798 518 : fprintf (file, "%s Object:\t", p);
799 518 : print_generic_expr (file, iv->base_object, TDF_SLIM);
800 518 : fprintf (file, "\n");
801 : }
802 :
803 2982 : fprintf (file, "%s Biv:\t%c\n", p, iv->biv_p ? 'Y' : 'N');
804 :
805 1649 : fprintf (file, "%s Overflowness wrto loop niter:\t%s\n",
806 1649 : p, iv->no_overflow ? "No-overflow" : "Overflow");
807 1649 : }
808 :
809 : /* Dumps information about the USE to FILE. */
810 :
811 : void
812 258 : dump_use (FILE *file, struct iv_use *use)
813 : {
814 258 : fprintf (file, " Use %d.%d:\n", use->group_id, use->id);
815 258 : fprintf (file, " At stmt:\t");
816 258 : print_gimple_stmt (file, use->stmt, 0);
817 258 : fprintf (file, " At pos:\t");
818 258 : if (use->op_p)
819 167 : print_generic_expr (file, *use->op_p, TDF_SLIM);
820 258 : fprintf (file, "\n");
821 258 : dump_iv (file, use->iv, false, 2);
822 258 : }
823 :
824 : /* Dumps information about the uses to FILE. */
825 :
826 : void
827 70 : dump_groups (FILE *file, struct ivopts_data *data)
828 : {
829 70 : unsigned i, j;
830 70 : struct iv_group *group;
831 :
832 297 : for (i = 0; i < data->vgroups.length (); i++)
833 : {
834 227 : group = data->vgroups[i];
835 227 : fprintf (file, "Group %d:\n", group->id);
836 227 : if (group->type == USE_NONLINEAR_EXPR)
837 91 : fprintf (file, " Type:\tGENERIC\n");
838 136 : else if (group->type == USE_REF_ADDRESS)
839 59 : fprintf (file, " Type:\tREFERENCE ADDRESS\n");
840 77 : else if (group->type == USE_PTR_ADDRESS)
841 0 : fprintf (file, " Type:\tPOINTER ARGUMENT ADDRESS\n");
842 : else
843 : {
844 77 : gcc_assert (group->type == USE_COMPARE);
845 77 : fprintf (file, " Type:\tCOMPARE\n");
846 : }
847 485 : for (j = 0; j < group->vuses.length (); j++)
848 258 : dump_use (file, group->vuses[j]);
849 : }
850 70 : }
851 :
852 : /* Dumps information about induction variable candidate CAND to FILE. */
853 :
854 : void
855 829 : dump_cand (FILE *file, struct iv_cand *cand)
856 : {
857 829 : struct iv *iv = cand->iv;
858 :
859 829 : fprintf (file, "Candidate %d:\n", cand->id);
860 829 : if (cand->inv_vars)
861 : {
862 26 : fprintf (file, " Depend on inv.vars: ");
863 26 : dump_bitmap (file, cand->inv_vars);
864 : }
865 829 : if (cand->inv_exprs)
866 : {
867 0 : fprintf (file, " Depend on inv.exprs: ");
868 0 : dump_bitmap (file, cand->inv_exprs);
869 : }
870 :
871 829 : if (cand->var_before)
872 : {
873 716 : fprintf (file, " Var before: ");
874 716 : print_generic_expr (file, cand->var_before, TDF_SLIM);
875 716 : fprintf (file, "\n");
876 : }
877 829 : if (cand->var_after)
878 : {
879 716 : fprintf (file, " Var after: ");
880 716 : print_generic_expr (file, cand->var_after, TDF_SLIM);
881 716 : fprintf (file, "\n");
882 : }
883 :
884 829 : switch (cand->pos)
885 : {
886 682 : case IP_NORMAL:
887 682 : fprintf (file, " Incr POS: before exit test\n");
888 682 : break;
889 :
890 0 : case IP_BEFORE_USE:
891 0 : fprintf (file, " Incr POS: before use %d\n", cand->ainc_use->id);
892 0 : break;
893 :
894 0 : case IP_AFTER_USE:
895 0 : fprintf (file, " Incr POS: after use %d\n", cand->ainc_use->id);
896 0 : break;
897 :
898 0 : case IP_END:
899 0 : fprintf (file, " Incr POS: at end\n");
900 0 : break;
901 :
902 147 : case IP_ORIGINAL:
903 147 : fprintf (file, " Incr POS: orig biv\n");
904 147 : break;
905 : }
906 :
907 829 : dump_iv (file, iv, false, 1);
908 829 : }
909 :
910 : /* Returns the info for ssa version VER. */
911 :
912 : static inline struct version_info *
913 118930454 : ver_info (struct ivopts_data *data, unsigned ver)
914 : {
915 118930454 : return data->version_info + ver;
916 : }
917 :
918 : /* Returns the info for ssa name NAME. */
919 :
920 : static inline struct version_info *
921 96292092 : name_info (struct ivopts_data *data, tree name)
922 : {
923 96292092 : return ver_info (data, SSA_NAME_VERSION (name));
924 : }
925 :
926 : /* Returns true if STMT is after the place where the IP_NORMAL ivs will be
927 : emitted in LOOP. */
928 :
929 : static bool
930 33837711 : stmt_after_ip_normal_pos (class loop *loop, gimple *stmt)
931 : {
932 33837711 : basic_block bb = ip_normal_pos (loop), sbb = gimple_bb (stmt);
933 :
934 33837711 : gcc_assert (bb);
935 :
936 33837711 : if (sbb == loop->latch)
937 : return true;
938 :
939 33724830 : if (sbb != bb)
940 : return false;
941 :
942 19946631 : return stmt == last_nondebug_stmt (bb);
943 : }
944 :
945 : /* Returns true if STMT if after the place where the original induction
946 : variable CAND is incremented. If TRUE_IF_EQUAL is set, we return true
947 : if the positions are identical. */
948 :
949 : static bool
950 8010739 : stmt_after_inc_pos (struct iv_cand *cand, gimple *stmt, bool true_if_equal)
951 : {
952 8010739 : basic_block cand_bb = gimple_bb (cand->incremented_at);
953 8010739 : basic_block stmt_bb = gimple_bb (stmt);
954 :
955 8010739 : if (!dominated_by_p (CDI_DOMINATORS, stmt_bb, cand_bb))
956 : return false;
957 :
958 5518593 : if (stmt_bb != cand_bb)
959 : return true;
960 :
961 5282301 : if (true_if_equal
962 5282301 : && gimple_uid (stmt) == gimple_uid (cand->incremented_at))
963 : return true;
964 5276085 : return gimple_uid (stmt) > gimple_uid (cand->incremented_at);
965 : }
966 :
967 : /* Returns true if STMT if after the place where the induction variable
968 : CAND is incremented in LOOP. */
969 :
970 : static bool
971 43034417 : stmt_after_increment (class loop *loop, struct iv_cand *cand, gimple *stmt)
972 : {
973 43034417 : switch (cand->pos)
974 : {
975 : case IP_END:
976 : return false;
977 :
978 33837711 : case IP_NORMAL:
979 33837711 : return stmt_after_ip_normal_pos (loop, stmt);
980 :
981 8001395 : case IP_ORIGINAL:
982 8001395 : case IP_AFTER_USE:
983 8001395 : return stmt_after_inc_pos (cand, stmt, false);
984 :
985 9344 : case IP_BEFORE_USE:
986 9344 : return stmt_after_inc_pos (cand, stmt, true);
987 :
988 0 : default:
989 0 : gcc_unreachable ();
990 : }
991 : }
992 :
993 : /* walk_tree callback for contains_abnormal_ssa_name_p. */
994 :
995 : static tree
996 15028912 : contains_abnormal_ssa_name_p_1 (tree *tp, int *walk_subtrees, void *)
997 : {
998 15028912 : if (TREE_CODE (*tp) == SSA_NAME
999 15028912 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (*tp))
1000 : return *tp;
1001 :
1002 15028896 : if (!EXPR_P (*tp))
1003 10270654 : *walk_subtrees = 0;
1004 :
1005 : return NULL_TREE;
1006 : }
1007 :
1008 : /* Returns true if EXPR contains a ssa name that occurs in an
1009 : abnormal phi node. */
1010 :
1011 : bool
1012 8061306 : contains_abnormal_ssa_name_p (tree expr)
1013 : {
1014 8061306 : return walk_tree_without_duplicates
1015 8061306 : (&expr, contains_abnormal_ssa_name_p_1, NULL) != NULL_TREE;
1016 : }
1017 :
1018 : /* Returns the structure describing number of iterations determined from
1019 : EXIT of DATA->current_loop, or NULL if something goes wrong. */
1020 :
1021 : static class tree_niter_desc *
1022 4422613 : niter_for_exit (struct ivopts_data *data, edge exit)
1023 : {
1024 4422613 : class tree_niter_desc *desc;
1025 4422613 : tree_niter_desc **slot;
1026 :
1027 4422613 : if (!data->niters)
1028 : {
1029 479425 : data->niters = new hash_map<edge, tree_niter_desc *>;
1030 479425 : slot = NULL;
1031 : }
1032 : else
1033 3943188 : slot = data->niters->get (exit);
1034 :
1035 4422613 : if (!slot)
1036 : {
1037 : /* Try to determine number of iterations. We cannot safely work with ssa
1038 : names that appear in phi nodes on abnormal edges, so that we do not
1039 : create overlapping life ranges for them (PR 27283). */
1040 491590 : desc = XNEW (class tree_niter_desc);
1041 491590 : ::new (static_cast<void*> (desc)) tree_niter_desc ();
1042 491590 : if (!number_of_iterations_exit (data->current_loop,
1043 : exit, desc, true)
1044 491590 : || contains_abnormal_ssa_name_p (desc->niter))
1045 : {
1046 40252 : desc->~tree_niter_desc ();
1047 40252 : XDELETE (desc);
1048 40252 : desc = NULL;
1049 : }
1050 491590 : data->niters->put (exit, desc);
1051 : }
1052 : else
1053 3931023 : desc = *slot;
1054 :
1055 4422613 : return desc;
1056 : }
1057 :
1058 : /* Returns the structure describing number of iterations determined from
1059 : single dominating exit of DATA->current_loop, or NULL if something
1060 : goes wrong. */
1061 :
1062 : static class tree_niter_desc *
1063 70 : niter_for_single_dom_exit (struct ivopts_data *data)
1064 : {
1065 70 : edge exit = single_dom_exit (data->current_loop);
1066 :
1067 70 : if (!exit)
1068 : return NULL;
1069 :
1070 60 : return niter_for_exit (data, exit);
1071 : }
1072 :
1073 : /* Initializes data structures used by the iv optimization pass, stored
1074 : in DATA. */
1075 :
1076 : static void
1077 247663 : tree_ssa_iv_optimize_init (struct ivopts_data *data)
1078 : {
1079 247663 : data->version_info_size = 2 * num_ssa_names;
1080 247663 : data->version_info = XCNEWVEC (struct version_info, data->version_info_size);
1081 247663 : data->relevant = BITMAP_ALLOC (NULL);
1082 247663 : data->important_candidates = BITMAP_ALLOC (NULL);
1083 247663 : data->max_inv_var_id = 0;
1084 247663 : data->max_inv_expr_id = 0;
1085 247663 : data->niters = NULL;
1086 247663 : data->vgroups.create (20);
1087 247663 : data->vcands.create (20);
1088 247663 : data->inv_expr_tab = new hash_table<iv_inv_expr_hasher> (10);
1089 247663 : data->name_expansion_cache = NULL;
1090 247663 : data->base_object_map = NULL;
1091 247663 : data->iv_common_cand_tab = new hash_table<iv_common_cand_hasher> (10);
1092 247663 : data->iv_common_cands.create (20);
1093 247663 : decl_rtl_to_reset.create (20);
1094 247663 : gcc_obstack_init (&data->iv_obstack);
1095 247663 : }
1096 :
1097 : /* walk_tree callback for determine_base_object. */
1098 :
1099 : static tree
1100 19664475 : determine_base_object_1 (tree *tp, int *walk_subtrees, void *wdata)
1101 : {
1102 19664475 : tree_code code = TREE_CODE (*tp);
1103 19664475 : tree obj = NULL_TREE;
1104 19664475 : if (code == ADDR_EXPR)
1105 : {
1106 1058796 : tree base = get_base_address (TREE_OPERAND (*tp, 0));
1107 1058796 : if (!base)
1108 0 : obj = *tp;
1109 1058796 : else if (TREE_CODE (base) != MEM_REF)
1110 1058772 : obj = fold_convert (ptr_type_node, build_fold_addr_expr (base));
1111 : }
1112 18605679 : else if (code == SSA_NAME && POINTER_TYPE_P (TREE_TYPE (*tp)))
1113 1957901 : obj = fold_convert (ptr_type_node, *tp);
1114 :
1115 3016673 : if (!obj)
1116 : {
1117 16647802 : if (!EXPR_P (*tp))
1118 7278366 : *walk_subtrees = 0;
1119 :
1120 : return NULL_TREE;
1121 : }
1122 : /* Record special node for multiple base objects and stop. */
1123 3016673 : if (*static_cast<tree *> (wdata))
1124 : {
1125 4283 : *static_cast<tree *> (wdata) = integer_zero_node;
1126 4283 : return integer_zero_node;
1127 : }
1128 : /* Record the base object and continue looking. */
1129 3012390 : *static_cast<tree *> (wdata) = obj;
1130 3012390 : return NULL_TREE;
1131 : }
1132 :
1133 : /* Returns a memory object to that EXPR points with caching. Return NULL if we
1134 : are able to determine that it does not point to any such object; specially
1135 : return integer_zero_node if EXPR contains multiple base objects. */
1136 :
1137 : static tree
1138 10549732 : determine_base_object (struct ivopts_data *data, tree expr)
1139 : {
1140 10549732 : tree *slot, obj = NULL_TREE;
1141 10549732 : if (data->base_object_map)
1142 : {
1143 10382611 : if ((slot = data->base_object_map->get(expr)) != NULL)
1144 4789528 : return *slot;
1145 : }
1146 : else
1147 167121 : data->base_object_map = new hash_map<tree, tree>;
1148 :
1149 5760204 : (void) walk_tree_without_duplicates (&expr, determine_base_object_1, &obj);
1150 5760204 : data->base_object_map->put (expr, obj);
1151 5760204 : return obj;
1152 : }
1153 :
1154 : /* Allocates an induction variable with given initial value BASE and step STEP
1155 : for loop LOOP. NO_OVERFLOW implies the iv doesn't overflow. */
1156 :
1157 : static struct iv *
1158 10549732 : alloc_iv (struct ivopts_data *data, tree base, tree step,
1159 : bool no_overflow = false)
1160 : {
1161 10549732 : tree expr = base;
1162 10549732 : struct iv *iv = (struct iv*) obstack_alloc (&data->iv_obstack,
1163 : sizeof (struct iv));
1164 10549732 : gcc_assert (step != NULL_TREE);
1165 :
1166 : /* Canonicalize the address expression in base if it were an unsigned
1167 : computation. That leads to more equalities being detected and results in:
1168 :
1169 : 1) More accurate cost can be computed for address expressions;
1170 : 2) Duplicate candidates won't be created for bases in different
1171 : forms, like &a[0] and &a.
1172 : 3) Duplicate candidates won't be created for IV expressions that differ
1173 : only in their sign. */
1174 10549732 : aff_tree comb;
1175 10549732 : STRIP_NOPS (expr);
1176 10549732 : expr = fold_convert (unsigned_type_for (TREE_TYPE (expr)), expr);
1177 10549732 : tree_to_aff_combination (expr, TREE_TYPE (expr), &comb);
1178 10549732 : base = fold_convert (TREE_TYPE (base), aff_combination_to_tree (&comb));
1179 :
1180 10549732 : iv->base = base;
1181 10549732 : iv->base_object = determine_base_object (data, base);
1182 10549732 : iv->step = step;
1183 10549732 : iv->biv_p = false;
1184 10549732 : iv->nonlin_use = NULL;
1185 10549732 : iv->ssa_name = NULL_TREE;
1186 10549732 : if (!no_overflow
1187 10549732 : && !iv_can_overflow_p (data->current_loop, TREE_TYPE (base),
1188 : base, step))
1189 : no_overflow = true;
1190 10549732 : iv->no_overflow = no_overflow;
1191 10549732 : iv->have_address_use = false;
1192 :
1193 21099464 : return iv;
1194 10549732 : }
1195 :
1196 : /* Sets STEP and BASE for induction variable IV. NO_OVERFLOW implies the IV
1197 : doesn't overflow. */
1198 :
1199 : static void
1200 4994771 : set_iv (struct ivopts_data *data, tree iv, tree base, tree step,
1201 : bool no_overflow)
1202 : {
1203 4994771 : struct version_info *info = name_info (data, iv);
1204 :
1205 4994771 : gcc_assert (!info->iv);
1206 :
1207 4994771 : bitmap_set_bit (data->relevant, SSA_NAME_VERSION (iv));
1208 4994771 : info->iv = alloc_iv (data, base, step, no_overflow);
1209 4994771 : info->iv->ssa_name = iv;
1210 4994771 : }
1211 :
1212 : /* Finds induction variable declaration for VAR. */
1213 :
1214 : static struct iv *
1215 45169093 : get_iv (struct ivopts_data *data, tree var)
1216 : {
1217 45169093 : basic_block bb;
1218 45169093 : tree type = TREE_TYPE (var);
1219 :
1220 45169093 : if (!POINTER_TYPE_P (type)
1221 35697017 : && !INTEGRAL_TYPE_P (type))
1222 : return NULL;
1223 :
1224 39390999 : if (!name_info (data, var)->iv)
1225 : {
1226 18289485 : bb = gimple_bb (SSA_NAME_DEF_STMT (var));
1227 :
1228 18289485 : if (!bb
1229 18289485 : || !flow_bb_inside_loop_p (data->current_loop, bb))
1230 : {
1231 818672 : if (POINTER_TYPE_P (type))
1232 328802 : type = sizetype;
1233 818672 : set_iv (data, var, var, build_int_cst (type, 0), true);
1234 : }
1235 : }
1236 :
1237 39390999 : return name_info (data, var)->iv;
1238 : }
1239 :
1240 : /* Return the first non-invariant ssa var found in EXPR. */
1241 :
1242 : static tree
1243 4150185 : extract_single_var_from_expr (tree expr)
1244 : {
1245 4150185 : int i, n;
1246 4150185 : tree tmp;
1247 4150185 : enum tree_code code;
1248 :
1249 4150185 : if (!expr || is_gimple_min_invariant (expr))
1250 : return NULL;
1251 :
1252 674194 : code = TREE_CODE (expr);
1253 674194 : if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code)))
1254 : {
1255 364445 : n = TREE_OPERAND_LENGTH (expr);
1256 728937 : for (i = 0; i < n; i++)
1257 : {
1258 364492 : tmp = extract_single_var_from_expr (TREE_OPERAND (expr, i));
1259 :
1260 364492 : if (tmp)
1261 : return tmp;
1262 : }
1263 : }
1264 309749 : return (TREE_CODE (expr) == SSA_NAME) ? expr : NULL;
1265 : }
1266 :
1267 : /* Finds basic ivs. */
1268 :
1269 : static bool
1270 641573 : find_bivs (struct ivopts_data *data)
1271 : {
1272 641573 : gphi *phi;
1273 641573 : affine_iv iv;
1274 641573 : tree step, type, base, stop;
1275 641573 : bool found = false;
1276 641573 : class loop *loop = data->current_loop;
1277 641573 : gphi_iterator psi;
1278 :
1279 2392519 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
1280 : {
1281 1750946 : phi = psi.phi ();
1282 :
1283 1750946 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
1284 248 : continue;
1285 :
1286 1750698 : if (virtual_operand_p (PHI_RESULT (phi)))
1287 419807 : continue;
1288 :
1289 1330891 : if (!simple_iv (loop, loop, PHI_RESULT (phi), &iv, true))
1290 446351 : continue;
1291 :
1292 884540 : if (integer_zerop (iv.step))
1293 0 : continue;
1294 :
1295 884540 : step = iv.step;
1296 884540 : base = PHI_ARG_DEF_FROM_EDGE (phi, loop_preheader_edge (loop));
1297 : /* Stop expanding iv base at the first ssa var referred by iv step.
1298 : Ideally we should stop at any ssa var, because that's expensive
1299 : and unusual to happen, we just do it on the first one.
1300 :
1301 : See PR64705 for the rationale. */
1302 884540 : stop = extract_single_var_from_expr (step);
1303 884540 : base = expand_simple_operations (base, stop);
1304 884540 : if (contains_abnormal_ssa_name_p (base)
1305 884540 : || contains_abnormal_ssa_name_p (step))
1306 11 : continue;
1307 :
1308 884529 : type = TREE_TYPE (PHI_RESULT (phi));
1309 884529 : base = fold_convert (type, base);
1310 884529 : if (step)
1311 : {
1312 884529 : if (POINTER_TYPE_P (type))
1313 169297 : step = convert_to_ptrofftype (step);
1314 : else
1315 715232 : step = fold_convert (type, step);
1316 : }
1317 :
1318 884529 : set_iv (data, PHI_RESULT (phi), base, step, iv.no_overflow);
1319 884529 : found = true;
1320 : }
1321 :
1322 641573 : return found;
1323 : }
1324 :
1325 : /* Marks basic ivs. */
1326 :
1327 : static void
1328 510605 : mark_bivs (struct ivopts_data *data)
1329 : {
1330 510605 : gphi *phi;
1331 510605 : gimple *def;
1332 510605 : tree var;
1333 510605 : struct iv *iv, *incr_iv;
1334 510605 : class loop *loop = data->current_loop;
1335 510605 : basic_block incr_bb;
1336 510605 : gphi_iterator psi;
1337 :
1338 510605 : data->bivs_not_used_in_addr = 0;
1339 1986442 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
1340 : {
1341 1475837 : phi = psi.phi ();
1342 :
1343 1475837 : iv = get_iv (data, PHI_RESULT (phi));
1344 1475837 : if (!iv)
1345 591308 : continue;
1346 :
1347 884529 : var = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
1348 884529 : def = SSA_NAME_DEF_STMT (var);
1349 : /* Don't mark iv peeled from other one as biv. */
1350 884652 : if (def
1351 884529 : && gimple_code (def) == GIMPLE_PHI
1352 885891 : && gimple_bb (def) == loop->header)
1353 123 : continue;
1354 :
1355 884406 : incr_iv = get_iv (data, var);
1356 884406 : if (!incr_iv)
1357 1250 : continue;
1358 :
1359 : /* If the increment is in the subloop, ignore it. */
1360 883156 : incr_bb = gimple_bb (SSA_NAME_DEF_STMT (var));
1361 883156 : if (incr_bb->loop_father != data->current_loop
1362 883156 : || (incr_bb->flags & BB_IRREDUCIBLE_LOOP))
1363 0 : continue;
1364 :
1365 883156 : iv->biv_p = true;
1366 883156 : incr_iv->biv_p = true;
1367 883156 : if (iv->no_overflow)
1368 590573 : data->bivs_not_used_in_addr++;
1369 883156 : if (incr_iv->no_overflow)
1370 581953 : data->bivs_not_used_in_addr++;
1371 : }
1372 510605 : }
1373 :
1374 : /* Checks whether STMT defines a linear induction variable and stores its
1375 : parameters to IV. */
1376 :
1377 : static bool
1378 12795434 : find_givs_in_stmt_scev (struct ivopts_data *data, gimple *stmt, affine_iv *iv)
1379 : {
1380 12795434 : tree lhs, stop;
1381 12795434 : class loop *loop = data->current_loop;
1382 :
1383 12795434 : iv->base = NULL_TREE;
1384 12795434 : iv->step = NULL_TREE;
1385 :
1386 12795434 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1387 : return false;
1388 :
1389 10705106 : lhs = gimple_assign_lhs (stmt);
1390 10705106 : if (TREE_CODE (lhs) != SSA_NAME)
1391 : return false;
1392 :
1393 19139752 : if (!simple_iv (loop, loop_containing_stmt (stmt), lhs, iv, true))
1394 : return false;
1395 :
1396 : /* Stop expanding iv base at the first ssa var referred by iv step.
1397 : Ideally we should stop at any ssa var, because that's expensive
1398 : and unusual to happen, we just do it on the first one.
1399 :
1400 : See PR64705 for the rationale. */
1401 2901153 : stop = extract_single_var_from_expr (iv->step);
1402 2901153 : iv->base = expand_simple_operations (iv->base, stop);
1403 2901153 : if (contains_abnormal_ssa_name_p (iv->base)
1404 2901153 : || contains_abnormal_ssa_name_p (iv->step))
1405 : return false;
1406 :
1407 : /* If STMT could throw, then do not consider STMT as defining a GIV.
1408 : While this will suppress optimizations, we cannot safely delete this
1409 : GIV and associated statements, even if it appears it is not used. */
1410 2901149 : if (stmt_could_throw_p (cfun, stmt))
1411 8 : return false;
1412 :
1413 : return true;
1414 : }
1415 :
1416 : /* Finds general ivs in statement STMT. */
1417 :
1418 : static void
1419 12795434 : find_givs_in_stmt (struct ivopts_data *data, gimple *stmt)
1420 : {
1421 12795434 : affine_iv iv;
1422 :
1423 12795434 : if (!find_givs_in_stmt_scev (data, stmt, &iv))
1424 : return;
1425 :
1426 2901141 : set_iv (data, gimple_assign_lhs (stmt), iv.base, iv.step, iv.no_overflow);
1427 : }
1428 :
1429 : /* Finds general ivs in basic block BB. */
1430 :
1431 : static void
1432 2873818 : find_givs_in_bb (struct ivopts_data *data, basic_block bb)
1433 : {
1434 2873818 : gimple_stmt_iterator bsi;
1435 :
1436 28729120 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
1437 22981484 : if (!is_gimple_debug (gsi_stmt (bsi)))
1438 12795434 : find_givs_in_stmt (data, gsi_stmt (bsi));
1439 2873818 : }
1440 :
1441 : /* Finds general ivs. */
1442 :
1443 : static void
1444 510605 : find_givs (struct ivopts_data *data, basic_block *body)
1445 : {
1446 510605 : class loop *loop = data->current_loop;
1447 510605 : unsigned i;
1448 :
1449 3384423 : for (i = 0; i < loop->num_nodes; i++)
1450 2873818 : find_givs_in_bb (data, body[i]);
1451 510605 : }
1452 :
1453 : /* For each ssa name defined in LOOP determines whether it is an induction
1454 : variable and if so, its initial value and step. */
1455 :
1456 : static bool
1457 641573 : find_induction_variables (struct ivopts_data *data, basic_block *body)
1458 : {
1459 641573 : unsigned i;
1460 641573 : bitmap_iterator bi;
1461 :
1462 641573 : if (!find_bivs (data))
1463 : return false;
1464 :
1465 510605 : find_givs (data, body);
1466 510605 : mark_bivs (data);
1467 :
1468 510605 : if (dump_file && (dump_flags & TDF_DETAILS))
1469 : {
1470 70 : class tree_niter_desc *niter = niter_for_single_dom_exit (data);
1471 :
1472 70 : if (niter)
1473 : {
1474 54 : fprintf (dump_file, " number of iterations ");
1475 54 : print_generic_expr (dump_file, niter->niter, TDF_SLIM);
1476 54 : if (!integer_zerop (niter->may_be_zero))
1477 : {
1478 1 : fprintf (dump_file, "; zero if ");
1479 1 : print_generic_expr (dump_file, niter->may_be_zero, TDF_SLIM);
1480 : }
1481 54 : fprintf (dump_file, "\n");
1482 70 : };
1483 :
1484 70 : fprintf (dump_file, "\n<Induction Vars>:\n");
1485 880 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
1486 : {
1487 810 : struct version_info *info = ver_info (data, i);
1488 810 : if (info->iv && info->iv->step && !integer_zerop (info->iv->step))
1489 562 : dump_iv (dump_file, ver_info (data, i)->iv, true, 0);
1490 : }
1491 : }
1492 :
1493 : return true;
1494 : }
1495 :
1496 : /* Records a use of TYPE at *USE_P in STMT whose value is IV in GROUP.
1497 : For address type use, ADDR_BASE is the stripped IV base, ADDR_OFFSET
1498 : is the const offset stripped from IV base and MEM_TYPE is the type
1499 : of the memory being addressed. For uses of other types, ADDR_BASE
1500 : and ADDR_OFFSET are zero by default and MEM_TYPE is NULL_TREE. */
1501 :
1502 : static struct iv_use *
1503 2106672 : record_use (struct iv_group *group, tree *use_p, struct iv *iv,
1504 : gimple *stmt, enum use_type type, tree mem_type,
1505 : tree addr_base, poly_uint64 addr_offset)
1506 : {
1507 2106672 : struct iv_use *use = XCNEW (struct iv_use);
1508 :
1509 2106672 : use->id = group->vuses.length ();
1510 2106672 : use->group_id = group->id;
1511 2106672 : use->type = type;
1512 2106672 : use->mem_type = mem_type;
1513 2106672 : use->iv = iv;
1514 2106672 : use->stmt = stmt;
1515 2106672 : use->op_p = use_p;
1516 2106672 : use->addr_base = addr_base;
1517 2106672 : use->addr_offset = addr_offset;
1518 :
1519 2106672 : group->vuses.safe_push (use);
1520 2106672 : return use;
1521 : }
1522 :
1523 : /* Checks whether OP is a loop-level invariant and if so, records it.
1524 : NONLINEAR_USE is true if the invariant is used in a way we do not
1525 : handle specially. */
1526 :
1527 : static void
1528 23063465 : record_invariant (struct ivopts_data *data, tree op, bool nonlinear_use)
1529 : {
1530 23063465 : basic_block bb;
1531 23063465 : struct version_info *info;
1532 :
1533 23063465 : if (TREE_CODE (op) != SSA_NAME
1534 23063465 : || virtual_operand_p (op))
1535 : return;
1536 :
1537 21863145 : bb = gimple_bb (SSA_NAME_DEF_STMT (op));
1538 21863145 : if (bb
1539 21863145 : && flow_bb_inside_loop_p (data->current_loop, bb))
1540 : return;
1541 :
1542 3958716 : info = name_info (data, op);
1543 3958716 : info->name = op;
1544 3958716 : info->has_nonlin_use |= nonlinear_use;
1545 3958716 : if (!info->inv_id)
1546 1353219 : info->inv_id = ++data->max_inv_var_id;
1547 3958716 : bitmap_set_bit (data->relevant, SSA_NAME_VERSION (op));
1548 : }
1549 :
1550 : /* Record a group of TYPE. */
1551 :
1552 : static struct iv_group *
1553 1823529 : record_group (struct ivopts_data *data, enum use_type type)
1554 : {
1555 1823529 : struct iv_group *group = XCNEW (struct iv_group);
1556 :
1557 1823529 : group->id = data->vgroups.length ();
1558 1823529 : group->type = type;
1559 1823529 : group->related_cands = BITMAP_ALLOC (NULL);
1560 1823529 : group->vuses.create (1);
1561 1823529 : group->doloop_p = false;
1562 :
1563 1823529 : data->vgroups.safe_push (group);
1564 1823529 : return group;
1565 : }
1566 :
1567 : /* Record a use of TYPE at *USE_P in STMT whose value is IV in a group.
1568 : New group will be created if there is no existing group for the use.
1569 : MEM_TYPE is the type of memory being addressed, or NULL if this
1570 : isn't an address reference. */
1571 :
1572 : static struct iv_use *
1573 2106672 : record_group_use (struct ivopts_data *data, tree *use_p,
1574 : struct iv *iv, gimple *stmt, enum use_type type,
1575 : tree mem_type)
1576 : {
1577 2106672 : tree addr_base = NULL;
1578 2106672 : struct iv_group *group = NULL;
1579 2106672 : poly_uint64 addr_offset = 0;
1580 :
1581 : /* Record non address type use in a new group. */
1582 2106672 : if (address_p (type))
1583 : {
1584 870345 : unsigned int i;
1585 :
1586 870345 : gcc_assert (POINTER_TYPE_P (TREE_TYPE (iv->base)));
1587 870345 : tree addr_toffset;
1588 870345 : split_constant_offset (iv->base, &addr_base, &addr_toffset);
1589 870345 : addr_offset = int_cst_value (addr_toffset);
1590 1627708 : for (i = 0; i < data->vgroups.length (); i++)
1591 : {
1592 1092260 : struct iv_use *use;
1593 :
1594 1092260 : group = data->vgroups[i];
1595 1092260 : use = group->vuses[0];
1596 1092260 : if (!address_p (use->type))
1597 332463 : continue;
1598 :
1599 : /* Check if it has the same stripped base and step. */
1600 759797 : if (operand_equal_p (iv->base_object, use->iv->base_object, 0)
1601 402359 : && operand_equal_p (iv->step, use->iv->step, OEP_ASSUME_WRAPV)
1602 1159047 : && operand_equal_p (addr_base, use->addr_base, OEP_ASSUME_WRAPV))
1603 : break;
1604 : }
1605 1740690 : if (i == data->vgroups.length ())
1606 535448 : group = NULL;
1607 : }
1608 :
1609 870345 : if (!group)
1610 1771775 : group = record_group (data, type);
1611 :
1612 2106672 : return record_use (group, use_p, iv, stmt, type, mem_type,
1613 2106672 : addr_base, addr_offset);
1614 : }
1615 :
1616 : /* Checks whether the use OP is interesting and if so, records it. */
1617 :
1618 : static struct iv_use *
1619 7438315 : find_interesting_uses_op (struct ivopts_data *data, tree op)
1620 : {
1621 7438315 : struct iv *iv;
1622 7438315 : gimple *stmt;
1623 7438315 : struct iv_use *use;
1624 :
1625 7438315 : if (TREE_CODE (op) != SSA_NAME)
1626 : return NULL;
1627 :
1628 5980283 : iv = get_iv (data, op);
1629 5980283 : if (!iv)
1630 : return NULL;
1631 :
1632 2629134 : if (iv->nonlin_use)
1633 : {
1634 212444 : gcc_assert (iv->nonlin_use->type == USE_NONLINEAR_EXPR);
1635 : return iv->nonlin_use;
1636 : }
1637 :
1638 2416690 : if (integer_zerop (iv->step))
1639 : {
1640 1788323 : record_invariant (data, op, true);
1641 1788323 : return NULL;
1642 : }
1643 :
1644 628367 : stmt = SSA_NAME_DEF_STMT (op);
1645 628367 : gcc_assert (gimple_code (stmt) == GIMPLE_PHI || is_gimple_assign (stmt));
1646 :
1647 628367 : use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR, NULL_TREE);
1648 628367 : iv->nonlin_use = use;
1649 628367 : return use;
1650 : }
1651 :
1652 : /* Indicate how compare type iv_use can be handled. */
1653 : enum comp_iv_rewrite
1654 : {
1655 : COMP_IV_NA,
1656 : /* We may rewrite compare type iv_use by expressing value of the iv_use. */
1657 : COMP_IV_EXPR,
1658 : /* We may rewrite compare type iv_uses on both sides of comparison by
1659 : expressing value of each iv_use. */
1660 : COMP_IV_EXPR_2,
1661 : /* We may rewrite compare type iv_use by expressing value of the iv_use
1662 : or by eliminating it with other iv_cand. */
1663 : COMP_IV_ELIM
1664 : };
1665 :
1666 : /* Given a condition in statement STMT, checks whether it is a compare
1667 : of an induction variable and an invariant. If this is the case,
1668 : CONTROL_VAR is set to location of the iv, BOUND to the location of
1669 : the invariant, IV_VAR and IV_BOUND are set to the corresponding
1670 : induction variable descriptions, and true is returned. If this is not
1671 : the case, CONTROL_VAR and BOUND are set to the arguments of the
1672 : condition and false is returned. */
1673 :
1674 : static enum comp_iv_rewrite
1675 7552048 : extract_cond_operands (struct ivopts_data *data, gimple *stmt,
1676 : tree **control_var, tree **bound,
1677 : struct iv **iv_var, struct iv **iv_bound)
1678 : {
1679 : /* The objects returned when COND has constant operands. */
1680 7552048 : static struct iv const_iv;
1681 7552048 : static tree zero;
1682 7552048 : tree *op0 = &zero, *op1 = &zero;
1683 7552048 : struct iv *iv0 = &const_iv, *iv1 = &const_iv;
1684 7552048 : enum comp_iv_rewrite rewrite_type = COMP_IV_NA;
1685 :
1686 7552048 : if (gimple_code (stmt) == GIMPLE_COND)
1687 : {
1688 7270896 : gcond *cond_stmt = as_a <gcond *> (stmt);
1689 7270896 : op0 = gimple_cond_lhs_ptr (cond_stmt);
1690 7270896 : op1 = gimple_cond_rhs_ptr (cond_stmt);
1691 : }
1692 : else
1693 : {
1694 281152 : op0 = gimple_assign_rhs1_ptr (stmt);
1695 281152 : op1 = gimple_assign_rhs2_ptr (stmt);
1696 : }
1697 :
1698 7552048 : zero = integer_zero_node;
1699 7552048 : const_iv.step = integer_zero_node;
1700 :
1701 7552048 : if (TREE_CODE (*op0) == SSA_NAME)
1702 7551891 : iv0 = get_iv (data, *op0);
1703 7552048 : if (TREE_CODE (*op1) == SSA_NAME)
1704 3381569 : iv1 = get_iv (data, *op1);
1705 :
1706 : /* If both sides of comparison are IVs. We can express ivs on both end. */
1707 7552048 : if (iv0 && iv1 && !integer_zerop (iv0->step) && !integer_zerop (iv1->step))
1708 : {
1709 87821 : rewrite_type = COMP_IV_EXPR_2;
1710 87821 : goto end;
1711 : }
1712 :
1713 : /* If none side of comparison is IV. */
1714 5817274 : if ((!iv0 || integer_zerop (iv0->step))
1715 8805964 : && (!iv1 || integer_zerop (iv1->step)))
1716 969052 : goto end;
1717 :
1718 : /* Control variable may be on the other side. */
1719 6495175 : if (!iv0 || integer_zerop (iv0->step))
1720 : {
1721 : std::swap (op0, op1);
1722 : std::swap (iv0, iv1);
1723 : }
1724 : /* If one side is IV and the other side isn't loop invariant. */
1725 6495175 : if (!iv1)
1726 : rewrite_type = COMP_IV_EXPR;
1727 : /* If one side is IV and the other side is loop invariant. */
1728 5485321 : else if (!integer_zerop (iv0->step) && integer_zerop (iv1->step))
1729 : rewrite_type = COMP_IV_ELIM;
1730 :
1731 7552048 : end:
1732 7552048 : if (control_var)
1733 7552048 : *control_var = op0;
1734 7552048 : if (iv_var)
1735 1574873 : *iv_var = iv0;
1736 7552048 : if (bound)
1737 7552048 : *bound = op1;
1738 7552048 : if (iv_bound)
1739 7552048 : *iv_bound = iv1;
1740 :
1741 7552048 : return rewrite_type;
1742 : }
1743 :
1744 : /* Checks whether the condition in STMT is interesting and if so,
1745 : records it. */
1746 :
1747 : static void
1748 1574873 : find_interesting_uses_cond (struct ivopts_data *data, gimple *stmt)
1749 : {
1750 1574873 : tree *var_p, *bound_p;
1751 1574873 : struct iv *var_iv, *bound_iv;
1752 1574873 : enum comp_iv_rewrite ret;
1753 :
1754 1574873 : ret = extract_cond_operands (data, stmt,
1755 : &var_p, &bound_p, &var_iv, &bound_iv);
1756 1574873 : if (ret == COMP_IV_NA)
1757 : {
1758 969052 : find_interesting_uses_op (data, *var_p);
1759 969052 : find_interesting_uses_op (data, *bound_p);
1760 969052 : return;
1761 : }
1762 :
1763 605821 : record_group_use (data, var_p, var_iv, stmt, USE_COMPARE, NULL_TREE);
1764 : /* Record compare type iv_use for iv on the other side of comparison. */
1765 605821 : if (ret == COMP_IV_EXPR_2)
1766 2139 : record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE, NULL_TREE);
1767 : }
1768 :
1769 : /* Returns the outermost loop EXPR is obviously invariant in
1770 : relative to the loop LOOP, i.e. if all its operands are defined
1771 : outside of the returned loop. Returns NULL if EXPR is not
1772 : even obviously invariant in LOOP. */
1773 :
1774 : class loop *
1775 375643 : outermost_invariant_loop_for_expr (class loop *loop, tree expr)
1776 : {
1777 375643 : basic_block def_bb;
1778 375643 : unsigned i, len;
1779 :
1780 375643 : if (is_gimple_min_invariant (expr))
1781 49055 : return current_loops->tree_root;
1782 :
1783 326588 : if (TREE_CODE (expr) == SSA_NAME)
1784 : {
1785 193282 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (expr));
1786 193282 : if (def_bb)
1787 : {
1788 115191 : if (flow_bb_inside_loop_p (loop, def_bb))
1789 : return NULL;
1790 230366 : return superloop_at_depth (loop,
1791 158545 : loop_depth (def_bb->loop_father) + 1);
1792 : }
1793 :
1794 78091 : return current_loops->tree_root;
1795 : }
1796 :
1797 133306 : if (!EXPR_P (expr))
1798 : return NULL;
1799 :
1800 133306 : unsigned maxdepth = 0;
1801 133306 : len = TREE_OPERAND_LENGTH (expr);
1802 347390 : for (i = 0; i < len; i++)
1803 : {
1804 214108 : class loop *ivloop;
1805 214108 : if (!TREE_OPERAND (expr, i))
1806 0 : continue;
1807 :
1808 214108 : ivloop = outermost_invariant_loop_for_expr (loop, TREE_OPERAND (expr, i));
1809 214108 : if (!ivloop)
1810 : return NULL;
1811 554755 : maxdepth = MAX (maxdepth, loop_depth (ivloop));
1812 : }
1813 :
1814 133282 : return superloop_at_depth (loop, maxdepth);
1815 : }
1816 :
1817 : /* Returns true if expression EXPR is obviously invariant in LOOP,
1818 : i.e. if all its operands are defined outside of the LOOP. LOOP
1819 : should not be the function body. */
1820 :
1821 : bool
1822 12493908 : expr_invariant_in_loop_p (class loop *loop, tree expr)
1823 : {
1824 12493908 : basic_block def_bb;
1825 12493908 : unsigned i, len;
1826 :
1827 12493908 : gcc_assert (loop_depth (loop) > 0);
1828 :
1829 12493908 : if (is_gimple_min_invariant (expr))
1830 : return true;
1831 :
1832 8639367 : if (TREE_CODE (expr) == SSA_NAME)
1833 : {
1834 8198329 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (expr));
1835 8198329 : if (def_bb
1836 8198329 : && flow_bb_inside_loop_p (loop, def_bb))
1837 3954583 : return false;
1838 :
1839 : return true;
1840 : }
1841 :
1842 441038 : if (!EXPR_P (expr))
1843 : return false;
1844 :
1845 441035 : len = TREE_OPERAND_LENGTH (expr);
1846 966939 : for (i = 0; i < len; i++)
1847 572261 : if (TREE_OPERAND (expr, i)
1848 572261 : && !expr_invariant_in_loop_p (loop, TREE_OPERAND (expr, i)))
1849 : return false;
1850 :
1851 : return true;
1852 : }
1853 :
1854 : /* Given expression EXPR which computes inductive values with respect
1855 : to loop recorded in DATA, this function returns biv from which EXPR
1856 : is derived by tracing definition chains of ssa variables in EXPR. */
1857 :
1858 : static struct iv*
1859 874900 : find_deriving_biv_for_expr (struct ivopts_data *data, tree expr)
1860 : {
1861 1415096 : struct iv *iv;
1862 1415096 : unsigned i, n;
1863 1415096 : tree e2, e1;
1864 1415096 : enum tree_code code;
1865 1415096 : gimple *stmt;
1866 :
1867 1415096 : if (expr == NULL_TREE)
1868 : return NULL;
1869 :
1870 1414848 : if (is_gimple_min_invariant (expr))
1871 : return NULL;
1872 :
1873 1130172 : code = TREE_CODE (expr);
1874 1130172 : if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code)))
1875 : {
1876 23395 : n = TREE_OPERAND_LENGTH (expr);
1877 25107 : for (i = 0; i < n; i++)
1878 : {
1879 24700 : iv = find_deriving_biv_for_expr (data, TREE_OPERAND (expr, i));
1880 24700 : if (iv)
1881 : return iv;
1882 : }
1883 : }
1884 :
1885 : /* Stop if it's not ssa name. */
1886 1107184 : if (code != SSA_NAME)
1887 : return NULL;
1888 :
1889 1106268 : iv = get_iv (data, expr);
1890 1106268 : if (!iv || integer_zerop (iv->step))
1891 : return NULL;
1892 1059284 : else if (iv->biv_p)
1893 : return iv;
1894 :
1895 787891 : stmt = SSA_NAME_DEF_STMT (expr);
1896 787891 : if (gphi *phi = dyn_cast <gphi *> (stmt))
1897 : {
1898 59 : ssa_op_iter iter;
1899 59 : use_operand_p use_p;
1900 59 : basic_block phi_bb = gimple_bb (phi);
1901 :
1902 : /* Skip loop header PHI that doesn't define biv. */
1903 59 : if (phi_bb->loop_father == data->current_loop)
1904 : return NULL;
1905 :
1906 0 : if (virtual_operand_p (gimple_phi_result (phi)))
1907 : return NULL;
1908 :
1909 0 : FOR_EACH_PHI_ARG (use_p, phi, iter, SSA_OP_USE)
1910 : {
1911 0 : tree use = USE_FROM_PTR (use_p);
1912 0 : iv = find_deriving_biv_for_expr (data, use);
1913 0 : if (iv)
1914 : return iv;
1915 : }
1916 : return NULL;
1917 : }
1918 787832 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1919 : return NULL;
1920 :
1921 787832 : e1 = gimple_assign_rhs1 (stmt);
1922 787832 : code = gimple_assign_rhs_code (stmt);
1923 787832 : if (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS)
1924 : return find_deriving_biv_for_expr (data, e1);
1925 :
1926 778868 : switch (code)
1927 : {
1928 577640 : case MULT_EXPR:
1929 577640 : case PLUS_EXPR:
1930 577640 : case MINUS_EXPR:
1931 577640 : case POINTER_PLUS_EXPR:
1932 : /* Increments, decrements and multiplications by a constant
1933 : are simple. */
1934 577640 : e2 = gimple_assign_rhs2 (stmt);
1935 577640 : iv = find_deriving_biv_for_expr (data, e2);
1936 577640 : if (iv)
1937 : return iv;
1938 : gcc_fallthrough ();
1939 :
1940 : CASE_CONVERT:
1941 : /* Casts are simple. */
1942 : return find_deriving_biv_for_expr (data, e1);
1943 :
1944 : default:
1945 : break;
1946 : }
1947 :
1948 : return NULL;
1949 : }
1950 :
1951 : /* Record BIV, its predecessor and successor that they are used in
1952 : address type uses. */
1953 :
1954 : static void
1955 606267 : record_biv_for_address_use (struct ivopts_data *data, struct iv *biv)
1956 : {
1957 606267 : unsigned i;
1958 606267 : tree type, base_1, base_2;
1959 606267 : bitmap_iterator bi;
1960 :
1961 605100 : if (!biv || !biv->biv_p || integer_zerop (biv->step)
1962 1211367 : || biv->have_address_use || !biv->no_overflow)
1963 339697 : return;
1964 :
1965 541593 : type = TREE_TYPE (biv->base);
1966 541593 : if (!INTEGRAL_TYPE_P (type))
1967 : return;
1968 :
1969 266570 : biv->have_address_use = true;
1970 266570 : data->bivs_not_used_in_addr--;
1971 266570 : base_1 = fold_build2 (PLUS_EXPR, type, biv->base, biv->step);
1972 2444449 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
1973 : {
1974 2177879 : struct iv *iv = ver_info (data, i)->iv;
1975 :
1976 1965687 : if (!iv || !iv->biv_p || integer_zerop (iv->step)
1977 3086007 : || iv->have_address_use || !iv->no_overflow)
1978 1884332 : continue;
1979 :
1980 293547 : if (type != TREE_TYPE (iv->base)
1981 293547 : || !INTEGRAL_TYPE_P (TREE_TYPE (iv->base)))
1982 30403 : continue;
1983 :
1984 263144 : if (!operand_equal_p (biv->step, iv->step, 0))
1985 5917 : continue;
1986 :
1987 257227 : base_2 = fold_build2 (PLUS_EXPR, type, iv->base, iv->step);
1988 257227 : if (operand_equal_p (base_1, iv->base, 0)
1989 257227 : || operand_equal_p (base_2, biv->base, 0))
1990 : {
1991 231766 : iv->have_address_use = true;
1992 231766 : data->bivs_not_used_in_addr--;
1993 : }
1994 : }
1995 : }
1996 :
1997 : /* Cumulates the steps of indices into DATA and replaces their values with the
1998 : initial ones. Returns false when the value of the index cannot be determined.
1999 : Callback for for_each_index. */
2000 :
2001 : struct ifs_ivopts_data
2002 : {
2003 : struct ivopts_data *ivopts_data;
2004 : gimple *stmt;
2005 : tree step;
2006 : };
2007 :
2008 : static bool
2009 2302839 : idx_find_step (tree base, tree *idx, void *data)
2010 : {
2011 2302839 : struct ifs_ivopts_data *dta = (struct ifs_ivopts_data *) data;
2012 2302839 : struct iv *iv;
2013 2302839 : bool use_overflow_semantics = false;
2014 2302839 : tree step, iv_base, iv_step, lbound, off;
2015 2302839 : class loop *loop = dta->ivopts_data->current_loop;
2016 :
2017 : /* If base is a component ref, require that the offset of the reference
2018 : be invariant. */
2019 2302839 : if (TREE_CODE (base) == COMPONENT_REF)
2020 : {
2021 78 : off = component_ref_field_offset (base);
2022 78 : return expr_invariant_in_loop_p (loop, off);
2023 : }
2024 :
2025 : /* If base is array, first check whether we will be able to move the
2026 : reference out of the loop (in order to take its address in strength
2027 : reduction). In order for this to work we need both lower bound
2028 : and step to be loop invariants. */
2029 2302761 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2030 : {
2031 : /* Moreover, for a range, the size needs to be invariant as well. */
2032 543430 : if (TREE_CODE (base) == ARRAY_RANGE_REF
2033 543430 : && !expr_invariant_in_loop_p (loop, TYPE_SIZE (TREE_TYPE (base))))
2034 : return false;
2035 :
2036 543430 : step = array_ref_element_size (base);
2037 543430 : lbound = array_ref_low_bound (base);
2038 :
2039 543430 : if (!expr_invariant_in_loop_p (loop, step)
2040 543430 : || !expr_invariant_in_loop_p (loop, lbound))
2041 : return false;
2042 : }
2043 :
2044 2302755 : if (TREE_CODE (*idx) != SSA_NAME)
2045 : return true;
2046 :
2047 1863686 : iv = get_iv (dta->ivopts_data, *idx);
2048 1863686 : if (!iv)
2049 : return false;
2050 :
2051 : /* XXX We produce for a base of *D42 with iv->base being &x[0]
2052 : *&x[0], which is not folded and does not trigger the
2053 : ARRAY_REF path below. */
2054 1203093 : *idx = iv->base;
2055 :
2056 1203093 : if (integer_zerop (iv->step))
2057 : return true;
2058 :
2059 886526 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2060 : {
2061 305693 : step = array_ref_element_size (base);
2062 :
2063 : /* We only handle addresses whose step is an integer constant. */
2064 305693 : if (TREE_CODE (step) != INTEGER_CST)
2065 : return false;
2066 : }
2067 : else
2068 : /* The step for pointer arithmetics already is 1 byte. */
2069 580833 : step = size_one_node;
2070 :
2071 886269 : iv_base = iv->base;
2072 886269 : iv_step = iv->step;
2073 886269 : if (iv->no_overflow && nowrap_type_p (TREE_TYPE (iv_step)))
2074 : use_overflow_semantics = true;
2075 :
2076 886269 : if (!convert_affine_scev (dta->ivopts_data->current_loop,
2077 : sizetype, &iv_base, &iv_step, dta->stmt,
2078 : use_overflow_semantics))
2079 : {
2080 : /* The index might wrap. */
2081 : return false;
2082 : }
2083 :
2084 883060 : step = fold_build2 (MULT_EXPR, sizetype, step, iv_step);
2085 883060 : dta->step = fold_build2 (PLUS_EXPR, sizetype, dta->step, step);
2086 :
2087 883060 : if (dta->ivopts_data->bivs_not_used_in_addr)
2088 : {
2089 606267 : if (!iv->biv_p)
2090 272560 : iv = find_deriving_biv_for_expr (dta->ivopts_data, iv->ssa_name);
2091 :
2092 606267 : record_biv_for_address_use (dta->ivopts_data, iv);
2093 : }
2094 : return true;
2095 : }
2096 :
2097 : /* Records use in index IDX. Callback for for_each_index. Ivopts data
2098 : object is passed to it in DATA. */
2099 :
2100 : static bool
2101 1870356 : idx_record_use (tree base, tree *idx,
2102 : void *vdata)
2103 : {
2104 1870356 : struct ivopts_data *data = (struct ivopts_data *) vdata;
2105 1870356 : find_interesting_uses_op (data, *idx);
2106 1870356 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2107 : {
2108 243688 : if (TREE_OPERAND (base, 2))
2109 5462 : find_interesting_uses_op (data, TREE_OPERAND (base, 2));
2110 243688 : if (TREE_OPERAND (base, 3))
2111 11482 : find_interesting_uses_op (data, TREE_OPERAND (base, 3));
2112 : }
2113 1870356 : return true;
2114 : }
2115 :
2116 : /* If we can prove that TOP = cst * BOT for some constant cst,
2117 : store cst to MUL and return true. Otherwise return false.
2118 : The returned value is always sign-extended, regardless of the
2119 : signedness of TOP and BOT. */
2120 :
2121 : static bool
2122 17387493 : constant_multiple_of (tree top, tree bot, widest_int *mul,
2123 : struct ivopts_data *data)
2124 : {
2125 34774986 : aff_tree aff_top, aff_bot;
2126 17387493 : tree_to_aff_combination_expand (top, TREE_TYPE (top), &aff_top,
2127 : &data->name_expansion_cache);
2128 17387493 : tree_to_aff_combination_expand (bot, TREE_TYPE (bot), &aff_bot,
2129 : &data->name_expansion_cache);
2130 :
2131 17387493 : poly_widest_int poly_mul;
2132 17387493 : if (aff_combination_constant_multiple_p (&aff_top, &aff_bot, &poly_mul)
2133 17387493 : && poly_mul.is_constant (mul))
2134 14407020 : return true;
2135 :
2136 : return false;
2137 17387493 : }
2138 :
2139 : /* Return true if memory reference REF with step STEP may be unaligned. */
2140 :
2141 : static bool
2142 0 : may_be_unaligned_p (tree ref, tree step)
2143 : {
2144 : /* TARGET_MEM_REFs are translated directly to valid MEMs on the target,
2145 : thus they are not misaligned. */
2146 0 : if (TREE_CODE (ref) == TARGET_MEM_REF)
2147 : return false;
2148 :
2149 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (ref));
2150 0 : if (GET_MODE_ALIGNMENT (TYPE_MODE (TREE_TYPE (ref))) > align)
2151 0 : align = GET_MODE_ALIGNMENT (TYPE_MODE (TREE_TYPE (ref)));
2152 :
2153 0 : unsigned HOST_WIDE_INT bitpos;
2154 0 : unsigned int ref_align;
2155 0 : get_object_alignment_1 (ref, &ref_align, &bitpos);
2156 0 : if (ref_align < align
2157 0 : || (bitpos % align) != 0
2158 0 : || (bitpos % BITS_PER_UNIT) != 0)
2159 : return true;
2160 :
2161 0 : unsigned int trailing_zeros = tree_ctz (step);
2162 0 : if (trailing_zeros < HOST_BITS_PER_INT
2163 0 : && (1U << trailing_zeros) * BITS_PER_UNIT < align)
2164 0 : return true;
2165 :
2166 : return false;
2167 : }
2168 :
2169 : /* Return true if EXPR may be non-addressable. */
2170 :
2171 : bool
2172 13429820 : may_be_nonaddressable_p (tree expr)
2173 : {
2174 14317020 : switch (TREE_CODE (expr))
2175 : {
2176 9607793 : case VAR_DECL:
2177 : /* Check if it's a register variable. */
2178 9607793 : return DECL_HARD_REGISTER (expr);
2179 :
2180 : case TARGET_MEM_REF:
2181 : /* TARGET_MEM_REFs are translated directly to valid MEMs on the
2182 : target, thus they are always addressable. */
2183 : return false;
2184 :
2185 1977888 : case MEM_REF:
2186 : /* Likewise for MEM_REFs, modulo the storage order. */
2187 1977888 : return REF_REVERSE_STORAGE_ORDER (expr);
2188 :
2189 76 : case BIT_FIELD_REF:
2190 76 : if (REF_REVERSE_STORAGE_ORDER (expr))
2191 : return true;
2192 76 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2193 :
2194 1271225 : case COMPONENT_REF:
2195 1271225 : if (TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_OPERAND (expr, 0))))
2196 : return true;
2197 1271225 : return DECL_NONADDRESSABLE_P (TREE_OPERAND (expr, 1))
2198 1271225 : || may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2199 :
2200 864258 : case ARRAY_REF:
2201 864258 : case ARRAY_RANGE_REF:
2202 864258 : if (TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_OPERAND (expr, 0))))
2203 : return true;
2204 864258 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2205 :
2206 22876 : case VIEW_CONVERT_EXPR:
2207 : /* This kind of view-conversions may wrap non-addressable objects
2208 : and make them look addressable. After some processing the
2209 : non-addressability may be uncovered again, causing ADDR_EXPRs
2210 : of inappropriate objects to be built. */
2211 22876 : if (is_gimple_reg (TREE_OPERAND (expr, 0))
2212 22876 : || !is_gimple_addressable (TREE_OPERAND (expr, 0)))
2213 : return true;
2214 22866 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2215 :
2216 : CASE_CONVERT:
2217 : return true;
2218 :
2219 : default:
2220 : break;
2221 : }
2222 :
2223 : return false;
2224 : }
2225 :
2226 : /* Finds addresses in *OP_P inside STMT. */
2227 :
2228 : static void
2229 2764229 : find_interesting_uses_address (struct ivopts_data *data, gimple *stmt,
2230 : tree *op_p)
2231 : {
2232 2764229 : tree base = *op_p, step = size_zero_node;
2233 2764229 : struct iv *civ;
2234 2764229 : struct ifs_ivopts_data ifs_ivopts_data;
2235 :
2236 : /* Do not play with volatile memory references. A bit too conservative,
2237 : perhaps, but safe. */
2238 5528458 : if (gimple_has_volatile_ops (stmt))
2239 7566 : goto fail;
2240 :
2241 : /* Ignore bitfields for now. Not really something terribly complicated
2242 : to handle. TODO. */
2243 2756663 : if (TREE_CODE (base) == BIT_FIELD_REF)
2244 92933 : goto fail;
2245 :
2246 2663730 : base = unshare_expr (base);
2247 :
2248 2663730 : if (TREE_CODE (base) == TARGET_MEM_REF)
2249 : {
2250 311228 : tree type = build_pointer_type (TREE_TYPE (base));
2251 311228 : tree astep;
2252 :
2253 311228 : if (TMR_BASE (base)
2254 311228 : && TREE_CODE (TMR_BASE (base)) == SSA_NAME)
2255 : {
2256 290199 : civ = get_iv (data, TMR_BASE (base));
2257 290199 : if (!civ)
2258 256169 : goto fail;
2259 :
2260 34030 : TMR_BASE (base) = civ->base;
2261 34030 : step = civ->step;
2262 : }
2263 55059 : if (TMR_INDEX2 (base)
2264 55059 : && TREE_CODE (TMR_INDEX2 (base)) == SSA_NAME)
2265 : {
2266 13692 : civ = get_iv (data, TMR_INDEX2 (base));
2267 13692 : if (!civ)
2268 4909 : goto fail;
2269 :
2270 8783 : TMR_INDEX2 (base) = civ->base;
2271 8783 : step = civ->step;
2272 : }
2273 50150 : if (TMR_INDEX (base)
2274 50150 : && TREE_CODE (TMR_INDEX (base)) == SSA_NAME)
2275 : {
2276 50150 : civ = get_iv (data, TMR_INDEX (base));
2277 50150 : if (!civ)
2278 50150 : goto fail;
2279 :
2280 0 : TMR_INDEX (base) = civ->base;
2281 0 : astep = civ->step;
2282 :
2283 0 : if (astep)
2284 : {
2285 0 : if (TMR_STEP (base))
2286 0 : astep = fold_build2 (MULT_EXPR, type, TMR_STEP (base), astep);
2287 :
2288 0 : step = fold_build2 (PLUS_EXPR, type, step, astep);
2289 : }
2290 : }
2291 :
2292 0 : if (integer_zerop (step))
2293 0 : goto fail;
2294 0 : base = tree_mem_ref_addr (type, base);
2295 : }
2296 : else
2297 : {
2298 2352502 : ifs_ivopts_data.ivopts_data = data;
2299 2352502 : ifs_ivopts_data.stmt = stmt;
2300 2352502 : ifs_ivopts_data.step = size_zero_node;
2301 2352502 : if (!for_each_index (&base, idx_find_step, &ifs_ivopts_data)
2302 2352502 : || integer_zerop (ifs_ivopts_data.step))
2303 1471218 : goto fail;
2304 881284 : step = ifs_ivopts_data.step;
2305 :
2306 : /* Check that the base expression is addressable. This needs
2307 : to be done after substituting bases of IVs into it. */
2308 881284 : if (may_be_nonaddressable_p (base))
2309 772 : goto fail;
2310 :
2311 : /* Moreover, on strict alignment platforms, check that it is
2312 : sufficiently aligned. */
2313 880512 : if (STRICT_ALIGNMENT && may_be_unaligned_p (base, step))
2314 : goto fail;
2315 :
2316 880512 : base = build_fold_addr_expr (base);
2317 :
2318 : /* Substituting bases of IVs into the base expression might
2319 : have caused folding opportunities. */
2320 880512 : if (TREE_CODE (base) == ADDR_EXPR)
2321 : {
2322 471090 : tree *ref = &TREE_OPERAND (base, 0);
2323 1610801 : while (handled_component_p (*ref))
2324 668621 : ref = &TREE_OPERAND (*ref, 0);
2325 471090 : if (TREE_CODE (*ref) == MEM_REF)
2326 : {
2327 306624 : tree tem = fold_binary (MEM_REF, TREE_TYPE (*ref),
2328 : TREE_OPERAND (*ref, 0),
2329 : TREE_OPERAND (*ref, 1));
2330 306624 : if (tem)
2331 0 : *ref = tem;
2332 : }
2333 : }
2334 : }
2335 :
2336 880512 : civ = alloc_iv (data, base, step);
2337 : /* Fail if base object of this memory reference is unknown. */
2338 880512 : if (civ->base_object == NULL_TREE)
2339 10804 : goto fail;
2340 :
2341 869708 : record_group_use (data, op_p, civ, stmt, USE_REF_ADDRESS, TREE_TYPE (*op_p));
2342 869708 : return;
2343 :
2344 1894521 : fail:
2345 1894521 : for_each_index (op_p, idx_record_use, data);
2346 : }
2347 :
2348 : /* Finds and records invariants used in STMT. */
2349 :
2350 : static void
2351 15704487 : find_invariants_stmt (struct ivopts_data *data, gimple *stmt)
2352 : {
2353 15704487 : ssa_op_iter iter;
2354 15704487 : use_operand_p use_p;
2355 15704487 : tree op;
2356 :
2357 52293687 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
2358 : {
2359 20884713 : op = USE_FROM_PTR (use_p);
2360 20884713 : record_invariant (data, op, false);
2361 : }
2362 15704487 : }
2363 :
2364 : /* CALL calls an internal function. If operand *OP_P will become an
2365 : address when the call is expanded, return the type of the memory
2366 : being addressed, otherwise return null. */
2367 :
2368 : static tree
2369 3506 : get_mem_type_for_internal_fn (gcall *call, tree *op_p)
2370 : {
2371 3506 : switch (gimple_call_internal_fn (call))
2372 : {
2373 272 : case IFN_MASK_LOAD:
2374 272 : case IFN_MASK_LOAD_LANES:
2375 272 : case IFN_MASK_LEN_LOAD_LANES:
2376 272 : case IFN_LEN_LOAD:
2377 272 : case IFN_MASK_LEN_LOAD:
2378 272 : if (op_p == gimple_call_arg_ptr (call, 0))
2379 272 : return TREE_TYPE (gimple_call_lhs (call));
2380 : return NULL_TREE;
2381 :
2382 365 : case IFN_MASK_STORE:
2383 365 : case IFN_MASK_STORE_LANES:
2384 365 : case IFN_MASK_LEN_STORE_LANES:
2385 365 : case IFN_LEN_STORE:
2386 365 : case IFN_MASK_LEN_STORE:
2387 365 : {
2388 365 : if (op_p == gimple_call_arg_ptr (call, 0))
2389 : {
2390 365 : internal_fn ifn = gimple_call_internal_fn (call);
2391 365 : int index = internal_fn_stored_value_index (ifn);
2392 365 : return TREE_TYPE (gimple_call_arg (call, index));
2393 : }
2394 : return NULL_TREE;
2395 : }
2396 :
2397 : default:
2398 : return NULL_TREE;
2399 : }
2400 : }
2401 :
2402 : /* IV is a (non-address) iv that describes operand *OP_P of STMT.
2403 : Return true if the operand will become an address when STMT
2404 : is expanded and record the associated address use if so. */
2405 :
2406 : static bool
2407 1779220 : find_address_like_use (struct ivopts_data *data, gimple *stmt, tree *op_p,
2408 : struct iv *iv)
2409 : {
2410 : /* Fail if base object of this memory reference is unknown. */
2411 1779220 : if (iv->base_object == NULL_TREE)
2412 : return false;
2413 :
2414 687174 : tree mem_type = NULL_TREE;
2415 687174 : if (gcall *call = dyn_cast <gcall *> (stmt))
2416 127802 : if (gimple_call_internal_p (call))
2417 3506 : mem_type = get_mem_type_for_internal_fn (call, op_p);
2418 3506 : if (mem_type)
2419 : {
2420 637 : iv = alloc_iv (data, iv->base, iv->step);
2421 637 : record_group_use (data, op_p, iv, stmt, USE_PTR_ADDRESS, mem_type);
2422 637 : return true;
2423 : }
2424 : return false;
2425 : }
2426 :
2427 : /* Finds interesting uses of induction variables in the statement STMT. */
2428 :
2429 : static void
2430 15704487 : find_interesting_uses_stmt (struct ivopts_data *data, gimple *stmt)
2431 : {
2432 15704487 : struct iv *iv;
2433 15704487 : tree op, *lhs, *rhs;
2434 15704487 : ssa_op_iter iter;
2435 15704487 : use_operand_p use_p;
2436 15704487 : enum tree_code code;
2437 :
2438 15704487 : find_invariants_stmt (data, stmt);
2439 :
2440 15704487 : if (gimple_code (stmt) == GIMPLE_COND)
2441 : {
2442 1494685 : find_interesting_uses_cond (data, stmt);
2443 9190406 : return;
2444 : }
2445 :
2446 14209802 : if (is_gimple_assign (stmt))
2447 : {
2448 10705106 : lhs = gimple_assign_lhs_ptr (stmt);
2449 10705106 : rhs = gimple_assign_rhs1_ptr (stmt);
2450 :
2451 10705106 : if (TREE_CODE (*lhs) == SSA_NAME)
2452 : {
2453 : /* If the statement defines an induction variable, the uses are not
2454 : interesting by themselves. */
2455 :
2456 9569876 : iv = get_iv (data, *lhs);
2457 :
2458 9569876 : if (iv && !integer_zerop (iv->step))
2459 : return;
2460 : }
2461 :
2462 8331942 : code = gimple_assign_rhs_code (stmt);
2463 8331942 : if (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS
2464 8331942 : && (REFERENCE_CLASS_P (*rhs)
2465 1284940 : || is_gimple_val (*rhs)))
2466 : {
2467 2863155 : if (REFERENCE_CLASS_P (*rhs))
2468 1784783 : find_interesting_uses_address (data, stmt, rhs);
2469 : else
2470 1078372 : find_interesting_uses_op (data, *rhs);
2471 :
2472 2863155 : if (REFERENCE_CLASS_P (*lhs))
2473 979446 : find_interesting_uses_address (data, stmt, lhs);
2474 : return;
2475 : }
2476 5468787 : else if (TREE_CODE_CLASS (code) == tcc_comparison)
2477 : {
2478 80188 : find_interesting_uses_cond (data, stmt);
2479 80188 : return;
2480 : }
2481 :
2482 : /* TODO -- we should also handle address uses of type
2483 :
2484 : memory = call (whatever);
2485 :
2486 : and
2487 :
2488 : call (memory). */
2489 : }
2490 :
2491 8893295 : if (gimple_code (stmt) == GIMPLE_PHI
2492 8893295 : && gimple_bb (stmt) == data->current_loop->header)
2493 : {
2494 1475837 : iv = get_iv (data, PHI_RESULT (stmt));
2495 :
2496 1475837 : if (iv && !integer_zerop (iv->step))
2497 : return;
2498 : }
2499 :
2500 26908654 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
2501 : {
2502 10891122 : op = USE_FROM_PTR (use_p);
2503 :
2504 10891122 : if (TREE_CODE (op) != SSA_NAME)
2505 528921 : continue;
2506 :
2507 10362201 : iv = get_iv (data, op);
2508 10362201 : if (!iv)
2509 8582981 : continue;
2510 :
2511 1779220 : if (!find_address_like_use (data, stmt, use_p->use, iv))
2512 1778583 : find_interesting_uses_op (data, op);
2513 : }
2514 : }
2515 :
2516 : /* Finds interesting uses of induction variables outside of loops
2517 : on loop exit edge EXIT. */
2518 :
2519 : static void
2520 913514 : find_interesting_uses_outside (struct ivopts_data *data, edge exit)
2521 : {
2522 913514 : gphi *phi;
2523 913514 : gphi_iterator psi;
2524 913514 : tree def;
2525 :
2526 2037584 : for (psi = gsi_start_phis (exit->dest); !gsi_end_p (psi); gsi_next (&psi))
2527 : {
2528 1124070 : phi = psi.phi ();
2529 1124070 : def = PHI_ARG_DEF_FROM_EDGE (phi, exit);
2530 2155954 : if (!virtual_operand_p (def))
2531 551984 : find_interesting_uses_op (data, def);
2532 : }
2533 913514 : }
2534 :
2535 : /* Return TRUE if OFFSET is within the range of [base + offset] addressing
2536 : mode for memory reference represented by USE. */
2537 :
2538 : static GTY (()) vec<rtx, va_gc> *addr_list;
2539 :
2540 : static bool
2541 225689 : addr_offset_valid_p (struct iv_use *use, poly_int64 offset)
2542 : {
2543 225689 : rtx reg, addr;
2544 225689 : unsigned list_index;
2545 225689 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
2546 225689 : machine_mode addr_mode, mem_mode = TYPE_MODE (use->mem_type);
2547 :
2548 225689 : list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
2549 225689 : if (list_index >= vec_safe_length (addr_list))
2550 10135 : vec_safe_grow_cleared (addr_list, list_index + MAX_MACHINE_MODE, true);
2551 :
2552 225689 : addr = (*addr_list)[list_index];
2553 225689 : if (!addr)
2554 : {
2555 13189 : addr_mode = targetm.addr_space.address_mode (as);
2556 13189 : reg = gen_raw_REG (addr_mode, LAST_VIRTUAL_REGISTER + 1);
2557 13189 : addr = gen_rtx_fmt_ee (PLUS, addr_mode, reg, NULL_RTX);
2558 13189 : (*addr_list)[list_index] = addr;
2559 : }
2560 : else
2561 212500 : addr_mode = GET_MODE (addr);
2562 :
2563 225689 : XEXP (addr, 1) = gen_int_mode (offset, addr_mode);
2564 225689 : return (memory_address_addr_space_p (mem_mode, addr, as));
2565 : }
2566 :
2567 : /* Comparison function to sort group in ascending order of addr_offset. */
2568 :
2569 : static int
2570 3335393 : group_compare_offset (const void *a, const void *b)
2571 : {
2572 3335393 : const struct iv_use *const *u1 = (const struct iv_use *const *) a;
2573 3335393 : const struct iv_use *const *u2 = (const struct iv_use *const *) b;
2574 :
2575 3335393 : return compare_sizes_for_sort ((*u1)->addr_offset, (*u2)->addr_offset);
2576 : }
2577 :
2578 : /* Check if small groups should be split. Return true if no group
2579 : contains more than two uses with distinct addr_offsets. Return
2580 : false otherwise. We want to split such groups because:
2581 :
2582 : 1) Small groups don't have much benefit and may interfere with
2583 : general candidate selection.
2584 : 2) Size for problem with only small groups is usually small and
2585 : general algorithm can handle it well.
2586 :
2587 : TODO -- Above claim may not hold when we want to merge memory
2588 : accesses with conseuctive addresses. */
2589 :
2590 : static bool
2591 510605 : split_small_address_groups_p (struct ivopts_data *data)
2592 : {
2593 510605 : unsigned int i, j, distinct = 1;
2594 510605 : struct iv_use *pre;
2595 510605 : struct iv_group *group;
2596 :
2597 2121425 : for (i = 0; i < data->vgroups.length (); i++)
2598 : {
2599 1610820 : group = data->vgroups[i];
2600 1610820 : if (group->vuses.length () == 1)
2601 1473813 : continue;
2602 :
2603 137007 : gcc_assert (address_p (group->type));
2604 137007 : if (group->vuses.length () == 2)
2605 : {
2606 78105 : if (compare_sizes_for_sort (group->vuses[0]->addr_offset,
2607 78105 : group->vuses[1]->addr_offset) > 0)
2608 19244 : std::swap (group->vuses[0], group->vuses[1]);
2609 : }
2610 : else
2611 58902 : group->vuses.qsort (group_compare_offset);
2612 :
2613 137007 : if (distinct > 2)
2614 13985 : continue;
2615 :
2616 123022 : distinct = 1;
2617 1799725 : for (pre = group->vuses[0], j = 1; j < group->vuses.length (); j++)
2618 : {
2619 188905 : if (maybe_ne (group->vuses[j]->addr_offset, pre->addr_offset))
2620 : {
2621 131567 : pre = group->vuses[j];
2622 131567 : distinct++;
2623 : }
2624 :
2625 188905 : if (distinct > 2)
2626 : break;
2627 : }
2628 : }
2629 :
2630 510605 : return (distinct <= 2);
2631 : }
2632 :
2633 : /* For each group of address type uses, this function further groups
2634 : these uses according to the maximum offset supported by target's
2635 : [base + offset] addressing mode. */
2636 :
2637 : static void
2638 510605 : split_address_groups (struct ivopts_data *data)
2639 : {
2640 510605 : unsigned int i, j;
2641 : /* Always split group. */
2642 510605 : bool split_p = split_small_address_groups_p (data);
2643 :
2644 2683784 : for (i = 0; i < data->vgroups.length (); i++)
2645 : {
2646 1662574 : struct iv_group *new_group = NULL;
2647 1662574 : struct iv_group *group = data->vgroups[i];
2648 1662574 : struct iv_use *use = group->vuses[0];
2649 :
2650 1662574 : use->id = 0;
2651 1662574 : use->group_id = group->id;
2652 1662574 : if (group->vuses.length () == 1)
2653 1519828 : continue;
2654 :
2655 142746 : gcc_assert (address_p (use->type));
2656 :
2657 2003656 : for (j = 1; j < group->vuses.length ();)
2658 : {
2659 341082 : struct iv_use *next = group->vuses[j];
2660 341082 : poly_int64 offset = next->addr_offset - use->addr_offset;
2661 :
2662 : /* Split group if asked to, or the offset against the first
2663 : use can't fit in offset part of addressing mode. IV uses
2664 : having the same offset are still kept in one group. */
2665 399021 : if (maybe_ne (offset, 0)
2666 341082 : && (split_p || !addr_offset_valid_p (use, offset)))
2667 : {
2668 57939 : if (!new_group)
2669 51754 : new_group = record_group (data, group->type);
2670 57939 : group->vuses.ordered_remove (j);
2671 57939 : new_group->vuses.safe_push (next);
2672 57939 : continue;
2673 : }
2674 :
2675 283143 : next->id = j;
2676 283143 : next->group_id = group->id;
2677 283143 : j++;
2678 : }
2679 : }
2680 510605 : }
2681 :
2682 : /* Finds uses of the induction variables that are interesting. */
2683 :
2684 : static void
2685 510605 : find_interesting_uses (struct ivopts_data *data, basic_block *body)
2686 : {
2687 510605 : basic_block bb;
2688 510605 : gimple_stmt_iterator bsi;
2689 510605 : unsigned i;
2690 510605 : edge e;
2691 :
2692 3384423 : for (i = 0; i < data->current_loop->num_nodes; i++)
2693 : {
2694 2873818 : edge_iterator ei;
2695 2873818 : bb = body[i];
2696 :
2697 7343322 : FOR_EACH_EDGE (e, ei, bb->succs)
2698 4469504 : if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
2699 4469504 : && !flow_bb_inside_loop_p (data->current_loop, e->dest))
2700 913514 : find_interesting_uses_outside (data, e);
2701 :
2702 5782871 : for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2703 2909053 : find_interesting_uses_stmt (data, gsi_stmt (bsi));
2704 28729120 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2705 22981484 : if (!is_gimple_debug (gsi_stmt (bsi)))
2706 12795434 : find_interesting_uses_stmt (data, gsi_stmt (bsi));
2707 : }
2708 :
2709 510605 : split_address_groups (data);
2710 :
2711 510605 : if (dump_file && (dump_flags & TDF_DETAILS))
2712 : {
2713 70 : fprintf (dump_file, "\n<IV Groups>:\n");
2714 70 : dump_groups (dump_file, data);
2715 70 : fprintf (dump_file, "\n");
2716 : }
2717 510605 : }
2718 :
2719 : /* Strips constant offsets from EXPR and stores them to OFFSET. If INSIDE_ADDR
2720 : is true, assume we are inside an address. If TOP_COMPREF is true, assume
2721 : we are at the top-level of the processed address. */
2722 :
2723 : static tree
2724 3416044 : strip_offset_1 (tree expr, bool inside_addr, bool top_compref,
2725 : poly_int64 *offset)
2726 : {
2727 3416044 : tree op0 = NULL_TREE, op1 = NULL_TREE, tmp, step;
2728 3416044 : enum tree_code code;
2729 3416044 : tree type, orig_type = TREE_TYPE (expr);
2730 3416044 : poly_int64 off0, off1;
2731 3416044 : HOST_WIDE_INT st;
2732 3416044 : tree orig_expr = expr;
2733 :
2734 3416044 : STRIP_NOPS (expr);
2735 :
2736 3416044 : type = TREE_TYPE (expr);
2737 3416044 : code = TREE_CODE (expr);
2738 3416044 : *offset = 0;
2739 :
2740 3416044 : switch (code)
2741 : {
2742 626643 : case POINTER_PLUS_EXPR:
2743 626643 : case PLUS_EXPR:
2744 626643 : case MINUS_EXPR:
2745 626643 : op0 = TREE_OPERAND (expr, 0);
2746 626643 : op1 = TREE_OPERAND (expr, 1);
2747 :
2748 626643 : op0 = strip_offset_1 (op0, false, false, &off0);
2749 626643 : op1 = strip_offset_1 (op1, false, false, &off1);
2750 :
2751 626643 : *offset = (code == MINUS_EXPR ? off0 - off1 : off0 + off1);
2752 626643 : if (op0 == TREE_OPERAND (expr, 0)
2753 626643 : && op1 == TREE_OPERAND (expr, 1))
2754 : return orig_expr;
2755 :
2756 389391 : if (integer_zerop (op1))
2757 : expr = op0;
2758 3247 : else if (integer_zerop (op0))
2759 : {
2760 691 : if (code == MINUS_EXPR)
2761 : {
2762 691 : if (TYPE_OVERFLOW_UNDEFINED (type))
2763 : {
2764 0 : type = unsigned_type_for (type);
2765 0 : op1 = fold_convert (type, op1);
2766 : }
2767 691 : expr = fold_build1 (NEGATE_EXPR, type, op1);
2768 : }
2769 : else
2770 : expr = op1;
2771 : }
2772 : else
2773 : {
2774 2556 : if (TYPE_OVERFLOW_UNDEFINED (type))
2775 : {
2776 0 : type = unsigned_type_for (type);
2777 0 : if (code == POINTER_PLUS_EXPR)
2778 0 : code = PLUS_EXPR;
2779 0 : op0 = fold_convert (type, op0);
2780 0 : op1 = fold_convert (type, op1);
2781 : }
2782 2556 : expr = fold_build2 (code, type, op0, op1);
2783 : }
2784 :
2785 389391 : return fold_convert (orig_type, expr);
2786 :
2787 218521 : case MULT_EXPR:
2788 218521 : op1 = TREE_OPERAND (expr, 1);
2789 218521 : if (!cst_and_fits_in_hwi (op1))
2790 : return orig_expr;
2791 :
2792 177692 : op0 = TREE_OPERAND (expr, 0);
2793 177692 : op0 = strip_offset_1 (op0, false, false, &off0);
2794 177692 : if (op0 == TREE_OPERAND (expr, 0))
2795 : return orig_expr;
2796 :
2797 7101 : *offset = off0 * int_cst_value (op1);
2798 7101 : if (integer_zerop (op0))
2799 : expr = op0;
2800 : else
2801 : {
2802 7101 : if (TYPE_OVERFLOW_UNDEFINED (type))
2803 : {
2804 0 : type = unsigned_type_for (type);
2805 0 : op0 = fold_convert (type, op0);
2806 0 : op1 = fold_convert (type, op1);
2807 : }
2808 7101 : expr = fold_build2 (MULT_EXPR, type, op0, op1);
2809 : }
2810 :
2811 7101 : return fold_convert (orig_type, expr);
2812 :
2813 9 : case ARRAY_REF:
2814 9 : case ARRAY_RANGE_REF:
2815 9 : if (!inside_addr)
2816 : return orig_expr;
2817 :
2818 9 : step = array_ref_element_size (expr);
2819 9 : if (!cst_and_fits_in_hwi (step))
2820 : break;
2821 :
2822 9 : st = int_cst_value (step);
2823 9 : op1 = TREE_OPERAND (expr, 1);
2824 9 : op1 = strip_offset_1 (op1, false, false, &off1);
2825 9 : *offset = off1 * st;
2826 :
2827 9 : if (top_compref
2828 9 : && integer_zerop (op1))
2829 : {
2830 : /* Strip the component reference completely. */
2831 7 : op0 = TREE_OPERAND (expr, 0);
2832 7 : op0 = strip_offset_1 (op0, inside_addr, top_compref, &off0);
2833 7 : *offset += off0;
2834 7 : return op0;
2835 : }
2836 : break;
2837 :
2838 1 : case COMPONENT_REF:
2839 1 : {
2840 1 : tree field;
2841 :
2842 1 : if (!inside_addr)
2843 : return orig_expr;
2844 :
2845 1 : tmp = component_ref_field_offset (expr);
2846 1 : field = TREE_OPERAND (expr, 1);
2847 1 : if (top_compref
2848 1 : && cst_and_fits_in_hwi (tmp)
2849 2 : && cst_and_fits_in_hwi (DECL_FIELD_BIT_OFFSET (field)))
2850 : {
2851 1 : HOST_WIDE_INT boffset, abs_off;
2852 :
2853 : /* Strip the component reference completely. */
2854 1 : op0 = TREE_OPERAND (expr, 0);
2855 1 : op0 = strip_offset_1 (op0, inside_addr, top_compref, &off0);
2856 1 : boffset = int_cst_value (DECL_FIELD_BIT_OFFSET (field));
2857 1 : abs_off = abs_hwi (boffset) / BITS_PER_UNIT;
2858 1 : if (boffset < 0)
2859 0 : abs_off = -abs_off;
2860 :
2861 1 : *offset = off0 + int_cst_value (tmp) + abs_off;
2862 1 : return op0;
2863 : }
2864 : }
2865 : break;
2866 :
2867 322780 : case ADDR_EXPR:
2868 322780 : op0 = TREE_OPERAND (expr, 0);
2869 322780 : op0 = strip_offset_1 (op0, true, true, &off0);
2870 322780 : *offset += off0;
2871 :
2872 322780 : if (op0 == TREE_OPERAND (expr, 0))
2873 : return orig_expr;
2874 :
2875 8 : expr = build_fold_addr_expr (op0);
2876 8 : return fold_convert (orig_type, expr);
2877 :
2878 : case MEM_REF:
2879 : /* ??? Offset operand? */
2880 : inside_addr = false;
2881 : break;
2882 :
2883 2248088 : default:
2884 2248088 : if (ptrdiff_tree_p (expr, offset) && maybe_ne (*offset, 0))
2885 876882 : return build_int_cst (orig_type, 0);
2886 : return orig_expr;
2887 : }
2888 :
2889 : /* Default handling of expressions for that we want to recurse into
2890 : the first operand. */
2891 4 : op0 = TREE_OPERAND (expr, 0);
2892 4 : op0 = strip_offset_1 (op0, inside_addr, false, &off0);
2893 4 : *offset += off0;
2894 :
2895 4 : if (op0 == TREE_OPERAND (expr, 0)
2896 4 : && (!op1 || op1 == TREE_OPERAND (expr, 1)))
2897 : return orig_expr;
2898 :
2899 1 : expr = copy_node (expr);
2900 1 : TREE_OPERAND (expr, 0) = op0;
2901 1 : if (op1)
2902 1 : TREE_OPERAND (expr, 1) = op1;
2903 :
2904 : /* Inside address, we might strip the top level component references,
2905 : thus changing type of the expression. Handling of ADDR_EXPR
2906 : will fix that. */
2907 1 : expr = fold_convert (orig_type, expr);
2908 :
2909 1 : return expr;
2910 : }
2911 :
2912 : /* Strips constant offsets from EXPR and stores them to OFFSET. */
2913 :
2914 : static tree
2915 1662265 : strip_offset (tree expr, poly_uint64 *offset)
2916 : {
2917 1662265 : poly_int64 off;
2918 1662265 : tree core = strip_offset_1 (expr, false, false, &off);
2919 1662265 : *offset = off;
2920 1662265 : return core;
2921 : }
2922 :
2923 : /* Returns variant of TYPE that can be used as base for different uses.
2924 : We return unsigned type with the same precision, which avoids problems
2925 : with overflows. */
2926 :
2927 : static tree
2928 8158846 : generic_type_for (tree type)
2929 : {
2930 8158846 : if (POINTER_TYPE_P (type))
2931 1456941 : return unsigned_type_for (type);
2932 :
2933 6701905 : if (TYPE_UNSIGNED (type))
2934 : return type;
2935 :
2936 3128804 : return unsigned_type_for (type);
2937 : }
2938 :
2939 : /* Private data for walk_tree. */
2940 :
2941 : struct walk_tree_data
2942 : {
2943 : bitmap *inv_vars;
2944 : struct ivopts_data *idata;
2945 : };
2946 :
2947 : /* Callback function for walk_tree, it records invariants and symbol
2948 : reference in *EXPR_P. DATA is the structure storing result info. */
2949 :
2950 : static tree
2951 34330169 : find_inv_vars_cb (tree *expr_p, int *ws ATTRIBUTE_UNUSED, void *data)
2952 : {
2953 34330169 : tree op = *expr_p;
2954 34330169 : struct version_info *info;
2955 34330169 : struct walk_tree_data *wdata = (struct walk_tree_data*) data;
2956 :
2957 34330169 : if (TREE_CODE (op) != SSA_NAME)
2958 : return NULL_TREE;
2959 :
2960 8020389 : info = name_info (wdata->idata, op);
2961 : /* Because we expand simple operations when finding IVs, loop invariant
2962 : variable that isn't referred by the original loop could be used now.
2963 : Record such invariant variables here. */
2964 8020389 : if (!info->iv)
2965 : {
2966 390429 : struct ivopts_data *idata = wdata->idata;
2967 390429 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
2968 :
2969 390429 : if (!bb || !flow_bb_inside_loop_p (idata->current_loop, bb))
2970 : {
2971 390429 : tree steptype = TREE_TYPE (op);
2972 390429 : if (POINTER_TYPE_P (steptype))
2973 194537 : steptype = sizetype;
2974 390429 : set_iv (idata, op, op, build_int_cst (steptype, 0), true);
2975 390429 : record_invariant (idata, op, false);
2976 : }
2977 : }
2978 8020389 : if (!info->inv_id || info->has_nonlin_use)
2979 : return NULL_TREE;
2980 :
2981 6636662 : if (!*wdata->inv_vars)
2982 5145804 : *wdata->inv_vars = BITMAP_ALLOC (NULL);
2983 6636662 : bitmap_set_bit (*wdata->inv_vars, info->inv_id);
2984 :
2985 6636662 : return NULL_TREE;
2986 : }
2987 :
2988 : /* Records invariants in *EXPR_P. INV_VARS is the bitmap to that we should
2989 : store it. */
2990 :
2991 : static inline void
2992 28049539 : find_inv_vars (struct ivopts_data *data, tree *expr_p, bitmap *inv_vars)
2993 : {
2994 28049539 : struct walk_tree_data wdata;
2995 :
2996 28049539 : if (!inv_vars)
2997 : return;
2998 :
2999 16104542 : wdata.idata = data;
3000 16104542 : wdata.inv_vars = inv_vars;
3001 16104542 : walk_tree (expr_p, find_inv_vars_cb, &wdata, NULL);
3002 : }
3003 :
3004 : /* Get entry from invariant expr hash table for INV_EXPR. New entry
3005 : will be recorded if it doesn't exist yet. Given below two exprs:
3006 : inv_expr + cst1, inv_expr + cst2
3007 : It's hard to make decision whether constant part should be stripped
3008 : or not. We choose to not strip based on below facts:
3009 : 1) We need to count ADD cost for constant part if it's stripped,
3010 : which isn't always trivial where this functions is called.
3011 : 2) Stripping constant away may be conflict with following loop
3012 : invariant hoisting pass.
3013 : 3) Not stripping constant away results in more invariant exprs,
3014 : which usually leads to decision preferring lower reg pressure. */
3015 :
3016 : static iv_inv_expr_ent *
3017 2612098 : get_loop_invariant_expr (struct ivopts_data *data, tree inv_expr)
3018 : {
3019 2612098 : STRIP_NOPS (inv_expr);
3020 :
3021 2612098 : if (poly_int_tree_p (inv_expr)
3022 2612098 : || TREE_CODE (inv_expr) == SSA_NAME)
3023 : return NULL;
3024 :
3025 : /* Don't strip constant part away as we used to. */
3026 :
3027 : /* Stores EXPR in DATA->inv_expr_tab, return pointer to iv_inv_expr_ent. */
3028 2523792 : struct iv_inv_expr_ent ent;
3029 2523792 : ent.expr = inv_expr;
3030 2523792 : ent.hash = iterative_hash_expr (inv_expr, 0);
3031 2523792 : struct iv_inv_expr_ent **slot = data->inv_expr_tab->find_slot (&ent, INSERT);
3032 :
3033 2523792 : if (!*slot)
3034 : {
3035 1156237 : *slot = XNEW (struct iv_inv_expr_ent);
3036 1156237 : (*slot)->expr = inv_expr;
3037 1156237 : (*slot)->hash = ent.hash;
3038 1156237 : (*slot)->id = ++data->max_inv_expr_id;
3039 : }
3040 :
3041 2523792 : return *slot;
3042 : }
3043 :
3044 :
3045 : /* Return *TP if it is an SSA_NAME marked with TREE_VISITED, i.e., as
3046 : unsuitable as ivopts candidates for potentially involving undefined
3047 : behavior. */
3048 :
3049 : static tree
3050 15514424 : find_ssa_undef (tree *tp, int *walk_subtrees, void *bb_)
3051 : {
3052 15514424 : basic_block bb = (basic_block) bb_;
3053 15514424 : if (TREE_CODE (*tp) == SSA_NAME
3054 2267953 : && ssa_name_maybe_undef_p (*tp)
3055 15523253 : && !ssa_name_any_use_dominates_bb_p (*tp, bb))
3056 3176 : return *tp;
3057 15511248 : if (!EXPR_P (*tp))
3058 10520755 : *walk_subtrees = 0;
3059 : return NULL;
3060 : }
3061 :
3062 : /* Adds a candidate BASE + STEP * i. Important field is set to IMPORTANT and
3063 : position to POS. If USE is not NULL, the candidate is set as related to
3064 : it. If both BASE and STEP are NULL, we add a pseudocandidate for the
3065 : replacement of the final value of the iv by a direct computation. */
3066 :
3067 : static struct iv_cand *
3068 9148217 : add_candidate_1 (struct ivopts_data *data, tree base, tree step, bool important,
3069 : enum iv_position pos, struct iv_use *use,
3070 : gimple *incremented_at, struct iv *orig_iv = NULL,
3071 : bool doloop = false)
3072 : {
3073 9148217 : unsigned i;
3074 9148217 : struct iv_cand *cand = NULL;
3075 9148217 : tree type, orig_type;
3076 :
3077 9148217 : gcc_assert (base && step);
3078 :
3079 : /* -fkeep-gc-roots-live means that we have to keep a real pointer
3080 : live, but the ivopts code may replace a real pointer with one
3081 : pointing before or after the memory block that is then adjusted
3082 : into the memory block during the loop. FIXME: It would likely be
3083 : better to actually force the pointer live and still use ivopts;
3084 : for example, it would be enough to write the pointer into memory
3085 : and keep it there until after the loop. */
3086 9148217 : if (flag_keep_gc_roots_live && POINTER_TYPE_P (TREE_TYPE (base)))
3087 : return NULL;
3088 :
3089 : /* If BASE contains undefined SSA names make sure we only record
3090 : the original IV. */
3091 9042437 : bool involves_undefs = false;
3092 9042437 : if (walk_tree (&base, find_ssa_undef, data->current_loop->header, NULL))
3093 : {
3094 3176 : if (pos != IP_ORIGINAL)
3095 : return NULL;
3096 : important = false;
3097 : involves_undefs = true;
3098 : }
3099 :
3100 : /* For non-original variables, make sure their values are computed in a type
3101 : that does not invoke undefined behavior on overflows (since in general,
3102 : we cannot prove that these induction variables are non-wrapping). */
3103 9039261 : if (pos != IP_ORIGINAL)
3104 : {
3105 8158846 : orig_type = TREE_TYPE (base);
3106 8158846 : type = generic_type_for (orig_type);
3107 8158846 : if (type != orig_type)
3108 : {
3109 4585745 : base = fold_convert (type, base);
3110 4585745 : step = fold_convert (type, step);
3111 : }
3112 : }
3113 :
3114 45136491 : for (i = 0; i < data->vcands.length (); i++)
3115 : {
3116 40462679 : cand = data->vcands[i];
3117 :
3118 40462679 : if (cand->pos != pos)
3119 9946042 : continue;
3120 :
3121 30516637 : if (cand->incremented_at != incremented_at
3122 30013674 : || ((pos == IP_AFTER_USE || pos == IP_BEFORE_USE)
3123 0 : && cand->ainc_use != use))
3124 502963 : continue;
3125 :
3126 30013674 : if (operand_equal_p (base, cand->iv->base, 0)
3127 9566092 : && operand_equal_p (step, cand->iv->step, 0)
3128 35785894 : && (TYPE_PRECISION (TREE_TYPE (base))
3129 5772220 : == TYPE_PRECISION (TREE_TYPE (cand->iv->base))))
3130 : break;
3131 : }
3132 :
3133 18079288 : if (i == data->vcands.length ())
3134 : {
3135 4673812 : cand = XCNEW (struct iv_cand);
3136 4673812 : cand->id = i;
3137 4673812 : cand->iv = alloc_iv (data, base, step);
3138 4673812 : cand->pos = pos;
3139 4673812 : if (pos != IP_ORIGINAL)
3140 : {
3141 3793190 : if (doloop)
3142 0 : cand->var_before = create_tmp_var_raw (TREE_TYPE (base), "doloop");
3143 : else
3144 3793190 : cand->var_before = create_tmp_var_raw (TREE_TYPE (base), "ivtmp");
3145 3793190 : cand->var_after = cand->var_before;
3146 : }
3147 4673812 : cand->important = important;
3148 4673812 : cand->involves_undefs = involves_undefs;
3149 4673812 : cand->incremented_at = incremented_at;
3150 4673812 : cand->doloop_p = doloop;
3151 4673812 : data->vcands.safe_push (cand);
3152 :
3153 4673812 : if (!poly_int_tree_p (step))
3154 : {
3155 175551 : find_inv_vars (data, &step, &cand->inv_vars);
3156 :
3157 175551 : iv_inv_expr_ent *inv_expr = get_loop_invariant_expr (data, step);
3158 : /* Share bitmap between inv_vars and inv_exprs for cand. */
3159 175551 : if (inv_expr != NULL)
3160 : {
3161 93625 : cand->inv_exprs = cand->inv_vars;
3162 93625 : cand->inv_vars = NULL;
3163 93625 : if (cand->inv_exprs)
3164 75254 : bitmap_clear (cand->inv_exprs);
3165 : else
3166 18371 : cand->inv_exprs = BITMAP_ALLOC (NULL);
3167 :
3168 93625 : bitmap_set_bit (cand->inv_exprs, inv_expr->id);
3169 : }
3170 : }
3171 :
3172 4673812 : if (pos == IP_AFTER_USE || pos == IP_BEFORE_USE)
3173 : cand->ainc_use = use;
3174 : else
3175 4673812 : cand->ainc_use = NULL;
3176 :
3177 4673812 : cand->orig_iv = orig_iv;
3178 4673812 : if (dump_file && (dump_flags & TDF_DETAILS))
3179 714 : dump_cand (dump_file, cand);
3180 : }
3181 :
3182 9039644 : cand->important |= important;
3183 9039644 : cand->doloop_p |= doloop;
3184 :
3185 : /* Relate candidate to the group for which it is added. */
3186 9039644 : if (use)
3187 2525168 : bitmap_set_bit (data->vgroups[use->group_id]->related_cands, i);
3188 :
3189 : return cand;
3190 : }
3191 :
3192 : /* Returns true if incrementing the induction variable at the end of the LOOP
3193 : is allowed.
3194 :
3195 : The purpose is to avoid splitting latch edge with a biv increment, thus
3196 : creating a jump, possibly confusing other optimization passes and leaving
3197 : less freedom to scheduler. So we allow IP_END only if IP_NORMAL is not
3198 : available (so we do not have a better alternative), or if the latch edge
3199 : is already nonempty. */
3200 :
3201 : static bool
3202 8040187 : allow_ip_end_pos_p (class loop *loop)
3203 : {
3204 : /* Do not allow IP_END when creating the IV would need to split the
3205 : latch edge as that makes all IP_NORMAL invalid. */
3206 8040187 : auto pos = gsi_last_bb (ip_end_pos (loop));
3207 8040187 : if (!gsi_end_p (pos) && stmt_ends_bb_p (*pos))
3208 : return false;
3209 :
3210 8040187 : if (!ip_normal_pos (loop))
3211 : return true;
3212 :
3213 7941326 : if (!empty_block_p (ip_end_pos (loop)))
3214 224877 : return true;
3215 :
3216 : return false;
3217 : }
3218 :
3219 : /* If possible, adds autoincrement candidates BASE + STEP * i based on use USE.
3220 : Important field is set to IMPORTANT. */
3221 :
3222 : static void
3223 587199 : add_autoinc_candidates (struct ivopts_data *data, tree base, tree step,
3224 : bool important, struct iv_use *use)
3225 : {
3226 587199 : basic_block use_bb = gimple_bb (use->stmt);
3227 587199 : machine_mode mem_mode;
3228 587199 : unsigned HOST_WIDE_INT cstepi;
3229 :
3230 : /* If we insert the increment in any position other than the standard
3231 : ones, we must ensure that it is incremented once per iteration.
3232 : It must not be in an inner nested loop, or one side of an if
3233 : statement. */
3234 587199 : if (use_bb->loop_father != data->current_loop
3235 585528 : || !dominated_by_p (CDI_DOMINATORS, data->current_loop->latch, use_bb)
3236 559224 : || stmt_can_throw_internal (cfun, use->stmt)
3237 1142601 : || !cst_and_fits_in_hwi (step))
3238 : return;
3239 :
3240 526026 : cstepi = int_cst_value (step);
3241 :
3242 526026 : mem_mode = TYPE_MODE (use->mem_type);
3243 : if (((USE_LOAD_PRE_INCREMENT (mem_mode)
3244 : || USE_STORE_PRE_INCREMENT (mem_mode))
3245 : && known_eq (GET_MODE_SIZE (mem_mode), cstepi))
3246 : || ((USE_LOAD_PRE_DECREMENT (mem_mode)
3247 : || USE_STORE_PRE_DECREMENT (mem_mode))
3248 : && known_eq (GET_MODE_SIZE (mem_mode), -cstepi)))
3249 : {
3250 : enum tree_code code = MINUS_EXPR;
3251 : tree new_base;
3252 : tree new_step = step;
3253 :
3254 : if (POINTER_TYPE_P (TREE_TYPE (base)))
3255 : {
3256 : new_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (step), step);
3257 : code = POINTER_PLUS_EXPR;
3258 : }
3259 : else
3260 : new_step = fold_convert (TREE_TYPE (base), new_step);
3261 : new_base = fold_build2 (code, TREE_TYPE (base), base, new_step);
3262 : add_candidate_1 (data, new_base, step, important, IP_BEFORE_USE, use,
3263 : use->stmt);
3264 : }
3265 : if (((USE_LOAD_POST_INCREMENT (mem_mode)
3266 : || USE_STORE_POST_INCREMENT (mem_mode))
3267 : && known_eq (GET_MODE_SIZE (mem_mode), cstepi))
3268 : || ((USE_LOAD_POST_DECREMENT (mem_mode)
3269 : || USE_STORE_POST_DECREMENT (mem_mode))
3270 : && known_eq (GET_MODE_SIZE (mem_mode), -cstepi)))
3271 : {
3272 : add_candidate_1 (data, base, step, important, IP_AFTER_USE, use,
3273 : use->stmt);
3274 : }
3275 : }
3276 :
3277 : /* Adds a candidate BASE + STEP * i. Important field is set to IMPORTANT and
3278 : position to POS. If USE is not NULL, the candidate is set as related to
3279 : it. The candidate computation is scheduled before exit condition and at
3280 : the end of loop. */
3281 :
3282 : static void
3283 7072596 : add_candidate (struct ivopts_data *data, tree base, tree step, bool important,
3284 : struct iv_use *use, struct iv *orig_iv = NULL,
3285 : bool doloop = false)
3286 : {
3287 7072596 : if (ip_normal_pos (data->current_loop))
3288 6988735 : add_candidate_1 (data, base, step, important, IP_NORMAL, use, NULL, orig_iv,
3289 : doloop);
3290 : /* Exclude doloop candidate here since it requires decrement then comparison
3291 : and jump, the IP_END position doesn't match. */
3292 7072596 : if (!doloop && ip_end_pos (data->current_loop)
3293 14145192 : && allow_ip_end_pos_p (data->current_loop))
3294 278147 : add_candidate_1 (data, base, step, important, IP_END, use, NULL, orig_iv);
3295 7072596 : }
3296 :
3297 : /* Adds standard iv candidates. */
3298 :
3299 : static void
3300 510604 : add_standard_iv_candidates (struct ivopts_data *data)
3301 : {
3302 510604 : add_candidate (data, integer_zero_node, integer_one_node, true, NULL);
3303 :
3304 : /* The same for a double-integer type if it is still fast enough. */
3305 510604 : if (TYPE_PRECISION
3306 510604 : (long_integer_type_node) > TYPE_PRECISION (integer_type_node)
3307 510604 : && TYPE_PRECISION (long_integer_type_node) <= BITS_PER_WORD)
3308 462450 : add_candidate (data, build_int_cst (long_integer_type_node, 0),
3309 : build_int_cst (long_integer_type_node, 1), true, NULL);
3310 :
3311 : /* The same for a double-integer type if it is still fast enough. */
3312 510604 : if (TYPE_PRECISION
3313 510604 : (long_long_integer_type_node) > TYPE_PRECISION (long_integer_type_node)
3314 558746 : && TYPE_PRECISION (long_long_integer_type_node) <= BITS_PER_WORD)
3315 12 : add_candidate (data, build_int_cst (long_long_integer_type_node, 0),
3316 : build_int_cst (long_long_integer_type_node, 1), true, NULL);
3317 510604 : }
3318 :
3319 :
3320 : /* Adds candidates bases on the old induction variable IV. */
3321 :
3322 : static void
3323 1766125 : add_iv_candidate_for_biv (struct ivopts_data *data, struct iv *iv)
3324 : {
3325 1766125 : gimple *phi;
3326 1766125 : tree def;
3327 1766125 : struct iv_cand *cand;
3328 :
3329 : /* Check if this biv is used in address type use. */
3330 1172339 : if (iv->no_overflow && iv->have_address_use
3331 498336 : && INTEGRAL_TYPE_P (TREE_TYPE (iv->base))
3332 2264461 : && TYPE_PRECISION (TREE_TYPE (iv->base)) < TYPE_PRECISION (sizetype))
3333 : {
3334 281734 : tree base = fold_convert (sizetype, iv->base);
3335 281734 : tree step = fold_convert (sizetype, iv->step);
3336 :
3337 : /* Add iv cand of same precision as index part in TARGET_MEM_REF. */
3338 281734 : add_candidate (data, base, step, true, NULL, iv);
3339 : /* Add iv cand of the original type only if it has nonlinear use. */
3340 281734 : if (iv->nonlin_use)
3341 27824 : add_candidate (data, iv->base, iv->step, true, NULL);
3342 : }
3343 : else
3344 1484391 : add_candidate (data, iv->base, iv->step, true, NULL);
3345 :
3346 : /* The same, but with initial value zero. */
3347 1766125 : if (POINTER_TYPE_P (TREE_TYPE (iv->base)))
3348 338092 : add_candidate (data, size_int (0), iv->step, true, NULL);
3349 : else
3350 1428033 : add_candidate (data, build_int_cst (TREE_TYPE (iv->base), 0),
3351 : iv->step, true, NULL);
3352 :
3353 1766125 : phi = SSA_NAME_DEF_STMT (iv->ssa_name);
3354 1766125 : if (gimple_code (phi) == GIMPLE_PHI)
3355 : {
3356 : /* Additionally record the possibility of leaving the original iv
3357 : untouched. */
3358 883153 : def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (data->current_loop));
3359 : /* Don't add candidate if it's from another PHI node because
3360 : it's an affine iv appearing in the form of PEELED_CHREC. */
3361 883153 : phi = SSA_NAME_DEF_STMT (def);
3362 883153 : if (gimple_code (phi) != GIMPLE_PHI)
3363 : {
3364 1766306 : cand = add_candidate_1 (data,
3365 : iv->base, iv->step, true, IP_ORIGINAL, NULL,
3366 883153 : SSA_NAME_DEF_STMT (def));
3367 883153 : if (cand)
3368 : {
3369 880798 : cand->var_before = iv->ssa_name;
3370 880798 : cand->var_after = def;
3371 : }
3372 : }
3373 : else
3374 0 : gcc_assert (gimple_bb (phi) == data->current_loop->header);
3375 : }
3376 1766125 : }
3377 :
3378 : /* Adds candidates based on the old induction variables. */
3379 :
3380 : static void
3381 510604 : add_iv_candidate_for_bivs (struct ivopts_data *data)
3382 : {
3383 510604 : unsigned i;
3384 510604 : struct iv *iv;
3385 510604 : bitmap_iterator bi;
3386 :
3387 5566180 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
3388 : {
3389 5055576 : iv = ver_info (data, i)->iv;
3390 5055576 : if (iv && iv->biv_p && !integer_zerop (iv->step))
3391 1766125 : add_iv_candidate_for_biv (data, iv);
3392 : }
3393 510604 : }
3394 :
3395 : /* Record common candidate {BASE, STEP} derived from USE in hashtable. */
3396 :
3397 : static void
3398 4201416 : record_common_cand (struct ivopts_data *data, tree base,
3399 : tree step, struct iv_use *use)
3400 : {
3401 4201416 : class iv_common_cand ent;
3402 4201416 : class iv_common_cand **slot;
3403 :
3404 4201416 : ent.base = base;
3405 4201416 : ent.step = step;
3406 4201416 : ent.hash = iterative_hash_expr (base, 0);
3407 4201416 : ent.hash = iterative_hash_expr (step, ent.hash);
3408 :
3409 4201416 : slot = data->iv_common_cand_tab->find_slot (&ent, INSERT);
3410 4201416 : if (*slot == NULL)
3411 : {
3412 2641902 : *slot = new iv_common_cand ();
3413 2641902 : (*slot)->base = base;
3414 2641902 : (*slot)->step = step;
3415 2641902 : (*slot)->uses.create (8);
3416 2641902 : (*slot)->hash = ent.hash;
3417 2641902 : data->iv_common_cands.safe_push ((*slot));
3418 : }
3419 :
3420 4201416 : gcc_assert (use != NULL);
3421 4201416 : (*slot)->uses.safe_push (use);
3422 4201416 : return;
3423 4201416 : }
3424 :
3425 : /* Comparison function used to sort common candidates. */
3426 :
3427 : static int
3428 19203299 : common_cand_cmp (const void *p1, const void *p2)
3429 : {
3430 19203299 : unsigned n1, n2;
3431 19203299 : const class iv_common_cand *const *const ccand1
3432 : = (const class iv_common_cand *const *)p1;
3433 19203299 : const class iv_common_cand *const *const ccand2
3434 : = (const class iv_common_cand *const *)p2;
3435 :
3436 19203299 : n1 = (*ccand1)->uses.length ();
3437 19203299 : n2 = (*ccand2)->uses.length ();
3438 19203299 : return n2 - n1;
3439 : }
3440 :
3441 : /* Adds IV candidates based on common candidated recorded. */
3442 :
3443 : static void
3444 510604 : add_iv_candidate_derived_from_uses (struct ivopts_data *data)
3445 : {
3446 510604 : unsigned i, j;
3447 510604 : struct iv_cand *cand_1, *cand_2;
3448 :
3449 510604 : data->iv_common_cands.qsort (common_cand_cmp);
3450 1478195 : for (i = 0; i < data->iv_common_cands.length (); i++)
3451 : {
3452 1461661 : class iv_common_cand *ptr = data->iv_common_cands[i];
3453 :
3454 : /* Only add IV candidate if it's derived from multiple uses. */
3455 1461661 : if (ptr->uses.length () <= 1)
3456 : break;
3457 :
3458 967591 : cand_1 = NULL;
3459 967591 : cand_2 = NULL;
3460 967591 : if (ip_normal_pos (data->current_loop))
3461 952591 : cand_1 = add_candidate_1 (data, ptr->base, ptr->step,
3462 : false, IP_NORMAL, NULL, NULL);
3463 :
3464 967591 : if (ip_end_pos (data->current_loop)
3465 967591 : && allow_ip_end_pos_p (data->current_loop))
3466 45591 : cand_2 = add_candidate_1 (data, ptr->base, ptr->step,
3467 : false, IP_END, NULL, NULL);
3468 :
3469 : /* Bind deriving uses and the new candidates. */
3470 3494696 : for (j = 0; j < ptr->uses.length (); j++)
3471 : {
3472 2527105 : struct iv_group *group = data->vgroups[ptr->uses[j]->group_id];
3473 2527105 : if (cand_1)
3474 2453475 : bitmap_set_bit (group->related_cands, cand_1->id);
3475 2527105 : if (cand_2)
3476 134981 : bitmap_set_bit (group->related_cands, cand_2->id);
3477 : }
3478 : }
3479 :
3480 : /* Release data since it is useless from this point. */
3481 510604 : data->iv_common_cand_tab->empty ();
3482 510604 : data->iv_common_cands.truncate (0);
3483 510604 : }
3484 :
3485 : /* Adds candidates based on the value of USE's iv. */
3486 :
3487 : static void
3488 1662570 : add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use)
3489 : {
3490 1662570 : poly_uint64 offset;
3491 1662570 : tree base;
3492 1662570 : struct iv *iv = use->iv;
3493 1662570 : tree basetype = TREE_TYPE (iv->base);
3494 :
3495 : /* Don't add candidate for iv_use with non integer, pointer or non-mode
3496 : precision types, instead, add candidate for the corresponding scev in
3497 : unsigned type with the same precision. See PR93674 for more info. */
3498 785344 : if ((TREE_CODE (basetype) != INTEGER_TYPE && !POINTER_TYPE_P (basetype))
3499 2447644 : || !type_has_mode_precision_p (basetype))
3500 : {
3501 305 : basetype = lang_hooks.types.type_for_mode (TYPE_MODE (basetype),
3502 305 : TYPE_UNSIGNED (basetype));
3503 305 : add_candidate (data, fold_convert (basetype, iv->base),
3504 : fold_convert (basetype, iv->step), false, NULL);
3505 305 : return;
3506 : }
3507 :
3508 1662265 : add_candidate (data, iv->base, iv->step, false, use);
3509 :
3510 : /* Record common candidate for use in case it can be shared by others. */
3511 1662265 : record_common_cand (data, iv->base, iv->step, use);
3512 :
3513 : /* Record common candidate with initial value zero. */
3514 1662265 : basetype = TREE_TYPE (iv->base);
3515 1662265 : if (POINTER_TYPE_P (basetype))
3516 785074 : basetype = sizetype;
3517 1662265 : record_common_cand (data, build_int_cst (basetype, 0), iv->step, use);
3518 :
3519 : /* Compare the cost of an address with an unscaled index with the cost of
3520 : an address with a scaled index and add candidate if useful. */
3521 1662265 : poly_int64 step;
3522 1662265 : if (use != NULL
3523 1662265 : && poly_int_tree_p (iv->step, &step)
3524 1428636 : && address_p (use->type))
3525 : {
3526 537768 : poly_int64 new_step;
3527 537768 : unsigned int fact = preferred_mem_scale_factor
3528 1075536 : (use->iv->base,
3529 537768 : TYPE_MODE (use->mem_type),
3530 : optimize_loop_for_speed_p (data->current_loop));
3531 :
3532 537768 : if (fact != 1
3533 537768 : && multiple_p (step, fact, &new_step))
3534 0 : add_candidate (data, size_int (0),
3535 0 : wide_int_to_tree (sizetype, new_step),
3536 : true, NULL);
3537 : }
3538 :
3539 : /* Record common candidate with constant offset stripped in base.
3540 : Like the use itself, we also add candidate directly for it. */
3541 1662265 : base = strip_offset (iv->base, &offset);
3542 1662265 : if (maybe_ne (offset, 0U) || base != iv->base)
3543 : {
3544 876886 : record_common_cand (data, base, iv->step, use);
3545 876886 : add_candidate (data, base, iv->step, false, use);
3546 : }
3547 :
3548 : /* Record common candidate with base_object removed in base. */
3549 1662265 : base = iv->base;
3550 1662265 : STRIP_NOPS (base);
3551 1662265 : if (iv->base_object != NULL && TREE_CODE (base) == POINTER_PLUS_EXPR)
3552 : {
3553 0 : tree step = iv->step;
3554 :
3555 0 : STRIP_NOPS (step);
3556 0 : base = TREE_OPERAND (base, 1);
3557 0 : step = fold_convert (sizetype, step);
3558 0 : record_common_cand (data, base, step, use);
3559 : /* Also record common candidate with offset stripped. */
3560 0 : tree alt_base, alt_offset;
3561 0 : split_constant_offset (base, &alt_base, &alt_offset);
3562 0 : if (!integer_zerop (alt_offset))
3563 0 : record_common_cand (data, alt_base, step, use);
3564 : }
3565 :
3566 : /* At last, add auto-incremental candidates. Make such variables
3567 : important since other iv uses with same base object may be based
3568 : on it. */
3569 1662265 : if (use != NULL && address_p (use->type))
3570 587199 : add_autoinc_candidates (data, iv->base, iv->step, true, use);
3571 : }
3572 :
3573 : /* Adds candidates based on the uses. */
3574 :
3575 : static void
3576 510604 : add_iv_candidate_for_groups (struct ivopts_data *data)
3577 : {
3578 510604 : unsigned i;
3579 :
3580 : /* Only add candidate for the first use in group. */
3581 2173174 : for (i = 0; i < data->vgroups.length (); i++)
3582 : {
3583 1662570 : struct iv_group *group = data->vgroups[i];
3584 :
3585 1662570 : gcc_assert (group->vuses[0] != NULL);
3586 1662570 : add_iv_candidate_for_use (data, group->vuses[0]);
3587 : }
3588 510604 : add_iv_candidate_derived_from_uses (data);
3589 510604 : }
3590 :
3591 : /* Record important candidates and add them to related_cands bitmaps. */
3592 :
3593 : static void
3594 510604 : record_important_candidates (struct ivopts_data *data)
3595 : {
3596 510604 : unsigned i;
3597 510604 : struct iv_group *group;
3598 :
3599 5184416 : for (i = 0; i < data->vcands.length (); i++)
3600 : {
3601 4673812 : struct iv_cand *cand = data->vcands[i];
3602 :
3603 4673812 : if (cand->important)
3604 3748385 : bitmap_set_bit (data->important_candidates, i);
3605 : }
3606 :
3607 510604 : data->consider_all_candidates = (data->vcands.length ()
3608 510604 : <= CONSIDER_ALL_CANDIDATES_BOUND);
3609 :
3610 : /* Add important candidates to groups' related_cands bitmaps. */
3611 2173174 : for (i = 0; i < data->vgroups.length (); i++)
3612 : {
3613 1662570 : group = data->vgroups[i];
3614 1662570 : bitmap_ior_into (group->related_cands, data->important_candidates);
3615 : }
3616 510604 : }
3617 :
3618 : /* Allocates the data structure mapping the (use, candidate) pairs to costs.
3619 : If consider_all_candidates is true, we use a two-dimensional array, otherwise
3620 : we allocate a simple list to every use. */
3621 :
3622 : static void
3623 510604 : alloc_use_cost_map (struct ivopts_data *data)
3624 : {
3625 510604 : unsigned i, size, s;
3626 :
3627 2173174 : for (i = 0; i < data->vgroups.length (); i++)
3628 : {
3629 1662570 : struct iv_group *group = data->vgroups[i];
3630 :
3631 1662570 : if (data->consider_all_candidates)
3632 1653163 : size = data->vcands.length ();
3633 : else
3634 : {
3635 9407 : s = bitmap_count_bits (group->related_cands);
3636 :
3637 : /* Round up to the power of two, so that moduling by it is fast. */
3638 18814 : size = s ? (1 << ceil_log2 (s)) : 1;
3639 : }
3640 :
3641 1662570 : group->n_map_members = size;
3642 1662570 : group->cost_map = XCNEWVEC (class cost_pair, size);
3643 : }
3644 510604 : }
3645 :
3646 : /* Sets cost of (GROUP, CAND) pair to COST and record that it depends
3647 : on invariants INV_VARS and that the value used in expressing it is
3648 : VALUE, and in case of iv elimination the comparison operator is COMP. */
3649 :
3650 : static void
3651 17836947 : set_group_iv_cost (struct ivopts_data *data,
3652 : struct iv_group *group, struct iv_cand *cand,
3653 : comp_cost cost, bitmap inv_vars, tree value,
3654 : enum tree_code comp, bitmap inv_exprs)
3655 : {
3656 17836947 : unsigned i, s;
3657 :
3658 17836947 : if (cost.infinite_cost_p ())
3659 : {
3660 6153889 : BITMAP_FREE (inv_vars);
3661 6153889 : BITMAP_FREE (inv_exprs);
3662 6153889 : return;
3663 : }
3664 :
3665 11683058 : if (data->consider_all_candidates)
3666 : {
3667 11546171 : group->cost_map[cand->id].cand = cand;
3668 11546171 : group->cost_map[cand->id].cost = cost;
3669 11546171 : group->cost_map[cand->id].inv_vars = inv_vars;
3670 11546171 : group->cost_map[cand->id].inv_exprs = inv_exprs;
3671 11546171 : group->cost_map[cand->id].value = value;
3672 11546171 : group->cost_map[cand->id].comp = comp;
3673 11546171 : return;
3674 : }
3675 :
3676 : /* n_map_members is a power of two, so this computes modulo. */
3677 136887 : s = cand->id & (group->n_map_members - 1);
3678 146755 : for (i = s; i < group->n_map_members; i++)
3679 146688 : if (!group->cost_map[i].cand)
3680 136820 : goto found;
3681 144 : for (i = 0; i < s; i++)
3682 144 : if (!group->cost_map[i].cand)
3683 67 : goto found;
3684 :
3685 0 : gcc_unreachable ();
3686 :
3687 136887 : found:
3688 136887 : group->cost_map[i].cand = cand;
3689 136887 : group->cost_map[i].cost = cost;
3690 136887 : group->cost_map[i].inv_vars = inv_vars;
3691 136887 : group->cost_map[i].inv_exprs = inv_exprs;
3692 136887 : group->cost_map[i].value = value;
3693 136887 : group->cost_map[i].comp = comp;
3694 : }
3695 :
3696 : /* Gets cost of (GROUP, CAND) pair. */
3697 :
3698 : static class cost_pair *
3699 212785878 : get_group_iv_cost (struct ivopts_data *data, struct iv_group *group,
3700 : struct iv_cand *cand)
3701 : {
3702 212785878 : unsigned i, s;
3703 212785878 : class cost_pair *ret;
3704 :
3705 212785878 : if (!cand)
3706 : return NULL;
3707 :
3708 206882942 : if (data->consider_all_candidates)
3709 : {
3710 193061861 : ret = group->cost_map + cand->id;
3711 193061861 : if (!ret->cand)
3712 : return NULL;
3713 :
3714 113022592 : return ret;
3715 : }
3716 :
3717 : /* n_map_members is a power of two, so this computes modulo. */
3718 13821081 : s = cand->id & (group->n_map_members - 1);
3719 19865226 : for (i = s; i < group->n_map_members; i++)
3720 19779010 : if (group->cost_map[i].cand == cand)
3721 : return group->cost_map + i;
3722 12294042 : else if (group->cost_map[i].cand == NULL)
3723 : return NULL;
3724 253151 : for (i = 0; i < s; i++)
3725 225947 : if (group->cost_map[i].cand == cand)
3726 : return group->cost_map + i;
3727 224435 : else if (group->cost_map[i].cand == NULL)
3728 : return NULL;
3729 :
3730 : return NULL;
3731 : }
3732 :
3733 : /* Produce DECL_RTL for object obj so it looks like it is stored in memory. */
3734 : static rtx
3735 42953 : produce_memory_decl_rtl (tree obj, int *regno)
3736 : {
3737 42953 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
3738 42953 : machine_mode address_mode = targetm.addr_space.address_mode (as);
3739 42953 : rtx x;
3740 :
3741 42953 : gcc_assert (obj);
3742 42953 : if (TREE_STATIC (obj) || DECL_EXTERNAL (obj))
3743 : {
3744 42953 : const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (obj));
3745 42953 : x = gen_rtx_SYMBOL_REF (address_mode, name);
3746 42953 : SET_SYMBOL_REF_DECL (x, obj);
3747 42953 : x = gen_rtx_MEM (DECL_MODE (obj), x);
3748 42953 : set_mem_addr_space (x, as);
3749 42953 : targetm.encode_section_info (obj, x, true);
3750 : }
3751 : else
3752 : {
3753 0 : x = gen_raw_REG (address_mode, (*regno)++);
3754 0 : x = gen_rtx_MEM (DECL_MODE (obj), x);
3755 0 : set_mem_addr_space (x, as);
3756 : }
3757 :
3758 42953 : return x;
3759 : }
3760 :
3761 : /* Prepares decl_rtl for variables referred in *EXPR_P. Callback for
3762 : walk_tree. DATA contains the actual fake register number. */
3763 :
3764 : static tree
3765 601342 : prepare_decl_rtl (tree *expr_p, int *ws, void *data)
3766 : {
3767 601342 : tree obj = NULL_TREE;
3768 601342 : rtx x = NULL_RTX;
3769 601342 : int *regno = (int *) data;
3770 :
3771 601342 : switch (TREE_CODE (*expr_p))
3772 : {
3773 171812 : case ADDR_EXPR:
3774 171812 : for (expr_p = &TREE_OPERAND (*expr_p, 0);
3775 171812 : handled_component_p (*expr_p);
3776 0 : expr_p = &TREE_OPERAND (*expr_p, 0))
3777 0 : continue;
3778 171812 : obj = *expr_p;
3779 171812 : if (DECL_P (obj) && HAS_RTL_P (obj) && !DECL_RTL_SET_P (obj))
3780 0 : x = produce_memory_decl_rtl (obj, regno);
3781 : break;
3782 :
3783 0 : case SSA_NAME:
3784 0 : *ws = 0;
3785 0 : obj = SSA_NAME_VAR (*expr_p);
3786 : /* Defer handling of anonymous SSA_NAMEs to the expander. */
3787 0 : if (!obj)
3788 : return NULL_TREE;
3789 0 : if (!DECL_RTL_SET_P (obj))
3790 0 : x = gen_raw_REG (DECL_MODE (obj), (*regno)++);
3791 : break;
3792 :
3793 171812 : case VAR_DECL:
3794 171812 : case PARM_DECL:
3795 171812 : case RESULT_DECL:
3796 171812 : *ws = 0;
3797 171812 : obj = *expr_p;
3798 :
3799 171812 : if (DECL_RTL_SET_P (obj))
3800 : break;
3801 :
3802 0 : if (DECL_MODE (obj) == BLKmode)
3803 0 : x = produce_memory_decl_rtl (obj, regno);
3804 : else
3805 0 : x = gen_raw_REG (DECL_MODE (obj), (*regno)++);
3806 :
3807 : break;
3808 :
3809 : default:
3810 : break;
3811 : }
3812 :
3813 0 : if (x)
3814 : {
3815 0 : decl_rtl_to_reset.safe_push (obj);
3816 0 : SET_DECL_RTL (obj, x);
3817 : }
3818 :
3819 : return NULL_TREE;
3820 : }
3821 :
3822 : /* Predict whether the given loop will be transformed in the RTL
3823 : doloop_optimize pass. Attempt to duplicate some doloop_optimize checks.
3824 : This is only for target independent checks, see targetm.predict_doloop_p
3825 : for the target dependent ones.
3826 :
3827 : Note that according to some initial investigation, some checks like costly
3828 : niter check and invalid stmt scanning don't have much gains among general
3829 : cases, so keep this as simple as possible first.
3830 :
3831 : Some RTL specific checks seems unable to be checked in gimple, if any new
3832 : checks or easy checks _are_ missing here, please add them. */
3833 :
3834 : static bool
3835 510604 : generic_predict_doloop_p (struct ivopts_data *data)
3836 : {
3837 510604 : class loop *loop = data->current_loop;
3838 :
3839 : /* Call target hook for target dependent checks. */
3840 510604 : if (!targetm.predict_doloop_p (loop))
3841 : {
3842 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
3843 70 : fprintf (dump_file, "Predict doloop failure due to"
3844 : " target specific checks.\n");
3845 : return false;
3846 : }
3847 :
3848 : /* Similar to doloop_optimize, check iteration description to know it's
3849 : suitable or not. Keep it as simple as possible, feel free to extend it
3850 : if you find any multiple exits cases matter. */
3851 0 : edge exit = single_dom_exit (loop);
3852 0 : class tree_niter_desc *niter_desc;
3853 0 : if (!exit || !(niter_desc = niter_for_exit (data, exit)))
3854 : {
3855 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3856 0 : fprintf (dump_file, "Predict doloop failure due to"
3857 : " unexpected niters.\n");
3858 : return false;
3859 : }
3860 :
3861 : /* Similar to doloop_optimize, check whether iteration count too small
3862 : and not profitable. */
3863 0 : HOST_WIDE_INT est_niter = get_estimated_loop_iterations_int (loop);
3864 0 : if (est_niter == -1)
3865 0 : est_niter = get_likely_max_loop_iterations_int (loop);
3866 0 : if (est_niter >= 0 && est_niter < 3)
3867 : {
3868 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3869 0 : fprintf (dump_file,
3870 : "Predict doloop failure due to"
3871 : " too few iterations (%u).\n",
3872 : (unsigned int) est_niter);
3873 : return false;
3874 : }
3875 :
3876 : return true;
3877 : }
3878 :
3879 : /* Determines cost of the computation of EXPR. */
3880 :
3881 : static unsigned
3882 257718 : computation_cost (tree expr, bool speed)
3883 : {
3884 257718 : rtx_insn *seq;
3885 257718 : rtx rslt;
3886 257718 : tree type = TREE_TYPE (expr);
3887 257718 : unsigned cost;
3888 : /* Avoid using hard regs in ways which may be unsupported. */
3889 257718 : int regno = LAST_VIRTUAL_REGISTER + 1;
3890 257718 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
3891 257718 : enum node_frequency real_frequency = node->frequency;
3892 :
3893 257718 : node->frequency = NODE_FREQUENCY_NORMAL;
3894 257718 : crtl->maybe_hot_insn_p = speed;
3895 257718 : walk_tree (&expr, prepare_decl_rtl, ®no, NULL);
3896 257718 : start_sequence ();
3897 257718 : rslt = expand_expr (expr, NULL_RTX, TYPE_MODE (type), EXPAND_NORMAL);
3898 257718 : seq = end_sequence ();
3899 257718 : default_rtl_profile ();
3900 257718 : node->frequency = real_frequency;
3901 :
3902 257718 : cost = seq_cost (seq, speed);
3903 257718 : if (MEM_P (rslt))
3904 0 : cost += address_cost (XEXP (rslt, 0), TYPE_MODE (type),
3905 0 : TYPE_ADDR_SPACE (type), speed);
3906 257718 : else if (!REG_P (rslt))
3907 515436 : cost += set_src_cost (rslt, TYPE_MODE (type), speed);
3908 :
3909 257718 : return cost;
3910 : }
3911 :
3912 : /* Returns variable containing the value of candidate CAND at statement AT. */
3913 :
3914 : static tree
3915 18664164 : var_at_stmt (class loop *loop, struct iv_cand *cand, gimple *stmt)
3916 : {
3917 18664164 : if (stmt_after_increment (loop, cand, stmt))
3918 4806747 : return cand->var_after;
3919 : else
3920 13857417 : return cand->var_before;
3921 : }
3922 :
3923 : /* If A is (TYPE) BA and B is (TYPE) BB, and the types of BA and BB have the
3924 : same precision that is at least as wide as the precision of TYPE, stores
3925 : BA to A and BB to B, and returns the type of BA. Otherwise, returns the
3926 : type of A and B. */
3927 :
3928 : static tree
3929 14401436 : determine_common_wider_type (tree *a, tree *b)
3930 : {
3931 14401436 : tree wider_type = NULL;
3932 14401436 : tree suba, subb;
3933 14401436 : tree atype = TREE_TYPE (*a);
3934 :
3935 14401436 : if (CONVERT_EXPR_P (*a))
3936 : {
3937 8131577 : suba = TREE_OPERAND (*a, 0);
3938 8131577 : wider_type = TREE_TYPE (suba);
3939 8131577 : if (TYPE_PRECISION (wider_type) < TYPE_PRECISION (atype))
3940 : return atype;
3941 : }
3942 : else
3943 : return atype;
3944 :
3945 8113533 : if (CONVERT_EXPR_P (*b))
3946 : {
3947 1654642 : subb = TREE_OPERAND (*b, 0);
3948 1654642 : if (TYPE_PRECISION (wider_type) != TYPE_PRECISION (TREE_TYPE (subb)))
3949 : return atype;
3950 : }
3951 : else
3952 : return atype;
3953 :
3954 1585954 : *a = suba;
3955 1585954 : *b = subb;
3956 1585954 : return wider_type;
3957 : }
3958 :
3959 : /* Determines the expression by that USE is expressed from induction variable
3960 : CAND at statement AT in DATA's current loop. The expression is stored in
3961 : two parts in a decomposed form. The invariant part is stored in AFF_INV;
3962 : while variant part in AFF_VAR. Store ratio of CAND.step over USE.step in
3963 : PRAT if it's non-null. Returns false if USE cannot be expressed using
3964 : CAND. */
3965 :
3966 : static bool
3967 17381809 : get_computation_aff_1 (struct ivopts_data *data, gimple *at, struct iv_use *use,
3968 : struct iv_cand *cand, class aff_tree *aff_inv,
3969 : class aff_tree *aff_var, widest_int *prat = NULL)
3970 : {
3971 17381809 : tree ubase = use->iv->base, ustep = use->iv->step;
3972 17381809 : tree cbase = cand->iv->base, cstep = cand->iv->step;
3973 17381809 : tree common_type, uutype, var, cstep_common;
3974 17381809 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
3975 17381809 : aff_tree aff_cbase;
3976 17381809 : widest_int rat;
3977 :
3978 : /* We must have a precision to express the values of use. */
3979 17381809 : if (TYPE_PRECISION (utype) > TYPE_PRECISION (ctype))
3980 : return false;
3981 :
3982 17380748 : var = var_at_stmt (data->current_loop, cand, at);
3983 17380748 : uutype = unsigned_type_for (utype);
3984 :
3985 : /* If the conversion is not noop, perform it. */
3986 17380748 : if (TYPE_PRECISION (utype) < TYPE_PRECISION (ctype))
3987 : {
3988 264074 : if (cand->orig_iv != NULL && CONVERT_EXPR_P (cbase)
3989 1637714 : && (CONVERT_EXPR_P (cstep) || poly_int_tree_p (cstep)))
3990 : {
3991 29989 : tree inner_base, inner_step, inner_type;
3992 29989 : inner_base = TREE_OPERAND (cbase, 0);
3993 29989 : if (CONVERT_EXPR_P (cstep))
3994 1415 : inner_step = TREE_OPERAND (cstep, 0);
3995 : else
3996 : inner_step = cstep;
3997 :
3998 29989 : inner_type = TREE_TYPE (inner_base);
3999 : /* If candidate is added from a biv whose type is smaller than
4000 : ctype, we know both candidate and the biv won't overflow.
4001 : In this case, it's safe to skip the conversion in candidate.
4002 : As an example, (unsigned short)((unsigned long)A) equals to
4003 : (unsigned short)A, if A has a type no larger than short. */
4004 29989 : if (TYPE_PRECISION (inner_type) <= TYPE_PRECISION (uutype))
4005 : {
4006 29068 : cbase = inner_base;
4007 29068 : cstep = inner_step;
4008 : }
4009 : }
4010 1607725 : cbase = fold_convert (uutype, cbase);
4011 1607725 : cstep = fold_convert (uutype, cstep);
4012 1607725 : var = fold_convert (uutype, var);
4013 : }
4014 :
4015 : /* Ratio is 1 when computing the value of biv cand by itself.
4016 : We can't rely on constant_multiple_of in this case because the
4017 : use is created after the original biv is selected. The call
4018 : could fail because of inconsistent fold behavior. See PR68021
4019 : for more information. */
4020 17380748 : if (cand->pos == IP_ORIGINAL && cand->incremented_at == use->stmt)
4021 : {
4022 2860 : gcc_assert (is_gimple_assign (use->stmt));
4023 2860 : gcc_assert (use->iv->ssa_name == cand->var_after);
4024 2860 : gcc_assert (gimple_assign_lhs (use->stmt) == cand->var_after);
4025 2860 : rat = 1;
4026 : }
4027 17377888 : else if (!constant_multiple_of (ustep, cstep, &rat, data))
4028 : return false;
4029 :
4030 14401436 : if (prat)
4031 12921052 : *prat = rat;
4032 :
4033 : /* In case both UBASE and CBASE are shortened to UUTYPE from some common
4034 : type, we achieve better folding by computing their difference in this
4035 : wider type, and cast the result to UUTYPE. We do not need to worry about
4036 : overflows, as all the arithmetics will in the end be performed in UUTYPE
4037 : anyway. */
4038 14401436 : common_type = determine_common_wider_type (&ubase, &cbase);
4039 :
4040 : /* use = ubase - ratio * cbase + ratio * var. */
4041 14401436 : tree_to_aff_combination (ubase, common_type, aff_inv, at);
4042 14401436 : tree_to_aff_combination (cbase, common_type, &aff_cbase, at);
4043 14401436 : tree_to_aff_combination (var, uutype, aff_var, at);
4044 :
4045 : /* We need to shift the value if we are after the increment. */
4046 14401436 : if (stmt_after_increment (data->current_loop, cand, at))
4047 : {
4048 3265899 : aff_tree cstep_aff;
4049 :
4050 3265899 : if (common_type != uutype)
4051 842766 : cstep_common = fold_convert (common_type, cstep);
4052 : else
4053 : cstep_common = cstep;
4054 :
4055 3265899 : tree_to_aff_combination (cstep_common, common_type, &cstep_aff, at);
4056 3265899 : aff_combination_add (&aff_cbase, &cstep_aff);
4057 3265899 : }
4058 :
4059 14401436 : aff_combination_scale (&aff_cbase, -rat);
4060 14401436 : aff_combination_add (aff_inv, &aff_cbase);
4061 14401436 : if (common_type != uutype)
4062 9761769 : aff_combination_convert (aff_inv, uutype, at);
4063 :
4064 14401436 : aff_combination_scale (aff_var, rat);
4065 14401436 : return true;
4066 17381809 : }
4067 :
4068 : /* Determines the expression by that USE is expressed from induction variable
4069 : CAND at statement AT in DATA's current loop. The expression is stored in a
4070 : decomposed form into AFF. Returns false if USE cannot be expressed using
4071 : CAND. */
4072 :
4073 : static bool
4074 1273566 : get_computation_aff (struct ivopts_data *data, gimple *at, struct iv_use *use,
4075 : struct iv_cand *cand, class aff_tree *aff)
4076 : {
4077 1273566 : aff_tree aff_var;
4078 :
4079 1273566 : if (!get_computation_aff_1 (data, at, use, cand, aff, &aff_var))
4080 : return false;
4081 :
4082 1141386 : aff_combination_add (aff, &aff_var);
4083 1141386 : return true;
4084 1273566 : }
4085 :
4086 : /* Return the type of USE. */
4087 :
4088 : static tree
4089 1031263 : get_use_type (struct iv_use *use)
4090 : {
4091 1031263 : tree base_type = TREE_TYPE (use->iv->base);
4092 1031263 : tree type;
4093 :
4094 1031263 : if (use->type == USE_REF_ADDRESS)
4095 : {
4096 : /* The base_type may be a void pointer. Create a pointer type based on
4097 : the mem_ref instead. */
4098 0 : type = build_pointer_type (TREE_TYPE (*use->op_p));
4099 0 : gcc_assert (TYPE_ADDR_SPACE (TREE_TYPE (type))
4100 : == TYPE_ADDR_SPACE (TREE_TYPE (base_type)));
4101 : }
4102 : else
4103 : type = base_type;
4104 :
4105 1031263 : return type;
4106 : }
4107 :
4108 : /* Determines the expression by that USE is expressed from induction variable
4109 : CAND at statement AT in DATA's current loop. The computation is
4110 : unshared. */
4111 :
4112 : static tree
4113 403486 : get_computation_at (struct ivopts_data *data, gimple *at,
4114 : struct iv_use *use, struct iv_cand *cand)
4115 : {
4116 403486 : aff_tree aff;
4117 403486 : tree type = get_use_type (use);
4118 :
4119 403486 : if (!get_computation_aff (data, at, use, cand, &aff))
4120 : return NULL_TREE;
4121 271306 : unshare_aff_combination (&aff);
4122 271306 : return fold_convert (type, aff_combination_to_tree (&aff));
4123 403486 : }
4124 :
4125 : /* Like get_computation_at, but try harder, even if the computation
4126 : is more expensive. Intended for debug stmts. */
4127 :
4128 : static tree
4129 201943 : get_debug_computation_at (struct ivopts_data *data, gimple *at,
4130 : struct iv_use *use, struct iv_cand *cand)
4131 : {
4132 201943 : if (tree ret = get_computation_at (data, at, use, cand))
4133 : return ret;
4134 :
4135 132180 : tree ubase = use->iv->base, ustep = use->iv->step;
4136 132180 : tree cbase = cand->iv->base, cstep = cand->iv->step;
4137 132180 : tree var;
4138 132180 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
4139 132180 : widest_int rat;
4140 :
4141 : /* We must have a precision to express the values of use. */
4142 132180 : if (TYPE_PRECISION (utype) >= TYPE_PRECISION (ctype))
4143 : return NULL_TREE;
4144 :
4145 : /* Try to handle the case that get_computation_at doesn't,
4146 : try to express
4147 : use = ubase + (var - cbase) / ratio. */
4148 9605 : if (!constant_multiple_of (cstep, fold_convert (TREE_TYPE (cstep), ustep),
4149 : &rat, data))
4150 : return NULL_TREE;
4151 :
4152 8444 : bool neg_p = false;
4153 8444 : if (wi::neg_p (rat))
4154 : {
4155 875 : if (TYPE_UNSIGNED (ctype))
4156 : return NULL_TREE;
4157 0 : neg_p = true;
4158 0 : rat = wi::neg (rat);
4159 : }
4160 :
4161 : /* If both IVs can wrap around and CAND doesn't have a power of two step,
4162 : it is unsafe. Consider uint16_t CAND with step 9, when wrapping around,
4163 : the values will be ... 0xfff0, 0xfff9, 2, 11 ... and when use is say
4164 : uint8_t with step 3, those values divided by 3 cast to uint8_t will be
4165 : ... 0x50, 0x53, 0, 3 ... rather than expected 0x50, 0x53, 0x56, 0x59. */
4166 7569 : if (!use->iv->no_overflow
4167 63 : && !cand->iv->no_overflow
4168 7621 : && !integer_pow2p (cstep))
4169 : return NULL_TREE;
4170 :
4171 7554 : int bits = wi::exact_log2 (rat);
4172 7554 : if (bits == -1)
4173 738 : bits = wi::floor_log2 (rat) + 1;
4174 7554 : if (!cand->iv->no_overflow
4175 7554 : && TYPE_PRECISION (utype) + bits > TYPE_PRECISION (ctype))
4176 : return NULL_TREE;
4177 :
4178 7554 : var = var_at_stmt (data->current_loop, cand, at);
4179 :
4180 7554 : if (POINTER_TYPE_P (ctype))
4181 : {
4182 135 : ctype = unsigned_type_for (ctype);
4183 135 : cbase = fold_convert (ctype, cbase);
4184 135 : cstep = fold_convert (ctype, cstep);
4185 135 : var = fold_convert (ctype, var);
4186 : }
4187 :
4188 7554 : if (stmt_after_increment (data->current_loop, cand, at))
4189 80 : var = fold_build2 (MINUS_EXPR, TREE_TYPE (var), var,
4190 : unshare_expr (cstep));
4191 :
4192 7554 : var = fold_build2 (MINUS_EXPR, TREE_TYPE (var), var, cbase);
4193 7554 : var = fold_build2 (EXACT_DIV_EXPR, TREE_TYPE (var), var,
4194 : wide_int_to_tree (TREE_TYPE (var), rat));
4195 7554 : if (POINTER_TYPE_P (utype))
4196 : {
4197 0 : var = fold_convert (sizetype, var);
4198 0 : if (neg_p)
4199 0 : var = fold_build1 (NEGATE_EXPR, sizetype, var);
4200 0 : var = fold_build2 (POINTER_PLUS_EXPR, utype, ubase, var);
4201 : }
4202 : else
4203 : {
4204 7554 : var = fold_convert (utype, var);
4205 15108 : var = fold_build2 (neg_p ? MINUS_EXPR : PLUS_EXPR, utype,
4206 : ubase, var);
4207 : }
4208 : return var;
4209 201943 : }
4210 :
4211 : /* Adjust the cost COST for being in loop setup rather than loop body.
4212 : If we're optimizing for space, the loop setup overhead is constant;
4213 : if we're optimizing for speed, amortize it over the per-iteration cost.
4214 : If ROUND_UP_P is true, the result is round up rather than to zero when
4215 : optimizing for speed. */
4216 : static int64_t
4217 10498408 : adjust_setup_cost (struct ivopts_data *data, int64_t cost,
4218 : bool round_up_p = false)
4219 : {
4220 10498408 : if (cost == INFTY)
4221 : return cost;
4222 10498408 : else if (optimize_loop_for_speed_p (data->current_loop))
4223 : {
4224 8834330 : uint64_t niters = avg_loop_niter (data->current_loop);
4225 8834330 : if (niters > (uint64_t) cost)
4226 7017059 : return (round_up_p && cost != 0) ? 1 : 0;
4227 1817271 : return (cost + (round_up_p ? niters - 1 : 0)) / niters;
4228 : }
4229 : else
4230 : return cost;
4231 : }
4232 :
4233 : /* Calculate the SPEED or size cost of shiftadd EXPR in MODE. MULT is the
4234 : EXPR operand holding the shift. COST0 and COST1 are the costs for
4235 : calculating the operands of EXPR. Returns true if successful, and returns
4236 : the cost in COST. */
4237 :
4238 : static bool
4239 1423705 : get_shiftadd_cost (tree expr, scalar_int_mode mode, comp_cost cost0,
4240 : comp_cost cost1, tree mult, bool speed, comp_cost *cost)
4241 : {
4242 1423705 : comp_cost res;
4243 1423705 : tree op1 = TREE_OPERAND (expr, 1);
4244 1423705 : tree cst = TREE_OPERAND (mult, 1);
4245 1423705 : tree multop = TREE_OPERAND (mult, 0);
4246 1423705 : int m = exact_log2 (int_cst_value (cst));
4247 4270849 : int maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (mode));
4248 1423705 : int as_cost, sa_cost;
4249 1423705 : bool mult_in_op1;
4250 :
4251 1423705 : if (!(m >= 0 && m < maxm))
4252 : return false;
4253 :
4254 946172 : STRIP_NOPS (op1);
4255 946172 : mult_in_op1 = operand_equal_p (op1, mult, 0);
4256 :
4257 946172 : as_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
4258 :
4259 : /* If the target has a cheap shift-and-add or shift-and-sub instruction,
4260 : use that in preference to a shift insn followed by an add insn. */
4261 946172 : sa_cost = (TREE_CODE (expr) != MINUS_EXPR
4262 946172 : ? shiftadd_cost (speed, mode, m)
4263 : : (mult_in_op1
4264 127747 : ? shiftsub1_cost (speed, mode, m)
4265 28274 : : shiftsub0_cost (speed, mode, m)));
4266 :
4267 946172 : res = comp_cost (MIN (as_cost, sa_cost), 0);
4268 1740431 : res += (mult_in_op1 ? cost0 : cost1);
4269 :
4270 946172 : STRIP_NOPS (multop);
4271 946172 : if (!is_gimple_val (multop))
4272 484357 : res += force_expr_to_var_cost (multop, speed);
4273 :
4274 946172 : *cost = res;
4275 946172 : return true;
4276 : }
4277 :
4278 : /* Estimates cost of forcing expression EXPR into a variable. */
4279 :
4280 : static comp_cost
4281 29000128 : force_expr_to_var_cost (tree expr, bool speed)
4282 : {
4283 29000128 : static bool costs_initialized = false;
4284 29000128 : static unsigned integer_cost [2];
4285 29000128 : static unsigned symbol_cost [2];
4286 29000128 : static unsigned address_cost [2];
4287 29000128 : tree op0, op1;
4288 29000128 : comp_cost cost0, cost1, cost;
4289 29000128 : machine_mode mode;
4290 29000128 : scalar_int_mode int_mode;
4291 :
4292 29000128 : if (!costs_initialized)
4293 : {
4294 42953 : tree type = build_pointer_type (integer_type_node);
4295 42953 : tree var, addr;
4296 42953 : rtx x;
4297 42953 : int i;
4298 :
4299 42953 : var = create_tmp_var_raw (integer_type_node, "test_var");
4300 42953 : TREE_STATIC (var) = 1;
4301 42953 : x = produce_memory_decl_rtl (var, NULL);
4302 42953 : SET_DECL_RTL (var, x);
4303 :
4304 42953 : addr = build1 (ADDR_EXPR, type, var);
4305 :
4306 :
4307 171812 : for (i = 0; i < 2; i++)
4308 : {
4309 85906 : integer_cost[i] = computation_cost (build_int_cst (integer_type_node,
4310 : 2000), i);
4311 :
4312 85906 : symbol_cost[i] = computation_cost (addr, i) + 1;
4313 :
4314 85906 : address_cost[i]
4315 85906 : = computation_cost (fold_build_pointer_plus_hwi (addr, 2000), i) + 1;
4316 85906 : if (dump_file && (dump_flags & TDF_DETAILS))
4317 : {
4318 108 : fprintf (dump_file, "force_expr_to_var_cost %s costs:\n", i ? "speed" : "size");
4319 72 : fprintf (dump_file, " integer %d\n", (int) integer_cost[i]);
4320 72 : fprintf (dump_file, " symbol %d\n", (int) symbol_cost[i]);
4321 72 : fprintf (dump_file, " address %d\n", (int) address_cost[i]);
4322 72 : fprintf (dump_file, " other %d\n", (int) target_spill_cost[i]);
4323 72 : fprintf (dump_file, "\n");
4324 : }
4325 : }
4326 :
4327 42953 : costs_initialized = true;
4328 : }
4329 :
4330 29000128 : STRIP_NOPS (expr);
4331 :
4332 29000128 : if (SSA_VAR_P (expr))
4333 5439061 : return no_cost;
4334 :
4335 23561067 : if (is_gimple_min_invariant (expr))
4336 : {
4337 14161040 : if (poly_int_tree_p (expr))
4338 12018276 : return comp_cost (integer_cost [speed], 0);
4339 :
4340 2142764 : if (TREE_CODE (expr) == ADDR_EXPR)
4341 : {
4342 2142764 : tree obj = TREE_OPERAND (expr, 0);
4343 :
4344 2142764 : if (VAR_P (obj)
4345 : || TREE_CODE (obj) == PARM_DECL
4346 : || TREE_CODE (obj) == RESULT_DECL)
4347 2077159 : return comp_cost (symbol_cost [speed], 0);
4348 : }
4349 :
4350 65605 : return comp_cost (address_cost [speed], 0);
4351 : }
4352 :
4353 9400027 : switch (TREE_CODE (expr))
4354 : {
4355 8082986 : case POINTER_PLUS_EXPR:
4356 8082986 : case PLUS_EXPR:
4357 8082986 : case MINUS_EXPR:
4358 8082986 : case MULT_EXPR:
4359 8082986 : case EXACT_DIV_EXPR:
4360 8082986 : case TRUNC_DIV_EXPR:
4361 8082986 : case BIT_AND_EXPR:
4362 8082986 : case BIT_IOR_EXPR:
4363 8082986 : case LSHIFT_EXPR:
4364 8082986 : case RSHIFT_EXPR:
4365 8082986 : op0 = TREE_OPERAND (expr, 0);
4366 8082986 : op1 = TREE_OPERAND (expr, 1);
4367 8082986 : STRIP_NOPS (op0);
4368 8082986 : STRIP_NOPS (op1);
4369 8082986 : break;
4370 :
4371 1317007 : CASE_CONVERT:
4372 1317007 : case NEGATE_EXPR:
4373 1317007 : case BIT_NOT_EXPR:
4374 1317007 : op0 = TREE_OPERAND (expr, 0);
4375 1317007 : STRIP_NOPS (op0);
4376 1317007 : op1 = NULL_TREE;
4377 1317007 : break;
4378 : /* See add_iv_candidate_for_doloop, for doloop may_be_zero case, we
4379 : introduce COND_EXPR for IV base, need to support better cost estimation
4380 : for this COND_EXPR and tcc_comparison. */
4381 0 : case COND_EXPR:
4382 0 : op0 = TREE_OPERAND (expr, 1);
4383 0 : STRIP_NOPS (op0);
4384 0 : op1 = TREE_OPERAND (expr, 2);
4385 0 : STRIP_NOPS (op1);
4386 0 : break;
4387 0 : case LT_EXPR:
4388 0 : case LE_EXPR:
4389 0 : case GT_EXPR:
4390 0 : case GE_EXPR:
4391 0 : case EQ_EXPR:
4392 0 : case NE_EXPR:
4393 0 : case UNORDERED_EXPR:
4394 0 : case ORDERED_EXPR:
4395 0 : case UNLT_EXPR:
4396 0 : case UNLE_EXPR:
4397 0 : case UNGT_EXPR:
4398 0 : case UNGE_EXPR:
4399 0 : case UNEQ_EXPR:
4400 0 : case LTGT_EXPR:
4401 0 : case MAX_EXPR:
4402 0 : case MIN_EXPR:
4403 0 : op0 = TREE_OPERAND (expr, 0);
4404 0 : STRIP_NOPS (op0);
4405 0 : op1 = TREE_OPERAND (expr, 1);
4406 0 : STRIP_NOPS (op1);
4407 0 : break;
4408 :
4409 34 : default:
4410 : /* Just an arbitrary value, FIXME. */
4411 34 : return comp_cost (target_spill_cost[speed], 0);
4412 : }
4413 :
4414 9399993 : if (op0 == NULL_TREE
4415 9399993 : || TREE_CODE (op0) == SSA_NAME || CONSTANT_CLASS_P (op0))
4416 4451776 : cost0 = no_cost;
4417 : else
4418 4948217 : cost0 = force_expr_to_var_cost (op0, speed);
4419 :
4420 9399993 : if (op1 == NULL_TREE
4421 8082986 : || TREE_CODE (op1) == SSA_NAME || CONSTANT_CLASS_P (op1))
4422 8660851 : cost1 = no_cost;
4423 : else
4424 739142 : cost1 = force_expr_to_var_cost (op1, speed);
4425 :
4426 9399993 : mode = TYPE_MODE (TREE_TYPE (expr));
4427 9399993 : switch (TREE_CODE (expr))
4428 : {
4429 5682521 : case POINTER_PLUS_EXPR:
4430 5682521 : case PLUS_EXPR:
4431 5682521 : case MINUS_EXPR:
4432 5682521 : case NEGATE_EXPR:
4433 5682521 : cost = comp_cost (add_cost (speed, mode), 0);
4434 5682521 : if (TREE_CODE (expr) != NEGATE_EXPR)
4435 : {
4436 5543044 : tree mult = NULL_TREE;
4437 5543044 : comp_cost sa_cost;
4438 5543044 : if (TREE_CODE (op1) == MULT_EXPR)
4439 : mult = op1;
4440 5178257 : else if (TREE_CODE (op0) == MULT_EXPR)
4441 : mult = op0;
4442 :
4443 : if (mult != NULL_TREE
4444 4596872 : && is_a <scalar_int_mode> (mode, &int_mode)
4445 1671399 : && cst_and_fits_in_hwi (TREE_OPERAND (mult, 1))
4446 1423705 : && get_shiftadd_cost (expr, int_mode, cost0, cost1, mult,
4447 : speed, &sa_cost))
4448 946172 : return sa_cost;
4449 : }
4450 : break;
4451 :
4452 1165306 : CASE_CONVERT:
4453 1165306 : {
4454 1165306 : tree inner_mode, outer_mode;
4455 1165306 : outer_mode = TREE_TYPE (expr);
4456 1165306 : inner_mode = TREE_TYPE (op0);
4457 1165306 : cost = comp_cost (convert_cost (TYPE_MODE (outer_mode),
4458 1165306 : TYPE_MODE (inner_mode), speed), 0);
4459 : }
4460 1165306 : break;
4461 :
4462 2451442 : case MULT_EXPR:
4463 2451442 : if (cst_and_fits_in_hwi (op0))
4464 0 : cost = comp_cost (mult_by_coeff_cost (int_cst_value (op0),
4465 0 : mode, speed), 0);
4466 2451442 : else if (cst_and_fits_in_hwi (op1))
4467 1974914 : cost = comp_cost (mult_by_coeff_cost (int_cst_value (op1),
4468 1974914 : mode, speed), 0);
4469 : else
4470 476528 : return comp_cost (target_spill_cost [speed], 0);
4471 : break;
4472 :
4473 45554 : case EXACT_DIV_EXPR:
4474 45554 : case TRUNC_DIV_EXPR:
4475 : /* Division by power of two is usually cheap, so we allow it. Forbid
4476 : anything else. */
4477 45554 : if (integer_pow2p (TREE_OPERAND (expr, 1)))
4478 45554 : cost = comp_cost (add_cost (speed, mode), 0);
4479 : else
4480 0 : cost = comp_cost (target_spill_cost[speed], 0);
4481 : break;
4482 :
4483 55170 : case BIT_AND_EXPR:
4484 55170 : case BIT_IOR_EXPR:
4485 55170 : case BIT_NOT_EXPR:
4486 55170 : case LSHIFT_EXPR:
4487 55170 : case RSHIFT_EXPR:
4488 55170 : cost = comp_cost (add_cost (speed, mode), 0);
4489 55170 : break;
4490 0 : case COND_EXPR:
4491 0 : op0 = TREE_OPERAND (expr, 0);
4492 0 : STRIP_NOPS (op0);
4493 0 : if (op0 == NULL_TREE || TREE_CODE (op0) == SSA_NAME
4494 0 : || CONSTANT_CLASS_P (op0))
4495 0 : cost = no_cost;
4496 : else
4497 0 : cost = force_expr_to_var_cost (op0, speed);
4498 : break;
4499 0 : case LT_EXPR:
4500 0 : case LE_EXPR:
4501 0 : case GT_EXPR:
4502 0 : case GE_EXPR:
4503 0 : case EQ_EXPR:
4504 0 : case NE_EXPR:
4505 0 : case UNORDERED_EXPR:
4506 0 : case ORDERED_EXPR:
4507 0 : case UNLT_EXPR:
4508 0 : case UNLE_EXPR:
4509 0 : case UNGT_EXPR:
4510 0 : case UNGE_EXPR:
4511 0 : case UNEQ_EXPR:
4512 0 : case LTGT_EXPR:
4513 0 : case MAX_EXPR:
4514 0 : case MIN_EXPR:
4515 : /* Simply use add cost for now, FIXME if there is some more accurate cost
4516 : evaluation way. */
4517 0 : cost = comp_cost (add_cost (speed, mode), 0);
4518 0 : break;
4519 :
4520 0 : default:
4521 0 : gcc_unreachable ();
4522 : }
4523 :
4524 7977293 : cost += cost0;
4525 7977293 : cost += cost1;
4526 7977293 : return cost;
4527 : }
4528 :
4529 : /* Estimates cost of forcing EXPR into a variable. INV_VARS is a set of the
4530 : invariants the computation depends on. */
4531 :
4532 : static comp_cost
4533 24879623 : force_var_cost (struct ivopts_data *data, tree expr, bitmap *inv_vars)
4534 : {
4535 24879623 : if (!expr)
4536 2051211 : return no_cost;
4537 :
4538 22828412 : find_inv_vars (data, &expr, inv_vars);
4539 22828412 : return force_expr_to_var_cost (expr, data->speed);
4540 : }
4541 :
4542 : /* Returns cost of auto-modifying address expression in shape base + offset.
4543 : AINC_STEP is step size of the address IV. AINC_OFFSET is offset of the
4544 : address expression. The address expression has ADDR_MODE in addr space
4545 : AS. The memory access has MEM_MODE. SPEED means we are optimizing for
4546 : speed or size. */
4547 :
4548 : enum ainc_type
4549 : {
4550 : AINC_PRE_INC, /* Pre increment. */
4551 : AINC_PRE_DEC, /* Pre decrement. */
4552 : AINC_POST_INC, /* Post increment. */
4553 : AINC_POST_DEC, /* Post decrement. */
4554 : AINC_NONE /* Also the number of auto increment types. */
4555 : };
4556 :
4557 : struct ainc_cost_data
4558 : {
4559 : int64_t costs[AINC_NONE];
4560 : };
4561 :
4562 : static comp_cost
4563 1891957 : get_address_cost_ainc (poly_int64 ainc_step, poly_int64 ainc_offset,
4564 : machine_mode addr_mode, machine_mode mem_mode,
4565 : addr_space_t as, bool speed)
4566 : {
4567 1891957 : if (!USE_LOAD_PRE_DECREMENT (mem_mode)
4568 : && !USE_STORE_PRE_DECREMENT (mem_mode)
4569 : && !USE_LOAD_POST_DECREMENT (mem_mode)
4570 : && !USE_STORE_POST_DECREMENT (mem_mode)
4571 : && !USE_LOAD_PRE_INCREMENT (mem_mode)
4572 : && !USE_STORE_PRE_INCREMENT (mem_mode)
4573 : && !USE_LOAD_POST_INCREMENT (mem_mode)
4574 : && !USE_STORE_POST_INCREMENT (mem_mode))
4575 1891957 : return infinite_cost;
4576 :
4577 : static vec<ainc_cost_data *> ainc_cost_data_list;
4578 : unsigned idx = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
4579 : if (idx >= ainc_cost_data_list.length ())
4580 : {
4581 : unsigned nsize = ((unsigned) as + 1) *MAX_MACHINE_MODE;
4582 :
4583 : gcc_assert (nsize > idx);
4584 : ainc_cost_data_list.safe_grow_cleared (nsize, true);
4585 : }
4586 :
4587 : ainc_cost_data *data = ainc_cost_data_list[idx];
4588 : if (data == NULL)
4589 : {
4590 : rtx reg = gen_raw_REG (addr_mode, LAST_VIRTUAL_REGISTER + 1);
4591 :
4592 : data = (ainc_cost_data *) xcalloc (1, sizeof (*data));
4593 : data->costs[AINC_PRE_DEC] = INFTY;
4594 : data->costs[AINC_POST_DEC] = INFTY;
4595 : data->costs[AINC_PRE_INC] = INFTY;
4596 : data->costs[AINC_POST_INC] = INFTY;
4597 : if (USE_LOAD_PRE_DECREMENT (mem_mode)
4598 : || USE_STORE_PRE_DECREMENT (mem_mode))
4599 : {
4600 : rtx addr = gen_rtx_PRE_DEC (addr_mode, reg);
4601 :
4602 : if (memory_address_addr_space_p (mem_mode, addr, as))
4603 : data->costs[AINC_PRE_DEC]
4604 : = address_cost (addr, mem_mode, as, speed);
4605 : }
4606 : if (USE_LOAD_POST_DECREMENT (mem_mode)
4607 : || USE_STORE_POST_DECREMENT (mem_mode))
4608 : {
4609 : rtx addr = gen_rtx_POST_DEC (addr_mode, reg);
4610 :
4611 : if (memory_address_addr_space_p (mem_mode, addr, as))
4612 : data->costs[AINC_POST_DEC]
4613 : = address_cost (addr, mem_mode, as, speed);
4614 : }
4615 : if (USE_LOAD_PRE_INCREMENT (mem_mode)
4616 : || USE_STORE_PRE_INCREMENT (mem_mode))
4617 : {
4618 : rtx addr = gen_rtx_PRE_INC (addr_mode, reg);
4619 :
4620 : if (memory_address_addr_space_p (mem_mode, addr, as))
4621 : data->costs[AINC_PRE_INC]
4622 : = address_cost (addr, mem_mode, as, speed);
4623 : }
4624 : if (USE_LOAD_POST_INCREMENT (mem_mode)
4625 : || USE_STORE_POST_INCREMENT (mem_mode))
4626 : {
4627 : rtx addr = gen_rtx_POST_INC (addr_mode, reg);
4628 :
4629 : if (memory_address_addr_space_p (mem_mode, addr, as))
4630 : data->costs[AINC_POST_INC]
4631 : = address_cost (addr, mem_mode, as, speed);
4632 : }
4633 : ainc_cost_data_list[idx] = data;
4634 : }
4635 :
4636 : poly_int64 msize = GET_MODE_SIZE (mem_mode);
4637 : if (known_eq (ainc_offset, 0) && known_eq (msize, ainc_step))
4638 : return comp_cost (data->costs[AINC_POST_INC], 0);
4639 : if (known_eq (ainc_offset, 0) && known_eq (msize, -ainc_step))
4640 : return comp_cost (data->costs[AINC_POST_DEC], 0);
4641 : if (known_eq (ainc_offset, msize) && known_eq (msize, ainc_step))
4642 : return comp_cost (data->costs[AINC_PRE_INC], 0);
4643 : if (known_eq (ainc_offset, -msize) && known_eq (msize, -ainc_step))
4644 : return comp_cost (data->costs[AINC_PRE_DEC], 0);
4645 :
4646 : return infinite_cost;
4647 : }
4648 :
4649 : /* Return cost of computing USE's address expression by using CAND.
4650 : AFF_INV and AFF_VAR represent invariant and variant parts of the
4651 : address expression, respectively. If AFF_INV is simple, store
4652 : the loop invariant variables which are depended by it in INV_VARS;
4653 : if AFF_INV is complicated, handle it as a new invariant expression
4654 : and record it in INV_EXPR. RATIO indicates multiple times between
4655 : steps of USE and CAND. If CAN_AUTOINC is nonNULL, store boolean
4656 : value to it indicating if this is an auto-increment address. */
4657 :
4658 : static comp_cost
4659 5630378 : get_address_cost (struct ivopts_data *data, struct iv_use *use,
4660 : struct iv_cand *cand, aff_tree *aff_inv,
4661 : aff_tree *aff_var, HOST_WIDE_INT ratio,
4662 : bitmap *inv_vars, iv_inv_expr_ent **inv_expr,
4663 : bool *can_autoinc, bool speed)
4664 : {
4665 5630378 : rtx addr;
4666 5630378 : bool simple_inv = true;
4667 5630378 : tree comp_inv = NULL_TREE, type = aff_var->type;
4668 5630378 : comp_cost var_cost = no_cost, cost = no_cost;
4669 5630378 : struct mem_address parts = {NULL_TREE, integer_one_node,
4670 5630378 : NULL_TREE, NULL_TREE, NULL_TREE};
4671 5630378 : machine_mode addr_mode = TYPE_MODE (type);
4672 5630378 : machine_mode mem_mode = TYPE_MODE (use->mem_type);
4673 5630378 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
4674 : /* Only true if ratio != 1. */
4675 5630378 : bool ok_with_ratio_p = false;
4676 5630378 : bool ok_without_ratio_p = false;
4677 5630378 : code_helper code = ERROR_MARK;
4678 :
4679 5630378 : if (use->type == USE_PTR_ADDRESS)
4680 : {
4681 3717 : gcall *call = as_a<gcall *> (use->stmt);
4682 3717 : gcc_assert (gimple_call_internal_p (call));
4683 3717 : code = gimple_call_internal_fn (call);
4684 : }
4685 :
4686 5630378 : if (!aff_combination_const_p (aff_inv))
4687 : {
4688 3671532 : parts.index = integer_one_node;
4689 : /* Addressing mode "base + index". */
4690 3671532 : ok_without_ratio_p = valid_mem_ref_p (mem_mode, as, &parts, code);
4691 3671532 : if (ratio != 1)
4692 : {
4693 2779902 : parts.step = wide_int_to_tree (type, ratio);
4694 : /* Addressing mode "base + index << scale". */
4695 2779902 : ok_with_ratio_p = valid_mem_ref_p (mem_mode, as, &parts, code);
4696 2779902 : if (!ok_with_ratio_p)
4697 1706598 : parts.step = NULL_TREE;
4698 : }
4699 2598228 : if (ok_with_ratio_p || ok_without_ratio_p)
4700 : {
4701 3671532 : if (maybe_ne (aff_inv->offset, 0))
4702 : {
4703 2401576 : parts.offset = wide_int_to_tree (sizetype, aff_inv->offset);
4704 : /* Addressing mode "base + index [<< scale] + offset". */
4705 2401576 : if (!valid_mem_ref_p (mem_mode, as, &parts, code))
4706 343 : parts.offset = NULL_TREE;
4707 : else
4708 2401233 : aff_inv->offset = 0;
4709 : }
4710 :
4711 3671532 : move_fixed_address_to_symbol (&parts, aff_inv);
4712 : /* Base is fixed address and is moved to symbol part. */
4713 3671532 : if (parts.symbol != NULL_TREE && aff_combination_zero_p (aff_inv))
4714 451025 : parts.base = NULL_TREE;
4715 :
4716 : /* Addressing mode "symbol + base + index [<< scale] [+ offset]". */
4717 3671532 : if (parts.symbol != NULL_TREE
4718 3671532 : && !valid_mem_ref_p (mem_mode, as, &parts, code))
4719 : {
4720 7171 : aff_combination_add_elt (aff_inv, parts.symbol, 1);
4721 7171 : parts.symbol = NULL_TREE;
4722 : /* Reset SIMPLE_INV since symbol address needs to be computed
4723 : outside of address expression in this case. */
4724 7171 : simple_inv = false;
4725 : /* Symbol part is moved back to base part, it can't be NULL. */
4726 7171 : parts.base = integer_one_node;
4727 : }
4728 : }
4729 : else
4730 0 : parts.index = NULL_TREE;
4731 : }
4732 : else
4733 : {
4734 1958846 : poly_int64 ainc_step;
4735 1958846 : if (can_autoinc
4736 1958846 : && ratio == 1
4737 3917684 : && ptrdiff_tree_p (cand->iv->step, &ainc_step))
4738 : {
4739 1891957 : poly_int64 ainc_offset = (aff_inv->offset).force_shwi ();
4740 :
4741 1891957 : if (stmt_after_increment (data->current_loop, cand, use->stmt))
4742 : ainc_offset += ainc_step;
4743 1891957 : cost = get_address_cost_ainc (ainc_step, ainc_offset,
4744 : addr_mode, mem_mode, as, speed);
4745 1891957 : if (!cost.infinite_cost_p ())
4746 : {
4747 0 : *can_autoinc = true;
4748 0 : return cost;
4749 : }
4750 1891957 : cost = no_cost;
4751 : }
4752 1958846 : if (!aff_combination_zero_p (aff_inv))
4753 : {
4754 1123141 : parts.offset = wide_int_to_tree (sizetype, aff_inv->offset);
4755 : /* Addressing mode "base + offset". */
4756 1123141 : if (!valid_mem_ref_p (mem_mode, as, &parts, code))
4757 38 : parts.offset = NULL_TREE;
4758 : else
4759 1123103 : aff_inv->offset = 0;
4760 : }
4761 : }
4762 :
4763 1966017 : if (simple_inv)
4764 5623207 : simple_inv = (aff_inv == NULL
4765 8850015 : || aff_combination_const_p (aff_inv)
4766 8842844 : || aff_combination_singleton_var_p (aff_inv));
4767 5630378 : if (!aff_combination_zero_p (aff_inv))
4768 3226876 : comp_inv = aff_combination_to_tree (aff_inv);
4769 3226876 : if (comp_inv != NULL_TREE)
4770 3226876 : cost = force_var_cost (data, comp_inv, inv_vars);
4771 5630378 : if (ratio != 1 && parts.step == NULL_TREE)
4772 1706606 : var_cost += mult_by_coeff_cost (ratio, addr_mode, speed);
4773 5630378 : if (comp_inv != NULL_TREE && parts.index == NULL_TREE)
4774 38 : var_cost += add_cost (speed, addr_mode);
4775 :
4776 5630378 : if (comp_inv && inv_expr && !simple_inv)
4777 : {
4778 737423 : *inv_expr = get_loop_invariant_expr (data, comp_inv);
4779 : /* Clear depends on. */
4780 737423 : if (*inv_expr != NULL && inv_vars && *inv_vars)
4781 426405 : bitmap_clear (*inv_vars);
4782 :
4783 : /* Cost of small invariant expression adjusted against loop niters
4784 : is usually zero, which makes it difficult to be differentiated
4785 : from candidate based on loop invariant variables. Secondly, the
4786 : generated invariant expression may not be hoisted out of loop by
4787 : following pass. We penalize the cost by rounding up in order to
4788 : neutralize such effects. */
4789 737423 : cost.cost = adjust_setup_cost (data, cost.cost, true);
4790 737423 : cost.scratch = cost.cost;
4791 : }
4792 :
4793 5630378 : cost += var_cost;
4794 5630378 : addr = addr_for_mem_ref (&parts, as, false);
4795 5630378 : gcc_assert (memory_address_addr_space_p (mem_mode, addr, as));
4796 5630378 : cost += address_cost (addr, mem_mode, as, speed);
4797 :
4798 5630378 : if (parts.symbol != NULL_TREE)
4799 504047 : cost.complexity += 1;
4800 : /* Don't increase the complexity of adding a scaled index if it's
4801 : the only kind of index that the target allows. */
4802 5630378 : if (parts.step != NULL_TREE && ok_without_ratio_p)
4803 1073304 : cost.complexity += 1;
4804 5630378 : if (parts.base != NULL_TREE && parts.index != NULL_TREE)
4805 3226838 : cost.complexity += 1;
4806 5630378 : if (parts.offset != NULL_TREE && !integer_zerop (parts.offset))
4807 3524336 : cost.complexity += 1;
4808 :
4809 : return cost;
4810 : }
4811 :
4812 : /* Scale (multiply) the computed COST (except scratch part that should be
4813 : hoisted out a loop) by header->frequency / AT->frequency, which makes
4814 : expected cost more accurate. */
4815 :
4816 : static comp_cost
4817 12921052 : get_scaled_computation_cost_at (ivopts_data *data, gimple *at, comp_cost cost)
4818 : {
4819 12921052 : if (data->speed
4820 12921052 : && data->current_loop->header->count.to_frequency (cfun) > 0)
4821 : {
4822 11303338 : basic_block bb = gimple_bb (at);
4823 11303338 : gcc_assert (cost.scratch <= cost.cost);
4824 11303338 : int scale_factor = (int)(intptr_t) bb->aux;
4825 11303338 : if (scale_factor == 1)
4826 10770815 : return cost;
4827 :
4828 532523 : int64_t scaled_cost
4829 532523 : = cost.scratch + (cost.cost - cost.scratch) * scale_factor;
4830 :
4831 532523 : if (dump_file && (dump_flags & TDF_DETAILS))
4832 93 : fprintf (dump_file, "Scaling cost based on bb prob by %2.2f: "
4833 : "%" PRId64 " (scratch: %" PRId64 ") -> %" PRId64 "\n",
4834 : 1.0f * scale_factor, cost.cost, cost.scratch, scaled_cost);
4835 :
4836 : cost.cost = scaled_cost;
4837 : }
4838 :
4839 2150237 : return cost;
4840 : }
4841 :
4842 : /* Determines the cost of the computation by that USE is expressed
4843 : from induction variable CAND. If ADDRESS_P is true, we just need
4844 : to create an address from it, otherwise we want to get it into
4845 : register. A set of invariants we depend on is stored in INV_VARS.
4846 : If CAN_AUTOINC is nonnull, use it to record whether autoinc
4847 : addressing is likely. If INV_EXPR is nonnull, record invariant
4848 : expr entry in it. */
4849 :
4850 : static comp_cost
4851 20336428 : get_computation_cost (struct ivopts_data *data, struct iv_use *use,
4852 : struct iv_cand *cand, bool address_p, bitmap *inv_vars,
4853 : bool *can_autoinc, iv_inv_expr_ent **inv_expr)
4854 : {
4855 20336428 : gimple *at = use->stmt;
4856 20336428 : tree ubase = use->iv->base, cbase = cand->iv->base;
4857 20336428 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
4858 20336428 : tree comp_inv = NULL_TREE;
4859 20336428 : HOST_WIDE_INT ratio, aratio;
4860 20336428 : comp_cost cost;
4861 20336428 : widest_int rat;
4862 40672856 : aff_tree aff_inv, aff_var;
4863 20336428 : bool speed = optimize_bb_for_speed_p (gimple_bb (at));
4864 :
4865 20336428 : if (inv_vars)
4866 17777812 : *inv_vars = NULL;
4867 20336428 : if (can_autoinc)
4868 8894170 : *can_autoinc = false;
4869 20336428 : if (inv_expr)
4870 19915339 : *inv_expr = NULL;
4871 :
4872 : /* Check if we have enough precision to express the values of use. */
4873 20336428 : if (TYPE_PRECISION (utype) > TYPE_PRECISION (ctype))
4874 3080403 : return infinite_cost;
4875 :
4876 17256025 : if (address_p
4877 17256025 : || (use->iv->base_object
4878 2160236 : && cand->iv->base_object
4879 1069113 : && POINTER_TYPE_P (TREE_TYPE (use->iv->base_object))
4880 1056850 : && POINTER_TYPE_P (TREE_TYPE (cand->iv->base_object))))
4881 : {
4882 : /* Do not try to express address of an object with computation based
4883 : on address of a different object. This may cause problems in rtl
4884 : level alias analysis (that does not expect this to be happening,
4885 : as this is illegal in C), and would be unlikely to be useful
4886 : anyway. */
4887 8080469 : if (use->iv->base_object
4888 8080469 : && cand->iv->base_object
4889 12464162 : && !operand_equal_p (use->iv->base_object, cand->iv->base_object, 0))
4890 1486780 : return infinite_cost;
4891 : }
4892 :
4893 15769245 : if (!get_computation_aff_1 (data, at, use, cand, &aff_inv, &aff_var, &rat)
4894 15769245 : || !wi::fits_shwi_p (rat))
4895 2848193 : return infinite_cost;
4896 :
4897 12921052 : ratio = rat.to_shwi ();
4898 12921052 : if (address_p)
4899 : {
4900 5630378 : cost = get_address_cost (data, use, cand, &aff_inv, &aff_var, ratio,
4901 : inv_vars, inv_expr, can_autoinc, speed);
4902 5630378 : cost = get_scaled_computation_cost_at (data, at, cost);
4903 : /* For doloop IV cand, add on the extra cost. */
4904 5630378 : cost += cand->doloop_p ? targetm.doloop_cost_for_address : 0;
4905 5630378 : return cost;
4906 : }
4907 :
4908 7290674 : bool simple_inv = (aff_combination_const_p (&aff_inv)
4909 1984458 : || aff_combination_singleton_var_p (&aff_inv));
4910 7290674 : tree signed_type = signed_type_for (aff_combination_type (&aff_inv));
4911 7290674 : aff_combination_convert (&aff_inv, signed_type);
4912 7290674 : if (!aff_combination_zero_p (&aff_inv))
4913 5239463 : comp_inv = aff_combination_to_tree (&aff_inv);
4914 :
4915 7290674 : cost = force_var_cost (data, comp_inv, inv_vars);
4916 7290674 : if (comp_inv && inv_expr && !simple_inv)
4917 : {
4918 1376087 : *inv_expr = get_loop_invariant_expr (data, comp_inv);
4919 : /* Clear depends on. */
4920 1376087 : if (*inv_expr != NULL && inv_vars && *inv_vars)
4921 824060 : bitmap_clear (*inv_vars);
4922 :
4923 1376087 : cost.cost = adjust_setup_cost (data, cost.cost);
4924 : /* Record setup cost in scratch field. */
4925 1376087 : cost.scratch = cost.cost;
4926 : }
4927 : /* Cost of constant integer can be covered when adding invariant part to
4928 : variant part. */
4929 5914587 : else if (comp_inv && CONSTANT_CLASS_P (comp_inv))
4930 3254980 : cost = no_cost;
4931 :
4932 : /* Need type narrowing to represent use with cand. */
4933 7290674 : if (TYPE_PRECISION (utype) < TYPE_PRECISION (ctype))
4934 : {
4935 803646 : machine_mode outer_mode = TYPE_MODE (utype);
4936 803646 : machine_mode inner_mode = TYPE_MODE (ctype);
4937 803646 : cost += comp_cost (convert_cost (outer_mode, inner_mode, speed), 0);
4938 : }
4939 :
4940 : /* Turn a + i * (-c) into a - i * c. */
4941 7290674 : if (ratio < 0 && comp_inv && !integer_zerop (comp_inv))
4942 1855526 : aratio = -ratio;
4943 : else
4944 : aratio = ratio;
4945 :
4946 7290674 : if (ratio != 1)
4947 2733741 : cost += mult_by_coeff_cost (aratio, TYPE_MODE (utype), speed);
4948 :
4949 : /* TODO: We may also need to check if we can compute a + i * 4 in one
4950 : instruction. */
4951 : /* Need to add up the invariant and variant parts. */
4952 7290674 : if (comp_inv && !integer_zerop (comp_inv))
4953 10472542 : cost += add_cost (speed, TYPE_MODE (utype));
4954 :
4955 7290674 : cost = get_scaled_computation_cost_at (data, at, cost);
4956 :
4957 : /* For doloop IV cand, add on the extra cost. */
4958 7290674 : if (cand->doloop_p && use->type == USE_NONLINEAR_EXPR)
4959 0 : cost += targetm.doloop_cost_for_generic;
4960 :
4961 7290674 : return cost;
4962 20336428 : }
4963 :
4964 : /* Determines cost of computing the use in GROUP with CAND in a generic
4965 : expression. */
4966 :
4967 : static bool
4968 5524445 : determine_group_iv_cost_generic (struct ivopts_data *data,
4969 : struct iv_group *group, struct iv_cand *cand)
4970 : {
4971 5524445 : comp_cost cost;
4972 5524445 : iv_inv_expr_ent *inv_expr = NULL;
4973 5524445 : bitmap inv_vars = NULL, inv_exprs = NULL;
4974 5524445 : struct iv_use *use = group->vuses[0];
4975 :
4976 : /* The simple case first -- if we need to express value of the preserved
4977 : original biv, the cost is 0. This also prevents us from counting the
4978 : cost of increment twice -- once at this use and once in the cost of
4979 : the candidate. */
4980 5524445 : if (cand->pos == IP_ORIGINAL && cand->incremented_at == use->stmt)
4981 59135 : cost = no_cost;
4982 : /* If the IV candidate involves undefined SSA values and is not the
4983 : same IV as on the USE avoid using that candidate here. */
4984 5465310 : else if (cand->involves_undefs
4985 5465310 : && (!use->iv || !operand_equal_p (cand->iv->base, use->iv->base, 0)))
4986 : return false;
4987 : else
4988 5465083 : cost = get_computation_cost (data, use, cand, false,
4989 : &inv_vars, NULL, &inv_expr);
4990 :
4991 5524218 : if (inv_expr)
4992 : {
4993 966382 : inv_exprs = BITMAP_ALLOC (NULL);
4994 966382 : bitmap_set_bit (inv_exprs, inv_expr->id);
4995 : }
4996 5524218 : set_group_iv_cost (data, group, cand, cost, inv_vars,
4997 : NULL_TREE, ERROR_MARK, inv_exprs);
4998 5524218 : return !cost.infinite_cost_p ();
4999 : }
5000 :
5001 : /* Determines cost of computing uses in GROUP with CAND in addresses. */
5002 :
5003 : static bool
5004 6335554 : determine_group_iv_cost_address (struct ivopts_data *data,
5005 : struct iv_group *group, struct iv_cand *cand)
5006 : {
5007 6335554 : unsigned i;
5008 6335554 : bitmap inv_vars = NULL, inv_exprs = NULL;
5009 6335554 : bool can_autoinc;
5010 6335554 : iv_inv_expr_ent *inv_expr = NULL;
5011 6335554 : struct iv_use *use = group->vuses[0];
5012 6335554 : comp_cost sum_cost = no_cost, cost;
5013 :
5014 6335554 : cost = get_computation_cost (data, use, cand, true,
5015 : &inv_vars, &can_autoinc, &inv_expr);
5016 :
5017 6335554 : if (inv_expr)
5018 : {
5019 453493 : inv_exprs = BITMAP_ALLOC (NULL);
5020 453493 : bitmap_set_bit (inv_exprs, inv_expr->id);
5021 : }
5022 6335554 : sum_cost = cost;
5023 6335554 : if (!sum_cost.infinite_cost_p () && cand->ainc_use == use)
5024 : {
5025 0 : if (can_autoinc)
5026 0 : sum_cost -= cand->cost_step;
5027 : /* If we generated the candidate solely for exploiting autoincrement
5028 : opportunities, and it turns out it can't be used, set the cost to
5029 : infinity to make sure we ignore it. */
5030 0 : else if (cand->pos == IP_AFTER_USE || cand->pos == IP_BEFORE_USE)
5031 0 : sum_cost = infinite_cost;
5032 : }
5033 :
5034 : /* Compute and add costs for rest uses of this group. */
5035 8473081 : for (i = 1; i < group->vuses.length () && !sum_cost.infinite_cost_p (); i++)
5036 : {
5037 2137527 : struct iv_use *next = group->vuses[i];
5038 :
5039 : /* TODO: We could skip computing cost for sub iv_use when it has the
5040 : same cost as the first iv_use, but the cost really depends on the
5041 : offset and where the iv_use is. */
5042 2137527 : cost = get_computation_cost (data, next, cand, true,
5043 : NULL, &can_autoinc, &inv_expr);
5044 2137527 : if (inv_expr)
5045 : {
5046 283697 : if (!inv_exprs)
5047 53 : inv_exprs = BITMAP_ALLOC (NULL);
5048 :
5049 : /* Uses in a group can share setup code,
5050 : so only add setup cost once. */
5051 283697 : if (bitmap_bit_p (inv_exprs, inv_expr->id))
5052 283325 : cost -= cost.scratch;
5053 : else
5054 372 : bitmap_set_bit (inv_exprs, inv_expr->id);
5055 : }
5056 2137527 : sum_cost += cost;
5057 : }
5058 6335554 : set_group_iv_cost (data, group, cand, sum_cost, inv_vars,
5059 : NULL_TREE, ERROR_MARK, inv_exprs);
5060 :
5061 6335554 : return !sum_cost.infinite_cost_p ();
5062 : }
5063 :
5064 : /* Computes value of candidate CAND at position AT in iteration DESC->NITER,
5065 : and stores it to VAL. */
5066 :
5067 : static void
5068 3928514 : cand_value_at (class loop *loop, struct iv_cand *cand, gimple *at,
5069 : class tree_niter_desc *desc, aff_tree *val)
5070 : {
5071 11785542 : aff_tree step, delta, nit;
5072 3928514 : struct iv *iv = cand->iv;
5073 3928514 : tree type = TREE_TYPE (iv->base);
5074 3928514 : tree niter = desc->niter;
5075 3928514 : bool after_adjust = stmt_after_increment (loop, cand, at);
5076 3928514 : tree steptype;
5077 :
5078 3928514 : if (POINTER_TYPE_P (type))
5079 111612 : steptype = sizetype;
5080 : else
5081 3816902 : steptype = unsigned_type_for (type);
5082 :
5083 : /* If AFTER_ADJUST is required, the code below generates the equivalent
5084 : of BASE + NITER * STEP + STEP, when ideally we'd prefer the expression
5085 : BASE + (NITER + 1) * STEP, especially when NITER is often of the form
5086 : SSA_NAME - 1. Unfortunately, guaranteeing that adding 1 to NITER
5087 : doesn't overflow is tricky, so we peek inside the TREE_NITER_DESC
5088 : class for common idioms that we know are safe. */
5089 3928514 : if (after_adjust
5090 3697761 : && desc->control.no_overflow
5091 3690797 : && integer_onep (desc->control.step)
5092 1008249 : && (desc->cmp == LT_EXPR
5093 38242 : || desc->cmp == NE_EXPR)
5094 4936763 : && TREE_CODE (desc->bound) == SSA_NAME)
5095 : {
5096 510142 : if (integer_onep (desc->control.base))
5097 : {
5098 383349 : niter = desc->bound;
5099 383349 : after_adjust = false;
5100 : }
5101 126793 : else if (TREE_CODE (niter) == MINUS_EXPR
5102 126793 : && integer_onep (TREE_OPERAND (niter, 1)))
5103 : {
5104 71254 : niter = TREE_OPERAND (niter, 0);
5105 71254 : after_adjust = false;
5106 : }
5107 : }
5108 :
5109 3928514 : tree_to_aff_combination (iv->step, TREE_TYPE (iv->step), &step);
5110 3928514 : aff_combination_convert (&step, steptype);
5111 3928514 : tree_to_aff_combination (niter, TREE_TYPE (niter), &nit);
5112 3928514 : aff_combination_convert (&nit, steptype);
5113 3928514 : aff_combination_mult (&nit, &step, &delta);
5114 3928514 : if (after_adjust)
5115 3243158 : aff_combination_add (&delta, &step);
5116 :
5117 3928514 : tree_to_aff_combination (iv->base, type, val);
5118 3928514 : if (!POINTER_TYPE_P (type))
5119 3816902 : aff_combination_convert (val, steptype);
5120 3928514 : aff_combination_add (val, &delta);
5121 3928514 : }
5122 :
5123 : /* Returns period of induction variable iv. */
5124 :
5125 : static tree
5126 4127247 : iv_period (struct iv *iv)
5127 : {
5128 4127247 : tree step = iv->step, period, type;
5129 4127247 : tree pow2div;
5130 :
5131 4127247 : gcc_assert (step && TREE_CODE (step) == INTEGER_CST);
5132 :
5133 4127247 : type = unsigned_type_for (TREE_TYPE (step));
5134 : /* Period of the iv is lcm (step, type_range)/step -1,
5135 : i.e., N*type_range/step - 1. Since type range is power
5136 : of two, N == (step >> num_of_ending_zeros_binary (step),
5137 : so the final result is
5138 :
5139 : (type_range >> num_of_ending_zeros_binary (step)) - 1
5140 :
5141 : */
5142 4127247 : pow2div = num_ending_zeros (step);
5143 :
5144 12381741 : period = build_low_bits_mask (type,
5145 4127247 : (TYPE_PRECISION (type)
5146 4127247 : - tree_to_uhwi (pow2div)));
5147 :
5148 4127247 : return period;
5149 : }
5150 :
5151 : /* Returns the comparison operator used when eliminating the iv USE. */
5152 :
5153 : static enum tree_code
5154 3928514 : iv_elimination_compare (struct ivopts_data *data, struct iv_use *use)
5155 : {
5156 3928514 : class loop *loop = data->current_loop;
5157 3928514 : basic_block ex_bb;
5158 3928514 : edge exit;
5159 :
5160 3928514 : ex_bb = gimple_bb (use->stmt);
5161 3928514 : exit = EDGE_SUCC (ex_bb, 0);
5162 3928514 : if (flow_bb_inside_loop_p (loop, exit->dest))
5163 2930484 : exit = EDGE_SUCC (ex_bb, 1);
5164 :
5165 3928514 : return (exit->flags & EDGE_TRUE_VALUE ? EQ_EXPR : NE_EXPR);
5166 : }
5167 :
5168 : /* Returns true if we can prove that BASE - OFFSET does not overflow. For now,
5169 : we only detect the situation that BASE = SOMETHING + OFFSET, where the
5170 : calculation is performed in non-wrapping type.
5171 :
5172 : TODO: More generally, we could test for the situation that
5173 : BASE = SOMETHING + OFFSET' and OFFSET is between OFFSET' and zero.
5174 : This would require knowing the sign of OFFSET. */
5175 :
5176 : static bool
5177 521 : difference_cannot_overflow_p (struct ivopts_data *data, tree base, tree offset)
5178 : {
5179 521 : enum tree_code code;
5180 521 : tree e1, e2;
5181 1563 : aff_tree aff_e1, aff_e2, aff_offset;
5182 :
5183 521 : if (!nowrap_type_p (TREE_TYPE (base)))
5184 : return false;
5185 :
5186 521 : base = expand_simple_operations (base);
5187 :
5188 521 : if (TREE_CODE (base) == SSA_NAME)
5189 : {
5190 519 : gimple *stmt = SSA_NAME_DEF_STMT (base);
5191 :
5192 519 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
5193 : return false;
5194 :
5195 47 : code = gimple_assign_rhs_code (stmt);
5196 47 : if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
5197 : return false;
5198 :
5199 34 : e1 = gimple_assign_rhs1 (stmt);
5200 34 : e2 = gimple_assign_rhs2 (stmt);
5201 : }
5202 : else
5203 : {
5204 2 : code = TREE_CODE (base);
5205 2 : if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
5206 : return false;
5207 0 : e1 = TREE_OPERAND (base, 0);
5208 0 : e2 = TREE_OPERAND (base, 1);
5209 : }
5210 :
5211 : /* Use affine expansion as deeper inspection to prove the equality. */
5212 34 : tree_to_aff_combination_expand (e2, TREE_TYPE (e2),
5213 : &aff_e2, &data->name_expansion_cache);
5214 34 : tree_to_aff_combination_expand (offset, TREE_TYPE (offset),
5215 : &aff_offset, &data->name_expansion_cache);
5216 34 : aff_combination_scale (&aff_offset, -1);
5217 34 : switch (code)
5218 : {
5219 0 : case PLUS_EXPR:
5220 0 : aff_combination_add (&aff_e2, &aff_offset);
5221 0 : if (aff_combination_zero_p (&aff_e2))
5222 : return true;
5223 :
5224 0 : tree_to_aff_combination_expand (e1, TREE_TYPE (e1),
5225 : &aff_e1, &data->name_expansion_cache);
5226 0 : aff_combination_add (&aff_e1, &aff_offset);
5227 0 : return aff_combination_zero_p (&aff_e1);
5228 :
5229 34 : case POINTER_PLUS_EXPR:
5230 34 : aff_combination_add (&aff_e2, &aff_offset);
5231 34 : return aff_combination_zero_p (&aff_e2);
5232 :
5233 : default:
5234 : return false;
5235 : }
5236 521 : }
5237 :
5238 : /* Return true if STEP * VAL, computed in OFF_TYPE, is known to be a
5239 : non-negative offset which does not overflow, i.e. if the value of VAL is
5240 : non-negative and multiplying it by STEP fits in the signed range of
5241 : OFF_TYPE. */
5242 :
5243 : static bool
5244 112 : nonneg_scaled_offset_p (tree val, HOST_WIDE_INT step, tree off_type)
5245 : {
5246 112 : if (!INTEGRAL_TYPE_P (TREE_TYPE (val)) || step == 0)
5247 : return false;
5248 :
5249 112 : signop sgn = TYPE_SIGN (TREE_TYPE (val));
5250 112 : int_range_max r;
5251 224 : if (!get_range_query (cfun)->range_of_expr (r, val)
5252 112 : || r.undefined_p ()
5253 336 : || wi::neg_p (r.lower_bound (), sgn))
5254 : return false;
5255 :
5256 112 : widest_int max = widest_int::from (r.upper_bound (), sgn);
5257 112 : widest_int limit
5258 112 : = widest_int::from (wi::max_value (TYPE_PRECISION (off_type), SIGNED),
5259 112 : SIGNED);
5260 112 : return wi::leu_p (max, wi::udiv_trunc (limit, absu_hwi (step)));
5261 112 : }
5262 :
5263 : /* Tries to replace loop exit by one formulated in terms of a LT_EXPR
5264 : comparison with CAND. NITER describes the number of iterations of
5265 : the loops. If successful, the comparison in COMP_P is altered accordingly
5266 : and the bound in BOUND_P is recomputed.
5267 :
5268 : We aim to handle the following situation:
5269 :
5270 : sometype *base, *p;
5271 : unsigned a, b, i;
5272 :
5273 : i = a;
5274 : p = p_0 = base + a;
5275 :
5276 : do
5277 : {
5278 : bla (*p);
5279 : p++;
5280 : i++;
5281 : }
5282 : while (i < b);
5283 :
5284 : Here, the number of iterations of the loop is (a + 1 > b) ? 0 : b - a - 1.
5285 : We aim to optimize this to
5286 :
5287 : p = p_0 = base + a;
5288 : do
5289 : {
5290 : bla (*p);
5291 : p++;
5292 : }
5293 : while (p < p_0 - (a + 1) + (b + 1));
5294 :
5295 : Note that the bound has to be computed as p_0 - (a + 1) + (b + 1) and not
5296 : from the number of iterations as p_0 + (b - a): the latter is only
5297 : equivalent if b - a does not wrap, which is not the case when the loop
5298 : rolls zero times.
5299 :
5300 : Note also that a + 1 must be used as the offset it is, i.e. converted to
5301 : the type of the offsets as a whole, and not rewritten into a and 1, since
5302 : it may wrap around in the type of a: this is precisely the case where the
5303 : loop rolls but b - a is not the number of iterations either. Conversely
5304 : b + 1 must be computed in the type of the offsets, where it cannot wrap
5305 : around, since nothing constrains b when the loop rolls zero times.
5306 :
5307 : For this to preserve correctness, we need to know that the values compared
5308 : in the transformed loop are ordered the same way as i and b are. Since the
5309 : comparison of the pointers is performed modulo the size of the address
5310 : space, this needs a + 1 > b to be an unsigned comparison, and the offsets
5311 : a, b, a + 1 and b + 1 scaled by the step of the candidate to be
5312 : non-negative and to not overflow. Then:
5313 :
5314 : 1) if a + 1 <= b, then the loop rolls b - (a + 1) times, so that (b + 1)
5315 : - (a + 1) computed on the offsets is its number of iterations, and
5316 : p_0 - (a + 1) + (b + 1) is the final value of p, hence there is no
5317 : overflow in computing it or the values of p, and the pointers increase
5318 : monotonically together with i.
5319 : 2) if a + 1 > b, then the loop exits at the first test and a + 1 does not
5320 : wrap around, so b <= a implies that p_0 - (a + 1) + (b + 1) = p_0 - a
5321 : + b lies between the valid addresses p_0 - a and p_0, so the test indeed
5322 : fails. Here we also need to verify that the expression p_0 - a does not
5323 : overflow, which we prove using p_0 = base + a. */
5324 :
5325 : static bool
5326 210402 : iv_elimination_compare_lt (struct ivopts_data *data, struct iv_use *use,
5327 : struct iv_cand *cand, enum tree_code *comp_p,
5328 : class tree_niter_desc *niter, tree *bound_p)
5329 : {
5330 210402 : tree cand_type, a, a1, b, b1, mbz, nit_type = TREE_TYPE (niter->niter);
5331 210402 : tree off_type, offset, bound;
5332 631206 : class aff_tree nit, tmpa, tmpb;
5333 210402 : enum tree_code comp;
5334 210402 : HOST_WIDE_INT step;
5335 :
5336 : /* We need to know that the candidate induction variable does not overflow.
5337 : While more complex analysis may be used to prove this, for now just
5338 : check that the variable appears in the original program and that it
5339 : is computed in a type that guarantees no overflows. */
5340 210402 : cand_type = TREE_TYPE (cand->iv->base);
5341 210402 : if (cand->pos != IP_ORIGINAL || !nowrap_type_p (cand_type))
5342 : return false;
5343 :
5344 : /* Make sure that the loop iterates till the loop bound is hit, as otherwise
5345 : the calculation of the BOUND could overflow, making the comparison
5346 : invalid. */
5347 19079 : if (!data->loop_single_exit_p)
5348 : return false;
5349 :
5350 : /* We need to be able to decide whether candidate is increasing or decreasing
5351 : in order to choose the right comparison operator. */
5352 13545 : if (!cst_and_fits_in_hwi (cand->iv->step))
5353 : return false;
5354 13545 : step = int_cst_value (cand->iv->step);
5355 :
5356 : /* The bound we compute below is the value the candidate has after the last
5357 : iteration, so the exit test has to see the incremented candidate. */
5358 13545 : if (!stmt_after_increment (data->current_loop, cand, use->stmt))
5359 : return false;
5360 :
5361 : /* Check that the number of iterations matches the expected pattern:
5362 : a + 1 > b ? 0 : b - a - 1. */
5363 9515 : mbz = niter->may_be_zero;
5364 9515 : if (TREE_CODE (mbz) == GT_EXPR)
5365 : {
5366 : /* Handle a + 1 > b. */
5367 1346 : tree op0 = TREE_OPERAND (mbz, 0);
5368 1346 : if (TREE_CODE (op0) == PLUS_EXPR && integer_onep (TREE_OPERAND (op0, 1)))
5369 : {
5370 804 : a1 = op0;
5371 804 : a = TREE_OPERAND (op0, 0);
5372 804 : b = TREE_OPERAND (mbz, 1);
5373 : }
5374 : else
5375 : return false;
5376 : }
5377 8169 : else if (TREE_CODE (mbz) == LT_EXPR)
5378 : {
5379 683 : tree op1 = TREE_OPERAND (mbz, 1);
5380 :
5381 : /* Handle b < a + 1. */
5382 683 : if (TREE_CODE (op1) == PLUS_EXPR && integer_onep (TREE_OPERAND (op1, 1)))
5383 : {
5384 108 : a1 = op1;
5385 108 : a = TREE_OPERAND (op1, 0);
5386 108 : b = TREE_OPERAND (mbz, 0);
5387 : }
5388 : else
5389 : return false;
5390 : }
5391 : else
5392 : return false;
5393 :
5394 : /* Expected number of iterations is B - A - 1. Check that it matches
5395 : the actual number, i.e., that B - A - NITER = 1. */
5396 912 : tree_to_aff_combination (niter->niter, nit_type, &nit);
5397 912 : tree_to_aff_combination (fold_convert (nit_type, a), nit_type, &tmpa);
5398 912 : tree_to_aff_combination (fold_convert (nit_type, b), nit_type, &tmpb);
5399 912 : aff_combination_scale (&nit, -1);
5400 912 : aff_combination_scale (&tmpa, -1);
5401 912 : aff_combination_add (&tmpb, &tmpa);
5402 912 : aff_combination_add (&tmpb, &nit);
5403 912 : if (tmpb.n != 0 || maybe_ne (tmpb.offset, 1))
5404 : return false;
5405 :
5406 : /* The comparison A + 1 > B only tells us that B is at most A if it is
5407 : an unsigned one; otherwise B may well be negative. */
5408 521 : if (!TYPE_UNSIGNED (TREE_TYPE (a)))
5409 : return false;
5410 :
5411 : /* Check that CAND->IV->BASE - CAND->IV->STEP * A does not overflow. */
5412 521 : off_type = TREE_TYPE (cand->iv->step);
5413 521 : offset = fold_build2 (MULT_EXPR, off_type, cand->iv->step,
5414 : fold_convert (off_type, a));
5415 521 : if (!difference_cannot_overflow_p (data, cand->iv->base, offset))
5416 : return false;
5417 :
5418 : /* Convert A + 1 as a whole, since it may wrap around in the type of A, but
5419 : compute B + 1 in OFF_TYPE, where it cannot wrap around. */
5420 34 : a1 = fold_convert (off_type, a1);
5421 34 : b1 = fold_build2 (PLUS_EXPR, off_type, fold_convert (off_type, b),
5422 : build_one_cst (off_type));
5423 :
5424 : /* The candidate is compared as an unsigned quantity, so the offsets by
5425 : which A + 1 and B + 1 move it away from CAND->IV->BASE - CAND->IV->STEP
5426 : * (A + 1) have to be ordered the same way as A + 1 and B + 1 themselves.
5427 : The same is required of A and B, in terms of which the bound is
5428 : expressed when the loop rolls zero times. */
5429 34 : if (!nonneg_scaled_offset_p (a, step, off_type)
5430 26 : || !nonneg_scaled_offset_p (b, step, off_type)
5431 26 : || !nonneg_scaled_offset_p (a1, step, off_type)
5432 60 : || !nonneg_scaled_offset_p (b1, step, off_type))
5433 : return false;
5434 :
5435 : /* Determine the new comparison operator. */
5436 26 : comp = step < 0 ? GT_EXPR : LT_EXPR;
5437 26 : if (*comp_p == NE_EXPR)
5438 26 : *comp_p = comp;
5439 0 : else if (*comp_p == EQ_EXPR)
5440 0 : *comp_p = invert_tree_comparison (comp, false);
5441 : else
5442 0 : gcc_unreachable ();
5443 :
5444 : /* Recompute the bound as CAND->IV->BASE - CAND->IV->STEP * (A + 1)
5445 : + CAND->IV->STEP * (B + 1). Deriving it from the number of iterations,
5446 : as cand_value_at does, is not correct here: B - A is computed in NIT_TYPE
5447 : and converting it to OFF_TYPE is not value preserving when the loop
5448 : rolls zero times and B - A is thus negative. */
5449 26 : bound = fold_build2 (MINUS_EXPR, off_type,
5450 : fold_build2 (MULT_EXPR, off_type, cand->iv->step, b1),
5451 : fold_build2 (MULT_EXPR, off_type, cand->iv->step, a1));
5452 26 : cand_type = TREE_TYPE (cand->iv->base);
5453 26 : if (POINTER_TYPE_P (cand_type))
5454 26 : *bound_p = fold_build_pointer_plus (cand->iv->base, bound);
5455 : else
5456 0 : *bound_p = fold_build2 (PLUS_EXPR, cand_type, cand->iv->base,
5457 : fold_convert (cand_type, bound));
5458 :
5459 : return true;
5460 210402 : }
5461 :
5462 : /* Check whether it is possible to express the condition in USE by comparison
5463 : of candidate CAND. If so, store the value compared with to BOUND, and the
5464 : comparison operator to COMP. */
5465 :
5466 : static bool
5467 4959894 : may_eliminate_iv (struct ivopts_data *data,
5468 : struct iv_use *use, struct iv_cand *cand, tree *bound,
5469 : enum tree_code *comp)
5470 : {
5471 4959894 : basic_block ex_bb;
5472 4959894 : edge exit;
5473 4959894 : tree period;
5474 4959894 : class loop *loop = data->current_loop;
5475 4959894 : aff_tree bnd;
5476 4959894 : class tree_niter_desc *desc = NULL;
5477 :
5478 : /* If the IV candidate involves undefs do not attempt to use it to
5479 : express a condition. */
5480 4959894 : if (cand->involves_undefs)
5481 : return false;
5482 :
5483 4959546 : if (TREE_CODE (cand->iv->step) != INTEGER_CST)
5484 : return false;
5485 :
5486 : /* For now works only for exits that dominate the loop latch.
5487 : TODO: extend to other conditions inside loop body. */
5488 4772335 : ex_bb = gimple_bb (use->stmt);
5489 4772335 : if (use->stmt != last_nondebug_stmt (ex_bb)
5490 4667897 : || gimple_code (use->stmt) != GIMPLE_COND
5491 9439233 : || !dominated_by_p (CDI_DOMINATORS, loop->latch, ex_bb))
5492 : return false;
5493 :
5494 4537058 : exit = EDGE_SUCC (ex_bb, 0);
5495 4537058 : if (flow_bb_inside_loop_p (loop, exit->dest))
5496 3429641 : exit = EDGE_SUCC (ex_bb, 1);
5497 4537058 : if (flow_bb_inside_loop_p (loop, exit->dest))
5498 : return false;
5499 :
5500 4422553 : desc = niter_for_exit (data, exit);
5501 4422553 : if (!desc)
5502 : return false;
5503 :
5504 : /* Determine whether we can use the variable to test the exit condition.
5505 : This is the case iff the period of the induction variable is greater
5506 : than the number of iterations for which the exit condition is true. */
5507 4127247 : period = iv_period (cand->iv);
5508 :
5509 : /* If the number of iterations is constant, compare against it directly. */
5510 4127247 : if (TREE_CODE (desc->niter) == INTEGER_CST)
5511 : {
5512 : /* See cand_value_at. */
5513 2709752 : if (stmt_after_increment (loop, cand, use->stmt))
5514 : {
5515 2651425 : if (!tree_int_cst_lt (desc->niter, period))
5516 : return false;
5517 : }
5518 : else
5519 : {
5520 58327 : if (tree_int_cst_lt (period, desc->niter))
5521 : return false;
5522 : }
5523 : }
5524 :
5525 : /* If not, and if this is the only possible exit of the loop, see whether
5526 : we can get a conservative estimate on the number of iterations of the
5527 : entire loop and compare against that instead. */
5528 : else
5529 : {
5530 1417495 : widest_int period_value, max_niter;
5531 :
5532 1417495 : max_niter = desc->max;
5533 1417495 : if (stmt_after_increment (loop, cand, use->stmt))
5534 1211743 : max_niter += 1;
5535 1417495 : period_value = wi::to_widest (period);
5536 1417495 : if (wi::gtu_p (max_niter, period_value))
5537 : {
5538 : /* See if we can take advantage of inferred loop bound
5539 : information. */
5540 285679 : if (data->loop_single_exit_p)
5541 : {
5542 165606 : if (!max_loop_iterations (loop, &max_niter))
5543 : return false;
5544 : /* The loop bound is already adjusted by adding 1. */
5545 165606 : if (wi::gtu_p (max_niter, period_value))
5546 : return false;
5547 : }
5548 : else
5549 : return false;
5550 : }
5551 1417495 : }
5552 :
5553 : /* For doloop IV cand, the bound would be zero. It's safe whether
5554 : may_be_zero set or not. */
5555 3928514 : if (cand->doloop_p)
5556 : {
5557 0 : *bound = build_int_cst (TREE_TYPE (cand->iv->base), 0);
5558 0 : *comp = iv_elimination_compare (data, use);
5559 0 : return true;
5560 : }
5561 :
5562 3928514 : cand_value_at (loop, cand, use->stmt, desc, &bnd);
5563 :
5564 3928514 : *bound = fold_convert (TREE_TYPE (cand->iv->base),
5565 : aff_combination_to_tree (&bnd));
5566 3928514 : *comp = iv_elimination_compare (data, use);
5567 :
5568 : /* Sometimes, it is possible to handle the situation that the number of
5569 : iterations may be zero unless additional assumptions by using <
5570 : instead of != in the exit condition.
5571 :
5572 : TODO: we could also calculate the value MAY_BE_ZERO ? 0 : NITER and
5573 : base the exit condition on it. However, that is often too
5574 : expensive. */
5575 3928514 : if (!integer_zerop (desc->may_be_zero)
5576 3928514 : && !iv_elimination_compare_lt (data, use, cand, comp, desc, bound))
5577 : return false;
5578 :
5579 : /* It is unlikely that computing the number of iterations using division
5580 : would be more profitable than keeping the original induction variable. */
5581 3718138 : bool cond_overflow_p;
5582 3718138 : if (expression_expensive_p (*bound, &cond_overflow_p))
5583 7052 : return false;
5584 :
5585 : return true;
5586 4959894 : }
5587 :
5588 : /* Calculates the cost of BOUND, if it is a PARM_DECL. A PARM_DECL must
5589 : be copied, if it is used in the loop body and DATA->body_includes_call. */
5590 :
5591 : static int
5592 8453653 : parm_decl_cost (struct ivopts_data *data, tree bound)
5593 : {
5594 8453653 : tree sbound = bound;
5595 8453653 : STRIP_NOPS (sbound);
5596 :
5597 8453653 : if (TREE_CODE (sbound) == SSA_NAME
5598 2915986 : && SSA_NAME_IS_DEFAULT_DEF (sbound)
5599 156140 : && TREE_CODE (SSA_NAME_VAR (sbound)) == PARM_DECL
5600 8607390 : && data->body_includes_call)
5601 36891 : return COSTS_N_INSNS (1);
5602 :
5603 : return 0;
5604 : }
5605 :
5606 : /* Determines cost of computing the use in GROUP with CAND in a condition. */
5607 :
5608 : static bool
5609 5977175 : determine_group_iv_cost_cond (struct ivopts_data *data,
5610 : struct iv_group *group, struct iv_cand *cand)
5611 : {
5612 5977175 : tree bound = NULL_TREE;
5613 5977175 : struct iv *cmp_iv;
5614 5977175 : bitmap inv_exprs = NULL;
5615 5977175 : bitmap inv_vars_elim = NULL, inv_vars_express = NULL, inv_vars;
5616 5977175 : comp_cost elim_cost = infinite_cost, express_cost, cost, bound_cost;
5617 5977175 : enum comp_iv_rewrite rewrite_type;
5618 5977175 : iv_inv_expr_ent *inv_expr_elim = NULL, *inv_expr_express = NULL, *inv_expr;
5619 5977175 : tree *control_var, *bound_cst;
5620 5977175 : enum tree_code comp = ERROR_MARK;
5621 5977175 : struct iv_use *use = group->vuses[0];
5622 :
5623 : /* Extract condition operands. */
5624 5977175 : rewrite_type = extract_cond_operands (data, use->stmt, &control_var,
5625 : &bound_cst, NULL, &cmp_iv);
5626 5977175 : gcc_assert (rewrite_type != COMP_IV_NA);
5627 :
5628 : /* Try iv elimination. */
5629 5977175 : if (rewrite_type == COMP_IV_ELIM
5630 5977175 : && may_eliminate_iv (data, use, cand, &bound, &comp))
5631 : {
5632 3711086 : elim_cost = force_var_cost (data, bound, &inv_vars_elim);
5633 3711086 : if (elim_cost.cost == 0)
5634 2497903 : elim_cost.cost = parm_decl_cost (data, bound);
5635 1213183 : else if (TREE_CODE (bound) == INTEGER_CST)
5636 0 : elim_cost.cost = 0;
5637 : /* If we replace a loop condition 'i < n' with 'p < base + n',
5638 : inv_vars_elim will have 'base' and 'n' set, which implies that both
5639 : 'base' and 'n' will be live during the loop. More likely,
5640 : 'base + n' will be loop invariant, resulting in only one live value
5641 : during the loop. So in that case we clear inv_vars_elim and set
5642 : inv_expr_elim instead. */
5643 3711086 : if (inv_vars_elim && bitmap_count_bits (inv_vars_elim) > 1)
5644 : {
5645 323037 : inv_expr_elim = get_loop_invariant_expr (data, bound);
5646 323037 : bitmap_clear (inv_vars_elim);
5647 : }
5648 : /* The bound is a loop invariant, so it will be only computed
5649 : once. */
5650 3711086 : elim_cost.cost = adjust_setup_cost (data, elim_cost.cost);
5651 : }
5652 :
5653 : /* When the condition is a comparison of the candidate IV against
5654 : zero, prefer this IV.
5655 :
5656 : TODO: The constant that we're subtracting from the cost should
5657 : be target-dependent. This information should be added to the
5658 : target costs for each backend. */
5659 5977175 : if (!elim_cost.infinite_cost_p () /* Do not try to decrease infinite! */
5660 3711086 : && integer_zerop (*bound_cst)
5661 8620464 : && (operand_equal_p (*control_var, cand->var_after, 0)
5662 2390136 : || operand_equal_p (*control_var, cand->var_before, 0)))
5663 259772 : elim_cost -= 1;
5664 :
5665 5977175 : express_cost = get_computation_cost (data, use, cand, false,
5666 : &inv_vars_express, NULL,
5667 : &inv_expr_express);
5668 5977175 : if (cmp_iv != NULL)
5669 5045576 : find_inv_vars (data, &cmp_iv->base, &inv_vars_express);
5670 :
5671 : /* Count the cost of the original bound as well. */
5672 5977175 : bound_cost = force_var_cost (data, *bound_cst, NULL);
5673 5977175 : if (bound_cost.cost == 0)
5674 5955750 : bound_cost.cost = parm_decl_cost (data, *bound_cst);
5675 21425 : else if (TREE_CODE (*bound_cst) == INTEGER_CST)
5676 0 : bound_cost.cost = 0;
5677 5977175 : express_cost += bound_cost;
5678 :
5679 : /* Choose the better approach, preferring the eliminated IV. */
5680 5977175 : if (elim_cost <= express_cost)
5681 : {
5682 4589920 : cost = elim_cost;
5683 4589920 : inv_vars = inv_vars_elim;
5684 4589920 : inv_vars_elim = NULL;
5685 4589920 : inv_expr = inv_expr_elim;
5686 : /* For doloop candidate/use pair, adjust to zero cost. */
5687 4589920 : if (group->doloop_p && cand->doloop_p && elim_cost.cost > no_cost.cost)
5688 0 : cost = no_cost;
5689 : }
5690 : else
5691 : {
5692 1387255 : cost = express_cost;
5693 1387255 : inv_vars = inv_vars_express;
5694 1387255 : inv_vars_express = NULL;
5695 1387255 : bound = NULL_TREE;
5696 1387255 : comp = ERROR_MARK;
5697 1387255 : inv_expr = inv_expr_express;
5698 : }
5699 :
5700 5977175 : if (inv_expr)
5701 : {
5702 601495 : inv_exprs = BITMAP_ALLOC (NULL);
5703 601495 : bitmap_set_bit (inv_exprs, inv_expr->id);
5704 : }
5705 5977175 : set_group_iv_cost (data, group, cand, cost,
5706 : inv_vars, bound, comp, inv_exprs);
5707 :
5708 5977175 : if (inv_vars_elim)
5709 23858 : BITMAP_FREE (inv_vars_elim);
5710 5977175 : if (inv_vars_express)
5711 1278729 : BITMAP_FREE (inv_vars_express);
5712 :
5713 5977175 : return !cost.infinite_cost_p ();
5714 : }
5715 :
5716 : /* Determines cost of computing uses in GROUP with CAND. Returns false
5717 : if USE cannot be represented with CAND. */
5718 :
5719 : static bool
5720 17837174 : determine_group_iv_cost (struct ivopts_data *data,
5721 : struct iv_group *group, struct iv_cand *cand)
5722 : {
5723 17837174 : switch (group->type)
5724 : {
5725 5524445 : case USE_NONLINEAR_EXPR:
5726 5524445 : return determine_group_iv_cost_generic (data, group, cand);
5727 :
5728 6335554 : case USE_REF_ADDRESS:
5729 6335554 : case USE_PTR_ADDRESS:
5730 6335554 : return determine_group_iv_cost_address (data, group, cand);
5731 :
5732 5977175 : case USE_COMPARE:
5733 5977175 : return determine_group_iv_cost_cond (data, group, cand);
5734 :
5735 0 : default:
5736 0 : gcc_unreachable ();
5737 : }
5738 : }
5739 :
5740 : /* Return true if get_computation_cost indicates that autoincrement is
5741 : a possibility for the pair of USE and CAND, false otherwise. */
5742 :
5743 : static bool
5744 1296811 : autoinc_possible_for_pair (struct ivopts_data *data, struct iv_use *use,
5745 : struct iv_cand *cand)
5746 : {
5747 1296811 : if (!address_p (use->type))
5748 : return false;
5749 :
5750 421089 : bool can_autoinc = false;
5751 421089 : get_computation_cost (data, use, cand, true, NULL, &can_autoinc, NULL);
5752 421089 : return can_autoinc;
5753 : }
5754 :
5755 : /* Examine IP_ORIGINAL candidates to see if they are incremented next to a
5756 : use that allows autoincrement, and set their AINC_USE if possible. */
5757 :
5758 : static void
5759 510604 : set_autoinc_for_original_candidates (struct ivopts_data *data)
5760 : {
5761 510604 : unsigned i, j;
5762 :
5763 5184416 : for (i = 0; i < data->vcands.length (); i++)
5764 : {
5765 4673812 : struct iv_cand *cand = data->vcands[i];
5766 4673812 : struct iv_use *closest_before = NULL;
5767 4673812 : struct iv_use *closest_after = NULL;
5768 4673812 : if (cand->pos != IP_ORIGINAL)
5769 3793190 : continue;
5770 :
5771 3863124 : for (j = 0; j < data->vgroups.length (); j++)
5772 : {
5773 2982502 : struct iv_group *group = data->vgroups[j];
5774 2982502 : struct iv_use *use = group->vuses[0];
5775 2982502 : unsigned uid = gimple_uid (use->stmt);
5776 :
5777 2982502 : if (gimple_bb (use->stmt) != gimple_bb (cand->incremented_at))
5778 1172911 : continue;
5779 :
5780 1809591 : if (uid < gimple_uid (cand->incremented_at)
5781 1809591 : && (closest_before == NULL
5782 380512 : || uid > gimple_uid (closest_before->stmt)))
5783 : closest_before = use;
5784 :
5785 1809591 : if (uid > gimple_uid (cand->incremented_at)
5786 1809591 : && (closest_after == NULL
5787 73133 : || uid < gimple_uid (closest_after->stmt)))
5788 : closest_after = use;
5789 : }
5790 :
5791 880622 : if (closest_before != NULL
5792 880622 : && autoinc_possible_for_pair (data, closest_before, cand))
5793 0 : cand->ainc_use = closest_before;
5794 880622 : else if (closest_after != NULL
5795 880622 : && autoinc_possible_for_pair (data, closest_after, cand))
5796 0 : cand->ainc_use = closest_after;
5797 : }
5798 510604 : }
5799 :
5800 : /* Relate compare use with all candidates. */
5801 :
5802 : static void
5803 328 : relate_compare_use_with_all_cands (struct ivopts_data *data)
5804 : {
5805 328 : unsigned i, count = data->vcands.length ();
5806 9735 : for (i = 0; i < data->vgroups.length (); i++)
5807 : {
5808 9407 : struct iv_group *group = data->vgroups[i];
5809 :
5810 9407 : if (group->type == USE_COMPARE)
5811 1938 : bitmap_set_range (group->related_cands, 0, count);
5812 : }
5813 328 : }
5814 :
5815 : /* If PREFERRED_MODE is suitable and profitable, use the preferred
5816 : PREFERRED_MODE to compute doloop iv base from niter: base = niter + 1. */
5817 :
5818 : static tree
5819 0 : compute_doloop_base_on_mode (machine_mode preferred_mode, tree niter,
5820 : const widest_int &iterations_max)
5821 : {
5822 0 : tree ntype = TREE_TYPE (niter);
5823 0 : tree pref_type = lang_hooks.types.type_for_mode (preferred_mode, 1);
5824 0 : if (!pref_type)
5825 0 : return fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5826 : build_int_cst (ntype, 1));
5827 :
5828 0 : gcc_assert (TREE_CODE (pref_type) == INTEGER_TYPE);
5829 :
5830 0 : int prec = TYPE_PRECISION (ntype);
5831 0 : int pref_prec = TYPE_PRECISION (pref_type);
5832 :
5833 0 : tree base;
5834 :
5835 : /* Check if the PREFERRED_MODED is able to present niter. */
5836 0 : if (pref_prec > prec
5837 0 : || wi::ltu_p (iterations_max,
5838 0 : widest_int::from (wi::max_value (pref_prec, UNSIGNED),
5839 : UNSIGNED)))
5840 : {
5841 : /* No wrap, it is safe to use preferred type after niter + 1. */
5842 0 : if (wi::ltu_p (iterations_max,
5843 0 : widest_int::from (wi::max_value (prec, UNSIGNED),
5844 : UNSIGNED)))
5845 : {
5846 : /* This could help to optimize "-1 +1" pair when niter looks
5847 : like "n-1": n is in original mode. "base = (n - 1) + 1"
5848 : in PREFERRED_MODED: it could be base = (PREFERRED_TYPE)n. */
5849 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5850 : build_int_cst (ntype, 1));
5851 0 : base = fold_convert (pref_type, base);
5852 : }
5853 :
5854 : /* To avoid wrap, convert niter to preferred type before plus 1. */
5855 : else
5856 : {
5857 0 : niter = fold_convert (pref_type, niter);
5858 0 : base = fold_build2 (PLUS_EXPR, pref_type, unshare_expr (niter),
5859 : build_int_cst (pref_type, 1));
5860 : }
5861 : }
5862 : else
5863 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5864 : build_int_cst (ntype, 1));
5865 : return base;
5866 : }
5867 :
5868 : /* Add one doloop dedicated IV candidate:
5869 : - Base is (may_be_zero ? 1 : (niter + 1)).
5870 : - Step is -1. */
5871 :
5872 : static void
5873 0 : add_iv_candidate_for_doloop (struct ivopts_data *data)
5874 : {
5875 0 : tree_niter_desc *niter_desc = niter_for_single_dom_exit (data);
5876 0 : gcc_assert (niter_desc && niter_desc->assumptions);
5877 :
5878 0 : tree niter = niter_desc->niter;
5879 0 : tree ntype = TREE_TYPE (niter);
5880 0 : gcc_assert (INTEGRAL_NB_TYPE_P (ntype));
5881 :
5882 0 : tree may_be_zero = niter_desc->may_be_zero;
5883 0 : if (may_be_zero && integer_zerop (may_be_zero))
5884 : may_be_zero = NULL_TREE;
5885 0 : if (may_be_zero)
5886 : {
5887 0 : if (COMPARISON_CLASS_P (may_be_zero))
5888 : {
5889 0 : niter = fold_build3 (COND_EXPR, ntype, may_be_zero,
5890 : build_int_cst (ntype, 0),
5891 : rewrite_to_non_trapping_overflow (niter));
5892 : }
5893 : /* Don't try to obtain the iteration count expression when may_be_zero is
5894 : integer_nonzerop (actually iteration count is one) or else. */
5895 : else
5896 : return;
5897 : }
5898 :
5899 0 : machine_mode mode = TYPE_MODE (ntype);
5900 0 : machine_mode pref_mode = targetm.preferred_doloop_mode (mode);
5901 :
5902 0 : tree base;
5903 0 : if (mode != pref_mode)
5904 : {
5905 0 : base = compute_doloop_base_on_mode (pref_mode, niter, niter_desc->max);
5906 0 : ntype = TREE_TYPE (base);
5907 : }
5908 : else
5909 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5910 : build_int_cst (ntype, 1));
5911 :
5912 : /* For non integer types or non-mode precision types,
5913 : convert directly to an integer type. */
5914 0 : if (TREE_CODE (ntype) != INTEGER_TYPE
5915 0 : || !type_has_mode_precision_p (ntype))
5916 : {
5917 0 : ntype = lang_hooks.types.type_for_mode (TYPE_MODE (ntype),
5918 0 : TYPE_UNSIGNED (ntype));
5919 0 : base = fold_convert (ntype, base);
5920 : }
5921 :
5922 0 : add_candidate (data, base, build_int_cst (ntype, -1), true, NULL, NULL, true);
5923 : }
5924 :
5925 : /* Finds the candidates for the induction variables. */
5926 :
5927 : static void
5928 510604 : find_iv_candidates (struct ivopts_data *data)
5929 : {
5930 : /* Add commonly used ivs. */
5931 510604 : add_standard_iv_candidates (data);
5932 :
5933 : /* Add doloop dedicated ivs. */
5934 510604 : if (data->doloop_use_p)
5935 0 : add_iv_candidate_for_doloop (data);
5936 :
5937 : /* Add old induction variables. */
5938 510604 : add_iv_candidate_for_bivs (data);
5939 :
5940 : /* Add induction variables derived from uses. */
5941 510604 : add_iv_candidate_for_groups (data);
5942 :
5943 510604 : set_autoinc_for_original_candidates (data);
5944 :
5945 : /* Record the important candidates. */
5946 510604 : record_important_candidates (data);
5947 :
5948 : /* Relate compare iv_use with all candidates. */
5949 510604 : if (!data->consider_all_candidates)
5950 328 : relate_compare_use_with_all_cands (data);
5951 :
5952 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
5953 : {
5954 70 : unsigned i;
5955 :
5956 70 : fprintf (dump_file, "\n<Important Candidates>:\t");
5957 854 : for (i = 0; i < data->vcands.length (); i++)
5958 714 : if (data->vcands[i]->important)
5959 509 : fprintf (dump_file, " %d,", data->vcands[i]->id);
5960 70 : fprintf (dump_file, "\n");
5961 :
5962 70 : fprintf (dump_file, "\n<Group, Cand> Related:\n");
5963 367 : for (i = 0; i < data->vgroups.length (); i++)
5964 : {
5965 227 : struct iv_group *group = data->vgroups[i];
5966 :
5967 227 : if (group->related_cands)
5968 : {
5969 227 : fprintf (dump_file, " Group %d:\t", group->id);
5970 227 : dump_bitmap (dump_file, group->related_cands);
5971 : }
5972 : }
5973 70 : fprintf (dump_file, "\n");
5974 : }
5975 510604 : }
5976 :
5977 : /* Determines costs of computing use of iv with an iv candidate. */
5978 :
5979 : static void
5980 510604 : determine_group_iv_costs (struct ivopts_data *data)
5981 : {
5982 510604 : unsigned i, j;
5983 510604 : struct iv_cand *cand;
5984 510604 : struct iv_group *group;
5985 510604 : bitmap to_clear = BITMAP_ALLOC (NULL);
5986 :
5987 510604 : alloc_use_cost_map (data);
5988 :
5989 2173174 : for (i = 0; i < data->vgroups.length (); i++)
5990 : {
5991 1662570 : group = data->vgroups[i];
5992 :
5993 1662570 : if (data->consider_all_candidates)
5994 : {
5995 19158234 : for (j = 0; j < data->vcands.length (); j++)
5996 : {
5997 17495664 : cand = data->vcands[j];
5998 17495664 : determine_group_iv_cost (data, group, cand);
5999 : }
6000 : }
6001 : else
6002 : {
6003 9407 : bitmap_iterator bi;
6004 :
6005 350917 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, j, bi)
6006 : {
6007 341510 : cand = data->vcands[j];
6008 341510 : if (!determine_group_iv_cost (data, group, cand))
6009 204623 : bitmap_set_bit (to_clear, j);
6010 : }
6011 :
6012 : /* Remove the candidates for that the cost is infinite from
6013 : the list of related candidates. */
6014 9407 : bitmap_and_compl_into (group->related_cands, to_clear);
6015 9407 : bitmap_clear (to_clear);
6016 : }
6017 : }
6018 :
6019 510604 : BITMAP_FREE (to_clear);
6020 :
6021 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6022 : {
6023 70 : bitmap_iterator bi;
6024 :
6025 : /* Dump invariant variables. */
6026 70 : fprintf (dump_file, "\n<Invariant Vars>:\n");
6027 1116 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
6028 : {
6029 1046 : struct version_info *info = ver_info (data, i);
6030 1046 : if (info->inv_id)
6031 : {
6032 236 : fprintf (dump_file, "Inv %d:\t", info->inv_id);
6033 236 : print_generic_expr (dump_file, info->name, TDF_SLIM);
6034 236 : fprintf (dump_file, "%s\n",
6035 236 : info->has_nonlin_use ? "" : "\t(eliminable)");
6036 : }
6037 : }
6038 :
6039 : /* Dump invariant expressions. */
6040 70 : fprintf (dump_file, "\n<Invariant Expressions>:\n");
6041 70 : auto_vec <iv_inv_expr_ent *> list (data->inv_expr_tab->elements ());
6042 :
6043 383 : for (hash_table<iv_inv_expr_hasher>::iterator it
6044 523 : = data->inv_expr_tab->begin (); it != data->inv_expr_tab->end ();
6045 383 : ++it)
6046 383 : list.safe_push (*it);
6047 :
6048 70 : list.qsort (sort_iv_inv_expr_ent);
6049 :
6050 453 : for (i = 0; i < list.length (); ++i)
6051 : {
6052 383 : fprintf (dump_file, "inv_expr %d: \t", list[i]->id);
6053 383 : print_generic_expr (dump_file, list[i]->expr, TDF_SLIM);
6054 383 : fprintf (dump_file, "\n");
6055 : }
6056 :
6057 70 : fprintf (dump_file, "\n<Group-candidate Costs>:\n");
6058 :
6059 297 : for (i = 0; i < data->vgroups.length (); i++)
6060 : {
6061 227 : group = data->vgroups[i];
6062 :
6063 227 : fprintf (dump_file, "Group %d:\n", i);
6064 227 : fprintf (dump_file, " cand\tcost\tcompl.\tinv.expr.\tinv.vars\n");
6065 3065 : for (j = 0; j < group->n_map_members; j++)
6066 : {
6067 3955 : if (!group->cost_map[j].cand
6068 2838 : || group->cost_map[j].cost.infinite_cost_p ())
6069 1117 : continue;
6070 :
6071 1721 : fprintf (dump_file, " %d\t%" PRId64 "\t%d\t",
6072 1721 : group->cost_map[j].cand->id,
6073 : group->cost_map[j].cost.cost,
6074 1721 : group->cost_map[j].cost.complexity);
6075 1721 : if (!group->cost_map[j].inv_exprs
6076 1721 : || bitmap_empty_p (group->cost_map[j].inv_exprs))
6077 1211 : fprintf (dump_file, "NIL;\t");
6078 : else
6079 510 : bitmap_print (dump_file,
6080 : group->cost_map[j].inv_exprs, "", ";\t");
6081 1721 : if (!group->cost_map[j].inv_vars
6082 1721 : || bitmap_empty_p (group->cost_map[j].inv_vars))
6083 1377 : fprintf (dump_file, "NIL;\n");
6084 : else
6085 344 : bitmap_print (dump_file,
6086 : group->cost_map[j].inv_vars, "", "\n");
6087 : }
6088 :
6089 227 : fprintf (dump_file, "\n");
6090 : }
6091 70 : fprintf (dump_file, "\n");
6092 70 : }
6093 510604 : }
6094 :
6095 : /* Determines cost of the candidate CAND. */
6096 :
6097 : static void
6098 4673812 : determine_iv_cost (struct ivopts_data *data, struct iv_cand *cand)
6099 : {
6100 4673812 : comp_cost cost_base;
6101 4673812 : int64_t cost, cost_step;
6102 4673812 : tree base;
6103 :
6104 4673812 : gcc_assert (cand->iv != NULL);
6105 :
6106 : /* There are two costs associated with the candidate -- its increment
6107 : and its initialization. The second is almost negligible for any loop
6108 : that rolls enough, so we take it just very little into account. */
6109 :
6110 4673812 : base = cand->iv->base;
6111 4673812 : cost_base = force_var_cost (data, base, NULL);
6112 : /* It will be exceptional that the iv register happens to be initialized with
6113 : the proper value at no cost. In general, there will at least be a regcopy
6114 : or a const set. */
6115 4673812 : if (cost_base.cost == 0)
6116 3709985 : cost_base.cost = COSTS_N_INSNS (1);
6117 : /* Doloop decrement should be considered as zero cost. */
6118 4673812 : if (cand->doloop_p)
6119 : cost_step = 0;
6120 : else
6121 4673812 : cost_step = add_cost (data->speed, TYPE_MODE (TREE_TYPE (base)));
6122 4673812 : cost = cost_step + adjust_setup_cost (data, cost_base.cost);
6123 :
6124 : /* Prefer the original ivs unless we may gain something by replacing it.
6125 : The reason is to make debugging simpler; so this is not relevant for
6126 : artificial ivs created by other optimization passes. */
6127 4673812 : if ((cand->pos != IP_ORIGINAL
6128 880622 : || !SSA_NAME_VAR (cand->var_before)
6129 444971 : || DECL_ARTIFICIAL (SSA_NAME_VAR (cand->var_before)))
6130 : /* Prefer doloop as well. */
6131 5204630 : && !cand->doloop_p)
6132 4324008 : cost++;
6133 :
6134 : /* Prefer not to insert statements into latch unless there are some
6135 : already (so that we do not create unnecessary jumps). */
6136 4673812 : if (cand->pos == IP_END
6137 4673812 : && empty_block_p (ip_end_pos (data->current_loop)))
6138 1851 : cost++;
6139 :
6140 4673812 : cand->cost = cost;
6141 4673812 : cand->cost_step = cost_step;
6142 4673812 : }
6143 :
6144 : /* Determines costs of computation of the candidates. */
6145 :
6146 : static void
6147 510604 : determine_iv_costs (struct ivopts_data *data)
6148 : {
6149 510604 : unsigned i;
6150 :
6151 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6152 : {
6153 70 : fprintf (dump_file, "<Candidate Costs>:\n");
6154 70 : fprintf (dump_file, " cand\tcost\n");
6155 : }
6156 :
6157 5184416 : for (i = 0; i < data->vcands.length (); i++)
6158 : {
6159 4673812 : struct iv_cand *cand = data->vcands[i];
6160 :
6161 4673812 : determine_iv_cost (data, cand);
6162 :
6163 4673812 : if (dump_file && (dump_flags & TDF_DETAILS))
6164 714 : fprintf (dump_file, " %d\t%d\n", i, cand->cost);
6165 : }
6166 :
6167 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6168 70 : fprintf (dump_file, "\n");
6169 510604 : }
6170 :
6171 : /* Estimate register pressure for loop having N_INVS invariants and N_CANDS
6172 : induction variables. Note N_INVS includes both invariant variables and
6173 : invariant expressions. */
6174 :
6175 : static unsigned
6176 430471637 : ivopts_estimate_reg_pressure (struct ivopts_data *data, unsigned n_invs,
6177 : unsigned n_cands)
6178 : {
6179 430471637 : unsigned cost;
6180 430471637 : unsigned n_old = data->regs_used, n_new = n_invs + n_cands;
6181 430471637 : unsigned regs_needed = n_new + n_old, available_regs = target_avail_regs;
6182 430471637 : bool speed = data->speed;
6183 :
6184 : /* If there is a call in the loop body, the call-clobbered registers
6185 : are not available for loop invariants. */
6186 430471637 : if (data->body_includes_call)
6187 98118268 : available_regs = available_regs - target_clobbered_regs;
6188 :
6189 : /* If we have enough registers. */
6190 430471637 : if (regs_needed <= available_regs)
6191 : cost = 0;
6192 : /* If we run out of available registers but the number of candidates
6193 : does not, we penalize extra registers using target_spill_cost.
6194 : As we tend to spill invariants here, only take loading the
6195 : invariant into account, because the invariant won't change for the
6196 : duration of the loop and storing it every iteration is unnecessary. */
6197 137259078 : else if (n_cands <= available_regs)
6198 117479729 : cost = target_spill_cost [speed] * (regs_needed - available_regs) / 2;
6199 : /* If both IV cands and invariants spill, calculate additional cost for
6200 : having to store spilled candidates. */
6201 : else
6202 19779349 : cost = (target_spill_cost [speed] * (regs_needed - available_regs) / 2
6203 19779349 : + target_spill_cost[speed] * (n_cands - available_regs) / 2);
6204 :
6205 430471637 : return cost;
6206 : }
6207 :
6208 : /* For each size of the induction variable set determine the penalty. */
6209 :
6210 : static void
6211 510604 : determine_set_costs (struct ivopts_data *data)
6212 : {
6213 510604 : unsigned j, n;
6214 510604 : gphi *phi;
6215 510604 : gphi_iterator psi;
6216 510604 : tree op;
6217 510604 : class loop *loop = data->current_loop;
6218 510604 : bitmap_iterator bi;
6219 :
6220 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6221 : {
6222 70 : fprintf (dump_file, "<Global Costs>:\n");
6223 70 : fprintf (dump_file, " target_avail_regs %d\n", target_avail_regs);
6224 70 : fprintf (dump_file, " target_clobbered_regs %d\n", target_clobbered_regs);
6225 70 : fprintf (dump_file, " target_reg_cost %d\n", target_reg_cost[data->speed]);
6226 70 : fprintf (dump_file, " target_spill_cost %d\n", target_spill_cost[data->speed]);
6227 : }
6228 :
6229 510604 : n = 0;
6230 1986437 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
6231 : {
6232 1475833 : phi = psi.phi ();
6233 1475833 : op = PHI_RESULT (phi);
6234 :
6235 2951666 : if (virtual_operand_p (op))
6236 312800 : continue;
6237 :
6238 1163033 : if (get_iv (data, op))
6239 884526 : continue;
6240 :
6241 510979 : if (!POINTER_TYPE_P (TREE_TYPE (op))
6242 510744 : && !INTEGRAL_TYPE_P (TREE_TYPE (op)))
6243 103108 : continue;
6244 :
6245 175399 : n++;
6246 : }
6247 :
6248 5649483 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
6249 : {
6250 5138879 : struct version_info *info = ver_info (data, j);
6251 :
6252 5138879 : if (info->inv_id && info->has_nonlin_use)
6253 526468 : n++;
6254 : }
6255 :
6256 510604 : data->regs_used = n;
6257 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6258 70 : fprintf (dump_file, " regs_used %d\n", n);
6259 :
6260 510604 : if (dump_file && (dump_flags & TDF_DETAILS))
6261 : {
6262 70 : fprintf (dump_file, " cost for size:\n");
6263 70 : fprintf (dump_file, " ivs\tcost\n");
6264 2240 : for (j = 0; j <= 2 * target_avail_regs; j++)
6265 2170 : fprintf (dump_file, " %d\t%d\n", j,
6266 : ivopts_estimate_reg_pressure (data, 0, j));
6267 70 : fprintf (dump_file, "\n");
6268 : }
6269 510604 : }
6270 :
6271 : /* Returns true if A is a cheaper cost pair than B. */
6272 :
6273 : static bool
6274 86500326 : cheaper_cost_pair (class cost_pair *a, class cost_pair *b)
6275 : {
6276 86500326 : if (!a)
6277 : return false;
6278 :
6279 81081502 : if (!b)
6280 : return true;
6281 :
6282 77851467 : if (a->cost < b->cost)
6283 : return true;
6284 :
6285 57240841 : if (b->cost < a->cost)
6286 : return false;
6287 :
6288 : /* In case the costs are the same, prefer the cheaper candidate. */
6289 32453570 : if (a->cand->cost < b->cand->cost)
6290 4841927 : return true;
6291 :
6292 : return false;
6293 : }
6294 :
6295 : /* Compare if A is a more expensive cost pair than B. Return 1, 0 and -1
6296 : for more expensive, equal and cheaper respectively. */
6297 :
6298 : static int
6299 30897461 : compare_cost_pair (class cost_pair *a, class cost_pair *b)
6300 : {
6301 30897461 : if (cheaper_cost_pair (a, b))
6302 : return -1;
6303 24572362 : if (cheaper_cost_pair (b, a))
6304 16163116 : return 1;
6305 :
6306 : return 0;
6307 : }
6308 :
6309 : /* Returns candidate by that USE is expressed in IVS. */
6310 :
6311 : static class cost_pair *
6312 288489337 : iv_ca_cand_for_group (class iv_ca *ivs, struct iv_group *group)
6313 : {
6314 288489337 : return ivs->cand_for_group[group->id];
6315 : }
6316 :
6317 : /* Computes the cost field of IVS structure. */
6318 :
6319 : static void
6320 430469206 : iv_ca_recount_cost (struct ivopts_data *data, class iv_ca *ivs)
6321 : {
6322 430469206 : comp_cost cost = ivs->cand_use_cost;
6323 :
6324 430469206 : cost += ivs->cand_cost;
6325 430469206 : cost += ivopts_estimate_reg_pressure (data, ivs->n_invs, ivs->n_cands);
6326 430469206 : ivs->cost = cost;
6327 430469206 : }
6328 :
6329 : /* Remove use of invariants in set INVS by decreasing counter in N_INV_USES
6330 : and IVS. */
6331 :
6332 : static void
6333 591672464 : iv_ca_set_remove_invs (class iv_ca *ivs, bitmap invs, unsigned *n_inv_uses)
6334 : {
6335 591672464 : bitmap_iterator bi;
6336 591672464 : unsigned iid;
6337 :
6338 591672464 : if (!invs)
6339 469851950 : return;
6340 :
6341 121820514 : gcc_assert (n_inv_uses != NULL);
6342 210959039 : EXECUTE_IF_SET_IN_BITMAP (invs, 0, iid, bi)
6343 : {
6344 89138525 : n_inv_uses[iid]--;
6345 89138525 : if (n_inv_uses[iid] == 0)
6346 66485316 : ivs->n_invs--;
6347 : }
6348 : }
6349 :
6350 : /* Set USE not to be expressed by any candidate in IVS. */
6351 :
6352 : static void
6353 213573458 : iv_ca_set_no_cp (struct ivopts_data *data, class iv_ca *ivs,
6354 : struct iv_group *group)
6355 : {
6356 213573458 : unsigned gid = group->id, cid;
6357 213573458 : class cost_pair *cp;
6358 :
6359 213573458 : cp = ivs->cand_for_group[gid];
6360 213573458 : if (!cp)
6361 : return;
6362 213573458 : cid = cp->cand->id;
6363 :
6364 213573458 : ivs->bad_groups++;
6365 213573458 : ivs->cand_for_group[gid] = NULL;
6366 213573458 : ivs->n_cand_uses[cid]--;
6367 :
6368 213573458 : if (ivs->n_cand_uses[cid] == 0)
6369 : {
6370 82262774 : bitmap_clear_bit (ivs->cands, cid);
6371 82262774 : if (!cp->cand->doloop_p || !targetm.have_count_reg_decr_p)
6372 82262774 : ivs->n_cands--;
6373 82262774 : ivs->cand_cost -= cp->cand->cost;
6374 82262774 : iv_ca_set_remove_invs (ivs, cp->cand->inv_vars, ivs->n_inv_var_uses);
6375 82262774 : iv_ca_set_remove_invs (ivs, cp->cand->inv_exprs, ivs->n_inv_expr_uses);
6376 : }
6377 :
6378 213573458 : ivs->cand_use_cost -= cp->cost;
6379 213573458 : iv_ca_set_remove_invs (ivs, cp->inv_vars, ivs->n_inv_var_uses);
6380 213573458 : iv_ca_set_remove_invs (ivs, cp->inv_exprs, ivs->n_inv_expr_uses);
6381 213573458 : iv_ca_recount_cost (data, ivs);
6382 : }
6383 :
6384 : /* Add use of invariants in set INVS by increasing counter in N_INV_USES and
6385 : IVS. */
6386 :
6387 : static void
6388 601234858 : iv_ca_set_add_invs (class iv_ca *ivs, bitmap invs, unsigned *n_inv_uses)
6389 : {
6390 601234858 : bitmap_iterator bi;
6391 601234858 : unsigned iid;
6392 :
6393 601234858 : if (!invs)
6394 478268099 : return;
6395 :
6396 122966759 : gcc_assert (n_inv_uses != NULL);
6397 213082658 : EXECUTE_IF_SET_IN_BITMAP (invs, 0, iid, bi)
6398 : {
6399 90115899 : n_inv_uses[iid]++;
6400 90115899 : if (n_inv_uses[iid] == 1)
6401 67389709 : ivs->n_invs++;
6402 : }
6403 : }
6404 :
6405 : /* Set cost pair for GROUP in set IVS to CP. */
6406 :
6407 : static void
6408 230001880 : iv_ca_set_cp (struct ivopts_data *data, class iv_ca *ivs,
6409 : struct iv_group *group, class cost_pair *cp)
6410 : {
6411 230001880 : unsigned gid = group->id, cid;
6412 :
6413 230001880 : if (ivs->cand_for_group[gid] == cp)
6414 : return;
6415 :
6416 216895748 : if (ivs->cand_for_group[gid])
6417 201271237 : iv_ca_set_no_cp (data, ivs, group);
6418 :
6419 216895748 : if (cp)
6420 : {
6421 216895748 : cid = cp->cand->id;
6422 :
6423 216895748 : ivs->bad_groups--;
6424 216895748 : ivs->cand_for_group[gid] = cp;
6425 216895748 : ivs->n_cand_uses[cid]++;
6426 216895748 : if (ivs->n_cand_uses[cid] == 1)
6427 : {
6428 83721681 : bitmap_set_bit (ivs->cands, cid);
6429 83721681 : if (!cp->cand->doloop_p || !targetm.have_count_reg_decr_p)
6430 83721681 : ivs->n_cands++;
6431 83721681 : ivs->cand_cost += cp->cand->cost;
6432 83721681 : iv_ca_set_add_invs (ivs, cp->cand->inv_vars, ivs->n_inv_var_uses);
6433 83721681 : iv_ca_set_add_invs (ivs, cp->cand->inv_exprs, ivs->n_inv_expr_uses);
6434 : }
6435 :
6436 216895748 : ivs->cand_use_cost += cp->cost;
6437 216895748 : iv_ca_set_add_invs (ivs, cp->inv_vars, ivs->n_inv_var_uses);
6438 216895748 : iv_ca_set_add_invs (ivs, cp->inv_exprs, ivs->n_inv_expr_uses);
6439 216895748 : iv_ca_recount_cost (data, ivs);
6440 : }
6441 : }
6442 :
6443 : /* Extend set IVS by expressing USE by some of the candidates in it
6444 : if possible. Consider all important candidates if candidates in
6445 : set IVS don't give any result. */
6446 :
6447 : static void
6448 3323194 : iv_ca_add_group (struct ivopts_data *data, class iv_ca *ivs,
6449 : struct iv_group *group)
6450 : {
6451 3323194 : class cost_pair *best_cp = NULL, *cp;
6452 3323194 : bitmap_iterator bi;
6453 3323194 : unsigned i;
6454 3323194 : struct iv_cand *cand;
6455 :
6456 3323194 : gcc_assert (ivs->upto >= group->id);
6457 3323194 : ivs->upto++;
6458 3323194 : ivs->bad_groups++;
6459 :
6460 6285775 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
6461 : {
6462 2962581 : cand = data->vcands[i];
6463 2962581 : cp = get_group_iv_cost (data, group, cand);
6464 2962581 : if (cheaper_cost_pair (cp, best_cp))
6465 2043274 : best_cp = cp;
6466 : }
6467 :
6468 3323194 : if (best_cp == NULL)
6469 : {
6470 11943121 : EXECUTE_IF_SET_IN_BITMAP (data->important_candidates, 0, i, bi)
6471 : {
6472 10583818 : cand = data->vcands[i];
6473 10583818 : cp = get_group_iv_cost (data, group, cand);
6474 10583818 : if (cheaper_cost_pair (cp, best_cp))
6475 2435323 : best_cp = cp;
6476 : }
6477 : }
6478 :
6479 3323194 : iv_ca_set_cp (data, ivs, group, best_cp);
6480 3323194 : }
6481 :
6482 : /* Get cost for assignment IVS. */
6483 :
6484 : static comp_cost
6485 86170475 : iv_ca_cost (class iv_ca *ivs)
6486 : {
6487 : /* This was a conditional expression but it triggered a bug in
6488 : Sun C 5.5. */
6489 0 : if (ivs->bad_groups)
6490 93159 : return infinite_cost;
6491 : else
6492 86077316 : return ivs->cost;
6493 : }
6494 :
6495 : /* Compare if applying NEW_CP to GROUP for IVS introduces more invariants
6496 : than OLD_CP. Return 1, 0 and -1 for more, equal and fewer invariants
6497 : respectively. */
6498 :
6499 : static int
6500 40898693 : iv_ca_compare_deps (struct ivopts_data *data, class iv_ca *ivs,
6501 : struct iv_group *group, class cost_pair *old_cp,
6502 : class cost_pair *new_cp)
6503 : {
6504 40898693 : gcc_assert (old_cp && new_cp && old_cp != new_cp);
6505 40898693 : unsigned old_n_invs = ivs->n_invs;
6506 40898693 : iv_ca_set_cp (data, ivs, group, new_cp);
6507 40898693 : unsigned new_n_invs = ivs->n_invs;
6508 40898693 : iv_ca_set_cp (data, ivs, group, old_cp);
6509 :
6510 40898693 : return new_n_invs > old_n_invs ? 1 : (new_n_invs < old_n_invs ? -1 : 0);
6511 : }
6512 :
6513 : /* Creates change of expressing GROUP by NEW_CP instead of OLD_CP and chains
6514 : it before NEXT. */
6515 :
6516 : static struct iv_ca_delta *
6517 48147576 : iv_ca_delta_add (struct iv_group *group, class cost_pair *old_cp,
6518 : class cost_pair *new_cp, struct iv_ca_delta *next)
6519 : {
6520 0 : struct iv_ca_delta *change = XNEW (struct iv_ca_delta);
6521 :
6522 48147576 : change->group = group;
6523 48147576 : change->old_cp = old_cp;
6524 48147576 : change->new_cp = new_cp;
6525 48147576 : change->next = next;
6526 :
6527 48147576 : return change;
6528 : }
6529 :
6530 : /* Joins two lists of changes L1 and L2. Destructive -- old lists
6531 : are rewritten. */
6532 :
6533 : static struct iv_ca_delta *
6534 8152517 : iv_ca_delta_join (struct iv_ca_delta *l1, struct iv_ca_delta *l2)
6535 : {
6536 8152517 : struct iv_ca_delta *last;
6537 :
6538 0 : if (!l2)
6539 : return l1;
6540 :
6541 0 : if (!l1)
6542 : return l2;
6543 :
6544 3306644 : for (last = l1; last->next; last = last->next)
6545 1092579 : continue;
6546 2214065 : last->next = l2;
6547 :
6548 2214065 : return l1;
6549 1092579 : }
6550 :
6551 : /* Reverse the list of changes DELTA, forming the inverse to it. */
6552 :
6553 : static struct iv_ca_delta *
6554 0 : iv_ca_delta_reverse (struct iv_ca_delta *delta)
6555 : {
6556 0 : struct iv_ca_delta *act, *next, *prev = NULL;
6557 :
6558 161358084 : for (act = delta; act; act = next)
6559 : {
6560 89930072 : next = act->next;
6561 89930072 : act->next = prev;
6562 89930072 : prev = act;
6563 :
6564 89930072 : std::swap (act->old_cp, act->new_cp);
6565 : }
6566 :
6567 0 : return prev;
6568 : }
6569 :
6570 : /* Commit changes in DELTA to IVS. If FORWARD is false, the changes are
6571 : reverted instead. */
6572 :
6573 : static void
6574 75277942 : iv_ca_delta_commit (struct ivopts_data *data, class iv_ca *ivs,
6575 : struct iv_ca_delta *delta, bool forward)
6576 : {
6577 75277942 : class cost_pair *from, *to;
6578 75277942 : struct iv_ca_delta *act;
6579 :
6580 75277942 : if (!forward)
6581 75277942 : delta = iv_ca_delta_reverse (delta);
6582 :
6583 170143913 : for (act = delta; act; act = act->next)
6584 : {
6585 94865971 : from = act->old_cp;
6586 94865971 : to = act->new_cp;
6587 94865971 : gcc_assert (iv_ca_cand_for_group (ivs, act->group) == from);
6588 94865971 : iv_ca_set_cp (data, ivs, act->group, to);
6589 : }
6590 :
6591 75277942 : if (!forward)
6592 75277942 : iv_ca_delta_reverse (delta);
6593 75277942 : }
6594 :
6595 : /* Returns true if CAND is used in IVS. */
6596 :
6597 : static bool
6598 29794364 : iv_ca_cand_used_p (class iv_ca *ivs, struct iv_cand *cand)
6599 : {
6600 29794364 : return ivs->n_cand_uses[cand->id] > 0;
6601 : }
6602 :
6603 : /* Returns number of induction variable candidates in the set IVS. */
6604 :
6605 : static unsigned
6606 13103817 : iv_ca_n_cands (class iv_ca *ivs)
6607 : {
6608 13103817 : return ivs->n_cands;
6609 : }
6610 :
6611 : /* Free the list of changes DELTA. */
6612 :
6613 : static void
6614 45177392 : iv_ca_delta_free (struct iv_ca_delta **delta)
6615 : {
6616 45177392 : struct iv_ca_delta *act, *next;
6617 :
6618 93324968 : for (act = *delta; act; act = next)
6619 : {
6620 48147576 : next = act->next;
6621 48147576 : free (act);
6622 : }
6623 :
6624 45177392 : *delta = NULL;
6625 45177392 : }
6626 :
6627 : /* Allocates new iv candidates assignment. */
6628 :
6629 : static class iv_ca *
6630 1021208 : iv_ca_new (struct ivopts_data *data)
6631 : {
6632 1021208 : class iv_ca *nw = XNEW (class iv_ca);
6633 :
6634 1021208 : nw->upto = 0;
6635 1021208 : nw->bad_groups = 0;
6636 2042416 : nw->cand_for_group = XCNEWVEC (class cost_pair *,
6637 : data->vgroups.length ());
6638 2042416 : nw->n_cand_uses = XCNEWVEC (unsigned, data->vcands.length ());
6639 1021208 : nw->cands = BITMAP_ALLOC (NULL);
6640 1021208 : nw->n_cands = 0;
6641 1021208 : nw->n_invs = 0;
6642 1021208 : nw->cand_use_cost = no_cost;
6643 1021208 : nw->cand_cost = 0;
6644 1021208 : nw->n_inv_var_uses = XCNEWVEC (unsigned, data->max_inv_var_id + 1);
6645 1021208 : nw->n_inv_expr_uses = XCNEWVEC (unsigned, data->max_inv_expr_id + 1);
6646 1021208 : nw->cost = no_cost;
6647 :
6648 1021208 : return nw;
6649 : }
6650 :
6651 : /* Free memory occupied by the set IVS. */
6652 :
6653 : static void
6654 1021208 : iv_ca_free (class iv_ca **ivs)
6655 : {
6656 1021208 : free ((*ivs)->cand_for_group);
6657 1021208 : free ((*ivs)->n_cand_uses);
6658 1021208 : BITMAP_FREE ((*ivs)->cands);
6659 1021208 : free ((*ivs)->n_inv_var_uses);
6660 1021208 : free ((*ivs)->n_inv_expr_uses);
6661 1021208 : free (*ivs);
6662 1021208 : *ivs = NULL;
6663 1021208 : }
6664 :
6665 : /* Dumps IVS to FILE. */
6666 :
6667 : static void
6668 261 : iv_ca_dump (struct ivopts_data *data, FILE *file, class iv_ca *ivs)
6669 : {
6670 261 : unsigned i;
6671 261 : comp_cost cost = iv_ca_cost (ivs);
6672 :
6673 261 : fprintf (file, " cost: %" PRId64 " (complexity %d)\n", cost.cost,
6674 : cost.complexity);
6675 261 : fprintf (file, " reg_cost: %d\n",
6676 : ivopts_estimate_reg_pressure (data, ivs->n_invs, ivs->n_cands));
6677 261 : fprintf (file, " cand_cost: %" PRId64 "\n cand_group_cost: "
6678 : "%" PRId64 " (complexity %d)\n", ivs->cand_cost,
6679 : ivs->cand_use_cost.cost, ivs->cand_use_cost.complexity);
6680 261 : bitmap_print (file, ivs->cands, " candidates: ","\n");
6681 :
6682 1620 : for (i = 0; i < ivs->upto; i++)
6683 : {
6684 1098 : struct iv_group *group = data->vgroups[i];
6685 1098 : class cost_pair *cp = iv_ca_cand_for_group (ivs, group);
6686 1098 : if (cp)
6687 1098 : fprintf (file, " group:%d --> iv_cand:%d, cost=("
6688 1098 : "%" PRId64 ",%d)\n", group->id, cp->cand->id,
6689 : cp->cost.cost, cp->cost.complexity);
6690 : else
6691 0 : fprintf (file, " group:%d --> ??\n", group->id);
6692 : }
6693 :
6694 261 : const char *pref = "";
6695 261 : fprintf (file, " invariant variables: ");
6696 1525 : for (i = 1; i <= data->max_inv_var_id; i++)
6697 1003 : if (ivs->n_inv_var_uses[i])
6698 : {
6699 154 : fprintf (file, "%s%d", pref, i);
6700 154 : pref = ", ";
6701 : }
6702 :
6703 261 : pref = "";
6704 261 : fprintf (file, "\n invariant expressions: ");
6705 2621 : for (i = 1; i <= data->max_inv_expr_id; i++)
6706 2099 : if (ivs->n_inv_expr_uses[i])
6707 : {
6708 309 : fprintf (file, "%s%d", pref, i);
6709 309 : pref = ", ";
6710 : }
6711 :
6712 261 : fprintf (file, "\n\n");
6713 261 : }
6714 :
6715 : /* Try changing candidate in IVS to CAND for each use. Return cost of the
6716 : new set, and store differences in DELTA. Number of induction variables
6717 : in the new set is stored to N_IVS. MIN_NCAND is a flag. When it is true
6718 : the function will try to find a solution with minimal iv candidates. */
6719 :
6720 : static comp_cost
6721 22176003 : iv_ca_extend (struct ivopts_data *data, class iv_ca *ivs,
6722 : struct iv_cand *cand, struct iv_ca_delta **delta,
6723 : unsigned *n_ivs, bool min_ncand)
6724 : {
6725 22176003 : unsigned i;
6726 22176003 : comp_cost cost;
6727 22176003 : struct iv_group *group;
6728 22176003 : class cost_pair *old_cp, *new_cp;
6729 :
6730 22176003 : *delta = NULL;
6731 125435437 : for (i = 0; i < ivs->upto; i++)
6732 : {
6733 103259434 : group = data->vgroups[i];
6734 103259434 : old_cp = iv_ca_cand_for_group (ivs, group);
6735 :
6736 103259434 : if (old_cp
6737 103259434 : && old_cp->cand == cand)
6738 9072186 : continue;
6739 :
6740 94187248 : new_cp = get_group_iv_cost (data, group, cand);
6741 94187248 : if (!new_cp)
6742 37803608 : continue;
6743 :
6744 56383640 : if (!min_ncand)
6745 : {
6746 40898693 : int cmp_invs = iv_ca_compare_deps (data, ivs, group, old_cp, new_cp);
6747 : /* Skip if new_cp depends on more invariants. */
6748 40898693 : if (cmp_invs > 0)
6749 10001232 : continue;
6750 :
6751 30897461 : int cmp_cost = compare_cost_pair (new_cp, old_cp);
6752 : /* Skip if new_cp is not cheaper. */
6753 30897461 : if (cmp_cost > 0 || (cmp_cost == 0 && cmp_invs == 0))
6754 24112647 : continue;
6755 : }
6756 :
6757 22269761 : *delta = iv_ca_delta_add (group, old_cp, new_cp, *delta);
6758 : }
6759 :
6760 22176003 : iv_ca_delta_commit (data, ivs, *delta, true);
6761 22176003 : cost = iv_ca_cost (ivs);
6762 22176003 : if (n_ivs)
6763 13103817 : *n_ivs = iv_ca_n_cands (ivs);
6764 22176003 : iv_ca_delta_commit (data, ivs, *delta, false);
6765 :
6766 22176003 : return cost;
6767 : }
6768 :
6769 : /* Try narrowing set IVS by removing CAND. Return the cost of
6770 : the new set and store the differences in DELTA. START is
6771 : the candidate with which we start narrowing. */
6772 :
6773 : static comp_cost
6774 15868402 : iv_ca_narrow (struct ivopts_data *data, class iv_ca *ivs,
6775 : struct iv_cand *cand, struct iv_cand *start,
6776 : struct iv_ca_delta **delta)
6777 : {
6778 15868402 : unsigned i, ci;
6779 15868402 : struct iv_group *group;
6780 15868402 : class cost_pair *old_cp, *new_cp, *cp;
6781 15868402 : bitmap_iterator bi;
6782 15868402 : struct iv_cand *cnd;
6783 15868402 : comp_cost cost, best_cost, acost;
6784 :
6785 15868402 : *delta = NULL;
6786 86228320 : for (i = 0; i < data->vgroups.length (); i++)
6787 : {
6788 80842834 : group = data->vgroups[i];
6789 :
6790 80842834 : old_cp = iv_ca_cand_for_group (ivs, group);
6791 80842834 : if (old_cp->cand != cand)
6792 58227885 : continue;
6793 :
6794 22614949 : best_cost = iv_ca_cost (ivs);
6795 : /* Start narrowing with START. */
6796 22614949 : new_cp = get_group_iv_cost (data, group, start);
6797 :
6798 22614949 : if (data->consider_all_candidates)
6799 : {
6800 99631629 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, ci, bi)
6801 : {
6802 78174832 : if (ci == cand->id || (start && ci == start->id))
6803 37065086 : continue;
6804 :
6805 41109746 : cnd = data->vcands[ci];
6806 :
6807 41109746 : cp = get_group_iv_cost (data, group, cnd);
6808 41109746 : if (!cp)
6809 24304919 : continue;
6810 :
6811 16804827 : iv_ca_set_cp (data, ivs, group, cp);
6812 16804827 : acost = iv_ca_cost (ivs);
6813 :
6814 16804827 : if (acost < best_cost)
6815 : {
6816 1849245 : best_cost = acost;
6817 1849245 : new_cp = cp;
6818 : }
6819 : }
6820 : }
6821 : else
6822 : {
6823 4594672 : EXECUTE_IF_AND_IN_BITMAP (group->related_cands, ivs->cands, 0, ci, bi)
6824 : {
6825 3436520 : if (ci == cand->id || (start && ci == start->id))
6826 1913153 : continue;
6827 :
6828 1523367 : cnd = data->vcands[ci];
6829 :
6830 1523367 : cp = get_group_iv_cost (data, group, cnd);
6831 1523367 : if (!cp)
6832 0 : continue;
6833 :
6834 1523367 : iv_ca_set_cp (data, ivs, group, cp);
6835 1523367 : acost = iv_ca_cost (ivs);
6836 :
6837 1523367 : if (acost < best_cost)
6838 : {
6839 34245 : best_cost = acost;
6840 34245 : new_cp = cp;
6841 : }
6842 : }
6843 : }
6844 : /* Restore to old cp for use. */
6845 22614949 : iv_ca_set_cp (data, ivs, group, old_cp);
6846 :
6847 22614949 : if (!new_cp)
6848 : {
6849 10482916 : iv_ca_delta_free (delta);
6850 10482916 : return infinite_cost;
6851 : }
6852 :
6853 12132033 : *delta = iv_ca_delta_add (group, old_cp, new_cp, *delta);
6854 : }
6855 :
6856 5385486 : iv_ca_delta_commit (data, ivs, *delta, true);
6857 5385486 : cost = iv_ca_cost (ivs);
6858 5385486 : iv_ca_delta_commit (data, ivs, *delta, false);
6859 :
6860 5385486 : return cost;
6861 : }
6862 :
6863 : /* Try optimizing the set of candidates IVS by removing candidates different
6864 : from to EXCEPT_CAND from it. Return cost of the new set, and store
6865 : differences in DELTA. */
6866 :
6867 : static comp_cost
6868 9207700 : iv_ca_prune (struct ivopts_data *data, class iv_ca *ivs,
6869 : struct iv_cand *except_cand, struct iv_ca_delta **delta)
6870 : {
6871 9207700 : bitmap_iterator bi;
6872 9207700 : struct iv_ca_delta *act_delta, *best_delta;
6873 9207700 : unsigned i;
6874 9207700 : comp_cost best_cost, acost;
6875 9207700 : struct iv_cand *cand;
6876 :
6877 9207700 : best_delta = NULL;
6878 9207700 : best_cost = iv_ca_cost (ivs);
6879 :
6880 31539981 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
6881 : {
6882 22332281 : cand = data->vcands[i];
6883 :
6884 22332281 : if (cand == except_cand)
6885 6463879 : continue;
6886 :
6887 15868402 : acost = iv_ca_narrow (data, ivs, cand, except_cand, &act_delta);
6888 :
6889 15868402 : if (acost < best_cost)
6890 : {
6891 2359028 : best_cost = acost;
6892 2359028 : iv_ca_delta_free (&best_delta);
6893 2359028 : best_delta = act_delta;
6894 : }
6895 : else
6896 13509374 : iv_ca_delta_free (&act_delta);
6897 : }
6898 :
6899 9207700 : if (!best_delta)
6900 : {
6901 6991685 : *delta = NULL;
6902 6991685 : return best_cost;
6903 : }
6904 :
6905 : /* Recurse to possibly remove other unnecessary ivs. */
6906 2216015 : iv_ca_delta_commit (data, ivs, best_delta, true);
6907 2216015 : best_cost = iv_ca_prune (data, ivs, except_cand, delta);
6908 2216015 : iv_ca_delta_commit (data, ivs, best_delta, false);
6909 2216015 : *delta = iv_ca_delta_join (best_delta, *delta);
6910 2216015 : return best_cost;
6911 : }
6912 :
6913 : /* Check if CAND_IDX is a candidate other than OLD_CAND and has
6914 : cheaper local cost for GROUP than BEST_CP. Return pointer to
6915 : the corresponding cost_pair, otherwise just return BEST_CP. */
6916 :
6917 : static class cost_pair*
6918 29408006 : cheaper_cost_with_cand (struct ivopts_data *data, struct iv_group *group,
6919 : unsigned int cand_idx, struct iv_cand *old_cand,
6920 : class cost_pair *best_cp)
6921 : {
6922 29408006 : struct iv_cand *cand;
6923 29408006 : class cost_pair *cp;
6924 :
6925 29408006 : gcc_assert (old_cand != NULL && best_cp != NULL);
6926 29408006 : if (cand_idx == old_cand->id)
6927 : return best_cp;
6928 :
6929 26568845 : cand = data->vcands[cand_idx];
6930 26568845 : cp = get_group_iv_cost (data, group, cand);
6931 26568845 : if (cp != NULL && cheaper_cost_pair (cp, best_cp))
6932 1715776 : return cp;
6933 :
6934 : return best_cp;
6935 : }
6936 :
6937 : /* Try breaking local optimal fixed-point for IVS by replacing candidates
6938 : which are used by more than one iv uses. For each of those candidates,
6939 : this function tries to represent iv uses under that candidate using
6940 : other ones with lower local cost, then tries to prune the new set.
6941 : If the new set has lower cost, It returns the new cost after recording
6942 : candidate replacement in list DELTA. */
6943 :
6944 : static comp_cost
6945 1020304 : iv_ca_replace (struct ivopts_data *data, class iv_ca *ivs,
6946 : struct iv_ca_delta **delta)
6947 : {
6948 1020304 : bitmap_iterator bi, bj;
6949 1020304 : unsigned int i, j, k;
6950 1020304 : struct iv_cand *cand;
6951 1020304 : comp_cost orig_cost, acost;
6952 1020304 : struct iv_ca_delta *act_delta, *tmp_delta;
6953 1020304 : class cost_pair *old_cp, *best_cp = NULL;
6954 :
6955 1020304 : *delta = NULL;
6956 1020304 : orig_cost = iv_ca_cost (ivs);
6957 :
6958 2404317 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
6959 : {
6960 1416942 : if (ivs->n_cand_uses[i] == 1
6961 1047278 : || ivs->n_cand_uses[i] > ALWAYS_PRUNE_CAND_SET_BOUND)
6962 374777 : continue;
6963 :
6964 1042165 : cand = data->vcands[i];
6965 :
6966 1042165 : act_delta = NULL;
6967 : /* Represent uses under current candidate using other ones with
6968 : lower local cost. */
6969 5462843 : for (j = 0; j < ivs->upto; j++)
6970 : {
6971 4420678 : struct iv_group *group = data->vgroups[j];
6972 4420678 : old_cp = iv_ca_cand_for_group (ivs, group);
6973 :
6974 4420678 : if (old_cp->cand != cand)
6975 1581517 : continue;
6976 :
6977 2839161 : best_cp = old_cp;
6978 2839161 : if (data->consider_all_candidates)
6979 32130947 : for (k = 0; k < data->vcands.length (); k++)
6980 29299651 : best_cp = cheaper_cost_with_cand (data, group, k,
6981 : old_cp->cand, best_cp);
6982 : else
6983 116220 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, k, bj)
6984 108355 : best_cp = cheaper_cost_with_cand (data, group, k,
6985 : old_cp->cand, best_cp);
6986 :
6987 2839161 : if (best_cp == old_cp)
6988 1395600 : continue;
6989 :
6990 1443561 : act_delta = iv_ca_delta_add (group, old_cp, best_cp, act_delta);
6991 : }
6992 : /* No need for further prune. */
6993 1042165 : if (!act_delta)
6994 258418 : continue;
6995 :
6996 : /* Prune the new candidate set. */
6997 783747 : iv_ca_delta_commit (data, ivs, act_delta, true);
6998 783747 : acost = iv_ca_prune (data, ivs, NULL, &tmp_delta);
6999 783747 : iv_ca_delta_commit (data, ivs, act_delta, false);
7000 783747 : act_delta = iv_ca_delta_join (act_delta, tmp_delta);
7001 :
7002 783747 : if (acost < orig_cost)
7003 : {
7004 32929 : *delta = act_delta;
7005 32929 : return acost;
7006 : }
7007 : else
7008 750818 : iv_ca_delta_free (&act_delta);
7009 : }
7010 :
7011 987375 : return orig_cost;
7012 : }
7013 :
7014 : /* Tries to extend the sets IVS in the best possible way in order to
7015 : express the GROUP. If ORIGINALP is true, prefer candidates from
7016 : the original set of IVs, otherwise favor important candidates not
7017 : based on any memory object. */
7018 :
7019 : static bool
7020 3323194 : try_add_cand_for (struct ivopts_data *data, class iv_ca *ivs,
7021 : struct iv_group *group, bool originalp)
7022 : {
7023 3323194 : comp_cost best_cost, act_cost;
7024 3323194 : unsigned i;
7025 3323194 : bitmap_iterator bi;
7026 3323194 : struct iv_cand *cand;
7027 3323194 : struct iv_ca_delta *best_delta = NULL, *act_delta;
7028 3323194 : class cost_pair *cp;
7029 :
7030 3323194 : iv_ca_add_group (data, ivs, group);
7031 3323194 : best_cost = iv_ca_cost (ivs);
7032 3323194 : cp = iv_ca_cand_for_group (ivs, group);
7033 3323194 : if (cp)
7034 : {
7035 3230035 : best_delta = iv_ca_delta_add (group, NULL, cp, NULL);
7036 3230035 : iv_ca_set_no_cp (data, ivs, group);
7037 : }
7038 :
7039 : /* If ORIGINALP is true, try to find the original IV for the use. Otherwise
7040 : first try important candidates not based on any memory object. Only if
7041 : this fails, try the specific ones. Rationale -- in loops with many
7042 : variables the best choice often is to use just one generic biv. If we
7043 : added here many ivs specific to the uses, the optimization algorithm later
7044 : would be likely to get stuck in a local minimum, thus causing us to create
7045 : too many ivs. The approach from few ivs to more seems more likely to be
7046 : successful -- starting from few ivs, replacing an expensive use by a
7047 : specific iv should always be a win. */
7048 30919762 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, i, bi)
7049 : {
7050 27596568 : cand = data->vcands[i];
7051 :
7052 27596568 : if (originalp && cand->pos !=IP_ORIGINAL)
7053 10873131 : continue;
7054 :
7055 13798284 : if (!originalp && cand->iv->base_object != NULL_TREE)
7056 2566604 : continue;
7057 :
7058 14156833 : if (iv_ca_cand_used_p (ivs, cand))
7059 1528834 : continue;
7060 :
7061 12627999 : cp = get_group_iv_cost (data, group, cand);
7062 12627999 : if (!cp)
7063 3671455 : continue;
7064 :
7065 8956544 : iv_ca_set_cp (data, ivs, group, cp);
7066 8956544 : act_cost = iv_ca_extend (data, ivs, cand, &act_delta, NULL,
7067 : true);
7068 8956544 : iv_ca_set_no_cp (data, ivs, group);
7069 8956544 : act_delta = iv_ca_delta_add (group, NULL, cp, act_delta);
7070 :
7071 8956544 : if (act_cost < best_cost)
7072 : {
7073 360679 : best_cost = act_cost;
7074 :
7075 360679 : iv_ca_delta_free (&best_delta);
7076 360679 : best_delta = act_delta;
7077 : }
7078 : else
7079 8595865 : iv_ca_delta_free (&act_delta);
7080 : }
7081 :
7082 3323194 : if (best_cost.infinite_cost_p ())
7083 : {
7084 733945 : for (i = 0; i < group->n_map_members; i++)
7085 : {
7086 666171 : cp = group->cost_map + i;
7087 666171 : cand = cp->cand;
7088 666171 : if (!cand)
7089 550529 : continue;
7090 :
7091 : /* Already tried this. */
7092 115642 : if (cand->important)
7093 : {
7094 0 : if (originalp && cand->pos == IP_ORIGINAL)
7095 0 : continue;
7096 0 : if (!originalp && cand->iv->base_object == NULL_TREE)
7097 0 : continue;
7098 : }
7099 :
7100 115642 : if (iv_ca_cand_used_p (ivs, cand))
7101 0 : continue;
7102 :
7103 115642 : act_delta = NULL;
7104 115642 : iv_ca_set_cp (data, ivs, group, cp);
7105 115642 : act_cost = iv_ca_extend (data, ivs, cand, &act_delta, NULL, true);
7106 115642 : iv_ca_set_no_cp (data, ivs, group);
7107 115642 : act_delta = iv_ca_delta_add (group,
7108 : iv_ca_cand_for_group (ivs, group),
7109 : cp, act_delta);
7110 :
7111 115642 : if (act_cost < best_cost)
7112 : {
7113 69233 : best_cost = act_cost;
7114 :
7115 69233 : if (best_delta)
7116 2363 : iv_ca_delta_free (&best_delta);
7117 69233 : best_delta = act_delta;
7118 : }
7119 : else
7120 46409 : iv_ca_delta_free (&act_delta);
7121 : }
7122 : }
7123 :
7124 3323194 : iv_ca_delta_commit (data, ivs, best_delta, true);
7125 3323194 : iv_ca_delta_free (&best_delta);
7126 :
7127 3323194 : return !best_cost.infinite_cost_p ();
7128 : }
7129 :
7130 : /* Finds an initial assignment of candidates to uses. */
7131 :
7132 : static class iv_ca *
7133 1021208 : get_initial_solution (struct ivopts_data *data, bool originalp)
7134 : {
7135 1021208 : unsigned i;
7136 1021208 : class iv_ca *ivs = iv_ca_new (data);
7137 :
7138 4343498 : for (i = 0; i < data->vgroups.length (); i++)
7139 3323194 : if (!try_add_cand_for (data, ivs, data->vgroups[i], originalp))
7140 : {
7141 904 : iv_ca_free (&ivs);
7142 904 : return NULL;
7143 : }
7144 :
7145 : return ivs;
7146 : }
7147 :
7148 : /* Tries to improve set of induction variables IVS. TRY_REPLACE_P
7149 : points to a bool variable, this function tries to break local
7150 : optimal fixed-point by replacing candidates in IVS if it's true. */
7151 :
7152 : static bool
7153 1547040 : try_improve_iv_set (struct ivopts_data *data,
7154 : class iv_ca *ivs, bool *try_replace_p)
7155 : {
7156 1547040 : unsigned i, n_ivs;
7157 1547040 : comp_cost acost, best_cost = iv_ca_cost (ivs);
7158 1547040 : struct iv_ca_delta *best_delta = NULL, *act_delta, *tmp_delta;
7159 1547040 : struct iv_cand *cand;
7160 :
7161 : /* Try extending the set of induction variables by one. */
7162 17068929 : for (i = 0; i < data->vcands.length (); i++)
7163 : {
7164 15521889 : cand = data->vcands[i];
7165 :
7166 15521889 : if (iv_ca_cand_used_p (ivs, cand))
7167 2418072 : continue;
7168 :
7169 13103817 : acost = iv_ca_extend (data, ivs, cand, &act_delta, &n_ivs, false);
7170 13103817 : if (!act_delta)
7171 7883807 : continue;
7172 :
7173 : /* If we successfully added the candidate and the set is small enough,
7174 : try optimizing it by removing other candidates. */
7175 5220010 : if (n_ivs <= ALWAYS_PRUNE_CAND_SET_BOUND)
7176 : {
7177 5152755 : iv_ca_delta_commit (data, ivs, act_delta, true);
7178 5152755 : acost = iv_ca_prune (data, ivs, cand, &tmp_delta);
7179 5152755 : iv_ca_delta_commit (data, ivs, act_delta, false);
7180 5152755 : act_delta = iv_ca_delta_join (act_delta, tmp_delta);
7181 : }
7182 :
7183 5220010 : if (acost < best_cost)
7184 : {
7185 634953 : best_cost = acost;
7186 634953 : iv_ca_delta_free (&best_delta);
7187 634953 : best_delta = act_delta;
7188 : }
7189 : else
7190 4585057 : iv_ca_delta_free (&act_delta);
7191 : }
7192 :
7193 1547040 : if (!best_delta)
7194 : {
7195 : /* Try removing the candidates from the set instead. */
7196 1055183 : best_cost = iv_ca_prune (data, ivs, NULL, &best_delta);
7197 :
7198 1055183 : if (!best_delta && *try_replace_p)
7199 : {
7200 1020304 : *try_replace_p = false;
7201 : /* So far candidate selecting algorithm tends to choose fewer IVs
7202 : so that it can handle cases in which loops have many variables
7203 : but the best choice is often to use only one general biv. One
7204 : weakness is it can't handle opposite cases, in which different
7205 : candidates should be chosen with respect to each use. To solve
7206 : the problem, we replace candidates in a manner described by the
7207 : comments of iv_ca_replace, thus give general algorithm a chance
7208 : to break local optimal fixed-point in these cases. */
7209 1020304 : best_cost = iv_ca_replace (data, ivs, &best_delta);
7210 : }
7211 :
7212 1055183 : if (!best_delta)
7213 : return false;
7214 : }
7215 :
7216 526736 : iv_ca_delta_commit (data, ivs, best_delta, true);
7217 526736 : iv_ca_delta_free (&best_delta);
7218 1053472 : return best_cost == iv_ca_cost (ivs);
7219 : }
7220 :
7221 : /* Attempts to find the optimal set of induction variables. We do simple
7222 : greedy heuristic -- we try to replace at most one candidate in the selected
7223 : solution and remove the unused ivs while this improves the cost. */
7224 :
7225 : static class iv_ca *
7226 1021208 : find_optimal_iv_set_1 (struct ivopts_data *data, bool originalp)
7227 : {
7228 1021208 : class iv_ca *set;
7229 1021208 : bool try_replace_p = true;
7230 :
7231 : /* Get the initial solution. */
7232 1021208 : set = get_initial_solution (data, originalp);
7233 1021208 : if (!set)
7234 : {
7235 904 : if (dump_file && (dump_flags & TDF_DETAILS))
7236 0 : fprintf (dump_file, "Unable to substitute for ivs, failed.\n");
7237 : return NULL;
7238 : }
7239 :
7240 1020304 : if (dump_file && (dump_flags & TDF_DETAILS))
7241 : {
7242 140 : fprintf (dump_file, "Initial set of candidates:\n");
7243 140 : iv_ca_dump (data, dump_file, set);
7244 : }
7245 :
7246 1547040 : while (try_improve_iv_set (data, set, &try_replace_p))
7247 : {
7248 526736 : if (dump_file && (dump_flags & TDF_DETAILS))
7249 : {
7250 121 : fprintf (dump_file, "Improved to:\n");
7251 121 : iv_ca_dump (data, dump_file, set);
7252 : }
7253 : }
7254 :
7255 : /* If the set has infinite_cost, it can't be optimal. */
7256 2040608 : if (iv_ca_cost (set).infinite_cost_p ())
7257 : {
7258 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7259 0 : fprintf (dump_file,
7260 : "Overflow to infinite cost in try_improve_iv_set.\n");
7261 0 : iv_ca_free (&set);
7262 : }
7263 1020304 : return set;
7264 : }
7265 :
7266 : static class iv_ca *
7267 510604 : find_optimal_iv_set (struct ivopts_data *data)
7268 : {
7269 510604 : unsigned i;
7270 510604 : comp_cost cost, origcost;
7271 510604 : class iv_ca *set, *origset;
7272 :
7273 : /* Determine the cost based on a strategy that starts with original IVs,
7274 : and try again using a strategy that prefers candidates not based
7275 : on any IVs. */
7276 510604 : origset = find_optimal_iv_set_1 (data, true);
7277 510604 : set = find_optimal_iv_set_1 (data, false);
7278 :
7279 510604 : if (!origset && !set)
7280 : return NULL;
7281 :
7282 510152 : origcost = origset ? iv_ca_cost (origset) : infinite_cost;
7283 510152 : cost = set ? iv_ca_cost (set) : infinite_cost;
7284 :
7285 510152 : if (dump_file && (dump_flags & TDF_DETAILS))
7286 : {
7287 70 : fprintf (dump_file, "Original cost %" PRId64 " (complexity %d)\n\n",
7288 : origcost.cost, origcost.complexity);
7289 70 : fprintf (dump_file, "Final cost %" PRId64 " (complexity %d)\n\n",
7290 : cost.cost, cost.complexity);
7291 : }
7292 :
7293 : /* Choose the one with the best cost. */
7294 510152 : if (origcost <= cost)
7295 : {
7296 476659 : if (set)
7297 476659 : iv_ca_free (&set);
7298 476659 : set = origset;
7299 : }
7300 33493 : else if (origset)
7301 33493 : iv_ca_free (&origset);
7302 :
7303 2170638 : for (i = 0; i < data->vgroups.length (); i++)
7304 : {
7305 1660486 : struct iv_group *group = data->vgroups[i];
7306 1660486 : group->selected = iv_ca_cand_for_group (set, group)->cand;
7307 : }
7308 :
7309 510152 : return set;
7310 : }
7311 :
7312 : /* Creates a new induction variable corresponding to CAND. */
7313 :
7314 : static void
7315 708680 : create_new_iv (struct ivopts_data *data, struct iv_cand *cand)
7316 : {
7317 708680 : gimple_stmt_iterator incr_pos;
7318 708680 : tree base;
7319 708680 : struct iv_use *use;
7320 708680 : struct iv_group *group;
7321 708680 : bool after = false;
7322 :
7323 708680 : gcc_assert (cand->iv != NULL);
7324 :
7325 708680 : switch (cand->pos)
7326 : {
7327 493272 : case IP_NORMAL:
7328 493272 : incr_pos = gsi_last_bb (ip_normal_pos (data->current_loop));
7329 493272 : break;
7330 :
7331 11436 : case IP_END:
7332 11436 : incr_pos = gsi_last_bb (ip_end_pos (data->current_loop));
7333 11436 : after = true;
7334 11436 : gcc_assert (gsi_end_p (incr_pos) || !stmt_ends_bb_p (*incr_pos));
7335 : break;
7336 :
7337 0 : case IP_AFTER_USE:
7338 0 : after = true;
7339 : /* fall through */
7340 0 : case IP_BEFORE_USE:
7341 0 : incr_pos = gsi_for_stmt (cand->incremented_at);
7342 0 : break;
7343 :
7344 203972 : case IP_ORIGINAL:
7345 : /* Mark that the iv is preserved. */
7346 203972 : name_info (data, cand->var_before)->preserve_biv = true;
7347 203972 : name_info (data, cand->var_after)->preserve_biv = true;
7348 :
7349 : /* Rewrite the increment so that it uses var_before directly. Missed
7350 : optimization can result in a use IV with zero step, avoid
7351 : crashing in that case. */
7352 203972 : use = find_interesting_uses_op (data, cand->var_after);
7353 203972 : if (use)
7354 : {
7355 203967 : group = data->vgroups[use->group_id];
7356 203967 : group->selected = cand;
7357 : }
7358 203972 : return;
7359 : }
7360 :
7361 504708 : gimple_add_tmp_var (cand->var_before);
7362 :
7363 504708 : base = unshare_expr (cand->iv->base);
7364 :
7365 : /* The step computation could invoke UB when the loop does not iterate.
7366 : Avoid inserting it on the preheader in its native form but rewrite
7367 : it to a well-defined form. This also helps masking SCEV issues
7368 : which freely re-associates the IV computations when building up
7369 : CHRECs without much regard for signed overflow invoking UB. */
7370 504708 : gimple_seq stmts = NULL;
7371 504708 : tree step = force_gimple_operand (unshare_expr (cand->iv->step), &stmts,
7372 : true, NULL_TREE);
7373 504708 : if (stmts)
7374 : {
7375 159176 : for (auto gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
7376 104894 : if (gimple_needing_rewrite_undefined (gsi_stmt (gsi)))
7377 12006 : rewrite_to_defined_unconditional (&gsi);
7378 54282 : gsi_insert_seq_on_edge_immediate
7379 54282 : (loop_preheader_edge (data->current_loop), stmts);
7380 : }
7381 :
7382 504708 : create_iv (base, PLUS_EXPR, step,
7383 : cand->var_before, data->current_loop,
7384 : &incr_pos, after, &cand->var_before, &cand->var_after);
7385 : }
7386 :
7387 : /* Creates new induction variables described in SET. */
7388 :
7389 : static void
7390 510152 : create_new_ivs (struct ivopts_data *data, class iv_ca *set)
7391 : {
7392 510152 : unsigned i;
7393 510152 : struct iv_cand *cand;
7394 510152 : bitmap_iterator bi;
7395 :
7396 1218832 : EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
7397 : {
7398 708680 : cand = data->vcands[i];
7399 708680 : create_new_iv (data, cand);
7400 : }
7401 :
7402 510152 : if (dump_file && (dump_flags & TDF_DETAILS))
7403 : {
7404 70 : fprintf (dump_file, "Selected IV set for loop %d",
7405 70 : data->current_loop->num);
7406 70 : if (data->loop_loc != UNKNOWN_LOCATION)
7407 68 : fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
7408 136 : LOCATION_LINE (data->loop_loc));
7409 70 : fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_UNSIGNED " avg niters",
7410 : avg_loop_niter (data->current_loop));
7411 70 : fprintf (dump_file, ", %lu IVs:\n", bitmap_count_bits (set->cands));
7412 185 : EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
7413 : {
7414 115 : cand = data->vcands[i];
7415 115 : dump_cand (dump_file, cand);
7416 : }
7417 70 : fprintf (dump_file, "\n");
7418 : }
7419 510152 : }
7420 :
7421 : /* Rewrites USE (definition of iv used in a nonlinear expression)
7422 : using candidate CAND. */
7423 :
7424 : static void
7425 627140 : rewrite_use_nonlinear_expr (struct ivopts_data *data,
7426 : struct iv_use *use, struct iv_cand *cand)
7427 : {
7428 627140 : gassign *ass;
7429 627140 : gimple_stmt_iterator bsi;
7430 627140 : tree comp, type = get_use_type (use), tgt;
7431 :
7432 : /* An important special case -- if we are asked to express value of
7433 : the original iv by itself, just exit; there is no need to
7434 : introduce a new computation (that might also need casting the
7435 : variable to unsigned and back). */
7436 627140 : if (cand->pos == IP_ORIGINAL
7437 334357 : && cand->incremented_at == use->stmt)
7438 : {
7439 203967 : tree op = NULL_TREE;
7440 203967 : enum tree_code stmt_code;
7441 :
7442 203967 : gcc_assert (is_gimple_assign (use->stmt));
7443 203967 : gcc_assert (gimple_assign_lhs (use->stmt) == cand->var_after);
7444 :
7445 : /* Check whether we may leave the computation unchanged.
7446 : This is the case only if it does not rely on other
7447 : computations in the loop -- otherwise, the computation
7448 : we rely upon may be removed in remove_unused_ivs,
7449 : thus leading to ICE. */
7450 203967 : stmt_code = gimple_assign_rhs_code (use->stmt);
7451 203967 : if (stmt_code == PLUS_EXPR
7452 203967 : || stmt_code == MINUS_EXPR
7453 203967 : || stmt_code == POINTER_PLUS_EXPR)
7454 : {
7455 201405 : if (gimple_assign_rhs1 (use->stmt) == cand->var_before)
7456 200606 : op = gimple_assign_rhs2 (use->stmt);
7457 799 : else if (gimple_assign_rhs2 (use->stmt) == cand->var_before)
7458 : op = gimple_assign_rhs1 (use->stmt);
7459 : }
7460 :
7461 201107 : if (op != NULL_TREE)
7462 : {
7463 201107 : if (expr_invariant_in_loop_p (data->current_loop, op))
7464 288142 : return;
7465 165 : if (TREE_CODE (op) == SSA_NAME)
7466 : {
7467 165 : struct iv *iv = get_iv (data, op);
7468 165 : if (iv != NULL && integer_zerop (iv->step))
7469 : return;
7470 : }
7471 : }
7472 : }
7473 :
7474 426033 : switch (gimple_code (use->stmt))
7475 : {
7476 128274 : case GIMPLE_PHI:
7477 128274 : tgt = PHI_RESULT (use->stmt);
7478 :
7479 : /* If we should keep the biv, do not replace it. */
7480 128274 : if (name_info (data, tgt)->preserve_biv)
7481 : return;
7482 :
7483 41239 : bsi = gsi_after_labels (gimple_bb (use->stmt));
7484 41239 : break;
7485 :
7486 297759 : case GIMPLE_ASSIGN:
7487 297759 : tgt = gimple_assign_lhs (use->stmt);
7488 297759 : bsi = gsi_for_stmt (use->stmt);
7489 297759 : break;
7490 :
7491 0 : default:
7492 0 : gcc_unreachable ();
7493 : }
7494 :
7495 1016994 : aff_tree aff_inv, aff_var;
7496 338998 : if (!get_computation_aff_1 (data, use->stmt, use, cand, &aff_inv, &aff_var))
7497 0 : gcc_unreachable ();
7498 :
7499 338998 : unshare_aff_combination (&aff_inv);
7500 338998 : unshare_aff_combination (&aff_var);
7501 : /* Prefer CSE opportunity than loop invariant by adding offset at last
7502 : so that iv_uses have different offsets can be CSEed. */
7503 677996 : poly_widest_int offset = aff_inv.offset;
7504 338998 : aff_inv.offset = 0;
7505 :
7506 338998 : gimple_seq stmt_list = NULL, seq = NULL;
7507 338998 : tree comp_op1 = aff_combination_to_tree (&aff_inv);
7508 338998 : tree comp_op2 = aff_combination_to_tree (&aff_var);
7509 338998 : gcc_assert (comp_op1 && comp_op2);
7510 :
7511 338998 : comp_op1 = force_gimple_operand (comp_op1, &seq, true, NULL);
7512 338998 : gimple_seq_add_seq (&stmt_list, seq);
7513 338998 : comp_op2 = force_gimple_operand (comp_op2, &seq, true, NULL);
7514 338998 : gimple_seq_add_seq (&stmt_list, seq);
7515 :
7516 338998 : if (POINTER_TYPE_P (TREE_TYPE (comp_op2)))
7517 : std::swap (comp_op1, comp_op2);
7518 :
7519 338998 : if (POINTER_TYPE_P (TREE_TYPE (comp_op1)))
7520 : {
7521 0 : comp = fold_build_pointer_plus (comp_op1,
7522 : fold_convert (sizetype, comp_op2));
7523 0 : comp = fold_build_pointer_plus (comp,
7524 : wide_int_to_tree (sizetype, offset));
7525 : }
7526 : else
7527 : {
7528 338998 : comp = fold_build2 (PLUS_EXPR, TREE_TYPE (comp_op1), comp_op1,
7529 : fold_convert (TREE_TYPE (comp_op1), comp_op2));
7530 338998 : comp = fold_build2 (PLUS_EXPR, TREE_TYPE (comp_op1), comp,
7531 : wide_int_to_tree (TREE_TYPE (comp_op1), offset));
7532 : }
7533 :
7534 338998 : comp = fold_convert (type, comp);
7535 338998 : comp = force_gimple_operand (comp, &seq, false, NULL);
7536 338998 : gimple_seq_add_seq (&stmt_list, seq);
7537 338998 : if (gimple_code (use->stmt) != GIMPLE_PHI
7538 : /* We can't allow re-allocating the stmt as it might be pointed
7539 : to still. */
7540 338998 : && (get_gimple_rhs_num_ops (TREE_CODE (comp))
7541 297759 : >= gimple_num_ops (gsi_stmt (bsi))))
7542 : {
7543 5757 : comp = force_gimple_operand (comp, &seq, true, NULL);
7544 5757 : gimple_seq_add_seq (&stmt_list, seq);
7545 5757 : if (POINTER_TYPE_P (TREE_TYPE (tgt)))
7546 : {
7547 0 : duplicate_ssa_name_ptr_info (comp, SSA_NAME_PTR_INFO (tgt));
7548 : /* As this isn't a plain copy we have to reset alignment
7549 : information. */
7550 0 : if (SSA_NAME_PTR_INFO (comp))
7551 0 : mark_ptr_info_alignment_unknown (SSA_NAME_PTR_INFO (comp));
7552 : }
7553 : }
7554 :
7555 338998 : gsi_insert_seq_before (&bsi, stmt_list, GSI_SAME_STMT);
7556 338998 : if (gimple_code (use->stmt) == GIMPLE_PHI)
7557 : {
7558 41239 : ass = gimple_build_assign (tgt, comp);
7559 41239 : gsi_insert_before (&bsi, ass, GSI_SAME_STMT);
7560 :
7561 41239 : bsi = gsi_for_stmt (use->stmt);
7562 41239 : remove_phi_node (&bsi, false);
7563 : }
7564 : else
7565 : {
7566 297759 : gimple_assign_set_rhs_from_tree (&bsi, comp);
7567 297759 : use->stmt = gsi_stmt (bsi);
7568 : }
7569 : }
7570 :
7571 : /* Performs a peephole optimization to reorder the iv update statement with
7572 : a mem ref to enable instruction combining in later phases. The mem ref uses
7573 : the iv value before the update, so the reordering transformation requires
7574 : adjustment of the offset. CAND is the selected IV_CAND.
7575 :
7576 : Example:
7577 :
7578 : t = MEM_REF (base, iv1, 8, 16); // base, index, stride, offset
7579 : iv2 = iv1 + 1;
7580 :
7581 : if (t < val) (1)
7582 : goto L;
7583 : goto Head;
7584 :
7585 :
7586 : directly propagating t over to (1) will introduce overlapping live range
7587 : thus increase register pressure. This peephole transform it into:
7588 :
7589 :
7590 : iv2 = iv1 + 1;
7591 : t = MEM_REF (base, iv2, 8, 8);
7592 : if (t < val)
7593 : goto L;
7594 : goto Head;
7595 : */
7596 :
7597 : static void
7598 870080 : adjust_iv_update_pos (struct iv_cand *cand, struct iv_use *use)
7599 : {
7600 870080 : tree var_after;
7601 870080 : gimple *iv_update, *stmt;
7602 870080 : basic_block bb;
7603 870080 : gimple_stmt_iterator gsi, gsi_iv;
7604 :
7605 870080 : if (cand->pos != IP_NORMAL)
7606 868008 : return;
7607 :
7608 665398 : var_after = cand->var_after;
7609 665398 : iv_update = SSA_NAME_DEF_STMT (var_after);
7610 :
7611 665398 : bb = gimple_bb (iv_update);
7612 665398 : gsi = gsi_last_nondebug_bb (bb);
7613 665398 : stmt = gsi_stmt (gsi);
7614 :
7615 : /* Only handle conditional statement for now. */
7616 665398 : if (gimple_code (stmt) != GIMPLE_COND)
7617 : return;
7618 :
7619 665398 : gsi_prev_nondebug (&gsi);
7620 665398 : stmt = gsi_stmt (gsi);
7621 665398 : if (stmt != iv_update)
7622 : return;
7623 :
7624 529525 : gsi_prev_nondebug (&gsi);
7625 529525 : if (gsi_end_p (gsi))
7626 : return;
7627 :
7628 526755 : stmt = gsi_stmt (gsi);
7629 526755 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
7630 : return;
7631 :
7632 526619 : if (stmt != use->stmt)
7633 : return;
7634 :
7635 4129 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
7636 : return;
7637 :
7638 2072 : if (dump_file && (dump_flags & TDF_DETAILS))
7639 : {
7640 0 : fprintf (dump_file, "Reordering \n");
7641 0 : print_gimple_stmt (dump_file, iv_update, 0);
7642 0 : print_gimple_stmt (dump_file, use->stmt, 0);
7643 0 : fprintf (dump_file, "\n");
7644 : }
7645 :
7646 2072 : gsi = gsi_for_stmt (use->stmt);
7647 2072 : gsi_iv = gsi_for_stmt (iv_update);
7648 2072 : gsi_move_before (&gsi_iv, &gsi);
7649 :
7650 2072 : cand->pos = IP_BEFORE_USE;
7651 2072 : cand->incremented_at = use->stmt;
7652 : }
7653 :
7654 : /* Return the alias pointer type that should be used for a MEM_REF
7655 : associated with USE, which has type USE_PTR_ADDRESS. */
7656 :
7657 : static tree
7658 637 : get_alias_ptr_type_for_ptr_address (iv_use *use)
7659 : {
7660 637 : gcall *call = as_a <gcall *> (use->stmt);
7661 637 : switch (gimple_call_internal_fn (call))
7662 : {
7663 637 : case IFN_MASK_LOAD:
7664 637 : case IFN_MASK_STORE:
7665 637 : case IFN_MASK_LOAD_LANES:
7666 637 : case IFN_MASK_STORE_LANES:
7667 637 : case IFN_MASK_LEN_LOAD_LANES:
7668 637 : case IFN_MASK_LEN_STORE_LANES:
7669 637 : case IFN_LEN_LOAD:
7670 637 : case IFN_LEN_STORE:
7671 637 : case IFN_MASK_LEN_LOAD:
7672 637 : case IFN_MASK_LEN_STORE:
7673 : /* The second argument contains the correct alias type. */
7674 637 : gcc_assert (use->op_p == gimple_call_arg_ptr (call, 0));
7675 637 : return TREE_TYPE (gimple_call_arg (call, 1));
7676 :
7677 0 : default:
7678 0 : gcc_unreachable ();
7679 : }
7680 : }
7681 :
7682 :
7683 : /* Rewrites USE (address that is an iv) using candidate CAND. */
7684 :
7685 : static void
7686 870080 : rewrite_use_address (struct ivopts_data *data,
7687 : struct iv_use *use, struct iv_cand *cand)
7688 : {
7689 870080 : aff_tree aff;
7690 870080 : bool ok;
7691 :
7692 870080 : adjust_iv_update_pos (cand, use);
7693 870080 : ok = get_computation_aff (data, use->stmt, use, cand, &aff);
7694 870080 : gcc_assert (ok);
7695 870080 : unshare_aff_combination (&aff);
7696 :
7697 : /* To avoid undefined overflow problems, all IV candidates use unsigned
7698 : integer types. The drawback is that this makes it impossible for
7699 : create_mem_ref to distinguish an IV that is based on a memory object
7700 : from one that represents simply an offset.
7701 :
7702 : To work around this problem, we pass a hint to create_mem_ref that
7703 : indicates which variable (if any) in aff is an IV based on a memory
7704 : object. Note that we only consider the candidate. If this is not
7705 : based on an object, the base of the reference is in some subexpression
7706 : of the use -- but these will use pointer types, so they are recognized
7707 : by the create_mem_ref heuristics anyway. */
7708 870080 : tree iv = var_at_stmt (data->current_loop, cand, use->stmt);
7709 870080 : tree base_hint = (cand->iv->base_object) ? iv : NULL_TREE;
7710 870080 : gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
7711 870080 : tree type = use->mem_type;
7712 870080 : tree alias_ptr_type;
7713 870080 : if (use->type == USE_PTR_ADDRESS)
7714 637 : alias_ptr_type = get_alias_ptr_type_for_ptr_address (use);
7715 : else
7716 : {
7717 869443 : gcc_assert (type == TREE_TYPE (*use->op_p));
7718 869443 : unsigned int align = get_object_alignment (*use->op_p);
7719 869443 : if (align != TYPE_ALIGN (type))
7720 34434 : type = build_aligned_type (type, align);
7721 869443 : alias_ptr_type = reference_alias_ptr_type (*use->op_p);
7722 : }
7723 870080 : tree ref = create_mem_ref (&bsi, type, &aff, alias_ptr_type,
7724 : iv, base_hint, data->speed);
7725 :
7726 870080 : if (use->type == USE_PTR_ADDRESS)
7727 : {
7728 637 : ref = fold_build1 (ADDR_EXPR, build_pointer_type (use->mem_type), ref);
7729 637 : ref = fold_convert (get_use_type (use), ref);
7730 637 : ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
7731 : true, GSI_SAME_STMT);
7732 : }
7733 : else
7734 : {
7735 : /* When we end up confused enough and have no suitable base but
7736 : stuffed everything to indexes use a LEA for the address and
7737 : create a plain MEM_REF to avoid basing a memory reference
7738 : on address zero which create_mem_ref_raw does as fallback. */
7739 869443 : if (TREE_CODE (ref) == TARGET_MEM_REF
7740 869443 : && integer_zerop (TREE_OPERAND (ref, 0)))
7741 : {
7742 24 : ref = fold_build1 (ADDR_EXPR, TREE_TYPE (TREE_OPERAND (ref, 0)), ref);
7743 24 : ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
7744 : true, GSI_SAME_STMT);
7745 24 : ref = build2 (MEM_REF, type, ref, build_zero_cst (alias_ptr_type));
7746 : }
7747 869443 : copy_ref_info (ref, *use->op_p);
7748 : }
7749 :
7750 870080 : *use->op_p = ref;
7751 870080 : }
7752 :
7753 : /* Rewrites USE (the condition such that one of the arguments is an iv) using
7754 : candidate CAND. */
7755 :
7756 : static void
7757 607325 : rewrite_use_compare (struct ivopts_data *data,
7758 : struct iv_use *use, struct iv_cand *cand)
7759 : {
7760 607325 : tree comp, op, bound;
7761 607325 : gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
7762 607325 : enum tree_code compare;
7763 607325 : struct iv_group *group = data->vgroups[use->group_id];
7764 607325 : class cost_pair *cp = get_group_iv_cost (data, group, cand);
7765 :
7766 607325 : bound = cp->value;
7767 607325 : if (bound)
7768 : {
7769 405782 : tree var = var_at_stmt (data->current_loop, cand, use->stmt);
7770 405782 : tree var_type = TREE_TYPE (var);
7771 405782 : gimple_seq stmts;
7772 :
7773 405782 : if (dump_file && (dump_flags & TDF_DETAILS))
7774 : {
7775 59 : fprintf (dump_file, "Replacing exit test: ");
7776 59 : print_gimple_stmt (dump_file, use->stmt, 0, TDF_SLIM);
7777 : }
7778 405782 : compare = cp->comp;
7779 405782 : bound = unshare_expr (fold_convert (var_type, bound));
7780 405782 : op = force_gimple_operand (bound, &stmts, true, NULL_TREE);
7781 405782 : if (stmts)
7782 191233 : gsi_insert_seq_on_edge_immediate (
7783 191233 : loop_preheader_edge (data->current_loop),
7784 : stmts);
7785 :
7786 405782 : gcond *cond_stmt = as_a <gcond *> (use->stmt);
7787 405782 : gimple_cond_set_lhs (cond_stmt, var);
7788 405782 : gimple_cond_set_code (cond_stmt, compare);
7789 405782 : gimple_cond_set_rhs (cond_stmt, op);
7790 405782 : return;
7791 : }
7792 :
7793 : /* The induction variable elimination failed; just express the original
7794 : giv. */
7795 201543 : comp = get_computation_at (data, use->stmt, use, cand);
7796 201543 : gcc_assert (comp != NULL_TREE);
7797 201543 : gcc_assert (use->op_p != NULL);
7798 201543 : *use->op_p = force_gimple_operand_gsi (&bsi, comp, true,
7799 201543 : SSA_NAME_VAR (*use->op_p),
7800 : true, GSI_SAME_STMT);
7801 : }
7802 :
7803 : /* Rewrite the groups using the selected induction variables. */
7804 :
7805 : static void
7806 510152 : rewrite_groups (struct ivopts_data *data)
7807 : {
7808 510152 : unsigned i, j;
7809 :
7810 2331593 : for (i = 0; i < data->vgroups.length (); i++)
7811 : {
7812 1821441 : struct iv_group *group = data->vgroups[i];
7813 1821441 : struct iv_cand *cand = group->selected;
7814 :
7815 1821441 : gcc_assert (cand);
7816 :
7817 1821441 : if (group->type == USE_NONLINEAR_EXPR)
7818 : {
7819 1254280 : for (j = 0; j < group->vuses.length (); j++)
7820 : {
7821 627140 : rewrite_use_nonlinear_expr (data, group->vuses[j], cand);
7822 627140 : update_stmt (group->vuses[j]->stmt);
7823 : }
7824 : }
7825 1194301 : else if (address_p (group->type))
7826 : {
7827 1457056 : for (j = 0; j < group->vuses.length (); j++)
7828 : {
7829 870080 : rewrite_use_address (data, group->vuses[j], cand);
7830 870080 : update_stmt (group->vuses[j]->stmt);
7831 : }
7832 : }
7833 : else
7834 : {
7835 607325 : gcc_assert (group->type == USE_COMPARE);
7836 :
7837 2428766 : for (j = 0; j < group->vuses.length (); j++)
7838 : {
7839 607325 : rewrite_use_compare (data, group->vuses[j], cand);
7840 607325 : update_stmt (group->vuses[j]->stmt);
7841 : }
7842 : }
7843 : }
7844 510152 : }
7845 :
7846 : /* Removes the ivs that are not used after rewriting. */
7847 :
7848 : static void
7849 510152 : remove_unused_ivs (struct ivopts_data *data, bitmap toremove)
7850 : {
7851 510152 : unsigned j;
7852 510152 : bitmap_iterator bi;
7853 :
7854 : /* Figure out an order in which to release SSA DEFs so that we don't
7855 : release something that we'd have to propagate into a debug stmt
7856 : afterwards. */
7857 5634873 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
7858 : {
7859 5124721 : struct version_info *info;
7860 :
7861 5124721 : info = ver_info (data, j);
7862 5124721 : if (info->iv
7863 4981345 : && !integer_zerop (info->iv->step)
7864 3252720 : && !info->inv_id
7865 3252720 : && !info->iv->nonlin_use
7866 7750301 : && !info->preserve_biv)
7867 : {
7868 2508643 : bitmap_set_bit (toremove, SSA_NAME_VERSION (info->iv->ssa_name));
7869 :
7870 2508643 : tree def = info->iv->ssa_name;
7871 :
7872 3269241 : if (MAY_HAVE_DEBUG_BIND_STMTS && SSA_NAME_DEF_STMT (def))
7873 : {
7874 760598 : imm_use_iterator imm_iter;
7875 760598 : use_operand_p use_p;
7876 760598 : gimple *stmt;
7877 760598 : int count = 0;
7878 :
7879 1498249 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, def)
7880 : {
7881 767073 : if (!gimple_debug_bind_p (stmt))
7882 644185 : continue;
7883 :
7884 : /* We just want to determine whether to do nothing
7885 : (count == 0), to substitute the computed
7886 : expression into a single use of the SSA DEF by
7887 : itself (count == 1), or to use a debug temp
7888 : because the SSA DEF is used multiple times or as
7889 : part of a larger expression (count > 1). */
7890 122888 : count++;
7891 122888 : if (gimple_debug_bind_get_value (stmt) != def)
7892 8093 : count++;
7893 :
7894 122888 : if (count > 1)
7895 : break;
7896 760598 : }
7897 :
7898 760598 : if (!count)
7899 685815 : continue;
7900 :
7901 101330 : struct iv_use dummy_use;
7902 101330 : struct iv_cand *best_cand = NULL, *cand;
7903 101330 : unsigned i, best_pref = 0, cand_pref;
7904 101330 : tree comp = NULL_TREE;
7905 :
7906 101330 : memset (&dummy_use, 0, sizeof (dummy_use));
7907 101330 : dummy_use.iv = info->iv;
7908 512434 : for (i = 0; i < data->vgroups.length () && i < 64; i++)
7909 : {
7910 411104 : cand = data->vgroups[i]->selected;
7911 411104 : if (cand == best_cand)
7912 156549 : continue;
7913 176039 : cand_pref = operand_equal_p (cand->iv->step,
7914 254555 : info->iv->step, 0)
7915 254555 : ? 4 : 0;
7916 254555 : cand_pref
7917 254555 : += TYPE_MODE (TREE_TYPE (cand->iv->base))
7918 254555 : == TYPE_MODE (TREE_TYPE (info->iv->base))
7919 254555 : ? 2 : 0;
7920 254555 : cand_pref
7921 509110 : += TREE_CODE (cand->iv->base) == INTEGER_CST
7922 254555 : ? 1 : 0;
7923 254555 : if (best_cand == NULL || best_pref < cand_pref)
7924 : {
7925 201943 : tree this_comp
7926 403886 : = get_debug_computation_at (data,
7927 201943 : SSA_NAME_DEF_STMT (def),
7928 : &dummy_use, cand);
7929 201943 : if (this_comp)
7930 : {
7931 411104 : best_cand = cand;
7932 411104 : best_pref = cand_pref;
7933 411104 : comp = this_comp;
7934 : }
7935 : }
7936 : }
7937 :
7938 101330 : if (!best_cand)
7939 26547 : continue;
7940 :
7941 74783 : comp = unshare_expr (comp);
7942 74783 : if (count > 1)
7943 : {
7944 24472 : tree vexpr = build_debug_expr_decl (TREE_TYPE (comp));
7945 : /* FIXME: Is setting the mode really necessary? */
7946 24472 : if (SSA_NAME_VAR (def))
7947 13556 : SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (def)));
7948 : else
7949 10916 : SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (vexpr)));
7950 24472 : gdebug *def_temp
7951 24472 : = gimple_build_debug_bind (vexpr, comp, NULL);
7952 24472 : gimple_stmt_iterator gsi;
7953 :
7954 24472 : if (gimple_code (SSA_NAME_DEF_STMT (def)) == GIMPLE_PHI)
7955 14100 : gsi = gsi_after_labels (gimple_bb
7956 14100 : (SSA_NAME_DEF_STMT (def)));
7957 : else
7958 10372 : gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (def));
7959 :
7960 24472 : gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
7961 24472 : comp = vexpr;
7962 : }
7963 :
7964 286037 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, def)
7965 : {
7966 211254 : if (!gimple_debug_bind_p (stmt))
7967 84419 : continue;
7968 :
7969 253722 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
7970 126861 : SET_USE (use_p, comp);
7971 :
7972 126835 : update_stmt (stmt);
7973 74783 : }
7974 : }
7975 : }
7976 : }
7977 510152 : }
7978 :
7979 : /* Frees memory occupied by class tree_niter_desc in *VALUE. Callback
7980 : for hash_map::traverse. */
7981 :
7982 : bool
7983 491590 : free_tree_niter_desc (edge const &, tree_niter_desc *const &value, void *)
7984 : {
7985 491590 : if (value)
7986 : {
7987 451338 : value->~tree_niter_desc ();
7988 451338 : free (value);
7989 : }
7990 491590 : return true;
7991 : }
7992 :
7993 : /* Frees data allocated by the optimization of a single loop. */
7994 :
7995 : static void
7996 889236 : free_loop_data (struct ivopts_data *data)
7997 : {
7998 889236 : unsigned i, j;
7999 889236 : bitmap_iterator bi;
8000 889236 : tree obj;
8001 :
8002 889236 : if (data->niters)
8003 : {
8004 971015 : data->niters->traverse<void *, free_tree_niter_desc> (NULL);
8005 958850 : delete data->niters;
8006 479425 : data->niters = NULL;
8007 : }
8008 :
8009 6028125 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
8010 : {
8011 5138889 : struct version_info *info;
8012 :
8013 5138889 : info = ver_info (data, i);
8014 5138889 : info->iv = NULL;
8015 5138889 : info->has_nonlin_use = false;
8016 5138889 : info->preserve_biv = false;
8017 5138889 : info->inv_id = 0;
8018 : }
8019 889236 : bitmap_clear (data->relevant);
8020 889236 : bitmap_clear (data->important_candidates);
8021 :
8022 2712765 : for (i = 0; i < data->vgroups.length (); i++)
8023 : {
8024 1823529 : struct iv_group *group = data->vgroups[i];
8025 :
8026 3930201 : for (j = 0; j < group->vuses.length (); j++)
8027 2106672 : free (group->vuses[j]);
8028 1823529 : group->vuses.release ();
8029 :
8030 1823529 : BITMAP_FREE (group->related_cands);
8031 19783449 : for (j = 0; j < group->n_map_members; j++)
8032 : {
8033 17959920 : if (group->cost_map[j].inv_vars)
8034 3693064 : BITMAP_FREE (group->cost_map[j].inv_vars);
8035 17959920 : if (group->cost_map[j].inv_exprs)
8036 2021423 : BITMAP_FREE (group->cost_map[j].inv_exprs);
8037 : }
8038 :
8039 1823529 : free (group->cost_map);
8040 1823529 : free (group);
8041 : }
8042 889236 : data->vgroups.truncate (0);
8043 :
8044 5563048 : for (i = 0; i < data->vcands.length (); i++)
8045 : {
8046 4673812 : struct iv_cand *cand = data->vcands[i];
8047 :
8048 4673812 : if (cand->inv_vars)
8049 74899 : BITMAP_FREE (cand->inv_vars);
8050 4673812 : if (cand->inv_exprs)
8051 93625 : BITMAP_FREE (cand->inv_exprs);
8052 4673812 : free (cand);
8053 : }
8054 889236 : data->vcands.truncate (0);
8055 :
8056 889236 : if (data->version_info_size < num_ssa_names)
8057 : {
8058 172 : data->version_info_size = 2 * num_ssa_names;
8059 172 : free (data->version_info);
8060 172 : data->version_info = XCNEWVEC (struct version_info, data->version_info_size);
8061 : }
8062 :
8063 889236 : data->max_inv_var_id = 0;
8064 889236 : data->max_inv_expr_id = 0;
8065 :
8066 889236 : FOR_EACH_VEC_ELT (decl_rtl_to_reset, i, obj)
8067 0 : SET_DECL_RTL (obj, NULL_RTX);
8068 :
8069 889236 : decl_rtl_to_reset.truncate (0);
8070 :
8071 889236 : data->inv_expr_tab->empty ();
8072 :
8073 889236 : data->iv_common_cand_tab->empty ();
8074 889236 : data->iv_common_cands.truncate (0);
8075 889236 : }
8076 :
8077 : /* Finalizes data structures used by the iv optimization pass. LOOPS is the
8078 : loop tree. */
8079 :
8080 : static void
8081 247663 : tree_ssa_iv_optimize_finalize (struct ivopts_data *data)
8082 : {
8083 247663 : free_loop_data (data);
8084 247663 : free (data->version_info);
8085 247663 : BITMAP_FREE (data->relevant);
8086 247663 : BITMAP_FREE (data->important_candidates);
8087 :
8088 247663 : decl_rtl_to_reset.release ();
8089 247663 : data->vgroups.release ();
8090 247663 : data->vcands.release ();
8091 247663 : delete data->inv_expr_tab;
8092 247663 : data->inv_expr_tab = NULL;
8093 247663 : free_affine_expand_cache (&data->name_expansion_cache);
8094 247663 : if (data->base_object_map)
8095 167121 : delete data->base_object_map;
8096 247663 : delete data->iv_common_cand_tab;
8097 247663 : data->iv_common_cand_tab = NULL;
8098 247663 : data->iv_common_cands.release ();
8099 247663 : obstack_free (&data->iv_obstack, NULL);
8100 247663 : }
8101 :
8102 : /* Returns true if the loop body BODY includes any function calls. */
8103 :
8104 : static bool
8105 641573 : loop_body_includes_call (basic_block *body, unsigned num_nodes)
8106 : {
8107 641573 : gimple_stmt_iterator gsi;
8108 641573 : unsigned i;
8109 :
8110 2882259 : for (i = 0; i < num_nodes; i++)
8111 24634180 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi); gsi_next (&gsi))
8112 : {
8113 19937965 : gimple *stmt = gsi_stmt (gsi);
8114 19937965 : if (is_gimple_call (stmt)
8115 286665 : && !gimple_call_internal_p (stmt)
8116 20159277 : && !is_inexpensive_builtin (gimple_call_fndecl (stmt)))
8117 : return true;
8118 : }
8119 : return false;
8120 : }
8121 :
8122 : /* Determine cost scaling factor for basic blocks in loop. */
8123 : #define COST_SCALING_FACTOR_BOUND (20)
8124 :
8125 : static void
8126 510604 : determine_scaling_factor (struct ivopts_data *data, basic_block *body)
8127 : {
8128 510604 : int lfreq = data->current_loop->header->count.to_frequency (cfun);
8129 510604 : if (!data->speed || lfreq <= 0)
8130 : return;
8131 :
8132 : int max_freq = lfreq;
8133 2935401 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8134 : {
8135 2512657 : body[i]->aux = (void *)(intptr_t) 1;
8136 2512657 : if (max_freq < body[i]->count.to_frequency (cfun))
8137 104616 : max_freq = body[i]->count.to_frequency (cfun);
8138 : }
8139 422744 : if (max_freq > lfreq)
8140 : {
8141 66128 : int divisor, factor;
8142 : /* Check if scaling factor itself needs to be scaled by the bound. This
8143 : is to avoid overflow when scaling cost according to profile info. */
8144 66128 : if (max_freq / lfreq > COST_SCALING_FACTOR_BOUND)
8145 : {
8146 : divisor = max_freq;
8147 : factor = COST_SCALING_FACTOR_BOUND;
8148 : }
8149 : else
8150 : {
8151 50122 : divisor = lfreq;
8152 50122 : factor = 1;
8153 : }
8154 1015141 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8155 : {
8156 949013 : int bfreq = body[i]->count.to_frequency (cfun);
8157 949013 : if (bfreq <= lfreq)
8158 523496 : continue;
8159 :
8160 425517 : body[i]->aux = (void*)(intptr_t) (factor * bfreq / divisor);
8161 : }
8162 : }
8163 : }
8164 :
8165 : /* Find doloop comparison use and set its doloop_p on if found. */
8166 :
8167 : static bool
8168 0 : find_doloop_use (struct ivopts_data *data)
8169 : {
8170 0 : struct loop *loop = data->current_loop;
8171 :
8172 0 : for (unsigned i = 0; i < data->vgroups.length (); i++)
8173 : {
8174 0 : struct iv_group *group = data->vgroups[i];
8175 0 : if (group->type == USE_COMPARE)
8176 : {
8177 0 : gcc_assert (group->vuses.length () == 1);
8178 0 : struct iv_use *use = group->vuses[0];
8179 0 : gimple *stmt = use->stmt;
8180 0 : if (gimple_code (stmt) == GIMPLE_COND)
8181 : {
8182 0 : basic_block bb = gimple_bb (stmt);
8183 0 : edge true_edge, false_edge;
8184 0 : extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
8185 : /* This comparison is used for loop latch. Require latch is empty
8186 : for now. */
8187 0 : if ((loop->latch == true_edge->dest
8188 0 : || loop->latch == false_edge->dest)
8189 0 : && empty_block_p (loop->latch))
8190 : {
8191 0 : group->doloop_p = true;
8192 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8193 : {
8194 0 : fprintf (dump_file, "Doloop cmp iv use: ");
8195 0 : print_gimple_stmt (dump_file, stmt, TDF_DETAILS);
8196 : }
8197 0 : return true;
8198 : }
8199 : }
8200 : }
8201 : }
8202 :
8203 : return false;
8204 : }
8205 :
8206 : /* For the targets which support doloop, to predict whether later RTL doloop
8207 : transformation will perform on this loop, further detect the doloop use and
8208 : mark the flag doloop_use_p if predicted. */
8209 :
8210 : void
8211 510604 : analyze_and_mark_doloop_use (struct ivopts_data *data)
8212 : {
8213 510604 : data->doloop_use_p = false;
8214 :
8215 510604 : if (!flag_branch_on_count_reg)
8216 : return;
8217 :
8218 510604 : if (data->current_loop->unroll == USHRT_MAX)
8219 : return;
8220 :
8221 510604 : if (!generic_predict_doloop_p (data))
8222 : return;
8223 :
8224 0 : if (find_doloop_use (data))
8225 : {
8226 0 : data->doloop_use_p = true;
8227 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8228 : {
8229 0 : struct loop *loop = data->current_loop;
8230 0 : fprintf (dump_file,
8231 : "Predict loop %d can perform"
8232 : " doloop optimization later.\n",
8233 : loop->num);
8234 0 : flow_loop_dump (loop, dump_file, NULL, 1);
8235 : }
8236 : }
8237 : }
8238 :
8239 : /* Optimizes the LOOP. Returns true if anything changed. */
8240 :
8241 : static bool
8242 641573 : tree_ssa_iv_optimize_loop (struct ivopts_data *data, class loop *loop,
8243 : bitmap toremove)
8244 : {
8245 641573 : bool changed = false;
8246 641573 : class iv_ca *iv_ca;
8247 641573 : edge exit = single_dom_exit (loop);
8248 641573 : basic_block *body;
8249 :
8250 641573 : gcc_assert (!data->niters);
8251 641573 : data->current_loop = loop;
8252 641573 : data->loop_loc = find_loop_location (loop).get_location_t ();
8253 641573 : data->speed = optimize_loop_for_speed_p (loop);
8254 :
8255 641573 : if (dump_file && (dump_flags & TDF_DETAILS))
8256 : {
8257 70 : fprintf (dump_file, "Processing loop %d", loop->num);
8258 70 : if (data->loop_loc != UNKNOWN_LOCATION)
8259 68 : fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
8260 136 : LOCATION_LINE (data->loop_loc));
8261 70 : fprintf (dump_file, "\n");
8262 :
8263 70 : if (exit)
8264 : {
8265 60 : fprintf (dump_file, " single exit %d -> %d, exit condition ",
8266 60 : exit->src->index, exit->dest->index);
8267 120 : print_gimple_stmt (dump_file, *gsi_last_bb (exit->src),
8268 : 0, TDF_SLIM);
8269 60 : fprintf (dump_file, "\n");
8270 : }
8271 :
8272 70 : fprintf (dump_file, "\n");
8273 : }
8274 :
8275 641573 : body = get_loop_body (loop);
8276 641573 : data->body_includes_call = loop_body_includes_call (body, loop->num_nodes);
8277 641573 : renumber_gimple_stmt_uids_in_blocks (body, loop->num_nodes);
8278 :
8279 641573 : data->loop_single_exit_p
8280 641573 : = exit != NULL && loop_only_exit_p (loop, body, exit);
8281 :
8282 : /* For each ssa name determines whether it behaves as an induction variable
8283 : in some loop. */
8284 641573 : if (!find_induction_variables (data, body))
8285 130968 : goto finish;
8286 :
8287 : /* Finds interesting uses (item 1). */
8288 510605 : find_interesting_uses (data, body);
8289 510605 : if (data->vgroups.length () > MAX_CONSIDERED_GROUPS)
8290 1 : goto finish;
8291 :
8292 : /* Determine cost scaling factor for basic blocks in loop. */
8293 510604 : determine_scaling_factor (data, body);
8294 :
8295 : /* Analyze doloop possibility and mark the doloop use if predicted. */
8296 510604 : analyze_and_mark_doloop_use (data);
8297 :
8298 : /* Finds candidates for the induction variables (item 2). */
8299 510604 : find_iv_candidates (data);
8300 :
8301 : /* Calculates the costs (item 3, part 1). */
8302 510604 : determine_iv_costs (data);
8303 510604 : determine_group_iv_costs (data);
8304 510604 : determine_set_costs (data);
8305 :
8306 : /* Find the optimal set of induction variables (item 3, part 2). */
8307 510604 : iv_ca = find_optimal_iv_set (data);
8308 : /* Cleanup basic block aux field. */
8309 3384420 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8310 2873816 : body[i]->aux = NULL;
8311 510604 : if (!iv_ca)
8312 452 : goto finish;
8313 510152 : changed = true;
8314 :
8315 : /* Create the new induction variables (item 4, part 1). */
8316 510152 : create_new_ivs (data, iv_ca);
8317 510152 : iv_ca_free (&iv_ca);
8318 :
8319 : /* Rewrite the uses (item 4, part 2). */
8320 510152 : rewrite_groups (data);
8321 :
8322 : /* Remove the ivs that are unused after rewriting. */
8323 510152 : remove_unused_ivs (data, toremove);
8324 :
8325 641573 : finish:
8326 641573 : free (body);
8327 641573 : free_loop_data (data);
8328 :
8329 641573 : return changed;
8330 : }
8331 :
8332 : /* Main entry point. Optimizes induction variables in loops. */
8333 :
8334 : void
8335 247663 : tree_ssa_iv_optimize (void)
8336 : {
8337 247663 : struct ivopts_data data;
8338 247663 : auto_bitmap toremove;
8339 :
8340 247663 : tree_ssa_iv_optimize_init (&data);
8341 247663 : mark_ssa_maybe_undefs ();
8342 :
8343 247663 : enable_ranger (cfun, false);
8344 :
8345 : /* Optimize the loops starting with the innermost ones. */
8346 1384562 : for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))
8347 : {
8348 641573 : if (!dbg_cnt (ivopts_loop))
8349 0 : continue;
8350 :
8351 641573 : if (dump_file && (dump_flags & TDF_DETAILS))
8352 70 : flow_loop_dump (loop, dump_file, NULL, 1);
8353 :
8354 641573 : tree_ssa_iv_optimize_loop (&data, loop, toremove);
8355 247663 : }
8356 :
8357 247663 : disable_ranger (cfun);
8358 :
8359 : /* Remove eliminated IV defs. */
8360 247663 : release_defs_bitset (toremove);
8361 :
8362 : /* We have changed the structure of induction variables; it might happen
8363 : that definitions in the scev database refer to some of them that were
8364 : eliminated. */
8365 247663 : scev_reset_htab ();
8366 : /* Likewise niter and control-IV information. */
8367 247663 : free_numbers_of_iterations_estimates (cfun);
8368 :
8369 247663 : tree_ssa_iv_optimize_finalize (&data);
8370 247663 : }
8371 :
8372 : #include "gt-tree-ssa-loop-ivopts.h"
|
