Line data Source code
1 : /* Induction variable optimizations.
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it
7 : under the terms of the GNU General Public License as published by the
8 : Free Software Foundation; either version 3, or (at your option) any
9 : later version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT
12 : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : /* This pass tries to find the optimal set of induction variables for the loop.
21 : It optimizes just the basic linear induction variables (although adding
22 : support for other types should not be too hard). It includes the
23 : optimizations commonly known as strength reduction, induction variable
24 : coalescing and induction variable elimination. It does it in the
25 : following steps:
26 :
27 : 1) The interesting uses of induction variables are found. This includes
28 :
29 : -- uses of induction variables in non-linear expressions
30 : -- addresses of arrays
31 : -- comparisons of induction variables
32 :
33 : Note the interesting uses are categorized and handled in group.
34 : Generally, address type uses are grouped together if their iv bases
35 : are different in constant offset.
36 :
37 : 2) Candidates for the induction variables are found. This includes
38 :
39 : -- old induction variables
40 : -- the variables defined by expressions derived from the "interesting
41 : groups/uses" above
42 :
43 : 3) The optimal (w.r. to a cost function) set of variables is chosen. The
44 : cost function assigns a cost to sets of induction variables and consists
45 : of three parts:
46 :
47 : -- The group/use costs. Each of the interesting groups/uses chooses
48 : the best induction variable in the set and adds its cost to the sum.
49 : The cost reflects the time spent on modifying the induction variables
50 : value to be usable for the given purpose (adding base and offset for
51 : arrays, etc.).
52 : -- The variable costs. Each of the variables has a cost assigned that
53 : reflects the costs associated with incrementing the value of the
54 : variable. The original variables are somewhat preferred.
55 : -- The set cost. Depending on the size of the set, extra cost may be
56 : added to reflect register pressure.
57 :
58 : All the costs are defined in a machine-specific way, using the target
59 : hooks and machine descriptions to determine them.
60 :
61 : 4) The trees are transformed to use the new variables, the dead code is
62 : removed.
63 :
64 : All of this is done loop by loop. Doing it globally is theoretically
65 : possible, it might give a better performance and it might enable us
66 : to decide costs more precisely, but getting all the interactions right
67 : would be complicated.
68 :
69 : For the targets supporting low-overhead loops, IVOPTs has to take care of
70 : the loops which will probably be transformed in RTL doloop optimization,
71 : to try to make selected IV candidate set optimal. The process of doloop
72 : support includes:
73 :
74 : 1) Analyze the current loop will be transformed to doloop or not, find and
75 : mark its compare type IV use as doloop use (iv_group field doloop_p), and
76 : set flag doloop_use_p of ivopts_data to notify subsequent processings on
77 : doloop. See analyze_and_mark_doloop_use and its callees for the details.
78 : The target hook predict_doloop_p can be used for target specific checks.
79 :
80 : 2) Add one doloop dedicated IV cand {(may_be_zero ? 1 : (niter + 1)), +, -1},
81 : set flag doloop_p of iv_cand, step cost is set as zero and no extra cost
82 : like biv. For cost determination between doloop IV cand and IV use, the
83 : target hooks doloop_cost_for_generic and doloop_cost_for_address are
84 : provided to add on extra costs for generic type and address type IV use.
85 : Zero cost is assigned to the pair between doloop IV cand and doloop IV
86 : use, and bound zero is set for IV elimination.
87 :
88 : 3) With the cost setting in step 2), the current cost model based IV
89 : selection algorithm will process as usual, pick up doloop dedicated IV if
90 : profitable. */
91 :
92 : #include "config.h"
93 : #include "system.h"
94 : #include "coretypes.h"
95 : #include "backend.h"
96 : #include "rtl.h"
97 : #include "tree.h"
98 : #include "gimple.h"
99 : #include "cfghooks.h"
100 : #include "tree-pass.h"
101 : #include "memmodel.h"
102 : #include "tm_p.h"
103 : #include "ssa.h"
104 : #include "expmed.h"
105 : #include "insn-config.h"
106 : #include "emit-rtl.h"
107 : #include "recog.h"
108 : #include "cgraph.h"
109 : #include "gimple-pretty-print.h"
110 : #include "alias.h"
111 : #include "fold-const.h"
112 : #include "stor-layout.h"
113 : #include "tree-eh.h"
114 : #include "gimplify.h"
115 : #include "gimple-iterator.h"
116 : #include "gimplify-me.h"
117 : #include "tree-cfg.h"
118 : #include "tree-ssa-loop-ivopts.h"
119 : #include "tree-ssa-loop-manip.h"
120 : #include "tree-ssa-loop-niter.h"
121 : #include "tree-ssa-loop.h"
122 : #include "explow.h"
123 : #include "expr.h"
124 : #include "tree-dfa.h"
125 : #include "tree-ssa.h"
126 : #include "cfgloop.h"
127 : #include "tree-scalar-evolution.h"
128 : #include "tree-affine.h"
129 : #include "tree-ssa-propagate.h"
130 : #include "tree-ssa-address.h"
131 : #include "builtins.h"
132 : #include "tree-vectorizer.h"
133 : #include "dbgcnt.h"
134 : #include "cfganal.h"
135 : #include "gimple-fold.h"
136 : #include "gimple-range.h"
137 :
138 : /* For lang_hooks.types.type_for_mode. */
139 : #include "langhooks.h"
140 :
141 : /* FIXME: Expressions are expanded to RTL in this pass to determine the
142 : cost of different addressing modes. This should be moved to a TBD
143 : interface between the GIMPLE and RTL worlds. */
144 :
145 : /* The infinite cost. */
146 : #define INFTY 1000000000
147 :
148 : /* Returns the expected number of loop iterations for LOOP.
149 : The average trip count is computed from profile data if it
150 : exists. */
151 :
152 : static inline unsigned HOST_WIDE_INT
153 8859793 : avg_loop_niter (class loop *loop)
154 : {
155 8859793 : HOST_WIDE_INT niter = estimated_stmt_executions_int (loop);
156 8859793 : if (niter == -1)
157 : {
158 4963992 : niter = likely_max_stmt_executions_int (loop);
159 :
160 4963992 : if (niter == -1 || niter > param_avg_loop_niter)
161 4159951 : return param_avg_loop_niter;
162 : }
163 :
164 4699842 : return niter;
165 : }
166 :
167 : struct iv_use;
168 :
169 : /* Representation of the induction variable. */
170 : struct iv
171 : {
172 : tree base; /* Initial value of the iv. */
173 : tree base_object; /* A memory object to that the induction variable points. */
174 : tree step; /* Step of the iv (constant only). */
175 : tree ssa_name; /* The ssa name with the value. */
176 : struct iv_use *nonlin_use; /* The identifier in the use if it is the case. */
177 : bool biv_p; /* Is it a biv? */
178 : bool no_overflow; /* True if the iv doesn't overflow. */
179 : bool have_address_use;/* For biv, indicate if it's used in any address
180 : type use. */
181 : };
182 :
183 : /* Per-ssa version information (induction variable descriptions, etc.). */
184 : struct version_info
185 : {
186 : tree name; /* The ssa name. */
187 : struct iv *iv; /* Induction variable description. */
188 : bool has_nonlin_use; /* For a loop-level invariant, whether it is used in
189 : an expression that is not an induction variable. */
190 : bool preserve_biv; /* For the original biv, whether to preserve it. */
191 : unsigned inv_id; /* Id of an invariant. */
192 : };
193 :
194 : /* Types of uses. */
195 : enum use_type
196 : {
197 : USE_NONLINEAR_EXPR, /* Use in a nonlinear expression. */
198 : USE_REF_ADDRESS, /* Use is an address for an explicit memory
199 : reference. */
200 : USE_PTR_ADDRESS, /* Use is a pointer argument to a function in
201 : cases where the expansion of the function
202 : will turn the argument into a normal address. */
203 : USE_COMPARE /* Use is a compare. */
204 : };
205 :
206 : /* Cost of a computation. */
207 : class comp_cost
208 : {
209 : public:
210 132844478 : comp_cost (): cost (0), complexity (0), scratch (0)
211 : {}
212 :
213 25445152 : comp_cost (int64_t cost, unsigned complexity, int64_t scratch = 0)
214 15517889 : : cost (cost), complexity (complexity), scratch (scratch)
215 : {}
216 :
217 : /* Returns true if COST is infinite. */
218 : bool infinite_cost_p ();
219 :
220 : /* Adds costs COST1 and COST2. */
221 : friend comp_cost operator+ (comp_cost cost1, comp_cost cost2);
222 :
223 : /* Adds COST to the comp_cost. */
224 : comp_cost operator+= (comp_cost cost);
225 :
226 : /* Adds constant C to this comp_cost. */
227 : comp_cost operator+= (HOST_WIDE_INT c);
228 :
229 : /* Subtracts constant C to this comp_cost. */
230 : comp_cost operator-= (HOST_WIDE_INT c);
231 :
232 : /* Divide the comp_cost by constant C. */
233 : comp_cost operator/= (HOST_WIDE_INT c);
234 :
235 : /* Multiply the comp_cost by constant C. */
236 : comp_cost operator*= (HOST_WIDE_INT c);
237 :
238 : /* Subtracts costs COST1 and COST2. */
239 : friend comp_cost operator- (comp_cost cost1, comp_cost cost2);
240 :
241 : /* Subtracts COST from this comp_cost. */
242 : comp_cost operator-= (comp_cost cost);
243 :
244 : /* Returns true if COST1 is smaller than COST2. */
245 : friend bool operator< (comp_cost cost1, comp_cost cost2);
246 :
247 : /* Returns true if COST1 and COST2 are equal. */
248 : friend bool operator== (comp_cost cost1, comp_cost cost2);
249 :
250 : /* Returns true if COST1 is smaller or equal than COST2. */
251 : friend bool operator<= (comp_cost cost1, comp_cost cost2);
252 :
253 : int64_t cost; /* The runtime cost. */
254 : unsigned complexity; /* The estimate of the complexity of the code for
255 : the computation (in no concrete units --
256 : complexity field should be larger for more
257 : complex expressions and addressing modes). */
258 : int64_t scratch; /* Scratch used during cost computation. */
259 : };
260 :
261 : static const comp_cost no_cost;
262 : static const comp_cost infinite_cost (INFTY, 0, INFTY);
263 :
264 : bool
265 1870177488 : comp_cost::infinite_cost_p ()
266 : {
267 1870177488 : return cost == INFTY;
268 : }
269 :
270 : comp_cost
271 249354864 : operator+ (comp_cost cost1, comp_cost cost2)
272 : {
273 249354864 : if (cost1.infinite_cost_p () || cost2.infinite_cost_p ())
274 1890219 : return infinite_cost;
275 :
276 247464645 : gcc_assert (cost1.cost + cost2.cost < infinite_cost.cost);
277 247464645 : cost1.cost += cost2.cost;
278 247464645 : cost1.complexity += cost2.complexity;
279 :
280 247464645 : return cost1;
281 : }
282 :
283 : comp_cost
284 214233377 : operator- (comp_cost cost1, comp_cost cost2)
285 : {
286 214233377 : if (cost1.infinite_cost_p ())
287 0 : return infinite_cost;
288 :
289 214233377 : gcc_assert (!cost2.infinite_cost_p ());
290 214233377 : gcc_assert (cost1.cost - cost2.cost < infinite_cost.cost);
291 :
292 214233377 : cost1.cost -= cost2.cost;
293 214233377 : cost1.complexity -= cost2.complexity;
294 :
295 214233377 : return cost1;
296 : }
297 :
298 : comp_cost
299 249354864 : comp_cost::operator+= (comp_cost cost)
300 : {
301 249354864 : *this = *this + cost;
302 249354864 : return *this;
303 : }
304 :
305 : comp_cost
306 884334237 : comp_cost::operator+= (HOST_WIDE_INT c)
307 : {
308 884334237 : if (c >= INFTY)
309 0 : this->cost = INFTY;
310 :
311 884334237 : if (infinite_cost_p ())
312 0 : return *this;
313 :
314 884334237 : gcc_assert (this->cost + c < infinite_cost.cost);
315 884334237 : this->cost += c;
316 :
317 884334237 : return *this;
318 : }
319 :
320 : comp_cost
321 545736 : comp_cost::operator-= (HOST_WIDE_INT c)
322 : {
323 545736 : if (infinite_cost_p ())
324 0 : return *this;
325 :
326 545736 : gcc_assert (this->cost - c < infinite_cost.cost);
327 545736 : this->cost -= c;
328 :
329 545736 : return *this;
330 : }
331 :
332 : comp_cost
333 0 : comp_cost::operator/= (HOST_WIDE_INT c)
334 : {
335 0 : gcc_assert (c != 0);
336 0 : if (infinite_cost_p ())
337 0 : return *this;
338 :
339 0 : this->cost /= c;
340 :
341 0 : return *this;
342 : }
343 :
344 : comp_cost
345 0 : comp_cost::operator*= (HOST_WIDE_INT c)
346 : {
347 0 : if (infinite_cost_p ())
348 0 : return *this;
349 :
350 0 : gcc_assert (this->cost * c < infinite_cost.cost);
351 0 : this->cost *= c;
352 :
353 0 : return *this;
354 : }
355 :
356 : comp_cost
357 214233377 : comp_cost::operator-= (comp_cost cost)
358 : {
359 214233377 : *this = *this - cost;
360 214233377 : return *this;
361 : }
362 :
363 : bool
364 191308649 : operator< (comp_cost cost1, comp_cost cost2)
365 : {
366 191308649 : if (cost1.cost == cost2.cost)
367 83414584 : return cost1.complexity < cost2.complexity;
368 :
369 107894065 : return cost1.cost < cost2.cost;
370 : }
371 :
372 : bool
373 3875651 : operator== (comp_cost cost1, comp_cost cost2)
374 : {
375 3875651 : return cost1.cost == cost2.cost
376 3875651 : && cost1.complexity == cost2.complexity;
377 : }
378 :
379 : bool
380 6459353 : operator<= (comp_cost cost1, comp_cost cost2)
381 : {
382 6459353 : return cost1 < cost2 || cost1 == cost2;
383 : }
384 :
385 : struct iv_inv_expr_ent;
386 :
387 : /* The candidate - cost pair. */
388 : class cost_pair
389 : {
390 : public:
391 : struct iv_cand *cand; /* The candidate. */
392 : comp_cost cost; /* The cost. */
393 : enum tree_code comp; /* For iv elimination, the comparison. */
394 : bitmap inv_vars; /* The list of invariant ssa_vars that have to be
395 : preserved when representing iv_use with iv_cand. */
396 : bitmap inv_exprs; /* The list of newly created invariant expressions
397 : when representing iv_use with iv_cand. */
398 : tree value; /* For final value elimination, the expression for
399 : the final value of the iv. For iv elimination,
400 : the new bound to compare with. */
401 : };
402 :
403 : /* Use. */
404 : struct iv_use
405 : {
406 : unsigned id; /* The id of the use. */
407 : unsigned group_id; /* The group id the use belongs to. */
408 : enum use_type type; /* Type of the use. */
409 : tree mem_type; /* The memory type to use when testing whether an
410 : address is legitimate, and what the address's
411 : cost is. */
412 : struct iv *iv; /* The induction variable it is based on. */
413 : gimple *stmt; /* Statement in that it occurs. */
414 : tree *op_p; /* The place where it occurs. */
415 :
416 : tree addr_base; /* Base address with const offset stripped. */
417 : poly_uint64 addr_offset;
418 : /* Const offset stripped from base address. */
419 : };
420 :
421 : /* Group of uses. */
422 : struct iv_group
423 : {
424 : /* The id of the group. */
425 : unsigned id;
426 : /* Uses of the group are of the same type. */
427 : enum use_type type;
428 : /* The set of "related" IV candidates, plus the important ones. */
429 : bitmap related_cands;
430 : /* Number of IV candidates in the cost_map. */
431 : unsigned n_map_members;
432 : /* The costs wrto the iv candidates. */
433 : class cost_pair *cost_map;
434 : /* The selected candidate for the group. */
435 : struct iv_cand *selected;
436 : /* To indicate this is a doloop use group. */
437 : bool doloop_p;
438 : /* Uses in the group. */
439 : vec<struct iv_use *> vuses;
440 : };
441 :
442 : /* The position where the iv is computed. */
443 : enum iv_position
444 : {
445 : IP_NORMAL, /* At the end, just before the exit condition. */
446 : IP_END, /* At the end of the latch block. */
447 : IP_BEFORE_USE, /* Immediately before a specific use. */
448 : IP_AFTER_USE, /* Immediately after a specific use. */
449 : IP_ORIGINAL /* The original biv. */
450 : };
451 :
452 : /* The induction variable candidate. */
453 : struct iv_cand
454 : {
455 : unsigned id; /* The number of the candidate. */
456 : bool important; /* Whether this is an "important" candidate, i.e. such
457 : that it should be considered by all uses. */
458 : bool involves_undefs; /* Whether the IV involves undefined values. */
459 : enum iv_position pos : 8;/* Where it is computed. */
460 : gimple *incremented_at;/* For original biv, the statement where it is
461 : incremented. */
462 : tree var_before; /* The variable used for it before increment. */
463 : tree var_after; /* The variable used for it after increment. */
464 : struct iv *iv; /* The value of the candidate. NULL for
465 : "pseudocandidate" used to indicate the possibility
466 : to replace the final value of an iv by direct
467 : computation of the value. */
468 : unsigned cost; /* Cost of the candidate. */
469 : unsigned cost_step; /* Cost of the candidate's increment operation. */
470 : unsigned nregs; /* Total number of registers used by the candidate. */
471 : struct iv_use *ainc_use; /* For IP_{BEFORE,AFTER}_USE candidates, the place
472 : where it is incremented. */
473 : bitmap inv_vars; /* The list of invariant ssa_vars used in step of the
474 : iv_cand. */
475 : bitmap inv_exprs; /* If step is more complicated than a single ssa_var,
476 : handle it as a new invariant expression which will
477 : be hoisted out of loop. */
478 : struct iv *orig_iv; /* The original iv if this cand is added from biv with
479 : smaller type. */
480 : bool doloop_p; /* Whether this is a doloop candidate. */
481 : };
482 :
483 : /* Hashtable entry for common candidate derived from iv uses. */
484 2652723 : class iv_common_cand
485 : {
486 : public:
487 : tree base;
488 : tree step;
489 : /* IV uses from which this common candidate is derived. */
490 : auto_vec<struct iv_use *> uses;
491 : hashval_t hash;
492 : };
493 :
494 : /* Hashtable helpers. */
495 :
496 : struct iv_common_cand_hasher : delete_ptr_hash <iv_common_cand>
497 : {
498 : static inline hashval_t hash (const iv_common_cand *);
499 : static inline bool equal (const iv_common_cand *, const iv_common_cand *);
500 : };
501 :
502 : /* Hash function for possible common candidates. */
503 :
504 : inline hashval_t
505 9734055 : iv_common_cand_hasher::hash (const iv_common_cand *ccand)
506 : {
507 9734055 : return ccand->hash;
508 : }
509 :
510 : /* Hash table equality function for common candidates. */
511 :
512 : inline bool
513 10992915 : iv_common_cand_hasher::equal (const iv_common_cand *ccand1,
514 : const iv_common_cand *ccand2)
515 : {
516 10992915 : return (ccand1->hash == ccand2->hash
517 1633266 : && operand_equal_p (ccand1->base, ccand2->base, 0)
518 1614425 : && operand_equal_p (ccand1->step, ccand2->step, 0)
519 12602508 : && (TYPE_PRECISION (TREE_TYPE (ccand1->base))
520 1609593 : == TYPE_PRECISION (TREE_TYPE (ccand2->base))));
521 : }
522 :
523 : /* Loop invariant expression hashtable entry. */
524 :
525 : struct iv_inv_expr_ent
526 : {
527 : /* Tree expression of the entry. */
528 : tree expr;
529 : /* Unique identifier. */
530 : int id;
531 : /* Hash value. */
532 : hashval_t hash;
533 : };
534 :
535 : /* Sort iv_inv_expr_ent pair A and B by id field. */
536 :
537 : static int
538 5824 : sort_iv_inv_expr_ent (const void *a, const void *b)
539 : {
540 5824 : const iv_inv_expr_ent * const *e1 = (const iv_inv_expr_ent * const *) (a);
541 5824 : const iv_inv_expr_ent * const *e2 = (const iv_inv_expr_ent * const *) (b);
542 :
543 5824 : unsigned id1 = (*e1)->id;
544 5824 : unsigned id2 = (*e2)->id;
545 :
546 5824 : if (id1 < id2)
547 : return -1;
548 2704 : else if (id1 > id2)
549 : return 1;
550 : else
551 0 : return 0;
552 : }
553 :
554 : /* Hashtable helpers. */
555 :
556 : struct iv_inv_expr_hasher : free_ptr_hash <iv_inv_expr_ent>
557 : {
558 : static inline hashval_t hash (const iv_inv_expr_ent *);
559 : static inline bool equal (const iv_inv_expr_ent *, const iv_inv_expr_ent *);
560 : };
561 :
562 : /* Return true if uses of type TYPE represent some form of address. */
563 :
564 : inline bool
565 9077619 : address_p (use_type type)
566 : {
567 9077619 : return type == USE_REF_ADDRESS || type == USE_PTR_ADDRESS;
568 : }
569 :
570 : /* Hash function for loop invariant expressions. */
571 :
572 : inline hashval_t
573 6531700 : iv_inv_expr_hasher::hash (const iv_inv_expr_ent *expr)
574 : {
575 6531700 : return expr->hash;
576 : }
577 :
578 : /* Hash table equality function for expressions. */
579 :
580 : inline bool
581 7838076 : iv_inv_expr_hasher::equal (const iv_inv_expr_ent *expr1,
582 : const iv_inv_expr_ent *expr2)
583 : {
584 7838076 : return expr1->hash == expr2->hash
585 7838076 : && operand_equal_p (expr1->expr, expr2->expr, 0);
586 : }
587 :
588 : struct ivopts_data
589 : {
590 : /* The currently optimized loop. */
591 : class loop *current_loop;
592 : location_t loop_loc;
593 :
594 : /* Numbers of iterations for all exits of the current loop. */
595 : hash_map<edge, tree_niter_desc *> *niters;
596 :
597 : /* Number of registers used in it. */
598 : unsigned regs_used;
599 :
600 : /* The size of version_info array allocated. */
601 : unsigned version_info_size;
602 :
603 : /* The array of information for the ssa names. */
604 : struct version_info *version_info;
605 :
606 : /* The hashtable of loop invariant expressions created
607 : by ivopt. */
608 : hash_table<iv_inv_expr_hasher> *inv_expr_tab;
609 :
610 : /* The bitmap of indices in version_info whose value was changed. */
611 : bitmap relevant;
612 :
613 : /* The uses of induction variables. */
614 : vec<iv_group *> vgroups;
615 :
616 : /* The candidates. */
617 : vec<iv_cand *> vcands;
618 :
619 : /* A bitmap of important candidates. */
620 : bitmap important_candidates;
621 :
622 : /* Cache used by tree_to_aff_combination_expand. */
623 : hash_map<tree, name_expansion *> *name_expansion_cache;
624 :
625 : /* The hashtable of common candidates derived from iv uses. */
626 : hash_table<iv_common_cand_hasher> *iv_common_cand_tab;
627 :
628 : /* The common candidates. */
629 : vec<iv_common_cand *> iv_common_cands;
630 :
631 : /* Hash map recording base object information of tree exp. */
632 : hash_map<tree, tree> *base_object_map;
633 :
634 : /* The maximum invariant variable id. */
635 : unsigned max_inv_var_id;
636 :
637 : /* The maximum invariant expression id. */
638 : unsigned max_inv_expr_id;
639 :
640 : /* Number of registers used by each invariant variable and expression. */
641 : vec<unsigned> inv_var_nregs;
642 : vec<unsigned> inv_expr_nregs;
643 :
644 : /* Number of no_overflow BIVs which are not used in memory address. */
645 : unsigned bivs_not_used_in_addr;
646 :
647 : /* Obstack for iv structure. */
648 : struct obstack iv_obstack;
649 :
650 : /* Whether to consider just related and important candidates when replacing a
651 : use. */
652 : bool consider_all_candidates;
653 :
654 : /* Are we optimizing for speed? */
655 : bool speed;
656 :
657 : /* Whether the loop body includes any function calls. */
658 : bool body_includes_call;
659 :
660 : /* Whether the loop body can only be exited via single exit. */
661 : bool loop_single_exit_p;
662 :
663 : /* Whether the loop has doloop comparison use. */
664 : bool doloop_use_p;
665 : };
666 :
667 : /* An assignment of iv candidates to uses. */
668 :
669 : class iv_ca
670 : {
671 : public:
672 : /* The number of uses covered by the assignment. */
673 : unsigned upto;
674 :
675 : /* Number of uses that cannot be expressed by the candidates in the set. */
676 : unsigned bad_groups;
677 :
678 : /* Candidate assigned to a use, together with the related costs. */
679 : class cost_pair **cand_for_group;
680 :
681 : /* Number of times each candidate is used. */
682 : unsigned *n_cand_uses;
683 :
684 : /* The candidates used. */
685 : bitmap cands;
686 :
687 : /* The number of candidates in the set. */
688 : unsigned n_cands;
689 :
690 : /* The number of invariants needed, including both invariant variants and
691 : invariant expressions. */
692 : unsigned n_invs;
693 :
694 : /* Total number of registers used by candidates. */
695 : unsigned n_cand_regs;
696 :
697 : /* Total number of registers used by invariants. */
698 : unsigned n_inv_regs;
699 :
700 : /* Total cost of expressing uses. */
701 : comp_cost cand_use_cost;
702 :
703 : /* Total cost of candidates. */
704 : int64_t cand_cost;
705 :
706 : /* Number of times each invariant variable is used. */
707 : unsigned *n_inv_var_uses;
708 :
709 : /* Number of times each invariant expression is used. */
710 : unsigned *n_inv_expr_uses;
711 :
712 : /* Total cost of the assignment. */
713 : comp_cost cost;
714 : };
715 :
716 : /* Difference of two iv candidate assignments. */
717 :
718 : struct iv_ca_delta
719 : {
720 : /* Changed group. */
721 : struct iv_group *group;
722 :
723 : /* An old assignment (for rollback purposes). */
724 : class cost_pair *old_cp;
725 :
726 : /* A new assignment. */
727 : class cost_pair *new_cp;
728 :
729 : /* Next change in the list. */
730 : struct iv_ca_delta *next;
731 : };
732 :
733 : /* Bound on number of candidates below that all candidates are considered. */
734 :
735 : #define CONSIDER_ALL_CANDIDATES_BOUND \
736 : ((unsigned) param_iv_consider_all_candidates_bound)
737 :
738 : /* If there are more iv occurrences, we just give up (it is quite unlikely that
739 : optimizing such a loop would help, and it would take ages). */
740 :
741 : #define MAX_CONSIDERED_GROUPS \
742 : ((unsigned) param_iv_max_considered_uses)
743 :
744 : /* If there are at most this number of ivs in the set, try removing unnecessary
745 : ivs from the set always. */
746 :
747 : #define ALWAYS_PRUNE_CAND_SET_BOUND \
748 : ((unsigned) param_iv_always_prune_cand_set_bound)
749 :
750 : /* The list of trees for that the decl_rtl field must be reset is stored
751 : here. */
752 :
753 : static vec<tree> decl_rtl_to_reset;
754 :
755 : static comp_cost force_expr_to_var_cost (tree, bool);
756 :
757 : /* The single loop exit if it dominates the latch, NULL otherwise. */
758 :
759 : edge
760 716412 : single_dom_exit (class loop *loop)
761 : {
762 716412 : edge exit = single_exit (loop);
763 :
764 716412 : if (!exit)
765 : return NULL;
766 :
767 474744 : if (!just_once_each_iteration_p (loop, exit->src))
768 5793 : return NULL;
769 :
770 : return exit;
771 : }
772 :
773 : /* Dumps information about the induction variable IV to FILE. Don't dump
774 : variable's name if DUMP_NAME is FALSE. The information is dumped with
775 : preceding spaces indicated by INDENT_LEVEL. */
776 :
777 : void
778 1690 : dump_iv (FILE *file, struct iv *iv, bool dump_name, unsigned indent_level)
779 : {
780 1690 : const char *p;
781 1690 : const char spaces[9] = {' ', ' ', ' ', ' ', ' ', ' ', ' ', ' ', '\0'};
782 :
783 1690 : if (indent_level > 4)
784 : indent_level = 4;
785 1690 : p = spaces + 8 - (indent_level << 1);
786 :
787 1690 : fprintf (file, "%sIV struct:\n", p);
788 1690 : if (iv->ssa_name && dump_name)
789 : {
790 573 : fprintf (file, "%s SSA_NAME:\t", p);
791 573 : print_generic_expr (file, iv->ssa_name, TDF_SLIM);
792 573 : fprintf (file, "\n");
793 : }
794 :
795 1690 : fprintf (file, "%s Type:\t", p);
796 1690 : print_generic_expr (file, TREE_TYPE (iv->base), TDF_SLIM);
797 1690 : fprintf (file, "\n");
798 :
799 1690 : fprintf (file, "%s Base:\t", p);
800 1690 : print_generic_expr (file, iv->base, TDF_SLIM);
801 1690 : fprintf (file, "\n");
802 :
803 1690 : fprintf (file, "%s Step:\t", p);
804 1690 : print_generic_expr (file, iv->step, TDF_SLIM);
805 1690 : fprintf (file, "\n");
806 :
807 1690 : if (iv->base_object)
808 : {
809 527 : fprintf (file, "%s Object:\t", p);
810 527 : print_generic_expr (file, iv->base_object, TDF_SLIM);
811 527 : fprintf (file, "\n");
812 : }
813 :
814 3056 : fprintf (file, "%s Biv:\t%c\n", p, iv->biv_p ? 'Y' : 'N');
815 :
816 1690 : fprintf (file, "%s Overflowness wrto loop niter:\t%s\n",
817 1690 : p, iv->no_overflow ? "No-overflow" : "Overflow");
818 1690 : }
819 :
820 : /* Dumps information about the USE to FILE. */
821 :
822 : void
823 264 : dump_use (FILE *file, struct iv_use *use)
824 : {
825 264 : fprintf (file, " Use %d.%d:\n", use->group_id, use->id);
826 264 : fprintf (file, " At stmt:\t");
827 264 : print_gimple_stmt (file, use->stmt, 0);
828 264 : fprintf (file, " At pos:\t");
829 264 : if (use->op_p)
830 172 : print_generic_expr (file, *use->op_p, TDF_SLIM);
831 264 : fprintf (file, "\n");
832 264 : dump_iv (file, use->iv, false, 2);
833 264 : }
834 :
835 : /* Dumps information about the uses to FILE. */
836 :
837 : void
838 72 : dump_groups (FILE *file, struct ivopts_data *data)
839 : {
840 72 : unsigned i, j;
841 72 : struct iv_group *group;
842 :
843 305 : for (i = 0; i < data->vgroups.length (); i++)
844 : {
845 233 : group = data->vgroups[i];
846 233 : fprintf (file, "Group %d:\n", group->id);
847 233 : if (group->type == USE_NONLINEAR_EXPR)
848 92 : fprintf (file, " Type:\tGENERIC\n");
849 141 : else if (group->type == USE_REF_ADDRESS)
850 62 : fprintf (file, " Type:\tREFERENCE ADDRESS\n");
851 79 : else if (group->type == USE_PTR_ADDRESS)
852 0 : fprintf (file, " Type:\tPOINTER ARGUMENT ADDRESS\n");
853 : else
854 : {
855 79 : gcc_assert (group->type == USE_COMPARE);
856 79 : fprintf (file, " Type:\tCOMPARE\n");
857 : }
858 497 : for (j = 0; j < group->vuses.length (); j++)
859 264 : dump_use (file, group->vuses[j]);
860 : }
861 72 : }
862 :
863 : /* Dumps information about induction variable candidate CAND to FILE. */
864 :
865 : void
866 853 : dump_cand (FILE *file, struct iv_cand *cand)
867 : {
868 853 : struct iv *iv = cand->iv;
869 :
870 853 : fprintf (file, "Candidate %d:\n", cand->id);
871 853 : if (cand->inv_vars)
872 : {
873 28 : fprintf (file, " Depend on inv.vars: ");
874 28 : dump_bitmap (file, cand->inv_vars);
875 : }
876 853 : if (cand->inv_exprs)
877 : {
878 4 : fprintf (file, " Depend on inv.exprs: ");
879 4 : dump_bitmap (file, cand->inv_exprs);
880 : }
881 :
882 853 : if (cand->var_before)
883 : {
884 737 : fprintf (file, " Var before: ");
885 737 : print_generic_expr (file, cand->var_before, TDF_SLIM);
886 737 : fprintf (file, "\n");
887 : }
888 853 : if (cand->var_after)
889 : {
890 737 : fprintf (file, " Var after: ");
891 737 : print_generic_expr (file, cand->var_after, TDF_SLIM);
892 737 : fprintf (file, "\n");
893 : }
894 :
895 853 : switch (cand->pos)
896 : {
897 701 : case IP_NORMAL:
898 701 : fprintf (file, " Incr POS: before exit test\n");
899 701 : break;
900 :
901 0 : case IP_BEFORE_USE:
902 0 : fprintf (file, " Incr POS: before use %d\n", cand->ainc_use->id);
903 0 : break;
904 :
905 0 : case IP_AFTER_USE:
906 0 : fprintf (file, " Incr POS: after use %d\n", cand->ainc_use->id);
907 0 : break;
908 :
909 0 : case IP_END:
910 0 : fprintf (file, " Incr POS: at end\n");
911 0 : break;
912 :
913 152 : case IP_ORIGINAL:
914 152 : fprintf (file, " Incr POS: orig biv\n");
915 152 : break;
916 : }
917 :
918 853 : dump_iv (file, iv, false, 1);
919 853 : }
920 :
921 : /* Returns the info for ssa version VER. */
922 :
923 : static inline struct version_info *
924 118926623 : ver_info (struct ivopts_data *data, unsigned ver)
925 : {
926 118926623 : return data->version_info + ver;
927 : }
928 :
929 : /* Returns the info for ssa name NAME. */
930 :
931 : static inline struct version_info *
932 96189746 : name_info (struct ivopts_data *data, tree name)
933 : {
934 96189746 : return ver_info (data, SSA_NAME_VERSION (name));
935 : }
936 :
937 : /* Returns true if STMT is after the place where the IP_NORMAL ivs will be
938 : emitted in LOOP. */
939 :
940 : static bool
941 33627759 : stmt_after_ip_normal_pos (class loop *loop, gimple *stmt)
942 : {
943 33627759 : basic_block bb = ip_normal_pos (loop), sbb = gimple_bb (stmt);
944 :
945 33627759 : gcc_assert (bb);
946 :
947 33627759 : if (sbb == loop->latch)
948 : return true;
949 :
950 33512111 : if (sbb != bb)
951 : return false;
952 :
953 20075185 : return stmt == last_nondebug_stmt (bb);
954 : }
955 :
956 : /* Returns true if STMT if after the place where the original induction
957 : variable CAND is incremented. If TRUE_IF_EQUAL is set, we return true
958 : if the positions are identical. */
959 :
960 : static bool
961 7958077 : stmt_after_inc_pos (struct iv_cand *cand, gimple *stmt, bool true_if_equal)
962 : {
963 7958077 : basic_block cand_bb = gimple_bb (cand->incremented_at);
964 7958077 : basic_block stmt_bb = gimple_bb (stmt);
965 :
966 7958077 : if (!dominated_by_p (CDI_DOMINATORS, stmt_bb, cand_bb))
967 : return false;
968 :
969 5547451 : if (stmt_bb != cand_bb)
970 : return true;
971 :
972 5306521 : if (true_if_equal
973 5306521 : && gimple_uid (stmt) == gimple_uid (cand->incremented_at))
974 : return true;
975 5299999 : return gimple_uid (stmt) > gimple_uid (cand->incremented_at);
976 : }
977 :
978 : /* Returns true if STMT if after the place where the induction variable
979 : CAND is incremented in LOOP. */
980 :
981 : static bool
982 42759154 : stmt_after_increment (class loop *loop, struct iv_cand *cand, gimple *stmt)
983 : {
984 42759154 : switch (cand->pos)
985 : {
986 : case IP_END:
987 : return false;
988 :
989 33627759 : case IP_NORMAL:
990 33627759 : return stmt_after_ip_normal_pos (loop, stmt);
991 :
992 7948239 : case IP_ORIGINAL:
993 7948239 : case IP_AFTER_USE:
994 7948239 : return stmt_after_inc_pos (cand, stmt, false);
995 :
996 9838 : case IP_BEFORE_USE:
997 9838 : return stmt_after_inc_pos (cand, stmt, true);
998 :
999 0 : default:
1000 0 : gcc_unreachable ();
1001 : }
1002 : }
1003 :
1004 : /* walk_tree callback for contains_abnormal_ssa_name_p. */
1005 :
1006 : static tree
1007 15088171 : contains_abnormal_ssa_name_p_1 (tree *tp, int *walk_subtrees, void *)
1008 : {
1009 15088171 : if (TREE_CODE (*tp) == SSA_NAME
1010 15088171 : && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (*tp))
1011 : return *tp;
1012 :
1013 15088155 : if (!EXPR_P (*tp))
1014 10314847 : *walk_subtrees = 0;
1015 :
1016 : return NULL_TREE;
1017 : }
1018 :
1019 : /* Returns true if EXPR contains a ssa name that occurs in an
1020 : abnormal phi node. */
1021 :
1022 : bool
1023 8097871 : contains_abnormal_ssa_name_p (tree expr)
1024 : {
1025 8097871 : return walk_tree_without_duplicates
1026 8097871 : (&expr, contains_abnormal_ssa_name_p_1, NULL) != NULL_TREE;
1027 : }
1028 :
1029 : /* Returns the structure describing number of iterations determined from
1030 : EXIT of DATA->current_loop, or NULL if something goes wrong. */
1031 :
1032 : static class tree_niter_desc *
1033 4439097 : niter_for_exit (struct ivopts_data *data, edge exit)
1034 : {
1035 4439097 : class tree_niter_desc *desc;
1036 4439097 : tree_niter_desc **slot;
1037 :
1038 4439097 : if (!data->niters)
1039 : {
1040 481252 : data->niters = new hash_map<edge, tree_niter_desc *>;
1041 481252 : slot = NULL;
1042 : }
1043 : else
1044 3957845 : slot = data->niters->get (exit);
1045 :
1046 4439097 : if (!slot)
1047 : {
1048 : /* Try to determine number of iterations. We cannot safely work with ssa
1049 : names that appear in phi nodes on abnormal edges, so that we do not
1050 : create overlapping life ranges for them (PR 27283). */
1051 493545 : desc = XNEW (class tree_niter_desc);
1052 493545 : ::new (static_cast<void*> (desc)) tree_niter_desc ();
1053 493545 : if (!number_of_iterations_exit (data->current_loop,
1054 : exit, desc, true)
1055 493545 : || contains_abnormal_ssa_name_p (desc->niter))
1056 : {
1057 38927 : desc->~tree_niter_desc ();
1058 38927 : XDELETE (desc);
1059 38927 : desc = NULL;
1060 : }
1061 493545 : data->niters->put (exit, desc);
1062 : }
1063 : else
1064 3945552 : desc = *slot;
1065 :
1066 4439097 : return desc;
1067 : }
1068 :
1069 : /* Returns the structure describing number of iterations determined from
1070 : single dominating exit of DATA->current_loop, or NULL if something
1071 : goes wrong. */
1072 :
1073 : static class tree_niter_desc *
1074 72 : niter_for_single_dom_exit (struct ivopts_data *data)
1075 : {
1076 72 : edge exit = single_dom_exit (data->current_loop);
1077 :
1078 72 : if (!exit)
1079 : return NULL;
1080 :
1081 62 : return niter_for_exit (data, exit);
1082 : }
1083 :
1084 : /* Initializes data structures used by the iv optimization pass, stored
1085 : in DATA. */
1086 :
1087 : static void
1088 248445 : tree_ssa_iv_optimize_init (struct ivopts_data *data)
1089 : {
1090 248445 : data->version_info_size = 2 * num_ssa_names;
1091 248445 : data->version_info = XCNEWVEC (struct version_info, data->version_info_size);
1092 248445 : data->relevant = BITMAP_ALLOC (NULL);
1093 248445 : data->important_candidates = BITMAP_ALLOC (NULL);
1094 248445 : data->max_inv_var_id = 0;
1095 248445 : data->max_inv_expr_id = 0;
1096 248445 : data->inv_var_nregs.create (0);
1097 248445 : data->inv_expr_nregs.create (0);
1098 248445 : data->niters = NULL;
1099 248445 : data->vgroups.create (20);
1100 248445 : data->vcands.create (20);
1101 248445 : data->inv_expr_tab = new hash_table<iv_inv_expr_hasher> (10);
1102 248445 : data->name_expansion_cache = NULL;
1103 248445 : data->base_object_map = NULL;
1104 248445 : data->iv_common_cand_tab = new hash_table<iv_common_cand_hasher> (10);
1105 248445 : data->iv_common_cands.create (20);
1106 248445 : decl_rtl_to_reset.create (20);
1107 248445 : gcc_obstack_init (&data->iv_obstack);
1108 248445 : }
1109 :
1110 : /* walk_tree callback for determine_base_object. */
1111 :
1112 : static tree
1113 19647866 : determine_base_object_1 (tree *tp, int *walk_subtrees, void *wdata)
1114 : {
1115 19647866 : tree_code code = TREE_CODE (*tp);
1116 19647866 : tree obj = NULL_TREE;
1117 19647866 : if (code == ADDR_EXPR)
1118 : {
1119 1065939 : tree base = get_base_address (TREE_OPERAND (*tp, 0));
1120 1065939 : if (!base)
1121 0 : obj = *tp;
1122 1065939 : else if (TREE_CODE (base) != MEM_REF)
1123 1065915 : obj = fold_convert (ptr_type_node, build_fold_addr_expr (base));
1124 : }
1125 18581927 : else if (code == SSA_NAME && POINTER_TYPE_P (TREE_TYPE (*tp)))
1126 1940848 : obj = fold_convert (ptr_type_node, *tp);
1127 :
1128 3006763 : if (!obj)
1129 : {
1130 16641103 : if (!EXPR_P (*tp))
1131 7293235 : *walk_subtrees = 0;
1132 :
1133 : return NULL_TREE;
1134 : }
1135 : /* Record special node for multiple base objects and stop. */
1136 3006763 : if (*static_cast<tree *> (wdata))
1137 : {
1138 4279 : *static_cast<tree *> (wdata) = integer_zero_node;
1139 4279 : return integer_zero_node;
1140 : }
1141 : /* Record the base object and continue looking. */
1142 3002484 : *static_cast<tree *> (wdata) = obj;
1143 3002484 : return NULL_TREE;
1144 : }
1145 :
1146 : /* Returns a memory object to that EXPR points with caching. Return NULL if we
1147 : are able to determine that it does not point to any such object; specially
1148 : return integer_zero_node if EXPR contains multiple base objects. */
1149 :
1150 : static tree
1151 10585954 : determine_base_object (struct ivopts_data *data, tree expr)
1152 : {
1153 10585954 : tree *slot, obj = NULL_TREE;
1154 10585954 : if (data->base_object_map)
1155 : {
1156 10418214 : if ((slot = data->base_object_map->get(expr)) != NULL)
1157 4818250 : return *slot;
1158 : }
1159 : else
1160 167740 : data->base_object_map = new hash_map<tree, tree>;
1161 :
1162 5767704 : (void) walk_tree_without_duplicates (&expr, determine_base_object_1, &obj);
1163 5767704 : data->base_object_map->put (expr, obj);
1164 5767704 : return obj;
1165 : }
1166 :
1167 : /* Allocates an induction variable with given initial value BASE and step STEP
1168 : for loop LOOP. NO_OVERFLOW implies the iv doesn't overflow. */
1169 :
1170 : static struct iv *
1171 10585954 : alloc_iv (struct ivopts_data *data, tree base, tree step,
1172 : bool no_overflow = false)
1173 : {
1174 10585954 : tree expr = base;
1175 10585954 : struct iv *iv = (struct iv*) obstack_alloc (&data->iv_obstack,
1176 : sizeof (struct iv));
1177 10585954 : gcc_assert (step != NULL_TREE);
1178 :
1179 : /* Canonicalize the address expression in base if it were an unsigned
1180 : computation. That leads to more equalities being detected and results in:
1181 :
1182 : 1) More accurate cost can be computed for address expressions;
1183 : 2) Duplicate candidates won't be created for bases in different
1184 : forms, like &a[0] and &a.
1185 : 3) Duplicate candidates won't be created for IV expressions that differ
1186 : only in their sign. */
1187 10585954 : aff_tree comb;
1188 10585954 : STRIP_NOPS (expr);
1189 10585954 : expr = fold_convert (unsigned_type_for (TREE_TYPE (expr)), expr);
1190 10585954 : tree_to_aff_combination (expr, TREE_TYPE (expr), &comb);
1191 10585954 : base = fold_convert (TREE_TYPE (base), aff_combination_to_tree (&comb));
1192 :
1193 10585954 : iv->base = base;
1194 10585954 : iv->base_object = determine_base_object (data, base);
1195 10585954 : iv->step = step;
1196 10585954 : iv->biv_p = false;
1197 10585954 : iv->nonlin_use = NULL;
1198 10585954 : iv->ssa_name = NULL_TREE;
1199 10585954 : if (!no_overflow
1200 10585954 : && !iv_can_overflow_p (data->current_loop, TREE_TYPE (base),
1201 : base, step))
1202 : no_overflow = true;
1203 10585954 : iv->no_overflow = no_overflow;
1204 10585954 : iv->have_address_use = false;
1205 :
1206 21171908 : return iv;
1207 10585954 : }
1208 :
1209 : /* Sets STEP and BASE for induction variable IV. NO_OVERFLOW implies the IV
1210 : doesn't overflow. */
1211 :
1212 : static void
1213 5016641 : set_iv (struct ivopts_data *data, tree iv, tree base, tree step,
1214 : bool no_overflow)
1215 : {
1216 5016641 : struct version_info *info = name_info (data, iv);
1217 :
1218 5016641 : gcc_assert (!info->iv);
1219 :
1220 5016641 : bitmap_set_bit (data->relevant, SSA_NAME_VERSION (iv));
1221 5016641 : info->iv = alloc_iv (data, base, step, no_overflow);
1222 5016641 : info->iv->ssa_name = iv;
1223 5016641 : }
1224 :
1225 : /* Finds induction variable declaration for VAR. */
1226 :
1227 : static struct iv *
1228 45078174 : get_iv (struct ivopts_data *data, tree var)
1229 : {
1230 45078174 : basic_block bb;
1231 45078174 : tree type = TREE_TYPE (var);
1232 :
1233 45078174 : if (!POINTER_TYPE_P (type)
1234 35804402 : && !INTEGRAL_TYPE_P (type))
1235 : return NULL;
1236 :
1237 39313330 : if (!name_info (data, var)->iv)
1238 : {
1239 18207434 : bb = gimple_bb (SSA_NAME_DEF_STMT (var));
1240 :
1241 18207434 : if (!bb
1242 18207434 : || !flow_bb_inside_loop_p (data->current_loop, bb))
1243 : {
1244 821477 : if (POINTER_TYPE_P (type))
1245 328905 : type = sizetype;
1246 821477 : set_iv (data, var, var, build_int_cst (type, 0), true);
1247 : }
1248 : }
1249 :
1250 39313330 : return name_info (data, var)->iv;
1251 : }
1252 :
1253 : /* Return the first non-invariant ssa var found in EXPR. */
1254 :
1255 : static tree
1256 4168160 : extract_single_var_from_expr (tree expr)
1257 : {
1258 4168160 : int i, n;
1259 4168160 : tree tmp;
1260 4168160 : enum tree_code code;
1261 :
1262 4168160 : if (!expr || is_gimple_min_invariant (expr))
1263 : return NULL;
1264 :
1265 677614 : code = TREE_CODE (expr);
1266 677614 : if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code)))
1267 : {
1268 366212 : n = TREE_OPERAND_LENGTH (expr);
1269 732471 : for (i = 0; i < n; i++)
1270 : {
1271 366259 : tmp = extract_single_var_from_expr (TREE_OPERAND (expr, i));
1272 :
1273 366259 : if (tmp)
1274 : return tmp;
1275 : }
1276 : }
1277 311402 : return (TREE_CODE (expr) == SSA_NAME) ? expr : NULL;
1278 : }
1279 :
1280 : /* Finds basic ivs. */
1281 :
1282 : static bool
1283 643678 : find_bivs (struct ivopts_data *data)
1284 : {
1285 643678 : gphi *phi;
1286 643678 : affine_iv iv;
1287 643678 : tree step, type, base, stop;
1288 643678 : bool found = false;
1289 643678 : class loop *loop = data->current_loop;
1290 643678 : gphi_iterator psi;
1291 :
1292 2401151 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
1293 : {
1294 1757473 : phi = psi.phi ();
1295 :
1296 1757473 : if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
1297 248 : continue;
1298 :
1299 1757225 : if (virtual_operand_p (PHI_RESULT (phi)))
1300 421840 : continue;
1301 :
1302 1335385 : if (!simple_iv (loop, loop, PHI_RESULT (phi), &iv, true))
1303 445807 : continue;
1304 :
1305 889578 : if (integer_zerop (iv.step))
1306 0 : continue;
1307 :
1308 889578 : step = iv.step;
1309 889578 : base = PHI_ARG_DEF_FROM_EDGE (phi, loop_preheader_edge (loop));
1310 : /* Stop expanding iv base at the first ssa var referred by iv step.
1311 : Ideally we should stop at any ssa var, because that's expensive
1312 : and unusual to happen, we just do it on the first one.
1313 :
1314 : See PR64705 for the rationale. */
1315 889578 : stop = extract_single_var_from_expr (step);
1316 889578 : base = expand_simple_operations (base, stop);
1317 889578 : if (contains_abnormal_ssa_name_p (base)
1318 889578 : || contains_abnormal_ssa_name_p (step))
1319 11 : continue;
1320 :
1321 889567 : type = TREE_TYPE (PHI_RESULT (phi));
1322 889567 : base = fold_convert (type, base);
1323 889567 : if (step)
1324 : {
1325 889567 : if (POINTER_TYPE_P (type))
1326 169252 : step = convert_to_ptrofftype (step);
1327 : else
1328 720315 : step = fold_convert (type, step);
1329 : }
1330 :
1331 889567 : set_iv (data, PHI_RESULT (phi), base, step, iv.no_overflow);
1332 889567 : found = true;
1333 : }
1334 :
1335 643678 : return found;
1336 : }
1337 :
1338 : /* Marks basic ivs. */
1339 :
1340 : static void
1341 513182 : mark_bivs (struct ivopts_data *data)
1342 : {
1343 513182 : gphi *phi;
1344 513182 : gimple *def;
1345 513182 : tree var;
1346 513182 : struct iv *iv, *incr_iv;
1347 513182 : class loop *loop = data->current_loop;
1348 513182 : basic_block incr_bb;
1349 513182 : gphi_iterator psi;
1350 :
1351 513182 : data->bivs_not_used_in_addr = 0;
1352 1995231 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
1353 : {
1354 1482049 : phi = psi.phi ();
1355 :
1356 1482049 : iv = get_iv (data, PHI_RESULT (phi));
1357 1482049 : if (!iv)
1358 592482 : continue;
1359 :
1360 889567 : var = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (loop));
1361 889567 : def = SSA_NAME_DEF_STMT (var);
1362 : /* Don't mark iv peeled from other one as biv. */
1363 889690 : if (def
1364 889567 : && gimple_code (def) == GIMPLE_PHI
1365 890929 : && gimple_bb (def) == loop->header)
1366 123 : continue;
1367 :
1368 889444 : incr_iv = get_iv (data, var);
1369 889444 : if (!incr_iv)
1370 1250 : continue;
1371 :
1372 : /* If the increment is in the subloop, ignore it. */
1373 888194 : incr_bb = gimple_bb (SSA_NAME_DEF_STMT (var));
1374 888194 : if (incr_bb->loop_father != data->current_loop
1375 888194 : || (incr_bb->flags & BB_IRREDUCIBLE_LOOP))
1376 0 : continue;
1377 :
1378 888194 : iv->biv_p = true;
1379 888194 : incr_iv->biv_p = true;
1380 888194 : if (iv->no_overflow)
1381 593987 : data->bivs_not_used_in_addr++;
1382 888194 : if (incr_iv->no_overflow)
1383 585394 : data->bivs_not_used_in_addr++;
1384 : }
1385 513182 : }
1386 :
1387 : /* Checks whether STMT defines a linear induction variable and stores its
1388 : parameters to IV. */
1389 :
1390 : static bool
1391 12842442 : find_givs_in_stmt_scev (struct ivopts_data *data, gimple *stmt, affine_iv *iv)
1392 : {
1393 12842442 : tree lhs, stop;
1394 12842442 : class loop *loop = data->current_loop;
1395 :
1396 12842442 : iv->base = NULL_TREE;
1397 12842442 : iv->step = NULL_TREE;
1398 :
1399 12842442 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1400 : return false;
1401 :
1402 10765173 : lhs = gimple_assign_lhs (stmt);
1403 10765173 : if (TREE_CODE (lhs) != SSA_NAME)
1404 : return false;
1405 :
1406 19142468 : if (!simple_iv (loop, loop_containing_stmt (stmt), lhs, iv, true))
1407 : return false;
1408 :
1409 : /* Stop expanding iv base at the first ssa var referred by iv step.
1410 : Ideally we should stop at any ssa var, because that's expensive
1411 : and unusual to happen, we just do it on the first one.
1412 :
1413 : See PR64705 for the rationale. */
1414 2912323 : stop = extract_single_var_from_expr (iv->step);
1415 2912323 : iv->base = expand_simple_operations (iv->base, stop);
1416 2912323 : if (contains_abnormal_ssa_name_p (iv->base)
1417 2912323 : || contains_abnormal_ssa_name_p (iv->step))
1418 : return false;
1419 :
1420 : /* If STMT could throw, then do not consider STMT as defining a GIV.
1421 : While this will suppress optimizations, we cannot safely delete this
1422 : GIV and associated statements, even if it appears it is not used. */
1423 2912319 : if (stmt_could_throw_p (cfun, stmt))
1424 8 : return false;
1425 :
1426 : return true;
1427 : }
1428 :
1429 : /* Finds general ivs in statement STMT. */
1430 :
1431 : static void
1432 12842442 : find_givs_in_stmt (struct ivopts_data *data, gimple *stmt)
1433 : {
1434 12842442 : affine_iv iv;
1435 :
1436 12842442 : if (!find_givs_in_stmt_scev (data, stmt, &iv))
1437 : return;
1438 :
1439 2912311 : set_iv (data, gimple_assign_lhs (stmt), iv.base, iv.step, iv.no_overflow);
1440 : }
1441 :
1442 : /* Finds general ivs in basic block BB. */
1443 :
1444 : static void
1445 2858946 : find_givs_in_bb (struct ivopts_data *data, basic_block bb)
1446 : {
1447 2858946 : gimple_stmt_iterator bsi;
1448 :
1449 28330281 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
1450 22612389 : if (!is_gimple_debug (gsi_stmt (bsi)))
1451 12842442 : find_givs_in_stmt (data, gsi_stmt (bsi));
1452 2858946 : }
1453 :
1454 : /* Finds general ivs. */
1455 :
1456 : static void
1457 513182 : find_givs (struct ivopts_data *data, basic_block *body)
1458 : {
1459 513182 : class loop *loop = data->current_loop;
1460 513182 : unsigned i;
1461 :
1462 3372128 : for (i = 0; i < loop->num_nodes; i++)
1463 2858946 : find_givs_in_bb (data, body[i]);
1464 513182 : }
1465 :
1466 : /* For each ssa name defined in LOOP determines whether it is an induction
1467 : variable and if so, its initial value and step. */
1468 :
1469 : static bool
1470 643678 : find_induction_variables (struct ivopts_data *data, basic_block *body)
1471 : {
1472 643678 : unsigned i;
1473 643678 : bitmap_iterator bi;
1474 :
1475 643678 : if (!find_bivs (data))
1476 : return false;
1477 :
1478 513182 : find_givs (data, body);
1479 513182 : mark_bivs (data);
1480 :
1481 513182 : if (dump_file && (dump_flags & TDF_DETAILS))
1482 : {
1483 72 : class tree_niter_desc *niter = niter_for_single_dom_exit (data);
1484 :
1485 72 : if (niter)
1486 : {
1487 56 : fprintf (dump_file, " number of iterations ");
1488 56 : print_generic_expr (dump_file, niter->niter, TDF_SLIM);
1489 56 : if (!integer_zerop (niter->may_be_zero))
1490 : {
1491 2 : fprintf (dump_file, "; zero if ");
1492 2 : print_generic_expr (dump_file, niter->may_be_zero, TDF_SLIM);
1493 : }
1494 56 : fprintf (dump_file, "\n");
1495 72 : };
1496 :
1497 72 : fprintf (dump_file, "\n<Induction Vars>:\n");
1498 895 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
1499 : {
1500 823 : struct version_info *info = ver_info (data, i);
1501 823 : if (info->iv && info->iv->step && !integer_zerop (info->iv->step))
1502 573 : dump_iv (dump_file, ver_info (data, i)->iv, true, 0);
1503 : }
1504 : }
1505 :
1506 : return true;
1507 : }
1508 :
1509 : /* Records a use of TYPE at *USE_P in STMT whose value is IV in GROUP.
1510 : For address type use, ADDR_BASE is the stripped IV base, ADDR_OFFSET
1511 : is the const offset stripped from IV base and MEM_TYPE is the type
1512 : of the memory being addressed. For uses of other types, ADDR_BASE
1513 : and ADDR_OFFSET are zero by default and MEM_TYPE is NULL_TREE. */
1514 :
1515 : static struct iv_use *
1516 2103543 : record_use (struct iv_group *group, tree *use_p, struct iv *iv,
1517 : gimple *stmt, enum use_type type, tree mem_type,
1518 : tree addr_base, poly_uint64 addr_offset)
1519 : {
1520 2103543 : struct iv_use *use = XCNEW (struct iv_use);
1521 :
1522 2103543 : use->id = group->vuses.length ();
1523 2103543 : use->group_id = group->id;
1524 2103543 : use->type = type;
1525 2103543 : use->mem_type = mem_type;
1526 2103543 : use->iv = iv;
1527 2103543 : use->stmt = stmt;
1528 2103543 : use->op_p = use_p;
1529 2103543 : use->addr_base = addr_base;
1530 2103543 : use->addr_offset = addr_offset;
1531 :
1532 2103543 : group->vuses.safe_push (use);
1533 2103543 : return use;
1534 : }
1535 :
1536 : /* Checks whether OP is a loop-level invariant and if so, records it.
1537 : NONLINEAR_USE is true if the invariant is used in a way we do not
1538 : handle specially. */
1539 :
1540 : static void
1541 23080986 : record_invariant (struct ivopts_data *data, tree op, bool nonlinear_use)
1542 : {
1543 23080986 : basic_block bb;
1544 23080986 : struct version_info *info;
1545 :
1546 23080986 : if (TREE_CODE (op) != SSA_NAME
1547 23080986 : || virtual_operand_p (op))
1548 : return;
1549 :
1550 21874381 : bb = gimple_bb (SSA_NAME_DEF_STMT (op));
1551 21874381 : if (bb
1552 21874381 : && flow_bb_inside_loop_p (data->current_loop, bb))
1553 : return;
1554 :
1555 3971685 : info = name_info (data, op);
1556 3971685 : info->name = op;
1557 3971685 : info->has_nonlin_use |= nonlinear_use;
1558 3971685 : if (!info->inv_id)
1559 1358572 : info->inv_id = ++data->max_inv_var_id;
1560 3971685 : bitmap_set_bit (data->relevant, SSA_NAME_VERSION (op));
1561 : }
1562 :
1563 : /* Record a group of TYPE. */
1564 :
1565 : static struct iv_group *
1566 1828985 : record_group (struct ivopts_data *data, enum use_type type)
1567 : {
1568 1828985 : struct iv_group *group = XCNEW (struct iv_group);
1569 :
1570 1828985 : group->id = data->vgroups.length ();
1571 1828985 : group->type = type;
1572 1828985 : group->related_cands = BITMAP_ALLOC (NULL);
1573 1828985 : group->vuses.create (1);
1574 1828985 : group->doloop_p = false;
1575 :
1576 1828985 : data->vgroups.safe_push (group);
1577 1828985 : return group;
1578 : }
1579 :
1580 : /* Record a use of TYPE at *USE_P in STMT whose value is IV in a group.
1581 : New group will be created if there is no existing group for the use.
1582 : MEM_TYPE is the type of memory being addressed, or NULL if this
1583 : isn't an address reference. */
1584 :
1585 : static struct iv_use *
1586 2103543 : record_group_use (struct ivopts_data *data, tree *use_p,
1587 : struct iv *iv, gimple *stmt, enum use_type type,
1588 : tree mem_type)
1589 : {
1590 2103543 : tree addr_base = NULL;
1591 2103543 : struct iv_group *group = NULL;
1592 2103543 : poly_uint64 addr_offset = 0;
1593 :
1594 : /* Record non address type use in a new group. */
1595 2103543 : if (address_p (type))
1596 : {
1597 866015 : unsigned int i;
1598 :
1599 866015 : gcc_assert (POINTER_TYPE_P (TREE_TYPE (iv->base)));
1600 866015 : tree addr_toffset;
1601 866015 : split_constant_offset (iv->base, &addr_base, &addr_toffset);
1602 866015 : addr_offset = int_cst_value (addr_toffset);
1603 1628873 : for (i = 0; i < data->vgroups.length (); i++)
1604 : {
1605 1090043 : struct iv_use *use;
1606 :
1607 1090043 : group = data->vgroups[i];
1608 1090043 : use = group->vuses[0];
1609 1090043 : if (!address_p (use->type))
1610 335483 : continue;
1611 :
1612 : /* Check if it has the same stripped base and step. */
1613 754560 : if (operand_equal_p (iv->base_object, use->iv->base_object, 0)
1614 394811 : && operand_equal_p (iv->step, use->iv->step, OEP_ASSUME_WRAPV)
1615 1146200 : && operand_equal_p (addr_base, use->addr_base, OEP_ASSUME_WRAPV))
1616 : break;
1617 : }
1618 1732030 : if (i == data->vgroups.length ())
1619 538830 : group = NULL;
1620 : }
1621 :
1622 866015 : if (!group)
1623 1776358 : group = record_group (data, type);
1624 :
1625 2103543 : return record_use (group, use_p, iv, stmt, type, mem_type,
1626 2103543 : addr_base, addr_offset);
1627 : }
1628 :
1629 : /* Checks whether the use OP is interesting and if so, records it. */
1630 :
1631 : static struct iv_use *
1632 7519758 : find_interesting_uses_op (struct ivopts_data *data, tree op)
1633 : {
1634 7519758 : struct iv *iv;
1635 7519758 : gimple *stmt;
1636 7519758 : struct iv_use *use;
1637 :
1638 7519758 : if (TREE_CODE (op) != SSA_NAME)
1639 : return NULL;
1640 :
1641 5960395 : iv = get_iv (data, op);
1642 5960395 : if (!iv)
1643 : return NULL;
1644 :
1645 2637746 : if (iv->nonlin_use)
1646 : {
1647 213086 : gcc_assert (iv->nonlin_use->type == USE_NONLINEAR_EXPR);
1648 : return iv->nonlin_use;
1649 : }
1650 :
1651 2424660 : if (integer_zerop (iv->step))
1652 : {
1653 1793464 : record_invariant (data, op, true);
1654 1793464 : return NULL;
1655 : }
1656 :
1657 631196 : stmt = SSA_NAME_DEF_STMT (op);
1658 631196 : gcc_assert (gimple_code (stmt) == GIMPLE_PHI || is_gimple_assign (stmt));
1659 :
1660 631196 : use = record_group_use (data, NULL, iv, stmt, USE_NONLINEAR_EXPR, NULL_TREE);
1661 631196 : iv->nonlin_use = use;
1662 631196 : return use;
1663 : }
1664 :
1665 : /* Indicate how compare type iv_use can be handled. */
1666 : enum comp_iv_rewrite
1667 : {
1668 : COMP_IV_NA,
1669 : /* We may rewrite compare type iv_use by expressing value of the iv_use. */
1670 : COMP_IV_EXPR,
1671 : /* We may rewrite compare type iv_uses on both sides of comparison by
1672 : expressing value of each iv_use. */
1673 : COMP_IV_EXPR_2,
1674 : /* We may rewrite compare type iv_use by expressing value of the iv_use
1675 : or by eliminating it with other iv_cand. */
1676 : COMP_IV_ELIM
1677 : };
1678 :
1679 : /* Given a condition in statement STMT, checks whether it is a compare
1680 : of an induction variable and an invariant. If this is the case,
1681 : CONTROL_VAR is set to location of the iv, BOUND to the location of
1682 : the invariant, IV_VAR and IV_BOUND are set to the corresponding
1683 : induction variable descriptions, and true is returned. If this is not
1684 : the case, CONTROL_VAR and BOUND are set to the arguments of the
1685 : condition and false is returned. */
1686 :
1687 : static enum comp_iv_rewrite
1688 7513629 : extract_cond_operands (struct ivopts_data *data, gimple *stmt,
1689 : tree **control_var, tree **bound,
1690 : struct iv **iv_var, struct iv **iv_bound)
1691 : {
1692 : /* The objects returned when COND has constant operands. */
1693 7513629 : static struct iv const_iv;
1694 7513629 : static tree zero;
1695 7513629 : tree *op0 = &zero, *op1 = &zero;
1696 7513629 : struct iv *iv0 = &const_iv, *iv1 = &const_iv;
1697 7513629 : enum comp_iv_rewrite rewrite_type = COMP_IV_NA;
1698 :
1699 7513629 : if (gimple_code (stmt) == GIMPLE_COND)
1700 : {
1701 7231632 : gcond *cond_stmt = as_a <gcond *> (stmt);
1702 7231632 : op0 = gimple_cond_lhs_ptr (cond_stmt);
1703 7231632 : op1 = gimple_cond_rhs_ptr (cond_stmt);
1704 : }
1705 : else
1706 : {
1707 281997 : op0 = gimple_assign_rhs1_ptr (stmt);
1708 281997 : op1 = gimple_assign_rhs2_ptr (stmt);
1709 : }
1710 :
1711 7513629 : zero = integer_zero_node;
1712 7513629 : const_iv.step = integer_zero_node;
1713 :
1714 7513629 : if (TREE_CODE (*op0) == SSA_NAME)
1715 7513472 : iv0 = get_iv (data, *op0);
1716 7513629 : if (TREE_CODE (*op1) == SSA_NAME)
1717 3330749 : iv1 = get_iv (data, *op1);
1718 :
1719 : /* If both sides of comparison are IVs. We can express ivs on both end. */
1720 7513629 : if (iv0 && iv1 && !integer_zerop (iv0->step) && !integer_zerop (iv1->step))
1721 : {
1722 84289 : rewrite_type = COMP_IV_EXPR_2;
1723 84289 : goto end;
1724 : }
1725 :
1726 : /* If none side of comparison is IV. */
1727 5828751 : if ((!iv0 || integer_zerop (iv0->step))
1728 8776808 : && (!iv1 || integer_zerop (iv1->step)))
1729 962684 : goto end;
1730 :
1731 : /* Control variable may be on the other side. */
1732 6466656 : if (!iv0 || integer_zerop (iv0->step))
1733 : {
1734 : std::swap (op0, op1);
1735 : std::swap (iv0, iv1);
1736 : }
1737 : /* If one side is IV and the other side isn't loop invariant. */
1738 6466656 : if (!iv1)
1739 : rewrite_type = COMP_IV_EXPR;
1740 : /* If one side is IV and the other side is loop invariant. */
1741 5505021 : else if (!integer_zerop (iv0->step) && integer_zerop (iv1->step))
1742 : rewrite_type = COMP_IV_ELIM;
1743 :
1744 7513629 : end:
1745 7513629 : if (control_var)
1746 7513629 : *control_var = op0;
1747 7513629 : if (iv_var)
1748 1567007 : *iv_var = iv0;
1749 7513629 : if (bound)
1750 7513629 : *bound = op1;
1751 7513629 : if (iv_bound)
1752 7513629 : *iv_bound = iv1;
1753 :
1754 7513629 : return rewrite_type;
1755 : }
1756 :
1757 : /* Checks whether the condition in STMT is interesting and if so,
1758 : records it. */
1759 :
1760 : static void
1761 1567007 : find_interesting_uses_cond (struct ivopts_data *data, gimple *stmt)
1762 : {
1763 1567007 : tree *var_p, *bound_p;
1764 1567007 : struct iv *var_iv, *bound_iv;
1765 1567007 : enum comp_iv_rewrite ret;
1766 :
1767 1567007 : ret = extract_cond_operands (data, stmt,
1768 : &var_p, &bound_p, &var_iv, &bound_iv);
1769 1567007 : if (ret == COMP_IV_NA)
1770 : {
1771 962684 : find_interesting_uses_op (data, *var_p);
1772 962684 : find_interesting_uses_op (data, *bound_p);
1773 962684 : return;
1774 : }
1775 :
1776 604323 : record_group_use (data, var_p, var_iv, stmt, USE_COMPARE, NULL_TREE);
1777 : /* Record compare type iv_use for iv on the other side of comparison. */
1778 604323 : if (ret == COMP_IV_EXPR_2)
1779 2009 : record_group_use (data, bound_p, bound_iv, stmt, USE_COMPARE, NULL_TREE);
1780 : }
1781 :
1782 : /* Returns the outermost loop EXPR is obviously invariant in
1783 : relative to the loop LOOP, i.e. if all its operands are defined
1784 : outside of the returned loop. Returns NULL if EXPR is not
1785 : even obviously invariant in LOOP. */
1786 :
1787 : class loop *
1788 383978 : outermost_invariant_loop_for_expr (class loop *loop, tree expr)
1789 : {
1790 383978 : basic_block def_bb;
1791 383978 : unsigned i, len;
1792 :
1793 383978 : if (is_gimple_min_invariant (expr))
1794 51557 : return current_loops->tree_root;
1795 :
1796 332421 : if (TREE_CODE (expr) == SSA_NAME)
1797 : {
1798 196685 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (expr));
1799 196685 : if (def_bb)
1800 : {
1801 117125 : if (flow_bb_inside_loop_p (loop, def_bb))
1802 : return NULL;
1803 234234 : return superloop_at_depth (loop,
1804 160669 : loop_depth (def_bb->loop_father) + 1);
1805 : }
1806 :
1807 79560 : return current_loops->tree_root;
1808 : }
1809 :
1810 135736 : if (!EXPR_P (expr))
1811 : return NULL;
1812 :
1813 135736 : unsigned maxdepth = 0;
1814 135736 : len = TREE_OPERAND_LENGTH (expr);
1815 353094 : for (i = 0; i < len; i++)
1816 : {
1817 217382 : class loop *ivloop;
1818 217382 : if (!TREE_OPERAND (expr, i))
1819 0 : continue;
1820 :
1821 217382 : ivloop = outermost_invariant_loop_for_expr (loop, TREE_OPERAND (expr, i));
1822 217382 : if (!ivloop)
1823 : return NULL;
1824 563475 : maxdepth = MAX (maxdepth, loop_depth (ivloop));
1825 : }
1826 :
1827 135712 : return superloop_at_depth (loop, maxdepth);
1828 : }
1829 :
1830 : /* Returns true if expression EXPR is obviously invariant in LOOP,
1831 : i.e. if all its operands are defined outside of the LOOP. LOOP
1832 : should not be the function body. */
1833 :
1834 : bool
1835 12462155 : expr_invariant_in_loop_p (class loop *loop, tree expr)
1836 : {
1837 12462155 : basic_block def_bb;
1838 12462155 : unsigned i, len;
1839 :
1840 12462155 : gcc_assert (loop_depth (loop) > 0);
1841 :
1842 12462155 : if (is_gimple_min_invariant (expr))
1843 : return true;
1844 :
1845 8636052 : if (TREE_CODE (expr) == SSA_NAME)
1846 : {
1847 8216438 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (expr));
1848 8216438 : if (def_bb
1849 8216438 : && flow_bb_inside_loop_p (loop, def_bb))
1850 3936926 : return false;
1851 :
1852 : return true;
1853 : }
1854 :
1855 419614 : if (!EXPR_P (expr))
1856 : return false;
1857 :
1858 419611 : len = TREE_OPERAND_LENGTH (expr);
1859 924521 : for (i = 0; i < len; i++)
1860 551181 : if (TREE_OPERAND (expr, i)
1861 551181 : && !expr_invariant_in_loop_p (loop, TREE_OPERAND (expr, i)))
1862 : return false;
1863 :
1864 : return true;
1865 : }
1866 :
1867 : /* Given expression EXPR which computes inductive values with respect
1868 : to loop recorded in DATA, this function returns biv from which EXPR
1869 : is derived by tracing definition chains of ssa variables in EXPR. */
1870 :
1871 : static struct iv*
1872 876807 : find_deriving_biv_for_expr (struct ivopts_data *data, tree expr)
1873 : {
1874 1418657 : struct iv *iv;
1875 1418657 : unsigned i, n;
1876 1418657 : tree e2, e1;
1877 1418657 : enum tree_code code;
1878 1418657 : gimple *stmt;
1879 :
1880 1418657 : if (expr == NULL_TREE)
1881 : return NULL;
1882 :
1883 1418409 : if (is_gimple_min_invariant (expr))
1884 : return NULL;
1885 :
1886 1133025 : code = TREE_CODE (expr);
1887 1133025 : if (IS_EXPR_CODE_CLASS (TREE_CODE_CLASS (code)))
1888 : {
1889 23192 : n = TREE_OPERAND_LENGTH (expr);
1890 24907 : for (i = 0; i < n; i++)
1891 : {
1892 24500 : iv = find_deriving_biv_for_expr (data, TREE_OPERAND (expr, i));
1893 24500 : if (iv)
1894 : return iv;
1895 : }
1896 : }
1897 :
1898 : /* Stop if it's not ssa name. */
1899 1110240 : if (code != SSA_NAME)
1900 : return NULL;
1901 :
1902 1109321 : iv = get_iv (data, expr);
1903 1109321 : if (!iv || integer_zerop (iv->step))
1904 : return NULL;
1905 1061944 : else if (iv->biv_p)
1906 : return iv;
1907 :
1908 790128 : stmt = SSA_NAME_DEF_STMT (expr);
1909 790128 : if (gphi *phi = dyn_cast <gphi *> (stmt))
1910 : {
1911 59 : ssa_op_iter iter;
1912 59 : use_operand_p use_p;
1913 59 : basic_block phi_bb = gimple_bb (phi);
1914 :
1915 : /* Skip loop header PHI that doesn't define biv. */
1916 59 : if (phi_bb->loop_father == data->current_loop)
1917 : return NULL;
1918 :
1919 0 : if (virtual_operand_p (gimple_phi_result (phi)))
1920 : return NULL;
1921 :
1922 0 : FOR_EACH_PHI_ARG (use_p, phi, iter, SSA_OP_USE)
1923 : {
1924 0 : tree use = USE_FROM_PTR (use_p);
1925 0 : iv = find_deriving_biv_for_expr (data, use);
1926 0 : if (iv)
1927 : return iv;
1928 : }
1929 : return NULL;
1930 : }
1931 790069 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
1932 : return NULL;
1933 :
1934 790069 : e1 = gimple_assign_rhs1 (stmt);
1935 790069 : code = gimple_assign_rhs_code (stmt);
1936 790069 : if (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS)
1937 : return find_deriving_biv_for_expr (data, e1);
1938 :
1939 781148 : switch (code)
1940 : {
1941 579320 : case MULT_EXPR:
1942 579320 : case PLUS_EXPR:
1943 579320 : case MINUS_EXPR:
1944 579320 : case POINTER_PLUS_EXPR:
1945 : /* Increments, decrements and multiplications by a constant
1946 : are simple. */
1947 579320 : e2 = gimple_assign_rhs2 (stmt);
1948 579320 : iv = find_deriving_biv_for_expr (data, e2);
1949 579320 : if (iv)
1950 : return iv;
1951 : gcc_fallthrough ();
1952 :
1953 : CASE_CONVERT:
1954 : /* Casts are simple. */
1955 : return find_deriving_biv_for_expr (data, e1);
1956 :
1957 : default:
1958 : break;
1959 : }
1960 :
1961 : return NULL;
1962 : }
1963 :
1964 : /* Record BIV, its predecessor and successor that they are used in
1965 : address type uses. */
1966 :
1967 : static void
1968 598415 : record_biv_for_address_use (struct ivopts_data *data, struct iv *biv)
1969 : {
1970 598415 : unsigned i;
1971 598415 : tree type, base_1, base_2;
1972 598415 : bitmap_iterator bi;
1973 :
1974 597244 : if (!biv || !biv->biv_p || integer_zerop (biv->step)
1975 1195659 : || biv->have_address_use || !biv->no_overflow)
1976 330346 : return;
1977 :
1978 533766 : type = TREE_TYPE (biv->base);
1979 533766 : if (!INTEGRAL_TYPE_P (type))
1980 : return;
1981 :
1982 268069 : biv->have_address_use = true;
1983 268069 : data->bivs_not_used_in_addr--;
1984 268069 : base_1 = fold_build2 (PLUS_EXPR, type, biv->base, biv->step);
1985 2458634 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
1986 : {
1987 2190565 : struct iv *iv = ver_info (data, i)->iv;
1988 :
1989 1977357 : if (!iv || !iv->biv_p || integer_zerop (iv->step)
1990 3103791 : || iv->have_address_use || !iv->no_overflow)
1991 1895249 : continue;
1992 :
1993 295316 : if (type != TREE_TYPE (iv->base)
1994 295316 : || !INTEGRAL_TYPE_P (TREE_TYPE (iv->base)))
1995 30527 : continue;
1996 :
1997 264789 : if (!operand_equal_p (biv->step, iv->step, 0))
1998 5919 : continue;
1999 :
2000 258870 : base_2 = fold_build2 (PLUS_EXPR, type, iv->base, iv->step);
2001 258870 : if (operand_equal_p (base_1, iv->base, 0)
2002 258870 : || operand_equal_p (base_2, biv->base, 0))
2003 : {
2004 233280 : iv->have_address_use = true;
2005 233280 : data->bivs_not_used_in_addr--;
2006 : }
2007 : }
2008 : }
2009 :
2010 : /* Cumulates the steps of indices into DATA and replaces their values with the
2011 : initial ones. Returns false when the value of the index cannot be determined.
2012 : Callback for for_each_index. */
2013 :
2014 : struct ifs_ivopts_data
2015 : {
2016 : struct ivopts_data *ivopts_data;
2017 : gimple *stmt;
2018 : tree step;
2019 : };
2020 :
2021 : static bool
2022 2276173 : idx_find_step (tree base, tree *idx, void *data)
2023 : {
2024 2276173 : struct ifs_ivopts_data *dta = (struct ifs_ivopts_data *) data;
2025 2276173 : struct iv *iv;
2026 2276173 : bool use_overflow_semantics = false;
2027 2276173 : tree step, iv_base, iv_step, lbound, off;
2028 2276173 : class loop *loop = dta->ivopts_data->current_loop;
2029 :
2030 : /* If base is a component ref, require that the offset of the reference
2031 : be invariant. */
2032 2276173 : if (TREE_CODE (base) == COMPONENT_REF)
2033 : {
2034 78 : off = component_ref_field_offset (base);
2035 78 : return expr_invariant_in_loop_p (loop, off);
2036 : }
2037 :
2038 : /* If base is array, first check whether we will be able to move the
2039 : reference out of the loop (in order to take its address in strength
2040 : reduction). In order for this to work we need both lower bound
2041 : and step to be loop invariants. */
2042 2276095 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2043 : {
2044 : /* Moreover, for a range, the size needs to be invariant as well. */
2045 549463 : if (TREE_CODE (base) == ARRAY_RANGE_REF
2046 549463 : && !expr_invariant_in_loop_p (loop, TYPE_SIZE (TREE_TYPE (base))))
2047 : return false;
2048 :
2049 549463 : step = array_ref_element_size (base);
2050 549463 : lbound = array_ref_low_bound (base);
2051 :
2052 549463 : if (!expr_invariant_in_loop_p (loop, step)
2053 549463 : || !expr_invariant_in_loop_p (loop, lbound))
2054 : return false;
2055 : }
2056 :
2057 2276089 : if (TREE_CODE (*idx) != SSA_NAME)
2058 : return true;
2059 :
2060 1843501 : iv = get_iv (dta->ivopts_data, *idx);
2061 1843501 : if (!iv)
2062 : return false;
2063 :
2064 : /* XXX We produce for a base of *D42 with iv->base being &x[0]
2065 : *&x[0], which is not folded and does not trigger the
2066 : ARRAY_REF path below. */
2067 1200950 : *idx = iv->base;
2068 :
2069 1200950 : if (integer_zerop (iv->step))
2070 : return true;
2071 :
2072 882272 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2073 : {
2074 308505 : step = array_ref_element_size (base);
2075 :
2076 : /* We only handle addresses whose step is an integer constant. */
2077 308505 : if (TREE_CODE (step) != INTEGER_CST)
2078 : return false;
2079 : }
2080 : else
2081 : /* The step for pointer arithmetics already is 1 byte. */
2082 573767 : step = size_one_node;
2083 :
2084 882015 : iv_base = iv->base;
2085 882015 : iv_step = iv->step;
2086 882015 : if (iv->no_overflow && nowrap_type_p (TREE_TYPE (iv_step)))
2087 : use_overflow_semantics = true;
2088 :
2089 882015 : if (!convert_affine_scev (dta->ivopts_data->current_loop,
2090 : sizetype, &iv_base, &iv_step, dta->stmt,
2091 : use_overflow_semantics))
2092 : {
2093 : /* The index might wrap. */
2094 : return false;
2095 : }
2096 :
2097 878807 : step = fold_build2 (MULT_EXPR, sizetype, step, iv_step);
2098 878807 : dta->step = fold_build2 (PLUS_EXPR, sizetype, dta->step, step);
2099 :
2100 878807 : if (dta->ivopts_data->bivs_not_used_in_addr)
2101 : {
2102 598415 : if (!iv->biv_p)
2103 272987 : iv = find_deriving_biv_for_expr (dta->ivopts_data, iv->ssa_name);
2104 :
2105 598415 : record_biv_for_address_use (dta->ivopts_data, iv);
2106 : }
2107 : return true;
2108 : }
2109 :
2110 : /* Records use in index IDX. Callback for for_each_index. Ivopts data
2111 : object is passed to it in DATA. */
2112 :
2113 : static bool
2114 1907300 : idx_record_use (tree base, tree *idx,
2115 : void *vdata)
2116 : {
2117 1907300 : struct ivopts_data *data = (struct ivopts_data *) vdata;
2118 1907300 : find_interesting_uses_op (data, *idx);
2119 1907300 : if (TREE_CODE (base) == ARRAY_REF || TREE_CODE (base) == ARRAY_RANGE_REF)
2120 : {
2121 304324 : if (TREE_OPERAND (base, 2))
2122 5464 : find_interesting_uses_op (data, TREE_OPERAND (base, 2));
2123 304324 : if (TREE_OPERAND (base, 3))
2124 11482 : find_interesting_uses_op (data, TREE_OPERAND (base, 3));
2125 : }
2126 1907300 : return true;
2127 : }
2128 :
2129 : /* If we can prove that TOP = cst * BOT for some constant cst,
2130 : store cst to MUL and return true. Otherwise return false.
2131 : The returned value is always sign-extended, regardless of the
2132 : signedness of TOP and BOT. */
2133 :
2134 : static bool
2135 17292416 : constant_multiple_of (tree top, tree bot, widest_int *mul,
2136 : struct ivopts_data *data)
2137 : {
2138 34584832 : aff_tree aff_top, aff_bot;
2139 17292416 : tree_to_aff_combination_expand (top, TREE_TYPE (top), &aff_top,
2140 : &data->name_expansion_cache);
2141 17292416 : tree_to_aff_combination_expand (bot, TREE_TYPE (bot), &aff_bot,
2142 : &data->name_expansion_cache);
2143 :
2144 17292416 : poly_widest_int poly_mul;
2145 17292416 : if (aff_combination_constant_multiple_p (&aff_top, &aff_bot, &poly_mul)
2146 17292416 : && poly_mul.is_constant (mul))
2147 14291669 : return true;
2148 :
2149 : return false;
2150 17292416 : }
2151 :
2152 : /* Return true if memory reference REF with step STEP may be unaligned. */
2153 :
2154 : static bool
2155 0 : may_be_unaligned_p (tree ref, tree step)
2156 : {
2157 : /* TARGET_MEM_REFs are translated directly to valid MEMs on the target,
2158 : thus they are not misaligned. */
2159 0 : if (TREE_CODE (ref) == TARGET_MEM_REF)
2160 : return false;
2161 :
2162 0 : unsigned int align = TYPE_ALIGN (TREE_TYPE (ref));
2163 0 : if (GET_MODE_ALIGNMENT (TYPE_MODE (TREE_TYPE (ref))) > align)
2164 0 : align = GET_MODE_ALIGNMENT (TYPE_MODE (TREE_TYPE (ref)));
2165 :
2166 0 : unsigned HOST_WIDE_INT bitpos;
2167 0 : unsigned int ref_align;
2168 0 : get_object_alignment_1 (ref, &ref_align, &bitpos);
2169 0 : if (ref_align < align
2170 0 : || (bitpos % align) != 0
2171 0 : || (bitpos % BITS_PER_UNIT) != 0)
2172 : return true;
2173 :
2174 0 : unsigned int trailing_zeros = tree_ctz (step);
2175 0 : if (trailing_zeros < HOST_BITS_PER_INT
2176 0 : && (1U << trailing_zeros) * BITS_PER_UNIT < align)
2177 0 : return true;
2178 :
2179 : return false;
2180 : }
2181 :
2182 : /* Return true if EXPR may be non-addressable. */
2183 :
2184 : bool
2185 13351150 : may_be_nonaddressable_p (tree expr)
2186 : {
2187 14383559 : switch (TREE_CODE (expr))
2188 : {
2189 9582078 : case VAR_DECL:
2190 : /* Check if it's a register variable. */
2191 9582078 : return DECL_HARD_REGISTER (expr);
2192 :
2193 : case TARGET_MEM_REF:
2194 : /* TARGET_MEM_REFs are translated directly to valid MEMs on the
2195 : target, thus they are always addressable. */
2196 : return false;
2197 :
2198 2050777 : case MEM_REF:
2199 : /* Likewise for MEM_REFs, modulo the storage order. */
2200 2050777 : return REF_REVERSE_STORAGE_ORDER (expr);
2201 :
2202 76 : case BIT_FIELD_REF:
2203 76 : if (REF_REVERSE_STORAGE_ORDER (expr))
2204 : return true;
2205 76 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2206 :
2207 1254488 : case COMPONENT_REF:
2208 1254488 : if (TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_OPERAND (expr, 0))))
2209 : return true;
2210 1254475 : return DECL_NONADDRESSABLE_P (TREE_OPERAND (expr, 1))
2211 1254475 : || may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2212 :
2213 896731 : case ARRAY_REF:
2214 896731 : case ARRAY_RANGE_REF:
2215 896731 : if (TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_OPERAND (expr, 0))))
2216 : return true;
2217 896731 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2218 :
2219 24081 : case VIEW_CONVERT_EXPR:
2220 : /* This kind of view-conversions may wrap non-addressable objects
2221 : and make them look addressable. After some processing the
2222 : non-addressability may be uncovered again, causing ADDR_EXPRs
2223 : of inappropriate objects to be built. */
2224 24081 : if (is_gimple_reg (TREE_OPERAND (expr, 0))
2225 24081 : || !is_gimple_addressable (TREE_OPERAND (expr, 0)))
2226 : return true;
2227 24071 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2228 :
2229 111531 : case REALPART_EXPR:
2230 111531 : case IMAGPART_EXPR:
2231 111531 : return may_be_nonaddressable_p (TREE_OPERAND (expr, 0));
2232 :
2233 : CASE_CONVERT:
2234 : return true;
2235 :
2236 : default:
2237 : break;
2238 : }
2239 :
2240 : return false;
2241 : }
2242 :
2243 : /* Finds addresses in *OP_P inside STMT. */
2244 :
2245 : static void
2246 2796245 : find_interesting_uses_address (struct ivopts_data *data, gimple *stmt,
2247 : tree *op_p)
2248 : {
2249 2796245 : tree base = *op_p, step = size_zero_node;
2250 2796245 : struct iv *civ;
2251 2796245 : struct ifs_ivopts_data ifs_ivopts_data;
2252 :
2253 : /* Do not play with volatile memory references. A bit too conservative,
2254 : perhaps, but safe. */
2255 5592490 : if (gimple_has_volatile_ops (stmt))
2256 66258 : goto fail;
2257 :
2258 : /* Ignore bitfields for now. Not really something terribly complicated
2259 : to handle. TODO. */
2260 2729987 : if (TREE_CODE (base) == BIT_FIELD_REF)
2261 93278 : goto fail;
2262 :
2263 2636709 : base = unshare_expr (base);
2264 :
2265 2636709 : if (TREE_CODE (base) == TARGET_MEM_REF)
2266 : {
2267 312252 : tree type = build_pointer_type (TREE_TYPE (base));
2268 312252 : tree astep;
2269 :
2270 312252 : if (TMR_BASE (base)
2271 312252 : && TREE_CODE (TMR_BASE (base)) == SSA_NAME)
2272 : {
2273 290716 : civ = get_iv (data, TMR_BASE (base));
2274 290716 : if (!civ)
2275 256555 : goto fail;
2276 :
2277 34161 : TMR_BASE (base) = civ->base;
2278 34161 : step = civ->step;
2279 : }
2280 55697 : if (TMR_INDEX2 (base)
2281 55697 : && TREE_CODE (TMR_INDEX2 (base)) == SSA_NAME)
2282 : {
2283 13730 : civ = get_iv (data, TMR_INDEX2 (base));
2284 13730 : if (!civ)
2285 4917 : goto fail;
2286 :
2287 8813 : TMR_INDEX2 (base) = civ->base;
2288 8813 : step = civ->step;
2289 : }
2290 50780 : if (TMR_INDEX (base)
2291 50780 : && TREE_CODE (TMR_INDEX (base)) == SSA_NAME)
2292 : {
2293 50780 : civ = get_iv (data, TMR_INDEX (base));
2294 50780 : if (!civ)
2295 50780 : goto fail;
2296 :
2297 0 : TMR_INDEX (base) = civ->base;
2298 0 : astep = civ->step;
2299 :
2300 0 : if (astep)
2301 : {
2302 0 : if (TMR_STEP (base))
2303 0 : astep = fold_build2 (MULT_EXPR, type, TMR_STEP (base), astep);
2304 :
2305 0 : step = fold_build2 (PLUS_EXPR, type, step, astep);
2306 : }
2307 : }
2308 :
2309 0 : if (integer_zerop (step))
2310 0 : goto fail;
2311 0 : base = tree_mem_ref_addr (type, base);
2312 : }
2313 : else
2314 : {
2315 2324457 : ifs_ivopts_data.ivopts_data = data;
2316 2324457 : ifs_ivopts_data.stmt = stmt;
2317 2324457 : ifs_ivopts_data.step = size_zero_node;
2318 2324457 : if (!for_each_index (&base, idx_find_step, &ifs_ivopts_data)
2319 2324457 : || integer_zerop (ifs_ivopts_data.step))
2320 1447428 : goto fail;
2321 877029 : step = ifs_ivopts_data.step;
2322 :
2323 : /* Check that the base expression is addressable. This needs
2324 : to be done after substituting bases of IVs into it. */
2325 877029 : if (may_be_nonaddressable_p (base))
2326 779 : goto fail;
2327 :
2328 : /* Moreover, on strict alignment platforms, check that it is
2329 : sufficiently aligned. */
2330 876250 : if (STRICT_ALIGNMENT && may_be_unaligned_p (base, step))
2331 : goto fail;
2332 :
2333 876250 : base = build_fold_addr_expr (base);
2334 :
2335 : /* Substituting bases of IVs into the base expression might
2336 : have caused folding opportunities. */
2337 876250 : if (TREE_CODE (base) == ADDR_EXPR)
2338 : {
2339 463438 : tree *ref = &TREE_OPERAND (base, 0);
2340 1585490 : while (handled_component_p (*ref))
2341 658614 : ref = &TREE_OPERAND (*ref, 0);
2342 463438 : if (TREE_CODE (*ref) == MEM_REF)
2343 : {
2344 296628 : tree tem = fold_binary (MEM_REF, TREE_TYPE (*ref),
2345 : TREE_OPERAND (*ref, 0),
2346 : TREE_OPERAND (*ref, 1));
2347 296628 : if (tem)
2348 0 : *ref = tem;
2349 : }
2350 : }
2351 : }
2352 :
2353 876250 : civ = alloc_iv (data, base, step);
2354 : /* Fail if base object of this memory reference is unknown. */
2355 876250 : if (civ->base_object == NULL_TREE)
2356 10897 : goto fail;
2357 :
2358 865353 : record_group_use (data, op_p, civ, stmt, USE_REF_ADDRESS, TREE_TYPE (*op_p));
2359 865353 : return;
2360 :
2361 1930892 : fail:
2362 1930892 : for_each_index (op_p, idx_record_use, data);
2363 : }
2364 :
2365 : /* Finds and records invariants used in STMT. */
2366 :
2367 : static void
2368 15756725 : find_invariants_stmt (struct ivopts_data *data, gimple *stmt)
2369 : {
2370 15756725 : ssa_op_iter iter;
2371 15756725 : use_operand_p use_p;
2372 15756725 : tree op;
2373 :
2374 52407686 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
2375 : {
2376 20894236 : op = USE_FROM_PTR (use_p);
2377 20894236 : record_invariant (data, op, false);
2378 : }
2379 15756725 : }
2380 :
2381 : /* CALL calls an internal function. If operand *OP_P will become an
2382 : address when the call is expanded, return the type of the memory
2383 : being addressed, otherwise return null. */
2384 :
2385 : static tree
2386 4134 : get_mem_type_for_internal_fn (gcall *call, tree *op_p)
2387 : {
2388 4134 : switch (gimple_call_internal_fn (call))
2389 : {
2390 285 : case IFN_MASK_LOAD:
2391 285 : case IFN_MASK_LOAD_LANES:
2392 285 : case IFN_MASK_LEN_LOAD_LANES:
2393 285 : case IFN_LEN_LOAD:
2394 285 : case IFN_MASK_LEN_LOAD:
2395 285 : if (op_p == gimple_call_arg_ptr (call, 0))
2396 285 : return TREE_TYPE (gimple_call_lhs (call));
2397 : return NULL_TREE;
2398 :
2399 377 : case IFN_MASK_STORE:
2400 377 : case IFN_MASK_STORE_LANES:
2401 377 : case IFN_MASK_LEN_STORE_LANES:
2402 377 : case IFN_LEN_STORE:
2403 377 : case IFN_MASK_LEN_STORE:
2404 377 : {
2405 377 : if (op_p == gimple_call_arg_ptr (call, 0))
2406 : {
2407 377 : internal_fn ifn = gimple_call_internal_fn (call);
2408 377 : int index = internal_fn_stored_value_index (ifn);
2409 377 : return TREE_TYPE (gimple_call_arg (call, index));
2410 : }
2411 : return NULL_TREE;
2412 : }
2413 :
2414 : default:
2415 : return NULL_TREE;
2416 : }
2417 : }
2418 :
2419 : /* IV is a (non-address) iv that describes operand *OP_P of STMT.
2420 : Return true if the operand will become an address when STMT
2421 : is expanded and record the associated address use if so. */
2422 :
2423 : static bool
2424 1782134 : find_address_like_use (struct ivopts_data *data, gimple *stmt, tree *op_p,
2425 : struct iv *iv)
2426 : {
2427 : /* Fail if base object of this memory reference is unknown. */
2428 1782134 : if (iv->base_object == NULL_TREE)
2429 : return false;
2430 :
2431 685296 : tree mem_type = NULL_TREE;
2432 685296 : if (gcall *call = dyn_cast <gcall *> (stmt))
2433 126625 : if (gimple_call_internal_p (call))
2434 4134 : mem_type = get_mem_type_for_internal_fn (call, op_p);
2435 4134 : if (mem_type)
2436 : {
2437 662 : iv = alloc_iv (data, iv->base, iv->step);
2438 662 : record_group_use (data, op_p, iv, stmt, USE_PTR_ADDRESS, mem_type);
2439 662 : return true;
2440 : }
2441 : return false;
2442 : }
2443 :
2444 : /* Finds interesting uses of induction variables in the statement STMT. */
2445 :
2446 : static void
2447 15756725 : find_interesting_uses_stmt (struct ivopts_data *data, gimple *stmt)
2448 : {
2449 15756725 : struct iv *iv;
2450 15756725 : tree op, *lhs, *rhs;
2451 15756725 : ssa_op_iter iter;
2452 15756725 : use_operand_p use_p;
2453 15756725 : enum tree_code code;
2454 :
2455 15756725 : find_invariants_stmt (data, stmt);
2456 :
2457 15756725 : if (gimple_code (stmt) == GIMPLE_COND)
2458 : {
2459 1486660 : find_interesting_uses_cond (data, stmt);
2460 9221993 : return;
2461 : }
2462 :
2463 14270065 : if (is_gimple_assign (stmt))
2464 : {
2465 10765173 : lhs = gimple_assign_lhs_ptr (stmt);
2466 10765173 : rhs = gimple_assign_rhs1_ptr (stmt);
2467 :
2468 10765173 : if (TREE_CODE (*lhs) == SSA_NAME)
2469 : {
2470 : /* If the statement defines an induction variable, the uses are not
2471 : interesting by themselves. */
2472 :
2473 9571234 : iv = get_iv (data, *lhs);
2474 :
2475 9571234 : if (iv && !integer_zerop (iv->step))
2476 : return;
2477 : }
2478 :
2479 8382628 : code = gimple_assign_rhs_code (stmt);
2480 8382628 : if (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS
2481 8382628 : && (REFERENCE_CLASS_P (*rhs)
2482 1344400 : || is_gimple_val (*rhs)))
2483 : {
2484 2896214 : if (REFERENCE_CLASS_P (*rhs))
2485 1757401 : find_interesting_uses_address (data, stmt, rhs);
2486 : else
2487 1138813 : find_interesting_uses_op (data, *rhs);
2488 :
2489 2896214 : if (REFERENCE_CLASS_P (*lhs))
2490 1038844 : find_interesting_uses_address (data, stmt, lhs);
2491 : return;
2492 : }
2493 5486414 : else if (TREE_CODE_CLASS (code) == tcc_comparison)
2494 : {
2495 80347 : find_interesting_uses_cond (data, stmt);
2496 80347 : return;
2497 : }
2498 :
2499 : /* TODO -- we should also handle address uses of type
2500 :
2501 : memory = call (whatever);
2502 :
2503 : and
2504 :
2505 : call (memory). */
2506 : }
2507 :
2508 8910959 : if (gimple_code (stmt) == GIMPLE_PHI
2509 8910959 : && gimple_bb (stmt) == data->current_loop->header)
2510 : {
2511 1482049 : iv = get_iv (data, PHI_RESULT (stmt));
2512 :
2513 1482049 : if (iv && !integer_zerop (iv->step))
2514 : return;
2515 : }
2516 :
2517 26947086 : FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE)
2518 : {
2519 10904302 : op = USE_FROM_PTR (use_p);
2520 :
2521 10904302 : if (TREE_CODE (op) != SSA_NAME)
2522 531615 : continue;
2523 :
2524 10372687 : iv = get_iv (data, op);
2525 10372687 : if (!iv)
2526 8590553 : continue;
2527 :
2528 1782134 : if (!find_address_like_use (data, stmt, use_p->use, iv))
2529 1781472 : find_interesting_uses_op (data, op);
2530 : }
2531 : }
2532 :
2533 : /* Finds interesting uses of induction variables outside of loops
2534 : on loop exit edge EXIT. */
2535 :
2536 : static void
2537 907375 : find_interesting_uses_outside (struct ivopts_data *data, edge exit)
2538 : {
2539 907375 : gphi *phi;
2540 907375 : gphi_iterator psi;
2541 907375 : tree def;
2542 :
2543 2023878 : for (psi = gsi_start_phis (exit->dest); !gsi_end_p (psi); gsi_next (&psi))
2544 : {
2545 1116503 : phi = psi.phi ();
2546 1116503 : def = PHI_ARG_DEF_FROM_EDGE (phi, exit);
2547 2140329 : if (!virtual_operand_p (def))
2548 545346 : find_interesting_uses_op (data, def);
2549 : }
2550 907375 : }
2551 :
2552 : /* Return TRUE if OFFSET is within the range of [base + offset] addressing
2553 : mode for memory reference represented by USE. */
2554 :
2555 : static GTY (()) vec<rtx, va_gc> *addr_list;
2556 :
2557 : static bool
2558 217014 : addr_offset_valid_p (struct iv_use *use, poly_int64 offset)
2559 : {
2560 217014 : rtx reg, addr;
2561 217014 : unsigned list_index;
2562 217014 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
2563 217014 : machine_mode addr_mode, mem_mode = TYPE_MODE (use->mem_type);
2564 :
2565 217014 : list_index = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
2566 217014 : if (list_index >= vec_safe_length (addr_list))
2567 10155 : vec_safe_grow_cleared (addr_list, list_index + MAX_MACHINE_MODE, true);
2568 :
2569 217014 : addr = (*addr_list)[list_index];
2570 217014 : if (!addr)
2571 : {
2572 13224 : addr_mode = targetm.addr_space.address_mode (as);
2573 13224 : reg = gen_raw_REG (addr_mode, LAST_VIRTUAL_REGISTER + 1);
2574 13224 : addr = gen_rtx_fmt_ee (PLUS, addr_mode, reg, NULL_RTX);
2575 13224 : (*addr_list)[list_index] = addr;
2576 : }
2577 : else
2578 203790 : addr_mode = GET_MODE (addr);
2579 :
2580 217014 : XEXP (addr, 1) = gen_int_mode (offset, addr_mode);
2581 217014 : return (memory_address_addr_space_p (mem_mode, addr, as));
2582 : }
2583 :
2584 : /* Comparison function to sort group in ascending order of addr_offset. */
2585 :
2586 : static int
2587 3129183 : group_compare_offset (const void *a, const void *b)
2588 : {
2589 3129183 : const struct iv_use *const *u1 = (const struct iv_use *const *) a;
2590 3129183 : const struct iv_use *const *u2 = (const struct iv_use *const *) b;
2591 :
2592 3129183 : return compare_sizes_for_sort ((*u1)->addr_offset, (*u2)->addr_offset);
2593 : }
2594 :
2595 : /* Check if small groups should be split. Return true if no group
2596 : contains more than two uses with distinct addr_offsets. Return
2597 : false otherwise. We want to split such groups because:
2598 :
2599 : 1) Small groups don't have much benefit and may interfere with
2600 : general candidate selection.
2601 : 2) Size for problem with only small groups is usually small and
2602 : general algorithm can handle it well.
2603 :
2604 : TODO -- Above claim may not hold when we want to merge memory
2605 : accesses with conseuctive addresses. */
2606 :
2607 : static bool
2608 513182 : split_small_address_groups_p (struct ivopts_data *data)
2609 : {
2610 513182 : unsigned int i, j, distinct = 1;
2611 513182 : struct iv_use *pre;
2612 513182 : struct iv_group *group;
2613 :
2614 2128558 : for (i = 0; i < data->vgroups.length (); i++)
2615 : {
2616 1615376 : group = data->vgroups[i];
2617 1615376 : if (group->vuses.length () == 1)
2618 1478006 : continue;
2619 :
2620 137370 : gcc_assert (address_p (group->type));
2621 137370 : if (group->vuses.length () == 2)
2622 : {
2623 79162 : if (compare_sizes_for_sort (group->vuses[0]->addr_offset,
2624 79162 : group->vuses[1]->addr_offset) > 0)
2625 19857 : std::swap (group->vuses[0], group->vuses[1]);
2626 : }
2627 : else
2628 58208 : group->vuses.qsort (group_compare_offset);
2629 :
2630 137370 : if (distinct > 2)
2631 13897 : continue;
2632 :
2633 123473 : distinct = 1;
2634 1803605 : for (pre = group->vuses[0], j = 1; j < group->vuses.length (); j++)
2635 : {
2636 188229 : if (maybe_ne (group->vuses[j]->addr_offset, pre->addr_offset))
2637 : {
2638 131064 : pre = group->vuses[j];
2639 131064 : distinct++;
2640 : }
2641 :
2642 188229 : if (distinct > 2)
2643 : break;
2644 : }
2645 : }
2646 :
2647 513182 : return (distinct <= 2);
2648 : }
2649 :
2650 : /* For each group of address type uses, this function further groups
2651 : these uses according to the maximum offset supported by target's
2652 : [base + offset] addressing mode. */
2653 :
2654 : static void
2655 513182 : split_address_groups (struct ivopts_data *data)
2656 : {
2657 513182 : unsigned int i, j;
2658 : /* Always split group. */
2659 513182 : bool split_p = split_small_address_groups_p (data);
2660 :
2661 2694367 : for (i = 0; i < data->vgroups.length (); i++)
2662 : {
2663 1668003 : struct iv_group *new_group = NULL;
2664 1668003 : struct iv_group *group = data->vgroups[i];
2665 1668003 : struct iv_use *use = group->vuses[0];
2666 :
2667 1668003 : use->id = 0;
2668 1668003 : use->group_id = group->id;
2669 1668003 : if (group->vuses.length () == 1)
2670 1524843 : continue;
2671 :
2672 143160 : gcc_assert (address_p (use->type));
2673 :
2674 2001422 : for (j = 1; j < group->vuses.length ();)
2675 : {
2676 333419 : struct iv_use *next = group->vuses[j];
2677 333419 : poly_int64 offset = next->addr_offset - use->addr_offset;
2678 :
2679 : /* Split group if asked to, or the offset against the first
2680 : use can't fit in offset part of addressing mode. IV uses
2681 : having the same offset are still kept in one group. */
2682 392280 : if (maybe_ne (offset, 0)
2683 333419 : && (split_p || !addr_offset_valid_p (use, offset)))
2684 : {
2685 58861 : if (!new_group)
2686 52627 : new_group = record_group (data, group->type);
2687 58861 : group->vuses.ordered_remove (j);
2688 58861 : new_group->vuses.safe_push (next);
2689 58861 : continue;
2690 : }
2691 :
2692 274558 : next->id = j;
2693 274558 : next->group_id = group->id;
2694 274558 : j++;
2695 : }
2696 : }
2697 513182 : }
2698 :
2699 : /* Finds uses of the induction variables that are interesting. */
2700 :
2701 : static void
2702 513182 : find_interesting_uses (struct ivopts_data *data, basic_block *body)
2703 : {
2704 513182 : basic_block bb;
2705 513182 : gimple_stmt_iterator bsi;
2706 513182 : unsigned i;
2707 513182 : edge e;
2708 :
2709 3372128 : for (i = 0; i < data->current_loop->num_nodes; i++)
2710 : {
2711 2858946 : edge_iterator ei;
2712 2858946 : bb = body[i];
2713 :
2714 7302657 : FOR_EACH_EDGE (e, ei, bb->succs)
2715 4443711 : if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun)
2716 4443711 : && !flow_bb_inside_loop_p (data->current_loop, e->dest))
2717 907375 : find_interesting_uses_outside (data, e);
2718 :
2719 5773229 : for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2720 2914283 : find_interesting_uses_stmt (data, gsi_stmt (bsi));
2721 28330281 : for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
2722 22612389 : if (!is_gimple_debug (gsi_stmt (bsi)))
2723 12842442 : find_interesting_uses_stmt (data, gsi_stmt (bsi));
2724 : }
2725 :
2726 513182 : split_address_groups (data);
2727 :
2728 513182 : if (dump_file && (dump_flags & TDF_DETAILS))
2729 : {
2730 72 : fprintf (dump_file, "\n<IV Groups>:\n");
2731 72 : dump_groups (dump_file, data);
2732 72 : fprintf (dump_file, "\n");
2733 : }
2734 513182 : }
2735 :
2736 : /* Strips constant offsets from EXPR and stores them to OFFSET. If INSIDE_ADDR
2737 : is true, assume we are inside an address. If TOP_COMPREF is true, assume
2738 : we are at the top-level of the processed address. */
2739 :
2740 : static tree
2741 3421129 : strip_offset_1 (tree expr, bool inside_addr, bool top_compref,
2742 : poly_int64 *offset)
2743 : {
2744 3421129 : tree op0 = NULL_TREE, op1 = NULL_TREE, tmp, step;
2745 3421129 : enum tree_code code;
2746 3421129 : tree type, orig_type = TREE_TYPE (expr);
2747 3421129 : poly_int64 off0, off1;
2748 3421129 : HOST_WIDE_INT st;
2749 3421129 : tree orig_expr = expr;
2750 :
2751 3421129 : STRIP_NOPS (expr);
2752 :
2753 3421129 : type = TREE_TYPE (expr);
2754 3421129 : code = TREE_CODE (expr);
2755 3421129 : *offset = 0;
2756 :
2757 3421129 : switch (code)
2758 : {
2759 624787 : case POINTER_PLUS_EXPR:
2760 624787 : case PLUS_EXPR:
2761 624787 : case MINUS_EXPR:
2762 624787 : op0 = TREE_OPERAND (expr, 0);
2763 624787 : op1 = TREE_OPERAND (expr, 1);
2764 :
2765 624787 : op0 = strip_offset_1 (op0, false, false, &off0);
2766 624787 : op1 = strip_offset_1 (op1, false, false, &off1);
2767 :
2768 624787 : *offset = (code == MINUS_EXPR ? off0 - off1 : off0 + off1);
2769 624787 : if (op0 == TREE_OPERAND (expr, 0)
2770 624787 : && op1 == TREE_OPERAND (expr, 1))
2771 : return orig_expr;
2772 :
2773 386499 : if (integer_zerop (op1))
2774 : expr = op0;
2775 3328 : else if (integer_zerop (op0))
2776 : {
2777 691 : if (code == MINUS_EXPR)
2778 : {
2779 691 : if (TYPE_OVERFLOW_UNDEFINED (type))
2780 : {
2781 0 : type = unsigned_type_for (type);
2782 0 : op1 = fold_convert (type, op1);
2783 : }
2784 691 : expr = fold_build1 (NEGATE_EXPR, type, op1);
2785 : }
2786 : else
2787 : expr = op1;
2788 : }
2789 : else
2790 : {
2791 2637 : if (TYPE_OVERFLOW_UNDEFINED (type))
2792 : {
2793 0 : type = unsigned_type_for (type);
2794 0 : if (code == POINTER_PLUS_EXPR)
2795 0 : code = PLUS_EXPR;
2796 0 : op0 = fold_convert (type, op0);
2797 0 : op1 = fold_convert (type, op1);
2798 : }
2799 2637 : expr = fold_build2 (code, type, op0, op1);
2800 : }
2801 :
2802 386499 : return fold_convert (orig_type, expr);
2803 :
2804 219208 : case MULT_EXPR:
2805 219208 : op1 = TREE_OPERAND (expr, 1);
2806 219208 : if (!cst_and_fits_in_hwi (op1))
2807 : return orig_expr;
2808 :
2809 178322 : op0 = TREE_OPERAND (expr, 0);
2810 178322 : op0 = strip_offset_1 (op0, false, false, &off0);
2811 178322 : if (op0 == TREE_OPERAND (expr, 0))
2812 : return orig_expr;
2813 :
2814 7092 : *offset = off0 * int_cst_value (op1);
2815 7092 : if (integer_zerop (op0))
2816 : expr = op0;
2817 : else
2818 : {
2819 7092 : if (TYPE_OVERFLOW_UNDEFINED (type))
2820 : {
2821 0 : type = unsigned_type_for (type);
2822 0 : op0 = fold_convert (type, op0);
2823 0 : op1 = fold_convert (type, op1);
2824 : }
2825 7092 : expr = fold_build2 (MULT_EXPR, type, op0, op1);
2826 : }
2827 :
2828 7092 : return fold_convert (orig_type, expr);
2829 :
2830 9 : case ARRAY_REF:
2831 9 : case ARRAY_RANGE_REF:
2832 9 : if (!inside_addr)
2833 : return orig_expr;
2834 :
2835 9 : step = array_ref_element_size (expr);
2836 9 : if (!cst_and_fits_in_hwi (step))
2837 : break;
2838 :
2839 9 : st = int_cst_value (step);
2840 9 : op1 = TREE_OPERAND (expr, 1);
2841 9 : op1 = strip_offset_1 (op1, false, false, &off1);
2842 9 : *offset = off1 * st;
2843 :
2844 9 : if (top_compref
2845 9 : && integer_zerop (op1))
2846 : {
2847 : /* Strip the component reference completely. */
2848 7 : op0 = TREE_OPERAND (expr, 0);
2849 7 : op0 = strip_offset_1 (op0, inside_addr, top_compref, &off0);
2850 7 : *offset += off0;
2851 7 : return op0;
2852 : }
2853 : break;
2854 :
2855 1 : case COMPONENT_REF:
2856 1 : {
2857 1 : tree field;
2858 :
2859 1 : if (!inside_addr)
2860 : return orig_expr;
2861 :
2862 1 : tmp = component_ref_field_offset (expr);
2863 1 : field = TREE_OPERAND (expr, 1);
2864 1 : if (top_compref
2865 1 : && cst_and_fits_in_hwi (tmp)
2866 2 : && cst_and_fits_in_hwi (DECL_FIELD_BIT_OFFSET (field)))
2867 : {
2868 1 : HOST_WIDE_INT boffset, abs_off;
2869 :
2870 : /* Strip the component reference completely. */
2871 1 : op0 = TREE_OPERAND (expr, 0);
2872 1 : op0 = strip_offset_1 (op0, inside_addr, top_compref, &off0);
2873 1 : boffset = int_cst_value (DECL_FIELD_BIT_OFFSET (field));
2874 1 : abs_off = abs_hwi (boffset) / BITS_PER_UNIT;
2875 1 : if (boffset < 0)
2876 0 : abs_off = -abs_off;
2877 :
2878 1 : *offset = off0 + int_cst_value (tmp) + abs_off;
2879 1 : return op0;
2880 : }
2881 : }
2882 : break;
2883 :
2884 325536 : case ADDR_EXPR:
2885 325536 : op0 = TREE_OPERAND (expr, 0);
2886 325536 : op0 = strip_offset_1 (op0, true, true, &off0);
2887 325536 : *offset += off0;
2888 :
2889 325536 : if (op0 == TREE_OPERAND (expr, 0))
2890 : return orig_expr;
2891 :
2892 8 : expr = build_fold_addr_expr (op0);
2893 8 : return fold_convert (orig_type, expr);
2894 :
2895 : case MEM_REF:
2896 : /* ??? Offset operand? */
2897 : inside_addr = false;
2898 : break;
2899 :
2900 2251586 : default:
2901 2251586 : if (ptrdiff_tree_p (expr, offset) && maybe_ne (*offset, 0))
2902 877200 : return build_int_cst (orig_type, 0);
2903 : return orig_expr;
2904 : }
2905 :
2906 : /* Default handling of expressions for that we want to recurse into
2907 : the first operand. */
2908 4 : op0 = TREE_OPERAND (expr, 0);
2909 4 : op0 = strip_offset_1 (op0, inside_addr, false, &off0);
2910 4 : *offset += off0;
2911 :
2912 4 : if (op0 == TREE_OPERAND (expr, 0)
2913 4 : && (!op1 || op1 == TREE_OPERAND (expr, 1)))
2914 : return orig_expr;
2915 :
2916 1 : expr = copy_node (expr);
2917 1 : TREE_OPERAND (expr, 0) = op0;
2918 1 : if (op1)
2919 1 : TREE_OPERAND (expr, 1) = op1;
2920 :
2921 : /* Inside address, we might strip the top level component references,
2922 : thus changing type of the expression. Handling of ADDR_EXPR
2923 : will fix that. */
2924 1 : expr = fold_convert (orig_type, expr);
2925 :
2926 1 : return expr;
2927 : }
2928 :
2929 : /* Strips constant offsets from EXPR and stores them to OFFSET. */
2930 :
2931 : static tree
2932 1667676 : strip_offset (tree expr, poly_uint64 *offset)
2933 : {
2934 1667676 : poly_int64 off;
2935 1667676 : tree core = strip_offset_1 (expr, false, false, &off);
2936 1667676 : *offset = off;
2937 1667676 : return core;
2938 : }
2939 :
2940 : /* Returns variant of TYPE that can be used as base for different uses.
2941 : We return unsigned type with the same precision, which avoids problems
2942 : with overflows. */
2943 :
2944 : static tree
2945 8186649 : generic_type_for (tree type)
2946 : {
2947 8186649 : if (POINTER_TYPE_P (type))
2948 1443688 : return unsigned_type_for (type);
2949 :
2950 6742961 : if (TYPE_UNSIGNED (type))
2951 : return type;
2952 :
2953 3152354 : return unsigned_type_for (type);
2954 : }
2955 :
2956 : /* Private data for walk_tree. */
2957 :
2958 : struct walk_tree_data
2959 : {
2960 : bitmap *inv_vars;
2961 : struct ivopts_data *idata;
2962 : };
2963 :
2964 : /* Callback function for walk_tree, it records invariants and symbol
2965 : reference in *EXPR_P. DATA is the structure storing result info. */
2966 :
2967 : static tree
2968 34447118 : find_inv_vars_cb (tree *expr_p, int *ws ATTRIBUTE_UNUSED, void *data)
2969 : {
2970 34447118 : tree op = *expr_p;
2971 34447118 : struct version_info *info;
2972 34447118 : struct walk_tree_data *wdata = (struct walk_tree_data*) data;
2973 :
2974 34447118 : if (TREE_CODE (op) != SSA_NAME)
2975 : return NULL_TREE;
2976 :
2977 8037370 : info = name_info (wdata->idata, op);
2978 : /* Because we expand simple operations when finding IVs, loop invariant
2979 : variable that isn't referred by the original loop could be used now.
2980 : Record such invariant variables here. */
2981 8037370 : if (!info->iv)
2982 : {
2983 393286 : struct ivopts_data *idata = wdata->idata;
2984 393286 : basic_block bb = gimple_bb (SSA_NAME_DEF_STMT (op));
2985 :
2986 393286 : if (!bb || !flow_bb_inside_loop_p (idata->current_loop, bb))
2987 : {
2988 393286 : tree steptype = TREE_TYPE (op);
2989 393286 : if (POINTER_TYPE_P (steptype))
2990 195477 : steptype = sizetype;
2991 393286 : set_iv (idata, op, op, build_int_cst (steptype, 0), true);
2992 393286 : record_invariant (idata, op, false);
2993 : }
2994 : }
2995 8037370 : if (!info->inv_id || info->has_nonlin_use)
2996 : return NULL_TREE;
2997 :
2998 6662095 : if (!*wdata->inv_vars)
2999 5155533 : *wdata->inv_vars = BITMAP_ALLOC (NULL);
3000 6662095 : bitmap_set_bit (*wdata->inv_vars, info->inv_id);
3001 :
3002 6662095 : return NULL_TREE;
3003 : }
3004 :
3005 : /* Records invariants in *EXPR_P. INV_VARS is the bitmap to that we should
3006 : store it. */
3007 :
3008 : static inline void
3009 28066546 : find_inv_vars (struct ivopts_data *data, tree *expr_p, bitmap *inv_vars)
3010 : {
3011 28066546 : struct walk_tree_data wdata;
3012 :
3013 28066546 : if (!inv_vars)
3014 : return;
3015 :
3016 16145430 : wdata.idata = data;
3017 16145430 : wdata.inv_vars = inv_vars;
3018 16145430 : walk_tree (expr_p, find_inv_vars_cb, &wdata, NULL);
3019 : }
3020 :
3021 : /* Get entry from invariant expr hash table for INV_EXPR. New entry
3022 : will be recorded if it doesn't exist yet. Given below two exprs:
3023 : inv_expr + cst1, inv_expr + cst2
3024 : It's hard to make decision whether constant part should be stripped
3025 : or not. We choose to not strip based on below facts:
3026 : 1) We need to count ADD cost for constant part if it's stripped,
3027 : which isn't always trivial where this functions is called.
3028 : 2) Stripping constant away may be conflict with following loop
3029 : invariant hoisting pass.
3030 : 3) Not stripping constant away results in more invariant exprs,
3031 : which usually leads to decision preferring lower reg pressure. */
3032 :
3033 : static iv_inv_expr_ent *
3034 2623338 : get_loop_invariant_expr (struct ivopts_data *data, tree inv_expr)
3035 : {
3036 2623338 : STRIP_NOPS (inv_expr);
3037 :
3038 2623338 : if (poly_int_tree_p (inv_expr)
3039 2623338 : || TREE_CODE (inv_expr) == SSA_NAME)
3040 : return NULL;
3041 :
3042 : /* Don't strip constant part away as we used to. */
3043 :
3044 : /* Stores EXPR in DATA->inv_expr_tab, return pointer to iv_inv_expr_ent. */
3045 2534774 : struct iv_inv_expr_ent ent;
3046 2534774 : ent.expr = inv_expr;
3047 2534774 : ent.hash = iterative_hash_expr (inv_expr, 0);
3048 2534774 : struct iv_inv_expr_ent **slot = data->inv_expr_tab->find_slot (&ent, INSERT);
3049 :
3050 2534774 : if (!*slot)
3051 : {
3052 1164347 : *slot = XNEW (struct iv_inv_expr_ent);
3053 1164347 : (*slot)->expr = inv_expr;
3054 1164347 : (*slot)->hash = ent.hash;
3055 1164347 : (*slot)->id = ++data->max_inv_expr_id;
3056 : }
3057 :
3058 2534774 : return *slot;
3059 : }
3060 :
3061 :
3062 : /* Return *TP if it is an SSA_NAME marked with TREE_VISITED, i.e., as
3063 : unsuitable as ivopts candidates for potentially involving undefined
3064 : behavior. */
3065 :
3066 : static tree
3067 15524108 : find_ssa_undef (tree *tp, int *walk_subtrees, void *bb_)
3068 : {
3069 15524108 : basic_block bb = (basic_block) bb_;
3070 15524108 : if (TREE_CODE (*tp) == SSA_NAME
3071 2261051 : && ssa_name_maybe_undef_p (*tp)
3072 15533018 : && !ssa_name_any_use_dominates_bb_p (*tp, bb))
3073 3167 : return *tp;
3074 15520941 : if (!EXPR_P (*tp))
3075 10548074 : *walk_subtrees = 0;
3076 : return NULL;
3077 : }
3078 :
3079 : /* Adds a candidate BASE + STEP * i. Important field is set to IMPORTANT and
3080 : position to POS. If USE is not NULL, the candidate is set as related to
3081 : it. If both BASE and STEP are NULL, we add a pseudocandidate for the
3082 : replacement of the final value of the iv by a direct computation. */
3083 :
3084 : static struct iv_cand *
3085 9180977 : add_candidate_1 (struct ivopts_data *data, tree base, tree step, bool important,
3086 : enum iv_position pos, struct iv_use *use,
3087 : gimple *incremented_at, struct iv *orig_iv = NULL,
3088 : bool doloop = false)
3089 : {
3090 9180977 : unsigned i;
3091 9180977 : struct iv_cand *cand = NULL;
3092 9180977 : tree type, orig_type;
3093 :
3094 9180977 : gcc_assert (base && step);
3095 :
3096 : /* -fkeep-gc-roots-live means that we have to keep a real pointer
3097 : live, but the ivopts code may replace a real pointer with one
3098 : pointing before or after the memory block that is then adjusted
3099 : into the memory block during the loop. FIXME: It would likely be
3100 : better to actually force the pointer live and still use ivopts;
3101 : for example, it would be enough to write the pointer into memory
3102 : and keep it there until after the loop. */
3103 9180977 : if (flag_keep_gc_roots_live && POINTER_TYPE_P (TREE_TYPE (base)))
3104 : return NULL;
3105 :
3106 : /* If BASE contains undefined SSA names make sure we only record
3107 : the original IV. */
3108 9075272 : bool involves_undefs = false;
3109 9075272 : if (walk_tree (&base, find_ssa_undef, data->current_loop->header, NULL))
3110 : {
3111 3167 : if (pos != IP_ORIGINAL)
3112 : return NULL;
3113 : important = false;
3114 : involves_undefs = true;
3115 : }
3116 :
3117 : /* For non-original variables, make sure their values are computed in a type
3118 : that does not invoke undefined behavior on overflows (since in general,
3119 : we cannot prove that these induction variables are non-wrapping). */
3120 9072105 : if (pos != IP_ORIGINAL)
3121 : {
3122 8186649 : orig_type = TREE_TYPE (base);
3123 8186649 : type = generic_type_for (orig_type);
3124 8186649 : if (type != orig_type)
3125 : {
3126 4596042 : base = fold_convert (type, base);
3127 4596042 : step = fold_convert (type, step);
3128 : }
3129 : }
3130 :
3131 45217686 : for (i = 0; i < data->vcands.length (); i++)
3132 : {
3133 40525285 : cand = data->vcands[i];
3134 :
3135 40525285 : if (cand->pos != pos)
3136 9923793 : continue;
3137 :
3138 30601492 : if (cand->incremented_at != incremented_at
3139 30099779 : || ((pos == IP_AFTER_USE || pos == IP_BEFORE_USE)
3140 0 : && cand->ainc_use != use))
3141 501713 : continue;
3142 :
3143 30099779 : if (operand_equal_p (base, cand->iv->base, 0)
3144 9606007 : && operand_equal_p (step, cand->iv->step, 0)
3145 35896861 : && (TYPE_PRECISION (TREE_TYPE (base))
3146 5797082 : == TYPE_PRECISION (TREE_TYPE (cand->iv->base))))
3147 : break;
3148 : }
3149 :
3150 18144974 : if (i == data->vcands.length ())
3151 : {
3152 4692401 : cand = XCNEW (struct iv_cand);
3153 4692401 : cand->id = i;
3154 4692401 : cand->iv = alloc_iv (data, base, step);
3155 4692401 : cand->pos = pos;
3156 4692401 : if (pos != IP_ORIGINAL)
3157 : {
3158 3806740 : if (doloop)
3159 0 : cand->var_before = create_tmp_var_raw (TREE_TYPE (base), "doloop");
3160 : else
3161 3806740 : cand->var_before = create_tmp_var_raw (TREE_TYPE (base), "ivtmp");
3162 3806740 : cand->var_after = cand->var_before;
3163 : }
3164 4692401 : cand->important = important;
3165 4692401 : cand->involves_undefs = involves_undefs;
3166 4692401 : cand->incremented_at = incremented_at;
3167 4692401 : cand->doloop_p = doloop;
3168 4692401 : data->vcands.safe_push (cand);
3169 :
3170 4692401 : if (!poly_int_tree_p (step))
3171 : {
3172 176173 : find_inv_vars (data, &step, &cand->inv_vars);
3173 :
3174 176173 : iv_inv_expr_ent *inv_expr = get_loop_invariant_expr (data, step);
3175 : /* Share bitmap between inv_vars and inv_exprs for cand. */
3176 176173 : if (inv_expr != NULL)
3177 : {
3178 94071 : cand->inv_exprs = cand->inv_vars;
3179 94071 : cand->inv_vars = NULL;
3180 94071 : if (cand->inv_exprs)
3181 75654 : bitmap_clear (cand->inv_exprs);
3182 : else
3183 18417 : cand->inv_exprs = BITMAP_ALLOC (NULL);
3184 :
3185 94071 : bitmap_set_bit (cand->inv_exprs, inv_expr->id);
3186 : }
3187 : }
3188 :
3189 4692401 : if (pos == IP_AFTER_USE || pos == IP_BEFORE_USE)
3190 : cand->ainc_use = use;
3191 : else
3192 4692401 : cand->ainc_use = NULL;
3193 :
3194 4692401 : cand->orig_iv = orig_iv;
3195 4692401 : if (dump_file && (dump_flags & TDF_DETAILS))
3196 733 : dump_cand (dump_file, cand);
3197 : }
3198 :
3199 9072487 : cand->important |= important;
3200 9072487 : cand->doloop_p |= doloop;
3201 :
3202 : /* Relate candidate to the group for which it is added. */
3203 9072487 : if (use)
3204 2527864 : bitmap_set_bit (data->vgroups[use->group_id]->related_cands, i);
3205 :
3206 : return cand;
3207 : }
3208 :
3209 : /* Returns true if incrementing the induction variable at the end of the LOOP
3210 : is allowed.
3211 :
3212 : The purpose is to avoid splitting latch edge with a biv increment, thus
3213 : creating a jump, possibly confusing other optimization passes and leaving
3214 : less freedom to scheduler. So we allow IP_END only if IP_NORMAL is not
3215 : available (so we do not have a better alternative), or if the latch edge
3216 : is already nonempty. */
3217 :
3218 : static bool
3219 8075783 : allow_ip_end_pos_p (class loop *loop)
3220 : {
3221 : /* Do not allow IP_END when creating the IV would need to split the
3222 : latch edge as that makes all IP_NORMAL invalid. */
3223 8075783 : auto pos = gsi_last_bb (ip_end_pos (loop));
3224 8075783 : if (!gsi_end_p (pos) && stmt_ends_bb_p (*pos))
3225 : return false;
3226 :
3227 8075783 : if (!ip_normal_pos (loop))
3228 : return true;
3229 :
3230 7964086 : if (!empty_block_p (ip_end_pos (loop)))
3231 217003 : return true;
3232 :
3233 : return false;
3234 : }
3235 :
3236 : /* If possible, adds autoincrement candidates BASE + STEP * i based on use USE.
3237 : Important field is set to IMPORTANT. */
3238 :
3239 : static void
3240 591454 : add_autoinc_candidates (struct ivopts_data *data, tree base, tree step,
3241 : bool important, struct iv_use *use)
3242 : {
3243 591454 : basic_block use_bb = gimple_bb (use->stmt);
3244 591454 : machine_mode mem_mode;
3245 591454 : unsigned HOST_WIDE_INT cstepi;
3246 :
3247 : /* If we insert the increment in any position other than the standard
3248 : ones, we must ensure that it is incremented once per iteration.
3249 : It must not be in an inner nested loop, or one side of an if
3250 : statement. */
3251 591454 : if (use_bb->loop_father != data->current_loop
3252 588875 : || !dominated_by_p (CDI_DOMINATORS, data->current_loop->latch, use_bb)
3253 562868 : || stmt_can_throw_internal (cfun, use->stmt)
3254 1150500 : || !cst_and_fits_in_hwi (step))
3255 : return;
3256 :
3257 529383 : cstepi = int_cst_value (step);
3258 :
3259 529383 : mem_mode = TYPE_MODE (use->mem_type);
3260 : if (((USE_LOAD_PRE_INCREMENT (mem_mode)
3261 : || USE_STORE_PRE_INCREMENT (mem_mode))
3262 : && known_eq (GET_MODE_SIZE (mem_mode), cstepi))
3263 : || ((USE_LOAD_PRE_DECREMENT (mem_mode)
3264 : || USE_STORE_PRE_DECREMENT (mem_mode))
3265 : && known_eq (GET_MODE_SIZE (mem_mode), -cstepi)))
3266 : {
3267 : enum tree_code code = MINUS_EXPR;
3268 : tree new_base;
3269 : tree new_step = step;
3270 :
3271 : if (POINTER_TYPE_P (TREE_TYPE (base)))
3272 : {
3273 : new_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (step), step);
3274 : code = POINTER_PLUS_EXPR;
3275 : }
3276 : else
3277 : new_step = fold_convert (TREE_TYPE (base), new_step);
3278 : new_base = fold_build2 (code, TREE_TYPE (base), base, new_step);
3279 : add_candidate_1 (data, new_base, step, important, IP_BEFORE_USE, use,
3280 : use->stmt);
3281 : }
3282 : if (((USE_LOAD_POST_INCREMENT (mem_mode)
3283 : || USE_STORE_POST_INCREMENT (mem_mode))
3284 : && known_eq (GET_MODE_SIZE (mem_mode), cstepi))
3285 : || ((USE_LOAD_POST_DECREMENT (mem_mode)
3286 : || USE_STORE_POST_DECREMENT (mem_mode))
3287 : && known_eq (GET_MODE_SIZE (mem_mode), -cstepi)))
3288 : {
3289 : add_candidate_1 (data, base, step, important, IP_AFTER_USE, use,
3290 : use->stmt);
3291 : }
3292 : }
3293 :
3294 : /* Adds a candidate BASE + STEP * i. Important field is set to IMPORTANT and
3295 : position to POS. If USE is not NULL, the candidate is set as related to
3296 : it. The candidate computation is scheduled before exit condition and at
3297 : the end of loop. */
3298 :
3299 : static void
3300 7103678 : add_candidate (struct ivopts_data *data, tree base, tree step, bool important,
3301 : struct iv_use *use, struct iv *orig_iv = NULL,
3302 : bool doloop = false)
3303 : {
3304 7103678 : if (ip_normal_pos (data->current_loop))
3305 7009003 : add_candidate_1 (data, base, step, important, IP_NORMAL, use, NULL, orig_iv,
3306 : doloop);
3307 : /* Exclude doloop candidate here since it requires decrement then comparison
3308 : and jump, the IP_END position doesn't match. */
3309 7103678 : if (!doloop && ip_end_pos (data->current_loop)
3310 14207356 : && allow_ip_end_pos_p (data->current_loop))
3311 282615 : add_candidate_1 (data, base, step, important, IP_END, use, NULL, orig_iv);
3312 7103678 : }
3313 :
3314 : /* Adds standard iv candidates. */
3315 :
3316 : static void
3317 513181 : add_standard_iv_candidates (struct ivopts_data *data)
3318 : {
3319 513181 : add_candidate (data, integer_zero_node, integer_one_node, true, NULL);
3320 :
3321 : /* The same for a double-integer type if it is still fast enough. */
3322 513181 : if (TYPE_PRECISION
3323 513181 : (long_integer_type_node) > TYPE_PRECISION (integer_type_node)
3324 513181 : && TYPE_PRECISION (long_integer_type_node) <= BITS_PER_WORD)
3325 464985 : add_candidate (data, build_int_cst (long_integer_type_node, 0),
3326 : build_int_cst (long_integer_type_node, 1), true, NULL);
3327 :
3328 : /* The same for a double-integer type if it is still fast enough. */
3329 513181 : if (TYPE_PRECISION
3330 513181 : (long_long_integer_type_node) > TYPE_PRECISION (long_integer_type_node)
3331 561365 : && TYPE_PRECISION (long_long_integer_type_node) <= BITS_PER_WORD)
3332 12 : add_candidate (data, build_int_cst (long_long_integer_type_node, 0),
3333 : build_int_cst (long_long_integer_type_node, 1), true, NULL);
3334 513181 : }
3335 :
3336 :
3337 : /* Adds candidates bases on the old induction variable IV. */
3338 :
3339 : static void
3340 1776200 : add_iv_candidate_for_biv (struct ivopts_data *data, struct iv *iv)
3341 : {
3342 1776200 : gimple *phi;
3343 1776200 : tree def;
3344 1776200 : struct iv_cand *cand;
3345 :
3346 : /* Check if this biv is used in address type use. */
3347 1179193 : if (iv->no_overflow && iv->have_address_use
3348 501349 : && INTEGRAL_TYPE_P (TREE_TYPE (iv->base))
3349 2277549 : && TYPE_PRECISION (TREE_TYPE (iv->base)) < TYPE_PRECISION (sizetype))
3350 : {
3351 283210 : tree base = fold_convert (sizetype, iv->base);
3352 283210 : tree step = fold_convert (sizetype, iv->step);
3353 :
3354 : /* Add iv cand of same precision as index part in TARGET_MEM_REF. */
3355 283210 : add_candidate (data, base, step, true, NULL, iv);
3356 : /* Add iv cand of the original type only if it has nonlinear use. */
3357 283210 : if (iv->nonlin_use)
3358 27897 : add_candidate (data, iv->base, iv->step, true, NULL);
3359 : }
3360 : else
3361 1492990 : add_candidate (data, iv->base, iv->step, true, NULL);
3362 :
3363 : /* The same, but with initial value zero. */
3364 1776200 : if (POINTER_TYPE_P (TREE_TYPE (iv->base)))
3365 338002 : add_candidate (data, size_int (0), iv->step, true, NULL);
3366 : else
3367 1438198 : add_candidate (data, build_int_cst (TREE_TYPE (iv->base), 0),
3368 : iv->step, true, NULL);
3369 :
3370 1776200 : phi = SSA_NAME_DEF_STMT (iv->ssa_name);
3371 1776200 : if (gimple_code (phi) == GIMPLE_PHI)
3372 : {
3373 : /* Additionally record the possibility of leaving the original iv
3374 : untouched. */
3375 888191 : def = PHI_ARG_DEF_FROM_EDGE (phi, loop_latch_edge (data->current_loop));
3376 : /* Don't add candidate if it's from another PHI node because
3377 : it's an affine iv appearing in the form of PEELED_CHREC. */
3378 888191 : phi = SSA_NAME_DEF_STMT (def);
3379 888191 : if (gimple_code (phi) != GIMPLE_PHI)
3380 : {
3381 1776382 : cand = add_candidate_1 (data,
3382 : iv->base, iv->step, true, IP_ORIGINAL, NULL,
3383 888191 : SSA_NAME_DEF_STMT (def));
3384 888191 : if (cand)
3385 : {
3386 885838 : cand->var_before = iv->ssa_name;
3387 885838 : cand->var_after = def;
3388 : }
3389 : }
3390 : else
3391 0 : gcc_assert (gimple_bb (phi) == data->current_loop->header);
3392 : }
3393 1776200 : }
3394 :
3395 : /* Adds candidates based on the old induction variables. */
3396 :
3397 : static void
3398 513181 : add_iv_candidate_for_bivs (struct ivopts_data *data)
3399 : {
3400 513181 : unsigned i;
3401 513181 : struct iv *iv;
3402 513181 : bitmap_iterator bi;
3403 :
3404 5589791 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
3405 : {
3406 5076610 : iv = ver_info (data, i)->iv;
3407 5076610 : if (iv && iv->biv_p && !integer_zerop (iv->step))
3408 1776200 : add_iv_candidate_for_biv (data, iv);
3409 : }
3410 513181 : }
3411 :
3412 : /* Record common candidate {BASE, STEP} derived from USE in hashtable. */
3413 :
3414 : static void
3415 4212556 : record_common_cand (struct ivopts_data *data, tree base,
3416 : tree step, struct iv_use *use)
3417 : {
3418 4212556 : class iv_common_cand ent;
3419 4212556 : class iv_common_cand **slot;
3420 :
3421 4212556 : ent.base = base;
3422 4212556 : ent.step = step;
3423 4212556 : ent.hash = iterative_hash_expr (base, 0);
3424 4212556 : ent.hash = iterative_hash_expr (step, ent.hash);
3425 :
3426 4212556 : slot = data->iv_common_cand_tab->find_slot (&ent, INSERT);
3427 4212556 : if (*slot == NULL)
3428 : {
3429 2652723 : *slot = new iv_common_cand ();
3430 2652723 : (*slot)->base = base;
3431 2652723 : (*slot)->step = step;
3432 2652723 : (*slot)->uses.create (8);
3433 2652723 : (*slot)->hash = ent.hash;
3434 2652723 : data->iv_common_cands.safe_push ((*slot));
3435 : }
3436 :
3437 4212556 : gcc_assert (use != NULL);
3438 4212556 : (*slot)->uses.safe_push (use);
3439 4212556 : return;
3440 4212556 : }
3441 :
3442 : /* Comparison function used to sort common candidates. */
3443 :
3444 : static int
3445 19274484 : common_cand_cmp (const void *p1, const void *p2)
3446 : {
3447 19274484 : unsigned n1, n2;
3448 19274484 : const class iv_common_cand *const *const ccand1
3449 : = (const class iv_common_cand *const *)p1;
3450 19274484 : const class iv_common_cand *const *const ccand2
3451 : = (const class iv_common_cand *const *)p2;
3452 :
3453 19274484 : n1 = (*ccand1)->uses.length ();
3454 19274484 : n2 = (*ccand2)->uses.length ();
3455 19274484 : return n2 - n1;
3456 : }
3457 :
3458 : /* Adds IV candidates based on common candidated recorded. */
3459 :
3460 : static void
3461 513181 : add_iv_candidate_derived_from_uses (struct ivopts_data *data)
3462 : {
3463 513181 : unsigned i, j;
3464 513181 : struct iv_cand *cand_1, *cand_2;
3465 :
3466 513181 : data->iv_common_cands.qsort (common_cand_cmp);
3467 1485286 : for (i = 0; i < data->iv_common_cands.length (); i++)
3468 : {
3469 1467946 : class iv_common_cand *ptr = data->iv_common_cands[i];
3470 :
3471 : /* Only add IV candidate if it's derived from multiple uses. */
3472 1467946 : if (ptr->uses.length () <= 1)
3473 : break;
3474 :
3475 972105 : cand_1 = NULL;
3476 972105 : cand_2 = NULL;
3477 972105 : if (ip_normal_pos (data->current_loop))
3478 955083 : cand_1 = add_candidate_1 (data, ptr->base, ptr->step,
3479 : false, IP_NORMAL, NULL, NULL);
3480 :
3481 972105 : if (ip_end_pos (data->current_loop)
3482 972105 : && allow_ip_end_pos_p (data->current_loop))
3483 46085 : cand_2 = add_candidate_1 (data, ptr->base, ptr->step,
3484 : false, IP_END, NULL, NULL);
3485 :
3486 : /* Bind deriving uses and the new candidates. */
3487 3504043 : for (j = 0; j < ptr->uses.length (); j++)
3488 : {
3489 2531938 : struct iv_group *group = data->vgroups[ptr->uses[j]->group_id];
3490 2531938 : if (cand_1)
3491 2452228 : bitmap_set_bit (group->related_cands, cand_1->id);
3492 2531938 : if (cand_2)
3493 137427 : bitmap_set_bit (group->related_cands, cand_2->id);
3494 : }
3495 : }
3496 :
3497 : /* Release data since it is useless from this point. */
3498 513181 : data->iv_common_cand_tab->empty ();
3499 513181 : data->iv_common_cands.truncate (0);
3500 513181 : }
3501 :
3502 : /* Adds candidates based on the value of USE's iv. */
3503 :
3504 : static void
3505 1667999 : add_iv_candidate_for_use (struct ivopts_data *data, struct iv_use *use)
3506 : {
3507 1667999 : poly_uint64 offset;
3508 1667999 : tree base;
3509 1667999 : struct iv *iv = use->iv;
3510 1667999 : tree basetype = TREE_TYPE (iv->base);
3511 :
3512 : /* Don't add candidate for iv_use with non integer, pointer or non-mode
3513 : precision types, instead, add candidate for the corresponding scev in
3514 : unsigned type with the same precision. See PR93674 for more info. */
3515 783504 : if ((TREE_CODE (basetype) != INTEGER_TYPE && !POINTER_TYPE_P (basetype))
3516 2451217 : || !type_has_mode_precision_p (basetype))
3517 : {
3518 323 : basetype = lang_hooks.types.type_for_mode (TYPE_MODE (basetype),
3519 323 : TYPE_UNSIGNED (basetype));
3520 323 : add_candidate (data, fold_convert (basetype, iv->base),
3521 : fold_convert (basetype, iv->step), false, NULL);
3522 323 : return;
3523 : }
3524 :
3525 1667676 : add_candidate (data, iv->base, iv->step, false, use);
3526 :
3527 : /* Record common candidate for use in case it can be shared by others. */
3528 1667676 : record_common_cand (data, iv->base, iv->step, use);
3529 :
3530 : /* Record common candidate with initial value zero. */
3531 1667676 : basetype = TREE_TYPE (iv->base);
3532 1667676 : if (POINTER_TYPE_P (basetype))
3533 783218 : basetype = sizetype;
3534 1667676 : record_common_cand (data, build_int_cst (basetype, 0), iv->step, use);
3535 :
3536 : /* Compare the cost of an address with an unscaled index with the cost of
3537 : an address with a scaled index and add candidate if useful. */
3538 1667676 : poly_int64 step;
3539 1667676 : if (use != NULL
3540 1667676 : && poly_int_tree_p (iv->step, &step)
3541 1433622 : && address_p (use->type))
3542 : {
3543 541754 : poly_int64 new_step;
3544 541754 : unsigned int fact = preferred_mem_scale_factor
3545 1083508 : (use->iv->base,
3546 541754 : TYPE_MODE (use->mem_type),
3547 : optimize_loop_for_speed_p (data->current_loop));
3548 :
3549 541754 : if (fact != 1
3550 541754 : && multiple_p (step, fact, &new_step))
3551 0 : add_candidate (data, size_int (0),
3552 0 : wide_int_to_tree (sizetype, new_step),
3553 : true, NULL);
3554 : }
3555 :
3556 : /* Record common candidate with constant offset stripped in base.
3557 : Like the use itself, we also add candidate directly for it. */
3558 1667676 : base = strip_offset (iv->base, &offset);
3559 1667676 : if (maybe_ne (offset, 0U) || base != iv->base)
3560 : {
3561 877204 : record_common_cand (data, base, iv->step, use);
3562 877204 : add_candidate (data, base, iv->step, false, use);
3563 : }
3564 :
3565 : /* Record common candidate with base_object removed in base. */
3566 1667676 : base = iv->base;
3567 1667676 : STRIP_NOPS (base);
3568 1667676 : if (iv->base_object != NULL && TREE_CODE (base) == POINTER_PLUS_EXPR)
3569 : {
3570 0 : tree step = iv->step;
3571 :
3572 0 : STRIP_NOPS (step);
3573 0 : base = TREE_OPERAND (base, 1);
3574 0 : step = fold_convert (sizetype, step);
3575 0 : record_common_cand (data, base, step, use);
3576 : /* Also record common candidate with offset stripped. */
3577 0 : tree alt_base, alt_offset;
3578 0 : split_constant_offset (base, &alt_base, &alt_offset);
3579 0 : if (!integer_zerop (alt_offset))
3580 0 : record_common_cand (data, alt_base, step, use);
3581 : }
3582 :
3583 : /* At last, add auto-incremental candidates. Make such variables
3584 : important since other iv uses with same base object may be based
3585 : on it. */
3586 1667676 : if (use != NULL && address_p (use->type))
3587 591454 : add_autoinc_candidates (data, iv->base, iv->step, true, use);
3588 : }
3589 :
3590 : /* Adds candidates based on the uses. */
3591 :
3592 : static void
3593 513181 : add_iv_candidate_for_groups (struct ivopts_data *data)
3594 : {
3595 513181 : unsigned i;
3596 :
3597 : /* Only add candidate for the first use in group. */
3598 2181180 : for (i = 0; i < data->vgroups.length (); i++)
3599 : {
3600 1667999 : struct iv_group *group = data->vgroups[i];
3601 :
3602 1667999 : gcc_assert (group->vuses[0] != NULL);
3603 1667999 : add_iv_candidate_for_use (data, group->vuses[0]);
3604 : }
3605 513181 : add_iv_candidate_derived_from_uses (data);
3606 513181 : }
3607 :
3608 : /* Record important candidates and add them to related_cands bitmaps. */
3609 :
3610 : static void
3611 513181 : record_important_candidates (struct ivopts_data *data)
3612 : {
3613 513181 : unsigned i;
3614 513181 : struct iv_group *group;
3615 :
3616 5205582 : for (i = 0; i < data->vcands.length (); i++)
3617 : {
3618 4692401 : struct iv_cand *cand = data->vcands[i];
3619 :
3620 4692401 : if (cand->important)
3621 3766302 : bitmap_set_bit (data->important_candidates, i);
3622 : }
3623 :
3624 513181 : data->consider_all_candidates = (data->vcands.length ()
3625 513181 : <= CONSIDER_ALL_CANDIDATES_BOUND);
3626 :
3627 : /* Add important candidates to groups' related_cands bitmaps. */
3628 2181180 : for (i = 0; i < data->vgroups.length (); i++)
3629 : {
3630 1667999 : group = data->vgroups[i];
3631 1667999 : bitmap_ior_into (group->related_cands, data->important_candidates);
3632 : }
3633 513181 : }
3634 :
3635 : /* Allocates the data structure mapping the (use, candidate) pairs to costs.
3636 : If consider_all_candidates is true, we use a two-dimensional array, otherwise
3637 : we allocate a simple list to every use. */
3638 :
3639 : static void
3640 513181 : alloc_use_cost_map (struct ivopts_data *data)
3641 : {
3642 513181 : unsigned i, size, s;
3643 :
3644 2181180 : for (i = 0; i < data->vgroups.length (); i++)
3645 : {
3646 1667999 : struct iv_group *group = data->vgroups[i];
3647 :
3648 1667999 : if (data->consider_all_candidates)
3649 1658678 : size = data->vcands.length ();
3650 : else
3651 : {
3652 9321 : s = bitmap_count_bits (group->related_cands);
3653 :
3654 : /* Round up to the power of two, so that moduling by it is fast. */
3655 18642 : size = s ? (1 << ceil_log2 (s)) : 1;
3656 : }
3657 :
3658 1667999 : group->n_map_members = size;
3659 1667999 : group->cost_map = XCNEWVEC (class cost_pair, size);
3660 : }
3661 513181 : }
3662 :
3663 : /* Sets cost of (GROUP, CAND) pair to COST and record that it depends
3664 : on invariants INV_VARS and that the value used in expressing it is
3665 : VALUE, and in case of iv elimination the comparison operator is COMP. */
3666 :
3667 : static void
3668 17888643 : set_group_iv_cost (struct ivopts_data *data,
3669 : struct iv_group *group, struct iv_cand *cand,
3670 : comp_cost cost, bitmap inv_vars, tree value,
3671 : enum tree_code comp, bitmap inv_exprs)
3672 : {
3673 17888643 : unsigned i, s;
3674 :
3675 17888643 : if (cost.infinite_cost_p ())
3676 : {
3677 6186034 : BITMAP_FREE (inv_vars);
3678 6186034 : BITMAP_FREE (inv_exprs);
3679 6186034 : return;
3680 : }
3681 :
3682 11702609 : if (data->consider_all_candidates)
3683 : {
3684 11568723 : group->cost_map[cand->id].cand = cand;
3685 11568723 : group->cost_map[cand->id].cost = cost;
3686 11568723 : group->cost_map[cand->id].inv_vars = inv_vars;
3687 11568723 : group->cost_map[cand->id].inv_exprs = inv_exprs;
3688 11568723 : group->cost_map[cand->id].value = value;
3689 11568723 : group->cost_map[cand->id].comp = comp;
3690 11568723 : return;
3691 : }
3692 :
3693 : /* n_map_members is a power of two, so this computes modulo. */
3694 133886 : s = cand->id & (group->n_map_members - 1);
3695 144474 : for (i = s; i < group->n_map_members; i++)
3696 144407 : if (!group->cost_map[i].cand)
3697 133819 : goto found;
3698 144 : for (i = 0; i < s; i++)
3699 144 : if (!group->cost_map[i].cand)
3700 67 : goto found;
3701 :
3702 0 : gcc_unreachable ();
3703 :
3704 133886 : found:
3705 133886 : group->cost_map[i].cand = cand;
3706 133886 : group->cost_map[i].cost = cost;
3707 133886 : group->cost_map[i].inv_vars = inv_vars;
3708 133886 : group->cost_map[i].inv_exprs = inv_exprs;
3709 133886 : group->cost_map[i].value = value;
3710 133886 : group->cost_map[i].comp = comp;
3711 : }
3712 :
3713 : /* Gets cost of (GROUP, CAND) pair. */
3714 :
3715 : static class cost_pair *
3716 213389027 : get_group_iv_cost (struct ivopts_data *data, struct iv_group *group,
3717 : struct iv_cand *cand)
3718 : {
3719 213389027 : unsigned i, s;
3720 213389027 : class cost_pair *ret;
3721 :
3722 213389027 : if (!cand)
3723 : return NULL;
3724 :
3725 207459189 : if (data->consider_all_candidates)
3726 : {
3727 193656799 : ret = group->cost_map + cand->id;
3728 193656799 : if (!ret->cand)
3729 : return NULL;
3730 :
3731 113183810 : return ret;
3732 : }
3733 :
3734 : /* n_map_members is a power of two, so this computes modulo. */
3735 13802390 : s = cand->id & (group->n_map_members - 1);
3736 19865161 : for (i = s; i < group->n_map_members; i++)
3737 19778575 : if (group->cost_map[i].cand == cand)
3738 : return group->cost_map + i;
3739 12301168 : else if (group->cost_map[i].cand == NULL)
3740 : return NULL;
3741 253521 : for (i = 0; i < s; i++)
3742 226317 : if (group->cost_map[i].cand == cand)
3743 : return group->cost_map + i;
3744 224805 : else if (group->cost_map[i].cand == NULL)
3745 : return NULL;
3746 :
3747 : return NULL;
3748 : }
3749 :
3750 : /* Produce DECL_RTL for object obj so it looks like it is stored in memory. */
3751 : static rtx
3752 43298 : produce_memory_decl_rtl (tree obj, int *regno)
3753 : {
3754 43298 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
3755 43298 : machine_mode address_mode = targetm.addr_space.address_mode (as);
3756 43298 : rtx x;
3757 :
3758 43298 : gcc_assert (obj);
3759 43298 : if (TREE_STATIC (obj) || DECL_EXTERNAL (obj))
3760 : {
3761 43298 : const char *name = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (obj));
3762 43298 : x = gen_rtx_SYMBOL_REF (address_mode, name);
3763 43298 : SET_SYMBOL_REF_DECL (x, obj);
3764 43298 : x = gen_rtx_MEM (DECL_MODE (obj), x);
3765 43298 : set_mem_addr_space (x, as);
3766 43298 : targetm.encode_section_info (obj, x, true);
3767 : }
3768 : else
3769 : {
3770 0 : x = gen_raw_REG (address_mode, (*regno)++);
3771 0 : x = gen_rtx_MEM (DECL_MODE (obj), x);
3772 0 : set_mem_addr_space (x, as);
3773 : }
3774 :
3775 43298 : return x;
3776 : }
3777 :
3778 : /* Prepares decl_rtl for variables referred in *EXPR_P. Callback for
3779 : walk_tree. DATA contains the actual fake register number. */
3780 :
3781 : static tree
3782 606172 : prepare_decl_rtl (tree *expr_p, int *ws, void *data)
3783 : {
3784 606172 : tree obj = NULL_TREE;
3785 606172 : rtx x = NULL_RTX;
3786 606172 : int *regno = (int *) data;
3787 :
3788 606172 : switch (TREE_CODE (*expr_p))
3789 : {
3790 173192 : case ADDR_EXPR:
3791 173192 : for (expr_p = &TREE_OPERAND (*expr_p, 0);
3792 173192 : handled_component_p (*expr_p);
3793 0 : expr_p = &TREE_OPERAND (*expr_p, 0))
3794 0 : continue;
3795 173192 : obj = *expr_p;
3796 173192 : if (DECL_P (obj) && HAS_RTL_P (obj) && !DECL_RTL_SET_P (obj))
3797 0 : x = produce_memory_decl_rtl (obj, regno);
3798 : break;
3799 :
3800 0 : case SSA_NAME:
3801 0 : *ws = 0;
3802 0 : obj = SSA_NAME_VAR (*expr_p);
3803 : /* Defer handling of anonymous SSA_NAMEs to the expander. */
3804 0 : if (!obj)
3805 : return NULL_TREE;
3806 0 : if (!DECL_RTL_SET_P (obj))
3807 0 : x = gen_raw_REG (DECL_MODE (obj), (*regno)++);
3808 : break;
3809 :
3810 173192 : case VAR_DECL:
3811 173192 : case PARM_DECL:
3812 173192 : case RESULT_DECL:
3813 173192 : *ws = 0;
3814 173192 : obj = *expr_p;
3815 :
3816 173192 : if (DECL_RTL_SET_P (obj))
3817 : break;
3818 :
3819 0 : if (DECL_MODE (obj) == BLKmode)
3820 0 : x = produce_memory_decl_rtl (obj, regno);
3821 : else
3822 0 : x = gen_raw_REG (DECL_MODE (obj), (*regno)++);
3823 :
3824 : break;
3825 :
3826 : default:
3827 : break;
3828 : }
3829 :
3830 0 : if (x)
3831 : {
3832 0 : decl_rtl_to_reset.safe_push (obj);
3833 0 : SET_DECL_RTL (obj, x);
3834 : }
3835 :
3836 : return NULL_TREE;
3837 : }
3838 :
3839 : /* Predict whether the given loop will be transformed in the RTL
3840 : doloop_optimize pass. Attempt to duplicate some doloop_optimize checks.
3841 : This is only for target independent checks, see targetm.predict_doloop_p
3842 : for the target dependent ones.
3843 :
3844 : Note that according to some initial investigation, some checks like costly
3845 : niter check and invalid stmt scanning don't have much gains among general
3846 : cases, so keep this as simple as possible first.
3847 :
3848 : Some RTL specific checks seems unable to be checked in gimple, if any new
3849 : checks or easy checks _are_ missing here, please add them. */
3850 :
3851 : static bool
3852 513181 : generic_predict_doloop_p (struct ivopts_data *data)
3853 : {
3854 513181 : class loop *loop = data->current_loop;
3855 :
3856 : /* Call target hook for target dependent checks. */
3857 513181 : if (!targetm.predict_doloop_p (loop))
3858 : {
3859 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
3860 72 : fprintf (dump_file, "Predict doloop failure due to"
3861 : " target specific checks.\n");
3862 : return false;
3863 : }
3864 :
3865 : /* Similar to doloop_optimize, check iteration description to know it's
3866 : suitable or not. Keep it as simple as possible, feel free to extend it
3867 : if you find any multiple exits cases matter. */
3868 0 : edge exit = single_dom_exit (loop);
3869 0 : class tree_niter_desc *niter_desc;
3870 0 : if (!exit || !(niter_desc = niter_for_exit (data, exit)))
3871 : {
3872 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3873 0 : fprintf (dump_file, "Predict doloop failure due to"
3874 : " unexpected niters.\n");
3875 : return false;
3876 : }
3877 :
3878 : /* Similar to doloop_optimize, check whether iteration count too small
3879 : and not profitable. */
3880 0 : HOST_WIDE_INT est_niter = get_estimated_loop_iterations_int (loop);
3881 0 : if (est_niter == -1)
3882 0 : est_niter = get_likely_max_loop_iterations_int (loop);
3883 0 : if (est_niter >= 0 && est_niter < 3)
3884 : {
3885 0 : if (dump_file && (dump_flags & TDF_DETAILS))
3886 0 : fprintf (dump_file,
3887 : "Predict doloop failure due to"
3888 : " too few iterations (%u).\n",
3889 : (unsigned int) est_niter);
3890 : return false;
3891 : }
3892 :
3893 : return true;
3894 : }
3895 :
3896 : /* Determines cost of the computation of EXPR. */
3897 :
3898 : static unsigned
3899 259788 : computation_cost (tree expr, bool speed)
3900 : {
3901 259788 : rtx_insn *seq;
3902 259788 : rtx rslt;
3903 259788 : tree type = TREE_TYPE (expr);
3904 259788 : unsigned cost;
3905 : /* Avoid using hard regs in ways which may be unsupported. */
3906 259788 : int regno = LAST_VIRTUAL_REGISTER + 1;
3907 259788 : struct cgraph_node *node = cgraph_node::get (current_function_decl);
3908 259788 : enum node_frequency real_frequency = node->frequency;
3909 :
3910 259788 : node->frequency = NODE_FREQUENCY_NORMAL;
3911 259788 : crtl->maybe_hot_insn_p = speed;
3912 259788 : walk_tree (&expr, prepare_decl_rtl, ®no, NULL);
3913 259788 : start_sequence ();
3914 259788 : rslt = expand_expr (expr, NULL_RTX, TYPE_MODE (type), EXPAND_NORMAL);
3915 259788 : seq = end_sequence ();
3916 259788 : default_rtl_profile ();
3917 259788 : node->frequency = real_frequency;
3918 :
3919 259788 : cost = seq_cost (seq, speed);
3920 259788 : if (MEM_P (rslt))
3921 0 : cost += address_cost (XEXP (rslt, 0), TYPE_MODE (type),
3922 0 : TYPE_ADDR_SPACE (type), speed);
3923 259788 : else if (!REG_P (rslt))
3924 519576 : cost += set_src_cost (rslt, TYPE_MODE (type), speed);
3925 :
3926 259788 : return cost;
3927 : }
3928 :
3929 : /* Returns variable containing the value of candidate CAND at statement AT. */
3930 :
3931 : static tree
3932 18566970 : var_at_stmt (class loop *loop, struct iv_cand *cand, gimple *stmt)
3933 : {
3934 18566970 : if (stmt_after_increment (loop, cand, stmt))
3935 4827579 : return cand->var_after;
3936 : else
3937 13739391 : return cand->var_before;
3938 : }
3939 :
3940 : /* If A is (TYPE) BA and B is (TYPE) BB, and the types of BA and BB have the
3941 : same precision that is at least as wide as the precision of TYPE, stores
3942 : BA to A and BB to B, and returns the type of BA. Otherwise, returns the
3943 : type of A and B. */
3944 :
3945 : static tree
3946 14286012 : determine_common_wider_type (tree *a, tree *b)
3947 : {
3948 14286012 : tree wider_type = NULL;
3949 14286012 : tree suba, subb;
3950 14286012 : tree atype = TREE_TYPE (*a);
3951 :
3952 14286012 : if (CONVERT_EXPR_P (*a))
3953 : {
3954 7971849 : suba = TREE_OPERAND (*a, 0);
3955 7971849 : wider_type = TREE_TYPE (suba);
3956 7971849 : if (TYPE_PRECISION (wider_type) < TYPE_PRECISION (atype))
3957 : return atype;
3958 : }
3959 : else
3960 : return atype;
3961 :
3962 7953597 : if (CONVERT_EXPR_P (*b))
3963 : {
3964 1633703 : subb = TREE_OPERAND (*b, 0);
3965 1633703 : if (TYPE_PRECISION (wider_type) != TYPE_PRECISION (TREE_TYPE (subb)))
3966 : return atype;
3967 : }
3968 : else
3969 : return atype;
3970 :
3971 1562429 : *a = suba;
3972 1562429 : *b = subb;
3973 1562429 : return wider_type;
3974 : }
3975 :
3976 : /* Determines the expression by that USE is expressed from induction variable
3977 : CAND at statement AT in DATA's current loop. The expression is stored in
3978 : two parts in a decomposed form. The invariant part is stored in AFF_INV;
3979 : while variant part in AFF_VAR. Store ratio of CAND.step over USE.step in
3980 : PRAT if it's non-null. Returns false if USE cannot be expressed using
3981 : CAND. */
3982 :
3983 : static bool
3984 17288720 : get_computation_aff_1 (struct ivopts_data *data, gimple *at, struct iv_use *use,
3985 : struct iv_cand *cand, class aff_tree *aff_inv,
3986 : class aff_tree *aff_var, widest_int *prat = NULL)
3987 : {
3988 17288720 : tree ubase = use->iv->base, ustep = use->iv->step;
3989 17288720 : tree cbase = cand->iv->base, cstep = cand->iv->step;
3990 17288720 : tree common_type, uutype, var, cstep_common;
3991 17288720 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
3992 17288720 : aff_tree aff_cbase;
3993 17288720 : widest_int rat;
3994 :
3995 : /* We must have a precision to express the values of use. */
3996 17288720 : if (TYPE_PRECISION (utype) > TYPE_PRECISION (ctype))
3997 : return false;
3998 :
3999 17285599 : var = var_at_stmt (data->current_loop, cand, at);
4000 17285599 : uutype = unsigned_type_for (utype);
4001 :
4002 : /* If the conversion is not noop, perform it. */
4003 17285599 : if (TYPE_PRECISION (utype) < TYPE_PRECISION (ctype))
4004 : {
4005 265836 : if (cand->orig_iv != NULL && CONVERT_EXPR_P (cbase)
4006 1663620 : && (CONVERT_EXPR_P (cstep) || poly_int_tree_p (cstep)))
4007 : {
4008 30017 : tree inner_base, inner_step, inner_type;
4009 30017 : inner_base = TREE_OPERAND (cbase, 0);
4010 30017 : if (CONVERT_EXPR_P (cstep))
4011 1416 : inner_step = TREE_OPERAND (cstep, 0);
4012 : else
4013 : inner_step = cstep;
4014 :
4015 30017 : inner_type = TREE_TYPE (inner_base);
4016 : /* If candidate is added from a biv whose type is smaller than
4017 : ctype, we know both candidate and the biv won't overflow.
4018 : In this case, it's safe to skip the conversion in candidate.
4019 : As an example, (unsigned short)((unsigned long)A) equals to
4020 : (unsigned short)A, if A has a type no larger than short. */
4021 30017 : if (TYPE_PRECISION (inner_type) <= TYPE_PRECISION (uutype))
4022 : {
4023 29096 : cbase = inner_base;
4024 29096 : cstep = inner_step;
4025 : }
4026 : }
4027 1633603 : cbase = fold_convert (uutype, cbase);
4028 1633603 : cstep = fold_convert (uutype, cstep);
4029 1633603 : var = fold_convert (uutype, var);
4030 : }
4031 :
4032 : /* Ratio is 1 when computing the value of biv cand by itself.
4033 : We can't rely on constant_multiple_of in this case because the
4034 : use is created after the original biv is selected. The call
4035 : could fail because of inconsistent fold behavior. See PR68021
4036 : for more information. */
4037 17285599 : if (cand->pos == IP_ORIGINAL && cand->incremented_at == use->stmt)
4038 : {
4039 2870 : gcc_assert (is_gimple_assign (use->stmt));
4040 2870 : gcc_assert (use->iv->ssa_name == cand->var_after);
4041 2870 : gcc_assert (gimple_assign_lhs (use->stmt) == cand->var_after);
4042 2870 : rat = 1;
4043 : }
4044 17282729 : else if (!constant_multiple_of (ustep, cstep, &rat, data))
4045 : return false;
4046 :
4047 14286012 : if (prat)
4048 12810988 : *prat = rat;
4049 :
4050 : /* In case both UBASE and CBASE are shortened to UUTYPE from some common
4051 : type, we achieve better folding by computing their difference in this
4052 : wider type, and cast the result to UUTYPE. We do not need to worry about
4053 : overflows, as all the arithmetics will in the end be performed in UUTYPE
4054 : anyway. */
4055 14286012 : common_type = determine_common_wider_type (&ubase, &cbase);
4056 :
4057 : /* use = ubase - ratio * cbase + ratio * var. */
4058 14286012 : tree_to_aff_combination (ubase, common_type, aff_inv, at);
4059 14286012 : tree_to_aff_combination (cbase, common_type, &aff_cbase, at);
4060 14286012 : tree_to_aff_combination (var, uutype, aff_var, at);
4061 :
4062 : /* We need to shift the value if we are after the increment. */
4063 14286012 : if (stmt_after_increment (data->current_loop, cand, at))
4064 : {
4065 3275960 : aff_tree cstep_aff;
4066 :
4067 3275960 : if (common_type != uutype)
4068 840501 : cstep_common = fold_convert (common_type, cstep);
4069 : else
4070 : cstep_common = cstep;
4071 :
4072 3275960 : tree_to_aff_combination (cstep_common, common_type, &cstep_aff, at);
4073 3275960 : aff_combination_add (&aff_cbase, &cstep_aff);
4074 3275960 : }
4075 :
4076 14286012 : aff_combination_scale (&aff_cbase, -rat);
4077 14286012 : aff_combination_add (aff_inv, &aff_cbase);
4078 14286012 : if (common_type != uutype)
4079 9634372 : aff_combination_convert (aff_inv, uutype, at);
4080 :
4081 14286012 : aff_combination_scale (aff_var, rat);
4082 14286012 : return true;
4083 17288720 : }
4084 :
4085 : /* Determines the expression by that USE is expressed from induction variable
4086 : CAND at statement AT in DATA's current loop. The expression is stored in a
4087 : decomposed form into AFF. Returns false if USE cannot be expressed using
4088 : CAND. */
4089 :
4090 : static bool
4091 1268555 : get_computation_aff (struct ivopts_data *data, gimple *at, struct iv_use *use,
4092 : struct iv_cand *cand, class aff_tree *aff)
4093 : {
4094 1268555 : aff_tree aff_var;
4095 :
4096 1268555 : if (!get_computation_aff_1 (data, at, use, cand, aff, &aff_var))
4097 : return false;
4098 :
4099 1133197 : aff_combination_add (aff, &aff_var);
4100 1133197 : return true;
4101 1268555 : }
4102 :
4103 : /* Return the type of USE. */
4104 :
4105 : static tree
4106 1033446 : get_use_type (struct iv_use *use)
4107 : {
4108 1033446 : tree base_type = TREE_TYPE (use->iv->base);
4109 1033446 : tree type;
4110 :
4111 1033446 : if (use->type == USE_REF_ADDRESS)
4112 : {
4113 : /* The base_type may be a void pointer. Create a pointer type based on
4114 : the mem_ref instead. */
4115 0 : type = build_pointer_type (TREE_TYPE (*use->op_p));
4116 0 : gcc_assert (TYPE_ADDR_SPACE (TREE_TYPE (type))
4117 : == TYPE_ADDR_SPACE (TREE_TYPE (base_type)));
4118 : }
4119 : else
4120 : type = base_type;
4121 :
4122 1033446 : return type;
4123 : }
4124 :
4125 : /* Determines the expression by that USE is expressed from induction variable
4126 : CAND at statement AT in DATA's current loop. The computation is
4127 : unshared. */
4128 :
4129 : static tree
4130 402807 : get_computation_at (struct ivopts_data *data, gimple *at,
4131 : struct iv_use *use, struct iv_cand *cand)
4132 : {
4133 402807 : aff_tree aff;
4134 402807 : tree type = get_use_type (use);
4135 :
4136 402807 : if (!get_computation_aff (data, at, use, cand, &aff))
4137 : return NULL_TREE;
4138 267449 : unshare_aff_combination (&aff);
4139 267449 : return fold_convert (type, aff_combination_to_tree (&aff));
4140 402807 : }
4141 :
4142 : /* Like get_computation_at, but try harder, even if the computation
4143 : is more expensive. Intended for debug stmts. */
4144 :
4145 : static tree
4146 205096 : get_debug_computation_at (struct ivopts_data *data, gimple *at,
4147 : struct iv_use *use, struct iv_cand *cand)
4148 : {
4149 205096 : if (tree ret = get_computation_at (data, at, use, cand))
4150 : return ret;
4151 :
4152 135358 : tree ubase = use->iv->base, ustep = use->iv->step;
4153 135358 : tree cbase = cand->iv->base, cstep = cand->iv->step;
4154 135358 : tree var;
4155 135358 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
4156 135358 : widest_int rat;
4157 :
4158 : /* We must have a precision to express the values of use. */
4159 135358 : if (TYPE_PRECISION (utype) >= TYPE_PRECISION (ctype))
4160 : return NULL_TREE;
4161 :
4162 : /* Try to handle the case that get_computation_at doesn't,
4163 : try to express
4164 : use = ubase + (var - cbase) / ratio. */
4165 9687 : if (!constant_multiple_of (cstep, fold_convert (TREE_TYPE (cstep), ustep),
4166 : &rat, data))
4167 : return NULL_TREE;
4168 :
4169 8527 : bool neg_p = false;
4170 8527 : if (wi::neg_p (rat))
4171 : {
4172 877 : if (TYPE_UNSIGNED (ctype))
4173 : return NULL_TREE;
4174 0 : neg_p = true;
4175 0 : rat = wi::neg (rat);
4176 : }
4177 :
4178 : /* If both IVs can wrap around and CAND doesn't have a power of two step,
4179 : it is unsafe. Consider uint16_t CAND with step 9, when wrapping around,
4180 : the values will be ... 0xfff0, 0xfff9, 2, 11 ... and when use is say
4181 : uint8_t with step 3, those values divided by 3 cast to uint8_t will be
4182 : ... 0x50, 0x53, 0, 3 ... rather than expected 0x50, 0x53, 0x56, 0x59. */
4183 7650 : if (!use->iv->no_overflow
4184 68 : && !cand->iv->no_overflow
4185 7706 : && !integer_pow2p (cstep))
4186 : return NULL_TREE;
4187 :
4188 7635 : int bits = wi::exact_log2 (rat);
4189 7635 : if (bits == -1)
4190 750 : bits = wi::floor_log2 (rat) + 1;
4191 7635 : if (!cand->iv->no_overflow
4192 7635 : && TYPE_PRECISION (utype) + bits > TYPE_PRECISION (ctype))
4193 : return NULL_TREE;
4194 :
4195 7635 : var = var_at_stmt (data->current_loop, cand, at);
4196 :
4197 7635 : if (POINTER_TYPE_P (ctype))
4198 : {
4199 135 : ctype = unsigned_type_for (ctype);
4200 135 : cbase = fold_convert (ctype, cbase);
4201 135 : cstep = fold_convert (ctype, cstep);
4202 135 : var = fold_convert (ctype, var);
4203 : }
4204 :
4205 7635 : if (stmt_after_increment (data->current_loop, cand, at))
4206 80 : var = fold_build2 (MINUS_EXPR, TREE_TYPE (var), var,
4207 : unshare_expr (cstep));
4208 :
4209 7635 : var = fold_build2 (MINUS_EXPR, TREE_TYPE (var), var, cbase);
4210 7635 : var = fold_build2 (EXACT_DIV_EXPR, TREE_TYPE (var), var,
4211 : wide_int_to_tree (TREE_TYPE (var), rat));
4212 7635 : if (POINTER_TYPE_P (utype))
4213 : {
4214 0 : var = fold_convert (sizetype, var);
4215 0 : if (neg_p)
4216 0 : var = fold_build1 (NEGATE_EXPR, sizetype, var);
4217 0 : var = fold_build2 (POINTER_PLUS_EXPR, utype, ubase, var);
4218 : }
4219 : else
4220 : {
4221 7635 : var = fold_convert (utype, var);
4222 15270 : var = fold_build2 (neg_p ? MINUS_EXPR : PLUS_EXPR, utype,
4223 : ubase, var);
4224 : }
4225 : return var;
4226 205096 : }
4227 :
4228 : /* Adjust the cost COST for being in loop setup rather than loop body.
4229 : If we're optimizing for space, the loop setup overhead is constant;
4230 : if we're optimizing for speed, amortize it over the per-iteration cost.
4231 : If ROUND_UP_P is true, the result is round up rather than to zero when
4232 : optimizing for speed. */
4233 : static int64_t
4234 10546120 : adjust_setup_cost (struct ivopts_data *data, int64_t cost,
4235 : bool round_up_p = false)
4236 : {
4237 10546120 : if (cost == INFTY)
4238 : return cost;
4239 10546120 : else if (optimize_loop_for_speed_p (data->current_loop))
4240 : {
4241 8859721 : uint64_t niters = avg_loop_niter (data->current_loop);
4242 8859721 : if (niters > (uint64_t) cost)
4243 7027533 : return (round_up_p && cost != 0) ? 1 : 0;
4244 1832188 : return (cost + (round_up_p ? niters - 1 : 0)) / niters;
4245 : }
4246 : else
4247 : return cost;
4248 : }
4249 :
4250 : /* Calculate the SPEED or size cost of shiftadd EXPR in MODE. MULT is the
4251 : EXPR operand holding the shift. COST0 and COST1 are the costs for
4252 : calculating the operands of EXPR. Returns true if successful, and returns
4253 : the cost in COST. */
4254 :
4255 : static bool
4256 1432672 : get_shiftadd_cost (tree expr, scalar_int_mode mode, comp_cost cost0,
4257 : comp_cost cost1, tree mult, bool speed, comp_cost *cost)
4258 : {
4259 1432672 : comp_cost res;
4260 1432672 : tree op1 = TREE_OPERAND (expr, 1);
4261 1432672 : tree cst = TREE_OPERAND (mult, 1);
4262 1432672 : tree multop = TREE_OPERAND (mult, 0);
4263 1432672 : int m = exact_log2 (int_cst_value (cst));
4264 4297750 : int maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (mode));
4265 1432672 : int as_cost, sa_cost;
4266 1432672 : bool mult_in_op1;
4267 :
4268 1432672 : if (!(m >= 0 && m < maxm))
4269 : return false;
4270 :
4271 953479 : STRIP_NOPS (op1);
4272 953479 : mult_in_op1 = operand_equal_p (op1, mult, 0);
4273 :
4274 953479 : as_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
4275 :
4276 : /* If the target has a cheap shift-and-add or shift-and-sub instruction,
4277 : use that in preference to a shift insn followed by an add insn. */
4278 953479 : sa_cost = (TREE_CODE (expr) != MINUS_EXPR
4279 953479 : ? shiftadd_cost (speed, mode, m)
4280 : : (mult_in_op1
4281 129973 : ? shiftsub1_cost (speed, mode, m)
4282 28328 : : shiftsub0_cost (speed, mode, m)));
4283 :
4284 953479 : res = comp_cost (MIN (as_cost, sa_cost), 0);
4285 1752095 : res += (mult_in_op1 ? cost0 : cost1);
4286 :
4287 953479 : STRIP_NOPS (multop);
4288 953479 : if (!is_gimple_val (multop))
4289 490353 : res += force_expr_to_var_cost (multop, speed);
4290 :
4291 953479 : *cost = res;
4292 953479 : return true;
4293 : }
4294 :
4295 : /* Estimates cost of forcing expression EXPR into a variable. */
4296 :
4297 : static comp_cost
4298 29047630 : force_expr_to_var_cost (tree expr, bool speed)
4299 : {
4300 29047630 : static bool costs_initialized = false;
4301 29047630 : static unsigned integer_cost [2];
4302 29047630 : static unsigned symbol_cost [2];
4303 29047630 : static unsigned address_cost [2];
4304 29047630 : tree op0, op1;
4305 29047630 : comp_cost cost0, cost1, cost;
4306 29047630 : machine_mode mode;
4307 29047630 : scalar_int_mode int_mode;
4308 :
4309 29047630 : if (!costs_initialized)
4310 : {
4311 43298 : tree type = build_pointer_type (integer_type_node);
4312 43298 : tree var, addr;
4313 43298 : rtx x;
4314 43298 : int i;
4315 :
4316 43298 : var = create_tmp_var_raw (integer_type_node, "test_var");
4317 43298 : TREE_STATIC (var) = 1;
4318 43298 : x = produce_memory_decl_rtl (var, NULL);
4319 43298 : SET_DECL_RTL (var, x);
4320 :
4321 43298 : addr = build1 (ADDR_EXPR, type, var);
4322 :
4323 :
4324 173192 : for (i = 0; i < 2; i++)
4325 : {
4326 86596 : integer_cost[i] = computation_cost (build_int_cst (integer_type_node,
4327 : 2000), i);
4328 :
4329 86596 : symbol_cost[i] = computation_cost (addr, i) + 1;
4330 :
4331 86596 : address_cost[i]
4332 86596 : = computation_cost (fold_build_pointer_plus_hwi (addr, 2000), i) + 1;
4333 86596 : if (dump_file && (dump_flags & TDF_DETAILS))
4334 : {
4335 114 : fprintf (dump_file, "force_expr_to_var_cost %s costs:\n", i ? "speed" : "size");
4336 76 : fprintf (dump_file, " integer %d\n", (int) integer_cost[i]);
4337 76 : fprintf (dump_file, " symbol %d\n", (int) symbol_cost[i]);
4338 76 : fprintf (dump_file, " address %d\n", (int) address_cost[i]);
4339 76 : fprintf (dump_file, " other %d\n", (int) target_spill_cost[i]);
4340 76 : fprintf (dump_file, "\n");
4341 : }
4342 : }
4343 :
4344 43298 : costs_initialized = true;
4345 : }
4346 :
4347 29047630 : STRIP_NOPS (expr);
4348 :
4349 29047630 : if (SSA_VAR_P (expr))
4350 5372191 : return no_cost;
4351 :
4352 23675439 : if (is_gimple_min_invariant (expr))
4353 : {
4354 14224880 : if (poly_int_tree_p (expr))
4355 12062386 : return comp_cost (integer_cost [speed], 0);
4356 :
4357 2162494 : if (TREE_CODE (expr) == ADDR_EXPR)
4358 : {
4359 2162494 : tree obj = TREE_OPERAND (expr, 0);
4360 :
4361 2162494 : if (VAR_P (obj)
4362 : || TREE_CODE (obj) == PARM_DECL
4363 : || TREE_CODE (obj) == RESULT_DECL)
4364 2097010 : return comp_cost (symbol_cost [speed], 0);
4365 : }
4366 :
4367 65484 : return comp_cost (address_cost [speed], 0);
4368 : }
4369 :
4370 9450559 : switch (TREE_CODE (expr))
4371 : {
4372 8110509 : case POINTER_PLUS_EXPR:
4373 8110509 : case PLUS_EXPR:
4374 8110509 : case MINUS_EXPR:
4375 8110509 : case MULT_EXPR:
4376 8110509 : case EXACT_DIV_EXPR:
4377 8110509 : case TRUNC_DIV_EXPR:
4378 8110509 : case BIT_AND_EXPR:
4379 8110509 : case BIT_IOR_EXPR:
4380 8110509 : case LSHIFT_EXPR:
4381 8110509 : case RSHIFT_EXPR:
4382 8110509 : op0 = TREE_OPERAND (expr, 0);
4383 8110509 : op1 = TREE_OPERAND (expr, 1);
4384 8110509 : STRIP_NOPS (op0);
4385 8110509 : STRIP_NOPS (op1);
4386 8110509 : break;
4387 :
4388 1340016 : CASE_CONVERT:
4389 1340016 : case NEGATE_EXPR:
4390 1340016 : case BIT_NOT_EXPR:
4391 1340016 : op0 = TREE_OPERAND (expr, 0);
4392 1340016 : STRIP_NOPS (op0);
4393 1340016 : op1 = NULL_TREE;
4394 1340016 : break;
4395 : /* See add_iv_candidate_for_doloop, for doloop may_be_zero case, we
4396 : introduce COND_EXPR for IV base, need to support better cost estimation
4397 : for this COND_EXPR and tcc_comparison. */
4398 0 : case COND_EXPR:
4399 0 : op0 = TREE_OPERAND (expr, 1);
4400 0 : STRIP_NOPS (op0);
4401 0 : op1 = TREE_OPERAND (expr, 2);
4402 0 : STRIP_NOPS (op1);
4403 0 : break;
4404 0 : case LT_EXPR:
4405 0 : case LE_EXPR:
4406 0 : case GT_EXPR:
4407 0 : case GE_EXPR:
4408 0 : case EQ_EXPR:
4409 0 : case NE_EXPR:
4410 0 : case UNORDERED_EXPR:
4411 0 : case ORDERED_EXPR:
4412 0 : case UNLT_EXPR:
4413 0 : case UNLE_EXPR:
4414 0 : case UNGT_EXPR:
4415 0 : case UNGE_EXPR:
4416 0 : case UNEQ_EXPR:
4417 0 : case LTGT_EXPR:
4418 0 : case MAX_EXPR:
4419 0 : case MIN_EXPR:
4420 0 : op0 = TREE_OPERAND (expr, 0);
4421 0 : STRIP_NOPS (op0);
4422 0 : op1 = TREE_OPERAND (expr, 1);
4423 0 : STRIP_NOPS (op1);
4424 0 : break;
4425 :
4426 34 : default:
4427 : /* Just an arbitrary value, FIXME. */
4428 34 : return comp_cost (target_spill_cost[speed], 0);
4429 : }
4430 :
4431 9450525 : if (op0 == NULL_TREE
4432 9450525 : || TREE_CODE (op0) == SSA_NAME || CONSTANT_CLASS_P (op0))
4433 4469190 : cost0 = no_cost;
4434 : else
4435 4981335 : cost0 = force_expr_to_var_cost (op0, speed);
4436 :
4437 9450525 : if (op1 == NULL_TREE
4438 8110509 : || TREE_CODE (op1) == SSA_NAME || CONSTANT_CLASS_P (op1))
4439 8705157 : cost1 = no_cost;
4440 : else
4441 745368 : cost1 = force_expr_to_var_cost (op1, speed);
4442 :
4443 9450525 : mode = TYPE_MODE (TREE_TYPE (expr));
4444 9450525 : switch (TREE_CODE (expr))
4445 : {
4446 5697439 : case POINTER_PLUS_EXPR:
4447 5697439 : case PLUS_EXPR:
4448 5697439 : case MINUS_EXPR:
4449 5697439 : case NEGATE_EXPR:
4450 5697439 : cost = comp_cost (add_cost (speed, mode), 0);
4451 5697439 : if (TREE_CODE (expr) != NEGATE_EXPR)
4452 : {
4453 5557243 : tree mult = NULL_TREE;
4454 5557243 : comp_cost sa_cost;
4455 5557243 : if (TREE_CODE (op1) == MULT_EXPR)
4456 : mult = op1;
4457 5189015 : else if (TREE_CODE (op0) == MULT_EXPR)
4458 : mult = op0;
4459 :
4460 : if (mult != NULL_TREE
4461 4603764 : && is_a <scalar_int_mode> (mode, &int_mode)
4462 1680554 : && cst_and_fits_in_hwi (TREE_OPERAND (mult, 1))
4463 1432672 : && get_shiftadd_cost (expr, int_mode, cost0, cost1, mult,
4464 : speed, &sa_cost))
4465 953479 : return sa_cost;
4466 : }
4467 : break;
4468 :
4469 1187510 : CASE_CONVERT:
4470 1187510 : {
4471 1187510 : tree inner_mode, outer_mode;
4472 1187510 : outer_mode = TREE_TYPE (expr);
4473 1187510 : inner_mode = TREE_TYPE (op0);
4474 1187510 : cost = comp_cost (convert_cost (TYPE_MODE (outer_mode),
4475 1187510 : TYPE_MODE (inner_mode), speed), 0);
4476 : }
4477 1187510 : break;
4478 :
4479 2463873 : case MULT_EXPR:
4480 2463873 : if (cst_and_fits_in_hwi (op0))
4481 0 : cost = comp_cost (mult_by_coeff_cost (int_cst_value (op0),
4482 0 : mode, speed), 0);
4483 2463873 : else if (cst_and_fits_in_hwi (op1))
4484 1987132 : cost = comp_cost (mult_by_coeff_cost (int_cst_value (op1),
4485 1987132 : mode, speed), 0);
4486 : else
4487 476741 : return comp_cost (target_spill_cost [speed], 0);
4488 : break;
4489 :
4490 45986 : case EXACT_DIV_EXPR:
4491 45986 : case TRUNC_DIV_EXPR:
4492 : /* Division by power of two is usually cheap, so we allow it. Forbid
4493 : anything else. */
4494 45986 : if (integer_pow2p (TREE_OPERAND (expr, 1)))
4495 45986 : cost = comp_cost (add_cost (speed, mode), 0);
4496 : else
4497 0 : cost = comp_cost (target_spill_cost[speed], 0);
4498 : break;
4499 :
4500 55717 : case BIT_AND_EXPR:
4501 55717 : case BIT_IOR_EXPR:
4502 55717 : case BIT_NOT_EXPR:
4503 55717 : case LSHIFT_EXPR:
4504 55717 : case RSHIFT_EXPR:
4505 55717 : cost = comp_cost (add_cost (speed, mode), 0);
4506 55717 : break;
4507 0 : case COND_EXPR:
4508 0 : op0 = TREE_OPERAND (expr, 0);
4509 0 : STRIP_NOPS (op0);
4510 0 : if (op0 == NULL_TREE || TREE_CODE (op0) == SSA_NAME
4511 0 : || CONSTANT_CLASS_P (op0))
4512 0 : cost = no_cost;
4513 : else
4514 0 : cost = force_expr_to_var_cost (op0, speed);
4515 : break;
4516 0 : case LT_EXPR:
4517 0 : case LE_EXPR:
4518 0 : case GT_EXPR:
4519 0 : case GE_EXPR:
4520 0 : case EQ_EXPR:
4521 0 : case NE_EXPR:
4522 0 : case UNORDERED_EXPR:
4523 0 : case ORDERED_EXPR:
4524 0 : case UNLT_EXPR:
4525 0 : case UNLE_EXPR:
4526 0 : case UNGT_EXPR:
4527 0 : case UNGE_EXPR:
4528 0 : case UNEQ_EXPR:
4529 0 : case LTGT_EXPR:
4530 0 : case MAX_EXPR:
4531 0 : case MIN_EXPR:
4532 : /* Simply use add cost for now, FIXME if there is some more accurate cost
4533 : evaluation way. */
4534 0 : cost = comp_cost (add_cost (speed, mode), 0);
4535 0 : break;
4536 :
4537 0 : default:
4538 0 : gcc_unreachable ();
4539 : }
4540 :
4541 8020305 : cost += cost0;
4542 8020305 : cost += cost1;
4543 8020305 : return cost;
4544 : }
4545 :
4546 : /* Estimates cost of forcing EXPR into a variable. INV_VARS is a set of the
4547 : invariants the computation depends on. */
4548 :
4549 : static comp_cost
4550 24887815 : force_var_cost (struct ivopts_data *data, tree expr, bitmap *inv_vars)
4551 : {
4552 24887815 : if (!expr)
4553 2057241 : return no_cost;
4554 :
4555 22830574 : find_inv_vars (data, &expr, inv_vars);
4556 22830574 : return force_expr_to_var_cost (expr, data->speed);
4557 : }
4558 :
4559 : /* Returns cost of auto-modifying address expression in shape base + offset.
4560 : AINC_STEP is step size of the address IV. AINC_OFFSET is offset of the
4561 : address expression. The address expression has ADDR_MODE in addr space
4562 : AS. The memory access has MEM_MODE. SPEED means we are optimizing for
4563 : speed or size. */
4564 :
4565 : enum ainc_type
4566 : {
4567 : AINC_PRE_INC, /* Pre increment. */
4568 : AINC_PRE_DEC, /* Pre decrement. */
4569 : AINC_POST_INC, /* Post increment. */
4570 : AINC_POST_DEC, /* Post decrement. */
4571 : AINC_NONE /* Also the number of auto increment types. */
4572 : };
4573 :
4574 : struct ainc_cost_data
4575 : {
4576 : int64_t costs[AINC_NONE];
4577 : };
4578 :
4579 : static comp_cost
4580 1772056 : get_address_cost_ainc (poly_int64 ainc_step, poly_int64 ainc_offset,
4581 : machine_mode addr_mode, machine_mode mem_mode,
4582 : addr_space_t as, bool speed)
4583 : {
4584 1772056 : if (!USE_LOAD_PRE_DECREMENT (mem_mode)
4585 : && !USE_STORE_PRE_DECREMENT (mem_mode)
4586 : && !USE_LOAD_POST_DECREMENT (mem_mode)
4587 : && !USE_STORE_POST_DECREMENT (mem_mode)
4588 : && !USE_LOAD_PRE_INCREMENT (mem_mode)
4589 : && !USE_STORE_PRE_INCREMENT (mem_mode)
4590 : && !USE_LOAD_POST_INCREMENT (mem_mode)
4591 : && !USE_STORE_POST_INCREMENT (mem_mode))
4592 1772056 : return infinite_cost;
4593 :
4594 : static vec<ainc_cost_data *> ainc_cost_data_list;
4595 : unsigned idx = (unsigned) as * MAX_MACHINE_MODE + (unsigned) mem_mode;
4596 : if (idx >= ainc_cost_data_list.length ())
4597 : {
4598 : unsigned nsize = ((unsigned) as + 1) *MAX_MACHINE_MODE;
4599 :
4600 : gcc_assert (nsize > idx);
4601 : ainc_cost_data_list.safe_grow_cleared (nsize, true);
4602 : }
4603 :
4604 : ainc_cost_data *data = ainc_cost_data_list[idx];
4605 : if (data == NULL)
4606 : {
4607 : rtx reg = gen_raw_REG (addr_mode, LAST_VIRTUAL_REGISTER + 1);
4608 :
4609 : data = (ainc_cost_data *) xcalloc (1, sizeof (*data));
4610 : data->costs[AINC_PRE_DEC] = INFTY;
4611 : data->costs[AINC_POST_DEC] = INFTY;
4612 : data->costs[AINC_PRE_INC] = INFTY;
4613 : data->costs[AINC_POST_INC] = INFTY;
4614 : if (USE_LOAD_PRE_DECREMENT (mem_mode)
4615 : || USE_STORE_PRE_DECREMENT (mem_mode))
4616 : {
4617 : rtx addr = gen_rtx_PRE_DEC (addr_mode, reg);
4618 :
4619 : if (memory_address_addr_space_p (mem_mode, addr, as))
4620 : data->costs[AINC_PRE_DEC]
4621 : = address_cost (addr, mem_mode, as, speed);
4622 : }
4623 : if (USE_LOAD_POST_DECREMENT (mem_mode)
4624 : || USE_STORE_POST_DECREMENT (mem_mode))
4625 : {
4626 : rtx addr = gen_rtx_POST_DEC (addr_mode, reg);
4627 :
4628 : if (memory_address_addr_space_p (mem_mode, addr, as))
4629 : data->costs[AINC_POST_DEC]
4630 : = address_cost (addr, mem_mode, as, speed);
4631 : }
4632 : if (USE_LOAD_PRE_INCREMENT (mem_mode)
4633 : || USE_STORE_PRE_INCREMENT (mem_mode))
4634 : {
4635 : rtx addr = gen_rtx_PRE_INC (addr_mode, reg);
4636 :
4637 : if (memory_address_addr_space_p (mem_mode, addr, as))
4638 : data->costs[AINC_PRE_INC]
4639 : = address_cost (addr, mem_mode, as, speed);
4640 : }
4641 : if (USE_LOAD_POST_INCREMENT (mem_mode)
4642 : || USE_STORE_POST_INCREMENT (mem_mode))
4643 : {
4644 : rtx addr = gen_rtx_POST_INC (addr_mode, reg);
4645 :
4646 : if (memory_address_addr_space_p (mem_mode, addr, as))
4647 : data->costs[AINC_POST_INC]
4648 : = address_cost (addr, mem_mode, as, speed);
4649 : }
4650 : ainc_cost_data_list[idx] = data;
4651 : }
4652 :
4653 : poly_int64 msize = GET_MODE_SIZE (mem_mode);
4654 : if (known_eq (ainc_offset, 0) && known_eq (msize, ainc_step))
4655 : return comp_cost (data->costs[AINC_POST_INC], 0);
4656 : if (known_eq (ainc_offset, 0) && known_eq (msize, -ainc_step))
4657 : return comp_cost (data->costs[AINC_POST_DEC], 0);
4658 : if (known_eq (ainc_offset, msize) && known_eq (msize, ainc_step))
4659 : return comp_cost (data->costs[AINC_PRE_INC], 0);
4660 : if (known_eq (ainc_offset, -msize) && known_eq (msize, -ainc_step))
4661 : return comp_cost (data->costs[AINC_PRE_DEC], 0);
4662 :
4663 : return infinite_cost;
4664 : }
4665 :
4666 : /* Return cost of computing USE's address expression by using CAND.
4667 : AFF_INV and AFF_VAR represent invariant and variant parts of the
4668 : address expression, respectively. If AFF_INV is simple, store
4669 : the loop invariant variables which are depended by it in INV_VARS;
4670 : if AFF_INV is complicated, handle it as a new invariant expression
4671 : and record it in INV_EXPR. RATIO indicates multiple times between
4672 : steps of USE and CAND. If CAN_AUTOINC is nonNULL, store boolean
4673 : value to it indicating if this is an auto-increment address. */
4674 :
4675 : static comp_cost
4676 5525918 : get_address_cost (struct ivopts_data *data, struct iv_use *use,
4677 : struct iv_cand *cand, aff_tree *aff_inv,
4678 : aff_tree *aff_var, HOST_WIDE_INT ratio,
4679 : bitmap *inv_vars, iv_inv_expr_ent **inv_expr,
4680 : bool *can_autoinc, bool speed)
4681 : {
4682 5525918 : rtx addr;
4683 5525918 : bool simple_inv = true;
4684 5525918 : tree comp_inv = NULL_TREE, type = aff_var->type;
4685 5525918 : comp_cost var_cost = no_cost, cost = no_cost;
4686 5525918 : struct mem_address parts = {NULL_TREE, integer_one_node,
4687 5525918 : NULL_TREE, NULL_TREE, NULL_TREE};
4688 5525918 : machine_mode addr_mode = TYPE_MODE (type);
4689 5525918 : machine_mode mem_mode = TYPE_MODE (use->mem_type);
4690 5525918 : addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (use->iv->base));
4691 : /* Only true if ratio != 1. */
4692 5525918 : bool ok_with_ratio_p = false;
4693 5525918 : bool ok_without_ratio_p = false;
4694 5525918 : code_helper code = ERROR_MARK;
4695 :
4696 5525918 : if (use->type == USE_PTR_ADDRESS)
4697 : {
4698 3877 : gcall *call = as_a<gcall *> (use->stmt);
4699 3877 : gcc_assert (gimple_call_internal_p (call));
4700 3877 : code = gimple_call_internal_fn (call);
4701 : }
4702 :
4703 5525918 : if (!aff_combination_const_p (aff_inv))
4704 : {
4705 3686624 : parts.index = integer_one_node;
4706 : /* Addressing mode "base + index". */
4707 3686624 : ok_without_ratio_p = valid_mem_ref_p (mem_mode, as, &parts, code);
4708 3686624 : if (ratio != 1)
4709 : {
4710 2802456 : parts.step = wide_int_to_tree (type, ratio);
4711 : /* Addressing mode "base + index << scale". */
4712 2802456 : ok_with_ratio_p = valid_mem_ref_p (mem_mode, as, &parts, code);
4713 2802456 : if (!ok_with_ratio_p)
4714 1709134 : parts.step = NULL_TREE;
4715 : }
4716 2593302 : if (ok_with_ratio_p || ok_without_ratio_p)
4717 : {
4718 3686624 : if (maybe_ne (aff_inv->offset, 0))
4719 : {
4720 2406743 : parts.offset = wide_int_to_tree (sizetype, aff_inv->offset);
4721 : /* Addressing mode "base + index [<< scale] + offset". */
4722 2406743 : if (!valid_mem_ref_p (mem_mode, as, &parts, code))
4723 409 : parts.offset = NULL_TREE;
4724 : else
4725 2406334 : aff_inv->offset = 0;
4726 : }
4727 :
4728 3686624 : move_fixed_address_to_symbol (&parts, aff_inv);
4729 : /* Base is fixed address and is moved to symbol part. */
4730 3686624 : if (parts.symbol != NULL_TREE && aff_combination_zero_p (aff_inv))
4731 466660 : parts.base = NULL_TREE;
4732 :
4733 : /* Addressing mode "symbol + base + index [<< scale] [+ offset]". */
4734 3686624 : if (parts.symbol != NULL_TREE
4735 3686624 : && !valid_mem_ref_p (mem_mode, as, &parts, code))
4736 : {
4737 7238 : aff_combination_add_elt (aff_inv, parts.symbol, 1);
4738 7238 : parts.symbol = NULL_TREE;
4739 : /* Reset SIMPLE_INV since symbol address needs to be computed
4740 : outside of address expression in this case. */
4741 7238 : simple_inv = false;
4742 : /* Symbol part is moved back to base part, it can't be NULL. */
4743 7238 : parts.base = integer_one_node;
4744 : }
4745 : }
4746 : else
4747 0 : parts.index = NULL_TREE;
4748 : }
4749 : else
4750 : {
4751 1839294 : poly_int64 ainc_step;
4752 1839294 : if (can_autoinc
4753 1839294 : && ratio == 1
4754 3678580 : && ptrdiff_tree_p (cand->iv->step, &ainc_step))
4755 : {
4756 1772056 : poly_int64 ainc_offset = (aff_inv->offset).force_shwi ();
4757 :
4758 1772056 : if (stmt_after_increment (data->current_loop, cand, use->stmt))
4759 : ainc_offset += ainc_step;
4760 1772056 : cost = get_address_cost_ainc (ainc_step, ainc_offset,
4761 : addr_mode, mem_mode, as, speed);
4762 1772056 : if (!cost.infinite_cost_p ())
4763 : {
4764 0 : *can_autoinc = true;
4765 0 : return cost;
4766 : }
4767 1772056 : cost = no_cost;
4768 : }
4769 1839294 : if (!aff_combination_zero_p (aff_inv))
4770 : {
4771 1002507 : parts.offset = wide_int_to_tree (sizetype, aff_inv->offset);
4772 : /* Addressing mode "base + offset". */
4773 1002507 : if (!valid_mem_ref_p (mem_mode, as, &parts, code))
4774 38 : parts.offset = NULL_TREE;
4775 : else
4776 1002469 : aff_inv->offset = 0;
4777 : }
4778 : }
4779 :
4780 1846532 : if (simple_inv)
4781 5518680 : simple_inv = (aff_inv == NULL
4782 8745012 : || aff_combination_const_p (aff_inv)
4783 8737774 : || aff_combination_singleton_var_p (aff_inv));
4784 5525918 : if (!aff_combination_zero_p (aff_inv))
4785 3226400 : comp_inv = aff_combination_to_tree (aff_inv);
4786 3226400 : if (comp_inv != NULL_TREE)
4787 3226400 : cost = force_var_cost (data, comp_inv, inv_vars);
4788 5525918 : if (ratio != 1 && parts.step == NULL_TREE)
4789 1709142 : var_cost += mult_by_coeff_cost (ratio, addr_mode, speed);
4790 5525918 : if (comp_inv != NULL_TREE && parts.index == NULL_TREE)
4791 38 : var_cost += add_cost (speed, addr_mode);
4792 :
4793 5525918 : if (comp_inv && inv_expr && !simple_inv)
4794 : {
4795 741357 : *inv_expr = get_loop_invariant_expr (data, comp_inv);
4796 : /* Clear depends on. */
4797 741357 : if (*inv_expr != NULL && inv_vars && *inv_vars)
4798 428780 : bitmap_clear (*inv_vars);
4799 :
4800 : /* Cost of small invariant expression adjusted against loop niters
4801 : is usually zero, which makes it difficult to be differentiated
4802 : from candidate based on loop invariant variables. Secondly, the
4803 : generated invariant expression may not be hoisted out of loop by
4804 : following pass. We penalize the cost by rounding up in order to
4805 : neutralize such effects. */
4806 741357 : cost.cost = adjust_setup_cost (data, cost.cost, true);
4807 741357 : cost.scratch = cost.cost;
4808 : }
4809 :
4810 5525918 : cost += var_cost;
4811 5525918 : addr = addr_for_mem_ref (&parts, as, false);
4812 5525918 : gcc_assert (memory_address_addr_space_p (mem_mode, addr, as));
4813 5525918 : cost += address_cost (addr, mem_mode, as, speed);
4814 :
4815 5525918 : if (parts.symbol != NULL_TREE)
4816 520167 : cost.complexity += 1;
4817 : /* Don't increase the complexity of adding a scaled index if it's
4818 : the only kind of index that the target allows. */
4819 5525918 : if (parts.step != NULL_TREE && ok_without_ratio_p)
4820 1093322 : cost.complexity += 1;
4821 5525918 : if (parts.base != NULL_TREE && parts.index != NULL_TREE)
4822 3226362 : cost.complexity += 1;
4823 5525918 : if (parts.offset != NULL_TREE && !integer_zerop (parts.offset))
4824 3408803 : cost.complexity += 1;
4825 :
4826 : return cost;
4827 : }
4828 :
4829 : /* Scale (multiply) the computed COST (except scratch part that should be
4830 : hoisted out a loop) by header->frequency / AT->frequency, which makes
4831 : expected cost more accurate. */
4832 :
4833 : static comp_cost
4834 12810988 : get_scaled_computation_cost_at (ivopts_data *data, gimple *at, comp_cost cost)
4835 : {
4836 12810988 : if (data->speed
4837 12810988 : && data->current_loop->header->count.to_frequency (cfun) > 0)
4838 : {
4839 11168777 : basic_block bb = gimple_bb (at);
4840 11168777 : gcc_assert (cost.scratch <= cost.cost);
4841 11168777 : int scale_factor = (int)(intptr_t) bb->aux;
4842 11168777 : if (scale_factor == 1)
4843 10629220 : return cost;
4844 :
4845 539557 : int64_t scaled_cost
4846 539557 : = cost.scratch + (cost.cost - cost.scratch) * scale_factor;
4847 :
4848 539557 : if (dump_file && (dump_flags & TDF_DETAILS))
4849 93 : fprintf (dump_file, "Scaling cost based on bb prob by %2.2f: "
4850 : "%" PRId64 " (scratch: %" PRId64 ") -> %" PRId64 "\n",
4851 : 1.0f * scale_factor, cost.cost, cost.scratch, scaled_cost);
4852 :
4853 : cost.cost = scaled_cost;
4854 : }
4855 :
4856 2181768 : return cost;
4857 : }
4858 :
4859 : /* Determines the cost of the computation by that USE is expressed
4860 : from induction variable CAND. If ADDRESS_P is true, we just need
4861 : to create an address from it, otherwise we want to get it into
4862 : register. A set of invariants we depend on is stored in INV_VARS.
4863 : If CAN_AUTOINC is nonnull, use it to record whether autoinc
4864 : addressing is likely. If INV_EXPR is nonnull, record invariant
4865 : expr entry in it. */
4866 :
4867 : static comp_cost
4868 20268915 : get_computation_cost (struct ivopts_data *data, struct iv_use *use,
4869 : struct iv_cand *cand, bool address_p, bitmap *inv_vars,
4870 : bool *can_autoinc, iv_inv_expr_ent **inv_expr)
4871 : {
4872 20268915 : gimple *at = use->stmt;
4873 20268915 : tree ubase = use->iv->base, cbase = cand->iv->base;
4874 20268915 : tree utype = TREE_TYPE (ubase), ctype = TREE_TYPE (cbase);
4875 20268915 : tree comp_inv = NULL_TREE;
4876 20268915 : HOST_WIDE_INT ratio, aratio;
4877 20268915 : comp_cost cost;
4878 20268915 : widest_int rat;
4879 40537830 : aff_tree aff_inv, aff_var;
4880 20268915 : bool speed = optimize_bb_for_speed_p (gimple_bb (at));
4881 :
4882 20268915 : if (inv_vars)
4883 17828735 : *inv_vars = NULL;
4884 20268915 : if (can_autoinc)
4885 8812190 : *can_autoinc = false;
4886 20268915 : if (inv_expr)
4887 19843846 : *inv_expr = NULL;
4888 :
4889 : /* Check if we have enough precision to express the values of use. */
4890 20268915 : if (TYPE_PRECISION (utype) > TYPE_PRECISION (ctype))
4891 3102804 : return infinite_cost;
4892 :
4893 17166111 : if (address_p
4894 17166111 : || (use->iv->base_object
4895 2087320 : && cand->iv->base_object
4896 1010494 : && POINTER_TYPE_P (TREE_TYPE (use->iv->base_object))
4897 998246 : && POINTER_TYPE_P (TREE_TYPE (cand->iv->base_object))))
4898 : {
4899 : /* Do not try to express address of an object with computation based
4900 : on address of a different object. This may cause problems in rtl
4901 : level alias analysis (that does not expect this to be happening,
4902 : as this is illegal in C), and would be unlikely to be useful
4903 : anyway. */
4904 7924353 : if (use->iv->base_object
4905 7924353 : && cand->iv->base_object
4906 12135735 : && !operand_equal_p (use->iv->base_object, cand->iv->base_object, 0))
4907 1487773 : return infinite_cost;
4908 : }
4909 :
4910 15678338 : if (!get_computation_aff_1 (data, at, use, cand, &aff_inv, &aff_var, &rat)
4911 15678338 : || !wi::fits_shwi_p (rat))
4912 2867350 : return infinite_cost;
4913 :
4914 12810988 : ratio = rat.to_shwi ();
4915 12810988 : if (address_p)
4916 : {
4917 5525918 : cost = get_address_cost (data, use, cand, &aff_inv, &aff_var, ratio,
4918 : inv_vars, inv_expr, can_autoinc, speed);
4919 5525918 : cost = get_scaled_computation_cost_at (data, at, cost);
4920 : /* For doloop IV cand, add on the extra cost. */
4921 5525918 : cost += cand->doloop_p ? targetm.doloop_cost_for_address : 0;
4922 5525918 : return cost;
4923 : }
4924 :
4925 7285070 : bool simple_inv = (aff_combination_const_p (&aff_inv)
4926 1981652 : || aff_combination_singleton_var_p (&aff_inv));
4927 7285070 : tree signed_type = signed_type_for (aff_combination_type (&aff_inv));
4928 7285070 : aff_combination_convert (&aff_inv, signed_type);
4929 7285070 : if (!aff_combination_zero_p (&aff_inv))
4930 5227829 : comp_inv = aff_combination_to_tree (&aff_inv);
4931 :
4932 7285070 : cost = force_var_cost (data, comp_inv, inv_vars);
4933 7285070 : if (comp_inv && inv_expr && !simple_inv)
4934 : {
4935 1375040 : *inv_expr = get_loop_invariant_expr (data, comp_inv);
4936 : /* Clear depends on. */
4937 1375040 : if (*inv_expr != NULL && inv_vars && *inv_vars)
4938 822652 : bitmap_clear (*inv_vars);
4939 :
4940 1375040 : cost.cost = adjust_setup_cost (data, cost.cost);
4941 : /* Record setup cost in scratch field. */
4942 1375040 : cost.scratch = cost.cost;
4943 : }
4944 : /* Cost of constant integer can be covered when adding invariant part to
4945 : variant part. */
4946 5910030 : else if (comp_inv && CONSTANT_CLASS_P (comp_inv))
4947 3246152 : cost = no_cost;
4948 :
4949 : /* Need type narrowing to represent use with cand. */
4950 7285070 : if (TYPE_PRECISION (utype) < TYPE_PRECISION (ctype))
4951 : {
4952 816234 : machine_mode outer_mode = TYPE_MODE (utype);
4953 816234 : machine_mode inner_mode = TYPE_MODE (ctype);
4954 816234 : cost += comp_cost (convert_cost (outer_mode, inner_mode, speed), 0);
4955 : }
4956 :
4957 : /* Turn a + i * (-c) into a - i * c. */
4958 7285070 : if (ratio < 0 && comp_inv && !integer_zerop (comp_inv))
4959 1873269 : aratio = -ratio;
4960 : else
4961 : aratio = ratio;
4962 :
4963 7285070 : if (ratio != 1)
4964 2748838 : cost += mult_by_coeff_cost (aratio, TYPE_MODE (utype), speed);
4965 :
4966 : /* TODO: We may also need to check if we can compute a + i * 4 in one
4967 : instruction. */
4968 : /* Need to add up the invariant and variant parts. */
4969 7285070 : if (comp_inv && !integer_zerop (comp_inv))
4970 10449110 : cost += add_cost (speed, TYPE_MODE (utype));
4971 :
4972 7285070 : cost = get_scaled_computation_cost_at (data, at, cost);
4973 :
4974 : /* For doloop IV cand, add on the extra cost. */
4975 7285070 : if (cand->doloop_p && use->type == USE_NONLINEAR_EXPR)
4976 0 : cost += targetm.doloop_cost_for_generic;
4977 :
4978 7285070 : return cost;
4979 20268915 : }
4980 :
4981 : /* Determines cost of computing the use in GROUP with CAND in a generic
4982 : expression. */
4983 :
4984 : static bool
4985 5570236 : determine_group_iv_cost_generic (struct ivopts_data *data,
4986 : struct iv_group *group, struct iv_cand *cand)
4987 : {
4988 5570236 : comp_cost cost;
4989 5570236 : iv_inv_expr_ent *inv_expr = NULL;
4990 5570236 : bitmap inv_vars = NULL, inv_exprs = NULL;
4991 5570236 : struct iv_use *use = group->vuses[0];
4992 :
4993 : /* The simple case first -- if we need to express value of the preserved
4994 : original biv, the cost is 0. This also prevents us from counting the
4995 : cost of increment twice -- once at this use and once in the cost of
4996 : the candidate. */
4997 5570236 : if (cand->pos == IP_ORIGINAL && cand->incremented_at == use->stmt)
4998 59908 : cost = no_cost;
4999 : /* If the IV candidate involves undefined SSA values and is not the
5000 : same IV as on the USE avoid using that candidate here. */
5001 5510328 : else if (cand->involves_undefs
5002 5510328 : && (!use->iv || !operand_equal_p (cand->iv->base, use->iv->base, 0)))
5003 : return false;
5004 : else
5005 5510103 : cost = get_computation_cost (data, use, cand, false,
5006 : &inv_vars, NULL, &inv_expr);
5007 :
5008 5570011 : if (inv_expr)
5009 : {
5010 971779 : inv_exprs = BITMAP_ALLOC (NULL);
5011 971779 : bitmap_set_bit (inv_exprs, inv_expr->id);
5012 : }
5013 5570011 : set_group_iv_cost (data, group, cand, cost, inv_vars,
5014 : NULL_TREE, ERROR_MARK, inv_exprs);
5015 5570011 : return !cost.infinite_cost_p ();
5016 : }
5017 :
5018 : /* Determines cost of computing uses in GROUP with CAND in addresses. */
5019 :
5020 : static bool
5021 6372010 : determine_group_iv_cost_address (struct ivopts_data *data,
5022 : struct iv_group *group, struct iv_cand *cand)
5023 : {
5024 6372010 : unsigned i;
5025 6372010 : bitmap inv_vars = NULL, inv_exprs = NULL;
5026 6372010 : bool can_autoinc;
5027 6372010 : iv_inv_expr_ent *inv_expr = NULL;
5028 6372010 : struct iv_use *use = group->vuses[0];
5029 6372010 : comp_cost sum_cost = no_cost, cost;
5030 :
5031 6372010 : cost = get_computation_cost (data, use, cand, true,
5032 : &inv_vars, &can_autoinc, &inv_expr);
5033 :
5034 6372010 : if (inv_expr)
5035 : {
5036 456295 : inv_exprs = BITMAP_ALLOC (NULL);
5037 456295 : bitmap_set_bit (inv_exprs, inv_expr->id);
5038 : }
5039 6372010 : sum_cost = cost;
5040 6372010 : if (!sum_cost.infinite_cost_p () && cand->ainc_use == use)
5041 : {
5042 0 : if (can_autoinc)
5043 0 : sum_cost -= cand->cost_step;
5044 : /* If we generated the candidate solely for exploiting autoincrement
5045 : opportunities, and it turns out it can't be used, set the cost to
5046 : infinity to make sure we ignore it. */
5047 0 : else if (cand->pos == IP_AFTER_USE || cand->pos == IP_BEFORE_USE)
5048 0 : sum_cost = infinite_cost;
5049 : }
5050 :
5051 : /* Compute and add costs for rest uses of this group. */
5052 8387121 : for (i = 1; i < group->vuses.length () && !sum_cost.infinite_cost_p (); i++)
5053 : {
5054 2015111 : struct iv_use *next = group->vuses[i];
5055 :
5056 : /* TODO: We could skip computing cost for sub iv_use when it has the
5057 : same cost as the first iv_use, but the cost really depends on the
5058 : offset and where the iv_use is. */
5059 2015111 : cost = get_computation_cost (data, next, cand, true,
5060 : NULL, &can_autoinc, &inv_expr);
5061 2015111 : if (inv_expr)
5062 : {
5063 284829 : if (!inv_exprs)
5064 53 : inv_exprs = BITMAP_ALLOC (NULL);
5065 :
5066 : /* Uses in a group can share setup code,
5067 : so only add setup cost once. */
5068 284829 : if (bitmap_bit_p (inv_exprs, inv_expr->id))
5069 284399 : cost -= cost.scratch;
5070 : else
5071 430 : bitmap_set_bit (inv_exprs, inv_expr->id);
5072 : }
5073 2015111 : sum_cost += cost;
5074 : }
5075 6372010 : set_group_iv_cost (data, group, cand, sum_cost, inv_vars,
5076 : NULL_TREE, ERROR_MARK, inv_exprs);
5077 :
5078 6372010 : return !sum_cost.infinite_cost_p ();
5079 : }
5080 :
5081 : /* Computes value of candidate CAND at position AT in iteration DESC->NITER,
5082 : and stores it to VAL. */
5083 :
5084 : static void
5085 3955620 : cand_value_at (class loop *loop, struct iv_cand *cand, gimple *at,
5086 : class tree_niter_desc *desc, aff_tree *val)
5087 : {
5088 11866860 : aff_tree step, delta, nit;
5089 3955620 : struct iv *iv = cand->iv;
5090 3955620 : tree type = TREE_TYPE (iv->base);
5091 3955620 : tree niter = desc->niter;
5092 3955620 : bool after_adjust = stmt_after_increment (loop, cand, at);
5093 3955620 : tree steptype;
5094 :
5095 3955620 : if (POINTER_TYPE_P (type))
5096 111594 : steptype = sizetype;
5097 : else
5098 3844026 : steptype = unsigned_type_for (type);
5099 :
5100 : /* If AFTER_ADJUST is required, the code below generates the equivalent
5101 : of BASE + NITER * STEP + STEP, when ideally we'd prefer the expression
5102 : BASE + (NITER + 1) * STEP, especially when NITER is often of the form
5103 : SSA_NAME - 1. Unfortunately, guaranteeing that adding 1 to NITER
5104 : doesn't overflow is tricky, so we peek inside the TREE_NITER_DESC
5105 : class for common idioms that we know are safe. */
5106 3955620 : if (after_adjust
5107 3722732 : && desc->control.no_overflow
5108 3715744 : && integer_onep (desc->control.step)
5109 1013414 : && (desc->cmp == LT_EXPR
5110 37704 : || desc->cmp == NE_EXPR)
5111 4969034 : && TREE_CODE (desc->bound) == SSA_NAME)
5112 : {
5113 512212 : if (integer_onep (desc->control.base))
5114 : {
5115 386149 : niter = desc->bound;
5116 386149 : after_adjust = false;
5117 : }
5118 126063 : else if (TREE_CODE (niter) == MINUS_EXPR
5119 126063 : && integer_onep (TREE_OPERAND (niter, 1)))
5120 : {
5121 74231 : niter = TREE_OPERAND (niter, 0);
5122 74231 : after_adjust = false;
5123 : }
5124 : }
5125 :
5126 3955620 : tree_to_aff_combination (iv->step, TREE_TYPE (iv->step), &step);
5127 3955620 : aff_combination_convert (&step, steptype);
5128 3955620 : tree_to_aff_combination (niter, TREE_TYPE (niter), &nit);
5129 3955620 : aff_combination_convert (&nit, steptype);
5130 3955620 : aff_combination_mult (&nit, &step, &delta);
5131 3955620 : if (after_adjust)
5132 3262352 : aff_combination_add (&delta, &step);
5133 :
5134 3955620 : tree_to_aff_combination (iv->base, type, val);
5135 3955620 : if (!POINTER_TYPE_P (type))
5136 3844026 : aff_combination_convert (val, steptype);
5137 3955620 : aff_combination_add (val, &delta);
5138 3955620 : }
5139 :
5140 : /* Returns period of induction variable iv. */
5141 :
5142 : static tree
5143 4157080 : iv_period (struct iv *iv)
5144 : {
5145 4157080 : tree step = iv->step, period, type;
5146 4157080 : tree pow2div;
5147 :
5148 4157080 : gcc_assert (step && TREE_CODE (step) == INTEGER_CST);
5149 :
5150 4157080 : type = unsigned_type_for (TREE_TYPE (step));
5151 : /* Period of the iv is lcm (step, type_range)/step -1,
5152 : i.e., N*type_range/step - 1. Since type range is power
5153 : of two, N == (step >> num_of_ending_zeros_binary (step),
5154 : so the final result is
5155 :
5156 : (type_range >> num_of_ending_zeros_binary (step)) - 1
5157 :
5158 : */
5159 4157080 : pow2div = num_ending_zeros (step);
5160 :
5161 12471240 : period = build_low_bits_mask (type,
5162 4157080 : (TYPE_PRECISION (type)
5163 4157080 : - tree_to_uhwi (pow2div)));
5164 :
5165 4157080 : return period;
5166 : }
5167 :
5168 : /* Returns the comparison operator used when eliminating the iv USE. */
5169 :
5170 : static enum tree_code
5171 3955620 : iv_elimination_compare (struct ivopts_data *data, struct iv_use *use)
5172 : {
5173 3955620 : class loop *loop = data->current_loop;
5174 3955620 : basic_block ex_bb;
5175 3955620 : edge exit;
5176 :
5177 3955620 : ex_bb = gimple_bb (use->stmt);
5178 3955620 : exit = EDGE_SUCC (ex_bb, 0);
5179 3955620 : if (flow_bb_inside_loop_p (loop, exit->dest))
5180 2945204 : exit = EDGE_SUCC (ex_bb, 1);
5181 :
5182 3955620 : return (exit->flags & EDGE_TRUE_VALUE ? EQ_EXPR : NE_EXPR);
5183 : }
5184 :
5185 : /* Returns true if we can prove that BASE - OFFSET does not overflow. For now,
5186 : we only detect the situation that BASE = SOMETHING + OFFSET, where the
5187 : calculation is performed in non-wrapping type.
5188 :
5189 : TODO: More generally, we could test for the situation that
5190 : BASE = SOMETHING + OFFSET' and OFFSET is between OFFSET' and zero.
5191 : This would require knowing the sign of OFFSET. */
5192 :
5193 : static bool
5194 521 : difference_cannot_overflow_p (struct ivopts_data *data, tree base, tree offset)
5195 : {
5196 521 : enum tree_code code;
5197 521 : tree e1, e2;
5198 1563 : aff_tree aff_e1, aff_e2, aff_offset;
5199 :
5200 521 : if (!nowrap_type_p (TREE_TYPE (base)))
5201 : return false;
5202 :
5203 521 : base = expand_simple_operations (base);
5204 :
5205 521 : if (TREE_CODE (base) == SSA_NAME)
5206 : {
5207 519 : gimple *stmt = SSA_NAME_DEF_STMT (base);
5208 :
5209 519 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
5210 : return false;
5211 :
5212 47 : code = gimple_assign_rhs_code (stmt);
5213 47 : if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
5214 : return false;
5215 :
5216 34 : e1 = gimple_assign_rhs1 (stmt);
5217 34 : e2 = gimple_assign_rhs2 (stmt);
5218 : }
5219 : else
5220 : {
5221 2 : code = TREE_CODE (base);
5222 2 : if (get_gimple_rhs_class (code) != GIMPLE_BINARY_RHS)
5223 : return false;
5224 0 : e1 = TREE_OPERAND (base, 0);
5225 0 : e2 = TREE_OPERAND (base, 1);
5226 : }
5227 :
5228 : /* Use affine expansion as deeper inspection to prove the equality. */
5229 34 : tree_to_aff_combination_expand (e2, TREE_TYPE (e2),
5230 : &aff_e2, &data->name_expansion_cache);
5231 34 : tree_to_aff_combination_expand (offset, TREE_TYPE (offset),
5232 : &aff_offset, &data->name_expansion_cache);
5233 34 : aff_combination_scale (&aff_offset, -1);
5234 34 : switch (code)
5235 : {
5236 0 : case PLUS_EXPR:
5237 0 : aff_combination_add (&aff_e2, &aff_offset);
5238 0 : if (aff_combination_zero_p (&aff_e2))
5239 : return true;
5240 :
5241 0 : tree_to_aff_combination_expand (e1, TREE_TYPE (e1),
5242 : &aff_e1, &data->name_expansion_cache);
5243 0 : aff_combination_add (&aff_e1, &aff_offset);
5244 0 : return aff_combination_zero_p (&aff_e1);
5245 :
5246 34 : case POINTER_PLUS_EXPR:
5247 34 : aff_combination_add (&aff_e2, &aff_offset);
5248 34 : return aff_combination_zero_p (&aff_e2);
5249 :
5250 : default:
5251 : return false;
5252 : }
5253 521 : }
5254 :
5255 : /* Return true if STEP * VAL, computed in OFF_TYPE, is known to be a
5256 : non-negative offset which does not overflow, i.e. if the value of VAL is
5257 : non-negative and multiplying it by STEP fits in the signed range of
5258 : OFF_TYPE. */
5259 :
5260 : static bool
5261 112 : nonneg_scaled_offset_p (tree val, HOST_WIDE_INT step, tree off_type)
5262 : {
5263 112 : if (!INTEGRAL_TYPE_P (TREE_TYPE (val)) || step == 0)
5264 : return false;
5265 :
5266 112 : signop sgn = TYPE_SIGN (TREE_TYPE (val));
5267 112 : int_range_max r;
5268 224 : if (!get_range_query (cfun)->range_of_expr (r, val)
5269 112 : || r.undefined_p ()
5270 336 : || wi::neg_p (r.lower_bound (), sgn))
5271 : return false;
5272 :
5273 112 : widest_int max = widest_int::from (r.upper_bound (), sgn);
5274 112 : widest_int limit
5275 112 : = widest_int::from (wi::max_value (TYPE_PRECISION (off_type), SIGNED),
5276 112 : SIGNED);
5277 112 : return wi::leu_p (max, wi::udiv_trunc (limit, absu_hwi (step)));
5278 112 : }
5279 :
5280 : /* Tries to replace loop exit by one formulated in terms of a LT_EXPR
5281 : comparison with CAND. NITER describes the number of iterations of
5282 : the loops. If successful, the comparison in COMP_P is altered accordingly
5283 : and the bound in BOUND_P is recomputed.
5284 :
5285 : We aim to handle the following situation:
5286 :
5287 : sometype *base, *p;
5288 : unsigned a, b, i;
5289 :
5290 : i = a;
5291 : p = p_0 = base + a;
5292 :
5293 : do
5294 : {
5295 : bla (*p);
5296 : p++;
5297 : i++;
5298 : }
5299 : while (i < b);
5300 :
5301 : Here, the number of iterations of the loop is (a + 1 > b) ? 0 : b - a - 1.
5302 : We aim to optimize this to
5303 :
5304 : p = p_0 = base + a;
5305 : do
5306 : {
5307 : bla (*p);
5308 : p++;
5309 : }
5310 : while (p < p_0 - (a + 1) + (b + 1));
5311 :
5312 : Note that the bound has to be computed as p_0 - (a + 1) + (b + 1) and not
5313 : from the number of iterations as p_0 + (b - a): the latter is only
5314 : equivalent if b - a does not wrap, which is not the case when the loop
5315 : rolls zero times.
5316 :
5317 : Note also that a + 1 must be used as the offset it is, i.e. converted to
5318 : the type of the offsets as a whole, and not rewritten into a and 1, since
5319 : it may wrap around in the type of a: this is precisely the case where the
5320 : loop rolls but b - a is not the number of iterations either. Conversely
5321 : b + 1 must be computed in the type of the offsets, where it cannot wrap
5322 : around, since nothing constrains b when the loop rolls zero times.
5323 :
5324 : For this to preserve correctness, we need to know that the values compared
5325 : in the transformed loop are ordered the same way as i and b are. Since the
5326 : comparison of the pointers is performed modulo the size of the address
5327 : space, this needs a + 1 > b to be an unsigned comparison, and the offsets
5328 : a, b, a + 1 and b + 1 scaled by the step of the candidate to be
5329 : non-negative and to not overflow. Then:
5330 :
5331 : 1) if a + 1 <= b, then the loop rolls b - (a + 1) times, so that (b + 1)
5332 : - (a + 1) computed on the offsets is its number of iterations, and
5333 : p_0 - (a + 1) + (b + 1) is the final value of p, hence there is no
5334 : overflow in computing it or the values of p, and the pointers increase
5335 : monotonically together with i.
5336 : 2) if a + 1 > b, then the loop exits at the first test and a + 1 does not
5337 : wrap around, so b <= a implies that p_0 - (a + 1) + (b + 1) = p_0 - a
5338 : + b lies between the valid addresses p_0 - a and p_0, so the test indeed
5339 : fails. Here we also need to verify that the expression p_0 - a does not
5340 : overflow, which we prove using p_0 = base + a. */
5341 :
5342 : static bool
5343 211272 : iv_elimination_compare_lt (struct ivopts_data *data, struct iv_use *use,
5344 : struct iv_cand *cand, enum tree_code *comp_p,
5345 : class tree_niter_desc *niter, tree *bound_p)
5346 : {
5347 211272 : tree cand_type, a, a1, b, b1, mbz, nit_type = TREE_TYPE (niter->niter);
5348 211272 : tree off_type, offset, bound;
5349 633816 : class aff_tree nit, tmpa, tmpb;
5350 211272 : enum tree_code comp;
5351 211272 : HOST_WIDE_INT step;
5352 :
5353 : /* We need to know that the candidate induction variable does not overflow.
5354 : While more complex analysis may be used to prove this, for now just
5355 : check that the variable appears in the original program and that it
5356 : is computed in a type that guarantees no overflows. */
5357 211272 : cand_type = TREE_TYPE (cand->iv->base);
5358 211272 : if (cand->pos != IP_ORIGINAL || !nowrap_type_p (cand_type))
5359 : return false;
5360 :
5361 : /* Make sure that the loop iterates till the loop bound is hit, as otherwise
5362 : the calculation of the BOUND could overflow, making the comparison
5363 : invalid. */
5364 19474 : if (!data->loop_single_exit_p)
5365 : return false;
5366 :
5367 : /* We need to be able to decide whether candidate is increasing or decreasing
5368 : in order to choose the right comparison operator. */
5369 13781 : if (!cst_and_fits_in_hwi (cand->iv->step))
5370 : return false;
5371 13781 : step = int_cst_value (cand->iv->step);
5372 :
5373 : /* The bound we compute below is the value the candidate has after the last
5374 : iteration, so the exit test has to see the incremented candidate. */
5375 13781 : if (!stmt_after_increment (data->current_loop, cand, use->stmt))
5376 : return false;
5377 :
5378 : /* Check that the number of iterations matches the expected pattern:
5379 : a + 1 > b ? 0 : b - a - 1. */
5380 9600 : mbz = niter->may_be_zero;
5381 9600 : if (TREE_CODE (mbz) == GT_EXPR)
5382 : {
5383 : /* Handle a + 1 > b. */
5384 1364 : tree op0 = TREE_OPERAND (mbz, 0);
5385 1364 : if (TREE_CODE (op0) == PLUS_EXPR && integer_onep (TREE_OPERAND (op0, 1)))
5386 : {
5387 803 : a1 = op0;
5388 803 : a = TREE_OPERAND (op0, 0);
5389 803 : b = TREE_OPERAND (mbz, 1);
5390 : }
5391 : else
5392 : return false;
5393 : }
5394 8236 : else if (TREE_CODE (mbz) == LT_EXPR)
5395 : {
5396 737 : tree op1 = TREE_OPERAND (mbz, 1);
5397 :
5398 : /* Handle b < a + 1. */
5399 737 : if (TREE_CODE (op1) == PLUS_EXPR && integer_onep (TREE_OPERAND (op1, 1)))
5400 : {
5401 108 : a1 = op1;
5402 108 : a = TREE_OPERAND (op1, 0);
5403 108 : b = TREE_OPERAND (mbz, 0);
5404 : }
5405 : else
5406 : return false;
5407 : }
5408 : else
5409 : return false;
5410 :
5411 : /* Expected number of iterations is B - A - 1. Check that it matches
5412 : the actual number, i.e., that B - A - NITER = 1. */
5413 911 : tree_to_aff_combination (niter->niter, nit_type, &nit);
5414 911 : tree_to_aff_combination (fold_convert (nit_type, a), nit_type, &tmpa);
5415 911 : tree_to_aff_combination (fold_convert (nit_type, b), nit_type, &tmpb);
5416 911 : aff_combination_scale (&nit, -1);
5417 911 : aff_combination_scale (&tmpa, -1);
5418 911 : aff_combination_add (&tmpb, &tmpa);
5419 911 : aff_combination_add (&tmpb, &nit);
5420 911 : if (tmpb.n != 0 || maybe_ne (tmpb.offset, 1))
5421 : return false;
5422 :
5423 : /* The comparison A + 1 > B only tells us that B is at most A if it is
5424 : an unsigned one; otherwise B may well be negative. */
5425 521 : if (!TYPE_UNSIGNED (TREE_TYPE (a)))
5426 : return false;
5427 :
5428 : /* Check that CAND->IV->BASE - CAND->IV->STEP * A does not overflow. */
5429 521 : off_type = TREE_TYPE (cand->iv->step);
5430 521 : offset = fold_build2 (MULT_EXPR, off_type, cand->iv->step,
5431 : fold_convert (off_type, a));
5432 521 : if (!difference_cannot_overflow_p (data, cand->iv->base, offset))
5433 : return false;
5434 :
5435 : /* Convert A + 1 as a whole, since it may wrap around in the type of A, but
5436 : compute B + 1 in OFF_TYPE, where it cannot wrap around. */
5437 34 : a1 = fold_convert (off_type, a1);
5438 34 : b1 = fold_build2 (PLUS_EXPR, off_type, fold_convert (off_type, b),
5439 : build_one_cst (off_type));
5440 :
5441 : /* The candidate is compared as an unsigned quantity, so the offsets by
5442 : which A + 1 and B + 1 move it away from CAND->IV->BASE - CAND->IV->STEP
5443 : * (A + 1) have to be ordered the same way as A + 1 and B + 1 themselves.
5444 : The same is required of A and B, in terms of which the bound is
5445 : expressed when the loop rolls zero times. */
5446 34 : if (!nonneg_scaled_offset_p (a, step, off_type)
5447 26 : || !nonneg_scaled_offset_p (b, step, off_type)
5448 26 : || !nonneg_scaled_offset_p (a1, step, off_type)
5449 60 : || !nonneg_scaled_offset_p (b1, step, off_type))
5450 : return false;
5451 :
5452 : /* Determine the new comparison operator. */
5453 26 : comp = step < 0 ? GT_EXPR : LT_EXPR;
5454 26 : if (*comp_p == NE_EXPR)
5455 26 : *comp_p = comp;
5456 0 : else if (*comp_p == EQ_EXPR)
5457 0 : *comp_p = invert_tree_comparison (comp, false);
5458 : else
5459 0 : gcc_unreachable ();
5460 :
5461 : /* Recompute the bound as CAND->IV->BASE - CAND->IV->STEP * (A + 1)
5462 : + CAND->IV->STEP * (B + 1). Deriving it from the number of iterations,
5463 : as cand_value_at does, is not correct here: B - A is computed in NIT_TYPE
5464 : and converting it to OFF_TYPE is not value preserving when the loop
5465 : rolls zero times and B - A is thus negative. */
5466 26 : bound = fold_build2 (MINUS_EXPR, off_type,
5467 : fold_build2 (MULT_EXPR, off_type, cand->iv->step, b1),
5468 : fold_build2 (MULT_EXPR, off_type, cand->iv->step, a1));
5469 26 : cand_type = TREE_TYPE (cand->iv->base);
5470 26 : if (POINTER_TYPE_P (cand_type))
5471 26 : *bound_p = fold_build_pointer_plus (cand->iv->base, bound);
5472 : else
5473 0 : *bound_p = fold_build2 (PLUS_EXPR, cand_type, cand->iv->base,
5474 : fold_convert (cand_type, bound));
5475 :
5476 : return true;
5477 211272 : }
5478 :
5479 : /* Check whether it is possible to express the condition in USE by comparison
5480 : of candidate CAND. If so, store the value compared with to BOUND, and the
5481 : comparison operator to COMP. */
5482 :
5483 : static bool
5484 4977519 : may_eliminate_iv (struct ivopts_data *data,
5485 : struct iv_use *use, struct iv_cand *cand, tree *bound,
5486 : enum tree_code *comp)
5487 : {
5488 4977519 : basic_block ex_bb;
5489 4977519 : edge exit;
5490 4977519 : tree period;
5491 4977519 : class loop *loop = data->current_loop;
5492 4977519 : aff_tree bnd;
5493 4977519 : class tree_niter_desc *desc = NULL;
5494 :
5495 : /* If the IV candidate involves undefs do not attempt to use it to
5496 : express a condition. */
5497 4977519 : if (cand->involves_undefs)
5498 : return false;
5499 :
5500 4977172 : if (TREE_CODE (cand->iv->step) != INTEGER_CST)
5501 : return false;
5502 :
5503 : /* For now works only for exits that dominate the loop latch.
5504 : TODO: extend to other conditions inside loop body. */
5505 4789286 : ex_bb = gimple_bb (use->stmt);
5506 4789286 : if (use->stmt != last_nondebug_stmt (ex_bb)
5507 4684264 : || gimple_code (use->stmt) != GIMPLE_COND
5508 9472551 : || !dominated_by_p (CDI_DOMINATORS, loop->latch, ex_bb))
5509 : return false;
5510 :
5511 4553881 : exit = EDGE_SUCC (ex_bb, 0);
5512 4553881 : if (flow_bb_inside_loop_p (loop, exit->dest))
5513 3431947 : exit = EDGE_SUCC (ex_bb, 1);
5514 4553881 : if (flow_bb_inside_loop_p (loop, exit->dest))
5515 : return false;
5516 :
5517 4439035 : desc = niter_for_exit (data, exit);
5518 4439035 : if (!desc)
5519 : return false;
5520 :
5521 : /* Determine whether we can use the variable to test the exit condition.
5522 : This is the case iff the period of the induction variable is greater
5523 : than the number of iterations for which the exit condition is true. */
5524 4157080 : period = iv_period (cand->iv);
5525 :
5526 : /* If the number of iterations is constant, compare against it directly. */
5527 4157080 : if (TREE_CODE (desc->niter) == INTEGER_CST)
5528 : {
5529 : /* See cand_value_at. */
5530 2725727 : if (stmt_after_increment (loop, cand, use->stmt))
5531 : {
5532 2666574 : if (!tree_int_cst_lt (desc->niter, period))
5533 : return false;
5534 : }
5535 : else
5536 : {
5537 59153 : if (tree_int_cst_lt (period, desc->niter))
5538 : return false;
5539 : }
5540 : }
5541 :
5542 : /* If not, and if this is the only possible exit of the loop, see whether
5543 : we can get a conservative estimate on the number of iterations of the
5544 : entire loop and compare against that instead. */
5545 : else
5546 : {
5547 1431353 : widest_int period_value, max_niter;
5548 :
5549 1431353 : max_niter = desc->max;
5550 1431353 : if (stmt_after_increment (loop, cand, use->stmt))
5551 1222811 : max_niter += 1;
5552 1431353 : period_value = wi::to_widest (period);
5553 1431353 : if (wi::gtu_p (max_niter, period_value))
5554 : {
5555 : /* See if we can take advantage of inferred loop bound
5556 : information. */
5557 289321 : if (data->loop_single_exit_p)
5558 : {
5559 167089 : if (!max_loop_iterations (loop, &max_niter))
5560 : return false;
5561 : /* The loop bound is already adjusted by adding 1. */
5562 167089 : if (wi::gtu_p (max_niter, period_value))
5563 : return false;
5564 : }
5565 : else
5566 : return false;
5567 : }
5568 1431353 : }
5569 :
5570 : /* For doloop IV cand, the bound would be zero. It's safe whether
5571 : may_be_zero set or not. */
5572 3955620 : if (cand->doloop_p)
5573 : {
5574 0 : *bound = build_int_cst (TREE_TYPE (cand->iv->base), 0);
5575 0 : *comp = iv_elimination_compare (data, use);
5576 0 : return true;
5577 : }
5578 :
5579 3955620 : cand_value_at (loop, cand, use->stmt, desc, &bnd);
5580 :
5581 3955620 : *bound = fold_convert (TREE_TYPE (cand->iv->base),
5582 : aff_combination_to_tree (&bnd));
5583 3955620 : *comp = iv_elimination_compare (data, use);
5584 :
5585 : /* Sometimes, it is possible to handle the situation that the number of
5586 : iterations may be zero unless additional assumptions by using <
5587 : instead of != in the exit condition.
5588 :
5589 : TODO: we could also calculate the value MAY_BE_ZERO ? 0 : NITER and
5590 : base the exit condition on it. However, that is often too
5591 : expensive. */
5592 3955620 : if (!integer_zerop (desc->may_be_zero)
5593 3955620 : && !iv_elimination_compare_lt (data, use, cand, comp, desc, bound))
5594 : return false;
5595 :
5596 : /* It is unlikely that computing the number of iterations using division
5597 : would be more profitable than keeping the original induction variable. */
5598 3744374 : bool cond_overflow_p;
5599 3744374 : if (expression_expensive_p (*bound, &cond_overflow_p))
5600 7052 : return false;
5601 :
5602 : return true;
5603 4977519 : }
5604 :
5605 : /* Calculates the cost of BOUND, if it is a PARM_DECL. A PARM_DECL must
5606 : be copied, if it is used in the loop body and DATA->body_includes_call. */
5607 :
5608 : static int
5609 8439571 : parm_decl_cost (struct ivopts_data *data, tree bound)
5610 : {
5611 8439571 : tree sbound = bound;
5612 8439571 : STRIP_NOPS (sbound);
5613 :
5614 8439571 : if (TREE_CODE (sbound) == SSA_NAME
5615 2866948 : && SSA_NAME_IS_DEFAULT_DEF (sbound)
5616 157202 : && TREE_CODE (SSA_NAME_VAR (sbound)) == PARM_DECL
5617 8594364 : && data->body_includes_call)
5618 35163 : return COSTS_N_INSNS (1);
5619 :
5620 : return 0;
5621 : }
5622 :
5623 : /* Determines cost of computing the use in GROUP with CAND in a condition. */
5624 :
5625 : static bool
5626 5946622 : determine_group_iv_cost_cond (struct ivopts_data *data,
5627 : struct iv_group *group, struct iv_cand *cand)
5628 : {
5629 5946622 : tree bound = NULL_TREE;
5630 5946622 : struct iv *cmp_iv;
5631 5946622 : bitmap inv_exprs = NULL;
5632 5946622 : bitmap inv_vars_elim = NULL, inv_vars_express = NULL, inv_vars;
5633 5946622 : comp_cost elim_cost = infinite_cost, express_cost, cost, bound_cost;
5634 5946622 : enum comp_iv_rewrite rewrite_type;
5635 5946622 : iv_inv_expr_ent *inv_expr_elim = NULL, *inv_expr_express = NULL, *inv_expr;
5636 5946622 : tree *control_var, *bound_cst;
5637 5946622 : enum tree_code comp = ERROR_MARK;
5638 5946622 : struct iv_use *use = group->vuses[0];
5639 :
5640 : /* Extract condition operands. */
5641 5946622 : rewrite_type = extract_cond_operands (data, use->stmt, &control_var,
5642 : &bound_cst, NULL, &cmp_iv);
5643 5946622 : gcc_assert (rewrite_type != COMP_IV_NA);
5644 :
5645 : /* Try iv elimination. */
5646 5946622 : if (rewrite_type == COMP_IV_ELIM
5647 5946622 : && may_eliminate_iv (data, use, cand, &bound, &comp))
5648 : {
5649 3737322 : elim_cost = force_var_cost (data, bound, &inv_vars_elim);
5650 3737322 : if (elim_cost.cost == 0)
5651 2514020 : elim_cost.cost = parm_decl_cost (data, bound);
5652 1223302 : else if (TREE_CODE (bound) == INTEGER_CST)
5653 0 : elim_cost.cost = 0;
5654 : /* If we replace a loop condition 'i < n' with 'p < base + n',
5655 : inv_vars_elim will have 'base' and 'n' set, which implies that both
5656 : 'base' and 'n' will be live during the loop. More likely,
5657 : 'base + n' will be loop invariant, resulting in only one live value
5658 : during the loop. So in that case we clear inv_vars_elim and set
5659 : inv_expr_elim instead. */
5660 3737322 : if (inv_vars_elim && bitmap_count_bits (inv_vars_elim) > 1)
5661 : {
5662 330768 : inv_expr_elim = get_loop_invariant_expr (data, bound);
5663 330768 : bitmap_clear (inv_vars_elim);
5664 : }
5665 : /* The bound is a loop invariant, so it will be only computed
5666 : once. */
5667 3737322 : elim_cost.cost = adjust_setup_cost (data, elim_cost.cost);
5668 : }
5669 :
5670 : /* When the condition is a comparison of the candidate IV against
5671 : zero, prefer this IV.
5672 :
5673 : TODO: The constant that we're subtracting from the cost should
5674 : be target-dependent. This information should be added to the
5675 : target costs for each backend. */
5676 5946622 : if (!elim_cost.infinite_cost_p () /* Do not try to decrease infinite! */
5677 3737322 : && integer_zerop (*bound_cst)
5678 8609571 : && (operand_equal_p (*control_var, cand->var_after, 0)
5679 2408196 : || operand_equal_p (*control_var, cand->var_before, 0)))
5680 261337 : elim_cost -= 1;
5681 :
5682 5946622 : express_cost = get_computation_cost (data, use, cand, false,
5683 : &inv_vars_express, NULL,
5684 : &inv_expr_express);
5685 5946622 : if (cmp_iv != NULL)
5686 5059799 : find_inv_vars (data, &cmp_iv->base, &inv_vars_express);
5687 :
5688 : /* Count the cost of the original bound as well. */
5689 5946622 : bound_cost = force_var_cost (data, *bound_cst, NULL);
5690 5946622 : if (bound_cost.cost == 0)
5691 5925551 : bound_cost.cost = parm_decl_cost (data, *bound_cst);
5692 21071 : else if (TREE_CODE (*bound_cst) == INTEGER_CST)
5693 0 : bound_cost.cost = 0;
5694 5946622 : express_cost += bound_cost;
5695 :
5696 : /* Choose the better approach, preferring the eliminated IV. */
5697 5946622 : if (elim_cost <= express_cost)
5698 : {
5699 4609273 : cost = elim_cost;
5700 4609273 : inv_vars = inv_vars_elim;
5701 4609273 : inv_vars_elim = NULL;
5702 4609273 : inv_expr = inv_expr_elim;
5703 : /* For doloop candidate/use pair, adjust to zero cost. */
5704 4609273 : if (group->doloop_p && cand->doloop_p && elim_cost.cost > no_cost.cost)
5705 0 : cost = no_cost;
5706 : }
5707 : else
5708 : {
5709 1337349 : cost = express_cost;
5710 1337349 : inv_vars = inv_vars_express;
5711 1337349 : inv_vars_express = NULL;
5712 1337349 : bound = NULL_TREE;
5713 1337349 : comp = ERROR_MARK;
5714 1337349 : inv_expr = inv_expr_express;
5715 : }
5716 :
5717 5946622 : if (inv_expr)
5718 : {
5719 601043 : inv_exprs = BITMAP_ALLOC (NULL);
5720 601043 : bitmap_set_bit (inv_exprs, inv_expr->id);
5721 : }
5722 5946622 : set_group_iv_cost (data, group, cand, cost,
5723 : inv_vars, bound, comp, inv_exprs);
5724 :
5725 5946622 : if (inv_vars_elim)
5726 25520 : BITMAP_FREE (inv_vars_elim);
5727 5946622 : if (inv_vars_express)
5728 1285416 : BITMAP_FREE (inv_vars_express);
5729 :
5730 5946622 : return !cost.infinite_cost_p ();
5731 : }
5732 :
5733 : /* Determines cost of computing uses in GROUP with CAND. Returns false
5734 : if USE cannot be represented with CAND. */
5735 :
5736 : static bool
5737 17888868 : determine_group_iv_cost (struct ivopts_data *data,
5738 : struct iv_group *group, struct iv_cand *cand)
5739 : {
5740 17888868 : switch (group->type)
5741 : {
5742 5570236 : case USE_NONLINEAR_EXPR:
5743 5570236 : return determine_group_iv_cost_generic (data, group, cand);
5744 :
5745 6372010 : case USE_REF_ADDRESS:
5746 6372010 : case USE_PTR_ADDRESS:
5747 6372010 : return determine_group_iv_cost_address (data, group, cand);
5748 :
5749 5946622 : case USE_COMPARE:
5750 5946622 : return determine_group_iv_cost_cond (data, group, cand);
5751 :
5752 0 : default:
5753 0 : gcc_unreachable ();
5754 : }
5755 : }
5756 :
5757 : /* Return true if get_computation_cost indicates that autoincrement is
5758 : a possibility for the pair of USE and CAND, false otherwise. */
5759 :
5760 : static bool
5761 1305277 : autoinc_possible_for_pair (struct ivopts_data *data, struct iv_use *use,
5762 : struct iv_cand *cand)
5763 : {
5764 1305277 : if (!address_p (use->type))
5765 : return false;
5766 :
5767 425069 : bool can_autoinc = false;
5768 425069 : get_computation_cost (data, use, cand, true, NULL, &can_autoinc, NULL);
5769 425069 : return can_autoinc;
5770 : }
5771 :
5772 : /* Examine IP_ORIGINAL candidates to see if they are incremented next to a
5773 : use that allows autoincrement, and set their AINC_USE if possible. */
5774 :
5775 : static void
5776 513181 : set_autoinc_for_original_candidates (struct ivopts_data *data)
5777 : {
5778 513181 : unsigned i, j;
5779 :
5780 5205582 : for (i = 0; i < data->vcands.length (); i++)
5781 : {
5782 4692401 : struct iv_cand *cand = data->vcands[i];
5783 4692401 : struct iv_use *closest_before = NULL;
5784 4692401 : struct iv_use *closest_after = NULL;
5785 4692401 : if (cand->pos != IP_ORIGINAL)
5786 3806740 : continue;
5787 :
5788 3882406 : for (j = 0; j < data->vgroups.length (); j++)
5789 : {
5790 2996745 : struct iv_group *group = data->vgroups[j];
5791 2996745 : struct iv_use *use = group->vuses[0];
5792 2996745 : unsigned uid = gimple_uid (use->stmt);
5793 :
5794 2996745 : if (gimple_bb (use->stmt) != gimple_bb (cand->incremented_at))
5795 1172018 : continue;
5796 :
5797 1824727 : if (uid < gimple_uid (cand->incremented_at)
5798 1824727 : && (closest_before == NULL
5799 385864 : || uid > gimple_uid (closest_before->stmt)))
5800 : closest_before = use;
5801 :
5802 1824727 : if (uid > gimple_uid (cand->incremented_at)
5803 1824727 : && (closest_after == NULL
5804 73678 : || uid < gimple_uid (closest_after->stmt)))
5805 : closest_after = use;
5806 : }
5807 :
5808 885661 : if (closest_before != NULL
5809 885661 : && autoinc_possible_for_pair (data, closest_before, cand))
5810 0 : cand->ainc_use = closest_before;
5811 885661 : else if (closest_after != NULL
5812 885661 : && autoinc_possible_for_pair (data, closest_after, cand))
5813 0 : cand->ainc_use = closest_after;
5814 : }
5815 513181 : }
5816 :
5817 : /* Relate compare use with all candidates. */
5818 :
5819 : static void
5820 301 : relate_compare_use_with_all_cands (struct ivopts_data *data)
5821 : {
5822 301 : unsigned i, count = data->vcands.length ();
5823 9622 : for (i = 0; i < data->vgroups.length (); i++)
5824 : {
5825 9321 : struct iv_group *group = data->vgroups[i];
5826 :
5827 9321 : if (group->type == USE_COMPARE)
5828 1911 : bitmap_set_range (group->related_cands, 0, count);
5829 : }
5830 301 : }
5831 :
5832 : /* If PREFERRED_MODE is suitable and profitable, use the preferred
5833 : PREFERRED_MODE to compute doloop iv base from niter: base = niter + 1. */
5834 :
5835 : static tree
5836 0 : compute_doloop_base_on_mode (machine_mode preferred_mode, tree niter,
5837 : const widest_int &iterations_max)
5838 : {
5839 0 : tree ntype = TREE_TYPE (niter);
5840 0 : tree pref_type = lang_hooks.types.type_for_mode (preferred_mode, 1);
5841 0 : if (!pref_type)
5842 0 : return fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5843 : build_int_cst (ntype, 1));
5844 :
5845 0 : gcc_assert (TREE_CODE (pref_type) == INTEGER_TYPE);
5846 :
5847 0 : int prec = TYPE_PRECISION (ntype);
5848 0 : int pref_prec = TYPE_PRECISION (pref_type);
5849 :
5850 0 : tree base;
5851 :
5852 : /* Check if the PREFERRED_MODED is able to present niter. */
5853 0 : if (pref_prec > prec
5854 0 : || wi::ltu_p (iterations_max,
5855 0 : widest_int::from (wi::max_value (pref_prec, UNSIGNED),
5856 : UNSIGNED)))
5857 : {
5858 : /* No wrap, it is safe to use preferred type after niter + 1. */
5859 0 : if (wi::ltu_p (iterations_max,
5860 0 : widest_int::from (wi::max_value (prec, UNSIGNED),
5861 : UNSIGNED)))
5862 : {
5863 : /* This could help to optimize "-1 +1" pair when niter looks
5864 : like "n-1": n is in original mode. "base = (n - 1) + 1"
5865 : in PREFERRED_MODED: it could be base = (PREFERRED_TYPE)n. */
5866 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5867 : build_int_cst (ntype, 1));
5868 0 : base = fold_convert (pref_type, base);
5869 : }
5870 :
5871 : /* To avoid wrap, convert niter to preferred type before plus 1. */
5872 : else
5873 : {
5874 0 : niter = fold_convert (pref_type, niter);
5875 0 : base = fold_build2 (PLUS_EXPR, pref_type, unshare_expr (niter),
5876 : build_int_cst (pref_type, 1));
5877 : }
5878 : }
5879 : else
5880 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5881 : build_int_cst (ntype, 1));
5882 : return base;
5883 : }
5884 :
5885 : /* Add one doloop dedicated IV candidate:
5886 : - Base is (may_be_zero ? 1 : (niter + 1)).
5887 : - Step is -1. */
5888 :
5889 : static void
5890 0 : add_iv_candidate_for_doloop (struct ivopts_data *data)
5891 : {
5892 0 : tree_niter_desc *niter_desc = niter_for_single_dom_exit (data);
5893 0 : gcc_assert (niter_desc && niter_desc->assumptions);
5894 :
5895 0 : tree niter = niter_desc->niter;
5896 0 : tree ntype = TREE_TYPE (niter);
5897 0 : gcc_assert (INTEGRAL_NB_TYPE_P (ntype));
5898 :
5899 0 : tree may_be_zero = niter_desc->may_be_zero;
5900 0 : if (may_be_zero && integer_zerop (may_be_zero))
5901 : may_be_zero = NULL_TREE;
5902 0 : if (may_be_zero)
5903 : {
5904 0 : if (COMPARISON_CLASS_P (may_be_zero))
5905 : {
5906 0 : niter = fold_build3 (COND_EXPR, ntype, may_be_zero,
5907 : build_int_cst (ntype, 0),
5908 : rewrite_to_non_trapping_overflow (niter));
5909 : }
5910 : /* Don't try to obtain the iteration count expression when may_be_zero is
5911 : integer_nonzerop (actually iteration count is one) or else. */
5912 : else
5913 : return;
5914 : }
5915 :
5916 0 : machine_mode mode = TYPE_MODE (ntype);
5917 0 : machine_mode pref_mode = targetm.preferred_doloop_mode (mode);
5918 :
5919 0 : tree base;
5920 0 : if (mode != pref_mode)
5921 : {
5922 0 : base = compute_doloop_base_on_mode (pref_mode, niter, niter_desc->max);
5923 0 : ntype = TREE_TYPE (base);
5924 : }
5925 : else
5926 0 : base = fold_build2 (PLUS_EXPR, ntype, unshare_expr (niter),
5927 : build_int_cst (ntype, 1));
5928 :
5929 : /* For non integer types or non-mode precision types,
5930 : convert directly to an integer type. */
5931 0 : if (TREE_CODE (ntype) != INTEGER_TYPE
5932 0 : || !type_has_mode_precision_p (ntype))
5933 : {
5934 0 : ntype = lang_hooks.types.type_for_mode (TYPE_MODE (ntype),
5935 0 : TYPE_UNSIGNED (ntype));
5936 0 : base = fold_convert (ntype, base);
5937 : }
5938 :
5939 0 : add_candidate (data, base, build_int_cst (ntype, -1), true, NULL, NULL, true);
5940 : }
5941 :
5942 : /* Finds the candidates for the induction variables. */
5943 :
5944 : static void
5945 513181 : find_iv_candidates (struct ivopts_data *data)
5946 : {
5947 : /* Add commonly used ivs. */
5948 513181 : add_standard_iv_candidates (data);
5949 :
5950 : /* Add doloop dedicated ivs. */
5951 513181 : if (data->doloop_use_p)
5952 0 : add_iv_candidate_for_doloop (data);
5953 :
5954 : /* Add old induction variables. */
5955 513181 : add_iv_candidate_for_bivs (data);
5956 :
5957 : /* Add induction variables derived from uses. */
5958 513181 : add_iv_candidate_for_groups (data);
5959 :
5960 513181 : set_autoinc_for_original_candidates (data);
5961 :
5962 : /* Record the important candidates. */
5963 513181 : record_important_candidates (data);
5964 :
5965 : /* Relate compare iv_use with all candidates. */
5966 513181 : if (!data->consider_all_candidates)
5967 301 : relate_compare_use_with_all_cands (data);
5968 :
5969 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
5970 : {
5971 72 : unsigned i;
5972 :
5973 72 : fprintf (dump_file, "\n<Important Candidates>:\t");
5974 877 : for (i = 0; i < data->vcands.length (); i++)
5975 733 : if (data->vcands[i]->important)
5976 521 : fprintf (dump_file, " %d,", data->vcands[i]->id);
5977 72 : fprintf (dump_file, "\n");
5978 :
5979 72 : fprintf (dump_file, "\n<Group, Cand> Related:\n");
5980 377 : for (i = 0; i < data->vgroups.length (); i++)
5981 : {
5982 233 : struct iv_group *group = data->vgroups[i];
5983 :
5984 233 : if (group->related_cands)
5985 : {
5986 233 : fprintf (dump_file, " Group %d:\t", group->id);
5987 233 : dump_bitmap (dump_file, group->related_cands);
5988 : }
5989 : }
5990 72 : fprintf (dump_file, "\n");
5991 : }
5992 513181 : }
5993 :
5994 : /* Return the number of registers needed to hold a value of a given TYPE. */
5995 :
5996 : static unsigned
5997 7390307 : nregs_per_value (tree type)
5998 : {
5999 7390307 : machine_mode mode = TYPE_MODE (type);
6000 7390307 : if (SCALAR_INT_MODE_P (mode))
6001 7338825 : return MAX (1u, targetm.class_max_nregs (GENERAL_REGS, mode));
6002 :
6003 : return 1;
6004 : }
6005 :
6006 : /* Determines costs of computing use of iv with an iv candidate. */
6007 :
6008 : static void
6009 513181 : determine_group_iv_costs (struct ivopts_data *data)
6010 : {
6011 513181 : unsigned i, j;
6012 513181 : struct iv_cand *cand;
6013 513181 : struct iv_group *group;
6014 513181 : bitmap to_clear = BITMAP_ALLOC (NULL);
6015 :
6016 513181 : alloc_use_cost_map (data);
6017 :
6018 2181180 : for (i = 0; i < data->vgroups.length (); i++)
6019 : {
6020 1667999 : group = data->vgroups[i];
6021 :
6022 1667999 : if (data->consider_all_candidates)
6023 : {
6024 19221521 : for (j = 0; j < data->vcands.length (); j++)
6025 : {
6026 17553522 : cand = data->vcands[j];
6027 17553522 : determine_group_iv_cost (data, group, cand);
6028 : }
6029 : }
6030 : else
6031 : {
6032 9321 : bitmap_iterator bi;
6033 :
6034 344667 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, j, bi)
6035 : {
6036 335346 : cand = data->vcands[j];
6037 335346 : if (!determine_group_iv_cost (data, group, cand))
6038 201460 : bitmap_set_bit (to_clear, j);
6039 : }
6040 :
6041 : /* Remove the candidates for that the cost is infinite from
6042 : the list of related candidates. */
6043 9321 : bitmap_and_compl_into (group->related_cands, to_clear);
6044 9321 : bitmap_clear (to_clear);
6045 : }
6046 : }
6047 :
6048 513181 : BITMAP_FREE (to_clear);
6049 :
6050 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6051 : {
6052 72 : bitmap_iterator bi;
6053 :
6054 : /* Dump invariant variables. */
6055 72 : fprintf (dump_file, "\n<Invariant Vars>:\n");
6056 1137 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
6057 : {
6058 1065 : struct version_info *info = ver_info (data, i);
6059 1065 : if (info->inv_id)
6060 : {
6061 242 : fprintf (dump_file, "Inv %d:\t", info->inv_id);
6062 242 : print_generic_expr (dump_file, info->name, TDF_SLIM);
6063 242 : fprintf (dump_file, "%s\n",
6064 242 : info->has_nonlin_use ? "" : "\t(eliminable)");
6065 : }
6066 : }
6067 :
6068 : /* Dump invariant expressions. */
6069 72 : fprintf (dump_file, "\n<Invariant Expressions>:\n");
6070 72 : auto_vec <iv_inv_expr_ent *> list (data->inv_expr_tab->elements ());
6071 :
6072 386 : for (hash_table<iv_inv_expr_hasher>::iterator it
6073 530 : = data->inv_expr_tab->begin (); it != data->inv_expr_tab->end ();
6074 386 : ++it)
6075 386 : list.safe_push (*it);
6076 :
6077 72 : list.qsort (sort_iv_inv_expr_ent);
6078 :
6079 458 : for (i = 0; i < list.length (); ++i)
6080 : {
6081 386 : fprintf (dump_file, "inv_expr %d: \t", list[i]->id);
6082 386 : print_generic_expr (dump_file, list[i]->expr, TDF_SLIM);
6083 386 : fprintf (dump_file, "\n");
6084 : }
6085 :
6086 72 : fprintf (dump_file, "\n<Group-candidate Costs>:\n");
6087 :
6088 305 : for (i = 0; i < data->vgroups.length (); i++)
6089 : {
6090 233 : group = data->vgroups[i];
6091 :
6092 233 : fprintf (dump_file, "Group %d:\n", i);
6093 233 : fprintf (dump_file, " cand\tcost\tcompl.\tinv.expr.\tinv.vars\n");
6094 3137 : for (j = 0; j < group->n_map_members; j++)
6095 : {
6096 4063 : if (!group->cost_map[j].cand
6097 2904 : || group->cost_map[j].cost.infinite_cost_p ())
6098 1159 : continue;
6099 :
6100 1745 : fprintf (dump_file, " %d\t%" PRId64 "\t%d\t",
6101 1745 : group->cost_map[j].cand->id,
6102 : group->cost_map[j].cost.cost,
6103 1745 : group->cost_map[j].cost.complexity);
6104 1745 : if (!group->cost_map[j].inv_exprs
6105 1745 : || bitmap_empty_p (group->cost_map[j].inv_exprs))
6106 1234 : fprintf (dump_file, "NIL;\t");
6107 : else
6108 511 : bitmap_print (dump_file,
6109 : group->cost_map[j].inv_exprs, "", ";\t");
6110 1745 : if (!group->cost_map[j].inv_vars
6111 1745 : || bitmap_empty_p (group->cost_map[j].inv_vars))
6112 1398 : fprintf (dump_file, "NIL;\n");
6113 : else
6114 347 : bitmap_print (dump_file,
6115 : group->cost_map[j].inv_vars, "", "\n");
6116 : }
6117 :
6118 233 : fprintf (dump_file, "\n");
6119 : }
6120 72 : fprintf (dump_file, "\n");
6121 72 : }
6122 513181 : }
6123 :
6124 : /* Determines cost of the candidate CAND. */
6125 :
6126 : static void
6127 4692401 : determine_iv_cost (struct ivopts_data *data, struct iv_cand *cand)
6128 : {
6129 4692401 : comp_cost cost_base;
6130 4692401 : int64_t cost, cost_step;
6131 4692401 : tree base;
6132 :
6133 4692401 : gcc_assert (cand->iv != NULL);
6134 :
6135 : /* There are two costs associated with the candidate -- its increment
6136 : and its initialization. The second is almost negligible for any loop
6137 : that rolls enough, so we take it just very little into account. */
6138 :
6139 4692401 : base = cand->iv->base;
6140 4692401 : cost_base = force_var_cost (data, base, NULL);
6141 : /* It will be exceptional that the iv register happens to be initialized with
6142 : the proper value at no cost. In general, there will at least be a regcopy
6143 : or a const set. */
6144 4692401 : if (cost_base.cost == 0)
6145 3731382 : cost_base.cost = COSTS_N_INSNS (1);
6146 : /* Doloop decrement should be considered as zero cost. */
6147 4692401 : if (cand->doloop_p)
6148 : cost_step = 0;
6149 : else
6150 4692401 : cost_step = add_cost (data->speed, TYPE_MODE (TREE_TYPE (base)));
6151 4692401 : cost = cost_step + adjust_setup_cost (data, cost_base.cost);
6152 :
6153 : /* Prefer the original ivs unless we may gain something by replacing it.
6154 : The reason is to make debugging simpler; so this is not relevant for
6155 : artificial ivs created by other optimization passes. */
6156 4692401 : if ((cand->pos != IP_ORIGINAL
6157 885661 : || !SSA_NAME_VAR (cand->var_before)
6158 448151 : || DECL_ARTIFICIAL (SSA_NAME_VAR (cand->var_before)))
6159 : /* Prefer doloop as well. */
6160 5228111 : && !cand->doloop_p)
6161 4342450 : cost++;
6162 :
6163 : /* Prefer not to insert statements into latch unless there are some
6164 : already (so that we do not create unnecessary jumps). */
6165 4692401 : if (cand->pos == IP_END
6166 4692401 : && empty_block_p (ip_end_pos (data->current_loop)))
6167 1837 : cost++;
6168 :
6169 4692401 : cand->cost = cost;
6170 4692401 : cand->cost_step = cost_step;
6171 4692401 : cand->nregs = nregs_per_value (TREE_TYPE (cand->iv->base));
6172 4692401 : }
6173 :
6174 : /* Determines costs of computation of the candidates. */
6175 :
6176 : static void
6177 513181 : determine_iv_costs (struct ivopts_data *data)
6178 : {
6179 513181 : unsigned i;
6180 :
6181 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6182 : {
6183 72 : fprintf (dump_file, "<Candidate Costs>:\n");
6184 72 : fprintf (dump_file, " cand\tcost\n");
6185 : }
6186 :
6187 5205582 : for (i = 0; i < data->vcands.length (); i++)
6188 : {
6189 4692401 : struct iv_cand *cand = data->vcands[i];
6190 :
6191 4692401 : determine_iv_cost (data, cand);
6192 :
6193 4692401 : if (dump_file && (dump_flags & TDF_DETAILS))
6194 733 : fprintf (dump_file, " %d\t%d\n", i, cand->cost);
6195 : }
6196 :
6197 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6198 72 : fprintf (dump_file, "\n");
6199 513181 : }
6200 :
6201 : /* Estimate register pressure for loop whose invariants occupy N_INV_REGS
6202 : registers and whose induction variables occupy N_CAND_REGS registers.
6203 : Note N_INV_REGS covers both invariant variables and invariant expressions.
6204 : All counts, including DATA->regs_used, are in GENERAL_REGS hard registers,
6205 : the same units as target_avail_regs. */
6206 :
6207 : static unsigned
6208 431802416 : ivopts_estimate_reg_pressure (struct ivopts_data *data, unsigned n_inv_regs,
6209 : unsigned n_cand_regs)
6210 : {
6211 431802416 : unsigned cost;
6212 431802416 : unsigned n_old = data->regs_used, n_new = n_inv_regs + n_cand_regs;
6213 431802416 : unsigned regs_needed = n_new + n_old, available_regs = target_avail_regs;
6214 431802416 : bool speed = data->speed;
6215 :
6216 : /* If there is a call in the loop body, the call-clobbered registers
6217 : are not available for loop invariants. */
6218 431802416 : if (data->body_includes_call)
6219 96168856 : available_regs = available_regs - target_clobbered_regs;
6220 :
6221 : /* If we have enough registers. */
6222 431802416 : if (regs_needed <= available_regs)
6223 : cost = 0;
6224 : /* If we run out of available registers but the number of candidates
6225 : does not, we penalize extra registers using target_spill_cost.
6226 : As we tend to spill invariants here, only take loading the
6227 : invariant into account, because the invariant won't change for the
6228 : duration of the loop and storing it every iteration is unnecessary. */
6229 137053629 : else if (n_cand_regs <= available_regs)
6230 117354845 : cost = target_spill_cost [speed] * (regs_needed - available_regs) / 2;
6231 : /* If both IV cands and invariants spill, calculate additional cost for
6232 : having to store spilled candidates. */
6233 : else
6234 19698784 : cost = (target_spill_cost[speed] * (regs_needed - available_regs) / 2
6235 19698784 : + target_spill_cost[speed] * (n_cand_regs - available_regs) / 2);
6236 :
6237 431802416 : return cost;
6238 : }
6239 :
6240 : /* For each size of the induction variable set determine the penalty. */
6241 :
6242 : static void
6243 513181 : determine_set_costs (struct ivopts_data *data)
6244 : {
6245 513181 : unsigned j, n, n_values;
6246 513181 : gphi *phi;
6247 513181 : gphi_iterator psi;
6248 513181 : tree op;
6249 513181 : class loop *loop = data->current_loop;
6250 513181 : bitmap_iterator bi;
6251 :
6252 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6253 : {
6254 72 : fprintf (dump_file, "<Global Costs>:\n");
6255 72 : fprintf (dump_file, " target_avail_regs %d\n", target_avail_regs);
6256 72 : fprintf (dump_file, " target_clobbered_regs %d\n", target_clobbered_regs);
6257 72 : fprintf (dump_file, " target_reg_cost %d\n", target_reg_cost[data->speed]);
6258 72 : fprintf (dump_file, " target_spill_cost %d\n", target_spill_cost[data->speed]);
6259 : }
6260 :
6261 513181 : n = 0;
6262 513181 : n_values = 0;
6263 1995226 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
6264 : {
6265 1482045 : phi = psi.phi ();
6266 1482045 : op = PHI_RESULT (phi);
6267 :
6268 2964090 : if (virtual_operand_p (op))
6269 314163 : continue;
6270 :
6271 1167882 : if (get_iv (data, op))
6272 889564 : continue;
6273 :
6274 511306 : if (!POINTER_TYPE_P (TREE_TYPE (op))
6275 511127 : && !INTEGRAL_TYPE_P (TREE_TYPE (op)))
6276 103328 : continue;
6277 :
6278 174990 : n += nregs_per_value (TREE_TYPE (op));
6279 174990 : n_values++;
6280 : }
6281 :
6282 513181 : data->inv_expr_nregs.safe_grow_cleared (data->max_inv_expr_id + 1);
6283 513181 : data->inv_var_nregs.safe_grow_cleared (data->max_inv_var_id + 1);
6284 :
6285 1677528 : for (hash_table<iv_inv_expr_hasher>::iterator it
6286 513181 : = data->inv_expr_tab->begin ();
6287 1677528 : it != data->inv_expr_tab->end (); ++it)
6288 1164347 : data->inv_expr_nregs[(*it)->id] = nregs_per_value (TREE_TYPE ((*it)->expr));
6289 :
6290 5673621 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
6291 : {
6292 5160440 : struct version_info *info = ver_info (data, j);
6293 :
6294 5160440 : if (info->inv_id)
6295 1358569 : data->inv_var_nregs[info->inv_id]
6296 2717138 : = nregs_per_value (TREE_TYPE (info->name));
6297 :
6298 5160440 : if (info->inv_id && info->has_nonlin_use)
6299 : {
6300 528021 : n += data->inv_var_nregs[info->inv_id];
6301 528021 : n_values++;
6302 : }
6303 : }
6304 :
6305 513181 : data->regs_used = n;
6306 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6307 72 : fprintf (dump_file, " regs_used %u (%u values)\n", n, n_values);
6308 :
6309 513181 : if (dump_file && (dump_flags & TDF_DETAILS))
6310 : {
6311 72 : fprintf (dump_file, " cost for size:\n");
6312 72 : fprintf (dump_file, " regs\tcost\n");
6313 2304 : for (j = 0; j <= 2 * target_avail_regs; j++)
6314 2232 : fprintf (dump_file, " %d\t%d\n", j,
6315 : ivopts_estimate_reg_pressure (data, 0, j));
6316 72 : fprintf (dump_file, "\n");
6317 : }
6318 513181 : }
6319 :
6320 : /* Returns true if A is a cheaper cost pair than B. */
6321 :
6322 : static bool
6323 86855307 : cheaper_cost_pair (class cost_pair *a, class cost_pair *b)
6324 : {
6325 86855307 : if (!a)
6326 : return false;
6327 :
6328 81406997 : if (!b)
6329 : return true;
6330 :
6331 78166798 : if (a->cost < b->cost)
6332 : return true;
6333 :
6334 57462616 : if (b->cost < a->cost)
6335 : return false;
6336 :
6337 : /* In case the costs are the same, prefer the cheaper candidate. */
6338 32592392 : if (a->cand->cost < b->cand->cost)
6339 4834646 : return true;
6340 :
6341 : return false;
6342 : }
6343 :
6344 : /* Compare if A is a more expensive cost pair than B. Return 1, 0 and -1
6345 : for more expensive, equal and cheaper respectively. */
6346 :
6347 : static int
6348 31039176 : compare_cost_pair (class cost_pair *a, class cost_pair *b)
6349 : {
6350 31039176 : if (cheaper_cost_pair (a, b))
6351 : return -1;
6352 24671793 : if (cheaper_cost_pair (b, a))
6353 16194582 : return 1;
6354 :
6355 : return 0;
6356 : }
6357 :
6358 : /* Returns candidate by that USE is expressed in IVS. */
6359 :
6360 : static class cost_pair *
6361 289690169 : iv_ca_cand_for_group (class iv_ca *ivs, struct iv_group *group)
6362 : {
6363 289690169 : return ivs->cand_for_group[group->id];
6364 : }
6365 :
6366 : /* Computes the cost field of IVS structure. */
6367 :
6368 : static void
6369 431799914 : iv_ca_recount_cost (struct ivopts_data *data, class iv_ca *ivs)
6370 : {
6371 431799914 : comp_cost cost = ivs->cand_use_cost;
6372 :
6373 431799914 : cost += ivs->cand_cost;
6374 431799914 : cost
6375 431799914 : += ivopts_estimate_reg_pressure (data, ivs->n_inv_regs, ivs->n_cand_regs);
6376 431799914 : ivs->cost = cost;
6377 :
6378 431799914 : gcc_checking_assert (ivs->n_cand_regs >= ivs->n_cands
6379 : && ivs->n_inv_regs >= ivs->n_invs);
6380 431799914 : }
6381 :
6382 : /* Remove use of invariants in set INVS by decreasing counter in N_INV_USES
6383 : and IVS. */
6384 :
6385 : static void
6386 593809726 : iv_ca_set_remove_invs (class iv_ca *ivs, bitmap invs, unsigned *n_inv_uses,
6387 : const vec<unsigned> &nregs)
6388 : {
6389 593809726 : bitmap_iterator bi;
6390 593809726 : unsigned iid;
6391 :
6392 593809726 : if (!invs)
6393 471798983 : return;
6394 :
6395 122010743 : gcc_assert (n_inv_uses != NULL);
6396 211192934 : EXECUTE_IF_SET_IN_BITMAP (invs, 0, iid, bi)
6397 : {
6398 89182191 : n_inv_uses[iid]--;
6399 89182191 : if (n_inv_uses[iid] == 0)
6400 : {
6401 66558451 : ivs->n_invs--;
6402 66558451 : ivs->n_inv_regs -= nregs[iid];
6403 : }
6404 : }
6405 : }
6406 :
6407 : /* Set USE not to be expressed by any candidate in IVS. */
6408 :
6409 : static void
6410 214233377 : iv_ca_set_no_cp (struct ivopts_data *data, class iv_ca *ivs,
6411 : struct iv_group *group)
6412 : {
6413 214233377 : unsigned gid = group->id, cid;
6414 214233377 : class cost_pair *cp;
6415 :
6416 214233377 : cp = ivs->cand_for_group[gid];
6417 214233377 : if (!cp)
6418 : return;
6419 214233377 : cid = cp->cand->id;
6420 :
6421 214233377 : ivs->bad_groups++;
6422 214233377 : ivs->cand_for_group[gid] = NULL;
6423 214233377 : ivs->n_cand_uses[cid]--;
6424 :
6425 214233377 : if (ivs->n_cand_uses[cid] == 0)
6426 : {
6427 82671486 : bitmap_clear_bit (ivs->cands, cid);
6428 82671486 : if (!cp->cand->doloop_p || !targetm.have_count_reg_decr_p)
6429 : {
6430 82671486 : ivs->n_cands--;
6431 82671486 : ivs->n_cand_regs -= cp->cand->nregs;
6432 : }
6433 82671486 : ivs->cand_cost -= cp->cand->cost;
6434 82671486 : iv_ca_set_remove_invs (ivs, cp->cand->inv_vars, ivs->n_inv_var_uses,
6435 82671486 : data->inv_var_nregs);
6436 82671486 : iv_ca_set_remove_invs (ivs, cp->cand->inv_exprs, ivs->n_inv_expr_uses,
6437 82671486 : data->inv_expr_nregs);
6438 : }
6439 :
6440 214233377 : ivs->cand_use_cost -= cp->cost;
6441 214233377 : iv_ca_set_remove_invs (ivs, cp->inv_vars, ivs->n_inv_var_uses,
6442 214233377 : data->inv_var_nregs);
6443 214233377 : iv_ca_set_remove_invs (ivs, cp->inv_exprs, ivs->n_inv_expr_uses,
6444 214233377 : data->inv_expr_nregs);
6445 214233377 : iv_ca_recount_cost (data, ivs);
6446 : }
6447 :
6448 : /* Add use of invariants in set INVS by increasing counter in N_INV_USES and
6449 : IVS. */
6450 :
6451 : static void
6452 603407996 : iv_ca_set_add_invs (class iv_ca *ivs, bitmap invs, unsigned *n_inv_uses,
6453 : const vec<unsigned> &nregs)
6454 : {
6455 603407996 : bitmap_iterator bi;
6456 603407996 : unsigned iid;
6457 :
6458 603407996 : if (!invs)
6459 480244596 : return;
6460 :
6461 123163400 : gcc_assert (n_inv_uses != NULL);
6462 213328172 : EXECUTE_IF_SET_IN_BITMAP (invs, 0, iid, bi)
6463 : {
6464 90164772 : n_inv_uses[iid]++;
6465 90164772 : if (n_inv_uses[iid] == 1)
6466 : {
6467 67467778 : ivs->n_invs++;
6468 67467778 : ivs->n_inv_regs += nregs[iid];
6469 : }
6470 : }
6471 : }
6472 :
6473 : /* Set cost pair for GROUP in set IVS to CP. */
6474 :
6475 : static void
6476 230765062 : iv_ca_set_cp (struct ivopts_data *data, class iv_ca *ivs,
6477 : struct iv_group *group, class cost_pair *cp)
6478 : {
6479 230765062 : unsigned gid = group->id, cid;
6480 :
6481 230765062 : if (ivs->cand_for_group[gid] == cp)
6482 : return;
6483 :
6484 217566537 : if (ivs->cand_for_group[gid])
6485 201859316 : iv_ca_set_no_cp (data, ivs, group);
6486 :
6487 217566537 : if (cp)
6488 : {
6489 217566537 : cid = cp->cand->id;
6490 :
6491 217566537 : ivs->bad_groups--;
6492 217566537 : ivs->cand_for_group[gid] = cp;
6493 217566537 : ivs->n_cand_uses[cid]++;
6494 217566537 : if (ivs->n_cand_uses[cid] == 1)
6495 : {
6496 84137461 : bitmap_set_bit (ivs->cands, cid);
6497 84137461 : if (!cp->cand->doloop_p || !targetm.have_count_reg_decr_p)
6498 : {
6499 84137461 : ivs->n_cands++;
6500 84137461 : ivs->n_cand_regs += cp->cand->nregs;
6501 : }
6502 84137461 : ivs->cand_cost += cp->cand->cost;
6503 84137461 : iv_ca_set_add_invs (ivs, cp->cand->inv_vars, ivs->n_inv_var_uses,
6504 84137461 : data->inv_var_nregs);
6505 84137461 : iv_ca_set_add_invs (ivs, cp->cand->inv_exprs, ivs->n_inv_expr_uses,
6506 84137461 : data->inv_expr_nregs);
6507 : }
6508 :
6509 217566537 : ivs->cand_use_cost += cp->cost;
6510 217566537 : iv_ca_set_add_invs (ivs, cp->inv_vars, ivs->n_inv_var_uses,
6511 217566537 : data->inv_var_nregs);
6512 217566537 : iv_ca_set_add_invs (ivs, cp->inv_exprs, ivs->n_inv_expr_uses,
6513 217566537 : data->inv_expr_nregs);
6514 217566537 : iv_ca_recount_cost (data, ivs);
6515 : }
6516 : }
6517 :
6518 : /* Extend set IVS by expressing USE by some of the candidates in it
6519 : if possible. Consider all important candidates if candidates in
6520 : set IVS don't give any result. */
6521 :
6522 : static void
6523 3334060 : iv_ca_add_group (struct ivopts_data *data, class iv_ca *ivs,
6524 : struct iv_group *group)
6525 : {
6526 3334060 : class cost_pair *best_cp = NULL, *cp;
6527 3334060 : bitmap_iterator bi;
6528 3334060 : unsigned i;
6529 3334060 : struct iv_cand *cand;
6530 :
6531 3334060 : gcc_assert (ivs->upto >= group->id);
6532 3334060 : ivs->upto++;
6533 3334060 : ivs->bad_groups++;
6534 :
6535 6304881 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
6536 : {
6537 2970821 : cand = data->vcands[i];
6538 2970821 : cp = get_group_iv_cost (data, group, cand);
6539 2970821 : if (cheaper_cost_pair (cp, best_cp))
6540 2046846 : best_cp = cp;
6541 : }
6542 :
6543 3334060 : if (best_cp == NULL)
6544 : {
6545 12008954 : EXECUTE_IF_SET_IN_BITMAP (data->important_candidates, 0, i, bi)
6546 : {
6547 10641725 : cand = data->vcands[i];
6548 10641725 : cp = get_group_iv_cost (data, group, cand);
6549 10641725 : if (cheaper_cost_pair (cp, best_cp))
6550 2443904 : best_cp = cp;
6551 : }
6552 : }
6553 :
6554 3334060 : iv_ca_set_cp (data, ivs, group, best_cp);
6555 3334060 : }
6556 :
6557 : /* Get cost for assignment IVS. */
6558 :
6559 : static comp_cost
6560 86297038 : iv_ca_cost (class iv_ca *ivs)
6561 : {
6562 : /* This was a conditional expression but it triggered a bug in
6563 : Sun C 5.5. */
6564 0 : if (ivs->bad_groups)
6565 93861 : return infinite_cost;
6566 : else
6567 86203177 : return ivs->cost;
6568 : }
6569 :
6570 : /* Compare if applying NEW_CP to GROUP for IVS introduces more invariants
6571 : than OLD_CP. Return 1, 0 and -1 for more, equal and fewer invariants
6572 : respectively. */
6573 :
6574 : static int
6575 41019091 : iv_ca_compare_deps (struct ivopts_data *data, class iv_ca *ivs,
6576 : struct iv_group *group, class cost_pair *old_cp,
6577 : class cost_pair *new_cp)
6578 : {
6579 41019091 : gcc_assert (old_cp && new_cp && old_cp != new_cp);
6580 41019091 : unsigned old_n_invs = ivs->n_invs;
6581 41019091 : iv_ca_set_cp (data, ivs, group, new_cp);
6582 41019091 : unsigned new_n_invs = ivs->n_invs;
6583 41019091 : iv_ca_set_cp (data, ivs, group, old_cp);
6584 :
6585 41019091 : return new_n_invs > old_n_invs ? 1 : (new_n_invs < old_n_invs ? -1 : 0);
6586 : }
6587 :
6588 : /* Creates change of expressing GROUP by NEW_CP instead of OLD_CP and chains
6589 : it before NEXT. */
6590 :
6591 : static struct iv_ca_delta *
6592 48404703 : iv_ca_delta_add (struct iv_group *group, class cost_pair *old_cp,
6593 : class cost_pair *new_cp, struct iv_ca_delta *next)
6594 : {
6595 0 : struct iv_ca_delta *change = XNEW (struct iv_ca_delta);
6596 :
6597 48404703 : change->group = group;
6598 48404703 : change->old_cp = old_cp;
6599 48404703 : change->new_cp = new_cp;
6600 48404703 : change->next = next;
6601 :
6602 48404703 : return change;
6603 : }
6604 :
6605 : /* Joins two lists of changes L1 and L2. Destructive -- old lists
6606 : are rewritten. */
6607 :
6608 : static struct iv_ca_delta *
6609 8209331 : iv_ca_delta_join (struct iv_ca_delta *l1, struct iv_ca_delta *l2)
6610 : {
6611 8209331 : struct iv_ca_delta *last;
6612 :
6613 0 : if (!l2)
6614 : return l1;
6615 :
6616 0 : if (!l1)
6617 : return l2;
6618 :
6619 3337604 : for (last = l1; last->next; last = last->next)
6620 1104727 : continue;
6621 2232877 : last->next = l2;
6622 :
6623 2232877 : return l1;
6624 1104727 : }
6625 :
6626 : /* Reverse the list of changes DELTA, forming the inverse to it. */
6627 :
6628 : static struct iv_ca_delta *
6629 0 : iv_ca_delta_reverse (struct iv_ca_delta *delta)
6630 : {
6631 0 : struct iv_ca_delta *act, *next, *prev = NULL;
6632 :
6633 162354188 : for (act = delta; act; act = next)
6634 : {
6635 90488852 : next = act->next;
6636 90488852 : act->next = prev;
6637 90488852 : prev = act;
6638 :
6639 90488852 : std::swap (act->old_cp, act->new_cp);
6640 : }
6641 :
6642 0 : return prev;
6643 : }
6644 :
6645 : /* Commit changes in DELTA to IVS. If FORWARD is false, the changes are
6646 : reverted instead. */
6647 :
6648 : static void
6649 75730107 : iv_ca_delta_commit (struct ivopts_data *data, class iv_ca *ivs,
6650 : struct iv_ca_delta *delta, bool forward)
6651 : {
6652 75730107 : class cost_pair *from, *to;
6653 75730107 : struct iv_ca_delta *act;
6654 :
6655 75730107 : if (!forward)
6656 75730107 : delta = iv_ca_delta_reverse (delta);
6657 :
6658 171183346 : for (act = delta; act; act = act->next)
6659 : {
6660 95453239 : from = act->old_cp;
6661 95453239 : to = act->new_cp;
6662 95453239 : gcc_assert (iv_ca_cand_for_group (ivs, act->group) == from);
6663 95453239 : iv_ca_set_cp (data, ivs, act->group, to);
6664 : }
6665 :
6666 75730107 : if (!forward)
6667 75730107 : iv_ca_delta_reverse (delta);
6668 75730107 : }
6669 :
6670 : /* Returns true if CAND is used in IVS. */
6671 :
6672 : static bool
6673 29960396 : iv_ca_cand_used_p (class iv_ca *ivs, struct iv_cand *cand)
6674 : {
6675 29960396 : return ivs->n_cand_uses[cand->id] > 0;
6676 : }
6677 :
6678 : /* Returns number of induction variable candidates in the set IVS. */
6679 :
6680 : static unsigned
6681 13170377 : iv_ca_n_cands (class iv_ca *ivs)
6682 : {
6683 13170377 : return ivs->n_cands;
6684 : }
6685 :
6686 : /* Free the list of changes DELTA. */
6687 :
6688 : static void
6689 45426669 : iv_ca_delta_free (struct iv_ca_delta **delta)
6690 : {
6691 45426669 : struct iv_ca_delta *act, *next;
6692 :
6693 93831372 : for (act = *delta; act; act = next)
6694 : {
6695 48404703 : next = act->next;
6696 48404703 : free (act);
6697 : }
6698 :
6699 45426669 : *delta = NULL;
6700 45426669 : }
6701 :
6702 : /* Allocates new iv candidates assignment. */
6703 :
6704 : static class iv_ca *
6705 1026362 : iv_ca_new (struct ivopts_data *data)
6706 : {
6707 1026362 : class iv_ca *nw = XNEW (class iv_ca);
6708 :
6709 1026362 : nw->upto = 0;
6710 1026362 : nw->bad_groups = 0;
6711 2052724 : nw->cand_for_group = XCNEWVEC (class cost_pair *,
6712 : data->vgroups.length ());
6713 2052724 : nw->n_cand_uses = XCNEWVEC (unsigned, data->vcands.length ());
6714 1026362 : nw->cands = BITMAP_ALLOC (NULL);
6715 1026362 : nw->n_cands = 0;
6716 1026362 : nw->n_invs = 0;
6717 1026362 : nw->n_cand_regs = 0;
6718 1026362 : nw->n_inv_regs = 0;
6719 1026362 : nw->cand_use_cost = no_cost;
6720 1026362 : nw->cand_cost = 0;
6721 1026362 : nw->n_inv_var_uses = XCNEWVEC (unsigned, data->max_inv_var_id + 1);
6722 1026362 : nw->n_inv_expr_uses = XCNEWVEC (unsigned, data->max_inv_expr_id + 1);
6723 1026362 : nw->cost = no_cost;
6724 :
6725 1026362 : return nw;
6726 : }
6727 :
6728 : /* Free memory occupied by the set IVS. */
6729 :
6730 : static void
6731 1026362 : iv_ca_free (class iv_ca **ivs)
6732 : {
6733 1026362 : free ((*ivs)->cand_for_group);
6734 1026362 : free ((*ivs)->n_cand_uses);
6735 1026362 : BITMAP_FREE ((*ivs)->cands);
6736 1026362 : free ((*ivs)->n_inv_var_uses);
6737 1026362 : free ((*ivs)->n_inv_expr_uses);
6738 1026362 : free (*ivs);
6739 1026362 : *ivs = NULL;
6740 1026362 : }
6741 :
6742 : /* Dumps IVS to FILE. */
6743 :
6744 : static void
6745 270 : iv_ca_dump (struct ivopts_data *data, FILE *file, class iv_ca *ivs)
6746 : {
6747 270 : unsigned i;
6748 270 : comp_cost cost = iv_ca_cost (ivs);
6749 :
6750 270 : fprintf (file, " cost: %" PRId64 " (complexity %d)\n", cost.cost,
6751 : cost.complexity);
6752 270 : fprintf (file, " reg_cost: %d\n",
6753 : ivopts_estimate_reg_pressure (data, ivs->n_inv_regs,
6754 : ivs->n_cand_regs));
6755 270 : fprintf (file,
6756 : " regs: %u (%u values) cands, %u (%u values) invs, "
6757 : "%u used\n",
6758 : ivs->n_cand_regs, ivs->n_cands, ivs->n_inv_regs, ivs->n_invs,
6759 : data->regs_used);
6760 270 : fprintf (file, " cand_cost: %" PRId64 "\n cand_group_cost: "
6761 : "%" PRId64 " (complexity %d)\n", ivs->cand_cost,
6762 : ivs->cand_use_cost.cost, ivs->cand_use_cost.complexity);
6763 270 : bitmap_print (file, ivs->cands, " candidates: ","\n");
6764 :
6765 1670 : for (i = 0; i < ivs->upto; i++)
6766 : {
6767 1130 : struct iv_group *group = data->vgroups[i];
6768 1130 : class cost_pair *cp = iv_ca_cand_for_group (ivs, group);
6769 1130 : if (cp)
6770 1130 : fprintf (file, " group:%d --> iv_cand:%d, cost=("
6771 1130 : "%" PRId64 ",%d)\n", group->id, cp->cand->id,
6772 : cp->cost.cost, cp->cost.complexity);
6773 : else
6774 0 : fprintf (file, " group:%d --> ??\n", group->id);
6775 : }
6776 :
6777 270 : const char *pref = "";
6778 270 : fprintf (file, " invariant variables: ");
6779 1570 : for (i = 1; i <= data->max_inv_var_id; i++)
6780 1030 : if (ivs->n_inv_var_uses[i])
6781 : {
6782 158 : fprintf (file, "%s%d", pref, i);
6783 158 : pref = ", ";
6784 : }
6785 :
6786 270 : pref = "";
6787 270 : fprintf (file, "\n invariant expressions: ");
6788 2660 : for (i = 1; i <= data->max_inv_expr_id; i++)
6789 2120 : if (ivs->n_inv_expr_uses[i])
6790 : {
6791 316 : fprintf (file, "%s%d", pref, i);
6792 316 : pref = ", ";
6793 : }
6794 :
6795 270 : fprintf (file, "\n\n");
6796 270 : }
6797 :
6798 : /* Try changing candidate in IVS to CAND for each use. Return cost of the
6799 : new set, and store differences in DELTA. Number of induction variables
6800 : in the new set is stored to N_IVS. MIN_NCAND is a flag. When it is true
6801 : the function will try to find a solution with minimal iv candidates. */
6802 :
6803 : static comp_cost
6804 22304239 : iv_ca_extend (struct ivopts_data *data, class iv_ca *ivs,
6805 : struct iv_cand *cand, struct iv_ca_delta **delta,
6806 : unsigned *n_ivs, bool min_ncand)
6807 : {
6808 22304239 : unsigned i;
6809 22304239 : comp_cost cost;
6810 22304239 : struct iv_group *group;
6811 22304239 : class cost_pair *old_cp, *new_cp;
6812 :
6813 22304239 : *delta = NULL;
6814 125978779 : for (i = 0; i < ivs->upto; i++)
6815 : {
6816 103674540 : group = data->vgroups[i];
6817 103674540 : old_cp = iv_ca_cand_for_group (ivs, group);
6818 :
6819 103674540 : if (old_cp
6820 103674540 : && old_cp->cand == cand)
6821 9133862 : continue;
6822 :
6823 94540678 : new_cp = get_group_iv_cost (data, group, cand);
6824 94540678 : if (!new_cp)
6825 37951245 : continue;
6826 :
6827 56589433 : if (!min_ncand)
6828 : {
6829 41019091 : int cmp_invs = iv_ca_compare_deps (data, ivs, group, old_cp, new_cp);
6830 : /* Skip if new_cp depends on more invariants. */
6831 41019091 : if (cmp_invs > 0)
6832 9979915 : continue;
6833 :
6834 31039176 : int cmp_cost = compare_cost_pair (new_cp, old_cp);
6835 : /* Skip if new_cp is not cheaper. */
6836 31039176 : if (cmp_cost > 0 || (cmp_cost == 0 && cmp_invs == 0))
6837 24209921 : continue;
6838 : }
6839 :
6840 22399597 : *delta = iv_ca_delta_add (group, old_cp, new_cp, *delta);
6841 : }
6842 :
6843 22304239 : iv_ca_delta_commit (data, ivs, *delta, true);
6844 22304239 : cost = iv_ca_cost (ivs);
6845 22304239 : if (n_ivs)
6846 13170377 : *n_ivs = iv_ca_n_cands (ivs);
6847 22304239 : iv_ca_delta_commit (data, ivs, *delta, false);
6848 :
6849 22304239 : return cost;
6850 : }
6851 :
6852 : /* Try narrowing set IVS by removing CAND. Return the cost of
6853 : the new set and store the differences in DELTA. START is
6854 : the candidate with which we start narrowing. */
6855 :
6856 : static comp_cost
6857 15953252 : iv_ca_narrow (struct ivopts_data *data, class iv_ca *ivs,
6858 : struct iv_cand *cand, struct iv_cand *start,
6859 : struct iv_ca_delta **delta)
6860 : {
6861 15953252 : unsigned i, ci;
6862 15953252 : struct iv_group *group;
6863 15953252 : class cost_pair *old_cp, *new_cp, *cp;
6864 15953252 : bitmap_iterator bi;
6865 15953252 : struct iv_cand *cnd;
6866 15953252 : comp_cost cost, best_cost, acost;
6867 :
6868 15953252 : *delta = NULL;
6869 86434414 : for (i = 0; i < data->vgroups.length (); i++)
6870 : {
6871 81015316 : group = data->vgroups[i];
6872 :
6873 81015316 : old_cp = iv_ca_cand_for_group (ivs, group);
6874 81015316 : if (old_cp->cand != cand)
6875 58300780 : continue;
6876 :
6877 22714536 : best_cost = iv_ca_cost (ivs);
6878 : /* Start narrowing with START. */
6879 22714536 : new_cp = get_group_iv_cost (data, group, start);
6880 :
6881 22714536 : if (data->consider_all_candidates)
6882 : {
6883 99810272 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, ci, bi)
6884 : {
6885 78253017 : if (ci == cand->id || (start && ci == start->id))
6886 37239175 : continue;
6887 :
6888 41013842 : cnd = data->vcands[ci];
6889 :
6890 41013842 : cp = get_group_iv_cost (data, group, cnd);
6891 41013842 : if (!cp)
6892 24451114 : continue;
6893 :
6894 16562728 : iv_ca_set_cp (data, ivs, group, cp);
6895 16562728 : acost = iv_ca_cost (ivs);
6896 :
6897 16562728 : if (acost < best_cost)
6898 : {
6899 1855364 : best_cost = acost;
6900 1855364 : new_cp = cp;
6901 : }
6902 : }
6903 : }
6904 : else
6905 : {
6906 4598078 : EXECUTE_IF_AND_IN_BITMAP (group->related_cands, ivs->cands, 0, ci, bi)
6907 : {
6908 3440797 : if (ci == cand->id || (start && ci == start->id))
6909 1912342 : continue;
6910 :
6911 1528455 : cnd = data->vcands[ci];
6912 :
6913 1528455 : cp = get_group_iv_cost (data, group, cnd);
6914 1528455 : if (!cp)
6915 0 : continue;
6916 :
6917 1528455 : iv_ca_set_cp (data, ivs, group, cp);
6918 1528455 : acost = iv_ca_cost (ivs);
6919 :
6920 1528455 : if (acost < best_cost)
6921 : {
6922 34173 : best_cost = acost;
6923 34173 : new_cp = cp;
6924 : }
6925 : }
6926 : }
6927 : /* Restore to old cp for use. */
6928 22714536 : iv_ca_set_cp (data, ivs, group, old_cp);
6929 :
6930 22714536 : if (!new_cp)
6931 : {
6932 10534154 : iv_ca_delta_free (delta);
6933 10534154 : return infinite_cost;
6934 : }
6935 :
6936 12180382 : *delta = iv_ca_delta_add (group, old_cp, new_cp, *delta);
6937 : }
6938 :
6939 5419098 : iv_ca_delta_commit (data, ivs, *delta, true);
6940 5419098 : cost = iv_ca_cost (ivs);
6941 5419098 : iv_ca_delta_commit (data, ivs, *delta, false);
6942 :
6943 5419098 : return cost;
6944 : }
6945 :
6946 : /* Try optimizing the set of candidates IVS by removing candidates different
6947 : from to EXCEPT_CAND from it. Return cost of the new set, and store
6948 : differences in DELTA. */
6949 :
6950 : static comp_cost
6951 9270382 : iv_ca_prune (struct ivopts_data *data, class iv_ca *ivs,
6952 : struct iv_cand *except_cand, struct iv_ca_delta **delta)
6953 : {
6954 9270382 : bitmap_iterator bi;
6955 9270382 : struct iv_ca_delta *act_delta, *best_delta;
6956 9270382 : unsigned i;
6957 9270382 : comp_cost best_cost, acost;
6958 9270382 : struct iv_cand *cand;
6959 :
6960 9270382 : best_delta = NULL;
6961 9270382 : best_cost = iv_ca_cost (ivs);
6962 :
6963 31737337 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
6964 : {
6965 22466955 : cand = data->vcands[i];
6966 :
6967 22466955 : if (cand == except_cand)
6968 6513703 : continue;
6969 :
6970 15953252 : acost = iv_ca_narrow (data, ivs, cand, except_cand, &act_delta);
6971 :
6972 15953252 : if (acost < best_cost)
6973 : {
6974 2379918 : best_cost = acost;
6975 2379918 : iv_ca_delta_free (&best_delta);
6976 2379918 : best_delta = act_delta;
6977 : }
6978 : else
6979 13573334 : iv_ca_delta_free (&act_delta);
6980 : }
6981 :
6982 9270382 : if (!best_delta)
6983 : {
6984 7035367 : *delta = NULL;
6985 7035367 : return best_cost;
6986 : }
6987 :
6988 : /* Recurse to possibly remove other unnecessary ivs. */
6989 2235015 : iv_ca_delta_commit (data, ivs, best_delta, true);
6990 2235015 : best_cost = iv_ca_prune (data, ivs, except_cand, delta);
6991 2235015 : iv_ca_delta_commit (data, ivs, best_delta, false);
6992 2235015 : *delta = iv_ca_delta_join (best_delta, *delta);
6993 2235015 : return best_cost;
6994 : }
6995 :
6996 : /* Check if CAND_IDX is a candidate other than OLD_CAND and has
6997 : cheaper local cost for GROUP than BEST_CP. Return pointer to
6998 : the corresponding cost_pair, otherwise just return BEST_CP. */
6999 :
7000 : static class cost_pair*
7001 29511832 : cheaper_cost_with_cand (struct ivopts_data *data, struct iv_group *group,
7002 : unsigned int cand_idx, struct iv_cand *old_cand,
7003 : class cost_pair *best_cp)
7004 : {
7005 29511832 : struct iv_cand *cand;
7006 29511832 : class cost_pair *cp;
7007 :
7008 29511832 : gcc_assert (old_cand != NULL && best_cp != NULL);
7009 29511832 : if (cand_idx == old_cand->id)
7010 : return best_cp;
7011 :
7012 26660876 : cand = data->vcands[cand_idx];
7013 26660876 : cp = get_group_iv_cost (data, group, cand);
7014 26660876 : if (cp != NULL && cheaper_cost_pair (cp, best_cp))
7015 1726312 : return cp;
7016 :
7017 : return best_cp;
7018 : }
7019 :
7020 : /* Try breaking local optimal fixed-point for IVS by replacing candidates
7021 : which are used by more than one iv uses. For each of those candidates,
7022 : this function tries to represent iv uses under that candidate using
7023 : other ones with lower local cost, then tries to prune the new set.
7024 : If the new set has lower cost, It returns the new cost after recording
7025 : candidate replacement in list DELTA. */
7026 :
7027 : static comp_cost
7028 1025462 : iv_ca_replace (struct ivopts_data *data, class iv_ca *ivs,
7029 : struct iv_ca_delta **delta)
7030 : {
7031 1025462 : bitmap_iterator bi, bj;
7032 1025462 : unsigned int i, j, k;
7033 1025462 : struct iv_cand *cand;
7034 1025462 : comp_cost orig_cost, acost;
7035 1025462 : struct iv_ca_delta *act_delta, *tmp_delta;
7036 1025462 : class cost_pair *old_cp, *best_cp = NULL;
7037 :
7038 1025462 : *delta = NULL;
7039 1025462 : orig_cost = iv_ca_cost (ivs);
7040 :
7041 2415495 : EXECUTE_IF_SET_IN_BITMAP (ivs->cands, 0, i, bi)
7042 : {
7043 1423484 : if (ivs->n_cand_uses[i] == 1
7044 1051666 : || ivs->n_cand_uses[i] > ALWAYS_PRUNE_CAND_SET_BOUND)
7045 376583 : continue;
7046 :
7047 1046901 : cand = data->vcands[i];
7048 :
7049 1046901 : act_delta = NULL;
7050 : /* Represent uses under current candidate using other ones with
7051 : lower local cost. */
7052 5476390 : for (j = 0; j < ivs->upto; j++)
7053 : {
7054 4429489 : struct iv_group *group = data->vgroups[j];
7055 4429489 : old_cp = iv_ca_cand_for_group (ivs, group);
7056 :
7057 4429489 : if (old_cp->cand != cand)
7058 1578533 : continue;
7059 :
7060 2850956 : best_cp = old_cp;
7061 2850956 : if (data->consider_all_candidates)
7062 32252402 : for (k = 0; k < data->vcands.length (); k++)
7063 29409163 : best_cp = cheaper_cost_with_cand (data, group, k,
7064 : old_cp->cand, best_cp);
7065 : else
7066 110386 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, k, bj)
7067 102669 : best_cp = cheaper_cost_with_cand (data, group, k,
7068 : old_cp->cand, best_cp);
7069 :
7070 2850956 : if (best_cp == old_cp)
7071 1400293 : continue;
7072 :
7073 1450663 : act_delta = iv_ca_delta_add (group, old_cp, best_cp, act_delta);
7074 : }
7075 : /* No need for further prune. */
7076 1046901 : if (!act_delta)
7077 260108 : continue;
7078 :
7079 : /* Prune the new candidate set. */
7080 786793 : iv_ca_delta_commit (data, ivs, act_delta, true);
7081 786793 : acost = iv_ca_prune (data, ivs, NULL, &tmp_delta);
7082 786793 : iv_ca_delta_commit (data, ivs, act_delta, false);
7083 786793 : act_delta = iv_ca_delta_join (act_delta, tmp_delta);
7084 :
7085 786793 : if (acost < orig_cost)
7086 : {
7087 33451 : *delta = act_delta;
7088 33451 : return acost;
7089 : }
7090 : else
7091 753342 : iv_ca_delta_free (&act_delta);
7092 : }
7093 :
7094 992011 : return orig_cost;
7095 : }
7096 :
7097 : /* Tries to extend the sets IVS in the best possible way in order to
7098 : express the GROUP. If ORIGINALP is true, prefer candidates from
7099 : the original set of IVs, otherwise favor important candidates not
7100 : based on any memory object. */
7101 :
7102 : static bool
7103 3334060 : try_add_cand_for (struct ivopts_data *data, class iv_ca *ivs,
7104 : struct iv_group *group, bool originalp)
7105 : {
7106 3334060 : comp_cost best_cost, act_cost;
7107 3334060 : unsigned i;
7108 3334060 : bitmap_iterator bi;
7109 3334060 : struct iv_cand *cand;
7110 3334060 : struct iv_ca_delta *best_delta = NULL, *act_delta;
7111 3334060 : class cost_pair *cp;
7112 :
7113 3334060 : iv_ca_add_group (data, ivs, group);
7114 3334060 : best_cost = iv_ca_cost (ivs);
7115 3334060 : cp = iv_ca_cand_for_group (ivs, group);
7116 3334060 : if (cp)
7117 : {
7118 3240199 : best_delta = iv_ca_delta_add (group, NULL, cp, NULL);
7119 3240199 : iv_ca_set_no_cp (data, ivs, group);
7120 : }
7121 :
7122 : /* If ORIGINALP is true, try to find the original IV for the use. Otherwise
7123 : first try important candidates not based on any memory object. Only if
7124 : this fails, try the specific ones. Rationale -- in loops with many
7125 : variables the best choice often is to use just one generic biv. If we
7126 : added here many ivs specific to the uses, the optimization algorithm later
7127 : would be likely to get stuck in a local minimum, thus causing us to create
7128 : too many ivs. The approach from few ivs to more seems more likely to be
7129 : successful -- starting from few ivs, replacing an expensive use by a
7130 : specific iv should always be a win. */
7131 31029104 : EXECUTE_IF_SET_IN_BITMAP (group->related_cands, 0, i, bi)
7132 : {
7133 27695044 : cand = data->vcands[i];
7134 :
7135 27695044 : if (originalp && cand->pos !=IP_ORIGINAL)
7136 10907072 : continue;
7137 :
7138 13847522 : if (!originalp && cand->iv->base_object != NULL_TREE)
7139 2545225 : continue;
7140 :
7141 14242747 : if (iv_ca_cand_used_p (ivs, cand))
7142 1530352 : continue;
7143 :
7144 12712395 : cp = get_group_iv_cost (data, group, cand);
7145 12712395 : if (!cp)
7146 3695005 : continue;
7147 :
7148 9017390 : iv_ca_set_cp (data, ivs, group, cp);
7149 9017390 : act_cost = iv_ca_extend (data, ivs, cand, &act_delta, NULL,
7150 : true);
7151 9017390 : iv_ca_set_no_cp (data, ivs, group);
7152 9017390 : act_delta = iv_ca_delta_add (group, NULL, cp, act_delta);
7153 :
7154 9017390 : if (act_cost < best_cost)
7155 : {
7156 365918 : best_cost = act_cost;
7157 :
7158 365918 : iv_ca_delta_free (&best_delta);
7159 365918 : best_delta = act_delta;
7160 : }
7161 : else
7162 8651472 : iv_ca_delta_free (&act_delta);
7163 : }
7164 :
7165 3334060 : if (best_cost.infinite_cost_p ())
7166 : {
7167 739361 : for (i = 0; i < group->n_map_members; i++)
7168 : {
7169 670957 : cp = group->cost_map + i;
7170 670957 : cand = cp->cand;
7171 670957 : if (!cand)
7172 554485 : continue;
7173 :
7174 : /* Already tried this. */
7175 116472 : if (cand->important)
7176 : {
7177 0 : if (originalp && cand->pos == IP_ORIGINAL)
7178 0 : continue;
7179 0 : if (!originalp && cand->iv->base_object == NULL_TREE)
7180 0 : continue;
7181 : }
7182 :
7183 116472 : if (iv_ca_cand_used_p (ivs, cand))
7184 0 : continue;
7185 :
7186 116472 : act_delta = NULL;
7187 116472 : iv_ca_set_cp (data, ivs, group, cp);
7188 116472 : act_cost = iv_ca_extend (data, ivs, cand, &act_delta, NULL, true);
7189 116472 : iv_ca_set_no_cp (data, ivs, group);
7190 116472 : act_delta = iv_ca_delta_add (group,
7191 : iv_ca_cand_for_group (ivs, group),
7192 : cp, act_delta);
7193 :
7194 116472 : if (act_cost < best_cost)
7195 : {
7196 69987 : best_cost = act_cost;
7197 :
7198 69987 : if (best_delta)
7199 2483 : iv_ca_delta_free (&best_delta);
7200 69987 : best_delta = act_delta;
7201 : }
7202 : else
7203 46485 : iv_ca_delta_free (&act_delta);
7204 : }
7205 : }
7206 :
7207 3334060 : iv_ca_delta_commit (data, ivs, best_delta, true);
7208 3334060 : iv_ca_delta_free (&best_delta);
7209 :
7210 3334060 : return !best_cost.infinite_cost_p ();
7211 : }
7212 :
7213 : /* Finds an initial assignment of candidates to uses. */
7214 :
7215 : static class iv_ca *
7216 1026362 : get_initial_solution (struct ivopts_data *data, bool originalp)
7217 : {
7218 1026362 : unsigned i;
7219 1026362 : class iv_ca *ivs = iv_ca_new (data);
7220 :
7221 4359522 : for (i = 0; i < data->vgroups.length (); i++)
7222 3334060 : if (!try_add_cand_for (data, ivs, data->vgroups[i], originalp))
7223 : {
7224 900 : iv_ca_free (&ivs);
7225 900 : return NULL;
7226 : }
7227 :
7228 : return ivs;
7229 : }
7230 :
7231 : /* Tries to improve set of induction variables IVS. TRY_REPLACE_P
7232 : points to a bool variable, this function tries to break local
7233 : optimal fixed-point by replacing candidates in IVS if it's true. */
7234 :
7235 : static bool
7236 1556173 : try_improve_iv_set (struct ivopts_data *data,
7237 : class iv_ca *ivs, bool *try_replace_p)
7238 : {
7239 1556173 : unsigned i, n_ivs;
7240 1556173 : comp_cost acost, best_cost = iv_ca_cost (ivs);
7241 1556173 : struct iv_ca_delta *best_delta = NULL, *act_delta, *tmp_delta;
7242 1556173 : struct iv_cand *cand;
7243 :
7244 : /* Try extending the set of induction variables by one. */
7245 17157350 : for (i = 0; i < data->vcands.length (); i++)
7246 : {
7247 15601177 : cand = data->vcands[i];
7248 :
7249 15601177 : if (iv_ca_cand_used_p (ivs, cand))
7250 2430800 : continue;
7251 :
7252 13170377 : acost = iv_ca_extend (data, ivs, cand, &act_delta, &n_ivs, false);
7253 13170377 : if (!act_delta)
7254 7915585 : continue;
7255 :
7256 : /* If we successfully added the candidate and the set is small enough,
7257 : try optimizing it by removing other candidates. */
7258 5254792 : if (n_ivs <= ALWAYS_PRUNE_CAND_SET_BOUND)
7259 : {
7260 5187523 : iv_ca_delta_commit (data, ivs, act_delta, true);
7261 5187523 : acost = iv_ca_prune (data, ivs, cand, &tmp_delta);
7262 5187523 : iv_ca_delta_commit (data, ivs, act_delta, false);
7263 5187523 : act_delta = iv_ca_delta_join (act_delta, tmp_delta);
7264 : }
7265 :
7266 5254792 : if (acost < best_cost)
7267 : {
7268 638912 : best_cost = acost;
7269 638912 : iv_ca_delta_free (&best_delta);
7270 638912 : best_delta = act_delta;
7271 : }
7272 : else
7273 4615880 : iv_ca_delta_free (&act_delta);
7274 : }
7275 :
7276 1556173 : if (!best_delta)
7277 : {
7278 : /* Try removing the candidates from the set instead. */
7279 1061051 : best_cost = iv_ca_prune (data, ivs, NULL, &best_delta);
7280 :
7281 1061051 : if (!best_delta && *try_replace_p)
7282 : {
7283 1025462 : *try_replace_p = false;
7284 : /* So far candidate selecting algorithm tends to choose fewer IVs
7285 : so that it can handle cases in which loops have many variables
7286 : but the best choice is often to use only one general biv. One
7287 : weakness is it can't handle opposite cases, in which different
7288 : candidates should be chosen with respect to each use. To solve
7289 : the problem, we replace candidates in a manner described by the
7290 : comments of iv_ca_replace, thus give general algorithm a chance
7291 : to break local optimal fixed-point in these cases. */
7292 1025462 : best_cost = iv_ca_replace (data, ivs, &best_delta);
7293 : }
7294 :
7295 1061051 : if (!best_delta)
7296 : return false;
7297 : }
7298 :
7299 530711 : iv_ca_delta_commit (data, ivs, best_delta, true);
7300 530711 : iv_ca_delta_free (&best_delta);
7301 1061422 : return best_cost == iv_ca_cost (ivs);
7302 : }
7303 :
7304 : /* Attempts to find the optimal set of induction variables. We do simple
7305 : greedy heuristic -- we try to replace at most one candidate in the selected
7306 : solution and remove the unused ivs while this improves the cost. */
7307 :
7308 : static class iv_ca *
7309 1026362 : find_optimal_iv_set_1 (struct ivopts_data *data, bool originalp)
7310 : {
7311 1026362 : class iv_ca *set;
7312 1026362 : bool try_replace_p = true;
7313 :
7314 : /* Get the initial solution. */
7315 1026362 : set = get_initial_solution (data, originalp);
7316 1026362 : if (!set)
7317 : {
7318 900 : if (dump_file && (dump_flags & TDF_DETAILS))
7319 0 : fprintf (dump_file, "Unable to substitute for ivs, failed.\n");
7320 : return NULL;
7321 : }
7322 :
7323 1025462 : if (dump_file && (dump_flags & TDF_DETAILS))
7324 : {
7325 144 : fprintf (dump_file, "Initial set of candidates:\n");
7326 144 : iv_ca_dump (data, dump_file, set);
7327 : }
7328 :
7329 1556173 : while (try_improve_iv_set (data, set, &try_replace_p))
7330 : {
7331 530711 : if (dump_file && (dump_flags & TDF_DETAILS))
7332 : {
7333 126 : fprintf (dump_file, "Improved to:\n");
7334 126 : iv_ca_dump (data, dump_file, set);
7335 : }
7336 : }
7337 :
7338 : /* If the set has infinite_cost, it can't be optimal. */
7339 2050924 : if (iv_ca_cost (set).infinite_cost_p ())
7340 : {
7341 0 : if (dump_file && (dump_flags & TDF_DETAILS))
7342 0 : fprintf (dump_file,
7343 : "Overflow to infinite cost in try_improve_iv_set.\n");
7344 0 : iv_ca_free (&set);
7345 : }
7346 1025462 : return set;
7347 : }
7348 :
7349 : static class iv_ca *
7350 513181 : find_optimal_iv_set (struct ivopts_data *data)
7351 : {
7352 513181 : unsigned i;
7353 513181 : comp_cost cost, origcost;
7354 513181 : class iv_ca *set, *origset;
7355 :
7356 : /* Determine the cost based on a strategy that starts with original IVs,
7357 : and try again using a strategy that prefers candidates not based
7358 : on any IVs. */
7359 513181 : origset = find_optimal_iv_set_1 (data, true);
7360 513181 : set = find_optimal_iv_set_1 (data, false);
7361 :
7362 513181 : if (!origset && !set)
7363 : return NULL;
7364 :
7365 512731 : origcost = origset ? iv_ca_cost (origset) : infinite_cost;
7366 512731 : cost = set ? iv_ca_cost (set) : infinite_cost;
7367 :
7368 512731 : if (dump_file && (dump_flags & TDF_DETAILS))
7369 : {
7370 72 : fprintf (dump_file, "Original cost %" PRId64 " (complexity %d)\n\n",
7371 : origcost.cost, origcost.complexity);
7372 72 : fprintf (dump_file, "Final cost %" PRId64 " (complexity %d)\n\n",
7373 : cost.cost, cost.complexity);
7374 : }
7375 :
7376 : /* Choose the one with the best cost. */
7377 512731 : if (origcost <= cost)
7378 : {
7379 478852 : if (set)
7380 478852 : iv_ca_free (&set);
7381 478852 : set = origset;
7382 : }
7383 33879 : else if (origset)
7384 33879 : iv_ca_free (&origset);
7385 :
7386 2178654 : for (i = 0; i < data->vgroups.length (); i++)
7387 : {
7388 1665923 : struct iv_group *group = data->vgroups[i];
7389 1665923 : group->selected = iv_ca_cand_for_group (set, group)->cand;
7390 : }
7391 :
7392 512731 : return set;
7393 : }
7394 :
7395 : /* Creates a new induction variable corresponding to CAND. */
7396 :
7397 : static void
7398 711993 : create_new_iv (struct ivopts_data *data, struct iv_cand *cand)
7399 : {
7400 711993 : gimple_stmt_iterator incr_pos;
7401 711993 : tree base;
7402 711993 : struct iv_use *use;
7403 711993 : struct iv_group *group;
7404 711993 : bool after = false;
7405 :
7406 711993 : gcc_assert (cand->iv != NULL);
7407 :
7408 711993 : switch (cand->pos)
7409 : {
7410 495824 : case IP_NORMAL:
7411 495824 : incr_pos = gsi_last_bb (ip_normal_pos (data->current_loop));
7412 495824 : break;
7413 :
7414 11656 : case IP_END:
7415 11656 : incr_pos = gsi_last_bb (ip_end_pos (data->current_loop));
7416 11656 : after = true;
7417 11656 : gcc_assert (gsi_end_p (incr_pos) || !stmt_ends_bb_p (*incr_pos));
7418 : break;
7419 :
7420 0 : case IP_AFTER_USE:
7421 0 : after = true;
7422 : /* fall through */
7423 0 : case IP_BEFORE_USE:
7424 0 : incr_pos = gsi_for_stmt (cand->incremented_at);
7425 0 : break;
7426 :
7427 204513 : case IP_ORIGINAL:
7428 : /* Mark that the iv is preserved. */
7429 204513 : name_info (data, cand->var_before)->preserve_biv = true;
7430 204513 : name_info (data, cand->var_after)->preserve_biv = true;
7431 :
7432 : /* Rewrite the increment so that it uses var_before directly. Missed
7433 : optimization can result in a use IV with zero step, avoid
7434 : crashing in that case. */
7435 204513 : use = find_interesting_uses_op (data, cand->var_after);
7436 204513 : if (use)
7437 : {
7438 204508 : group = data->vgroups[use->group_id];
7439 204508 : group->selected = cand;
7440 : }
7441 204513 : return;
7442 : }
7443 :
7444 507480 : gimple_add_tmp_var (cand->var_before);
7445 :
7446 507480 : base = unshare_expr (cand->iv->base);
7447 :
7448 : /* The step computation could invoke UB when the loop does not iterate.
7449 : Avoid inserting it on the preheader in its native form but rewrite
7450 : it to a well-defined form. This also helps masking SCEV issues
7451 : which freely re-associates the IV computations when building up
7452 : CHRECs without much regard for signed overflow invoking UB. */
7453 507480 : gimple_seq stmts = NULL;
7454 507480 : tree step = force_gimple_operand (unshare_expr (cand->iv->step), &stmts,
7455 : true, NULL_TREE);
7456 507480 : if (stmts)
7457 : {
7458 160544 : for (auto gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
7459 105878 : if (gimple_needing_rewrite_undefined (gsi_stmt (gsi)))
7460 12281 : rewrite_to_defined_unconditional (&gsi);
7461 54666 : gsi_insert_seq_on_edge_immediate
7462 54666 : (loop_preheader_edge (data->current_loop), stmts);
7463 : }
7464 :
7465 507480 : create_iv (base, PLUS_EXPR, step,
7466 : cand->var_before, data->current_loop,
7467 : &incr_pos, after, &cand->var_before, &cand->var_after);
7468 : }
7469 :
7470 : /* Creates new induction variables described in SET. */
7471 :
7472 : static void
7473 512731 : create_new_ivs (struct ivopts_data *data, class iv_ca *set)
7474 : {
7475 512731 : unsigned i;
7476 512731 : struct iv_cand *cand;
7477 512731 : bitmap_iterator bi;
7478 :
7479 1224724 : EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
7480 : {
7481 711993 : cand = data->vcands[i];
7482 711993 : create_new_iv (data, cand);
7483 : }
7484 :
7485 512731 : if (dump_file && (dump_flags & TDF_DETAILS))
7486 : {
7487 72 : fprintf (dump_file, "Selected IV set for loop %d",
7488 72 : data->current_loop->num);
7489 72 : if (data->loop_loc != UNKNOWN_LOCATION)
7490 70 : fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
7491 140 : LOCATION_LINE (data->loop_loc));
7492 72 : fprintf (dump_file, ", " HOST_WIDE_INT_PRINT_UNSIGNED " avg niters",
7493 : avg_loop_niter (data->current_loop));
7494 72 : fprintf (dump_file, ", %lu IVs:\n", bitmap_count_bits (set->cands));
7495 192 : EXECUTE_IF_SET_IN_BITMAP (set->cands, 0, i, bi)
7496 : {
7497 120 : cand = data->vcands[i];
7498 120 : dump_cand (dump_file, cand);
7499 : }
7500 72 : fprintf (dump_file, "\n");
7501 : }
7502 512731 : }
7503 :
7504 : /* Rewrites USE (definition of iv used in a nonlinear expression)
7505 : using candidate CAND. */
7506 :
7507 : static void
7508 629977 : rewrite_use_nonlinear_expr (struct ivopts_data *data,
7509 : struct iv_use *use, struct iv_cand *cand)
7510 : {
7511 629977 : gassign *ass;
7512 629977 : gimple_stmt_iterator bsi;
7513 629977 : tree comp, type = get_use_type (use), tgt;
7514 :
7515 : /* An important special case -- if we are asked to express value of
7516 : the original iv by itself, just exit; there is no need to
7517 : introduce a new computation (that might also need casting the
7518 : variable to unsigned and back). */
7519 629977 : if (cand->pos == IP_ORIGINAL
7520 334287 : && cand->incremented_at == use->stmt)
7521 : {
7522 204508 : tree op = NULL_TREE;
7523 204508 : enum tree_code stmt_code;
7524 :
7525 204508 : gcc_assert (is_gimple_assign (use->stmt));
7526 204508 : gcc_assert (gimple_assign_lhs (use->stmt) == cand->var_after);
7527 :
7528 : /* Check whether we may leave the computation unchanged.
7529 : This is the case only if it does not rely on other
7530 : computations in the loop -- otherwise, the computation
7531 : we rely upon may be removed in remove_unused_ivs,
7532 : thus leading to ICE. */
7533 204508 : stmt_code = gimple_assign_rhs_code (use->stmt);
7534 204508 : if (stmt_code == PLUS_EXPR
7535 204508 : || stmt_code == MINUS_EXPR
7536 204508 : || stmt_code == POINTER_PLUS_EXPR)
7537 : {
7538 201945 : if (gimple_assign_rhs1 (use->stmt) == cand->var_before)
7539 201125 : op = gimple_assign_rhs2 (use->stmt);
7540 820 : else if (gimple_assign_rhs2 (use->stmt) == cand->var_before)
7541 : op = gimple_assign_rhs1 (use->stmt);
7542 : }
7543 :
7544 201638 : if (op != NULL_TREE)
7545 : {
7546 201638 : if (expr_invariant_in_loop_p (data->current_loop, op))
7547 288150 : return;
7548 165 : if (TREE_CODE (op) == SSA_NAME)
7549 : {
7550 165 : struct iv *iv = get_iv (data, op);
7551 165 : if (iv != NULL && integer_zerop (iv->step))
7552 : return;
7553 : }
7554 : }
7555 : }
7556 :
7557 428339 : switch (gimple_code (use->stmt))
7558 : {
7559 128364 : case GIMPLE_PHI:
7560 128364 : tgt = PHI_RESULT (use->stmt);
7561 :
7562 : /* If we should keep the biv, do not replace it. */
7563 128364 : if (name_info (data, tgt)->preserve_biv)
7564 : return;
7565 :
7566 41852 : bsi = gsi_after_labels (gimple_bb (use->stmt));
7567 41852 : break;
7568 :
7569 299975 : case GIMPLE_ASSIGN:
7570 299975 : tgt = gimple_assign_lhs (use->stmt);
7571 299975 : bsi = gsi_for_stmt (use->stmt);
7572 299975 : break;
7573 :
7574 0 : default:
7575 0 : gcc_unreachable ();
7576 : }
7577 :
7578 1025481 : aff_tree aff_inv, aff_var;
7579 341827 : if (!get_computation_aff_1 (data, use->stmt, use, cand, &aff_inv, &aff_var))
7580 0 : gcc_unreachable ();
7581 :
7582 341827 : unshare_aff_combination (&aff_inv);
7583 341827 : unshare_aff_combination (&aff_var);
7584 : /* Prefer CSE opportunity than loop invariant by adding offset at last
7585 : so that iv_uses have different offsets can be CSEed. */
7586 683654 : poly_widest_int offset = aff_inv.offset;
7587 341827 : aff_inv.offset = 0;
7588 :
7589 341827 : gimple_seq stmt_list = NULL, seq = NULL;
7590 341827 : tree comp_op1 = aff_combination_to_tree (&aff_inv);
7591 341827 : tree comp_op2 = aff_combination_to_tree (&aff_var);
7592 341827 : gcc_assert (comp_op1 && comp_op2);
7593 :
7594 341827 : comp_op1 = force_gimple_operand (comp_op1, &seq, true, NULL);
7595 341827 : gimple_seq_add_seq (&stmt_list, seq);
7596 341827 : comp_op2 = force_gimple_operand (comp_op2, &seq, true, NULL);
7597 341827 : gimple_seq_add_seq (&stmt_list, seq);
7598 :
7599 341827 : if (POINTER_TYPE_P (TREE_TYPE (comp_op2)))
7600 : std::swap (comp_op1, comp_op2);
7601 :
7602 341827 : if (POINTER_TYPE_P (TREE_TYPE (comp_op1)))
7603 : {
7604 0 : comp = fold_build_pointer_plus (comp_op1,
7605 : fold_convert (sizetype, comp_op2));
7606 0 : comp = fold_build_pointer_plus (comp,
7607 : wide_int_to_tree (sizetype, offset));
7608 : }
7609 : else
7610 : {
7611 341827 : comp = fold_build2 (PLUS_EXPR, TREE_TYPE (comp_op1), comp_op1,
7612 : fold_convert (TREE_TYPE (comp_op1), comp_op2));
7613 341827 : comp = fold_build2 (PLUS_EXPR, TREE_TYPE (comp_op1), comp,
7614 : wide_int_to_tree (TREE_TYPE (comp_op1), offset));
7615 : }
7616 :
7617 341827 : comp = fold_convert (type, comp);
7618 341827 : comp = force_gimple_operand (comp, &seq, false, NULL);
7619 341827 : gimple_seq_add_seq (&stmt_list, seq);
7620 341827 : if (gimple_code (use->stmt) != GIMPLE_PHI
7621 : /* We can't allow re-allocating the stmt as it might be pointed
7622 : to still. */
7623 341827 : && (get_gimple_rhs_num_ops (TREE_CODE (comp))
7624 299975 : >= gimple_num_ops (gsi_stmt (bsi))))
7625 : {
7626 5800 : comp = force_gimple_operand (comp, &seq, true, NULL);
7627 5800 : gimple_seq_add_seq (&stmt_list, seq);
7628 5800 : if (POINTER_TYPE_P (TREE_TYPE (tgt)))
7629 0 : duplicate_ssa_name_pt_solution (comp, tgt);
7630 : }
7631 :
7632 341827 : gsi_insert_seq_before (&bsi, stmt_list, GSI_SAME_STMT);
7633 341827 : if (gimple_code (use->stmt) == GIMPLE_PHI)
7634 : {
7635 41852 : ass = gimple_build_assign (tgt, comp);
7636 41852 : gsi_insert_before (&bsi, ass, GSI_SAME_STMT);
7637 :
7638 41852 : bsi = gsi_for_stmt (use->stmt);
7639 41852 : remove_phi_node (&bsi, false);
7640 : }
7641 : else
7642 : {
7643 299975 : gimple_assign_set_rhs_from_tree (&bsi, comp);
7644 299975 : use->stmt = gsi_stmt (bsi);
7645 : }
7646 : }
7647 :
7648 : /* Performs a peephole optimization to reorder the iv update statement with
7649 : a mem ref to enable instruction combining in later phases. The mem ref uses
7650 : the iv value before the update, so the reordering transformation requires
7651 : adjustment of the offset. CAND is the selected IV_CAND.
7652 :
7653 : Example:
7654 :
7655 : t = MEM_REF (base, iv1, 8, 16); // base, index, stride, offset
7656 : iv2 = iv1 + 1;
7657 :
7658 : if (t < val) (1)
7659 : goto L;
7660 : goto Head;
7661 :
7662 :
7663 : directly propagating t over to (1) will introduce overlapping live range
7664 : thus increase register pressure. This peephole transform it into:
7665 :
7666 :
7667 : iv2 = iv1 + 1;
7668 : t = MEM_REF (base, iv2, 8, 8);
7669 : if (t < val)
7670 : goto L;
7671 : goto Head;
7672 : */
7673 :
7674 : static void
7675 865748 : adjust_iv_update_pos (struct iv_cand *cand, struct iv_use *use)
7676 : {
7677 865748 : tree var_after;
7678 865748 : gimple *iv_update, *stmt;
7679 865748 : basic_block bb;
7680 865748 : gimple_stmt_iterator gsi, gsi_iv;
7681 :
7682 865748 : if (cand->pos != IP_NORMAL)
7683 863574 : return;
7684 :
7685 669608 : var_after = cand->var_after;
7686 669608 : iv_update = SSA_NAME_DEF_STMT (var_after);
7687 :
7688 669608 : bb = gimple_bb (iv_update);
7689 669608 : gsi = gsi_last_nondebug_bb (bb);
7690 669608 : stmt = gsi_stmt (gsi);
7691 :
7692 : /* Only handle conditional statement for now. */
7693 669608 : if (gimple_code (stmt) != GIMPLE_COND)
7694 : return;
7695 :
7696 669608 : gsi_prev_nondebug (&gsi);
7697 669608 : stmt = gsi_stmt (gsi);
7698 669608 : if (stmt != iv_update)
7699 : return;
7700 :
7701 532264 : gsi_prev_nondebug (&gsi);
7702 532264 : if (gsi_end_p (gsi))
7703 : return;
7704 :
7705 529444 : stmt = gsi_stmt (gsi);
7706 529444 : if (gimple_code (stmt) != GIMPLE_ASSIGN)
7707 : return;
7708 :
7709 529310 : if (stmt != use->stmt)
7710 : return;
7711 :
7712 4237 : if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME)
7713 : return;
7714 :
7715 2174 : if (dump_file && (dump_flags & TDF_DETAILS))
7716 : {
7717 0 : fprintf (dump_file, "Reordering \n");
7718 0 : print_gimple_stmt (dump_file, iv_update, 0);
7719 0 : print_gimple_stmt (dump_file, use->stmt, 0);
7720 0 : fprintf (dump_file, "\n");
7721 : }
7722 :
7723 2174 : gsi = gsi_for_stmt (use->stmt);
7724 2174 : gsi_iv = gsi_for_stmt (iv_update);
7725 2174 : gsi_move_before (&gsi_iv, &gsi);
7726 :
7727 2174 : cand->pos = IP_BEFORE_USE;
7728 2174 : cand->incremented_at = use->stmt;
7729 : }
7730 :
7731 : /* Return the alias pointer type that should be used for a MEM_REF
7732 : associated with USE, which has type USE_PTR_ADDRESS. */
7733 :
7734 : static tree
7735 662 : get_alias_ptr_type_for_ptr_address (iv_use *use)
7736 : {
7737 662 : gcall *call = as_a <gcall *> (use->stmt);
7738 662 : switch (gimple_call_internal_fn (call))
7739 : {
7740 662 : case IFN_MASK_LOAD:
7741 662 : case IFN_MASK_STORE:
7742 662 : case IFN_MASK_LOAD_LANES:
7743 662 : case IFN_MASK_STORE_LANES:
7744 662 : case IFN_MASK_LEN_LOAD_LANES:
7745 662 : case IFN_MASK_LEN_STORE_LANES:
7746 662 : case IFN_LEN_LOAD:
7747 662 : case IFN_LEN_STORE:
7748 662 : case IFN_MASK_LEN_LOAD:
7749 662 : case IFN_MASK_LEN_STORE:
7750 : /* The second argument contains the correct alias type. */
7751 662 : gcc_assert (use->op_p == gimple_call_arg_ptr (call, 0));
7752 662 : return TREE_TYPE (gimple_call_arg (call, 1));
7753 :
7754 0 : default:
7755 0 : gcc_unreachable ();
7756 : }
7757 : }
7758 :
7759 :
7760 : /* Rewrites USE (address that is an iv) using candidate CAND. */
7761 :
7762 : static void
7763 865748 : rewrite_use_address (struct ivopts_data *data,
7764 : struct iv_use *use, struct iv_cand *cand)
7765 : {
7766 865748 : aff_tree aff;
7767 865748 : bool ok;
7768 :
7769 865748 : adjust_iv_update_pos (cand, use);
7770 865748 : ok = get_computation_aff (data, use->stmt, use, cand, &aff);
7771 865748 : gcc_assert (ok);
7772 865748 : unshare_aff_combination (&aff);
7773 :
7774 : /* To avoid undefined overflow problems, all IV candidates use unsigned
7775 : integer types. The drawback is that this makes it impossible for
7776 : create_mem_ref to distinguish an IV that is based on a memory object
7777 : from one that represents simply an offset.
7778 :
7779 : To work around this problem, we pass a hint to create_mem_ref that
7780 : indicates which variable (if any) in aff is an IV based on a memory
7781 : object. Note that we only consider the candidate. If this is not
7782 : based on an object, the base of the reference is in some subexpression
7783 : of the use -- but these will use pointer types, so they are recognized
7784 : by the create_mem_ref heuristics anyway. */
7785 865748 : tree iv = var_at_stmt (data->current_loop, cand, use->stmt);
7786 865748 : tree base_hint = (cand->iv->base_object) ? iv : NULL_TREE;
7787 865748 : gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
7788 865748 : tree type = use->mem_type;
7789 865748 : tree alias_ptr_type;
7790 865748 : if (use->type == USE_PTR_ADDRESS)
7791 662 : alias_ptr_type = get_alias_ptr_type_for_ptr_address (use);
7792 : else
7793 : {
7794 865086 : gcc_assert (type == TREE_TYPE (*use->op_p));
7795 865086 : unsigned int align = get_object_alignment (*use->op_p);
7796 865086 : if (align != TYPE_ALIGN (type))
7797 34831 : type = build_aligned_type (type, align);
7798 865086 : alias_ptr_type = reference_alias_ptr_type (*use->op_p);
7799 : }
7800 865748 : tree ref = create_mem_ref (&bsi, type, &aff, alias_ptr_type,
7801 : iv, base_hint, data->speed);
7802 :
7803 865748 : if (use->type == USE_PTR_ADDRESS)
7804 : {
7805 662 : ref = fold_build1 (ADDR_EXPR, build_pointer_type (use->mem_type), ref);
7806 662 : ref = fold_convert (get_use_type (use), ref);
7807 662 : ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
7808 : true, GSI_SAME_STMT);
7809 : }
7810 : else
7811 : {
7812 : /* When we end up confused enough and have no suitable base but
7813 : stuffed everything to indexes use a LEA for the address and
7814 : create a plain MEM_REF to avoid basing a memory reference
7815 : on address zero which create_mem_ref_raw does as fallback. */
7816 865086 : if (TREE_CODE (ref) == TARGET_MEM_REF
7817 865086 : && integer_zerop (TREE_OPERAND (ref, 0)))
7818 : {
7819 24 : ref = fold_build1 (ADDR_EXPR, TREE_TYPE (TREE_OPERAND (ref, 0)), ref);
7820 24 : ref = force_gimple_operand_gsi (&bsi, ref, true, NULL_TREE,
7821 : true, GSI_SAME_STMT);
7822 24 : ref = build2 (MEM_REF, type, ref, build_zero_cst (alias_ptr_type));
7823 : }
7824 865086 : copy_ref_info (ref, *use->op_p);
7825 : }
7826 :
7827 865748 : *use->op_p = ref;
7828 865748 : }
7829 :
7830 : /* Rewrites USE (the condition such that one of the arguments is an iv) using
7831 : candidate CAND. */
7832 :
7833 : static void
7834 605699 : rewrite_use_compare (struct ivopts_data *data,
7835 : struct iv_use *use, struct iv_cand *cand)
7836 : {
7837 605699 : tree comp, op, bound;
7838 605699 : gimple_stmt_iterator bsi = gsi_for_stmt (use->stmt);
7839 605699 : enum tree_code compare;
7840 605699 : struct iv_group *group = data->vgroups[use->group_id];
7841 605699 : class cost_pair *cp = get_group_iv_cost (data, group, cand);
7842 :
7843 605699 : bound = cp->value;
7844 605699 : if (bound)
7845 : {
7846 407988 : tree var = var_at_stmt (data->current_loop, cand, use->stmt);
7847 407988 : tree var_type = TREE_TYPE (var);
7848 407988 : gimple_seq stmts;
7849 :
7850 407988 : if (dump_file && (dump_flags & TDF_DETAILS))
7851 : {
7852 60 : fprintf (dump_file, "Replacing exit test: ");
7853 60 : print_gimple_stmt (dump_file, use->stmt, 0, TDF_SLIM);
7854 : }
7855 407988 : compare = cp->comp;
7856 407988 : bound = unshare_expr (fold_convert (var_type, bound));
7857 407988 : op = force_gimple_operand (bound, &stmts, true, NULL_TREE);
7858 407988 : if (stmts)
7859 192577 : gsi_insert_seq_on_edge_immediate (
7860 192577 : loop_preheader_edge (data->current_loop),
7861 : stmts);
7862 :
7863 407988 : gcond *cond_stmt = as_a <gcond *> (use->stmt);
7864 407988 : gimple_cond_set_lhs (cond_stmt, var);
7865 407988 : gimple_cond_set_code (cond_stmt, compare);
7866 407988 : gimple_cond_set_rhs (cond_stmt, op);
7867 407988 : return;
7868 : }
7869 :
7870 : /* The induction variable elimination failed; just express the original
7871 : giv. */
7872 197711 : comp = get_computation_at (data, use->stmt, use, cand);
7873 197711 : gcc_assert (comp != NULL_TREE);
7874 197711 : gcc_assert (use->op_p != NULL);
7875 197711 : *use->op_p = force_gimple_operand_gsi (&bsi, comp, true,
7876 197711 : SSA_NAME_VAR (*use->op_p),
7877 : true, GSI_SAME_STMT);
7878 : }
7879 :
7880 : /* Rewrite the groups using the selected induction variables. */
7881 :
7882 : static void
7883 512731 : rewrite_groups (struct ivopts_data *data)
7884 : {
7885 512731 : unsigned i, j;
7886 :
7887 2339636 : for (i = 0; i < data->vgroups.length (); i++)
7888 : {
7889 1826905 : struct iv_group *group = data->vgroups[i];
7890 1826905 : struct iv_cand *cand = group->selected;
7891 :
7892 1826905 : gcc_assert (cand);
7893 :
7894 1826905 : if (group->type == USE_NONLINEAR_EXPR)
7895 : {
7896 1259954 : for (j = 0; j < group->vuses.length (); j++)
7897 : {
7898 629977 : rewrite_use_nonlinear_expr (data, group->vuses[j], cand);
7899 629977 : update_stmt (group->vuses[j]->stmt);
7900 : }
7901 : }
7902 1196928 : else if (address_p (group->type))
7903 : {
7904 1456977 : for (j = 0; j < group->vuses.length (); j++)
7905 : {
7906 865748 : rewrite_use_address (data, group->vuses[j], cand);
7907 865748 : update_stmt (group->vuses[j]->stmt);
7908 : }
7909 : }
7910 : else
7911 : {
7912 605699 : gcc_assert (group->type == USE_COMPARE);
7913 :
7914 2432604 : for (j = 0; j < group->vuses.length (); j++)
7915 : {
7916 605699 : rewrite_use_compare (data, group->vuses[j], cand);
7917 605699 : update_stmt (group->vuses[j]->stmt);
7918 : }
7919 : }
7920 : }
7921 512731 : }
7922 :
7923 : /* Removes the ivs that are not used after rewriting. */
7924 :
7925 : static void
7926 512731 : remove_unused_ivs (struct ivopts_data *data, bitmap toremove)
7927 : {
7928 512731 : unsigned j;
7929 512731 : bitmap_iterator bi;
7930 :
7931 : /* Figure out an order in which to release SSA DEFs so that we don't
7932 : release something that we'd have to propagate into a debug stmt
7933 : afterwards. */
7934 5659082 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, j, bi)
7935 : {
7936 5146351 : struct version_info *info;
7937 :
7938 5146351 : info = ver_info (data, j);
7939 5146351 : if (info->iv
7940 5003278 : && !integer_zerop (info->iv->step)
7941 3267161 : && !info->inv_id
7942 3267161 : && !info->iv->nonlin_use
7943 7783535 : && !info->preserve_biv)
7944 : {
7945 2519183 : bitmap_set_bit (toremove, SSA_NAME_VERSION (info->iv->ssa_name));
7946 :
7947 2519183 : tree def = info->iv->ssa_name;
7948 :
7949 3278290 : if (MAY_HAVE_DEBUG_BIND_STMTS && SSA_NAME_DEF_STMT (def))
7950 : {
7951 759107 : imm_use_iterator imm_iter;
7952 759107 : use_operand_p use_p;
7953 759107 : gimple *stmt;
7954 759107 : int count = 0;
7955 :
7956 1498805 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, def)
7957 : {
7958 769169 : if (!gimple_debug_bind_p (stmt))
7959 645481 : continue;
7960 :
7961 : /* We just want to determine whether to do nothing
7962 : (count == 0), to substitute the computed
7963 : expression into a single use of the SSA DEF by
7964 : itself (count == 1), or to use a debug temp
7965 : because the SSA DEF is used multiple times or as
7966 : part of a larger expression (count > 1). */
7967 123688 : count++;
7968 123688 : if (gimple_debug_bind_get_value (stmt) != def)
7969 8071 : count++;
7970 :
7971 123688 : if (count > 1)
7972 : break;
7973 759107 : }
7974 :
7975 759107 : if (!count)
7976 684268 : continue;
7977 :
7978 102059 : struct iv_use dummy_use;
7979 102059 : struct iv_cand *best_cand = NULL, *cand;
7980 102059 : unsigned i, best_pref = 0, cand_pref;
7981 102059 : tree comp = NULL_TREE;
7982 :
7983 102059 : memset (&dummy_use, 0, sizeof (dummy_use));
7984 102059 : dummy_use.iv = info->iv;
7985 517391 : for (i = 0; i < data->vgroups.length () && i < 64; i++)
7986 : {
7987 415332 : cand = data->vgroups[i]->selected;
7988 415332 : if (cand == best_cand)
7989 156605 : continue;
7990 180600 : cand_pref = operand_equal_p (cand->iv->step,
7991 258727 : info->iv->step, 0)
7992 258727 : ? 4 : 0;
7993 258727 : cand_pref
7994 258727 : += TYPE_MODE (TREE_TYPE (cand->iv->base))
7995 258727 : == TYPE_MODE (TREE_TYPE (info->iv->base))
7996 258727 : ? 2 : 0;
7997 258727 : cand_pref
7998 517454 : += TREE_CODE (cand->iv->base) == INTEGER_CST
7999 258727 : ? 1 : 0;
8000 258727 : if (best_cand == NULL || best_pref < cand_pref)
8001 : {
8002 205096 : tree this_comp
8003 410192 : = get_debug_computation_at (data,
8004 205096 : SSA_NAME_DEF_STMT (def),
8005 : &dummy_use, cand);
8006 205096 : if (this_comp)
8007 : {
8008 415332 : best_cand = cand;
8009 415332 : best_pref = cand_pref;
8010 415332 : comp = this_comp;
8011 : }
8012 : }
8013 : }
8014 :
8015 102059 : if (!best_cand)
8016 27220 : continue;
8017 :
8018 74839 : comp = unshare_expr (comp);
8019 74839 : if (count > 1)
8020 : {
8021 24306 : tree vexpr = build_debug_expr_decl (TREE_TYPE (comp));
8022 : /* FIXME: Is setting the mode really necessary? */
8023 24306 : if (SSA_NAME_VAR (def))
8024 13735 : SET_DECL_MODE (vexpr, DECL_MODE (SSA_NAME_VAR (def)));
8025 : else
8026 10571 : SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (vexpr)));
8027 24306 : gdebug *def_temp
8028 24306 : = gimple_build_debug_bind (vexpr, comp, NULL);
8029 24306 : gimple_stmt_iterator gsi;
8030 :
8031 24306 : if (gimple_code (SSA_NAME_DEF_STMT (def)) == GIMPLE_PHI)
8032 13385 : gsi = gsi_after_labels (gimple_bb
8033 13385 : (SSA_NAME_DEF_STMT (def)));
8034 : else
8035 10921 : gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (def));
8036 :
8037 24306 : gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
8038 24306 : comp = vexpr;
8039 : }
8040 :
8041 283027 : FOR_EACH_IMM_USE_STMT (stmt, imm_iter, def)
8042 : {
8043 208188 : if (!gimple_debug_bind_p (stmt))
8044 83943 : continue;
8045 :
8046 248542 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
8047 124271 : SET_USE (use_p, comp);
8048 :
8049 124245 : update_stmt (stmt);
8050 74839 : }
8051 : }
8052 : }
8053 : }
8054 512731 : }
8055 :
8056 : /* Frees memory occupied by class tree_niter_desc in *VALUE. Callback
8057 : for hash_map::traverse. */
8058 :
8059 : bool
8060 493545 : free_tree_niter_desc (edge const &, tree_niter_desc *const &value, void *)
8061 : {
8062 493545 : if (value)
8063 : {
8064 454618 : value->~tree_niter_desc ();
8065 454618 : free (value);
8066 : }
8067 493545 : return true;
8068 : }
8069 :
8070 : /* Frees data allocated by the optimization of a single loop. */
8071 :
8072 : static void
8073 892123 : free_loop_data (struct ivopts_data *data)
8074 : {
8075 892123 : unsigned i, j;
8076 892123 : bitmap_iterator bi;
8077 892123 : tree obj;
8078 :
8079 892123 : if (data->niters)
8080 : {
8081 974797 : data->niters->traverse<void *, free_tree_niter_desc> (NULL);
8082 962504 : delete data->niters;
8083 481252 : data->niters = NULL;
8084 : }
8085 :
8086 6052573 : EXECUTE_IF_SET_IN_BITMAP (data->relevant, 0, i, bi)
8087 : {
8088 5160450 : struct version_info *info;
8089 :
8090 5160450 : info = ver_info (data, i);
8091 5160450 : info->iv = NULL;
8092 5160450 : info->has_nonlin_use = false;
8093 5160450 : info->preserve_biv = false;
8094 5160450 : info->inv_id = 0;
8095 : }
8096 892123 : bitmap_clear (data->relevant);
8097 892123 : bitmap_clear (data->important_candidates);
8098 :
8099 2721108 : for (i = 0; i < data->vgroups.length (); i++)
8100 : {
8101 1828985 : struct iv_group *group = data->vgroups[i];
8102 :
8103 3932528 : for (j = 0; j < group->vuses.length (); j++)
8104 2103543 : free (group->vuses[j]);
8105 1828985 : group->vuses.release ();
8106 :
8107 1828985 : BITMAP_FREE (group->related_cands);
8108 19840331 : for (j = 0; j < group->n_map_members; j++)
8109 : {
8110 18011346 : if (group->cost_map[j].inv_vars)
8111 3693881 : BITMAP_FREE (group->cost_map[j].inv_vars);
8112 18011346 : if (group->cost_map[j].inv_exprs)
8113 2029170 : BITMAP_FREE (group->cost_map[j].inv_exprs);
8114 : }
8115 :
8116 1828985 : free (group->cost_map);
8117 1828985 : free (group);
8118 : }
8119 892123 : data->vgroups.truncate (0);
8120 :
8121 5584524 : for (i = 0; i < data->vcands.length (); i++)
8122 : {
8123 4692401 : struct iv_cand *cand = data->vcands[i];
8124 :
8125 4692401 : if (cand->inv_vars)
8126 75062 : BITMAP_FREE (cand->inv_vars);
8127 4692401 : if (cand->inv_exprs)
8128 94071 : BITMAP_FREE (cand->inv_exprs);
8129 4692401 : free (cand);
8130 : }
8131 892123 : data->vcands.truncate (0);
8132 :
8133 892123 : if (data->version_info_size < num_ssa_names)
8134 : {
8135 176 : data->version_info_size = 2 * num_ssa_names;
8136 176 : free (data->version_info);
8137 176 : data->version_info = XCNEWVEC (struct version_info, data->version_info_size);
8138 : }
8139 :
8140 892123 : data->max_inv_var_id = 0;
8141 892123 : data->max_inv_expr_id = 0;
8142 :
8143 892123 : data->inv_var_nregs.truncate (0);
8144 892123 : data->inv_expr_nregs.truncate (0);
8145 :
8146 892123 : FOR_EACH_VEC_ELT (decl_rtl_to_reset, i, obj)
8147 0 : SET_DECL_RTL (obj, NULL_RTX);
8148 :
8149 892123 : decl_rtl_to_reset.truncate (0);
8150 :
8151 892123 : data->inv_expr_tab->empty ();
8152 :
8153 892123 : data->iv_common_cand_tab->empty ();
8154 892123 : data->iv_common_cands.truncate (0);
8155 892123 : }
8156 :
8157 : /* Finalizes data structures used by the iv optimization pass. LOOPS is the
8158 : loop tree. */
8159 :
8160 : static void
8161 248445 : tree_ssa_iv_optimize_finalize (struct ivopts_data *data)
8162 : {
8163 248445 : free_loop_data (data);
8164 248445 : free (data->version_info);
8165 248445 : BITMAP_FREE (data->relevant);
8166 248445 : BITMAP_FREE (data->important_candidates);
8167 :
8168 248445 : decl_rtl_to_reset.release ();
8169 248445 : data->vgroups.release ();
8170 248445 : data->vcands.release ();
8171 248445 : data->inv_var_nregs.release ();
8172 248445 : data->inv_expr_nregs.release ();
8173 248445 : delete data->inv_expr_tab;
8174 248445 : data->inv_expr_tab = NULL;
8175 248445 : free_affine_expand_cache (&data->name_expansion_cache);
8176 248445 : if (data->base_object_map)
8177 167740 : delete data->base_object_map;
8178 248445 : delete data->iv_common_cand_tab;
8179 248445 : data->iv_common_cand_tab = NULL;
8180 248445 : data->iv_common_cands.release ();
8181 248445 : obstack_free (&data->iv_obstack, NULL);
8182 248445 : }
8183 :
8184 : /* Returns true if the loop body BODY includes any function calls. */
8185 :
8186 : static bool
8187 643678 : loop_body_includes_call (basic_block *body, unsigned num_nodes)
8188 : {
8189 643678 : gimple_stmt_iterator gsi;
8190 643678 : unsigned i;
8191 :
8192 2905784 : for (i = 0; i < num_nodes; i++)
8193 24751894 : for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi); gsi_next (&gsi))
8194 : {
8195 20014813 : gimple *stmt = gsi_stmt (gsi);
8196 20014813 : if (is_gimple_call (stmt)
8197 285341 : && !gimple_call_internal_p (stmt)
8198 20234146 : && !is_inexpensive_builtin (gimple_call_fndecl (stmt)))
8199 : return true;
8200 : }
8201 : return false;
8202 : }
8203 :
8204 : /* Determine cost scaling factor for basic blocks in loop. */
8205 : #define COST_SCALING_FACTOR_BOUND (20)
8206 :
8207 : static void
8208 513181 : determine_scaling_factor (struct ivopts_data *data, basic_block *body)
8209 : {
8210 513181 : int lfreq = data->current_loop->header->count.to_frequency (cfun);
8211 513181 : if (!data->speed || lfreq <= 0)
8212 : return;
8213 :
8214 : int max_freq = lfreq;
8215 2918658 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8216 : {
8217 2494383 : body[i]->aux = (void *)(intptr_t) 1;
8218 2494383 : if (max_freq < body[i]->count.to_frequency (cfun))
8219 106681 : max_freq = body[i]->count.to_frequency (cfun);
8220 : }
8221 424275 : if (max_freq > lfreq)
8222 : {
8223 67292 : int divisor, factor;
8224 : /* Check if scaling factor itself needs to be scaled by the bound. This
8225 : is to avoid overflow when scaling cost according to profile info. */
8226 67292 : if (max_freq / lfreq > COST_SCALING_FACTOR_BOUND)
8227 : {
8228 : divisor = max_freq;
8229 : factor = COST_SCALING_FACTOR_BOUND;
8230 : }
8231 : else
8232 : {
8233 51176 : divisor = lfreq;
8234 51176 : factor = 1;
8235 : }
8236 1015295 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8237 : {
8238 948003 : int bfreq = body[i]->count.to_frequency (cfun);
8239 948003 : if (bfreq <= lfreq)
8240 518865 : continue;
8241 :
8242 429138 : body[i]->aux = (void*)(intptr_t) (factor * bfreq / divisor);
8243 : }
8244 : }
8245 : }
8246 :
8247 : /* Find doloop comparison use and set its doloop_p on if found. */
8248 :
8249 : static bool
8250 0 : find_doloop_use (struct ivopts_data *data)
8251 : {
8252 0 : struct loop *loop = data->current_loop;
8253 :
8254 0 : for (unsigned i = 0; i < data->vgroups.length (); i++)
8255 : {
8256 0 : struct iv_group *group = data->vgroups[i];
8257 0 : if (group->type == USE_COMPARE)
8258 : {
8259 0 : gcc_assert (group->vuses.length () == 1);
8260 0 : struct iv_use *use = group->vuses[0];
8261 0 : gimple *stmt = use->stmt;
8262 0 : if (gimple_code (stmt) == GIMPLE_COND)
8263 : {
8264 0 : basic_block bb = gimple_bb (stmt);
8265 0 : edge true_edge, false_edge;
8266 0 : extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
8267 : /* This comparison is used for loop latch. Require latch is empty
8268 : for now. */
8269 0 : if ((loop->latch == true_edge->dest
8270 0 : || loop->latch == false_edge->dest)
8271 0 : && empty_block_p (loop->latch))
8272 : {
8273 0 : group->doloop_p = true;
8274 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8275 : {
8276 0 : fprintf (dump_file, "Doloop cmp iv use: ");
8277 0 : print_gimple_stmt (dump_file, stmt, TDF_DETAILS);
8278 : }
8279 0 : return true;
8280 : }
8281 : }
8282 : }
8283 : }
8284 :
8285 : return false;
8286 : }
8287 :
8288 : /* For the targets which support doloop, to predict whether later RTL doloop
8289 : transformation will perform on this loop, further detect the doloop use and
8290 : mark the flag doloop_use_p if predicted. */
8291 :
8292 : void
8293 513181 : analyze_and_mark_doloop_use (struct ivopts_data *data)
8294 : {
8295 513181 : data->doloop_use_p = false;
8296 :
8297 513181 : if (!flag_branch_on_count_reg)
8298 : return;
8299 :
8300 513181 : if (data->current_loop->unroll == USHRT_MAX)
8301 : return;
8302 :
8303 513181 : if (!generic_predict_doloop_p (data))
8304 : return;
8305 :
8306 0 : if (find_doloop_use (data))
8307 : {
8308 0 : data->doloop_use_p = true;
8309 0 : if (dump_file && (dump_flags & TDF_DETAILS))
8310 : {
8311 0 : struct loop *loop = data->current_loop;
8312 0 : fprintf (dump_file,
8313 : "Predict loop %d can perform"
8314 : " doloop optimization later.\n",
8315 : loop->num);
8316 0 : flow_loop_dump (loop, dump_file, NULL, 1);
8317 : }
8318 : }
8319 : }
8320 :
8321 : /* Optimizes the LOOP. Returns true if anything changed. */
8322 :
8323 : static bool
8324 643678 : tree_ssa_iv_optimize_loop (struct ivopts_data *data, class loop *loop,
8325 : bitmap toremove)
8326 : {
8327 643678 : bool changed = false;
8328 643678 : class iv_ca *iv_ca;
8329 643678 : edge exit = single_dom_exit (loop);
8330 643678 : basic_block *body;
8331 :
8332 643678 : gcc_assert (!data->niters);
8333 643678 : data->current_loop = loop;
8334 643678 : data->loop_loc = find_loop_location (loop).get_location_t ();
8335 643678 : data->speed = optimize_loop_for_speed_p (loop);
8336 :
8337 643678 : if (dump_file && (dump_flags & TDF_DETAILS))
8338 : {
8339 72 : fprintf (dump_file, "Processing loop %d", loop->num);
8340 72 : if (data->loop_loc != UNKNOWN_LOCATION)
8341 70 : fprintf (dump_file, " at %s:%d", LOCATION_FILE (data->loop_loc),
8342 140 : LOCATION_LINE (data->loop_loc));
8343 72 : fprintf (dump_file, "\n");
8344 :
8345 72 : if (exit)
8346 : {
8347 62 : fprintf (dump_file, " single exit %d -> %d, exit condition ",
8348 62 : exit->src->index, exit->dest->index);
8349 124 : print_gimple_stmt (dump_file, *gsi_last_bb (exit->src),
8350 : 0, TDF_SLIM);
8351 62 : fprintf (dump_file, "\n");
8352 : }
8353 :
8354 72 : fprintf (dump_file, "\n");
8355 : }
8356 :
8357 643678 : body = get_loop_body (loop);
8358 643678 : data->body_includes_call = loop_body_includes_call (body, loop->num_nodes);
8359 643678 : renumber_gimple_stmt_uids_in_blocks (body, loop->num_nodes);
8360 :
8361 643678 : data->loop_single_exit_p
8362 643678 : = exit != NULL && loop_only_exit_p (loop, body, exit);
8363 :
8364 : /* For each ssa name determines whether it behaves as an induction variable
8365 : in some loop. */
8366 643678 : if (!find_induction_variables (data, body))
8367 130496 : goto finish;
8368 :
8369 : /* Finds interesting uses (item 1). */
8370 513182 : find_interesting_uses (data, body);
8371 513182 : if (data->vgroups.length () > MAX_CONSIDERED_GROUPS)
8372 1 : goto finish;
8373 :
8374 : /* Determine cost scaling factor for basic blocks in loop. */
8375 513181 : determine_scaling_factor (data, body);
8376 :
8377 : /* Analyze doloop possibility and mark the doloop use if predicted. */
8378 513181 : analyze_and_mark_doloop_use (data);
8379 :
8380 : /* Finds candidates for the induction variables (item 2). */
8381 513181 : find_iv_candidates (data);
8382 :
8383 : /* Calculates the costs (item 3, part 1). */
8384 513181 : determine_iv_costs (data);
8385 513181 : determine_group_iv_costs (data);
8386 513181 : determine_set_costs (data);
8387 :
8388 : /* Find the optimal set of induction variables (item 3, part 2). */
8389 513181 : iv_ca = find_optimal_iv_set (data);
8390 : /* Cleanup basic block aux field. */
8391 3372125 : for (unsigned i = 0; i < data->current_loop->num_nodes; i++)
8392 2858944 : body[i]->aux = NULL;
8393 513181 : if (!iv_ca)
8394 450 : goto finish;
8395 512731 : changed = true;
8396 :
8397 : /* Create the new induction variables (item 4, part 1). */
8398 512731 : create_new_ivs (data, iv_ca);
8399 512731 : iv_ca_free (&iv_ca);
8400 :
8401 : /* Rewrite the uses (item 4, part 2). */
8402 512731 : rewrite_groups (data);
8403 :
8404 : /* Remove the ivs that are unused after rewriting. */
8405 512731 : remove_unused_ivs (data, toremove);
8406 :
8407 643678 : finish:
8408 643678 : free (body);
8409 643678 : free_loop_data (data);
8410 :
8411 643678 : return changed;
8412 : }
8413 :
8414 : /* Main entry point. Optimizes induction variables in loops. */
8415 :
8416 : void
8417 248445 : tree_ssa_iv_optimize (void)
8418 : {
8419 248445 : struct ivopts_data data;
8420 248445 : auto_bitmap toremove;
8421 :
8422 248445 : tree_ssa_iv_optimize_init (&data);
8423 248445 : mark_ssa_maybe_undefs ();
8424 :
8425 248445 : enable_ranger (cfun, false);
8426 :
8427 : /* Optimize the loops starting with the innermost ones. */
8428 1389013 : for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))
8429 : {
8430 643678 : if (!dbg_cnt (ivopts_loop))
8431 0 : continue;
8432 :
8433 643678 : if (dump_file && (dump_flags & TDF_DETAILS))
8434 72 : flow_loop_dump (loop, dump_file, NULL, 1);
8435 :
8436 643678 : tree_ssa_iv_optimize_loop (&data, loop, toremove);
8437 248445 : }
8438 :
8439 248445 : disable_ranger (cfun);
8440 :
8441 : /* Remove eliminated IV defs. */
8442 248445 : release_defs_bitset (toremove);
8443 :
8444 : /* We have changed the structure of induction variables; it might happen
8445 : that definitions in the scev database refer to some of them that were
8446 : eliminated. */
8447 248445 : scev_reset_htab ();
8448 : /* Likewise niter and control-IV information. */
8449 248445 : free_numbers_of_iterations_estimates (cfun);
8450 :
8451 248445 : tree_ssa_iv_optimize_finalize (&data);
8452 248445 : }
8453 :
8454 : #include "gt-tree-ssa-loop-ivopts.h"
|