Line data Source code
1 : /* Natural loop discovery code for GNU compiler.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "cfghooks.h"
28 : #include "gimple-ssa.h"
29 : #include "diagnostic-core.h"
30 : #include "cfganal.h"
31 : #include "cfgloop.h"
32 : #include "gimple-iterator.h"
33 : #include "dumpfile.h"
34 : #include "tree-ssa.h"
35 : #include "tree-pretty-print.h"
36 : #include "sreal.h"
37 :
38 : static void flow_loops_cfg_dump (FILE *);
39 :
40 : /* Dump loop related CFG information. */
41 :
42 : static void
43 5956 : flow_loops_cfg_dump (FILE *file)
44 : {
45 5956 : basic_block bb;
46 :
47 5956 : if (!file)
48 : return;
49 :
50 27294 : FOR_EACH_BB_FN (bb, cfun)
51 : {
52 21338 : edge succ;
53 21338 : edge_iterator ei;
54 :
55 21338 : fprintf (file, ";; %d succs { ", bb->index);
56 49478 : FOR_EACH_EDGE (succ, ei, bb->succs)
57 28140 : fprintf (file, "%d ", succ->dest->index);
58 21338 : fprintf (file, "}\n");
59 : }
60 : }
61 :
62 : /* Return nonzero if the nodes of LOOP are a subset of OUTER. */
63 :
64 : bool
65 774176369 : flow_loop_nested_p (const class loop *outer, const class loop *loop)
66 : {
67 774176369 : unsigned odepth = loop_depth (outer);
68 :
69 774176369 : return (loop_depth (loop) > odepth
70 506589295 : && (*loop->superloops)[odepth] == outer);
71 : }
72 :
73 : /* Returns the loop such that LOOP is nested DEPTH (indexed from zero)
74 : loops within LOOP. */
75 :
76 : class loop *
77 22180290 : superloop_at_depth (class loop *loop, unsigned depth)
78 : {
79 22180290 : unsigned ldepth = loop_depth (loop);
80 :
81 22180290 : gcc_assert (depth <= ldepth);
82 :
83 22180290 : if (depth == ldepth)
84 : return loop;
85 :
86 5776458 : return (*loop->superloops)[depth];
87 : }
88 :
89 : /* Returns the list of the latch edges of LOOP. */
90 :
91 : static vec<edge>
92 48192 : get_loop_latch_edges (const class loop *loop)
93 : {
94 48192 : edge_iterator ei;
95 48192 : edge e;
96 48192 : vec<edge> ret = vNULL;
97 :
98 205320 : FOR_EACH_EDGE (e, ei, loop->header->preds)
99 : {
100 157128 : if (dominated_by_p (CDI_DOMINATORS, e->src, loop->header))
101 107029 : ret.safe_push (e);
102 : }
103 :
104 48192 : return ret;
105 : }
106 :
107 : /* Dump the loop information specified by LOOP to the stream FILE
108 : using auxiliary dump callback function LOOP_DUMP_AUX if non null. */
109 :
110 : void
111 7581 : flow_loop_dump (const class loop *loop, FILE *file,
112 : void (*loop_dump_aux) (const class loop *, FILE *, int),
113 : int verbose)
114 : {
115 7581 : basic_block *bbs;
116 7581 : unsigned i;
117 7581 : vec<edge> latches;
118 7581 : edge e;
119 :
120 7581 : if (! loop || ! loop->header)
121 0 : return;
122 :
123 7581 : fprintf (file, ";;\n;; Loop %d\n", loop->num);
124 :
125 7581 : fprintf (file, ";; header %d, ", loop->header->index);
126 7581 : if (loop->latch)
127 7577 : fprintf (file, "latch %d\n", loop->latch->index);
128 : else
129 : {
130 4 : fprintf (file, "multiple latches:");
131 4 : latches = get_loop_latch_edges (loop);
132 16 : FOR_EACH_VEC_ELT (latches, i, e)
133 8 : fprintf (file, " %d", e->src->index);
134 4 : latches.release ();
135 4 : fprintf (file, "\n");
136 : }
137 :
138 15162 : fprintf (file, ";; depth %d, outer %ld",
139 7581 : loop_depth (loop), (long) (loop_outer (loop)
140 1597 : ? loop_outer (loop)->num : -1));
141 7581 : print_loop_info (file, loop, ";; ");
142 :
143 7581 : fprintf (file, "\n;; nodes:");
144 7581 : bbs = get_loop_body (loop);
145 53849 : for (i = 0; i < loop->num_nodes; i++)
146 38687 : fprintf (file, " %d", bbs[i]->index);
147 7581 : free (bbs);
148 7581 : fprintf (file, "\n");
149 :
150 7581 : if (loop_dump_aux)
151 0 : loop_dump_aux (loop, file, verbose);
152 : }
153 :
154 : /* Dump the loop information about loops to the stream FILE,
155 : using auxiliary dump callback function LOOP_DUMP_AUX if non null. */
156 :
157 : void
158 49776629 : flow_loops_dump (FILE *file, void (*loop_dump_aux) (const class loop *, FILE *, int), int verbose)
159 : {
160 49776629 : if (!current_loops || ! file)
161 : return;
162 :
163 11968 : fprintf (file, ";; %d loops found\n", number_of_loops (cfun));
164 :
165 25466 : for (auto loop : loops_list (cfun, LI_INCLUDE_ROOT))
166 : {
167 7514 : flow_loop_dump (loop, file, loop_dump_aux, verbose);
168 5984 : }
169 :
170 5984 : if (verbose)
171 5956 : flow_loops_cfg_dump (file);
172 : }
173 :
174 : /* Free data allocated for LOOP. */
175 :
176 : void
177 17344088 : flow_loop_free (class loop *loop)
178 : {
179 17344088 : struct loop_exit *exit, *next;
180 :
181 17344088 : vec_free (loop->superloops);
182 :
183 : /* Break the list of the loop exit records. They will be freed when the
184 : corresponding edge is rescanned or removed, and this avoids
185 : accessing the (already released) head of the list stored in the
186 : loop structure. */
187 17357378 : for (exit = loop->exits->next; exit != loop->exits; exit = next)
188 : {
189 13290 : next = exit->next;
190 13290 : exit->next = exit;
191 13290 : exit->prev = exit;
192 : }
193 :
194 17344088 : ggc_free (loop->exits);
195 17344088 : ggc_free (loop);
196 17344088 : }
197 :
198 : /* Free all the memory allocated for LOOPS. */
199 :
200 : void
201 10853899 : flow_loops_free (struct loops *loops)
202 : {
203 10853899 : if (loops->larray)
204 : {
205 : unsigned i;
206 : loop_p loop;
207 :
208 : /* Free the loop descriptors. */
209 28288815 : FOR_EACH_VEC_SAFE_ELT (loops->larray, i, loop)
210 : {
211 17434916 : if (!loop)
212 518616 : continue;
213 :
214 16916300 : flow_loop_free (loop);
215 : }
216 :
217 21707798 : vec_free (loops->larray);
218 : }
219 10853899 : }
220 :
221 : /* Find the nodes contained within the LOOP with header HEADER.
222 : Return the number of nodes within the loop. */
223 :
224 : int
225 22357324 : flow_loop_nodes_find (basic_block header, class loop *loop)
226 : {
227 22357324 : vec<basic_block> stack = vNULL;
228 22357324 : int num_nodes = 1;
229 22357324 : edge latch;
230 22357324 : edge_iterator latch_ei;
231 :
232 22357324 : header->loop_father = loop;
233 :
234 67846318 : FOR_EACH_EDGE (latch, latch_ei, loop->header->preds)
235 : {
236 74177424 : if (latch->src->loop_father == loop
237 45488994 : || !dominated_by_p (CDI_DOMINATORS, latch->src, loop->header))
238 28688430 : continue;
239 :
240 16800564 : num_nodes++;
241 16800564 : stack.safe_push (latch->src);
242 16800564 : latch->src->loop_father = loop;
243 :
244 169810284 : while (!stack.is_empty ())
245 : {
246 107520726 : basic_block node;
247 107520726 : edge e;
248 107520726 : edge_iterator ei;
249 :
250 107520726 : node = stack.pop ();
251 :
252 255853261 : FOR_EACH_EDGE (e, ei, node->preds)
253 : {
254 148332535 : basic_block ancestor = e->src;
255 :
256 148332535 : if (ancestor->loop_father != loop)
257 : {
258 90720162 : ancestor->loop_father = loop;
259 90720162 : num_nodes++;
260 90720162 : stack.safe_push (ancestor);
261 : }
262 : }
263 : }
264 : }
265 22357324 : stack.release ();
266 :
267 22357324 : return num_nodes;
268 : }
269 :
270 : /* Records the vector of superloops of the loop LOOP, whose immediate
271 : superloop is FATHER. */
272 :
273 : static void
274 23330409 : establish_preds (class loop *loop, class loop *father)
275 : {
276 23330409 : loop_p ploop;
277 23330409 : unsigned depth = loop_depth (father) + 1;
278 23330409 : unsigned i;
279 :
280 23330409 : loop->superloops = 0;
281 23330409 : vec_alloc (loop->superloops, depth);
282 56038310 : FOR_EACH_VEC_SAFE_ELT (father->superloops, i, ploop)
283 9377492 : loop->superloops->quick_push (ploop);
284 23330409 : loop->superloops->quick_push (father);
285 :
286 23416553 : for (ploop = loop->inner; ploop; ploop = ploop->next)
287 86144 : establish_preds (ploop, loop);
288 23330409 : }
289 :
290 : /* Add LOOP to the loop hierarchy tree where FATHER is father of the
291 : added loop. If LOOP has some children, take care of that their
292 : pred field will be initialized correctly. If AFTER is non-null
293 : then it's expected it's a pointer into FATHERs inner sibling
294 : list and LOOP is added behind AFTER, otherwise it's added in front
295 : of FATHERs siblings. */
296 :
297 : void
298 23244265 : flow_loop_tree_node_add (class loop *father, class loop *loop,
299 : class loop *after)
300 : {
301 23244265 : if (after)
302 : {
303 4787 : loop->next = after->next;
304 4787 : after->next = loop;
305 : }
306 : else
307 : {
308 23239478 : loop->next = father->inner;
309 23239478 : father->inner = loop;
310 : }
311 :
312 23244265 : establish_preds (loop, father);
313 23244265 : }
314 :
315 : /* Remove LOOP from the loop hierarchy tree. */
316 :
317 : void
318 17167389 : flow_loop_tree_node_remove (class loop *loop)
319 : {
320 17167389 : class loop *prev, *father;
321 :
322 17167389 : father = loop_outer (loop);
323 :
324 : /* Remove loop from the list of sons. */
325 17167389 : if (father->inner == loop)
326 7978046 : father->inner = loop->next;
327 : else
328 : {
329 232891130 : for (prev = father->inner; prev->next != loop; prev = prev->next)
330 223701787 : continue;
331 9189343 : prev->next = loop->next;
332 : }
333 :
334 17167389 : loop->superloops = NULL;
335 17167389 : }
336 :
337 : /* Allocates and returns new loop structure. */
338 :
339 : class loop *
340 17464189 : alloc_loop (void)
341 : {
342 17464189 : class loop *loop = ggc_cleared_alloc<class loop> ();
343 :
344 17464189 : loop->exits = ggc_cleared_alloc<loop_exit> ();
345 17464189 : loop->exits->next = loop->exits->prev = loop->exits;
346 17464189 : loop->can_be_parallel = false;
347 17464189 : loop->constraints = 0;
348 17464189 : loop->nb_iterations_upper_bound = 0;
349 17464189 : loop->nb_iterations_likely_upper_bound = 0;
350 17464189 : loop->nb_iterations_estimate = 0;
351 17464189 : return loop;
352 : }
353 :
354 : /* Initializes loops structure LOOPS, reserving place for NUM_LOOPS loops
355 : (including the root of the loop tree). */
356 :
357 : void
358 10959525 : init_loops_structure (struct function *fn,
359 : struct loops *loops, unsigned num_loops)
360 : {
361 10959525 : class loop *root;
362 :
363 10959525 : memset (loops, 0, sizeof *loops);
364 10959525 : vec_alloc (loops->larray, num_loops);
365 :
366 : /* Dummy loop containing whole function. */
367 10959525 : root = alloc_loop ();
368 10959525 : root->num_nodes = n_basic_blocks_for_fn (fn);
369 10959525 : root->latch = EXIT_BLOCK_PTR_FOR_FN (fn);
370 10959525 : root->header = ENTRY_BLOCK_PTR_FOR_FN (fn);
371 10959525 : ENTRY_BLOCK_PTR_FOR_FN (fn)->loop_father = root;
372 10959525 : EXIT_BLOCK_PTR_FOR_FN (fn)->loop_father = root;
373 :
374 10959525 : loops->larray->quick_push (root);
375 10959525 : loops->tree_root = root;
376 10959525 : }
377 :
378 : /* Returns whether HEADER is a loop header. */
379 :
380 : bool
381 3934602365 : bb_loop_header_p (basic_block header)
382 : {
383 3934602365 : edge_iterator ei;
384 3934602365 : edge e;
385 :
386 : /* If we have an abnormal predecessor, do not consider the
387 : loop (not worth the problems). */
388 3934602365 : if (bb_has_abnormal_pred (header))
389 : return false;
390 :
391 : /* Look for back edges where a predecessor is dominated
392 : by this block. A natural loop has a single entry
393 : node (header) that dominates all the nodes in the
394 : loop. It also has single back edge to the header
395 : from a latch node. */
396 8936872140 : FOR_EACH_EDGE (e, ei, header->preds)
397 : {
398 5447883788 : basic_block latch = e->src;
399 5447883788 : if (latch != ENTRY_BLOCK_PTR_FOR_FN (cfun)
400 5447883788 : && dominated_by_p (CDI_DOMINATORS, latch, header))
401 : return true;
402 : }
403 :
404 : return false;
405 : }
406 :
407 : /* Find all the natural loops in the function and save in LOOPS structure and
408 : recalculate loop_father information in basic block structures.
409 : If LOOPS is non-NULL then the loop structures for already recorded loops
410 : will be re-used and their number will not change. We assume that no
411 : stale loops exist in LOOPS.
412 : When LOOPS is NULL it is allocated and re-built from scratch.
413 : Return the built LOOPS structure. */
414 :
415 : struct loops *
416 21763405 : flow_loops_find (struct loops *loops)
417 : {
418 21763405 : bool from_scratch = (loops == NULL);
419 21763405 : int *rc_order;
420 21763405 : int b;
421 21763405 : unsigned i;
422 :
423 : /* Ensure that the dominators are computed. */
424 21763405 : calculate_dominance_info (CDI_DOMINATORS);
425 :
426 21763405 : if (!loops)
427 : {
428 10812569 : loops = ggc_cleared_alloc<struct loops> ();
429 10812569 : init_loops_structure (cfun, loops, 1);
430 : }
431 :
432 : /* Ensure that loop exits were released. */
433 21763405 : gcc_assert (loops->exits == NULL);
434 :
435 : /* Taking care of this degenerate case makes the rest of
436 : this code simpler. */
437 21763405 : if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
438 : return loops;
439 :
440 : /* The root loop node contains all basic-blocks. */
441 21531284 : loops->tree_root->num_nodes = n_basic_blocks_for_fn (cfun);
442 :
443 : /* Compute depth first search order of the CFG so that outer
444 : natural loops will be found before inner natural loops. */
445 21531284 : rc_order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
446 21531284 : pre_and_rev_post_order_compute (NULL, rc_order, false);
447 :
448 : /* Gather all loop headers in reverse completion order and allocate
449 : loop structures for loops that are not already present. */
450 21531284 : auto_vec<loop_p> larray (loops->larray->length ());
451 365283554 : for (b = 0; b < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; b++)
452 : {
453 343752270 : basic_block header = BASIC_BLOCK_FOR_FN (cfun, rc_order[b]);
454 343752270 : if (bb_loop_header_p (header))
455 : {
456 22357324 : class loop *loop;
457 :
458 : /* The current active loop tree has valid loop-fathers for
459 : header blocks. */
460 22357324 : if (!from_scratch
461 16724788 : && header->loop_father->header == header)
462 : {
463 16656045 : loop = header->loop_father;
464 : /* If we found an existing loop remove it from the
465 : loop tree. It is going to be inserted again
466 : below. */
467 16656045 : flow_loop_tree_node_remove (loop);
468 : }
469 : else
470 : {
471 : /* Otherwise allocate a new loop structure for the loop. */
472 5701279 : loop = alloc_loop ();
473 : /* ??? We could re-use unused loop slots here. */
474 5701279 : loop->num = loops->larray->length ();
475 5701279 : vec_safe_push (loops->larray, loop);
476 5701279 : loop->header = header;
477 :
478 5701279 : if (!from_scratch
479 5701279 : && dump_file && (dump_flags & TDF_DETAILS))
480 3 : fprintf (dump_file, "flow_loops_find: discovered new "
481 : "loop %d with header %d\n",
482 : loop->num, header->index);
483 : }
484 : /* Reset latch, we recompute it below. */
485 22357324 : loop->latch = NULL;
486 22357324 : larray.safe_push (loop);
487 : }
488 :
489 : /* Make blocks part of the loop root node at start. */
490 343752270 : header->loop_father = loops->tree_root;
491 : }
492 :
493 21531284 : free (rc_order);
494 :
495 : /* Now iterate over the loops found, insert them into the loop tree
496 : and assign basic-block ownership. */
497 43888608 : for (i = 0; i < larray.length (); ++i)
498 : {
499 22357324 : class loop *loop = larray[i];
500 22357324 : basic_block header = loop->header;
501 22357324 : edge_iterator ei;
502 22357324 : edge e;
503 :
504 22357324 : flow_loop_tree_node_add (header->loop_father, loop);
505 22357324 : loop->num_nodes = flow_loop_nodes_find (loop->header, loop);
506 :
507 : /* Look for the latch for this header block, if it has just a
508 : single one. */
509 67450601 : FOR_EACH_EDGE (e, ei, header->preds)
510 : {
511 45323667 : basic_block latch = e->src;
512 :
513 45323667 : if (flow_bb_inside_loop_p (loop, latch))
514 : {
515 22587714 : if (loop->latch != NULL)
516 : {
517 : /* More than one latch edge. */
518 230390 : loop->latch = NULL;
519 230390 : break;
520 : }
521 22357324 : loop->latch = latch;
522 : }
523 : }
524 : }
525 :
526 21531284 : return loops;
527 21531284 : }
528 :
529 : /* qsort helper for sort_sibling_loops. */
530 :
531 : static int *sort_sibling_loops_cmp_rpo;
532 : static int
533 2635 : sort_sibling_loops_cmp (const void *la_, const void *lb_)
534 : {
535 2635 : const class loop *la = *(const class loop * const *)la_;
536 2635 : const class loop *lb = *(const class loop * const *)lb_;
537 2635 : return (sort_sibling_loops_cmp_rpo[la->header->index]
538 2635 : - sort_sibling_loops_cmp_rpo[lb->header->index]);
539 : }
540 :
541 : /* Sort sibling loops in RPO order. */
542 :
543 : void
544 556 : sort_sibling_loops (function *fn)
545 : {
546 : /* Match flow_loops_find in the order we sort sibling loops. */
547 556 : sort_sibling_loops_cmp_rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
548 556 : int *rc_order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
549 556 : pre_and_rev_post_order_compute_fn (fn, NULL, rc_order, false);
550 7570 : for (int i = 0; i < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; ++i)
551 7014 : sort_sibling_loops_cmp_rpo[rc_order[i]] = i;
552 556 : free (rc_order);
553 :
554 556 : auto_vec<loop_p, 3> siblings;
555 3597 : for (auto loop : loops_list (fn, LI_INCLUDE_ROOT))
556 1929 : if (loop->inner && loop->inner->next)
557 : {
558 224 : loop_p sibling = loop->inner;
559 598 : do
560 : {
561 598 : siblings.safe_push (sibling);
562 598 : sibling = sibling->next;
563 : }
564 598 : while (sibling);
565 224 : siblings.qsort (sort_sibling_loops_cmp);
566 224 : loop_p *siblingp = &loop->inner;
567 822 : for (unsigned i = 0; i < siblings.length (); ++i)
568 : {
569 598 : *siblingp = siblings[i];
570 598 : siblingp = &(*siblingp)->next;
571 : }
572 224 : *siblingp = NULL;
573 224 : siblings.truncate (0);
574 556 : }
575 :
576 556 : free (sort_sibling_loops_cmp_rpo);
577 556 : sort_sibling_loops_cmp_rpo = NULL;
578 556 : }
579 :
580 : /* Ratio of frequencies of edges so that one of more latch edges is
581 : considered to belong to inner loop with same header. */
582 : #define HEAVY_EDGE_RATIO 8
583 :
584 : /* Minimum number of samples for that we apply
585 : find_subloop_latch_edge_by_profile heuristics. */
586 : #define HEAVY_EDGE_MIN_SAMPLES 10
587 :
588 : /* If the profile info is available, finds an edge in LATCHES that much more
589 : frequent than the remaining edges. Returns such an edge, or NULL if we do
590 : not find one.
591 :
592 : We do not use guessed profile here, only the measured one. The guessed
593 : profile is usually too flat and unreliable for this (and it is mostly based
594 : on the loop structure of the program, so it does not make much sense to
595 : derive the loop structure from it). */
596 :
597 : static edge
598 20231 : find_subloop_latch_edge_by_profile (vec<edge> latches)
599 : {
600 20231 : unsigned i;
601 20231 : edge e, me = NULL;
602 20231 : profile_count mcount = profile_count::zero (), tcount = profile_count::zero ();
603 :
604 71732 : FOR_EACH_VEC_ELT (latches, i, e)
605 : {
606 51501 : if (e->count ()> mcount)
607 : {
608 8587 : me = e;
609 8587 : mcount = e->count();
610 : }
611 51501 : tcount += e->count();
612 : }
613 :
614 20231 : if (!tcount.initialized_p () || !(tcount.ipa () > HEAVY_EDGE_MIN_SAMPLES)
615 20251 : || (tcount - mcount) * HEAVY_EDGE_RATIO > tcount)
616 20217 : return NULL;
617 :
618 14 : if (dump_file)
619 0 : fprintf (dump_file,
620 : "Found latch edge %d -> %d using profile information.\n",
621 0 : me->src->index, me->dest->index);
622 : return me;
623 : }
624 :
625 : /* Among LATCHES, guesses a latch edge of LOOP corresponding to subloop, based
626 : on the structure of induction variables. Returns this edge, or NULL if we
627 : do not find any.
628 :
629 : We are quite conservative, and look just for an obvious simple innermost
630 : loop (which is the case where we would lose the most performance by not
631 : disambiguating the loop). More precisely, we look for the following
632 : situation: The source of the chosen latch edge dominates sources of all
633 : the other latch edges. Additionally, the header does not contain a phi node
634 : such that the argument from the chosen edge is equal to the argument from
635 : another edge. */
636 :
637 : static edge
638 19507 : find_subloop_latch_edge_by_ivs (class loop *loop ATTRIBUTE_UNUSED, vec<edge> latches)
639 : {
640 19507 : edge e, latch = latches[0];
641 19507 : unsigned i;
642 19507 : gphi *phi;
643 19507 : gphi_iterator psi;
644 19507 : tree lop;
645 19507 : basic_block bb;
646 :
647 : /* Find the candidate for the latch edge. */
648 50044 : for (i = 1; latches.iterate (i, &e); i++)
649 30537 : if (dominated_by_p (CDI_DOMINATORS, latch->src, e->src))
650 5668 : latch = e;
651 :
652 : /* Verify that it dominates all the latch edges. */
653 50660 : FOR_EACH_VEC_ELT (latches, i, e)
654 42359 : if (!dominated_by_p (CDI_DOMINATORS, e->src, latch->src))
655 : return NULL;
656 :
657 : /* Check for a phi node that would deny that this is a latch edge of
658 : a subloop. */
659 20430 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
660 : {
661 17082 : phi = psi.phi ();
662 17082 : lop = PHI_ARG_DEF_FROM_EDGE (phi, latch);
663 :
664 : /* Ignore the values that are not changed inside the subloop. */
665 20629 : if (TREE_CODE (lop) != SSA_NAME
666 17082 : || SSA_NAME_DEF_STMT (lop) == phi)
667 3547 : continue;
668 13535 : bb = gimple_bb (SSA_NAME_DEF_STMT (lop));
669 13535 : if (!bb || !flow_bb_inside_loop_p (loop, bb))
670 16 : continue;
671 :
672 36521 : FOR_EACH_VEC_ELT (latches, i, e)
673 24392 : if (e != latch
674 24392 : && PHI_ARG_DEF_FROM_EDGE (phi, e) == lop)
675 : return NULL;
676 : }
677 :
678 3348 : if (dump_file)
679 1 : fprintf (dump_file,
680 : "Found latch edge %d -> %d using iv structure.\n",
681 1 : latch->src->index, latch->dest->index);
682 : return latch;
683 : }
684 :
685 : /* If we can determine that one of the several latch edges of LOOP behaves
686 : as a latch edge of a separate subloop, returns this edge. Otherwise
687 : returns NULL. */
688 :
689 : static edge
690 25775 : find_subloop_latch_edge (class loop *loop)
691 : {
692 25775 : vec<edge> latches = get_loop_latch_edges (loop);
693 25775 : edge latch = NULL;
694 :
695 25775 : if (latches.length () > 1)
696 : {
697 20231 : latch = find_subloop_latch_edge_by_profile (latches);
698 :
699 20231 : if (!latch
700 : /* We consider ivs to guess the latch edge only in SSA. Perhaps we
701 : should use cfghook for this, but it is hard to imagine it would
702 : be useful elsewhere. */
703 20231 : && current_ir_type () == IR_GIMPLE)
704 19507 : latch = find_subloop_latch_edge_by_ivs (loop, latches);
705 : }
706 :
707 25775 : latches.release ();
708 25775 : return latch;
709 : }
710 :
711 : /* Callback for make_forwarder_block. Returns true if the edge E is marked
712 : in the set MFB_REIS_SET. */
713 :
714 : static hash_set<edge> *mfb_reis_set;
715 : static bool
716 72674 : mfb_redirect_edges_in_set (edge e)
717 : {
718 72674 : return mfb_reis_set->contains (e);
719 : }
720 :
721 : /* Creates a subloop of LOOP with latch edge LATCH. */
722 :
723 : static void
724 3362 : form_subloop (class loop *loop, edge latch)
725 : {
726 3362 : edge_iterator ei;
727 3362 : edge e, new_entry;
728 3362 : class loop *new_loop;
729 :
730 3362 : mfb_reis_set = new hash_set<edge>;
731 13816 : FOR_EACH_EDGE (e, ei, loop->header->preds)
732 : {
733 10454 : if (e != latch)
734 7092 : mfb_reis_set->add (e);
735 : }
736 3362 : new_entry = make_forwarder_block (loop->header, mfb_redirect_edges_in_set,
737 : NULL);
738 6724 : delete mfb_reis_set;
739 :
740 3362 : loop->header = new_entry->src;
741 :
742 : /* Find the blocks and subloops that belong to the new loop, and add it to
743 : the appropriate place in the loop tree. */
744 3362 : new_loop = alloc_loop ();
745 3362 : new_loop->header = new_entry->dest;
746 3362 : new_loop->latch = latch->src;
747 3362 : add_loop (new_loop, loop);
748 3362 : }
749 :
750 : /* Make all the latch edges of LOOP to go to a single forwarder block --
751 : a new latch of LOOP. */
752 :
753 : static void
754 22413 : merge_latch_edges (class loop *loop)
755 : {
756 22413 : vec<edge> latches = get_loop_latch_edges (loop);
757 22413 : edge latch, e;
758 22413 : unsigned i;
759 :
760 22413 : gcc_assert (latches.length () > 0);
761 :
762 22413 : if (latches.length () == 1)
763 5544 : loop->latch = latches[0]->src;
764 : else
765 : {
766 16869 : if (dump_file)
767 8 : fprintf (dump_file, "Merged latch edges of loop %d\n", loop->num);
768 :
769 16869 : mfb_reis_set = new hash_set<edge>;
770 61301 : FOR_EACH_VEC_ELT (latches, i, e)
771 44432 : mfb_reis_set->add (e);
772 16869 : latch = make_forwarder_block (loop->header, mfb_redirect_edges_in_set,
773 : NULL);
774 33738 : delete mfb_reis_set;
775 :
776 16869 : loop->header = latch->dest;
777 16869 : loop->latch = latch->src;
778 : }
779 :
780 22413 : latches.release ();
781 22413 : }
782 :
783 : /* LOOP may have several latch edges. Transform it into (possibly several)
784 : loops with single latch edge. */
785 :
786 : static void
787 22413 : disambiguate_multiple_latches (class loop *loop)
788 : {
789 22413 : edge e;
790 :
791 : /* We eliminate the multiple latches by splitting the header to the forwarder
792 : block F and the rest R, and redirecting the edges. There are two cases:
793 :
794 : 1) If there is a latch edge E that corresponds to a subloop (we guess
795 : that based on profile -- if it is taken much more often than the
796 : remaining edges; and on trees, using the information about induction
797 : variables of the loops), we redirect E to R, all the remaining edges to
798 : F, then rescan the loops and try again for the outer loop.
799 : 2) If there is no such edge, we redirect all latch edges to F, and the
800 : entry edges to R, thus making F the single latch of the loop. */
801 :
802 22413 : if (dump_file)
803 9 : fprintf (dump_file, "Disambiguating loop %d with multiple latches\n",
804 : loop->num);
805 :
806 : /* During latch merging, we may need to redirect the entry edges to a new
807 : block. This would cause problems if the entry edge was the one from the
808 : entry block. To avoid having to handle this case specially, split
809 : such entry edge. */
810 22413 : e = find_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), loop->header);
811 22413 : if (e)
812 897 : split_edge (e);
813 :
814 29137 : while (1)
815 : {
816 25775 : e = find_subloop_latch_edge (loop);
817 25775 : if (!e)
818 : break;
819 :
820 3362 : form_subloop (loop, e);
821 : }
822 :
823 22413 : merge_latch_edges (loop);
824 22413 : }
825 :
826 : /* Split loops with multiple latch edges. */
827 :
828 : void
829 28537356 : disambiguate_loops_with_multiple_latches (void)
830 : {
831 103349685 : for (auto loop : loops_list (cfun, 0))
832 : {
833 17737617 : if (!loop->latch)
834 22413 : disambiguate_multiple_latches (loop);
835 28537356 : }
836 28537356 : }
837 :
838 : /* Return nonzero if basic block BB belongs to LOOP. */
839 : bool
840 3258501994 : flow_bb_inside_loop_p (const class loop *loop, const_basic_block bb)
841 : {
842 3258501994 : class loop *source_loop;
843 :
844 3258501994 : if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun)
845 3258489308 : || bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
846 : return 0;
847 :
848 3240400436 : source_loop = bb->loop_father;
849 3240400436 : return loop == source_loop || flow_loop_nested_p (loop, source_loop);
850 : }
851 :
852 : /* Enumeration predicate for get_loop_body_with_size. */
853 : static bool
854 1229113346 : glb_enum_p (const_basic_block bb, const void *glb_loop)
855 : {
856 1229113346 : const class loop *const loop = (const class loop *) glb_loop;
857 1229113346 : return (bb != loop->header
858 1229113346 : && dominated_by_p (CDI_DOMINATORS, bb, loop->header));
859 : }
860 :
861 : /* Gets basic blocks of a LOOP. Header is the 0-th block, rest is in dfs
862 : order against direction of edges from latch. Specially, if
863 : header != latch, latch is the 1-st block. LOOP cannot be the fake
864 : loop tree root, and its size must be at most MAX_SIZE. The blocks
865 : in the LOOP body are stored to BODY, and the size of the LOOP is
866 : returned. */
867 :
868 : unsigned
869 220563558 : get_loop_body_with_size (const class loop *loop, basic_block *body,
870 : unsigned max_size)
871 : {
872 220563558 : return dfs_enumerate_from (loop->header, 1, glb_enum_p,
873 220563558 : body, max_size, loop);
874 : }
875 :
876 : /* Gets basic blocks of a LOOP. Header is the 0-th block, rest is in dfs
877 : order against direction of edges from latch. Specially, if
878 : header != latch, latch is the 1-st block. */
879 :
880 : basic_block *
881 21976077 : get_loop_body (const class loop *loop)
882 : {
883 21976077 : basic_block *body, bb;
884 21976077 : unsigned tv = 0;
885 :
886 21976077 : gcc_assert (loop->num_nodes);
887 :
888 21976077 : body = XNEWVEC (basic_block, loop->num_nodes);
889 :
890 21976077 : if (loop->latch == EXIT_BLOCK_PTR_FOR_FN (cfun))
891 : {
892 : /* There may be blocks unreachable from EXIT_BLOCK, hence we need to
893 : special-case the fake loop that contains the whole function. */
894 6232 : gcc_assert (loop->num_nodes == (unsigned) n_basic_blocks_for_fn (cfun));
895 6232 : body[tv++] = loop->header;
896 6232 : body[tv++] = EXIT_BLOCK_PTR_FOR_FN (cfun);
897 28430 : FOR_EACH_BB_FN (bb, cfun)
898 22198 : body[tv++] = bb;
899 : }
900 : else
901 21969845 : tv = get_loop_body_with_size (loop, body, loop->num_nodes);
902 :
903 21976077 : gcc_assert (tv == loop->num_nodes);
904 21976077 : return body;
905 : }
906 :
907 : /* Fills dominance descendants inside LOOP of the basic block BB into
908 : array TOVISIT from index *TV. */
909 :
910 : static void
911 5716377 : fill_sons_in_loop (const class loop *loop, basic_block bb,
912 : basic_block *tovisit, int *tv)
913 : {
914 9473289 : basic_block son, postpone = NULL;
915 :
916 9473289 : tovisit[(*tv)++] = bb;
917 9473289 : for (son = first_dom_son (CDI_DOMINATORS, bb);
918 19188398 : son;
919 9715109 : son = next_dom_son (CDI_DOMINATORS, son))
920 : {
921 9715109 : if (!flow_bb_inside_loop_p (loop, son))
922 2232029 : continue;
923 :
924 7483080 : if (dominated_by_p (CDI_DOMINATORS, loop->latch, son))
925 : {
926 3756912 : postpone = son;
927 3756912 : continue;
928 : }
929 3726168 : fill_sons_in_loop (loop, son, tovisit, tv);
930 : }
931 :
932 9473289 : if (postpone)
933 : fill_sons_in_loop (loop, postpone, tovisit, tv);
934 5716377 : }
935 :
936 : /* Gets body of a LOOP (that must be different from the outermost loop)
937 : sorted by dominance relation. Additionally, if a basic block s dominates
938 : the latch, then only blocks dominated by s are be after it. */
939 :
940 : basic_block *
941 1990209 : get_loop_body_in_dom_order (const class loop *loop)
942 : {
943 1990209 : basic_block *tovisit;
944 1990209 : int tv;
945 :
946 1990209 : gcc_assert (loop->num_nodes);
947 :
948 1990209 : tovisit = XNEWVEC (basic_block, loop->num_nodes);
949 :
950 1990209 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
951 :
952 1990209 : tv = 0;
953 1990209 : fill_sons_in_loop (loop, loop->header, tovisit, &tv);
954 :
955 1990209 : gcc_assert (tv == (int) loop->num_nodes);
956 :
957 1990209 : return tovisit;
958 : }
959 :
960 : /* Gets body of a LOOP sorted via provided BB_COMPARATOR. */
961 :
962 : basic_block *
963 56 : get_loop_body_in_custom_order (const class loop *loop,
964 : int (*bb_comparator) (const void *, const void *))
965 : {
966 56 : basic_block *bbs = get_loop_body (loop);
967 :
968 56 : qsort (bbs, loop->num_nodes, sizeof (basic_block), bb_comparator);
969 :
970 56 : return bbs;
971 : }
972 :
973 : /* Same as above, but use gcc_sort_r instead of qsort. */
974 :
975 : basic_block *
976 147701 : get_loop_body_in_custom_order (const class loop *loop, void *data,
977 : int (*bb_comparator) (const void *, const void *, void *))
978 : {
979 147701 : basic_block *bbs = get_loop_body (loop);
980 :
981 147701 : gcc_sort_r (bbs, loop->num_nodes, sizeof (basic_block), bb_comparator, data);
982 :
983 147701 : return bbs;
984 : }
985 :
986 : /* Get body of a LOOP in breadth first sort order. */
987 :
988 : basic_block *
989 405007 : get_loop_body_in_bfs_order (const class loop *loop)
990 : {
991 405007 : basic_block *blocks;
992 405007 : basic_block bb;
993 405007 : unsigned int i = 1;
994 405007 : unsigned int vc = 0;
995 :
996 405007 : gcc_assert (loop->num_nodes);
997 405007 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
998 :
999 405007 : blocks = XNEWVEC (basic_block, loop->num_nodes);
1000 405007 : auto_bitmap visited;
1001 405007 : blocks[0] = loop->header;
1002 405007 : bitmap_set_bit (visited, loop->header->index);
1003 2128852 : while (i < loop->num_nodes)
1004 : {
1005 1318838 : edge e;
1006 1318838 : edge_iterator ei;
1007 1318838 : gcc_assert (i > vc);
1008 1318838 : bb = blocks[vc++];
1009 :
1010 3614639 : FOR_EACH_EDGE (e, ei, bb->succs)
1011 : {
1012 2295801 : if (flow_bb_inside_loop_p (loop, e->dest))
1013 : {
1014 : /* This bb is now visited. */
1015 1577246 : if (bitmap_set_bit (visited, e->dest->index))
1016 1370280 : blocks[i++] = e->dest;
1017 : }
1018 : }
1019 : }
1020 :
1021 405007 : return blocks;
1022 405007 : }
1023 :
1024 : /* Hash function for struct loop_exit. */
1025 :
1026 : hashval_t
1027 254554515 : loop_exit_hasher::hash (loop_exit *exit)
1028 : {
1029 254554515 : return htab_hash_pointer (exit->e);
1030 : }
1031 :
1032 : /* Equality function for struct loop_exit. Compares with edge. */
1033 :
1034 : bool
1035 301847953 : loop_exit_hasher::equal (loop_exit *exit, edge e)
1036 : {
1037 301847953 : return exit->e == e;
1038 : }
1039 :
1040 : /* Frees the list of loop exit descriptions EX. */
1041 :
1042 : void
1043 19455968 : loop_exit_hasher::remove (loop_exit *exit)
1044 : {
1045 19455968 : loop_exit *next;
1046 40389265 : for (; exit; exit = next)
1047 : {
1048 20933297 : next = exit->next_e;
1049 :
1050 20933297 : exit->next->prev = exit->prev;
1051 20933297 : exit->prev->next = exit->next;
1052 :
1053 20933297 : ggc_free (exit);
1054 : }
1055 19455968 : }
1056 :
1057 : /* Returns the list of records for E as an exit of a loop. */
1058 :
1059 : static struct loop_exit *
1060 41291121 : get_exit_descriptions (edge e)
1061 : {
1062 41291121 : return current_loops->exits->find_with_hash (e, htab_hash_pointer (e));
1063 : }
1064 :
1065 : /* Updates the lists of loop exits in that E appears.
1066 : If REMOVED is true, E is being removed, and we
1067 : just remove it from the lists of exits.
1068 : If NEW_EDGE is true and E is not a loop exit, we
1069 : do not try to remove it from loop exit lists. */
1070 :
1071 : void
1072 557323505 : rescan_loop_exit (edge e, bool new_edge, bool removed)
1073 : {
1074 557323505 : struct loop_exit *exits = NULL, *exit;
1075 557323505 : class loop *aloop, *cloop;
1076 :
1077 557323505 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1078 : return;
1079 :
1080 255666759 : if (!removed
1081 240166243 : && e->src->loop_father != NULL
1082 240166243 : && e->dest->loop_father != NULL
1083 493798291 : && !flow_bb_inside_loop_p (e->src->loop_father, e->dest))
1084 : {
1085 37539107 : cloop = find_common_loop (e->src->loop_father, e->dest->loop_father);
1086 37539107 : for (aloop = e->src->loop_father;
1087 58472404 : aloop != cloop;
1088 20933297 : aloop = loop_outer (aloop))
1089 : {
1090 20933297 : exit = ggc_alloc<loop_exit> ();
1091 20933297 : exit->e = e;
1092 :
1093 20933297 : exit->next = aloop->exits->next;
1094 20933297 : exit->prev = aloop->exits;
1095 20933297 : exit->next->prev = exit;
1096 20933297 : exit->prev->next = exit;
1097 :
1098 20933297 : exit->next_e = exits;
1099 20933297 : exits = exit;
1100 : }
1101 : }
1102 :
1103 255666759 : if (!exits && new_edge)
1104 : return;
1105 :
1106 38597704 : loop_exit **slot
1107 57739440 : = current_loops->exits->find_slot_with_hash (e, htab_hash_pointer (e),
1108 : exits ? INSERT : NO_INSERT);
1109 38597704 : if (!slot)
1110 : return;
1111 :
1112 20777373 : if (exits)
1113 : {
1114 19455968 : if (*slot)
1115 404061 : loop_exit_hasher::remove (*slot);
1116 19455968 : *slot = exits;
1117 : }
1118 : else
1119 1321405 : current_loops->exits->clear_slot (slot);
1120 : }
1121 :
1122 : /* For each loop, record list of exit edges, and start maintaining these
1123 : lists. */
1124 :
1125 : void
1126 17323394 : record_loop_exits (void)
1127 : {
1128 17323394 : basic_block bb;
1129 17323394 : edge_iterator ei;
1130 17323394 : edge e;
1131 :
1132 17323394 : if (!current_loops)
1133 0 : return;
1134 :
1135 17323394 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1136 : return;
1137 17323394 : loops_state_set (LOOPS_HAVE_RECORDED_EXITS);
1138 :
1139 17323394 : gcc_assert (current_loops->exits == NULL);
1140 17323394 : current_loops->exits
1141 34646788 : = hash_table<loop_exit_hasher>::create_ggc (2 * number_of_loops (cfun));
1142 :
1143 174862447 : FOR_EACH_BB_FN (bb, cfun)
1144 : {
1145 378548241 : FOR_EACH_EDGE (e, ei, bb->succs)
1146 : {
1147 221009188 : rescan_loop_exit (e, true, false);
1148 : }
1149 : }
1150 : }
1151 :
1152 : /* Dumps information about the exit in *SLOT to FILE.
1153 : Callback for htab_traverse. */
1154 :
1155 : int
1156 0 : dump_recorded_exit (loop_exit **slot, FILE *file)
1157 : {
1158 0 : struct loop_exit *exit = *slot;
1159 0 : unsigned n = 0;
1160 0 : edge e = exit->e;
1161 :
1162 0 : for (; exit != NULL; exit = exit->next_e)
1163 0 : n++;
1164 :
1165 0 : fprintf (file, "Edge %d->%d exits %u loops\n",
1166 0 : e->src->index, e->dest->index, n);
1167 :
1168 0 : return 1;
1169 : }
1170 :
1171 : /* Dumps the recorded exits of loops to FILE. */
1172 :
1173 : extern void dump_recorded_exits (FILE *);
1174 : void
1175 0 : dump_recorded_exits (FILE *file)
1176 : {
1177 0 : if (!current_loops->exits)
1178 : return;
1179 0 : current_loops->exits->traverse<FILE *, dump_recorded_exit> (file);
1180 : }
1181 :
1182 : /* Releases lists of loop exits. */
1183 :
1184 : void
1185 17323394 : release_recorded_exits (function *fn)
1186 : {
1187 17323394 : gcc_assert (loops_state_satisfies_p (fn, LOOPS_HAVE_RECORDED_EXITS));
1188 17323394 : loops_for_fn (fn)->exits->empty ();
1189 17323394 : loops_for_fn (fn)->exits = NULL;
1190 17323394 : loops_state_clear (fn, LOOPS_HAVE_RECORDED_EXITS);
1191 17323394 : }
1192 :
1193 : /* Returns the list of the exit edges of a LOOP. */
1194 :
1195 : auto_vec<edge>
1196 36560176 : get_loop_exit_edges (const class loop *loop, basic_block *body)
1197 : {
1198 36560176 : auto_vec<edge> edges;
1199 36560176 : edge e;
1200 36560176 : unsigned i;
1201 36560176 : edge_iterator ei;
1202 36560176 : struct loop_exit *exit;
1203 :
1204 36560176 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
1205 :
1206 : /* If we maintain the lists of exits, use them. Otherwise we must
1207 : scan the body of the loop. */
1208 36560176 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1209 : {
1210 110656913 : for (exit = loop->exits->next; exit->e; exit = exit->next)
1211 74761245 : edges.safe_push (exit->e);
1212 : }
1213 : else
1214 : {
1215 664508 : bool body_from_caller = true;
1216 664508 : if (!body)
1217 : {
1218 654918 : body = get_loop_body (loop);
1219 654918 : body_from_caller = false;
1220 : }
1221 4790586 : for (i = 0; i < loop->num_nodes; i++)
1222 10758599 : FOR_EACH_EDGE (e, ei, body[i]->succs)
1223 : {
1224 6632521 : if (!flow_bb_inside_loop_p (loop, e->dest))
1225 1605477 : edges.safe_push (e);
1226 : }
1227 664508 : if (!body_from_caller)
1228 654918 : free (body);
1229 : }
1230 :
1231 36560176 : return edges;
1232 : }
1233 :
1234 : /* Counts the number of conditional branches inside LOOP. */
1235 :
1236 : unsigned
1237 133 : num_loop_branches (const class loop *loop)
1238 : {
1239 133 : unsigned i, n;
1240 133 : basic_block * body;
1241 :
1242 133 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
1243 :
1244 133 : body = get_loop_body (loop);
1245 133 : n = 0;
1246 632 : for (i = 0; i < loop->num_nodes; i++)
1247 568 : if (EDGE_COUNT (body[i]->succs) >= 2)
1248 202 : n++;
1249 133 : free (body);
1250 :
1251 133 : return n;
1252 : }
1253 :
1254 : /* Adds basic block BB to LOOP. */
1255 : void
1256 45711200 : add_bb_to_loop (basic_block bb, class loop *loop)
1257 : {
1258 45711200 : unsigned i;
1259 45711200 : loop_p ploop;
1260 45711200 : edge_iterator ei;
1261 45711200 : edge e;
1262 :
1263 45711200 : gcc_assert (bb->loop_father == NULL);
1264 45711200 : bb->loop_father = loop;
1265 45711200 : loop->num_nodes++;
1266 66227677 : FOR_EACH_VEC_SAFE_ELT (loop->superloops, i, ploop)
1267 20516477 : ploop->num_nodes++;
1268 :
1269 95686816 : FOR_EACH_EDGE (e, ei, bb->succs)
1270 : {
1271 49975616 : rescan_loop_exit (e, true, false);
1272 : }
1273 76849524 : FOR_EACH_EDGE (e, ei, bb->preds)
1274 : {
1275 31138324 : rescan_loop_exit (e, true, false);
1276 : }
1277 45711200 : }
1278 :
1279 : /* Remove basic block BB from loops. */
1280 : void
1281 57099413 : remove_bb_from_loops (basic_block bb)
1282 : {
1283 57099413 : unsigned i;
1284 57099413 : class loop *loop = bb->loop_father;
1285 57099413 : loop_p ploop;
1286 57099413 : edge_iterator ei;
1287 57099413 : edge e;
1288 :
1289 57099413 : gcc_assert (loop != NULL);
1290 57099413 : loop->num_nodes--;
1291 78800949 : FOR_EACH_VEC_SAFE_ELT (loop->superloops, i, ploop)
1292 21701536 : ploop->num_nodes--;
1293 57099413 : bb->loop_father = NULL;
1294 :
1295 85642800 : FOR_EACH_EDGE (e, ei, bb->succs)
1296 : {
1297 28543387 : rescan_loop_exit (e, false, true);
1298 : }
1299 77518170 : FOR_EACH_EDGE (e, ei, bb->preds)
1300 : {
1301 20418757 : rescan_loop_exit (e, false, true);
1302 : }
1303 57099413 : }
1304 :
1305 : /* Finds nearest common ancestor in loop tree for given loops. */
1306 : class loop *
1307 206380490 : find_common_loop (class loop *loop_s, class loop *loop_d)
1308 : {
1309 206380490 : unsigned sdepth, ddepth;
1310 :
1311 206380490 : if (!loop_s) return loop_d;
1312 206379945 : if (!loop_d) return loop_s;
1313 :
1314 206379945 : sdepth = loop_depth (loop_s);
1315 206379945 : ddepth = loop_depth (loop_d);
1316 :
1317 84123583 : if (sdepth < ddepth)
1318 6447060 : loop_d = (*loop_d->superloops)[sdepth];
1319 199932885 : else if (sdepth > ddepth)
1320 100969177 : loop_s = (*loop_s->superloops)[ddepth];
1321 :
1322 207293593 : while (loop_s != loop_d)
1323 : {
1324 913648 : loop_s = loop_outer (loop_s);
1325 913648 : loop_d = loop_outer (loop_d);
1326 : }
1327 : return loop_s;
1328 : }
1329 :
1330 : /* Removes LOOP from structures and frees its data. */
1331 :
1332 : void
1333 147620 : delete_loop (class loop *loop)
1334 : {
1335 : /* Remove the loop from structure. */
1336 147620 : flow_loop_tree_node_remove (loop);
1337 :
1338 : /* Remove loop from loops array. */
1339 147620 : (*current_loops->larray)[loop->num] = NULL;
1340 :
1341 : /* Free loop data. */
1342 147620 : flow_loop_free (loop);
1343 147620 : }
1344 :
1345 : /* Cancels the LOOP; it must be innermost one. */
1346 :
1347 : static void
1348 10645 : cancel_loop (class loop *loop)
1349 : {
1350 10645 : basic_block *bbs;
1351 10645 : unsigned i;
1352 10645 : class loop *outer = loop_outer (loop);
1353 :
1354 10645 : gcc_assert (!loop->inner);
1355 :
1356 : /* Move blocks up one level (they should be removed as soon as possible). */
1357 10645 : bbs = get_loop_body (loop);
1358 44091 : for (i = 0; i < loop->num_nodes; i++)
1359 22801 : bbs[i]->loop_father = outer;
1360 :
1361 10645 : free (bbs);
1362 10645 : delete_loop (loop);
1363 10645 : }
1364 :
1365 : /* Cancels LOOP and all its subloops. */
1366 : void
1367 10645 : cancel_loop_tree (class loop *loop)
1368 : {
1369 11133 : while (loop->inner)
1370 488 : cancel_loop_tree (loop->inner);
1371 10645 : cancel_loop (loop);
1372 10645 : }
1373 :
1374 : /* Disable warnings about missing quoting in GCC diagnostics for
1375 : the verification errors. Their format strings don't follow GCC
1376 : diagnostic conventions and the calls are ultimately followed by
1377 : a deliberate ICE triggered by a failed assertion. */
1378 : #if __GNUC__ >= 10
1379 : # pragma GCC diagnostic push
1380 : # pragma GCC diagnostic ignored "-Wformat-diag"
1381 : #endif
1382 :
1383 : /* Checks that information about loops is correct
1384 : -- sizes of loops are all right
1385 : -- results of get_loop_body really belong to the loop
1386 : -- loop header have just single entry edge and single latch edge
1387 : -- loop latches have only single successor that is header of their loop
1388 : -- irreducible loops are correctly marked
1389 : -- the cached loop depth and loop father of each bb is correct
1390 : */
1391 : DEBUG_FUNCTION void
1392 371047692 : verify_loop_structure (void)
1393 : {
1394 371047692 : unsigned *sizes, i, j;
1395 371047692 : basic_block bb, *bbs;
1396 371047692 : int err = 0;
1397 371047692 : edge e;
1398 371047692 : unsigned num = number_of_loops (cfun);
1399 371047692 : struct loop_exit *exit, *mexit;
1400 371047692 : bool dom_available = dom_info_available_p (CDI_DOMINATORS);
1401 :
1402 371047692 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1403 : {
1404 0 : error ("loop verification on loop tree that needs fixup");
1405 0 : err = 1;
1406 : }
1407 :
1408 : /* We need up-to-date dominators, compute or verify them. */
1409 371047692 : if (!dom_available)
1410 100010870 : calculate_dominance_info (CDI_DOMINATORS);
1411 : else
1412 271036822 : verify_dominators (CDI_DOMINATORS);
1413 :
1414 : /* Check the loop tree root. */
1415 371047692 : if (current_loops->tree_root->header != ENTRY_BLOCK_PTR_FOR_FN (cfun)
1416 371047692 : || current_loops->tree_root->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
1417 371047692 : || (current_loops->tree_root->num_nodes
1418 371047692 : != (unsigned) n_basic_blocks_for_fn (cfun)))
1419 : {
1420 0 : error ("corrupt loop tree root");
1421 0 : err = 1;
1422 : }
1423 :
1424 : /* Check the headers. */
1425 3712302682 : FOR_EACH_BB_FN (bb, cfun)
1426 3341254990 : if (bb_loop_header_p (bb))
1427 : {
1428 198017494 : if (bb->loop_father->header == NULL)
1429 : {
1430 0 : error ("loop with header %d marked for removal", bb->index);
1431 0 : err = 1;
1432 : }
1433 198017494 : else if (bb->loop_father->header != bb)
1434 : {
1435 0 : error ("loop with header %d not in loop tree", bb->index);
1436 0 : err = 1;
1437 : }
1438 : }
1439 3143237496 : else if (bb->loop_father->header == bb)
1440 : {
1441 0 : error ("non-loop with header %d not marked for removal", bb->index);
1442 0 : err = 1;
1443 : }
1444 :
1445 : /* Check the recorded loop father and sizes of loops. */
1446 371047692 : auto_sbitmap visited (last_basic_block_for_fn (cfun));
1447 371047692 : bitmap_clear (visited);
1448 371047692 : bbs = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
1449 1311160570 : for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))
1450 : {
1451 198017494 : unsigned n;
1452 :
1453 198017494 : if (loop->header == NULL)
1454 : {
1455 0 : error ("removed loop %d in loop tree", loop->num);
1456 0 : err = 1;
1457 0 : continue;
1458 : }
1459 :
1460 198017494 : n = get_loop_body_with_size (loop, bbs, n_basic_blocks_for_fn (cfun));
1461 198017494 : if (loop->num_nodes != n)
1462 : {
1463 0 : error ("size of loop %d should be %d, not %d",
1464 : loop->num, n, loop->num_nodes);
1465 0 : err = 1;
1466 : }
1467 :
1468 1309567936 : for (j = 0; j < n; j++)
1469 : {
1470 1111550442 : bb = bbs[j];
1471 :
1472 1111550442 : if (!flow_bb_inside_loop_p (loop, bb))
1473 : {
1474 0 : error ("bb %d does not belong to loop %d",
1475 : bb->index, loop->num);
1476 0 : err = 1;
1477 : }
1478 :
1479 : /* Ignore this block if it is in an inner loop. */
1480 1111550442 : if (bitmap_bit_p (visited, bb->index))
1481 261301646 : continue;
1482 850248796 : bitmap_set_bit (visited, bb->index);
1483 :
1484 850248796 : if (bb->loop_father != loop)
1485 : {
1486 0 : error ("bb %d has father loop %d, should be loop %d",
1487 : bb->index, bb->loop_father->num, loop->num);
1488 0 : err = 1;
1489 : }
1490 : }
1491 371047692 : }
1492 371047692 : free (bbs);
1493 :
1494 : /* Check headers and latches. */
1495 1311160570 : for (auto loop : loops_list (cfun, 0))
1496 : {
1497 198017494 : i = loop->num;
1498 198017494 : if (loop->header == NULL)
1499 0 : continue;
1500 198017494 : if (!bb_loop_header_p (loop->header))
1501 : {
1502 0 : error ("loop %d%'s header is not a loop header", i);
1503 0 : err = 1;
1504 : }
1505 198017494 : if (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
1506 198017494 : && EDGE_COUNT (loop->header->preds) != 2)
1507 : {
1508 0 : error ("loop %d%'s header does not have exactly 2 entries", i);
1509 0 : err = 1;
1510 : }
1511 198017494 : if (loop->latch)
1512 : {
1513 197126833 : if (!find_edge (loop->latch, loop->header))
1514 : {
1515 0 : error ("loop %d%'s latch does not have an edge to its header", i);
1516 0 : err = 1;
1517 : }
1518 197126833 : if (!dominated_by_p (CDI_DOMINATORS, loop->latch, loop->header))
1519 : {
1520 0 : error ("loop %d%'s latch is not dominated by its header", i);
1521 0 : err = 1;
1522 : }
1523 : }
1524 198017494 : if (loops_state_satisfies_p (LOOPS_HAVE_SIMPLE_LATCHES))
1525 : {
1526 30506460 : if (!single_succ_p (loop->latch))
1527 : {
1528 0 : error ("loop %d%'s latch does not have exactly 1 successor", i);
1529 0 : err = 1;
1530 : }
1531 30506460 : if (single_succ (loop->latch) != loop->header)
1532 : {
1533 0 : error ("loop %d%'s latch does not have header as successor", i);
1534 0 : err = 1;
1535 : }
1536 30506460 : if (loop->latch->loop_father != loop)
1537 : {
1538 0 : error ("loop %d%'s latch does not belong directly to it", i);
1539 0 : err = 1;
1540 : }
1541 : }
1542 198017494 : if (loop->header->loop_father != loop)
1543 : {
1544 0 : error ("loop %d%'s header does not belong directly to it", i);
1545 0 : err = 1;
1546 : }
1547 198017494 : if (loops_state_satisfies_p (LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS))
1548 : {
1549 27208644 : edge_iterator ei;
1550 81626194 : FOR_EACH_EDGE (e, ei, loop->header->preds)
1551 54417550 : if (dominated_by_p (CDI_DOMINATORS, e->src, loop->header)
1552 54417550 : && e->flags & EDGE_IRREDUCIBLE_LOOP)
1553 : {
1554 0 : error ("loop %d%'s latch is marked as part of irreducible"
1555 : " region", i);
1556 0 : err = 1;
1557 : }
1558 : }
1559 :
1560 : /* Check cached number of iterations for released SSA names. */
1561 198017494 : tree ref;
1562 198017494 : if (loop->nb_iterations
1563 198017494 : && (ref = walk_tree (&loop->nb_iterations,
1564 : find_released_ssa_name, NULL, NULL)))
1565 : {
1566 0 : error ("loop %d%'s number of iterations %qE references the"
1567 : " released SSA name %qE", i, loop->nb_iterations, ref);
1568 0 : err = 1;
1569 : }
1570 371047692 : }
1571 :
1572 : /* Check irreducible loops. */
1573 371047692 : if (loops_state_satisfies_p (LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS))
1574 : {
1575 33572638 : auto_edge_flag saved_edge_irr (cfun);
1576 33572638 : auto_bb_flag saved_bb_irr (cfun);
1577 : /* Save old info. */
1578 435673712 : FOR_EACH_BB_FN (bb, cfun)
1579 : {
1580 402101074 : edge_iterator ei;
1581 402101074 : if (bb->flags & BB_IRREDUCIBLE_LOOP)
1582 582259 : bb->flags |= saved_bb_irr;
1583 966803207 : FOR_EACH_EDGE (e, ei, bb->succs)
1584 564702133 : if (e->flags & EDGE_IRREDUCIBLE_LOOP)
1585 835405 : e->flags |= saved_edge_irr;
1586 : }
1587 :
1588 : /* Recount it. */
1589 33572638 : mark_irreducible_loops ();
1590 :
1591 : /* Compare. */
1592 435673712 : FOR_EACH_BB_FN (bb, cfun)
1593 : {
1594 402101074 : edge_iterator ei;
1595 :
1596 402101074 : if ((bb->flags & BB_IRREDUCIBLE_LOOP)
1597 402101074 : && !(bb->flags & saved_bb_irr))
1598 : {
1599 0 : error ("basic block %d should be marked irreducible", bb->index);
1600 0 : err = 1;
1601 : }
1602 402101074 : else if (!(bb->flags & BB_IRREDUCIBLE_LOOP)
1603 402101074 : && (bb->flags & saved_bb_irr))
1604 : {
1605 0 : error ("basic block %d should not be marked irreducible", bb->index);
1606 0 : err = 1;
1607 : }
1608 402101074 : bb->flags &= ~saved_bb_irr;
1609 966803207 : FOR_EACH_EDGE (e, ei, bb->succs)
1610 : {
1611 564702133 : if ((e->flags & EDGE_IRREDUCIBLE_LOOP)
1612 564702133 : && !(e->flags & saved_edge_irr))
1613 : {
1614 0 : error ("edge from %d to %d should be marked irreducible",
1615 0 : e->src->index, e->dest->index);
1616 0 : err = 1;
1617 : }
1618 564702133 : else if (!(e->flags & EDGE_IRREDUCIBLE_LOOP)
1619 564702133 : && (e->flags & saved_edge_irr))
1620 : {
1621 0 : error ("edge from %d to %d should not be marked irreducible",
1622 0 : e->src->index, e->dest->index);
1623 0 : err = 1;
1624 : }
1625 564702133 : e->flags &= ~saved_edge_irr;
1626 : }
1627 : }
1628 33572638 : }
1629 :
1630 : /* Check the recorded loop exits. */
1631 1311160570 : for (auto loop : loops_list (cfun, 0))
1632 : {
1633 198017494 : if (!loop->exits || loop->exits->e != NULL)
1634 : {
1635 0 : error ("corrupted head of the exits list of loop %d",
1636 : loop->num);
1637 0 : err = 1;
1638 : }
1639 : else
1640 : {
1641 : /* Check that the list forms a cycle, and all elements except
1642 : for the head are nonnull. */
1643 198017494 : for (mexit = loop->exits, exit = mexit->next, i = 0;
1644 242365374 : exit->e && exit != mexit;
1645 44347880 : exit = exit->next)
1646 : {
1647 44347880 : if (i++ & 1)
1648 13209368 : mexit = mexit->next;
1649 : }
1650 :
1651 198017494 : if (exit != loop->exits)
1652 : {
1653 0 : error ("corrupted exits list of loop %d", loop->num);
1654 0 : err = 1;
1655 : }
1656 : }
1657 :
1658 198017494 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1659 : {
1660 173740480 : if (loop->exits->next != loop->exits)
1661 : {
1662 0 : error ("nonempty exits list of loop %d, but exits are not recorded",
1663 : loop->num);
1664 0 : err = 1;
1665 : }
1666 : }
1667 371047692 : }
1668 :
1669 371047692 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1670 : {
1671 25581237 : unsigned n_exits = 0, eloops;
1672 :
1673 25581237 : sizes = XCNEWVEC (unsigned, num);
1674 25581237 : memset (sizes, 0, sizeof (unsigned) * num);
1675 365922379 : FOR_EACH_BB_FN (bb, cfun)
1676 : {
1677 340341142 : edge_iterator ei;
1678 340341142 : if (bb->loop_father == current_loops->tree_root)
1679 227166860 : continue;
1680 289299140 : FOR_EACH_EDGE (e, ei, bb->succs)
1681 : {
1682 176124858 : if (flow_bb_inside_loop_p (bb->loop_father, e->dest))
1683 134833737 : continue;
1684 :
1685 41291121 : n_exits++;
1686 41291121 : exit = get_exit_descriptions (e);
1687 41291121 : if (!exit)
1688 : {
1689 0 : error ("exit %d->%d not recorded",
1690 0 : e->src->index, e->dest->index);
1691 0 : err = 1;
1692 : }
1693 41291121 : eloops = 0;
1694 85639001 : for (; exit; exit = exit->next_e)
1695 44347880 : eloops++;
1696 :
1697 41291121 : for (class loop *loop = bb->loop_father;
1698 85639001 : loop != e->dest->loop_father
1699 : /* When a loop exit is also an entry edge which
1700 : can happen when avoiding CFG manipulations
1701 : then the last loop exited is the outer loop
1702 : of the loop entered. */
1703 85639001 : && loop != loop_outer (e->dest->loop_father);
1704 44347880 : loop = loop_outer (loop))
1705 : {
1706 44347880 : eloops--;
1707 44347880 : sizes[loop->num]++;
1708 : }
1709 :
1710 41291121 : if (eloops != 0)
1711 : {
1712 0 : error ("wrong list of exited loops for edge %d->%d",
1713 0 : e->src->index, e->dest->index);
1714 0 : err = 1;
1715 : }
1716 : }
1717 : }
1718 :
1719 25581237 : if (n_exits != current_loops->exits->elements ())
1720 : {
1721 0 : error ("too many loop exits recorded");
1722 0 : err = 1;
1723 : }
1724 :
1725 101020725 : for (auto loop : loops_list (cfun, 0))
1726 : {
1727 24277014 : eloops = 0;
1728 68624894 : for (exit = loop->exits->next; exit->e; exit = exit->next)
1729 44347880 : eloops++;
1730 24277014 : if (eloops != sizes[loop->num])
1731 : {
1732 0 : error ("%d exits recorded for loop %d (having %d exits)",
1733 : eloops, loop->num, sizes[loop->num]);
1734 0 : err = 1;
1735 : }
1736 25581237 : }
1737 :
1738 25581237 : free (sizes);
1739 : }
1740 :
1741 371047692 : gcc_assert (!err);
1742 :
1743 371047692 : if (!dom_available)
1744 100010870 : free_dominance_info (CDI_DOMINATORS);
1745 371047692 : }
1746 :
1747 : #if __GNUC__ >= 10
1748 : # pragma GCC diagnostic pop
1749 : #endif
1750 :
1751 : /* Returns latch edge of LOOP. */
1752 : edge
1753 12862755 : loop_latch_edge (const class loop *loop)
1754 : {
1755 12862755 : return find_edge (loop->latch, loop->header);
1756 : }
1757 :
1758 : /* Returns preheader edge of LOOP. */
1759 : edge
1760 431275170 : loop_preheader_edge (const class loop *loop)
1761 : {
1762 431275170 : edge e;
1763 431275170 : edge_iterator ei;
1764 :
1765 431275170 : gcc_assert (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
1766 : && ! loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
1767 :
1768 658111911 : FOR_EACH_EDGE (e, ei, loop->header->preds)
1769 657277351 : if (e->src != loop->latch)
1770 : break;
1771 :
1772 431275170 : if (! e)
1773 : {
1774 834560 : gcc_assert (! loop_outer (loop));
1775 834560 : return single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1776 : }
1777 :
1778 : return e;
1779 : }
1780 :
1781 : /* Returns true if E is an exit of LOOP. */
1782 :
1783 : bool
1784 36307119 : loop_exit_edge_p (const class loop *loop, const_edge e)
1785 : {
1786 36307119 : return (flow_bb_inside_loop_p (loop, e->src)
1787 36307119 : && !flow_bb_inside_loop_p (loop, e->dest));
1788 : }
1789 :
1790 : /* Returns the single exit edge of LOOP, or NULL if LOOP has either no exit
1791 : or more than one exit. If loops do not have the exits recorded, NULL
1792 : is returned always. */
1793 :
1794 : edge
1795 35418612 : single_exit (const class loop *loop)
1796 : {
1797 35418612 : struct loop_exit *exit = loop->exits->next;
1798 :
1799 35418612 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1800 : return NULL;
1801 :
1802 35389088 : if (exit->e && exit->next == loop->exits)
1803 : return exit->e;
1804 : else
1805 10654614 : return NULL;
1806 : }
1807 :
1808 : /* Returns incoming edge when BB has an incoming edge exiting LOOP, else return
1809 : NULL. */
1810 :
1811 : edge
1812 0 : loop_exits_to_bb_p (class loop *loop, basic_block bb)
1813 : {
1814 0 : edge e;
1815 0 : edge_iterator ei;
1816 :
1817 0 : FOR_EACH_EDGE (e, ei, bb->preds)
1818 0 : if (loop_exit_edge_p (loop, e))
1819 : return e;
1820 :
1821 : return NULL;
1822 : }
1823 :
1824 : /* Returns outgoing edge when BB has an outgoing edge exiting LOOP, else return
1825 : NULL. */
1826 :
1827 : edge
1828 1453589 : loop_exits_from_bb_p (class loop *loop, basic_block bb)
1829 : {
1830 1453589 : edge e;
1831 1453589 : edge_iterator ei;
1832 :
1833 3040579 : FOR_EACH_EDGE (e, ei, bb->succs)
1834 2545621 : if (loop_exit_edge_p (loop, e))
1835 : return e;
1836 :
1837 : return NULL;
1838 : }
1839 :
1840 : /* Return location corresponding to the loop control condition if possible. */
1841 :
1842 : dump_user_location_t
1843 591598 : get_loop_location (class loop *loop)
1844 : {
1845 591598 : rtx_insn *insn = NULL;
1846 591598 : class niter_desc *desc = NULL;
1847 591598 : edge exit;
1848 :
1849 : /* For a for or while loop, we would like to return the location
1850 : of the for or while statement, if possible. To do this, look
1851 : for the branch guarding the loop back-edge. */
1852 :
1853 : /* If this is a simple loop with an in_edge, then the loop control
1854 : branch is typically at the end of its source. */
1855 591598 : desc = get_simple_loop_desc (loop);
1856 591598 : if (desc->in_edge)
1857 : {
1858 590428 : FOR_BB_INSNS_REVERSE (desc->in_edge->src, insn)
1859 : {
1860 585402 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1861 407521 : return insn;
1862 : }
1863 : }
1864 : /* If loop has a single exit, then the loop control branch
1865 : must be at the end of its source. */
1866 184077 : if ((exit = single_exit (loop)))
1867 : {
1868 206691 : FOR_BB_INSNS_REVERSE (exit->src, insn)
1869 : {
1870 198105 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1871 102064 : return insn;
1872 : }
1873 : }
1874 : /* Next check the latch, to see if it is non-empty. */
1875 193612 : FOR_BB_INSNS_REVERSE (loop->latch, insn)
1876 : {
1877 134475 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1878 22876 : return insn;
1879 : }
1880 : /* Finally, if none of the above identifies the loop control branch,
1881 : return the first location in the loop header. */
1882 283587 : FOR_BB_INSNS (loop->header, insn)
1883 : {
1884 271918 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1885 47468 : return insn;
1886 : }
1887 : /* If all else fails, simply return the current function location. */
1888 11669 : return dump_user_location_t::from_function_decl (current_function_decl);
1889 : }
1890 :
1891 : /* Records that every statement in LOOP is executed I_BOUND times.
1892 : REALISTIC is true if I_BOUND is expected to be close to the real number
1893 : of iterations. UPPER is true if we are sure the loop iterates at most
1894 : I_BOUND times. */
1895 :
1896 : void
1897 17328496 : record_niter_bound (class loop *loop, const widest_int &i_bound,
1898 : bool realistic, bool upper)
1899 : {
1900 17328496 : if (wi::min_precision (i_bound, SIGNED) > bound_wide_int ().get_precision ())
1901 0 : return;
1902 :
1903 17328496 : bound_wide_int bound = bound_wide_int::from (i_bound, SIGNED);
1904 :
1905 : /* Update the bounds only when there is no previous estimation, or when the
1906 : current estimation is smaller. */
1907 17328496 : if (upper
1908 17328496 : && (!loop->any_upper_bound
1909 15863587 : || wi::ltu_p (bound, loop->nb_iterations_upper_bound)))
1910 : {
1911 880616 : loop->any_upper_bound = true;
1912 880616 : loop->nb_iterations_upper_bound = bound;
1913 880616 : if (!loop->any_likely_upper_bound)
1914 : {
1915 508137 : loop->any_likely_upper_bound = true;
1916 508137 : loop->nb_iterations_likely_upper_bound = bound;
1917 : }
1918 : }
1919 17328496 : if (realistic
1920 17328496 : && (!loop->any_estimate
1921 2816703 : || wi::ltu_p (bound, loop->nb_iterations_estimate)))
1922 : {
1923 239186 : loop->any_estimate = true;
1924 239186 : loop->nb_iterations_estimate = bound;
1925 : }
1926 17328496 : if (!realistic
1927 17328496 : && (!loop->any_likely_upper_bound
1928 14277830 : || wi::ltu_p (bound, loop->nb_iterations_likely_upper_bound)))
1929 : {
1930 244814 : loop->any_likely_upper_bound = true;
1931 244814 : loop->nb_iterations_likely_upper_bound = bound;
1932 : }
1933 :
1934 : /* If an upper bound is smaller than the realistic estimate of the
1935 : number of iterations, use the upper bound instead. */
1936 17328496 : if (loop->any_upper_bound
1937 17324417 : && loop->any_estimate
1938 26417964 : && wi::ltu_p (loop->nb_iterations_upper_bound,
1939 9089468 : loop->nb_iterations_estimate))
1940 6353 : loop->nb_iterations_estimate = loop->nb_iterations_upper_bound;
1941 17328496 : if (loop->any_upper_bound
1942 17324417 : && loop->any_likely_upper_bound
1943 34652913 : && wi::ltu_p (loop->nb_iterations_upper_bound,
1944 17324417 : loop->nb_iterations_likely_upper_bound))
1945 37606 : loop->nb_iterations_likely_upper_bound = loop->nb_iterations_upper_bound;
1946 : }
1947 :
1948 : /* Similar to get_estimated_loop_iterations, but returns the estimate only
1949 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
1950 : on the number of iterations of LOOP could not be derived, returns -1. */
1951 :
1952 : HOST_WIDE_INT
1953 231 : get_estimated_loop_iterations_int (class loop *loop)
1954 : {
1955 231 : widest_int nit;
1956 231 : HOST_WIDE_INT hwi_nit;
1957 :
1958 231 : if (!get_estimated_loop_iterations (loop, &nit))
1959 : return -1;
1960 :
1961 46 : if (!wi::fits_shwi_p (nit))
1962 : return -1;
1963 46 : hwi_nit = nit.to_shwi ();
1964 :
1965 46 : return hwi_nit < 0 ? -1 : hwi_nit;
1966 231 : }
1967 :
1968 : /* Returns an upper bound on the number of executions of statements
1969 : in the LOOP. For statements before the loop exit, this exceeds
1970 : the number of execution of the latch by one. */
1971 :
1972 : HOST_WIDE_INT
1973 290739 : max_stmt_executions_int (class loop *loop)
1974 : {
1975 290739 : HOST_WIDE_INT nit = get_max_loop_iterations_int (loop);
1976 290739 : HOST_WIDE_INT snit;
1977 :
1978 290739 : if (nit == -1)
1979 : return -1;
1980 :
1981 266822 : snit = (HOST_WIDE_INT) ((unsigned HOST_WIDE_INT) nit + 1);
1982 :
1983 : /* If the computation overflows, return -1. */
1984 266822 : return snit < 0 ? -1 : snit;
1985 : }
1986 :
1987 : /* Returns an likely upper bound on the number of executions of statements
1988 : in the LOOP. For statements before the loop exit, this exceeds
1989 : the number of execution of the latch by one. */
1990 :
1991 : HOST_WIDE_INT
1992 4963805 : likely_max_stmt_executions_int (class loop *loop)
1993 : {
1994 4963805 : HOST_WIDE_INT nit = get_likely_max_loop_iterations_int (loop);
1995 4963805 : HOST_WIDE_INT snit;
1996 :
1997 4963805 : if (nit == -1)
1998 : return -1;
1999 :
2000 4184021 : snit = (HOST_WIDE_INT) ((unsigned HOST_WIDE_INT) nit + 1);
2001 :
2002 : /* If the computation overflows, return -1. */
2003 4184021 : return snit < 0 ? -1 : snit;
2004 : }
2005 :
2006 : /* Sets NIT to the estimated number of executions of the latch of the
2007 : LOOP. If we have no reliable estimate, the function returns false, otherwise
2008 : returns true. */
2009 :
2010 : bool
2011 9266191 : get_estimated_loop_iterations (class loop *loop, widest_int *nit)
2012 : {
2013 : /* Even if the bound is not recorded, possibly we can derrive one from
2014 : profile. */
2015 9266191 : if (!loop->any_estimate)
2016 : {
2017 5347515 : sreal snit;
2018 5347515 : bool reliable;
2019 5347515 : if (expected_loop_iterations_by_profile (loop, &snit, &reliable)
2020 5347515 : && reliable)
2021 : {
2022 3 : *nit = snit.to_nearest_int ();
2023 3 : return true;
2024 : }
2025 : return false;
2026 : }
2027 :
2028 3918676 : *nit = widest_int::from (loop->nb_iterations_estimate, SIGNED);
2029 3918676 : return true;
2030 : }
2031 :
2032 : /* Sets NIT to an upper bound for the maximum number of executions of the
2033 : latch of the LOOP. If we have no reliable estimate, the function returns
2034 : false, otherwise returns true. */
2035 :
2036 : bool
2037 23024912 : get_max_loop_iterations (const class loop *loop, widest_int *nit)
2038 : {
2039 23024912 : if (!loop->any_upper_bound)
2040 : return false;
2041 :
2042 20465932 : *nit = widest_int::from (loop->nb_iterations_upper_bound, SIGNED);
2043 20465932 : return true;
2044 : }
2045 :
2046 : /* Similar to get_max_loop_iterations, but returns the estimate only
2047 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
2048 : on the number of iterations of LOOP could not be derived, returns -1. */
2049 :
2050 : HOST_WIDE_INT
2051 1743873 : get_max_loop_iterations_int (const class loop *loop)
2052 : {
2053 1743873 : widest_int nit;
2054 1743873 : HOST_WIDE_INT hwi_nit;
2055 :
2056 1743873 : if (!get_max_loop_iterations (loop, &nit))
2057 : return -1;
2058 :
2059 1355483 : if (!wi::fits_shwi_p (nit))
2060 : return -1;
2061 1293164 : hwi_nit = nit.to_shwi ();
2062 :
2063 1293164 : return hwi_nit < 0 ? -1 : hwi_nit;
2064 1743873 : }
2065 :
2066 : /* Sets NIT to an upper bound for the maximum number of executions of the
2067 : latch of the LOOP. If we have no reliable estimate, the function returns
2068 : false, otherwise returns true. */
2069 :
2070 : bool
2071 5408457 : get_likely_max_loop_iterations (class loop *loop, widest_int *nit)
2072 : {
2073 5408457 : if (!loop->any_likely_upper_bound)
2074 : return false;
2075 :
2076 4795295 : *nit = widest_int::from (loop->nb_iterations_likely_upper_bound, SIGNED);
2077 4795295 : return true;
2078 : }
2079 :
2080 : /* Similar to get_max_loop_iterations, but returns the estimate only
2081 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
2082 : on the number of iterations of LOOP could not be derived, returns -1. */
2083 :
2084 : HOST_WIDE_INT
2085 5026402 : get_likely_max_loop_iterations_int (class loop *loop)
2086 : {
2087 5026402 : widest_int nit;
2088 5026402 : HOST_WIDE_INT hwi_nit;
2089 :
2090 5026402 : if (!get_likely_max_loop_iterations (loop, &nit))
2091 : return -1;
2092 :
2093 4545807 : if (!wi::fits_shwi_p (nit))
2094 : return -1;
2095 4246063 : hwi_nit = nit.to_shwi ();
2096 :
2097 4246063 : return hwi_nit < 0 ? -1 : hwi_nit;
2098 5026402 : }
2099 :
2100 : /* Returns the loop depth of the loop BB belongs to. */
2101 :
2102 : int
2103 55033935 : bb_loop_depth (const_basic_block bb)
2104 : {
2105 67247138 : return bb->loop_father ? loop_depth (bb->loop_father) : 0;
2106 : }
2107 :
2108 : /* Marks LOOP for removal and sets LOOPS_NEED_FIXUP. */
2109 :
2110 : void
2111 279665 : mark_loop_for_removal (loop_p loop)
2112 : {
2113 279665 : if (loop->header == NULL)
2114 : return;
2115 279665 : loop->former_header = loop->header;
2116 279665 : loop->header = NULL;
2117 279665 : loop->latch = NULL;
2118 279665 : loops_state_set (LOOPS_NEED_FIXUP);
2119 : }
2120 :
2121 : /* Starting from loop tree ROOT, walk loop tree as the visiting
2122 : order specified by FLAGS. The supported visiting orders
2123 : are:
2124 : - LI_ONLY_INNERMOST
2125 : - LI_FROM_INNERMOST
2126 : - Preorder (if neither of above is specified) */
2127 :
2128 : void
2129 1364430266 : loops_list::walk_loop_tree (class loop *root, unsigned flags)
2130 : {
2131 1364430266 : bool only_innermost_p = flags & LI_ONLY_INNERMOST;
2132 1364430266 : bool from_innermost_p = flags & LI_FROM_INNERMOST;
2133 1364430266 : bool preorder_p = !(only_innermost_p || from_innermost_p);
2134 :
2135 : /* Early handle root without any inner loops, make later
2136 : processing simpler, that is all loops processed in the
2137 : following while loop are impossible to be root. */
2138 1364430266 : if (!root->inner)
2139 : {
2140 1100317025 : if (flags & LI_INCLUDE_ROOT)
2141 11878 : this->to_visit.quick_push (root->num);
2142 1100317025 : return;
2143 : }
2144 264113241 : else if (preorder_p && flags & LI_INCLUDE_ROOT)
2145 50959 : this->to_visit.quick_push (root->num);
2146 :
2147 : class loop *aloop;
2148 68548446 : for (aloop = root->inner;
2149 332661687 : aloop->inner != NULL;
2150 68548446 : aloop = aloop->inner)
2151 : {
2152 68548446 : if (preorder_p)
2153 49762210 : this->to_visit.quick_push (aloop->num);
2154 68548446 : continue;
2155 : }
2156 :
2157 919987480 : while (1)
2158 : {
2159 919987480 : gcc_assert (aloop != root);
2160 919987480 : if (from_innermost_p || aloop->inner == NULL)
2161 790648837 : this->to_visit.quick_push (aloop->num);
2162 :
2163 919987480 : if (aloop->next)
2164 : {
2165 100148049 : for (aloop = aloop->next;
2166 587325793 : aloop->inner != NULL;
2167 100148049 : aloop = aloop->inner)
2168 : {
2169 100148049 : if (preorder_p)
2170 79576433 : this->to_visit.quick_push (aloop->num);
2171 100148049 : continue;
2172 : }
2173 : }
2174 432809736 : else if (loop_outer (aloop) == root)
2175 : break;
2176 : else
2177 : aloop = loop_outer (aloop);
2178 : }
2179 :
2180 : /* When visiting from innermost, we need to consider root here
2181 : since the previous while loop doesn't handle it. */
2182 264113241 : if (from_innermost_p && flags & LI_INCLUDE_ROOT)
2183 0 : this->to_visit.quick_push (root->num);
2184 68548446 : }
2185 :
|
