Line data Source code
1 : /* Natural loop discovery code for GNU compiler.
2 : Copyright (C) 2000-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "rtl.h"
25 : #include "tree.h"
26 : #include "gimple.h"
27 : #include "cfghooks.h"
28 : #include "gimple-ssa.h"
29 : #include "diagnostic-core.h"
30 : #include "cfganal.h"
31 : #include "cfgloop.h"
32 : #include "gimple-iterator.h"
33 : #include "dumpfile.h"
34 : #include "tree-ssa.h"
35 : #include "tree-pretty-print.h"
36 : #include "sreal.h"
37 :
38 : static void flow_loops_cfg_dump (FILE *);
39 :
40 : /* Dump loop related CFG information. */
41 :
42 : static void
43 5957 : flow_loops_cfg_dump (FILE *file)
44 : {
45 5957 : basic_block bb;
46 :
47 5957 : if (!file)
48 : return;
49 :
50 27296 : FOR_EACH_BB_FN (bb, cfun)
51 : {
52 21339 : edge succ;
53 21339 : edge_iterator ei;
54 :
55 21339 : fprintf (file, ";; %d succs { ", bb->index);
56 49480 : FOR_EACH_EDGE (succ, ei, bb->succs)
57 28141 : fprintf (file, "%d ", succ->dest->index);
58 21339 : fprintf (file, "}\n");
59 : }
60 : }
61 :
62 : /* Return nonzero if the nodes of LOOP are a subset of OUTER. */
63 :
64 : bool
65 772815834 : flow_loop_nested_p (const class loop *outer, const class loop *loop)
66 : {
67 772815834 : unsigned odepth = loop_depth (outer);
68 :
69 772815834 : return (loop_depth (loop) > odepth
70 505328637 : && (*loop->superloops)[odepth] == outer);
71 : }
72 :
73 : /* Returns the loop such that LOOP is nested DEPTH (indexed from zero)
74 : loops within LOOP. */
75 :
76 : class loop *
77 22147294 : superloop_at_depth (class loop *loop, unsigned depth)
78 : {
79 22147294 : unsigned ldepth = loop_depth (loop);
80 :
81 22147294 : gcc_assert (depth <= ldepth);
82 :
83 22147294 : if (depth == ldepth)
84 : return loop;
85 :
86 5767342 : return (*loop->superloops)[depth];
87 : }
88 :
89 : /* Returns the list of the latch edges of LOOP. */
90 :
91 : static vec<edge>
92 48241 : get_loop_latch_edges (const class loop *loop)
93 : {
94 48241 : edge_iterator ei;
95 48241 : edge e;
96 48241 : vec<edge> ret = vNULL;
97 :
98 205356 : FOR_EACH_EDGE (e, ei, loop->header->preds)
99 : {
100 157115 : if (dominated_by_p (CDI_DOMINATORS, e->src, loop->header))
101 106961 : ret.safe_push (e);
102 : }
103 :
104 48241 : return ret;
105 : }
106 :
107 : /* Dump the loop information specified by LOOP to the stream FILE
108 : using auxiliary dump callback function LOOP_DUMP_AUX if non null. */
109 :
110 : void
111 7582 : flow_loop_dump (const class loop *loop, FILE *file,
112 : void (*loop_dump_aux) (const class loop *, FILE *, int),
113 : int verbose)
114 : {
115 7582 : basic_block *bbs;
116 7582 : unsigned i;
117 7582 : vec<edge> latches;
118 7582 : edge e;
119 :
120 7582 : if (! loop || ! loop->header)
121 0 : return;
122 :
123 7582 : fprintf (file, ";;\n;; Loop %d\n", loop->num);
124 :
125 7582 : fprintf (file, ";; header %d, ", loop->header->index);
126 7582 : if (loop->latch)
127 7578 : fprintf (file, "latch %d\n", loop->latch->index);
128 : else
129 : {
130 4 : fprintf (file, "multiple latches:");
131 4 : latches = get_loop_latch_edges (loop);
132 16 : FOR_EACH_VEC_ELT (latches, i, e)
133 8 : fprintf (file, " %d", e->src->index);
134 4 : latches.release ();
135 4 : fprintf (file, "\n");
136 : }
137 :
138 15164 : fprintf (file, ";; depth %d, outer %ld",
139 7582 : loop_depth (loop), (long) (loop_outer (loop)
140 1597 : ? loop_outer (loop)->num : -1));
141 7582 : print_loop_info (file, loop, ";; ");
142 :
143 7582 : fprintf (file, "\n;; nodes:");
144 7582 : bbs = get_loop_body (loop);
145 53854 : for (i = 0; i < loop->num_nodes; i++)
146 38690 : fprintf (file, " %d", bbs[i]->index);
147 7582 : free (bbs);
148 7582 : fprintf (file, "\n");
149 :
150 7582 : if (loop_dump_aux)
151 0 : loop_dump_aux (loop, file, verbose);
152 : }
153 :
154 : /* Dump the loop information about loops to the stream FILE,
155 : using auxiliary dump callback function LOOP_DUMP_AUX if non null. */
156 :
157 : void
158 49988542 : flow_loops_dump (FILE *file, void (*loop_dump_aux) (const class loop *, FILE *, int), int verbose)
159 : {
160 49988542 : if (!current_loops || ! file)
161 : return;
162 :
163 11970 : fprintf (file, ";; %d loops found\n", number_of_loops (cfun));
164 :
165 25470 : for (auto loop : loops_list (cfun, LI_INCLUDE_ROOT))
166 : {
167 7515 : flow_loop_dump (loop, file, loop_dump_aux, verbose);
168 5985 : }
169 :
170 5985 : if (verbose)
171 5957 : flow_loops_cfg_dump (file);
172 : }
173 :
174 : /* Free data allocated for LOOP. */
175 :
176 : void
177 17369421 : flow_loop_free (class loop *loop)
178 : {
179 17369421 : struct loop_exit *exit, *next;
180 :
181 17369421 : vec_free (loop->superloops);
182 :
183 : /* Break the list of the loop exit records. They will be freed when the
184 : corresponding edge is rescanned or removed, and this avoids
185 : accessing the (already released) head of the list stored in the
186 : loop structure. */
187 17382739 : for (exit = loop->exits->next; exit != loop->exits; exit = next)
188 : {
189 13318 : next = exit->next;
190 13318 : exit->next = exit;
191 13318 : exit->prev = exit;
192 : }
193 :
194 17369421 : ggc_free (loop->exits);
195 17369421 : ggc_free (loop);
196 17369421 : }
197 :
198 : /* Free all the memory allocated for LOOPS. */
199 :
200 : void
201 10909295 : flow_loops_free (struct loops *loops)
202 : {
203 10909295 : if (loops->larray)
204 : {
205 : unsigned i;
206 : loop_p loop;
207 :
208 : /* Free the loop descriptors. */
209 28369561 : FOR_EACH_VEC_SAFE_ELT (loops->larray, i, loop)
210 : {
211 17460266 : if (!loop)
212 514220 : continue;
213 :
214 16946046 : flow_loop_free (loop);
215 : }
216 :
217 21818590 : vec_free (loops->larray);
218 : }
219 10909295 : }
220 :
221 : /* Find the nodes contained within the LOOP with header HEADER.
222 : Return the number of nodes within the loop. */
223 :
224 : int
225 22229721 : flow_loop_nodes_find (basic_block header, class loop *loop)
226 : {
227 22229721 : vec<basic_block> stack = vNULL;
228 22229721 : int num_nodes = 1;
229 22229721 : edge latch;
230 22229721 : edge_iterator latch_ei;
231 :
232 22229721 : header->loop_father = loop;
233 :
234 67461753 : FOR_EACH_EDGE (latch, latch_ei, loop->header->preds)
235 : {
236 73749728 : if (latch->src->loop_father == loop
237 45232032 : || !dominated_by_p (CDI_DOMINATORS, latch->src, loop->header))
238 28517696 : continue;
239 :
240 16714336 : num_nodes++;
241 16714336 : stack.safe_push (latch->src);
242 16714336 : latch->src->loop_father = loop;
243 :
244 169875360 : while (!stack.is_empty ())
245 : {
246 107928992 : basic_block node;
247 107928992 : edge e;
248 107928992 : edge_iterator ei;
249 :
250 107928992 : node = stack.pop ();
251 :
252 256851531 : FOR_EACH_EDGE (e, ei, node->preds)
253 : {
254 148922539 : basic_block ancestor = e->src;
255 :
256 148922539 : if (ancestor->loop_father != loop)
257 : {
258 91214656 : ancestor->loop_father = loop;
259 91214656 : num_nodes++;
260 91214656 : stack.safe_push (ancestor);
261 : }
262 : }
263 : }
264 : }
265 22229721 : stack.release ();
266 :
267 22229721 : return num_nodes;
268 : }
269 :
270 : /* Records the vector of superloops of the loop LOOP, whose immediate
271 : superloop is FATHER. */
272 :
273 : static void
274 23195442 : establish_preds (class loop *loop, class loop *father)
275 : {
276 23195442 : loop_p ploop;
277 23195442 : unsigned depth = loop_depth (father) + 1;
278 23195442 : unsigned i;
279 :
280 23195442 : loop->superloops = 0;
281 23195442 : vec_alloc (loop->superloops, depth);
282 55735328 : FOR_EACH_VEC_SAFE_ELT (father->superloops, i, ploop)
283 9344444 : loop->superloops->quick_push (ploop);
284 23195442 : loop->superloops->quick_push (father);
285 :
286 23281672 : for (ploop = loop->inner; ploop; ploop = ploop->next)
287 86230 : establish_preds (ploop, loop);
288 23195442 : }
289 :
290 : /* Add LOOP to the loop hierarchy tree where FATHER is father of the
291 : added loop. If LOOP has some children, take care of that their
292 : pred field will be initialized correctly. If AFTER is non-null
293 : then it's expected it's a pointer into FATHERs inner sibling
294 : list and LOOP is added behind AFTER, otherwise it's added in front
295 : of FATHERs siblings. */
296 :
297 : void
298 23109212 : flow_loop_tree_node_add (class loop *father, class loop *loop,
299 : class loop *after)
300 : {
301 23109212 : if (after)
302 : {
303 4817 : loop->next = after->next;
304 4817 : after->next = loop;
305 : }
306 : else
307 : {
308 23104395 : loop->next = father->inner;
309 23104395 : father->inner = loop;
310 : }
311 :
312 23109212 : establish_preds (loop, father);
313 23109212 : }
314 :
315 : /* Remove LOOP from the loop hierarchy tree. */
316 :
317 : void
318 17058014 : flow_loop_tree_node_remove (class loop *loop)
319 : {
320 17058014 : class loop *prev, *father;
321 :
322 17058014 : father = loop_outer (loop);
323 :
324 : /* Remove loop from the list of sons. */
325 17058014 : if (father->inner == loop)
326 7934736 : father->inner = loop->next;
327 : else
328 : {
329 233164447 : for (prev = father->inner; prev->next != loop; prev = prev->next)
330 224041169 : continue;
331 9123278 : prev->next = loop->next;
332 : }
333 :
334 17058014 : loop->superloops = NULL;
335 17058014 : }
336 :
337 : /* Allocates and returns new loop structure. */
338 :
339 : class loop *
340 17489686 : alloc_loop (void)
341 : {
342 17489686 : class loop *loop = ggc_cleared_alloc<class loop> ();
343 :
344 17489686 : loop->exits = ggc_cleared_alloc<loop_exit> ();
345 17489686 : loop->exits->next = loop->exits->prev = loop->exits;
346 17489686 : loop->can_be_parallel = false;
347 17489686 : loop->constraints = 0;
348 17489686 : loop->nb_iterations_upper_bound = 0;
349 17489686 : loop->nb_iterations_likely_upper_bound = 0;
350 17489686 : loop->nb_iterations_estimate = 0;
351 17489686 : return loop;
352 : }
353 :
354 : /* Initializes loops structure LOOPS, reserving place for NUM_LOOPS loops
355 : (including the root of the loop tree). */
356 :
357 : void
358 11015113 : init_loops_structure (struct function *fn,
359 : struct loops *loops, unsigned num_loops)
360 : {
361 11015113 : class loop *root;
362 :
363 11015113 : memset (loops, 0, sizeof *loops);
364 11015113 : vec_alloc (loops->larray, num_loops);
365 :
366 : /* Dummy loop containing whole function. */
367 11015113 : root = alloc_loop ();
368 11015113 : root->num_nodes = n_basic_blocks_for_fn (fn);
369 11015113 : root->latch = EXIT_BLOCK_PTR_FOR_FN (fn);
370 11015113 : root->header = ENTRY_BLOCK_PTR_FOR_FN (fn);
371 11015113 : ENTRY_BLOCK_PTR_FOR_FN (fn)->loop_father = root;
372 11015113 : EXIT_BLOCK_PTR_FOR_FN (fn)->loop_father = root;
373 :
374 11015113 : loops->larray->quick_push (root);
375 11015113 : loops->tree_root = root;
376 11015113 : }
377 :
378 : /* Returns whether HEADER is a loop header. */
379 :
380 : bool
381 3926543986 : bb_loop_header_p (basic_block header)
382 : {
383 3926543986 : edge_iterator ei;
384 3926543986 : edge e;
385 :
386 : /* If we have an abnormal predecessor, do not consider the
387 : loop (not worth the problems). */
388 3926543986 : if (bb_has_abnormal_pred (header))
389 : return false;
390 :
391 : /* Look for back edges where a predecessor is dominated
392 : by this block. A natural loop has a single entry
393 : node (header) that dominates all the nodes in the
394 : loop. It also has single back edge to the header
395 : from a latch node. */
396 8916068351 : FOR_EACH_EDGE (e, ei, header->preds)
397 : {
398 5433462660 : basic_block latch = e->src;
399 5433462660 : if (latch != ENTRY_BLOCK_PTR_FOR_FN (cfun)
400 5433462660 : && dominated_by_p (CDI_DOMINATORS, latch, header))
401 : return true;
402 : }
403 :
404 : return false;
405 : }
406 :
407 : /* Find all the natural loops in the function and save in LOOPS structure and
408 : recalculate loop_father information in basic block structures.
409 : If LOOPS is non-NULL then the loop structures for already recorded loops
410 : will be re-used and their number will not change. We assume that no
411 : stale loops exist in LOOPS.
412 : When LOOPS is NULL it is allocated and re-built from scratch.
413 : Return the built LOOPS structure. */
414 :
415 : struct loops *
416 21831572 : flow_loops_find (struct loops *loops)
417 : {
418 21831572 : bool from_scratch = (loops == NULL);
419 21831572 : int *rc_order;
420 21831572 : int b;
421 21831572 : unsigned i;
422 :
423 : /* Ensure that the dominators are computed. */
424 21831572 : calculate_dominance_info (CDI_DOMINATORS);
425 :
426 21831572 : if (!loops)
427 : {
428 10867732 : loops = ggc_cleared_alloc<struct loops> ();
429 10867732 : init_loops_structure (cfun, loops, 1);
430 : }
431 :
432 : /* Ensure that loop exits were released. */
433 21831572 : gcc_assert (loops->exits == NULL);
434 :
435 : /* Taking care of this degenerate case makes the rest of
436 : this code simpler. */
437 21831572 : if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS)
438 : return loops;
439 :
440 : /* The root loop node contains all basic-blocks. */
441 21593221 : loops->tree_root->num_nodes = n_basic_blocks_for_fn (cfun);
442 :
443 : /* Compute depth first search order of the CFG so that outer
444 : natural loops will be found before inner natural loops. */
445 21593221 : rc_order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
446 21593221 : pre_and_rev_post_order_compute (NULL, rc_order, false);
447 :
448 : /* Gather all loop headers in reverse completion order and allocate
449 : loop structures for loops that are not already present. */
450 21593221 : auto_vec<loop_p> larray (loops->larray->length ());
451 364345482 : for (b = 0; b < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; b++)
452 : {
453 342752261 : basic_block header = BASIC_BLOCK_FOR_FN (cfun, rc_order[b]);
454 342752261 : if (bb_loop_header_p (header))
455 : {
456 22229721 : class loop *loop;
457 :
458 : /* The current active loop tree has valid loop-fathers for
459 : header blocks. */
460 22229721 : if (!from_scratch
461 16620280 : && header->loop_father->header == header)
462 : {
463 16550997 : loop = header->loop_father;
464 : /* If we found an existing loop remove it from the
465 : loop tree. It is going to be inserted again
466 : below. */
467 16550997 : flow_loop_tree_node_remove (loop);
468 : }
469 : else
470 : {
471 : /* Otherwise allocate a new loop structure for the loop. */
472 5678724 : loop = alloc_loop ();
473 : /* ??? We could re-use unused loop slots here. */
474 5678724 : loop->num = loops->larray->length ();
475 5678724 : vec_safe_push (loops->larray, loop);
476 5678724 : loop->header = header;
477 :
478 5678724 : if (!from_scratch
479 5678724 : && dump_file && (dump_flags & TDF_DETAILS))
480 3 : fprintf (dump_file, "flow_loops_find: discovered new "
481 : "loop %d with header %d\n",
482 : loop->num, header->index);
483 : }
484 : /* Reset latch, we recompute it below. */
485 22229721 : loop->latch = NULL;
486 22229721 : larray.safe_push (loop);
487 : }
488 :
489 : /* Make blocks part of the loop root node at start. */
490 342752261 : header->loop_father = loops->tree_root;
491 : }
492 :
493 21593221 : free (rc_order);
494 :
495 : /* Now iterate over the loops found, insert them into the loop tree
496 : and assign basic-block ownership. */
497 43822942 : for (i = 0; i < larray.length (); ++i)
498 : {
499 22229721 : class loop *loop = larray[i];
500 22229721 : basic_block header = loop->header;
501 22229721 : edge_iterator ei;
502 22229721 : edge e;
503 :
504 22229721 : flow_loop_tree_node_add (header->loop_father, loop);
505 22229721 : loop->num_nodes = flow_loop_nodes_find (loop->header, loop);
506 :
507 : /* Look for the latch for this header block, if it has just a
508 : single one. */
509 67069144 : FOR_EACH_EDGE (e, ei, header->preds)
510 : {
511 45067413 : basic_block latch = e->src;
512 :
513 45067413 : if (flow_bb_inside_loop_p (loop, latch))
514 : {
515 22457711 : if (loop->latch != NULL)
516 : {
517 : /* More than one latch edge. */
518 227990 : loop->latch = NULL;
519 227990 : break;
520 : }
521 22229721 : loop->latch = latch;
522 : }
523 : }
524 : }
525 :
526 21593221 : return loops;
527 21593221 : }
528 :
529 : /* qsort helper for sort_sibling_loops. */
530 :
531 : static int *sort_sibling_loops_cmp_rpo;
532 : static int
533 2635 : sort_sibling_loops_cmp (const void *la_, const void *lb_)
534 : {
535 2635 : const class loop *la = *(const class loop * const *)la_;
536 2635 : const class loop *lb = *(const class loop * const *)lb_;
537 2635 : return (sort_sibling_loops_cmp_rpo[la->header->index]
538 2635 : - sort_sibling_loops_cmp_rpo[lb->header->index]);
539 : }
540 :
541 : /* Sort sibling loops in RPO order. */
542 :
543 : void
544 556 : sort_sibling_loops (function *fn)
545 : {
546 : /* Match flow_loops_find in the order we sort sibling loops. */
547 556 : sort_sibling_loops_cmp_rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
548 556 : int *rc_order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
549 556 : pre_and_rev_post_order_compute_fn (fn, NULL, rc_order, false);
550 7570 : for (int i = 0; i < n_basic_blocks_for_fn (cfun) - NUM_FIXED_BLOCKS; ++i)
551 7014 : sort_sibling_loops_cmp_rpo[rc_order[i]] = i;
552 556 : free (rc_order);
553 :
554 556 : auto_vec<loop_p, 3> siblings;
555 3597 : for (auto loop : loops_list (fn, LI_INCLUDE_ROOT))
556 1929 : if (loop->inner && loop->inner->next)
557 : {
558 224 : loop_p sibling = loop->inner;
559 598 : do
560 : {
561 598 : siblings.safe_push (sibling);
562 598 : sibling = sibling->next;
563 : }
564 598 : while (sibling);
565 224 : siblings.qsort (sort_sibling_loops_cmp);
566 224 : loop_p *siblingp = &loop->inner;
567 822 : for (unsigned i = 0; i < siblings.length (); ++i)
568 : {
569 598 : *siblingp = siblings[i];
570 598 : siblingp = &(*siblingp)->next;
571 : }
572 224 : *siblingp = NULL;
573 224 : siblings.truncate (0);
574 556 : }
575 :
576 556 : free (sort_sibling_loops_cmp_rpo);
577 556 : sort_sibling_loops_cmp_rpo = NULL;
578 556 : }
579 :
580 : /* Ratio of frequencies of edges so that one of more latch edges is
581 : considered to belong to inner loop with same header. */
582 : #define HEAVY_EDGE_RATIO 8
583 :
584 : /* Minimum number of samples for that we apply
585 : find_subloop_latch_edge_by_profile heuristics. */
586 : #define HEAVY_EDGE_MIN_SAMPLES 10
587 :
588 : /* If the profile info is available, finds an edge in LATCHES that much more
589 : frequent than the remaining edges. Returns such an edge, or NULL if we do
590 : not find one.
591 :
592 : We do not use guessed profile here, only the measured one. The guessed
593 : profile is usually too flat and unreliable for this (and it is mostly based
594 : on the loop structure of the program, so it does not make much sense to
595 : derive the loop structure from it). */
596 :
597 : static edge
598 20197 : find_subloop_latch_edge_by_profile (vec<edge> latches)
599 : {
600 20197 : unsigned i;
601 20197 : edge e, me = NULL;
602 20197 : profile_count mcount = profile_count::zero (), tcount = profile_count::zero ();
603 :
604 71614 : FOR_EACH_VEC_ELT (latches, i, e)
605 : {
606 51417 : if (e->count ()> mcount)
607 : {
608 8600 : me = e;
609 8600 : mcount = e->count();
610 : }
611 51417 : tcount += e->count();
612 : }
613 :
614 20197 : if (!tcount.initialized_p () || !(tcount.ipa () > HEAVY_EDGE_MIN_SAMPLES)
615 20217 : || (tcount - mcount) * HEAVY_EDGE_RATIO > tcount)
616 20183 : return NULL;
617 :
618 14 : if (dump_file)
619 0 : fprintf (dump_file,
620 : "Found latch edge %d -> %d using profile information.\n",
621 0 : me->src->index, me->dest->index);
622 : return me;
623 : }
624 :
625 : /* Among LATCHES, guesses a latch edge of LOOP corresponding to subloop, based
626 : on the structure of induction variables. Returns this edge, or NULL if we
627 : do not find any.
628 :
629 : We are quite conservative, and look just for an obvious simple innermost
630 : loop (which is the case where we would lose the most performance by not
631 : disambiguating the loop). More precisely, we look for the following
632 : situation: The source of the chosen latch edge dominates sources of all
633 : the other latch edges. Additionally, the header does not contain a phi node
634 : such that the argument from the chosen edge is equal to the argument from
635 : another edge. */
636 :
637 : static edge
638 19478 : find_subloop_latch_edge_by_ivs (class loop *loop ATTRIBUTE_UNUSED, vec<edge> latches)
639 : {
640 19478 : edge e, latch = latches[0];
641 19478 : unsigned i;
642 19478 : gphi *phi;
643 19478 : gphi_iterator psi;
644 19478 : tree lop;
645 19478 : basic_block bb;
646 :
647 : /* Find the candidate for the latch edge. */
648 49970 : for (i = 1; latches.iterate (i, &e); i++)
649 30492 : if (dominated_by_p (CDI_DOMINATORS, latch->src, e->src))
650 5683 : latch = e;
651 :
652 : /* Verify that it dominates all the latch edges. */
653 50590 : FOR_EACH_VEC_ELT (latches, i, e)
654 42279 : if (!dominated_by_p (CDI_DOMINATORS, e->src, latch->src))
655 : return NULL;
656 :
657 : /* Check for a phi node that would deny that this is a latch edge of
658 : a subloop. */
659 20452 : for (psi = gsi_start_phis (loop->header); !gsi_end_p (psi); gsi_next (&psi))
660 : {
661 17087 : phi = psi.phi ();
662 17087 : lop = PHI_ARG_DEF_FROM_EDGE (phi, latch);
663 :
664 : /* Ignore the values that are not changed inside the subloop. */
665 20636 : if (TREE_CODE (lop) != SSA_NAME
666 17087 : || SSA_NAME_DEF_STMT (lop) == phi)
667 3549 : continue;
668 13538 : bb = gimple_bb (SSA_NAME_DEF_STMT (lop));
669 13538 : if (!bb || !flow_bb_inside_loop_p (loop, bb))
670 16 : continue;
671 :
672 36538 : FOR_EACH_VEC_ELT (latches, i, e)
673 24397 : if (e != latch
674 24397 : && PHI_ARG_DEF_FROM_EDGE (phi, e) == lop)
675 : return NULL;
676 : }
677 :
678 3365 : if (dump_file)
679 1 : fprintf (dump_file,
680 : "Found latch edge %d -> %d using iv structure.\n",
681 1 : latch->src->index, latch->dest->index);
682 : return latch;
683 : }
684 :
685 : /* If we can determine that one of the several latch edges of LOOP behaves
686 : as a latch edge of a separate subloop, returns this edge. Otherwise
687 : returns NULL. */
688 :
689 : static edge
690 25808 : find_subloop_latch_edge (class loop *loop)
691 : {
692 25808 : vec<edge> latches = get_loop_latch_edges (loop);
693 25808 : edge latch = NULL;
694 :
695 25808 : if (latches.length () > 1)
696 : {
697 20197 : latch = find_subloop_latch_edge_by_profile (latches);
698 :
699 20197 : if (!latch
700 : /* We consider ivs to guess the latch edge only in SSA. Perhaps we
701 : should use cfghook for this, but it is hard to imagine it would
702 : be useful elsewhere. */
703 20197 : && current_ir_type () == IR_GIMPLE)
704 19478 : latch = find_subloop_latch_edge_by_ivs (loop, latches);
705 : }
706 :
707 25808 : latches.release ();
708 25808 : return latch;
709 : }
710 :
711 : /* Callback for make_forwarder_block. Returns true if the edge E is marked
712 : in the set MFB_REIS_SET. */
713 :
714 : static hash_set<edge> *mfb_reis_set;
715 : static bool
716 72559 : mfb_redirect_edges_in_set (edge e)
717 : {
718 72559 : return mfb_reis_set->contains (e);
719 : }
720 :
721 : /* Creates a subloop of LOOP with latch edge LATCH. */
722 :
723 : static void
724 3379 : form_subloop (class loop *loop, edge latch)
725 : {
726 3379 : edge_iterator ei;
727 3379 : edge e, new_entry;
728 3379 : class loop *new_loop;
729 :
730 3379 : mfb_reis_set = new hash_set<edge>;
731 13884 : FOR_EACH_EDGE (e, ei, loop->header->preds)
732 : {
733 10505 : if (e != latch)
734 7126 : mfb_reis_set->add (e);
735 : }
736 3379 : new_entry = make_forwarder_block (loop->header, mfb_redirect_edges_in_set,
737 : NULL);
738 6758 : delete mfb_reis_set;
739 :
740 3379 : loop->header = new_entry->src;
741 :
742 : /* Find the blocks and subloops that belong to the new loop, and add it to
743 : the appropriate place in the loop tree. */
744 3379 : new_loop = alloc_loop ();
745 3379 : new_loop->header = new_entry->dest;
746 3379 : new_loop->latch = latch->src;
747 3379 : add_loop (new_loop, loop);
748 3379 : }
749 :
750 : /* Make all the latch edges of LOOP to go to a single forwarder block --
751 : a new latch of LOOP. */
752 :
753 : static void
754 22429 : merge_latch_edges (class loop *loop)
755 : {
756 22429 : vec<edge> latches = get_loop_latch_edges (loop);
757 22429 : edge latch, e;
758 22429 : unsigned i;
759 :
760 22429 : gcc_assert (latches.length () > 0);
761 :
762 22429 : if (latches.length () == 1)
763 5611 : loop->latch = latches[0]->src;
764 : else
765 : {
766 16818 : if (dump_file)
767 8 : fprintf (dump_file, "Merged latch edges of loop %d\n", loop->num);
768 :
769 16818 : mfb_reis_set = new hash_set<edge>;
770 61132 : FOR_EACH_VEC_ELT (latches, i, e)
771 44314 : mfb_reis_set->add (e);
772 16818 : latch = make_forwarder_block (loop->header, mfb_redirect_edges_in_set,
773 : NULL);
774 33636 : delete mfb_reis_set;
775 :
776 16818 : loop->header = latch->dest;
777 16818 : loop->latch = latch->src;
778 : }
779 :
780 22429 : latches.release ();
781 22429 : }
782 :
783 : /* LOOP may have several latch edges. Transform it into (possibly several)
784 : loops with single latch edge. */
785 :
786 : static void
787 22429 : disambiguate_multiple_latches (class loop *loop)
788 : {
789 22429 : edge e;
790 :
791 : /* We eliminate the multiple latches by splitting the header to the forwarder
792 : block F and the rest R, and redirecting the edges. There are two cases:
793 :
794 : 1) If there is a latch edge E that corresponds to a subloop (we guess
795 : that based on profile -- if it is taken much more often than the
796 : remaining edges; and on trees, using the information about induction
797 : variables of the loops), we redirect E to R, all the remaining edges to
798 : F, then rescan the loops and try again for the outer loop.
799 : 2) If there is no such edge, we redirect all latch edges to F, and the
800 : entry edges to R, thus making F the single latch of the loop. */
801 :
802 22429 : if (dump_file)
803 9 : fprintf (dump_file, "Disambiguating loop %d with multiple latches\n",
804 : loop->num);
805 :
806 : /* During latch merging, we may need to redirect the entry edges to a new
807 : block. This would cause problems if the entry edge was the one from the
808 : entry block. To avoid having to handle this case specially, split
809 : such entry edge. */
810 22429 : e = find_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), loop->header);
811 22429 : if (e)
812 897 : split_edge (e);
813 :
814 29187 : while (1)
815 : {
816 25808 : e = find_subloop_latch_edge (loop);
817 25808 : if (!e)
818 : break;
819 :
820 3379 : form_subloop (loop, e);
821 : }
822 :
823 22429 : merge_latch_edges (loop);
824 22429 : }
825 :
826 : /* Split loops with multiple latch edges. */
827 :
828 : void
829 28635159 : disambiguate_loops_with_multiple_latches (void)
830 : {
831 103558371 : for (auto loop : loops_list (cfun, 0))
832 : {
833 17652894 : if (!loop->latch)
834 22429 : disambiguate_multiple_latches (loop);
835 28635159 : }
836 28635159 : }
837 :
838 : /* Return nonzero if basic block BB belongs to LOOP. */
839 : bool
840 3254036998 : flow_bb_inside_loop_p (const class loop *loop, const_basic_block bb)
841 : {
842 3254036998 : class loop *source_loop;
843 :
844 3254036998 : if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun)
845 3254024268 : || bb == EXIT_BLOCK_PTR_FOR_FN (cfun))
846 : return 0;
847 :
848 3235854826 : source_loop = bb->loop_father;
849 3235854826 : return loop == source_loop || flow_loop_nested_p (loop, source_loop);
850 : }
851 :
852 : /* Enumeration predicate for get_loop_body_with_size. */
853 : static bool
854 1230960251 : glb_enum_p (const_basic_block bb, const void *glb_loop)
855 : {
856 1230960251 : const class loop *const loop = (const class loop *) glb_loop;
857 1230960251 : return (bb != loop->header
858 1230960251 : && dominated_by_p (CDI_DOMINATORS, bb, loop->header));
859 : }
860 :
861 : /* Gets basic blocks of a LOOP. Header is the 0-th block, rest is in dfs
862 : order against direction of edges from latch. Specially, if
863 : header != latch, latch is the 1-st block. LOOP cannot be the fake
864 : loop tree root, and its size must be at most MAX_SIZE. The blocks
865 : in the LOOP body are stored to BODY, and the size of the LOOP is
866 : returned. */
867 :
868 : unsigned
869 219691668 : get_loop_body_with_size (const class loop *loop, basic_block *body,
870 : unsigned max_size)
871 : {
872 219691668 : return dfs_enumerate_from (loop->header, 1, glb_enum_p,
873 219691668 : body, max_size, loop);
874 : }
875 :
876 : /* Gets basic blocks of a LOOP. Header is the 0-th block, rest is in dfs
877 : order against direction of edges from latch. Specially, if
878 : header != latch, latch is the 1-st block. */
879 :
880 : basic_block *
881 21919417 : get_loop_body (const class loop *loop)
882 : {
883 21919417 : basic_block *body, bb;
884 21919417 : unsigned tv = 0;
885 :
886 21919417 : gcc_assert (loop->num_nodes);
887 :
888 21919417 : body = XNEWVEC (basic_block, loop->num_nodes);
889 :
890 21919417 : if (loop->latch == EXIT_BLOCK_PTR_FOR_FN (cfun))
891 : {
892 : /* There may be blocks unreachable from EXIT_BLOCK, hence we need to
893 : special-case the fake loop that contains the whole function. */
894 6233 : gcc_assert (loop->num_nodes == (unsigned) n_basic_blocks_for_fn (cfun));
895 6233 : body[tv++] = loop->header;
896 6233 : body[tv++] = EXIT_BLOCK_PTR_FOR_FN (cfun);
897 28432 : FOR_EACH_BB_FN (bb, cfun)
898 22199 : body[tv++] = bb;
899 : }
900 : else
901 21913184 : tv = get_loop_body_with_size (loop, body, loop->num_nodes);
902 :
903 21919417 : gcc_assert (tv == loop->num_nodes);
904 21919417 : return body;
905 : }
906 :
907 : /* Fills dominance descendants inside LOOP of the basic block BB into
908 : array TOVISIT from index *TV. */
909 :
910 : static void
911 5751286 : fill_sons_in_loop (const class loop *loop, basic_block bb,
912 : basic_block *tovisit, int *tv)
913 : {
914 9493996 : basic_block son, postpone = NULL;
915 :
916 9493996 : tovisit[(*tv)++] = bb;
917 9493996 : for (son = first_dom_son (CDI_DOMINATORS, bb);
918 19228393 : son;
919 9734397 : son = next_dom_son (CDI_DOMINATORS, son))
920 : {
921 9734397 : if (!flow_bb_inside_loop_p (loop, son))
922 2220809 : continue;
923 :
924 7513588 : if (dominated_by_p (CDI_DOMINATORS, loop->latch, son))
925 : {
926 3742710 : postpone = son;
927 3742710 : continue;
928 : }
929 3770878 : fill_sons_in_loop (loop, son, tovisit, tv);
930 : }
931 :
932 9493996 : if (postpone)
933 : fill_sons_in_loop (loop, postpone, tovisit, tv);
934 5751286 : }
935 :
936 : /* Gets body of a LOOP (that must be different from the outermost loop)
937 : sorted by dominance relation. Additionally, if a basic block s dominates
938 : the latch, then only blocks dominated by s are be after it. */
939 :
940 : basic_block *
941 1980408 : get_loop_body_in_dom_order (const class loop *loop)
942 : {
943 1980408 : basic_block *tovisit;
944 1980408 : int tv;
945 :
946 1980408 : gcc_assert (loop->num_nodes);
947 :
948 1980408 : tovisit = XNEWVEC (basic_block, loop->num_nodes);
949 :
950 1980408 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
951 :
952 1980408 : tv = 0;
953 1980408 : fill_sons_in_loop (loop, loop->header, tovisit, &tv);
954 :
955 1980408 : gcc_assert (tv == (int) loop->num_nodes);
956 :
957 1980408 : return tovisit;
958 : }
959 :
960 : /* Gets body of a LOOP sorted via provided BB_COMPARATOR. */
961 :
962 : basic_block *
963 56 : get_loop_body_in_custom_order (const class loop *loop,
964 : int (*bb_comparator) (const void *, const void *))
965 : {
966 56 : basic_block *bbs = get_loop_body (loop);
967 :
968 56 : qsort (bbs, loop->num_nodes, sizeof (basic_block), bb_comparator);
969 :
970 56 : return bbs;
971 : }
972 :
973 : /* Same as above, but use gcc_sort_r instead of qsort. */
974 :
975 : basic_block *
976 146627 : get_loop_body_in_custom_order (const class loop *loop, void *data,
977 : int (*bb_comparator) (const void *, const void *, void *))
978 : {
979 146627 : basic_block *bbs = get_loop_body (loop);
980 :
981 146627 : gcc_sort_r (bbs, loop->num_nodes, sizeof (basic_block), bb_comparator, data);
982 :
983 146627 : return bbs;
984 : }
985 :
986 : /* Get body of a LOOP in breadth first sort order. */
987 :
988 : basic_block *
989 401237 : get_loop_body_in_bfs_order (const class loop *loop)
990 : {
991 401237 : basic_block *blocks;
992 401237 : basic_block bb;
993 401237 : unsigned int i = 1;
994 401237 : unsigned int vc = 0;
995 :
996 401237 : gcc_assert (loop->num_nodes);
997 401237 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
998 :
999 401237 : blocks = XNEWVEC (basic_block, loop->num_nodes);
1000 401237 : auto_bitmap visited;
1001 401237 : blocks[0] = loop->header;
1002 401237 : bitmap_set_bit (visited, loop->header->index);
1003 2121426 : while (i < loop->num_nodes)
1004 : {
1005 1318952 : edge e;
1006 1318952 : edge_iterator ei;
1007 1318952 : gcc_assert (i > vc);
1008 1318952 : bb = blocks[vc++];
1009 :
1010 3615091 : FOR_EACH_EDGE (e, ei, bb->succs)
1011 : {
1012 2296139 : if (flow_bb_inside_loop_p (loop, e->dest))
1013 : {
1014 : /* This bb is now visited. */
1015 1580069 : if (bitmap_set_bit (visited, e->dest->index))
1016 1370407 : blocks[i++] = e->dest;
1017 : }
1018 : }
1019 : }
1020 :
1021 401237 : return blocks;
1022 401237 : }
1023 :
1024 : /* Hash function for struct loop_exit. */
1025 :
1026 : hashval_t
1027 253953522 : loop_exit_hasher::hash (loop_exit *exit)
1028 : {
1029 253953522 : return htab_hash_pointer (exit->e);
1030 : }
1031 :
1032 : /* Equality function for struct loop_exit. Compares with edge. */
1033 :
1034 : bool
1035 301301013 : loop_exit_hasher::equal (loop_exit *exit, edge e)
1036 : {
1037 301301013 : return exit->e == e;
1038 : }
1039 :
1040 : /* Frees the list of loop exit descriptions EX. */
1041 :
1042 : void
1043 19442272 : loop_exit_hasher::remove (loop_exit *exit)
1044 : {
1045 19442272 : loop_exit *next;
1046 40366338 : for (; exit; exit = next)
1047 : {
1048 20924066 : next = exit->next_e;
1049 :
1050 20924066 : exit->next->prev = exit->prev;
1051 20924066 : exit->prev->next = exit->next;
1052 :
1053 20924066 : ggc_free (exit);
1054 : }
1055 19442272 : }
1056 :
1057 : /* Returns the list of records for E as an exit of a loop. */
1058 :
1059 : static struct loop_exit *
1060 41234892 : get_exit_descriptions (edge e)
1061 : {
1062 41234892 : return current_loops->exits->find_with_hash (e, htab_hash_pointer (e));
1063 : }
1064 :
1065 : /* Updates the lists of loop exits in that E appears.
1066 : If REMOVED is true, E is being removed, and we
1067 : just remove it from the lists of exits.
1068 : If NEW_EDGE is true and E is not a loop exit, we
1069 : do not try to remove it from loop exit lists. */
1070 :
1071 : void
1072 556567268 : rescan_loop_exit (edge e, bool new_edge, bool removed)
1073 : {
1074 556567268 : struct loop_exit *exits = NULL, *exit;
1075 556567268 : class loop *aloop, *cloop;
1076 :
1077 556567268 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1078 : return;
1079 :
1080 255065637 : if (!removed
1081 239497383 : && e->src->loop_father != NULL
1082 239497383 : && e->dest->loop_father != NULL
1083 492531467 : && !flow_bb_inside_loop_p (e->src->loop_father, e->dest))
1084 : {
1085 37605971 : cloop = find_common_loop (e->src->loop_father, e->dest->loop_father);
1086 37605971 : for (aloop = e->src->loop_father;
1087 58530037 : aloop != cloop;
1088 20924066 : aloop = loop_outer (aloop))
1089 : {
1090 20924066 : exit = ggc_alloc<loop_exit> ();
1091 20924066 : exit->e = e;
1092 :
1093 20924066 : exit->next = aloop->exits->next;
1094 20924066 : exit->prev = aloop->exits;
1095 20924066 : exit->next->prev = exit;
1096 20924066 : exit->prev->next = exit;
1097 :
1098 20924066 : exit->next_e = exits;
1099 20924066 : exits = exit;
1100 : }
1101 : }
1102 :
1103 255065637 : if (!exits && new_edge)
1104 : return;
1105 :
1106 38647566 : loop_exit **slot
1107 57852860 : = current_loops->exits->find_slot_with_hash (e, htab_hash_pointer (e),
1108 : exits ? INSERT : NO_INSERT);
1109 38647566 : if (!slot)
1110 : return;
1111 :
1112 20766205 : if (exits)
1113 : {
1114 19442272 : if (*slot)
1115 406445 : loop_exit_hasher::remove (*slot);
1116 19442272 : *slot = exits;
1117 : }
1118 : else
1119 1323933 : current_loops->exits->clear_slot (slot);
1120 : }
1121 :
1122 : /* For each loop, record list of exit edges, and start maintaining these
1123 : lists. */
1124 :
1125 : void
1126 17403255 : record_loop_exits (void)
1127 : {
1128 17403255 : basic_block bb;
1129 17403255 : edge_iterator ei;
1130 17403255 : edge e;
1131 :
1132 17403255 : if (!current_loops)
1133 0 : return;
1134 :
1135 17403255 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1136 : return;
1137 17403255 : loops_state_set (LOOPS_HAVE_RECORDED_EXITS);
1138 :
1139 17403255 : gcc_assert (current_loops->exits == NULL);
1140 17403255 : current_loops->exits
1141 34806510 : = hash_table<loop_exit_hasher>::create_ggc (2 * number_of_loops (cfun));
1142 :
1143 174542542 : FOR_EACH_BB_FN (bb, cfun)
1144 : {
1145 377448180 : FOR_EACH_EDGE (e, ei, bb->succs)
1146 : {
1147 220308893 : rescan_loop_exit (e, true, false);
1148 : }
1149 : }
1150 : }
1151 :
1152 : /* Dumps information about the exit in *SLOT to FILE.
1153 : Callback for htab_traverse. */
1154 :
1155 : int
1156 0 : dump_recorded_exit (loop_exit **slot, FILE *file)
1157 : {
1158 0 : struct loop_exit *exit = *slot;
1159 0 : unsigned n = 0;
1160 0 : edge e = exit->e;
1161 :
1162 0 : for (; exit != NULL; exit = exit->next_e)
1163 0 : n++;
1164 :
1165 0 : fprintf (file, "Edge %d->%d exits %u loops\n",
1166 0 : e->src->index, e->dest->index, n);
1167 :
1168 0 : return 1;
1169 : }
1170 :
1171 : /* Dumps the recorded exits of loops to FILE. */
1172 :
1173 : extern void dump_recorded_exits (FILE *);
1174 : void
1175 0 : dump_recorded_exits (FILE *file)
1176 : {
1177 0 : if (!current_loops->exits)
1178 : return;
1179 0 : current_loops->exits->traverse<FILE *, dump_recorded_exit> (file);
1180 : }
1181 :
1182 : /* Releases lists of loop exits. */
1183 :
1184 : void
1185 17403255 : release_recorded_exits (function *fn)
1186 : {
1187 17403255 : gcc_assert (loops_state_satisfies_p (fn, LOOPS_HAVE_RECORDED_EXITS));
1188 17403255 : loops_for_fn (fn)->exits->empty ();
1189 17403255 : loops_for_fn (fn)->exits = NULL;
1190 17403255 : loops_state_clear (fn, LOOPS_HAVE_RECORDED_EXITS);
1191 17403255 : }
1192 :
1193 : /* Returns the list of the exit edges of a LOOP. */
1194 :
1195 : auto_vec<edge>
1196 36598664 : get_loop_exit_edges (const class loop *loop, basic_block *body)
1197 : {
1198 36598664 : auto_vec<edge> edges;
1199 36598664 : edge e;
1200 36598664 : unsigned i;
1201 36598664 : edge_iterator ei;
1202 36598664 : struct loop_exit *exit;
1203 :
1204 36598664 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
1205 :
1206 : /* If we maintain the lists of exits, use them. Otherwise we must
1207 : scan the body of the loop. */
1208 36598664 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1209 : {
1210 111801019 : for (exit = loop->exits->next; exit->e; exit = exit->next)
1211 75867395 : edges.safe_push (exit->e);
1212 : }
1213 : else
1214 : {
1215 665040 : bool body_from_caller = true;
1216 665040 : if (!body)
1217 : {
1218 655863 : body = get_loop_body (loop);
1219 655863 : body_from_caller = false;
1220 : }
1221 4797441 : for (i = 0; i < loop->num_nodes; i++)
1222 10775520 : FOR_EACH_EDGE (e, ei, body[i]->succs)
1223 : {
1224 6643119 : if (!flow_bb_inside_loop_p (loop, e->dest))
1225 1608224 : edges.safe_push (e);
1226 : }
1227 665040 : if (!body_from_caller)
1228 655863 : free (body);
1229 : }
1230 :
1231 36598664 : return edges;
1232 : }
1233 :
1234 : /* Counts the number of conditional branches inside LOOP. */
1235 :
1236 : unsigned
1237 133 : num_loop_branches (const class loop *loop)
1238 : {
1239 133 : unsigned i, n;
1240 133 : basic_block * body;
1241 :
1242 133 : gcc_assert (loop->latch != EXIT_BLOCK_PTR_FOR_FN (cfun));
1243 :
1244 133 : body = get_loop_body (loop);
1245 133 : n = 0;
1246 632 : for (i = 0; i < loop->num_nodes; i++)
1247 568 : if (EDGE_COUNT (body[i]->succs) >= 2)
1248 202 : n++;
1249 133 : free (body);
1250 :
1251 133 : return n;
1252 : }
1253 :
1254 : /* Adds basic block BB to LOOP. */
1255 : void
1256 45567959 : add_bb_to_loop (basic_block bb, class loop *loop)
1257 : {
1258 45567959 : unsigned i;
1259 45567959 : loop_p ploop;
1260 45567959 : edge_iterator ei;
1261 45567959 : edge e;
1262 :
1263 45567959 : gcc_assert (bb->loop_father == NULL);
1264 45567959 : bb->loop_father = loop;
1265 45567959 : loop->num_nodes++;
1266 66064181 : FOR_EACH_VEC_SAFE_ELT (loop->superloops, i, ploop)
1267 20496222 : ploop->num_nodes++;
1268 :
1269 95376397 : FOR_EACH_EDGE (e, ei, bb->succs)
1270 : {
1271 49808438 : rescan_loop_exit (e, true, false);
1272 : }
1273 76431289 : FOR_EACH_EDGE (e, ei, bb->preds)
1274 : {
1275 30863330 : rescan_loop_exit (e, true, false);
1276 : }
1277 45567959 : }
1278 :
1279 : /* Remove basic block BB from loops. */
1280 : void
1281 57227486 : remove_bb_from_loops (basic_block bb)
1282 : {
1283 57227486 : unsigned i;
1284 57227486 : class loop *loop = bb->loop_father;
1285 57227486 : loop_p ploop;
1286 57227486 : edge_iterator ei;
1287 57227486 : edge e;
1288 :
1289 57227486 : gcc_assert (loop != NULL);
1290 57227486 : loop->num_nodes--;
1291 78975506 : FOR_EACH_VEC_SAFE_ELT (loop->superloops, i, ploop)
1292 21748020 : ploop->num_nodes--;
1293 57227486 : bb->loop_father = NULL;
1294 :
1295 86049535 : FOR_EACH_EDGE (e, ei, bb->succs)
1296 : {
1297 28822049 : rescan_loop_exit (e, false, true);
1298 : }
1299 77867134 : FOR_EACH_EDGE (e, ei, bb->preds)
1300 : {
1301 20639648 : rescan_loop_exit (e, false, true);
1302 : }
1303 57227486 : }
1304 :
1305 : /* Finds nearest common ancestor in loop tree for given loops. */
1306 : class loop *
1307 207217828 : find_common_loop (class loop *loop_s, class loop *loop_d)
1308 : {
1309 207217828 : unsigned sdepth, ddepth;
1310 :
1311 207217828 : if (!loop_s) return loop_d;
1312 207217283 : if (!loop_d) return loop_s;
1313 :
1314 207217283 : sdepth = loop_depth (loop_s);
1315 207217283 : ddepth = loop_depth (loop_d);
1316 :
1317 83980726 : if (sdepth < ddepth)
1318 6456228 : loop_d = (*loop_d->superloops)[sdepth];
1319 200761055 : else if (sdepth > ddepth)
1320 101387636 : loop_s = (*loop_s->superloops)[ddepth];
1321 :
1322 208132433 : while (loop_s != loop_d)
1323 : {
1324 915150 : loop_s = loop_outer (loop_s);
1325 915150 : loop_d = loop_outer (loop_d);
1326 : }
1327 : return loop_s;
1328 : }
1329 :
1330 : /* Removes LOOP from structures and frees its data. */
1331 :
1332 : void
1333 148747 : delete_loop (class loop *loop)
1334 : {
1335 : /* Remove the loop from structure. */
1336 148747 : flow_loop_tree_node_remove (loop);
1337 :
1338 : /* Remove loop from loops array. */
1339 148747 : (*current_loops->larray)[loop->num] = NULL;
1340 :
1341 : /* Free loop data. */
1342 148747 : flow_loop_free (loop);
1343 148747 : }
1344 :
1345 : /* Cancels the LOOP; it must be innermost one. */
1346 :
1347 : static void
1348 10656 : cancel_loop (class loop *loop)
1349 : {
1350 10656 : basic_block *bbs;
1351 10656 : unsigned i;
1352 10656 : class loop *outer = loop_outer (loop);
1353 :
1354 10656 : gcc_assert (!loop->inner);
1355 :
1356 : /* Move blocks up one level (they should be removed as soon as possible). */
1357 10656 : bbs = get_loop_body (loop);
1358 44135 : for (i = 0; i < loop->num_nodes; i++)
1359 22823 : bbs[i]->loop_father = outer;
1360 :
1361 10656 : free (bbs);
1362 10656 : delete_loop (loop);
1363 10656 : }
1364 :
1365 : /* Cancels LOOP and all its subloops. */
1366 : void
1367 10656 : cancel_loop_tree (class loop *loop)
1368 : {
1369 11144 : while (loop->inner)
1370 488 : cancel_loop_tree (loop->inner);
1371 10656 : cancel_loop (loop);
1372 10656 : }
1373 :
1374 : /* Disable warnings about missing quoting in GCC diagnostics for
1375 : the verification errors. Their format strings don't follow GCC
1376 : diagnostic conventions and the calls are ultimately followed by
1377 : a deliberate ICE triggered by a failed assertion. */
1378 : #if __GNUC__ >= 10
1379 : # pragma GCC diagnostic push
1380 : # pragma GCC diagnostic ignored "-Wformat-diag"
1381 : #endif
1382 :
1383 : /* Checks that information about loops is correct
1384 : -- sizes of loops are all right
1385 : -- results of get_loop_body really belong to the loop
1386 : -- loop header have just single entry edge and single latch edge
1387 : -- loop latches have only single successor that is header of their loop
1388 : -- irreducible loops are correctly marked
1389 : -- the cached loop depth and loop father of each bb is correct
1390 : */
1391 : DEBUG_FUNCTION void
1392 373042945 : verify_loop_structure (void)
1393 : {
1394 373042945 : unsigned *sizes, i, j;
1395 373042945 : basic_block bb, *bbs;
1396 373042945 : int err = 0;
1397 373042945 : edge e;
1398 373042945 : unsigned num = number_of_loops (cfun);
1399 373042945 : struct loop_exit *exit, *mexit;
1400 373042945 : bool dom_available = dom_info_available_p (CDI_DOMINATORS);
1401 :
1402 373042945 : if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
1403 : {
1404 0 : error ("loop verification on loop tree that needs fixup");
1405 0 : err = 1;
1406 : }
1407 :
1408 : /* We need up-to-date dominators, compute or verify them. */
1409 373042945 : if (!dom_available)
1410 100674704 : calculate_dominance_info (CDI_DOMINATORS);
1411 : else
1412 272368241 : verify_dominators (CDI_DOMINATORS);
1413 :
1414 : /* Check the loop tree root. */
1415 373042945 : if (current_loops->tree_root->header != ENTRY_BLOCK_PTR_FOR_FN (cfun)
1416 373042945 : || current_loops->tree_root->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
1417 373042945 : || (current_loops->tree_root->num_nodes
1418 373042945 : != (unsigned) n_basic_blocks_for_fn (cfun)))
1419 : {
1420 0 : error ("corrupt loop tree root");
1421 0 : err = 1;
1422 : }
1423 :
1424 : /* Check the headers. */
1425 3708452419 : FOR_EACH_BB_FN (bb, cfun)
1426 3335409474 : if (bb_loop_header_p (bb))
1427 : {
1428 197202370 : if (bb->loop_father->header == NULL)
1429 : {
1430 0 : error ("loop with header %d marked for removal", bb->index);
1431 0 : err = 1;
1432 : }
1433 197202370 : else if (bb->loop_father->header != bb)
1434 : {
1435 0 : error ("loop with header %d not in loop tree", bb->index);
1436 0 : err = 1;
1437 : }
1438 : }
1439 3138207104 : else if (bb->loop_father->header == bb)
1440 : {
1441 0 : error ("non-loop with header %d not marked for removal", bb->index);
1442 0 : err = 1;
1443 : }
1444 :
1445 : /* Check the recorded loop father and sizes of loops. */
1446 373042945 : auto_sbitmap visited (last_basic_block_for_fn (cfun));
1447 373042945 : bitmap_clear (visited);
1448 373042945 : bbs = XNEWVEC (basic_block, n_basic_blocks_for_fn (cfun));
1449 1316331205 : for (auto loop : loops_list (cfun, LI_FROM_INNERMOST))
1450 : {
1451 197202370 : unsigned n;
1452 :
1453 197202370 : if (loop->header == NULL)
1454 : {
1455 0 : error ("removed loop %d in loop tree", loop->num);
1456 0 : err = 1;
1457 0 : continue;
1458 : }
1459 :
1460 197202370 : n = get_loop_body_with_size (loop, bbs, n_basic_blocks_for_fn (cfun));
1461 197202370 : if (loop->num_nodes != n)
1462 : {
1463 0 : error ("size of loop %d should be %d, not %d",
1464 : loop->num, n, loop->num_nodes);
1465 0 : err = 1;
1466 : }
1467 :
1468 1310447568 : for (j = 0; j < n; j++)
1469 : {
1470 1113245198 : bb = bbs[j];
1471 :
1472 1113245198 : if (!flow_bb_inside_loop_p (loop, bb))
1473 : {
1474 0 : error ("bb %d does not belong to loop %d",
1475 : bb->index, loop->num);
1476 0 : err = 1;
1477 : }
1478 :
1479 : /* Ignore this block if it is in an inner loop. */
1480 1113245198 : if (bitmap_bit_p (visited, bb->index))
1481 260691638 : continue;
1482 852553560 : bitmap_set_bit (visited, bb->index);
1483 :
1484 852553560 : if (bb->loop_father != loop)
1485 : {
1486 0 : error ("bb %d has father loop %d, should be loop %d",
1487 : bb->index, bb->loop_father->num, loop->num);
1488 0 : err = 1;
1489 : }
1490 : }
1491 373042945 : }
1492 373042945 : free (bbs);
1493 :
1494 : /* Check headers and latches. */
1495 1316331205 : for (auto loop : loops_list (cfun, 0))
1496 : {
1497 197202370 : i = loop->num;
1498 197202370 : if (loop->header == NULL)
1499 0 : continue;
1500 197202370 : if (!bb_loop_header_p (loop->header))
1501 : {
1502 0 : error ("loop %d%'s header is not a loop header", i);
1503 0 : err = 1;
1504 : }
1505 197202370 : if (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
1506 197202370 : && EDGE_COUNT (loop->header->preds) != 2)
1507 : {
1508 0 : error ("loop %d%'s header does not have exactly 2 entries", i);
1509 0 : err = 1;
1510 : }
1511 197202370 : if (loop->latch)
1512 : {
1513 196312582 : if (!find_edge (loop->latch, loop->header))
1514 : {
1515 0 : error ("loop %d%'s latch does not have an edge to its header", i);
1516 0 : err = 1;
1517 : }
1518 196312582 : if (!dominated_by_p (CDI_DOMINATORS, loop->latch, loop->header))
1519 : {
1520 0 : error ("loop %d%'s latch is not dominated by its header", i);
1521 0 : err = 1;
1522 : }
1523 : }
1524 197202370 : if (loops_state_satisfies_p (LOOPS_HAVE_SIMPLE_LATCHES))
1525 : {
1526 30349973 : if (!single_succ_p (loop->latch))
1527 : {
1528 0 : error ("loop %d%'s latch does not have exactly 1 successor", i);
1529 0 : err = 1;
1530 : }
1531 30349973 : if (single_succ (loop->latch) != loop->header)
1532 : {
1533 0 : error ("loop %d%'s latch does not have header as successor", i);
1534 0 : err = 1;
1535 : }
1536 30349973 : if (loop->latch->loop_father != loop)
1537 : {
1538 0 : error ("loop %d%'s latch does not belong directly to it", i);
1539 0 : err = 1;
1540 : }
1541 : }
1542 197202370 : if (loop->header->loop_father != loop)
1543 : {
1544 0 : error ("loop %d%'s header does not belong directly to it", i);
1545 0 : err = 1;
1546 : }
1547 197202370 : if (loops_state_satisfies_p (LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS))
1548 : {
1549 27066639 : edge_iterator ei;
1550 81200181 : FOR_EACH_EDGE (e, ei, loop->header->preds)
1551 54133542 : if (dominated_by_p (CDI_DOMINATORS, e->src, loop->header)
1552 54133542 : && e->flags & EDGE_IRREDUCIBLE_LOOP)
1553 : {
1554 0 : error ("loop %d%'s latch is marked as part of irreducible"
1555 : " region", i);
1556 0 : err = 1;
1557 : }
1558 : }
1559 :
1560 : /* Check cached number of iterations for released SSA names. */
1561 197202370 : tree ref;
1562 197202370 : if (loop->nb_iterations
1563 197202370 : && (ref = walk_tree (&loop->nb_iterations,
1564 : find_released_ssa_name, NULL, NULL)))
1565 : {
1566 0 : error ("loop %d%'s number of iterations %qE references the"
1567 : " released SSA name %qE", i, loop->nb_iterations, ref);
1568 0 : err = 1;
1569 : }
1570 373042945 : }
1571 :
1572 : /* Check irreducible loops. */
1573 373042945 : if (loops_state_satisfies_p (LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS))
1574 : {
1575 33663620 : auto_edge_flag saved_edge_irr (cfun);
1576 33663620 : auto_bb_flag saved_bb_irr (cfun);
1577 : /* Save old info. */
1578 433747750 : FOR_EACH_BB_FN (bb, cfun)
1579 : {
1580 400084130 : edge_iterator ei;
1581 400084130 : if (bb->flags & BB_IRREDUCIBLE_LOOP)
1582 548369 : bb->flags |= saved_bb_irr;
1583 961619262 : FOR_EACH_EDGE (e, ei, bb->succs)
1584 561535132 : if (e->flags & EDGE_IRREDUCIBLE_LOOP)
1585 788904 : e->flags |= saved_edge_irr;
1586 : }
1587 :
1588 : /* Recount it. */
1589 33663620 : mark_irreducible_loops ();
1590 :
1591 : /* Compare. */
1592 433747750 : FOR_EACH_BB_FN (bb, cfun)
1593 : {
1594 400084130 : edge_iterator ei;
1595 :
1596 400084130 : if ((bb->flags & BB_IRREDUCIBLE_LOOP)
1597 400084130 : && !(bb->flags & saved_bb_irr))
1598 : {
1599 0 : error ("basic block %d should be marked irreducible", bb->index);
1600 0 : err = 1;
1601 : }
1602 400084130 : else if (!(bb->flags & BB_IRREDUCIBLE_LOOP)
1603 400084130 : && (bb->flags & saved_bb_irr))
1604 : {
1605 0 : error ("basic block %d should not be marked irreducible", bb->index);
1606 0 : err = 1;
1607 : }
1608 400084130 : bb->flags &= ~saved_bb_irr;
1609 961619262 : FOR_EACH_EDGE (e, ei, bb->succs)
1610 : {
1611 561535132 : if ((e->flags & EDGE_IRREDUCIBLE_LOOP)
1612 561535132 : && !(e->flags & saved_edge_irr))
1613 : {
1614 0 : error ("edge from %d to %d should be marked irreducible",
1615 0 : e->src->index, e->dest->index);
1616 0 : err = 1;
1617 : }
1618 561535132 : else if (!(e->flags & EDGE_IRREDUCIBLE_LOOP)
1619 561535132 : && (e->flags & saved_edge_irr))
1620 : {
1621 0 : error ("edge from %d to %d should not be marked irreducible",
1622 0 : e->src->index, e->dest->index);
1623 0 : err = 1;
1624 : }
1625 561535132 : e->flags &= ~saved_edge_irr;
1626 : }
1627 : }
1628 33663620 : }
1629 :
1630 : /* Check the recorded loop exits. */
1631 1316331205 : for (auto loop : loops_list (cfun, 0))
1632 : {
1633 197202370 : if (!loop->exits || loop->exits->e != NULL)
1634 : {
1635 0 : error ("corrupted head of the exits list of loop %d",
1636 : loop->num);
1637 0 : err = 1;
1638 : }
1639 : else
1640 : {
1641 : /* Check that the list forms a cycle, and all elements except
1642 : for the head are nonnull. */
1643 197202370 : for (mexit = loop->exits, exit = mexit->next, i = 0;
1644 241505323 : exit->e && exit != mexit;
1645 44302953 : exit = exit->next)
1646 : {
1647 44302953 : if (i++ & 1)
1648 13229553 : mexit = mexit->next;
1649 : }
1650 :
1651 197202370 : if (exit != loop->exits)
1652 : {
1653 0 : error ("corrupted exits list of loop %d", loop->num);
1654 0 : err = 1;
1655 : }
1656 : }
1657 :
1658 197202370 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1659 : {
1660 173046931 : if (loop->exits->next != loop->exits)
1661 : {
1662 0 : error ("nonempty exits list of loop %d, but exits are not recorded",
1663 : loop->num);
1664 0 : err = 1;
1665 : }
1666 : }
1667 373042945 : }
1668 :
1669 373042945 : if (loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1670 : {
1671 25635646 : unsigned n_exits = 0, eloops;
1672 :
1673 25635646 : sizes = XCNEWVEC (unsigned, num);
1674 25635646 : memset (sizes, 0, sizeof (unsigned) * num);
1675 364062668 : FOR_EACH_BB_FN (bb, cfun)
1676 : {
1677 338427022 : edge_iterator ei;
1678 338427022 : if (bb->loop_father == current_loops->tree_root)
1679 225158219 : continue;
1680 289619030 : FOR_EACH_EDGE (e, ei, bb->succs)
1681 : {
1682 176350227 : if (flow_bb_inside_loop_p (bb->loop_father, e->dest))
1683 135115335 : continue;
1684 :
1685 41234892 : n_exits++;
1686 41234892 : exit = get_exit_descriptions (e);
1687 41234892 : if (!exit)
1688 : {
1689 0 : error ("exit %d->%d not recorded",
1690 0 : e->src->index, e->dest->index);
1691 0 : err = 1;
1692 : }
1693 41234892 : eloops = 0;
1694 85537845 : for (; exit; exit = exit->next_e)
1695 44302953 : eloops++;
1696 :
1697 41234892 : for (class loop *loop = bb->loop_father;
1698 85537845 : loop != e->dest->loop_father
1699 : /* When a loop exit is also an entry edge which
1700 : can happen when avoiding CFG manipulations
1701 : then the last loop exited is the outer loop
1702 : of the loop entered. */
1703 85537845 : && loop != loop_outer (e->dest->loop_father);
1704 44302953 : loop = loop_outer (loop))
1705 : {
1706 44302953 : eloops--;
1707 44302953 : sizes[loop->num]++;
1708 : }
1709 :
1710 41234892 : if (eloops != 0)
1711 : {
1712 0 : error ("wrong list of exited loops for edge %d->%d",
1713 0 : e->src->index, e->dest->index);
1714 0 : err = 1;
1715 : }
1716 : }
1717 : }
1718 :
1719 25635646 : if (n_exits != current_loops->exits->elements ())
1720 : {
1721 0 : error ("too many loop exits recorded");
1722 0 : err = 1;
1723 : }
1724 :
1725 101062377 : for (auto loop : loops_list (cfun, 0))
1726 : {
1727 24155439 : eloops = 0;
1728 68458392 : for (exit = loop->exits->next; exit->e; exit = exit->next)
1729 44302953 : eloops++;
1730 24155439 : if (eloops != sizes[loop->num])
1731 : {
1732 0 : error ("%d exits recorded for loop %d (having %d exits)",
1733 : eloops, loop->num, sizes[loop->num]);
1734 0 : err = 1;
1735 : }
1736 25635646 : }
1737 :
1738 25635646 : free (sizes);
1739 : }
1740 :
1741 373042945 : gcc_assert (!err);
1742 :
1743 373042945 : if (!dom_available)
1744 100674704 : free_dominance_info (CDI_DOMINATORS);
1745 373042945 : }
1746 :
1747 : #if __GNUC__ >= 10
1748 : # pragma GCC diagnostic pop
1749 : #endif
1750 :
1751 : /* Returns latch edge of LOOP. */
1752 : edge
1753 12797439 : loop_latch_edge (const class loop *loop)
1754 : {
1755 12797439 : return find_edge (loop->latch, loop->header);
1756 : }
1757 :
1758 : /* Returns preheader edge of LOOP. */
1759 : edge
1760 430290597 : loop_preheader_edge (const class loop *loop)
1761 : {
1762 430290597 : edge e;
1763 430290597 : edge_iterator ei;
1764 :
1765 430290597 : gcc_assert (loops_state_satisfies_p (LOOPS_HAVE_PREHEADERS)
1766 : && ! loops_state_satisfies_p (LOOPS_MAY_HAVE_MULTIPLE_LATCHES));
1767 :
1768 656391972 : FOR_EACH_EDGE (e, ei, loop->header->preds)
1769 655551373 : if (e->src != loop->latch)
1770 : break;
1771 :
1772 430290597 : if (! e)
1773 : {
1774 840599 : gcc_assert (! loop_outer (loop));
1775 840599 : return single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
1776 : }
1777 :
1778 : return e;
1779 : }
1780 :
1781 : /* Returns true if E is an exit of LOOP. */
1782 :
1783 : bool
1784 36205558 : loop_exit_edge_p (const class loop *loop, const_edge e)
1785 : {
1786 36205558 : return (flow_bb_inside_loop_p (loop, e->src)
1787 36205558 : && !flow_bb_inside_loop_p (loop, e->dest));
1788 : }
1789 :
1790 : /* Returns the single exit edge of LOOP, or NULL if LOOP has either no exit
1791 : or more than one exit. If loops do not have the exits recorded, NULL
1792 : is returned always. */
1793 :
1794 : edge
1795 35319793 : single_exit (const class loop *loop)
1796 : {
1797 35319793 : struct loop_exit *exit = loop->exits->next;
1798 :
1799 35319793 : if (!loops_state_satisfies_p (LOOPS_HAVE_RECORDED_EXITS))
1800 : return NULL;
1801 :
1802 35291016 : if (exit->e && exit->next == loop->exits)
1803 : return exit->e;
1804 : else
1805 10633624 : return NULL;
1806 : }
1807 :
1808 : /* Returns incoming edge when BB has an incoming edge exiting LOOP, else return
1809 : NULL. */
1810 :
1811 : edge
1812 0 : loop_exits_to_bb_p (class loop *loop, basic_block bb)
1813 : {
1814 0 : edge e;
1815 0 : edge_iterator ei;
1816 :
1817 0 : FOR_EACH_EDGE (e, ei, bb->preds)
1818 0 : if (loop_exit_edge_p (loop, e))
1819 : return e;
1820 :
1821 : return NULL;
1822 : }
1823 :
1824 : /* Returns outgoing edge when BB has an outgoing edge exiting LOOP, else return
1825 : NULL. */
1826 :
1827 : edge
1828 1474637 : loop_exits_from_bb_p (class loop *loop, basic_block bb)
1829 : {
1830 1474637 : edge e;
1831 1474637 : edge_iterator ei;
1832 :
1833 3105362 : FOR_EACH_EDGE (e, ei, bb->succs)
1834 2588359 : if (loop_exit_edge_p (loop, e))
1835 : return e;
1836 :
1837 : return NULL;
1838 : }
1839 :
1840 : /* Return location corresponding to the loop control condition if possible. */
1841 :
1842 : dump_user_location_t
1843 588249 : get_loop_location (class loop *loop)
1844 : {
1845 588249 : rtx_insn *insn = NULL;
1846 588249 : class niter_desc *desc = NULL;
1847 588249 : edge exit;
1848 :
1849 : /* For a for or while loop, we would like to return the location
1850 : of the for or while statement, if possible. To do this, look
1851 : for the branch guarding the loop back-edge. */
1852 :
1853 : /* If this is a simple loop with an in_edge, then the loop control
1854 : branch is typically at the end of its source. */
1855 588249 : desc = get_simple_loop_desc (loop);
1856 588249 : if (desc->in_edge)
1857 : {
1858 590485 : FOR_BB_INSNS_REVERSE (desc->in_edge->src, insn)
1859 : {
1860 585456 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1861 406949 : return insn;
1862 : }
1863 : }
1864 : /* If loop has a single exit, then the loop control branch
1865 : must be at the end of its source. */
1866 181300 : if ((exit = single_exit (loop)))
1867 : {
1868 204651 : FOR_BB_INSNS_REVERSE (exit->src, insn)
1869 : {
1870 196062 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1871 99985 : return insn;
1872 : }
1873 : }
1874 : /* Next check the latch, to see if it is non-empty. */
1875 192690 : FOR_BB_INSNS_REVERSE (loop->latch, insn)
1876 : {
1877 133640 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1878 22265 : return insn;
1879 : }
1880 : /* Finally, if none of the above identifies the loop control branch,
1881 : return the first location in the loop header. */
1882 283373 : FOR_BB_INSNS (loop->header, insn)
1883 : {
1884 271694 : if (INSN_P (insn) && INSN_HAS_LOCATION (insn))
1885 47371 : return insn;
1886 : }
1887 : /* If all else fails, simply return the current function location. */
1888 11679 : return dump_user_location_t::from_function_decl (current_function_decl);
1889 : }
1890 :
1891 : /* Records that every statement in LOOP is executed I_BOUND times.
1892 : REALISTIC is true if I_BOUND is expected to be close to the real number
1893 : of iterations. UPPER is true if we are sure the loop iterates at most
1894 : I_BOUND times. */
1895 :
1896 : void
1897 17348704 : record_niter_bound (class loop *loop, const widest_int &i_bound,
1898 : bool realistic, bool upper)
1899 : {
1900 17348704 : if (wi::min_precision (i_bound, SIGNED) > bound_wide_int ().get_precision ())
1901 0 : return;
1902 :
1903 17348704 : bound_wide_int bound = bound_wide_int::from (i_bound, SIGNED);
1904 :
1905 : /* Update the bounds only when there is no previous estimation, or when the
1906 : current estimation is smaller. */
1907 17348704 : if (upper
1908 17348704 : && (!loop->any_upper_bound
1909 15887252 : || wi::ltu_p (bound, loop->nb_iterations_upper_bound)))
1910 : {
1911 880979 : loop->any_upper_bound = true;
1912 880979 : loop->nb_iterations_upper_bound = bound;
1913 880979 : if (!loop->any_likely_upper_bound)
1914 : {
1915 507424 : loop->any_likely_upper_bound = true;
1916 507424 : loop->nb_iterations_likely_upper_bound = bound;
1917 : }
1918 : }
1919 17348704 : if (realistic
1920 17348704 : && (!loop->any_estimate
1921 2825007 : || wi::ltu_p (bound, loop->nb_iterations_estimate)))
1922 : {
1923 240603 : loop->any_estimate = true;
1924 240603 : loop->nb_iterations_estimate = bound;
1925 : }
1926 17348704 : if (!realistic
1927 17348704 : && (!loop->any_likely_upper_bound
1928 14288273 : || wi::ltu_p (bound, loop->nb_iterations_likely_upper_bound)))
1929 : {
1930 245407 : loop->any_likely_upper_bound = true;
1931 245407 : loop->nb_iterations_likely_upper_bound = bound;
1932 : }
1933 :
1934 : /* If an upper bound is smaller than the realistic estimate of the
1935 : number of iterations, use the upper bound instead. */
1936 17348704 : if (loop->any_upper_bound
1937 17344617 : && loop->any_estimate
1938 26458615 : && wi::ltu_p (loop->nb_iterations_upper_bound,
1939 9109911 : loop->nb_iterations_estimate))
1940 6364 : loop->nb_iterations_estimate = loop->nb_iterations_upper_bound;
1941 17348704 : if (loop->any_upper_bound
1942 17344617 : && loop->any_likely_upper_bound
1943 34693321 : && wi::ltu_p (loop->nb_iterations_upper_bound,
1944 17344617 : loop->nb_iterations_likely_upper_bound))
1945 37685 : loop->nb_iterations_likely_upper_bound = loop->nb_iterations_upper_bound;
1946 : }
1947 :
1948 : /* Similar to get_estimated_loop_iterations, but returns the estimate only
1949 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
1950 : on the number of iterations of LOOP could not be derived, returns -1. */
1951 :
1952 : HOST_WIDE_INT
1953 231 : get_estimated_loop_iterations_int (class loop *loop)
1954 : {
1955 231 : widest_int nit;
1956 231 : HOST_WIDE_INT hwi_nit;
1957 :
1958 231 : if (!get_estimated_loop_iterations (loop, &nit))
1959 : return -1;
1960 :
1961 46 : if (!wi::fits_shwi_p (nit))
1962 : return -1;
1963 46 : hwi_nit = nit.to_shwi ();
1964 :
1965 46 : return hwi_nit < 0 ? -1 : hwi_nit;
1966 231 : }
1967 :
1968 : /* Returns an upper bound on the number of executions of statements
1969 : in the LOOP. For statements before the loop exit, this exceeds
1970 : the number of execution of the latch by one. */
1971 :
1972 : HOST_WIDE_INT
1973 291112 : max_stmt_executions_int (class loop *loop)
1974 : {
1975 291112 : HOST_WIDE_INT nit = get_max_loop_iterations_int (loop);
1976 291112 : HOST_WIDE_INT snit;
1977 :
1978 291112 : if (nit == -1)
1979 : return -1;
1980 :
1981 267191 : snit = (HOST_WIDE_INT) ((unsigned HOST_WIDE_INT) nit + 1);
1982 :
1983 : /* If the computation overflows, return -1. */
1984 267191 : return snit < 0 ? -1 : snit;
1985 : }
1986 :
1987 : /* Returns an likely upper bound on the number of executions of statements
1988 : in the LOOP. For statements before the loop exit, this exceeds
1989 : the number of execution of the latch by one. */
1990 :
1991 : HOST_WIDE_INT
1992 4928388 : likely_max_stmt_executions_int (class loop *loop)
1993 : {
1994 4928388 : HOST_WIDE_INT nit = get_likely_max_loop_iterations_int (loop);
1995 4928388 : HOST_WIDE_INT snit;
1996 :
1997 4928388 : if (nit == -1)
1998 : return -1;
1999 :
2000 4179389 : snit = (HOST_WIDE_INT) ((unsigned HOST_WIDE_INT) nit + 1);
2001 :
2002 : /* If the computation overflows, return -1. */
2003 4179389 : return snit < 0 ? -1 : snit;
2004 : }
2005 :
2006 : /* Sets NIT to the estimated number of executions of the latch of the
2007 : LOOP. If we have no reliable estimate, the function returns false, otherwise
2008 : returns true. */
2009 :
2010 : bool
2011 9242448 : get_estimated_loop_iterations (class loop *loop, widest_int *nit)
2012 : {
2013 : /* Even if the bound is not recorded, possibly we can derrive one from
2014 : profile. */
2015 9242448 : if (!loop->any_estimate)
2016 : {
2017 5311875 : sreal snit;
2018 5311875 : bool reliable;
2019 5311875 : if (expected_loop_iterations_by_profile (loop, &snit, &reliable)
2020 5311875 : && reliable)
2021 : {
2022 3 : *nit = snit.to_nearest_int ();
2023 3 : return true;
2024 : }
2025 : return false;
2026 : }
2027 :
2028 3930573 : *nit = widest_int::from (loop->nb_iterations_estimate, SIGNED);
2029 3930573 : return true;
2030 : }
2031 :
2032 : /* Sets NIT to an upper bound for the maximum number of executions of the
2033 : latch of the LOOP. If we have no reliable estimate, the function returns
2034 : false, otherwise returns true. */
2035 :
2036 : bool
2037 22929572 : get_max_loop_iterations (const class loop *loop, widest_int *nit)
2038 : {
2039 22929572 : if (!loop->any_upper_bound)
2040 : return false;
2041 :
2042 20453570 : *nit = widest_int::from (loop->nb_iterations_upper_bound, SIGNED);
2043 20453570 : return true;
2044 : }
2045 :
2046 : /* Similar to get_max_loop_iterations, but returns the estimate only
2047 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
2048 : on the number of iterations of LOOP could not be derived, returns -1. */
2049 :
2050 : HOST_WIDE_INT
2051 1734539 : get_max_loop_iterations_int (const class loop *loop)
2052 : {
2053 1734539 : widest_int nit;
2054 1734539 : HOST_WIDE_INT hwi_nit;
2055 :
2056 1734539 : if (!get_max_loop_iterations (loop, &nit))
2057 : return -1;
2058 :
2059 1353836 : if (!wi::fits_shwi_p (nit))
2060 : return -1;
2061 1294865 : hwi_nit = nit.to_shwi ();
2062 :
2063 1294865 : return hwi_nit < 0 ? -1 : hwi_nit;
2064 1734539 : }
2065 :
2066 : /* Sets NIT to an upper bound for the maximum number of executions of the
2067 : latch of the LOOP. If we have no reliable estimate, the function returns
2068 : false, otherwise returns true. */
2069 :
2070 : bool
2071 5372979 : get_likely_max_loop_iterations (class loop *loop, widest_int *nit)
2072 : {
2073 5372979 : if (!loop->any_likely_upper_bound)
2074 : return false;
2075 :
2076 4773147 : *nit = widest_int::from (loop->nb_iterations_likely_upper_bound, SIGNED);
2077 4773147 : return true;
2078 : }
2079 :
2080 : /* Similar to get_max_loop_iterations, but returns the estimate only
2081 : if it fits to HOST_WIDE_INT. If this is not the case, or the estimate
2082 : on the number of iterations of LOOP could not be derived, returns -1. */
2083 :
2084 : HOST_WIDE_INT
2085 4991141 : get_likely_max_loop_iterations_int (class loop *loop)
2086 : {
2087 4991141 : widest_int nit;
2088 4991141 : HOST_WIDE_INT hwi_nit;
2089 :
2090 4991141 : if (!get_likely_max_loop_iterations (loop, &nit))
2091 : return -1;
2092 :
2093 4523753 : if (!wi::fits_shwi_p (nit))
2094 : return -1;
2095 4241587 : hwi_nit = nit.to_shwi ();
2096 :
2097 4241587 : return hwi_nit < 0 ? -1 : hwi_nit;
2098 4991141 : }
2099 :
2100 : /* Returns the loop depth of the loop BB belongs to. */
2101 :
2102 : int
2103 55090685 : bb_loop_depth (const_basic_block bb)
2104 : {
2105 67352914 : return bb->loop_father ? loop_depth (bb->loop_father) : 0;
2106 : }
2107 :
2108 : /* Marks LOOP for removal and sets LOOPS_NEED_FIXUP. */
2109 :
2110 : void
2111 274124 : mark_loop_for_removal (loop_p loop)
2112 : {
2113 274124 : if (loop->header == NULL)
2114 : return;
2115 274124 : loop->former_header = loop->header;
2116 274124 : loop->header = NULL;
2117 274124 : loop->latch = NULL;
2118 274124 : loops_state_set (LOOPS_NEED_FIXUP);
2119 : }
2120 :
2121 : /* Starting from loop tree ROOT, walk loop tree as the visiting
2122 : order specified by FLAGS. The supported visiting orders
2123 : are:
2124 : - LI_ONLY_INNERMOST
2125 : - LI_FROM_INNERMOST
2126 : - Preorder (if neither of above is specified) */
2127 :
2128 : void
2129 1371188425 : loops_list::walk_loop_tree (class loop *root, unsigned flags)
2130 : {
2131 1371188425 : bool only_innermost_p = flags & LI_ONLY_INNERMOST;
2132 1371188425 : bool from_innermost_p = flags & LI_FROM_INNERMOST;
2133 1371188425 : bool preorder_p = !(only_innermost_p || from_innermost_p);
2134 :
2135 : /* Early handle root without any inner loops, make later
2136 : processing simpler, that is all loops processed in the
2137 : following while loop are impossible to be root. */
2138 1371188425 : if (!root->inner)
2139 : {
2140 1108171379 : if (flags & LI_INCLUDE_ROOT)
2141 11909 : this->to_visit.quick_push (root->num);
2142 1108171379 : return;
2143 : }
2144 263017046 : else if (preorder_p && flags & LI_INCLUDE_ROOT)
2145 51111 : this->to_visit.quick_push (root->num);
2146 :
2147 : class loop *aloop;
2148 68393976 : for (aloop = root->inner;
2149 331411022 : aloop->inner != NULL;
2150 68393976 : aloop = aloop->inner)
2151 : {
2152 68393976 : if (preorder_p)
2153 49649220 : this->to_visit.quick_push (aloop->num);
2154 68393976 : continue;
2155 : }
2156 :
2157 916678078 : while (1)
2158 : {
2159 916678078 : gcc_assert (aloop != root);
2160 916678078 : if (from_innermost_p || aloop->inner == NULL)
2161 787585966 : this->to_visit.quick_push (aloop->num);
2162 :
2163 916678078 : if (aloop->next)
2164 : {
2165 99959280 : for (aloop = aloop->next;
2166 585267056 : aloop->inner != NULL;
2167 99959280 : aloop = aloop->inner)
2168 : {
2169 99959280 : if (preorder_p)
2170 79442892 : this->to_visit.quick_push (aloop->num);
2171 99959280 : continue;
2172 : }
2173 : }
2174 431370302 : else if (loop_outer (aloop) == root)
2175 : break;
2176 : else
2177 : aloop = loop_outer (aloop);
2178 : }
2179 :
2180 : /* When visiting from innermost, we need to consider root here
2181 : since the previous while loop doesn't handle it. */
2182 263017046 : if (from_innermost_p && flags & LI_INCLUDE_ROOT)
2183 0 : this->to_visit.quick_push (root->num);
2184 68393976 : }
2185 :
|
