Line data Source code
1 : /* Callgraph transformations to handle inlining
2 : Copyright (C) 2003-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : /* The inline decisions are stored in callgraph in "inline plan" and
22 : applied later.
23 :
24 : To mark given call inline, use inline_call function.
25 : The function marks the edge inlinable and, if necessary, produces
26 : virtual clone in the callgraph representing the new copy of callee's
27 : function body.
28 :
29 : The inline plan is applied on given function body by inline_transform. */
30 :
31 : #define INCLUDE_ALGORITHM
32 : #include "config.h"
33 : #include "system.h"
34 : #include "coretypes.h"
35 : #include "tm.h"
36 : #include "function.h"
37 : #include "tree.h"
38 : #include "alloc-pool.h"
39 : #include "tree-pass.h"
40 : #include "cgraph.h"
41 : #include "tree-cfg.h"
42 : #include "symbol-summary.h"
43 : #include "tree-vrp.h"
44 : #include "sreal.h"
45 : #include "ipa-cp.h"
46 : #include "ipa-prop.h"
47 : #include "ipa-fnsummary.h"
48 : #include "ipa-inline.h"
49 : #include "tree-inline.h"
50 : #include "cfg.h"
51 : #include "basic-block.h"
52 : #include "ipa-utils.h"
53 : #include "ipa-modref-tree.h"
54 : #include "ipa-modref.h"
55 : #include "symtab-thunks.h"
56 : #include "symtab-clones.h"
57 :
58 : int ncalls_inlined;
59 : int nfunctions_inlined;
60 :
61 : /* We removed or are going to remove the last call to NODE.
62 : Return true if we can and want proactively remove the NODE now.
63 : This is important to do, since we want inliner to know when offline
64 : copy of function was removed. */
65 :
66 : static bool
67 2922647 : can_remove_node_now_p_1 (struct cgraph_node *node, struct cgraph_edge *e)
68 : {
69 2922647 : ipa_ref *ref;
70 :
71 3195751 : FOR_EACH_ALIAS (node, ref)
72 : {
73 843529 : cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
74 556367 : if ((alias->callers && alias->callers != e)
75 843529 : || !can_remove_node_now_p_1 (alias, e))
76 : return false;
77 : }
78 : /* FIXME: When address is taken of DECL_EXTERNAL function we still
79 : can remove its offline copy, but we would need to keep unanalyzed node in
80 : the callgraph so references can point to it.
81 :
82 : Also for comdat group we can ignore references inside a group as we
83 : want to prove the group as a whole to be dead. */
84 2352222 : return (!node->address_taken
85 2314346 : && node->can_remove_if_no_direct_calls_and_refs_p ()
86 : /* Inlining might enable more devirtualizing, so we want to remove
87 : those only after all devirtualizable virtual calls are processed.
88 : Lacking may edges in callgraph we just preserve them post
89 : inlining. */
90 2245223 : && (!DECL_VIRTUAL_P (node->decl)
91 6657 : || !opt_for_fn (node->decl, flag_devirtualize))
92 : /* During early inlining some unanalyzed cgraph nodes might be in the
93 : callgraph and they might refer the function in question. */
94 4590876 : && !cgraph_new_nodes.exists ());
95 : }
96 :
97 : /* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
98 : Verify that the NODE can be removed from unit and if it is contained in comdat
99 : group that the whole comdat group is removable. */
100 :
101 : static bool
102 2435649 : can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
103 : {
104 2435649 : struct cgraph_node *next;
105 2435649 : if (!can_remove_node_now_p_1 (node, e))
106 : return false;
107 :
108 : /* When we see same comdat group, we need to be sure that all
109 : items can be removed. */
110 1765740 : if (!node->same_comdat_group || !node->externally_visible)
111 : return true;
112 276010 : for (next = dyn_cast<cgraph_node *> (node->same_comdat_group);
113 822910 : next != node; next = dyn_cast<cgraph_node *> (next->same_comdat_group))
114 : {
115 276570 : if (next->alias)
116 73640 : continue;
117 202503 : if ((next->callers && next->callers != e)
118 402339 : || !can_remove_node_now_p_1 (next, e))
119 : return false;
120 : }
121 : return true;
122 : }
123 :
124 : /* Return true if NODE is a master clone with non-inline clones. */
125 :
126 : static bool
127 1537459 : master_clone_with_noninline_clones_p (struct cgraph_node *node)
128 : {
129 1537459 : if (node->clone_of)
130 : return false;
131 :
132 1959750 : for (struct cgraph_node *n = node->clones; n; n = n->next_sibling_clone)
133 537051 : if (n->decl != node->decl)
134 : return true;
135 :
136 : return false;
137 : }
138 :
139 : /* E is expected to be an edge being inlined. Clone destination node of
140 : the edge and redirect it to the new clone.
141 : DUPLICATE is used for bookkeeping on whether we are actually creating new
142 : clones or re-using node originally representing out-of-line function call.
143 : By default the offline copy is removed, when it appears dead after inlining.
144 : KEEP_OFFLINE_COPY prevents this transformation.
145 : If UPDATE_ORIGINAL is set, clones profile is subtracted from the offline version.
146 : If OVERALL_SIZE is non-NULL, the size is updated to reflect the
147 : transformation. */
148 :
149 : void
150 5070815 : clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
151 : bool keep_offline_copy,
152 : bool update_original, int *overall_size)
153 : {
154 5070815 : struct cgraph_node *inlining_into;
155 5070815 : struct cgraph_edge *next;
156 :
157 5070815 : if (e->caller->inlined_to)
158 : inlining_into = e->caller->inlined_to;
159 : else
160 3995462 : inlining_into = e->caller;
161 :
162 5070815 : if (duplicate)
163 : {
164 : /* We may eliminate the need for out-of-line copy to be output.
165 : In that case just go ahead and re-use it. This is not just an
166 : memory optimization. Making offline copy of function disappear
167 : from the program will improve future decisions on inlining. */
168 4818247 : if (!e->callee->callers->next_caller
169 : /* Recursive inlining never wants the master clone to
170 : be overwritten. */
171 2209724 : && !keep_offline_copy
172 2197605 : && can_remove_node_now_p (e->callee, e)
173 : /* We cannot overwrite a master clone with non-inline clones
174 : until after these clones are materialized. */
175 6355706 : && !master_clone_with_noninline_clones_p (e->callee))
176 : {
177 : /* TODO: When callee is in a comdat group, we could remove all of it,
178 : including all inline clones inlined into it. That would however
179 : need small function inlining to register edge removal hook to
180 : maintain the priority queue.
181 :
182 : For now we keep the other functions in the group in program until
183 : cgraph_remove_unreachable_functions gets rid of them. */
184 1537234 : gcc_assert (!e->callee->inlined_to);
185 1537234 : e->callee->remove_from_same_comdat_group ();
186 1537234 : if (e->callee->definition
187 1537234 : && inline_account_function_p (e->callee))
188 : {
189 1349086 : gcc_assert (!e->callee->alias);
190 1349086 : if (overall_size)
191 270086 : *overall_size -= ipa_size_summaries->get (e->callee)->size;
192 1349086 : nfunctions_inlined++;
193 : }
194 1537234 : duplicate = false;
195 1537234 : e->callee->externally_visible = false;
196 1537234 : profile_count num = e->count;
197 1537234 : profile_count den = e->callee->count;
198 1537234 : profile_count::adjust_for_ipa_scaling (&num, &den);
199 1537234 : e->callee->apply_scale (num, den);
200 :
201 1537234 : dump_callgraph_transformation (e->callee, inlining_into,
202 : "inlining to");
203 : }
204 : else
205 : {
206 3281013 : struct cgraph_node *n;
207 :
208 3281013 : n = e->callee->create_clone (e->callee->decl,
209 : e->count,
210 3281013 : update_original, vNULL, true,
211 : inlining_into,
212 : NULL, NULL);
213 3281013 : n->used_as_abstract_origin = e->callee->used_as_abstract_origin;
214 3281013 : e->redirect_callee (n);
215 : }
216 : }
217 : else
218 252568 : e->callee->remove_from_same_comdat_group ();
219 :
220 5070815 : e->callee->inlined_to = inlining_into;
221 5070815 : if (e->callee->ipa_transforms_to_apply.length ())
222 : {
223 316248 : e->callee->ipa_transforms_to_apply.release ();
224 316248 : e->callee->ipa_transforms_to_apply = vNULL;
225 : }
226 :
227 : /* Recursively clone all bodies. */
228 9866189 : for (e = e->callee->callees; e; e = next)
229 : {
230 4795374 : next = e->next_callee;
231 4795374 : if (!e->inline_failed)
232 773521 : clone_inlined_nodes (e, duplicate, keep_offline_copy,
233 : update_original, overall_size);
234 : }
235 5070815 : }
236 :
237 : /* Check all speculations in N and if any seem useless, resolve them. When a
238 : first edge is resolved, pop all edges from NEW_EDGES and insert them to
239 : EDGE_SET. Then remove each resolved edge from EDGE_SET, if it is there. */
240 :
241 : static bool
242 5069313 : check_speculations_1 (cgraph_node *n, vec<cgraph_edge *> *new_edges,
243 : hash_set <cgraph_edge *> *edge_set)
244 : {
245 5069313 : bool speculation_removed = false;
246 5069313 : cgraph_edge *next;
247 :
248 9865802 : for (cgraph_edge *e = n->callees; e; e = next)
249 : {
250 4796489 : next = e->next_callee;
251 4796489 : if (e->speculative && !speculation_useful_p (e, true))
252 : {
253 54 : while (new_edges && !new_edges->is_empty ())
254 6 : edge_set->add (new_edges->pop ());
255 48 : edge_set->remove (e);
256 :
257 48 : cgraph_edge::resolve_speculation (e, NULL);
258 48 : speculation_removed = true;
259 : }
260 4796441 : else if (!e->inline_failed)
261 772527 : speculation_removed |= check_speculations_1 (e->callee, new_edges,
262 : edge_set);
263 : }
264 5069313 : return speculation_removed;
265 : }
266 :
267 : /* Push E to NEW_EDGES. Called from hash_set traverse method, which
268 : unfortunately means this function has to have external linkage, otherwise
269 : the code will not compile with gcc 4.8. */
270 :
271 : bool
272 0 : push_all_edges_in_set_to_vec (cgraph_edge * const &e,
273 : vec<cgraph_edge *> *new_edges)
274 : {
275 0 : new_edges->safe_push (e);
276 0 : return true;
277 : }
278 :
279 : /* Check all speculations in N and if any seem useless, resolve them and remove
280 : them from NEW_EDGES. */
281 :
282 : static bool
283 4296786 : check_speculations (cgraph_node *n, vec<cgraph_edge *> *new_edges)
284 : {
285 4296786 : hash_set <cgraph_edge *> edge_set;
286 4296786 : bool res = check_speculations_1 (n, new_edges, &edge_set);
287 4296786 : if (!edge_set.is_empty ())
288 0 : edge_set.traverse <vec<cgraph_edge *> *,
289 0 : push_all_edges_in_set_to_vec> (new_edges);
290 4296786 : return res;
291 4296786 : }
292 :
293 : /* Mark all call graph edges coming out of NODE and all nodes that have been
294 : inlined to it as in_polymorphic_cdtor. */
295 :
296 : static void
297 136592 : mark_all_inlined_calls_cdtor (cgraph_node *node)
298 : {
299 287920 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
300 : {
301 151328 : cs->in_polymorphic_cdtor = true;
302 151328 : if (!cs->inline_failed)
303 28243 : mark_all_inlined_calls_cdtor (cs->callee);
304 : }
305 144181 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
306 7589 : cs->in_polymorphic_cdtor = true;
307 136592 : }
308 :
309 :
310 : /* Mark edge E as inlined and update callgraph accordingly. UPDATE_ORIGINAL
311 : specify whether profile of original function should be updated and whether
312 : offline copy should be removed if unnecessary. If any new
313 : indirect edges are discovered in the process, add them to NEW_EDGES, unless
314 : it is NULL. If UPDATE_OVERALL_SUMMARY is false, do not bother to recompute overall
315 : size of caller after inlining. Caller is required to eventually do it via
316 : ipa_update_overall_fn_summary.
317 : If callee_removed is non-NULL, set it to true if we removed callee node.
318 :
319 : Return true iff any new callgraph edges were discovered as a
320 : result of inlining. */
321 :
322 : bool
323 4296786 : inline_call (struct cgraph_edge *e, bool update_original,
324 : vec<cgraph_edge *> *new_edges,
325 : int *overall_size, bool update_overall_summary,
326 : bool *callee_removed)
327 : {
328 4296786 : int old_size = 0, new_size = 0;
329 4296786 : struct cgraph_node *to = NULL;
330 4296786 : struct cgraph_edge *curr = e;
331 4296786 : bool comdat_local = e->callee->comdat_local_p ();
332 4296786 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
333 4296786 : bool new_edges_found = false;
334 4296786 : bool keep_offline_copy = !update_original;
335 :
336 4296786 : int estimated_growth = 0;
337 4296786 : if (! update_overall_summary)
338 3323455 : estimated_growth = estimate_edge_growth (e);
339 : /* This is used only for assert below. */
340 : #if 0
341 : bool predicated = inline_edge_summary (e)->predicate != NULL;
342 : #endif
343 :
344 : /* Don't inline inlined edges. */
345 4296786 : gcc_assert (e->inline_failed);
346 : /* Don't even think of inlining inline clone. */
347 4296786 : gcc_assert (!callee->inlined_to);
348 :
349 4296786 : to = e->caller;
350 4296786 : if (to->inlined_to)
351 301832 : to = to->inlined_to;
352 :
353 : /* In case callee has AFDO profile but caller has GLOBAL0 we need
354 : to re-scale it so it can have non-zero AFDO profile. */
355 4296786 : if (callee->count.quality () == AFDO
356 0 : && e->count.nonzero_p ()
357 4296786 : && (to->count.quality () == GUESSED_GLOBAL0_AFDO
358 0 : || to->count.quality () == GUESSED_GLOBAL0_ADJUSTED))
359 : {
360 0 : profile_count num = callee->count;
361 0 : profile_count den = e->count;
362 0 : profile_count::adjust_for_ipa_scaling (&num, &den);
363 0 : if (dump_file)
364 : {
365 0 : fprintf (dump_file, "Rescalling profile of caller %s "
366 : "to allow non-zero AFDO counts:",
367 : to->dump_name ());
368 0 : den.dump (dump_file);
369 0 : fprintf (dump_file, " -> ");
370 0 : num.dump (dump_file);
371 0 : fprintf (dump_file, "\n");
372 : }
373 0 : to->apply_scale (num, den);
374 0 : to->frequency = std::max (to->frequency, callee->frequency);
375 : /* Do not update original, so possible additional calls of callee
376 : are handled reasonably well. */
377 0 : update_original = false;
378 0 : gcc_checking_assert (to->count.quality () == AFDO);
379 0 : if (dump_file)
380 : {
381 0 : fprintf (dump_file, "Scaled profile of %s: ", to->dump_name ());
382 0 : to->count.dump (dump_file);
383 0 : fprintf (dump_file, "\n");
384 : }
385 : }
386 : /* Do sanity checking of the profile and in case of inconsistencies do not
387 : update profile of original. This reduces the chances that inlining
388 : turns callee cold while in reality it is still hot. */
389 4296786 : if (!(callee->count.ipa ().force_nonzero () == callee->count.ipa ()))
390 : {
391 9823 : if (dump_file)
392 0 : fprintf (dump_file, "Callee count is 0; not updating callee profile\n");
393 : update_original = false;
394 : }
395 4286963 : else if (e->count.ipa ().quality () == AFDO
396 4286963 : && !(e->count.ipa ().force_nonzero () == e->count.ipa ()))
397 : {
398 0 : if (dump_file)
399 0 : fprintf (dump_file, "Edge count is AFDO 0; not updating callee profile\n");
400 : update_original = false;
401 : }
402 4296786 : if (e->count.ipa () > callee->count.ipa ().apply_scale (9, 8))
403 : {
404 0 : if (dump_file)
405 0 : fprintf (dump_file, "Calee count is too small (profile is inconsistent);"
406 : " not updating callee profile\n");
407 : update_original = false;
408 : }
409 4296786 : if (to->thunk)
410 : {
411 1188 : struct cgraph_node *target = to->callees->callee;
412 1188 : thunk_expansion = true;
413 :
414 : /* Remove all annotations, but keep thunk info. */
415 1188 : thunk_info info = *thunk_info::get (to);
416 1188 : symtab->call_cgraph_removal_hooks (to);
417 1188 : *thunk_info::get_create (to) = info;
418 1188 : if (in_lto_p)
419 58 : to->get_untransformed_body ();
420 1188 : expand_thunk (to, false, true);
421 : /* When thunk is instrumented we may have multiple callees. */
422 1188 : for (e = to->callees; e && e->callee != target; e = e->next_callee)
423 : ;
424 1188 : symtab->call_cgraph_insertion_hooks (to);
425 1188 : thunk_expansion = false;
426 1188 : gcc_assert (e);
427 : }
428 :
429 :
430 4296786 : e->inline_failed = CIF_OK;
431 4296786 : DECL_POSSIBLY_INLINED (callee->decl) = true;
432 :
433 4296786 : if (DECL_FUNCTION_PERSONALITY (callee->decl))
434 311168 : DECL_FUNCTION_PERSONALITY (to->decl)
435 155584 : = DECL_FUNCTION_PERSONALITY (callee->decl);
436 :
437 4296786 : bool reload_optimization_node = false;
438 4296786 : bool remove_strict_aliasing
439 4296786 : = (!opt_for_fn (callee->decl, flag_strict_aliasing)
440 4296786 : && opt_for_fn (to->decl, flag_strict_aliasing));
441 4296786 : bool remove_assume_sane_operators_new_delete
442 4296786 : = (!opt_for_fn (callee->decl, flag_assume_sane_operators_new_delete)
443 4296786 : && opt_for_fn (to->decl, flag_assume_sane_operators_new_delete));
444 4296768 : if (remove_strict_aliasing || remove_assume_sane_operators_new_delete)
445 : {
446 31 : struct gcc_options opts = global_options;
447 31 : struct gcc_options opts_set = global_options_set;
448 :
449 31 : cl_optimization_restore (&opts, &opts_set, opts_for_fn (to->decl));
450 31 : if (remove_strict_aliasing)
451 : {
452 13 : opts.x_flag_strict_aliasing = false;
453 13 : if (dump_file)
454 0 : fprintf (dump_file, "Dropping flag_strict_aliasing on %s\n",
455 : to->dump_name ());
456 : }
457 31 : if (remove_assume_sane_operators_new_delete)
458 : {
459 18 : opts.x_flag_assume_sane_operators_new_delete = false;
460 18 : if (dump_file)
461 0 : fprintf (dump_file,
462 : "Dropping flag_assume_sane_operators_new_delete on %s\n",
463 : to->dump_name ());
464 : }
465 31 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
466 31 : = build_optimization_node (&opts, &opts_set);
467 31 : reload_optimization_node = true;
468 : }
469 :
470 4296786 : ipa_fn_summary *caller_info = ipa_fn_summaries->get (to);
471 4296786 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (callee);
472 4296786 : if (!caller_info->fp_expressions && callee_info->fp_expressions)
473 : {
474 20611 : caller_info->fp_expressions = true;
475 20611 : if (opt_for_fn (callee->decl, flag_rounding_math)
476 20611 : != opt_for_fn (to->decl, flag_rounding_math)
477 20611 : || opt_for_fn (callee->decl, flag_trapping_math)
478 20611 : != opt_for_fn (to->decl, flag_trapping_math)
479 20604 : || opt_for_fn (callee->decl, flag_unsafe_math_optimizations)
480 20604 : != opt_for_fn (to->decl, flag_unsafe_math_optimizations)
481 20604 : || opt_for_fn (callee->decl, flag_finite_math_only)
482 20604 : != opt_for_fn (to->decl, flag_finite_math_only)
483 20604 : || opt_for_fn (callee->decl, flag_signaling_nans)
484 20604 : != opt_for_fn (to->decl, flag_signaling_nans)
485 20604 : || opt_for_fn (callee->decl, flag_complex_method)
486 20604 : != opt_for_fn (to->decl, flag_complex_method)
487 20604 : || opt_for_fn (callee->decl, flag_signed_zeros)
488 20604 : != opt_for_fn (to->decl, flag_signed_zeros)
489 20604 : || opt_for_fn (callee->decl, flag_associative_math)
490 20604 : != opt_for_fn (to->decl, flag_associative_math)
491 20603 : || opt_for_fn (callee->decl, flag_reciprocal_math)
492 20603 : != opt_for_fn (to->decl, flag_reciprocal_math)
493 20603 : || opt_for_fn (callee->decl, flag_fp_int_builtin_inexact)
494 20603 : != opt_for_fn (to->decl, flag_fp_int_builtin_inexact)
495 20611 : || opt_for_fn (callee->decl, flag_errno_math)
496 20603 : != opt_for_fn (to->decl, flag_errno_math))
497 : {
498 8 : struct gcc_options opts = global_options;
499 8 : struct gcc_options opts_set = global_options_set;
500 :
501 8 : cl_optimization_restore (&opts, &opts_set, opts_for_fn (to->decl));
502 8 : opts.x_flag_rounding_math
503 8 : = opt_for_fn (callee->decl, flag_rounding_math);
504 8 : opts.x_flag_trapping_math
505 8 : = opt_for_fn (callee->decl, flag_trapping_math);
506 8 : opts.x_flag_unsafe_math_optimizations
507 8 : = opt_for_fn (callee->decl, flag_unsafe_math_optimizations);
508 8 : opts.x_flag_finite_math_only
509 8 : = opt_for_fn (callee->decl, flag_finite_math_only);
510 8 : opts.x_flag_signaling_nans
511 8 : = opt_for_fn (callee->decl, flag_signaling_nans);
512 8 : opts.x_flag_complex_method
513 8 : = opt_for_fn (callee->decl, flag_complex_method);
514 8 : opts.x_flag_signed_zeros
515 8 : = opt_for_fn (callee->decl, flag_signed_zeros);
516 8 : opts.x_flag_associative_math
517 8 : = opt_for_fn (callee->decl, flag_associative_math);
518 8 : opts.x_flag_reciprocal_math
519 8 : = opt_for_fn (callee->decl, flag_reciprocal_math);
520 8 : opts.x_flag_fp_int_builtin_inexact
521 8 : = opt_for_fn (callee->decl, flag_fp_int_builtin_inexact);
522 8 : opts.x_flag_errno_math
523 8 : = opt_for_fn (callee->decl, flag_errno_math);
524 8 : if (dump_file)
525 0 : fprintf (dump_file, "Copying FP flags from %s to %s\n",
526 : callee->dump_name (), to->dump_name ());
527 8 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
528 8 : = build_optimization_node (&opts, &opts_set);
529 8 : reload_optimization_node = true;
530 : }
531 : }
532 :
533 : /* Reload global optimization flags. */
534 4296786 : if (reload_optimization_node && DECL_STRUCT_FUNCTION (to->decl) == cfun)
535 34 : set_cfun (cfun, true);
536 :
537 : /* If aliases are involved, redirect edge to the actual destination and
538 : possibly remove the aliases. */
539 4296786 : if (e->callee != callee)
540 : {
541 787075 : struct cgraph_node *alias = e->callee, *next_alias;
542 787075 : e->redirect_callee (callee);
543 1799311 : while (alias && alias != callee)
544 : {
545 787094 : if (!alias->callers
546 1025138 : && can_remove_node_now_p (alias,
547 238044 : !e->next_caller && !e->prev_caller ? e : NULL))
548 : {
549 225161 : next_alias = alias->get_alias_target ();
550 225161 : alias->remove ();
551 225161 : if (callee_removed)
552 1287 : *callee_removed = true;
553 225161 : alias = next_alias;
554 : }
555 : else
556 : break;
557 : }
558 : }
559 :
560 4296786 : if (callee->must_remain_in_tu_body)
561 : {
562 0 : gcc_assert (callee->lto_file_data == to->lto_file_data);
563 0 : to->must_remain_in_tu_body = true;
564 : }
565 :
566 4296786 : clone_inlined_nodes (e, true, keep_offline_copy,
567 : update_original, overall_size);
568 :
569 4296786 : gcc_assert (curr->callee->inlined_to == to);
570 :
571 4296786 : old_size = ipa_size_summaries->get (to)->size;
572 4296786 : ipa_merge_modref_summary_after_inlining (e);
573 4296786 : ipa_merge_fn_summary_after_inlining (e);
574 4296786 : if (e->in_polymorphic_cdtor)
575 108349 : mark_all_inlined_calls_cdtor (e->callee);
576 4296786 : if (opt_for_fn (e->caller->decl, optimize))
577 4267107 : new_edges_found = ipa_propagate_indirect_call_infos (curr, new_edges);
578 4296786 : bool removed_p = check_speculations (e->callee, new_edges);
579 4296786 : if (update_overall_summary)
580 973331 : ipa_update_overall_fn_summary (to, new_edges_found || removed_p);
581 : else
582 : /* Update self size by the estimate so overall function growth limits
583 : work for further inlining into this function. Before inlining
584 : the function we inlined to again we expect the caller to update
585 : the overall summary. */
586 3323455 : ipa_size_summaries->get (to)->size += estimated_growth;
587 4296786 : new_size = ipa_size_summaries->get (to)->size;
588 :
589 4296786 : if (callee->calls_comdat_local)
590 0 : to->calls_comdat_local = true;
591 4296786 : else if (to->calls_comdat_local && comdat_local)
592 3637 : to->calls_comdat_local = to->check_calls_comdat_local_p ();
593 :
594 : /* FIXME: This assert suffers from roundoff errors, disable it for GCC 5
595 : and revisit it after conversion to sreals in GCC 6.
596 : See PR 65654. */
597 : #if 0
598 : /* Verify that estimated growth match real growth. Allow off-by-one
599 : error due to ipa_fn_summary::size_scale roudoff errors. */
600 : gcc_assert (!update_overall_summary || !overall_size || new_edges_found
601 : || abs (estimated_growth - (new_size - old_size)) <= 1
602 : || speculation_removed
603 : /* FIXME: a hack. Edges with false predicate are accounted
604 : wrong, we should remove them from callgraph. */
605 : || predicated);
606 : #endif
607 :
608 : /* Account the change of overall unit size; external functions will be
609 : removed and are thus not accounted. */
610 4296786 : if (overall_size && inline_account_function_p (to))
611 955795 : *overall_size += new_size - old_size;
612 4296786 : ncalls_inlined++;
613 :
614 : /* This must happen after ipa_merge_fn_summary_after_inlining that rely on jump
615 : functions of callee to not be updated. */
616 4296786 : return new_edges_found;
617 : }
618 :
619 : /* For each node that was made the holder of function body by
620 : save_inline_function_body, this summary contains pointer to the previous
621 : holder of the body. */
622 :
623 : function_summary <tree *> *ipa_saved_clone_sources;
624 :
625 : /* Copy function body of NODE and redirect all inline clones to it.
626 : This is done before inline plan is applied to NODE when there are
627 : still some inline clones if it.
628 :
629 : This is necessary because inline decisions are not really transitive
630 : and the other inline clones may have different bodies. */
631 :
632 : static struct cgraph_node *
633 52671 : save_inline_function_body (struct cgraph_node *node)
634 : {
635 52671 : struct cgraph_node *first_clone, *n;
636 :
637 52671 : if (dump_file)
638 153 : fprintf (dump_file, "\nSaving body of %s for later reuse\n",
639 : node->dump_name ());
640 :
641 52671 : gcc_assert (node == cgraph_node::get (node->decl));
642 :
643 : /* first_clone will be turned into real function. */
644 52671 : first_clone = node->clones;
645 :
646 : /* Arrange first clone to not be thunk as those do not have bodies. */
647 52671 : if (first_clone->thunk)
648 : {
649 0 : while (first_clone->thunk)
650 0 : first_clone = first_clone->next_sibling_clone;
651 0 : first_clone->prev_sibling_clone->next_sibling_clone
652 0 : = first_clone->next_sibling_clone;
653 0 : if (first_clone->next_sibling_clone)
654 0 : first_clone->next_sibling_clone->prev_sibling_clone
655 0 : = first_clone->prev_sibling_clone;
656 0 : first_clone->next_sibling_clone = node->clones;
657 0 : first_clone->prev_sibling_clone = NULL;
658 0 : node->clones->prev_sibling_clone = first_clone;
659 0 : node->clones = first_clone;
660 : }
661 52671 : first_clone->decl = copy_node (node->decl);
662 52671 : first_clone->decl->decl_with_vis.symtab_node = first_clone;
663 52671 : gcc_assert (first_clone == cgraph_node::get (first_clone->decl));
664 :
665 : /* Now reshape the clone tree, so all other clones descends from
666 : first_clone. */
667 52671 : if (first_clone->next_sibling_clone)
668 : {
669 180972 : for (n = first_clone->next_sibling_clone; n->next_sibling_clone;
670 154857 : n = n->next_sibling_clone)
671 154857 : n->clone_of = first_clone;
672 26115 : n->clone_of = first_clone;
673 26115 : n->next_sibling_clone = first_clone->clones;
674 26115 : if (first_clone->clones)
675 547 : first_clone->clones->prev_sibling_clone = n;
676 26115 : first_clone->clones = first_clone->next_sibling_clone;
677 26115 : first_clone->next_sibling_clone->prev_sibling_clone = NULL;
678 26115 : first_clone->next_sibling_clone = NULL;
679 26115 : gcc_assert (!first_clone->prev_sibling_clone);
680 : }
681 :
682 52671 : tree prev_body_holder = node->decl;
683 52671 : if (!ipa_saved_clone_sources)
684 : {
685 16368 : ipa_saved_clone_sources = new function_summary <tree *> (symtab);
686 16368 : ipa_saved_clone_sources->disable_insertion_hook ();
687 : }
688 : else
689 : {
690 36303 : tree *p = ipa_saved_clone_sources->get (node);
691 36303 : if (p)
692 : {
693 0 : prev_body_holder = *p;
694 0 : gcc_assert (prev_body_holder);
695 : }
696 : }
697 52671 : *ipa_saved_clone_sources->get_create (first_clone) = prev_body_holder;
698 52671 : first_clone->former_clone_of
699 52671 : = node->former_clone_of ? node->former_clone_of : node->decl;
700 52671 : first_clone->clone_of = NULL;
701 :
702 : /* Now node in question has no clones. */
703 52671 : node->clones = NULL;
704 :
705 : /* Inline clones share decl with the function they are cloned
706 : from. Walk the whole clone tree and redirect them all to the
707 : new decl. */
708 52671 : if (first_clone->clones)
709 265731 : for (n = first_clone->clones; n != first_clone;)
710 : {
711 237339 : gcc_assert (n->decl == node->decl);
712 237339 : n->decl = first_clone->decl;
713 237339 : if (n->clones)
714 : n = n->clones;
715 230684 : else if (n->next_sibling_clone)
716 : n = n->next_sibling_clone;
717 : else
718 : {
719 66682 : while (n != first_clone && !n->next_sibling_clone)
720 35047 : n = n->clone_of;
721 31635 : if (n != first_clone)
722 3243 : n = n->next_sibling_clone;
723 : }
724 : }
725 :
726 : /* Copy the OLD_VERSION_NODE function tree to the new version. */
727 52671 : tree_function_versioning (node->decl, first_clone->decl,
728 : NULL, NULL, true, NULL, NULL);
729 :
730 : /* The function will be short lived and removed after we inline all the
731 : clones, but make it internal so we won't confuse ourself. */
732 52671 : DECL_EXTERNAL (first_clone->decl) = 0;
733 52671 : TREE_PUBLIC (first_clone->decl) = 0;
734 52671 : DECL_COMDAT (first_clone->decl) = 0;
735 52671 : first_clone->ipa_transforms_to_apply.release ();
736 :
737 : /* When doing recursive inlining, the clone may become unnecessary.
738 : This is possible i.e. in the case when the recursive function is proved to
739 : be non-throwing and the recursion happens only in the EH landing pad.
740 : We cannot remove the clone until we are done with saving the body.
741 : Remove it now. */
742 52671 : if (!first_clone->callers)
743 : {
744 0 : first_clone->remove_symbol_and_inline_clones ();
745 0 : first_clone = NULL;
746 : }
747 52671 : else if (flag_checking)
748 52671 : first_clone->verify ();
749 :
750 52671 : return first_clone;
751 : }
752 :
753 : /* Return true when function body of DECL still needs to be kept around
754 : for later re-use. */
755 : static bool
756 1508255 : preserve_function_body_p (struct cgraph_node *node)
757 : {
758 1508255 : gcc_assert (symtab->global_info_ready);
759 1508255 : gcc_assert (!node->alias && !node->thunk);
760 :
761 : /* Look if there is any non-thunk clone around. */
762 1508302 : for (node = node->clones; node; node = node->next_sibling_clone)
763 52718 : if (!node->thunk)
764 : return true;
765 : return false;
766 : }
767 :
768 : /* tree-inline can not recurse; materialize all function bodie we will need
769 : during inlining. This includes inlined functions, but also called functions
770 : with param manipulation because IPA param manipulation attaches debug
771 : statements to PARM_DECLs of called clone. Materialize them if needed.
772 :
773 : FIXME: This is somewhat broken by design because it does not play well
774 : with partitioning. */
775 :
776 : static void
777 3016379 : maybe_materialize_called_clones (cgraph_node *node)
778 : {
779 12262357 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
780 : {
781 9245978 : clone_info *info;
782 :
783 9245978 : if (!e->inline_failed)
784 1508124 : maybe_materialize_called_clones (e->callee);
785 :
786 9245978 : cgraph_node *callee = cgraph_node::get (e->callee->decl);
787 9245978 : if (callee->clone_of
788 9245978 : && (info = clone_info::get (callee)) && info->param_adjustments)
789 111658 : callee->get_untransformed_body ();
790 : }
791 3016379 : }
792 :
793 : /* Apply inline plan to function. */
794 :
795 : unsigned int
796 1508255 : inline_transform (struct cgraph_node *node)
797 : {
798 1508255 : unsigned int todo = 0;
799 1508255 : struct cgraph_edge *e, *next;
800 1508255 : bool has_inline = false;
801 :
802 : /* FIXME: Currently the pass manager is adding inline transform more than
803 : once to some clones. This needs revisiting after WPA cleanups. */
804 1508255 : if (cfun->after_inlining)
805 : return 0;
806 :
807 1508255 : cgraph_node *next_clone;
808 1741945 : for (cgraph_node *n = node->clones; n; n = next_clone)
809 : {
810 233690 : next_clone = n->next_sibling_clone;
811 233690 : if (n->decl != node->decl)
812 0 : n->materialize_clone ();
813 : }
814 1508255 : node->clear_stmts_in_references ();
815 :
816 : /* We might need the body of this function so that we can expand
817 : it inline somewhere else. */
818 1508255 : if (preserve_function_body_p (node))
819 52671 : save_inline_function_body (node);
820 :
821 1508255 : profile_count num = node->count;
822 1508255 : profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
823 1508255 : bool scale = num.initialized_p () && !(num == den);
824 105 : if (scale)
825 : {
826 105 : profile_count::adjust_for_ipa_scaling (&num, &den);
827 105 : if (dump_file)
828 : {
829 0 : fprintf (dump_file, "Applying count scale ");
830 0 : num.dump (dump_file);
831 0 : fprintf (dump_file, "/");
832 0 : den.dump (dump_file);
833 0 : fprintf (dump_file, "\n");
834 : }
835 :
836 105 : basic_block bb;
837 105 : cfun->cfg->count_max = profile_count::uninitialized ();
838 813 : FOR_ALL_BB_FN (bb, cfun)
839 : {
840 708 : bb->count = bb->count.apply_scale (num, den);
841 708 : cfun->cfg->count_max = profile_count::max_prefer_initialized
842 708 : (cfun->cfg->count_max, bb->count);
843 : }
844 105 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
845 : }
846 :
847 1508255 : maybe_materialize_called_clones (node);
848 7612546 : for (e = node->callees; e; e = next)
849 : {
850 6104291 : if (!e->inline_failed)
851 556819 : has_inline = true;
852 6104291 : next = e->next_callee;
853 6104291 : if (e->has_callback)
854 : {
855 : /* Redirect callback edges when redirecting their carrying edge. */
856 13363 : cgraph_edge *cbe;
857 13363 : cgraph_edge::redirect_call_stmt_to_callee (e);
858 14756 : for (cbe = e->first_callback_edge (); cbe;
859 1393 : cbe = cbe->next_callback_edge ())
860 1393 : cgraph_edge::redirect_call_stmt_to_callee (cbe);
861 : }
862 : else
863 6090928 : cgraph_edge::redirect_call_stmt_to_callee (e);
864 : }
865 1508255 : node->remove_all_references ();
866 :
867 1508255 : timevar_push (TV_INTEGRATION);
868 1508255 : if (node->callees && (opt_for_fn (node->decl, optimize) || has_inline))
869 : {
870 827257 : todo = optimize_inline_calls (current_function_decl);
871 : }
872 1508255 : timevar_pop (TV_INTEGRATION);
873 :
874 1508255 : cfun->always_inline_functions_inlined = true;
875 1508255 : cfun->after_inlining = true;
876 1508255 : todo |= execute_fixup_cfg ();
877 :
878 1508255 : if (!(todo & TODO_update_ssa_any))
879 : /* Redirecting edges might lead to a need for vops to be recomputed. */
880 1124787 : todo |= TODO_update_ssa_only_virtuals;
881 :
882 : return todo;
883 : }
|
