Branch data Line data Source code
1 : : /* Callgraph transformations to handle inlining
2 : : Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : /* The inline decisions are stored in callgraph in "inline plan" and
22 : : applied later.
23 : :
24 : : To mark given call inline, use inline_call function.
25 : : The function marks the edge inlinable and, if necessary, produces
26 : : virtual clone in the callgraph representing the new copy of callee's
27 : : function body.
28 : :
29 : : The inline plan is applied on given function body by inline_transform. */
30 : :
31 : : #include "config.h"
32 : : #include "system.h"
33 : : #include "coretypes.h"
34 : : #include "tm.h"
35 : : #include "function.h"
36 : : #include "tree.h"
37 : : #include "alloc-pool.h"
38 : : #include "tree-pass.h"
39 : : #include "cgraph.h"
40 : : #include "tree-cfg.h"
41 : : #include "symbol-summary.h"
42 : : #include "tree-vrp.h"
43 : : #include "sreal.h"
44 : : #include "ipa-cp.h"
45 : : #include "ipa-prop.h"
46 : : #include "ipa-fnsummary.h"
47 : : #include "ipa-inline.h"
48 : : #include "tree-inline.h"
49 : : #include "function.h"
50 : : #include "cfg.h"
51 : : #include "basic-block.h"
52 : : #include "ipa-utils.h"
53 : : #include "ipa-modref-tree.h"
54 : : #include "ipa-modref.h"
55 : : #include "symtab-thunks.h"
56 : : #include "symtab-clones.h"
57 : :
58 : : int ncalls_inlined;
59 : : int nfunctions_inlined;
60 : :
61 : : /* Scale counts of NODE edges by NUM/DEN. */
62 : :
63 : : static void
64 : 1427393 : update_noncloned_counts (struct cgraph_node *node,
65 : : profile_count num, profile_count den)
66 : : {
67 : 1427393 : struct cgraph_edge *e;
68 : :
69 : 1427393 : profile_count::adjust_for_ipa_scaling (&num, &den);
70 : :
71 : 2919984 : for (e = node->callees; e; e = e->next_callee)
72 : : {
73 : 1492591 : if (!e->inline_failed)
74 : 159640 : update_noncloned_counts (e->callee, num, den);
75 : 1492591 : e->count = e->count.apply_scale (num, den);
76 : : }
77 : 1460926 : for (e = node->indirect_calls; e; e = e->next_callee)
78 : 33533 : e->count = e->count.apply_scale (num, den);
79 : 1427393 : node->count = node->count.apply_scale (num, den);
80 : 1427393 : }
81 : :
82 : : /* We removed or are going to remove the last call to NODE.
83 : : Return true if we can and want proactively remove the NODE now.
84 : : This is important to do, since we want inliner to know when offline
85 : : copy of function was removed. */
86 : :
87 : : static bool
88 : 2444564 : can_remove_node_now_p_1 (struct cgraph_node *node, struct cgraph_edge *e)
89 : : {
90 : 2444564 : ipa_ref *ref;
91 : :
92 : 2710079 : FOR_EACH_ALIAS (node, ref)
93 : : {
94 : 707604 : cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
95 : 429450 : if ((alias->callers && alias->callers != e)
96 : 707604 : || !can_remove_node_now_p_1 (alias, e))
97 : 442089 : return false;
98 : : }
99 : : /* FIXME: When address is taken of DECL_EXTERNAL function we still
100 : : can remove its offline copy, but we would need to keep unanalyzed node in
101 : : the callgraph so references can point to it.
102 : :
103 : : Also for comdat group we can ignore references inside a group as we
104 : : want to prove the group as a whole to be dead. */
105 : 2002475 : return (!node->address_taken
106 : 1973106 : && node->can_remove_if_no_direct_calls_and_refs_p ()
107 : : /* Inlining might enable more devirtualizing, so we want to remove
108 : : those only after all devirtualizable virtual calls are processed.
109 : : Lacking may edges in callgraph we just preserve them post
110 : : inlining. */
111 : 1912326 : && (!DECL_VIRTUAL_P (node->decl)
112 : 6272 : || !opt_for_fn (node->decl, flag_devirtualize))
113 : : /* During early inlining some unanalyzed cgraph nodes might be in the
114 : : callgraph and they might refer the function in question. */
115 : 3908619 : && !cgraph_new_nodes.exists ());
116 : : }
117 : :
118 : : /* We are going to eliminate last direct call to NODE (or alias of it) via edge E.
119 : : Verify that the NODE can be removed from unit and if it is contained in comdat
120 : : group that the whole comdat group is removable. */
121 : :
122 : : static bool
123 : 1993695 : can_remove_node_now_p (struct cgraph_node *node, struct cgraph_edge *e)
124 : : {
125 : 1993695 : struct cgraph_node *next;
126 : 1993695 : if (!can_remove_node_now_p_1 (node, e))
127 : : return false;
128 : :
129 : : /* When we see same comdat group, we need to be sure that all
130 : : items can be removed. */
131 : 1467936 : if (!node->same_comdat_group || !node->externally_visible)
132 : : return true;
133 : 257775 : for (next = dyn_cast<cgraph_node *> (node->same_comdat_group);
134 : 782077 : next != node; next = dyn_cast<cgraph_node *> (next->same_comdat_group))
135 : : {
136 : 264455 : if (next->alias)
137 : 89458 : continue;
138 : 174622 : if ((next->callers && next->callers != e)
139 : 347337 : || !can_remove_node_now_p_1 (next, e))
140 : : return false;
141 : : }
142 : : return true;
143 : : }
144 : :
145 : : /* Return true if NODE is a master clone with non-inline clones. */
146 : :
147 : : static bool
148 : 1267794 : master_clone_with_noninline_clones_p (struct cgraph_node *node)
149 : : {
150 : 1267794 : if (node->clone_of)
151 : : return false;
152 : :
153 : 1612116 : for (struct cgraph_node *n = node->clones; n; n = n->next_sibling_clone)
154 : 430451 : if (n->decl != node->decl)
155 : : return true;
156 : :
157 : : return false;
158 : : }
159 : :
160 : : /* E is expected to be an edge being inlined. Clone destination node of
161 : : the edge and redirect it to the new clone.
162 : : DUPLICATE is used for bookkeeping on whether we are actually creating new
163 : : clones or re-using node originally representing out-of-line function call.
164 : : By default the offline copy is removed, when it appears dead after inlining.
165 : : UPDATE_ORIGINAL prevents this transformation.
166 : : If OVERALL_SIZE is non-NULL, the size is updated to reflect the
167 : : transformation. */
168 : :
169 : : void
170 : 3912880 : clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
171 : : bool update_original, int *overall_size)
172 : : {
173 : 3912880 : struct cgraph_node *inlining_into;
174 : 3912880 : struct cgraph_edge *next;
175 : :
176 : 3912880 : if (e->caller->inlined_to)
177 : : inlining_into = e->caller->inlined_to;
178 : : else
179 : 3192858 : inlining_into = e->caller;
180 : :
181 : 3912880 : if (duplicate)
182 : : {
183 : : /* We may eliminate the need for out-of-line copy to be output.
184 : : In that case just go ahead and re-use it. This is not just an
185 : : memory optimization. Making offline copy of function disappear
186 : : from the program will improve future decisions on inlining. */
187 : 3753240 : if (!e->callee->callers->next_caller
188 : : /* Recursive inlining never wants the master clone to
189 : : be overwritten. */
190 : 1795727 : && update_original
191 : 1784626 : && can_remove_node_now_p (e->callee, e)
192 : : /* We cannot overwrite a master clone with non-inline clones
193 : : until after these clones are materialized. */
194 : 5021034 : && !master_clone_with_noninline_clones_p (e->callee))
195 : : {
196 : : /* TODO: When callee is in a comdat group, we could remove all of it,
197 : : including all inline clones inlined into it. That would however
198 : : need small function inlining to register edge removal hook to
199 : : maintain the priority queue.
200 : :
201 : : For now we keep the other functions in the group in program until
202 : : cgraph_remove_unreachable_functions gets rid of them. */
203 : 1267753 : gcc_assert (!e->callee->inlined_to);
204 : 1267753 : e->callee->remove_from_same_comdat_group ();
205 : 1267753 : if (e->callee->definition
206 : 1267753 : && inline_account_function_p (e->callee))
207 : : {
208 : 1106959 : gcc_assert (!e->callee->alias);
209 : 1106959 : if (overall_size)
210 : 217981 : *overall_size -= ipa_size_summaries->get (e->callee)->size;
211 : 1106959 : nfunctions_inlined++;
212 : : }
213 : 1267753 : duplicate = false;
214 : 1267753 : e->callee->externally_visible = false;
215 : 1267753 : update_noncloned_counts (e->callee, e->count, e->callee->count);
216 : :
217 : 1267753 : dump_callgraph_transformation (e->callee, inlining_into,
218 : : "inlining to");
219 : : }
220 : : else
221 : : {
222 : 2485487 : struct cgraph_node *n;
223 : :
224 : 2485487 : n = e->callee->create_clone (e->callee->decl,
225 : : e->count,
226 : : update_original, vNULL, true,
227 : : inlining_into,
228 : : NULL);
229 : 2485487 : n->used_as_abstract_origin = e->callee->used_as_abstract_origin;
230 : 2485487 : e->redirect_callee (n);
231 : : }
232 : : }
233 : : else
234 : 159640 : e->callee->remove_from_same_comdat_group ();
235 : :
236 : 3912880 : e->callee->inlined_to = inlining_into;
237 : 3912880 : if (e->callee->ipa_transforms_to_apply.length ())
238 : : {
239 : 261369 : e->callee->ipa_transforms_to_apply.release ();
240 : 261369 : e->callee->ipa_transforms_to_apply = vNULL;
241 : : }
242 : :
243 : : /* Recursively clone all bodies. */
244 : 7317338 : for (e = e->callee->callees; e; e = next)
245 : : {
246 : 3404458 : next = e->next_callee;
247 : 3404458 : if (!e->inline_failed)
248 : 510177 : clone_inlined_nodes (e, duplicate, update_original, overall_size);
249 : : }
250 : 3912880 : }
251 : :
252 : : /* Check all speculations in N and if any seem useless, resolve them. When a
253 : : first edge is resolved, pop all edges from NEW_EDGES and insert them to
254 : : EDGE_SET. Then remove each resolved edge from EDGE_SET, if it is there. */
255 : :
256 : : static bool
257 : 3911921 : check_speculations_1 (cgraph_node *n, vec<cgraph_edge *> *new_edges,
258 : : hash_set <cgraph_edge *> *edge_set)
259 : : {
260 : 3911921 : bool speculation_removed = false;
261 : 3911921 : cgraph_edge *next;
262 : :
263 : 7317206 : for (cgraph_edge *e = n->callees; e; e = next)
264 : : {
265 : 3405285 : next = e->next_callee;
266 : 3405285 : if (e->speculative && !speculation_useful_p (e, true))
267 : : {
268 : 15 : while (new_edges && !new_edges->is_empty ())
269 : 6 : edge_set->add (new_edges->pop ());
270 : 9 : edge_set->remove (e);
271 : :
272 : 9 : cgraph_edge::resolve_speculation (e, NULL);
273 : 9 : speculation_removed = true;
274 : : }
275 : 3405276 : else if (!e->inline_failed)
276 : 509684 : speculation_removed |= check_speculations_1 (e->callee, new_edges,
277 : : edge_set);
278 : : }
279 : 3911921 : return speculation_removed;
280 : : }
281 : :
282 : : /* Push E to NEW_EDGES. Called from hash_set traverse method, which
283 : : unfortunately means this function has to have external linkage, otherwise
284 : : the code will not compile with gcc 4.8. */
285 : :
286 : : bool
287 : 0 : push_all_edges_in_set_to_vec (cgraph_edge * const &e,
288 : : vec<cgraph_edge *> *new_edges)
289 : : {
290 : 0 : new_edges->safe_push (e);
291 : 0 : return true;
292 : : }
293 : :
294 : : /* Check all speculations in N and if any seem useless, resolve them and remove
295 : : them from NEW_EDGES. */
296 : :
297 : : static bool
298 : 3402237 : check_speculations (cgraph_node *n, vec<cgraph_edge *> *new_edges)
299 : : {
300 : 3402237 : hash_set <cgraph_edge *> edge_set;
301 : 3402237 : bool res = check_speculations_1 (n, new_edges, &edge_set);
302 : 3402237 : if (!edge_set.is_empty ())
303 : 0 : edge_set.traverse <vec<cgraph_edge *> *,
304 : 0 : push_all_edges_in_set_to_vec> (new_edges);
305 : 3402237 : return res;
306 : 3402237 : }
307 : :
308 : : /* Mark all call graph edges coming out of NODE and all nodes that have been
309 : : inlined to it as in_polymorphic_cdtor. */
310 : :
311 : : static void
312 : 114572 : mark_all_inlined_calls_cdtor (cgraph_node *node)
313 : : {
314 : 233620 : for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
315 : : {
316 : 119048 : cs->in_polymorphic_cdtor = true;
317 : 119048 : if (!cs->inline_failed)
318 : 19730 : mark_all_inlined_calls_cdtor (cs->callee);
319 : : }
320 : 119407 : for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
321 : 4835 : cs->in_polymorphic_cdtor = true;
322 : 114572 : }
323 : :
324 : :
325 : : /* Mark edge E as inlined and update callgraph accordingly. UPDATE_ORIGINAL
326 : : specify whether profile of original function should be updated. If any new
327 : : indirect edges are discovered in the process, add them to NEW_EDGES, unless
328 : : it is NULL. If UPDATE_OVERALL_SUMMARY is false, do not bother to recompute overall
329 : : size of caller after inlining. Caller is required to eventually do it via
330 : : ipa_update_overall_fn_summary.
331 : : If callee_removed is non-NULL, set it to true if we removed callee node.
332 : :
333 : : Return true iff any new callgraph edges were discovered as a
334 : : result of inlining. */
335 : :
336 : : bool
337 : 3402237 : inline_call (struct cgraph_edge *e, bool update_original,
338 : : vec<cgraph_edge *> *new_edges,
339 : : int *overall_size, bool update_overall_summary,
340 : : bool *callee_removed)
341 : : {
342 : 3402237 : int old_size = 0, new_size = 0;
343 : 3402237 : struct cgraph_node *to = NULL;
344 : 3402237 : struct cgraph_edge *curr = e;
345 : 3402237 : bool comdat_local = e->callee->comdat_local_p ();
346 : 3402237 : struct cgraph_node *callee = e->callee->ultimate_alias_target ();
347 : 3402237 : bool new_edges_found = false;
348 : :
349 : 3402237 : int estimated_growth = 0;
350 : 3402237 : if (! update_overall_summary)
351 : 2635484 : estimated_growth = estimate_edge_growth (e);
352 : : /* This is used only for assert bellow. */
353 : : #if 0
354 : : bool predicated = inline_edge_summary (e)->predicate != NULL;
355 : : #endif
356 : :
357 : : /* Don't inline inlined edges. */
358 : 3402237 : gcc_assert (e->inline_failed);
359 : : /* Don't even think of inlining inline clone. */
360 : 3402237 : gcc_assert (!callee->inlined_to);
361 : :
362 : 3402237 : to = e->caller;
363 : 3402237 : if (to->inlined_to)
364 : 209845 : to = to->inlined_to;
365 : 3402237 : if (to->thunk)
366 : : {
367 : 1125 : struct cgraph_node *target = to->callees->callee;
368 : 1125 : thunk_expansion = true;
369 : :
370 : : /* Remove all annotations, but keep thunk info. */
371 : 1125 : thunk_info info = *thunk_info::get (to);
372 : 1125 : symtab->call_cgraph_removal_hooks (to);
373 : 1125 : *thunk_info::get_create (to) = info;
374 : 1125 : if (in_lto_p)
375 : 49 : to->get_untransformed_body ();
376 : 1125 : expand_thunk (to, false, true);
377 : : /* When thunk is instrumented we may have multiple callees. */
378 : 1125 : for (e = to->callees; e && e->callee != target; e = e->next_callee)
379 : : ;
380 : 1125 : symtab->call_cgraph_insertion_hooks (to);
381 : 1125 : thunk_expansion = false;
382 : 1125 : gcc_assert (e);
383 : : }
384 : :
385 : :
386 : 3402237 : e->inline_failed = CIF_OK;
387 : 3402237 : DECL_POSSIBLY_INLINED (callee->decl) = true;
388 : :
389 : 3402237 : if (DECL_FUNCTION_PERSONALITY (callee->decl))
390 : 234096 : DECL_FUNCTION_PERSONALITY (to->decl)
391 : 117048 : = DECL_FUNCTION_PERSONALITY (callee->decl);
392 : :
393 : 3402237 : bool reload_optimization_node = false;
394 : 3402237 : bool remove_strict_aliasing
395 : 3402237 : = (!opt_for_fn (callee->decl, flag_strict_aliasing)
396 : 3402237 : && opt_for_fn (to->decl, flag_strict_aliasing));
397 : 3402237 : bool remove_assume_sane_operators_new_delete
398 : 3402237 : = (!opt_for_fn (callee->decl, flag_assume_sane_operators_new_delete)
399 : 3402237 : && opt_for_fn (to->decl, flag_assume_sane_operators_new_delete));
400 : 3402219 : if (remove_strict_aliasing || remove_assume_sane_operators_new_delete)
401 : : {
402 : 31 : struct gcc_options opts = global_options;
403 : 31 : struct gcc_options opts_set = global_options_set;
404 : :
405 : 31 : cl_optimization_restore (&opts, &opts_set, opts_for_fn (to->decl));
406 : 31 : if (remove_strict_aliasing)
407 : : {
408 : 13 : opts.x_flag_strict_aliasing = false;
409 : 13 : if (dump_file)
410 : 0 : fprintf (dump_file, "Dropping flag_strict_aliasing on %s\n",
411 : : to->dump_name ());
412 : : }
413 : 31 : if (remove_assume_sane_operators_new_delete)
414 : : {
415 : 18 : opts.x_flag_assume_sane_operators_new_delete = false;
416 : 18 : if (dump_file)
417 : 0 : fprintf (dump_file,
418 : : "Dropping flag_assume_sane_operators_new_delete on %s\n",
419 : : to->dump_name ());
420 : : }
421 : 31 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
422 : 31 : = build_optimization_node (&opts, &opts_set);
423 : 31 : reload_optimization_node = true;
424 : : }
425 : :
426 : 3402237 : ipa_fn_summary *caller_info = ipa_fn_summaries->get (to);
427 : 3402237 : ipa_fn_summary *callee_info = ipa_fn_summaries->get (callee);
428 : 3402237 : if (!caller_info->fp_expressions && callee_info->fp_expressions)
429 : : {
430 : 10811 : caller_info->fp_expressions = true;
431 : 10811 : if (opt_for_fn (callee->decl, flag_rounding_math)
432 : 10811 : != opt_for_fn (to->decl, flag_rounding_math)
433 : 10811 : || opt_for_fn (callee->decl, flag_trapping_math)
434 : 10811 : != opt_for_fn (to->decl, flag_trapping_math)
435 : 10804 : || opt_for_fn (callee->decl, flag_unsafe_math_optimizations)
436 : 10804 : != opt_for_fn (to->decl, flag_unsafe_math_optimizations)
437 : 10804 : || opt_for_fn (callee->decl, flag_finite_math_only)
438 : 10804 : != opt_for_fn (to->decl, flag_finite_math_only)
439 : 10804 : || opt_for_fn (callee->decl, flag_signaling_nans)
440 : 10804 : != opt_for_fn (to->decl, flag_signaling_nans)
441 : 10804 : || opt_for_fn (callee->decl, flag_cx_limited_range)
442 : 10804 : != opt_for_fn (to->decl, flag_cx_limited_range)
443 : 10804 : || opt_for_fn (callee->decl, flag_signed_zeros)
444 : 10804 : != opt_for_fn (to->decl, flag_signed_zeros)
445 : 10804 : || opt_for_fn (callee->decl, flag_associative_math)
446 : 10804 : != opt_for_fn (to->decl, flag_associative_math)
447 : 10803 : || opt_for_fn (callee->decl, flag_reciprocal_math)
448 : 10803 : != opt_for_fn (to->decl, flag_reciprocal_math)
449 : 10803 : || opt_for_fn (callee->decl, flag_fp_int_builtin_inexact)
450 : 10803 : != opt_for_fn (to->decl, flag_fp_int_builtin_inexact)
451 : 10811 : || opt_for_fn (callee->decl, flag_errno_math)
452 : 10803 : != opt_for_fn (to->decl, flag_errno_math))
453 : : {
454 : 8 : struct gcc_options opts = global_options;
455 : 8 : struct gcc_options opts_set = global_options_set;
456 : :
457 : 8 : cl_optimization_restore (&opts, &opts_set, opts_for_fn (to->decl));
458 : 8 : opts.x_flag_rounding_math
459 : 8 : = opt_for_fn (callee->decl, flag_rounding_math);
460 : 8 : opts.x_flag_trapping_math
461 : 8 : = opt_for_fn (callee->decl, flag_trapping_math);
462 : 8 : opts.x_flag_unsafe_math_optimizations
463 : 8 : = opt_for_fn (callee->decl, flag_unsafe_math_optimizations);
464 : 8 : opts.x_flag_finite_math_only
465 : 8 : = opt_for_fn (callee->decl, flag_finite_math_only);
466 : 8 : opts.x_flag_signaling_nans
467 : 8 : = opt_for_fn (callee->decl, flag_signaling_nans);
468 : 8 : opts.x_flag_cx_limited_range
469 : 8 : = opt_for_fn (callee->decl, flag_cx_limited_range);
470 : 8 : opts.x_flag_signed_zeros
471 : 8 : = opt_for_fn (callee->decl, flag_signed_zeros);
472 : 8 : opts.x_flag_associative_math
473 : 8 : = opt_for_fn (callee->decl, flag_associative_math);
474 : 8 : opts.x_flag_reciprocal_math
475 : 8 : = opt_for_fn (callee->decl, flag_reciprocal_math);
476 : 8 : opts.x_flag_fp_int_builtin_inexact
477 : 8 : = opt_for_fn (callee->decl, flag_fp_int_builtin_inexact);
478 : 8 : opts.x_flag_errno_math
479 : 8 : = opt_for_fn (callee->decl, flag_errno_math);
480 : 8 : if (dump_file)
481 : 0 : fprintf (dump_file, "Copying FP flags from %s to %s\n",
482 : : callee->dump_name (), to->dump_name ());
483 : 8 : DECL_FUNCTION_SPECIFIC_OPTIMIZATION (to->decl)
484 : 8 : = build_optimization_node (&opts, &opts_set);
485 : 8 : reload_optimization_node = true;
486 : : }
487 : : }
488 : :
489 : : /* Reload global optimization flags. */
490 : 3402237 : if (reload_optimization_node && DECL_STRUCT_FUNCTION (to->decl) == cfun)
491 : 34 : set_cfun (cfun, true);
492 : :
493 : : /* If aliases are involved, redirect edge to the actual destination and
494 : : possibly remove the aliases. */
495 : 3402237 : if (e->callee != callee)
496 : : {
497 : 630902 : struct cgraph_node *alias = e->callee, *next_alias;
498 : 630902 : e->redirect_callee (callee);
499 : 1459642 : while (alias && alias != callee)
500 : : {
501 : 630921 : if (!alias->callers
502 : 839990 : && can_remove_node_now_p (alias,
503 : 209069 : !e->next_caller && !e->prev_caller ? e : NULL))
504 : : {
505 : 197838 : next_alias = alias->get_alias_target ();
506 : 197838 : alias->remove ();
507 : 197838 : if (callee_removed)
508 : 942 : *callee_removed = true;
509 : : alias = next_alias;
510 : : }
511 : : else
512 : : break;
513 : : }
514 : : }
515 : :
516 : 3402237 : clone_inlined_nodes (e, true, update_original, overall_size);
517 : :
518 : 3402237 : gcc_assert (curr->callee->inlined_to == to);
519 : :
520 : 3402237 : old_size = ipa_size_summaries->get (to)->size;
521 : 3402237 : ipa_merge_modref_summary_after_inlining (e);
522 : 3402237 : ipa_merge_fn_summary_after_inlining (e);
523 : 3402237 : if (e->in_polymorphic_cdtor)
524 : 94842 : mark_all_inlined_calls_cdtor (e->callee);
525 : 3402237 : if (opt_for_fn (e->caller->decl, optimize))
526 : 3378021 : new_edges_found = ipa_propagate_indirect_call_infos (curr, new_edges);
527 : 3402237 : bool removed_p = check_speculations (e->callee, new_edges);
528 : 3402237 : if (update_overall_summary)
529 : 766753 : ipa_update_overall_fn_summary (to, new_edges_found || removed_p);
530 : : else
531 : : /* Update self size by the estimate so overall function growth limits
532 : : work for further inlining into this function. Before inlining
533 : : the function we inlined to again we expect the caller to update
534 : : the overall summary. */
535 : 2635484 : ipa_size_summaries->get (to)->size += estimated_growth;
536 : 3402237 : new_size = ipa_size_summaries->get (to)->size;
537 : :
538 : 3402237 : if (callee->calls_comdat_local)
539 : 0 : to->calls_comdat_local = true;
540 : 3402237 : else if (to->calls_comdat_local && comdat_local)
541 : 3172 : to->calls_comdat_local = to->check_calls_comdat_local_p ();
542 : :
543 : : /* FIXME: This assert suffers from roundoff errors, disable it for GCC 5
544 : : and revisit it after conversion to sreals in GCC 6.
545 : : See PR 65654. */
546 : : #if 0
547 : : /* Verify that estimated growth match real growth. Allow off-by-one
548 : : error due to ipa_fn_summary::size_scale roudoff errors. */
549 : : gcc_assert (!update_overall_summary || !overall_size || new_edges_found
550 : : || abs (estimated_growth - (new_size - old_size)) <= 1
551 : : || speculation_removed
552 : : /* FIXME: a hack. Edges with false predicate are accounted
553 : : wrong, we should remove them from callgraph. */
554 : : || predicated);
555 : : #endif
556 : :
557 : : /* Account the change of overall unit size; external functions will be
558 : : removed and are thus not accounted. */
559 : 3402237 : if (overall_size && inline_account_function_p (to))
560 : 752046 : *overall_size += new_size - old_size;
561 : 3402237 : ncalls_inlined++;
562 : :
563 : : /* This must happen after ipa_merge_fn_summary_after_inlining that rely on jump
564 : : functions of callee to not be updated. */
565 : 3402237 : return new_edges_found;
566 : : }
567 : :
568 : : /* For each node that was made the holder of function body by
569 : : save_inline_function_body, this summary contains pointer to the previous
570 : : holder of the body. */
571 : :
572 : : function_summary <tree *> *ipa_saved_clone_sources;
573 : :
574 : : /* Copy function body of NODE and redirect all inline clones to it.
575 : : This is done before inline plan is applied to NODE when there are
576 : : still some inline clones if it.
577 : :
578 : : This is necessary because inline decisions are not really transitive
579 : : and the other inline clones may have different bodies. */
580 : :
581 : : static struct cgraph_node *
582 : 39546 : save_inline_function_body (struct cgraph_node *node)
583 : : {
584 : 39546 : struct cgraph_node *first_clone, *n;
585 : :
586 : 39546 : if (dump_file)
587 : 126 : fprintf (dump_file, "\nSaving body of %s for later reuse\n",
588 : : node->dump_name ());
589 : :
590 : 39546 : gcc_assert (node == cgraph_node::get (node->decl));
591 : :
592 : : /* first_clone will be turned into real function. */
593 : 39546 : first_clone = node->clones;
594 : :
595 : : /* Arrange first clone to not be thunk as those do not have bodies. */
596 : 39546 : if (first_clone->thunk)
597 : : {
598 : 0 : while (first_clone->thunk)
599 : 0 : first_clone = first_clone->next_sibling_clone;
600 : 0 : first_clone->prev_sibling_clone->next_sibling_clone
601 : 0 : = first_clone->next_sibling_clone;
602 : 0 : if (first_clone->next_sibling_clone)
603 : 0 : first_clone->next_sibling_clone->prev_sibling_clone
604 : 0 : = first_clone->prev_sibling_clone;
605 : 0 : first_clone->next_sibling_clone = node->clones;
606 : 0 : first_clone->prev_sibling_clone = NULL;
607 : 0 : node->clones->prev_sibling_clone = first_clone;
608 : 0 : node->clones = first_clone;
609 : : }
610 : 39546 : first_clone->decl = copy_node (node->decl);
611 : 39546 : first_clone->decl->decl_with_vis.symtab_node = first_clone;
612 : 39546 : gcc_assert (first_clone == cgraph_node::get (first_clone->decl));
613 : :
614 : : /* Now reshape the clone tree, so all other clones descends from
615 : : first_clone. */
616 : 39546 : if (first_clone->next_sibling_clone)
617 : : {
618 : 159186 : for (n = first_clone->next_sibling_clone; n->next_sibling_clone;
619 : 138600 : n = n->next_sibling_clone)
620 : 138600 : n->clone_of = first_clone;
621 : 20586 : n->clone_of = first_clone;
622 : 20586 : n->next_sibling_clone = first_clone->clones;
623 : 20586 : if (first_clone->clones)
624 : 361 : first_clone->clones->prev_sibling_clone = n;
625 : 20586 : first_clone->clones = first_clone->next_sibling_clone;
626 : 20586 : first_clone->next_sibling_clone->prev_sibling_clone = NULL;
627 : 20586 : first_clone->next_sibling_clone = NULL;
628 : 20586 : gcc_assert (!first_clone->prev_sibling_clone);
629 : : }
630 : :
631 : 39546 : tree prev_body_holder = node->decl;
632 : 39546 : if (!ipa_saved_clone_sources)
633 : : {
634 : 11784 : ipa_saved_clone_sources = new function_summary <tree *> (symtab);
635 : 11784 : ipa_saved_clone_sources->disable_insertion_hook ();
636 : : }
637 : : else
638 : : {
639 : 27762 : tree *p = ipa_saved_clone_sources->get (node);
640 : 27762 : if (p)
641 : : {
642 : 0 : prev_body_holder = *p;
643 : 0 : gcc_assert (prev_body_holder);
644 : : }
645 : : }
646 : 39546 : *ipa_saved_clone_sources->get_create (first_clone) = prev_body_holder;
647 : 39546 : first_clone->former_clone_of
648 : 39546 : = node->former_clone_of ? node->former_clone_of : node->decl;
649 : 39546 : first_clone->clone_of = NULL;
650 : :
651 : : /* Now node in question has no clones. */
652 : 39546 : node->clones = NULL;
653 : :
654 : : /* Inline clones share decl with the function they are cloned
655 : : from. Walk the whole clone tree and redirect them all to the
656 : : new decl. */
657 : 39546 : if (first_clone->clones)
658 : 220382 : for (n = first_clone->clones; n != first_clone;)
659 : : {
660 : 198471 : gcc_assert (n->decl == node->decl);
661 : 198471 : n->decl = first_clone->decl;
662 : 198471 : if (n->clones)
663 : : n = n->clones;
664 : 193378 : else if (n->next_sibling_clone)
665 : : n = n->next_sibling_clone;
666 : : else
667 : : {
668 : 51979 : while (n != first_clone && !n->next_sibling_clone)
669 : 27004 : n = n->clone_of;
670 : 24975 : if (n != first_clone)
671 : 3064 : n = n->next_sibling_clone;
672 : : }
673 : : }
674 : :
675 : : /* Copy the OLD_VERSION_NODE function tree to the new version. */
676 : 39546 : tree_function_versioning (node->decl, first_clone->decl,
677 : : NULL, NULL, true, NULL, NULL);
678 : :
679 : : /* The function will be short lived and removed after we inline all the
680 : : clones, but make it internal so we won't confuse ourself. */
681 : 39546 : DECL_EXTERNAL (first_clone->decl) = 0;
682 : 39546 : TREE_PUBLIC (first_clone->decl) = 0;
683 : 39546 : DECL_COMDAT (first_clone->decl) = 0;
684 : 39546 : first_clone->ipa_transforms_to_apply.release ();
685 : :
686 : : /* When doing recursive inlining, the clone may become unnecessary.
687 : : This is possible i.e. in the case when the recursive function is proved to
688 : : be non-throwing and the recursion happens only in the EH landing pad.
689 : : We cannot remove the clone until we are done with saving the body.
690 : : Remove it now. */
691 : 39546 : if (!first_clone->callers)
692 : : {
693 : 0 : first_clone->remove_symbol_and_inline_clones ();
694 : 0 : first_clone = NULL;
695 : : }
696 : 39546 : else if (flag_checking)
697 : 39546 : first_clone->verify ();
698 : :
699 : 39546 : return first_clone;
700 : : }
701 : :
702 : : /* Return true when function body of DECL still needs to be kept around
703 : : for later re-use. */
704 : : static bool
705 : 1407860 : preserve_function_body_p (struct cgraph_node *node)
706 : : {
707 : 1407860 : gcc_assert (symtab->global_info_ready);
708 : 1407860 : gcc_assert (!node->alias && !node->thunk);
709 : :
710 : : /* Look if there is any non-thunk clone around. */
711 : 1407899 : for (node = node->clones; node; node = node->next_sibling_clone)
712 : 39585 : if (!node->thunk)
713 : : return true;
714 : : return false;
715 : : }
716 : :
717 : : /* tree-inline can not recurse; materialize all function bodie we will need
718 : : during inlining. This includes inlined functions, but also called functions
719 : : with param manipulation because IPA param manipulation attaches debug
720 : : statements to PARM_DECLs of called clone. Materialize them if needed.
721 : :
722 : : FIXME: This is somehwat broken by design because it does not play well
723 : : with partitioning. */
724 : :
725 : : static void
726 : 2535448 : maybe_materialize_called_clones (cgraph_node *node)
727 : : {
728 : 10467645 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
729 : : {
730 : 7932197 : clone_info *info;
731 : :
732 : 7932197 : if (!e->inline_failed)
733 : 1127588 : maybe_materialize_called_clones (e->callee);
734 : :
735 : 7932197 : cgraph_node *callee = cgraph_node::get (e->callee->decl);
736 : 7932197 : if (callee->clone_of
737 : 7932197 : && (info = clone_info::get (callee)) && info->param_adjustments)
738 : 81348 : callee->get_untransformed_body ();
739 : : }
740 : 2535448 : }
741 : :
742 : : /* Apply inline plan to function. */
743 : :
744 : : unsigned int
745 : 1407860 : inline_transform (struct cgraph_node *node)
746 : : {
747 : 1407860 : unsigned int todo = 0;
748 : 1407860 : struct cgraph_edge *e, *next;
749 : 1407860 : bool has_inline = false;
750 : :
751 : : /* FIXME: Currently the pass manager is adding inline transform more than
752 : : once to some clones. This needs revisiting after WPA cleanups. */
753 : 1407860 : if (cfun->after_inlining)
754 : : return 0;
755 : :
756 : 1407860 : cgraph_node *next_clone;
757 : 1606631 : for (cgraph_node *n = node->clones; n; n = next_clone)
758 : : {
759 : 198771 : next_clone = n->next_sibling_clone;
760 : 198771 : if (n->decl != node->decl)
761 : 0 : n->materialize_clone ();
762 : : }
763 : 1407860 : node->clear_stmts_in_references ();
764 : :
765 : : /* We might need the body of this function so that we can expand
766 : : it inline somewhere else. */
767 : 1407860 : if (preserve_function_body_p (node))
768 : 39546 : save_inline_function_body (node);
769 : :
770 : 1407860 : profile_count num = node->count;
771 : 1407860 : profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
772 : 1407860 : bool scale = num.initialized_p () && !(num == den);
773 : 31 : if (scale)
774 : : {
775 : 31 : profile_count::adjust_for_ipa_scaling (&num, &den);
776 : 31 : if (dump_file)
777 : : {
778 : 0 : fprintf (dump_file, "Applying count scale ");
779 : 0 : num.dump (dump_file);
780 : 0 : fprintf (dump_file, "/");
781 : 0 : den.dump (dump_file);
782 : 0 : fprintf (dump_file, "\n");
783 : : }
784 : :
785 : 31 : basic_block bb;
786 : 31 : cfun->cfg->count_max = profile_count::uninitialized ();
787 : 259 : FOR_ALL_BB_FN (bb, cfun)
788 : : {
789 : 228 : bb->count = bb->count.apply_scale (num, den);
790 : 228 : cfun->cfg->count_max = cfun->cfg->count_max.max (bb->count);
791 : : }
792 : 31 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = node->count;
793 : : }
794 : :
795 : 1407860 : maybe_materialize_called_clones (node);
796 : 7030405 : for (e = node->callees; e; e = next)
797 : : {
798 : 5622545 : if (!e->inline_failed)
799 : 482159 : has_inline = true;
800 : 5622545 : next = e->next_callee;
801 : 5622545 : cgraph_edge::redirect_call_stmt_to_callee (e);
802 : : }
803 : 1407860 : node->remove_all_references ();
804 : :
805 : 1407860 : timevar_push (TV_INTEGRATION);
806 : 1407860 : if (node->callees && (opt_for_fn (node->decl, optimize) || has_inline))
807 : : {
808 : 775999 : todo = optimize_inline_calls (current_function_decl);
809 : : }
810 : 1407860 : timevar_pop (TV_INTEGRATION);
811 : :
812 : 1407860 : cfun->always_inline_functions_inlined = true;
813 : 1407860 : cfun->after_inlining = true;
814 : 1407860 : todo |= execute_fixup_cfg ();
815 : :
816 : 1407860 : if (!(todo & TODO_update_ssa_any))
817 : : /* Redirecting edges might lead to a need for vops to be recomputed. */
818 : 1054183 : todo |= TODO_update_ssa_only_virtuals;
819 : :
820 : : return todo;
821 : : }
|
