GCC Middle and Back End API Reference
ipa-inline.cc File Reference
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "target.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "alloc-pool.h"
#include "tree-pass.h"
#include "gimple-ssa.h"
#include "cgraph.h"
#include "lto-streamer.h"
#include "trans-mem.h"
#include "calls.h"
#include "tree-inline.h"
#include "profile.h"
#include "symbol-summary.h"
#include "tree-vrp.h"
#include "sreal.h"
#include "ipa-cp.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
#include "ipa-inline.h"
#include "ipa-utils.h"
#include "auto-profile.h"
#include "builtins.h"
#include "fibonacci_heap.h"
#include "stringpool.h"
#include "attribs.h"
#include "asan.h"
#include "ipa-strub.h"
Include dependency graph for ipa-inline.cc:

Data Structures

class  inline_badness
 

Macros

#define check_maybe_up(flag)
 
#define check_maybe_down(flag)
 
#define check_match(flag)
 

Typedefs

typedef fibonacci_heap< inline_badness, cgraph_edgeedge_heap_t
 
typedef fibonacci_node< inline_badness, cgraph_edgeedge_heap_node_t
 

Enumerations

enum  can_inline_edge_by_limits_flags { CAN_INLINE_EARLY = 1 , CAN_INLINE_DISREGARD_LIMITS = 2 , CAN_INLINE_FORCE_LIMITS = 4 , CAN_INLINE_REPORT = 8 }
 

Functions

static bool caller_growth_limits (struct cgraph_edge *e)
 
static void report_inline_failed_reason (struct cgraph_edge *e)
 
static bool sanitize_attrs_match_for_inline_p (const_tree caller, const_tree callee)
 
static bool can_inline_edge_p (struct cgraph_edge *e, bool report, bool early=false)
 
static int inline_insns_single (cgraph_node *n, bool hint, bool hint2)
 
static int inline_insns_auto (cgraph_node *n, bool hint, bool hint2)
 
static bool can_inline_edge_by_limits_p (struct cgraph_edge *e, int flags)
 
static bool can_early_inline_edge_p (struct cgraph_edge *e)
 
static int num_calls (struct cgraph_node *n)
 
static bool want_early_inline_function_p (struct cgraph_edge *e)
 
sreal compute_uninlined_call_time (struct cgraph_edge *edge, sreal uninlined_call_time, sreal freq)
 
sreal compute_inlined_call_time (struct cgraph_edge *edge, sreal time, sreal freq)
 
sreal inlining_speedup (struct cgraph_edge *edge, sreal freq, sreal uninlined_time, sreal inlined_time)
 
static bool big_speedup_p (struct cgraph_edge *e)
 
static bool want_inline_small_function_p (struct cgraph_edge *e, bool report)
 
static bool want_inline_self_recursive_call_p (struct cgraph_edge *edge, struct cgraph_node *outer_node, bool peeling, int depth)
 
static bool check_callers (struct cgraph_node *node, void *has_hot_call)
 
static bool has_caller_p (struct cgraph_node *node, void *data)
 
static bool want_inline_function_to_all_callers_p (struct cgraph_node *node, bool cold)
 
static bool wrapper_heuristics_may_apply (struct cgraph_node *where, int size)
 
static sreal edge_badness (struct cgraph_edge *edge, bool dump)
 
static void update_edge_key (edge_heap_t *heap, struct cgraph_edge *edge)
 
static void reset_edge_caches (struct cgraph_node *node)
 
static void update_caller_keys (edge_heap_t *heap, struct cgraph_node *node, bitmap updated_nodes, struct cgraph_edge *check_inlinablity_for)
 
static void update_callee_keys (edge_heap_t *heap, struct cgraph_node *node, struct cgraph_node *update_since, bitmap updated_nodes)
 
static void lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where, edge_heap_t *heap)
 
static bool recursive_inlining (struct cgraph_edge *edge, vec< cgraph_edge * > *new_edges)
 
static int64_t compute_max_insns (cgraph_node *node, int insns)
 
static void add_new_edges_to_heap (edge_heap_t *heap, vec< cgraph_edge * > &new_edges)
 
static void heap_edge_removal_hook (struct cgraph_edge *e, void *data)
 
bool speculation_useful_p (struct cgraph_edge *e, bool anticipate_inlining)
 
static void resolve_noninline_speculation (edge_heap_t *edge_heap, struct cgraph_edge *edge)
 
bool inline_account_function_p (struct cgraph_node *node)
 
static bool sum_callers (struct cgraph_node *node, void *data)
 
bool ignore_edge_p (struct cgraph_edge *e)
 
static void inline_small_functions (void)
 
static void flatten_function (struct cgraph_node *node, bool early, bool update)
 
static bool inline_to_all_callers_1 (struct cgraph_node *node, void *data, hash_set< cgraph_node * > *callers)
 
static bool inline_to_all_callers (struct cgraph_node *node, void *data)
 
static void dump_overall_stats (void)
 
static void dump_inline_stats (void)
 
static void flatten_remove_node_hook (struct cgraph_node *node, void *data)
 
static unsigned int ipa_inline (void)
 
static bool inline_always_inline_functions (struct cgraph_node *node)
 
static bool early_inline_small_functions (struct cgraph_node *node)
 
unsigned int early_inliner (function *fun)
 
gimple_opt_passmake_pass_early_inline (gcc::context *ctxt)
 
ipa_opt_pass_dmake_pass_ipa_inline (gcc::context *ctxt)
 

Variables

static int overall_size
 
static profile_count max_count
 
static profile_count spec_rem
 

Macro Definition Documentation

◆ check_match

#define check_match ( flag)
Value:
(opts_for_fn (caller->decl)->x_##flag \
!= opts_for_fn (callee->decl)->x_##flag)
struct cl_optimization * opts_for_fn(const_tree fndecl)
Definition tree.h:6209
Used for flags where exact match is needed for correctness.   

Referenced by can_inline_edge_by_limits_p().

◆ check_maybe_down

#define check_maybe_down ( flag)
Value:
(opts_for_fn (caller->decl)->x_##flag \
!= opts_for_fn (callee->decl)->x_##flag \
&& (!always_inline \
|| opts_for_fn (caller->decl)->x_##flag \
> opts_for_fn (callee->decl)->x_##flag))
Used for flags where it is safe to inline when caller's value is
smaller than callee's.   

Referenced by can_inline_edge_by_limits_p().

◆ check_maybe_up

#define check_maybe_up ( flag)
Value:
(opts_for_fn (caller->decl)->x_##flag \
!= opts_for_fn (callee->decl)->x_##flag \
&& (!always_inline \
|| opts_for_fn (caller->decl)->x_##flag \
< opts_for_fn (callee->decl)->x_##flag))
Used for flags where it is safe to inline when caller's value is
grater than callee's.   

Referenced by can_inline_edge_by_limits_p().

Typedef Documentation

◆ edge_heap_node_t

◆ edge_heap_t

Enumeration Type Documentation

◆ can_inline_edge_by_limits_flags

Enumerator
CAN_INLINE_EARLY 
CAN_INLINE_DISREGARD_LIMITS 
CAN_INLINE_FORCE_LIMITS 
CAN_INLINE_REPORT 

Function Documentation

◆ add_new_edges_to_heap()

static void add_new_edges_to_heap ( edge_heap_t * heap,
vec< cgraph_edge * > & new_edges )
static

◆ big_speedup_p()

static bool big_speedup_p ( struct cgraph_edge * e)
static

◆ caller_growth_limits()

static bool caller_growth_limits ( struct cgraph_edge * e)
static
Return false when inlining edge E would lead to violating
limits on function unit growth or stack usage growth.

The relative function body growth limit is present generally
to avoid problems with non-linear behavior of the compiler.
To allow inlining huge functions into tiny wrapper, the limit
is always based on the bigger of the two functions considered.

For stack growth limits we always base the growth in stack usage
of the callers.  We want to prevent applications from segfaulting
on stack overflow when functions with huge stack frames gets
inlined.  

References cgraph_edge::callee, cgraph_edge::caller, cgraph_node::callers, symtab_node::decl, estimate_size_after_inlining(), ipa_size_summary::estimated_self_stack_size, ipa_fn_summary::estimated_stack_size, cgraph_edge::inline_failed, cgraph_node::inlined_to, ipa_fn_summaries, ipa_get_stack_frame_offset(), ipa_size_summaries, opt_for_fn, ipa_size_summary::self_size, and cgraph_node::ultimate_alias_target().

Referenced by can_inline_edge_by_limits_p().

◆ can_early_inline_edge_p()

◆ can_inline_edge_by_limits_p()

◆ can_inline_edge_p()

◆ check_callers()

static bool check_callers ( struct cgraph_node * node,
void * has_hot_call )
static

◆ compute_inlined_call_time()

sreal compute_inlined_call_time ( struct cgraph_edge * edge,
sreal time,
sreal freq )
inline
Same as compute_uinlined_call_time but compute time when inlining
does happen.   

References gcc_checking_assert, ipa_call_summaries, and ipa_fn_summaries.

Referenced by big_speedup_p().

◆ compute_max_insns()

static int64_t compute_max_insns ( cgraph_node * node,
int insns )
static
Given whole compilation unit estimate of INSNS, compute how large we can
allow the unit to grow.   

References symtab_node::decl, insns, and opt_for_fn.

Referenced by inline_small_functions().

◆ compute_uninlined_call_time()

sreal compute_uninlined_call_time ( struct cgraph_edge * edge,
sreal uninlined_call_time,
sreal freq )
inline
Compute time of the edge->caller + edge->callee execution when inlining
does not happen.   

References ipa_fn_summaries.

Referenced by big_speedup_p().

◆ dump_inline_stats()

◆ dump_overall_stats()

◆ early_inline_small_functions()

◆ early_inliner()

◆ edge_badness()

◆ flatten_function()

◆ flatten_remove_node_hook()

static void flatten_remove_node_hook ( struct cgraph_node * node,
void * data )
static
Called when node is removed.   

References hash_set< KeyId, Lazy, Traits >::add(), symtab_node::decl, DECL_ATTRIBUTES, lookup_attribute(), and NULL.

Referenced by ipa_inline().

◆ has_caller_p()

static bool has_caller_p ( struct cgraph_node * node,
void * data )
static
If NODE has a caller, return true.   

References cgraph_node::callers.

Referenced by want_inline_function_to_all_callers_p().

◆ heap_edge_removal_hook()

static void heap_edge_removal_hook ( struct cgraph_edge * e,
void * data )
static
Remove EDGE from the fibheap.   

References cgraph_edge::aux, and NULL.

Referenced by inline_small_functions().

◆ ignore_edge_p()

bool ignore_edge_p ( struct cgraph_edge * e)
inline
We only propagate across edges with non-interposable callee.   

References AVAIL_INTERPOSABLE, cgraph_edge::callee, cgraph_edge::caller, and cgraph_node::function_or_virtual_thunk_symbol().

Referenced by inline_small_functions().

◆ inline_account_function_p()

bool inline_account_function_p ( struct cgraph_node * node)
Return true if NODE should be accounted for overall size estimate.
Skip all nodes optimized for size so we can measure the growth of hot
part of program no matter of the padding.   

References symtab_node::decl, DECL_EXTERNAL, cgraph_node::frequency, NODE_FREQUENCY_UNLIKELY_EXECUTED, and opt_for_fn.

Referenced by clone_inlined_nodes(), inline_call(), and inline_small_functions().

◆ inline_always_inline_functions()

◆ inline_insns_auto()

static int inline_insns_auto ( cgraph_node * n,
bool hint,
bool hint2 )
static
Return inlining_insns_auto limit for function N.  If HINT or HINT2 is true
scale up the bound.    

References symtab_node::decl, opt_for_fn, and spd.

Referenced by can_inline_edge_by_limits_p(), want_inline_small_function_p(), and wrapper_heuristics_may_apply().

◆ inline_insns_single()

static int inline_insns_single ( cgraph_node * n,
bool hint,
bool hint2 )
static
Return inlining_insns_single limit for function N.  If HINT or HINT2 is true
scale up the bound.   

References symtab_node::decl, opt_for_fn, and spd.

Referenced by can_inline_edge_by_limits_p(), want_inline_small_function_p(), and wrapper_heuristics_may_apply().

◆ inline_small_functions()

static void inline_small_functions ( void )
static
We use greedy algorithm for inlining of small functions:
All inline candidates are put into prioritized heap ordered in
increasing badness.

The inlining of small functions is bounded by unit growth parameters.   

References symbol_table::add_edge_removal_hook(), add_new_edges_to_heap(), symtab_node::alias, symtab_node::analyzed, symtab_node::aux, b, bitmap_clear(), BUILTINS_LOCATION, cgraph_node::call_for_symbol_and_aliases(), cgraph_node::callees, cgraph_edge::caller, cgraph_node::callers, can_inline_edge_by_limits_p(), can_inline_edge_p(), CAN_INLINE_REPORT, symbol_table::cgraph_count, compute_max_insns(), cgraph_node::count, cross_module_call_p(), symtab_node::decl, DECL_DISREGARD_INLINE_LIMITS, dump_enabled_p(), dump_file, dump_flags, symtab_node::dump_name(), dump_printf(), dump_printf_loc(), edge_badness(), edge_growth_cache, fibonacci_heap< K, V >::empty(), estimate_edge_growth(), estimate_edge_hints(), estimate_edge_size(), estimate_edge_time(), estimate_growth(), fibonacci_heap< K, V >::extract_min(), FOR_EACH_DEFINED_FUNCTION, free(), free_growth_caches(), gcc_assert, gcc_checking_assert, cgraph_node::get(), gimple_filename(), gimple_lineno(), gimple_location(), ipa_fn_summary::growth, cgraph_node::has_gimple_body_p(), heap_edge_removal_hook(), ignore_edge_p(), initialize_growth_caches(), profile_count::initialized_p(), inline_account_function_p(), inline_call(), cgraph_node::inlined_to, fibonacci_heap< K, V >::insert(), ipa_fn_summaries, ipa_free_postorder_info(), profile_count::ipa_p(), ipa_reduced_postorder(), ipa_size_summaries, ipa_update_overall_fn_summary(), LOCATION_LOCUS, profile_count::max(), max_count, fibonacci_heap< K, V >::min(), fibonacci_heap< K, V >::min_key(), MSG_NOTE, MSG_OPTIMIZED_LOCATIONS, cgraph_edge::next_callee, cgraph_edge::next_caller, ipa_dfs_info::next_cycle, profile_count::nonzero_p(), NULL, num_calls(), opt_for_fn, symtab_node::order, overall_size, profile_info, recursive_inlining(), symbol_table::remove_edge_removal_hook(), report_inline_failed_reason(), reset_edge_caches(), reset_node_cache(), resolve_noninline_speculation(), cgraph_edge::resolve_speculation(), ipa_dfs_info::scc_no, ipa_fn_summary::scc_no, ipa_fn_summary::single_caller, speculation_useful_p(), sum_callers(), symtab, TDF_DETAILS, cgraph_node::thunk, ipa_fn_summary::time, sreal::to_double(), cgraph_node::ultimate_alias_target(), profile_count::uninitialized(), update_callee_keys(), update_caller_keys(), update_edge_key(), want_inline_self_recursive_call_p(), want_inline_small_function_p(), and wrapper_heuristics_may_apply().

Referenced by ipa_inline().

◆ inline_to_all_callers()

static bool inline_to_all_callers ( struct cgraph_node * node,
void * data )
static
Wrapper around inline_to_all_callers_1 doing delayed overall summary
update.   

References cgraph_node::callers, i, inline_to_all_callers_1(), and ipa_update_overall_fn_summary().

Referenced by ipa_inline().

◆ inline_to_all_callers_1()

static bool inline_to_all_callers_1 ( struct cgraph_node * node,
void * data,
hash_set< cgraph_node * > * callers )
static
Inline NODE to all callers.  Worker for cgraph_for_node_and_aliases.
DATA points to number of calls originally found so we avoid infinite
recursion.   

References cgraph_edge::caller, cgraph_node::callers, can_inline_edge_by_limits_p(), can_inline_edge_p(), CAN_INLINE_REPORT, dump_file, symtab_node::dump_name(), inline_call(), cgraph_node::inlined_to, ipa_size_summaries, NULL, num_calls(), cgraph_edge::recursive_p(), and cgraph_node::ultimate_alias_target().

Referenced by inline_to_all_callers().

◆ inlining_speedup()

sreal inlining_speedup ( struct cgraph_edge * edge,
sreal freq,
sreal uninlined_time,
sreal inlined_time )
inline
Determine time saved by inlining EDGE of frequency FREQ
where callee's runtime w/o inlining is UNINLINED_TYPE
and with inlined is INLINED_TYPE.   

References gcc_checking_assert, and ipa_call_summaries.

Referenced by edge_badness().

◆ ipa_inline()

◆ lookup_recursive_calls()

static void lookup_recursive_calls ( struct cgraph_node * node,
struct cgraph_node * where,
edge_heap_t * heap )
static

◆ make_pass_early_inline()

gimple_opt_pass * make_pass_early_inline ( gcc::context * ctxt)

◆ make_pass_ipa_inline()

ipa_opt_pass_d * make_pass_ipa_inline ( gcc::context * ctxt)

◆ num_calls()

◆ recursive_inlining()

◆ report_inline_failed_reason()

◆ reset_edge_caches()

static void reset_edge_caches ( struct cgraph_node * node)
static

◆ resolve_noninline_speculation()

static void resolve_noninline_speculation ( edge_heap_t * edge_heap,
struct cgraph_edge * edge )
static

◆ sanitize_attrs_match_for_inline_p()

◆ speculation_useful_p()

◆ sum_callers()

static bool sum_callers ( struct cgraph_node * node,
void * data )
static
Count number of callers of NODE and store it into DATA (that
points to int.  Worker for cgraph_for_node_and_aliases.   

References cgraph_node::callers, cgraph_edge::next_caller, and num_calls().

Referenced by inline_small_functions(), and ipa_inline().

◆ update_callee_keys()

static void update_callee_keys ( edge_heap_t * heap,
struct cgraph_node * node,
struct cgraph_node * update_since,
bitmap updated_nodes )
static
Recompute HEAP nodes for each uninlined call in NODE
If UPDATE_SINCE is non-NULL check if edges called within that function
are inlinable (typically UPDATE_SINCE is the inline clone we introduced
where all edges have new context).

This is used when we know that edge badnesses are going only to increase
(we introduced new call site) and thus all we need is to insert newly
created edges into heap.   

References cgraph_edge::aux, AVAIL_AVAILABLE, bitmap_bit_p, cgraph_edge::callee, cgraph_node::callees, cgraph_edge::caller, cgraph_node::callers, can_inline_edge_by_limits_p(), can_inline_edge_p(), fibonacci_heap< K, V >::delete_node(), gcc_checking_assert, cgraph_node::get_uid(), cgraph_edge::inline_failed, ipa_fn_summaries, cgraph_edge::next_callee, NULL, report_inline_failed_reason(), cgraph_node::ultimate_alias_target(), update_edge_key(), and want_inline_small_function_p().

Referenced by inline_small_functions(), and resolve_noninline_speculation().

◆ update_caller_keys()

static void update_caller_keys ( edge_heap_t * heap,
struct cgraph_node * node,
bitmap updated_nodes,
struct cgraph_edge * check_inlinablity_for )
static
Recompute HEAP nodes for each of caller of NODE.
UPDATED_NODES track nodes we already visited, to avoid redundant work.
When CHECK_INLINABLITY_FOR is set, re-check for specified edge that
it is inlinable. Otherwise check all edges.   

References symtab_node::alias, bitmap_set_bit, cgraph_node::callers, can_inline_edge_by_limits_p(), can_inline_edge_p(), fibonacci_heap< K, V >::delete_node(), dyn_cast(), FOR_EACH_ALIAS, cgraph_node::get_uid(), cgraph_node::inlined_to, ipa_fn_summaries, cgraph_edge::next_caller, NULL, ipa_ref::referring, report_inline_failed_reason(), update_caller_keys(), update_edge_key(), and want_inline_small_function_p().

Referenced by inline_small_functions(), resolve_noninline_speculation(), and update_caller_keys().

◆ update_edge_key()

◆ want_early_inline_function_p()

◆ want_inline_function_to_all_callers_p()

static bool want_inline_function_to_all_callers_p ( struct cgraph_node * node,
bool cold )
static
Decide if inlining NODE would reduce unit size by eliminating
the offline copy of function.
When COLD is true the cold calls are considered, too.   

References symtab_node::alias, cgraph_node::call_for_symbol_and_aliases(), check_callers(), growth_positive_p(), has_caller_p(), cgraph_node::inlined_to, INT_MIN, and NULL.

Referenced by ipa_inline().

◆ want_inline_self_recursive_call_p()

static bool want_inline_self_recursive_call_p ( struct cgraph_edge * edge,
struct cgraph_node * outer_node,
bool peeling,
int depth )
static
EDGE is self recursive edge.
We handle two cases - when function A is inlining into itself
or when function A is being inlined into another inliner copy of function
A within function B.

In first case OUTER_NODE points to the toplevel copy of A, while
in the second case OUTER_NODE points to the outermost copy of A in B.

In both cases we want to be extra selective since
inlining the call will just introduce new recursive calls to appear.   

References cgraph_node::callers, can_inline_edge_by_limits_p(), CAN_INLINE_FORCE_LIMITS, CAN_INLINE_REPORT, symtab_node::decl, DECL_DECLARED_INLINE_P, dump_enabled_p(), dump_printf_loc(), i, cgraph_node::inlined_to, MSG_MISSED_OPTIMIZATION, NULL, opt_for_fn, and cgraph_edge::sreal_frequency().

Referenced by inline_small_functions(), and recursive_inlining().

◆ want_inline_small_function_p()

◆ wrapper_heuristics_may_apply()

static bool wrapper_heuristics_may_apply ( struct cgraph_node * where,
int size )
static
Return true if WHERE of SIZE is a possible candidate for wrapper heuristics
in estimate_edge_badness.   

References symtab_node::decl, DECL_DECLARED_INLINE_P, inline_insns_auto(), and inline_insns_single().

Referenced by edge_badness(), and inline_small_functions().

Variable Documentation

◆ max_count

◆ overall_size

int overall_size
static
Statistics we collect about inlining algorithm.   

Referenced by inline_small_functions(), and recursive_inlining().

◆ spec_rem