GCC Middle and Back End API Reference
|
Data Structures | |
class | autofdo_source_profile |
class | count_info |
struct | decl_lineno |
class | edge_info |
class | function_instance |
struct | scale |
struct | string_compare |
class | string_table |
Typedefs | |
typedef auto_vec< decl_lineno, 20 > | inline_stack |
typedef auto_vec< char * > | string_vector |
typedef std::map< unsigned, gcov_type > | icall_target_map |
typedef std::set< gimple * > | stmt_set |
typedef hash_set< int_hash< int, -1, -2 > > | name_index_set |
typedef hash_map< int_hash< int, -1, -2 >, int > | name_index_map |
typedef std::set< basic_block > | bb_set |
Functions | |
static char * | get_original_name (const char *name, bool alloc=true) |
static unsigned | get_combined_location (location_t loc, tree decl) |
static tree | get_function_decl_from_block (tree block) |
static void | dump_afdo_loc (FILE *f, unsigned loc) |
static const char * | raw_symbol_name (const char *asmname) |
static const char * | raw_symbol_name (tree decl) |
static void | dump_inline_stack (FILE *f, inline_stack *stack) |
static void | get_inline_stack (location_t locus, inline_stack *stack, tree fn=current_function_decl) |
static void | get_inline_stack_in_node (location_t locus, inline_stack *stack, cgraph_node *node) |
static unsigned | get_relative_location_for_locus (tree fn, tree block, location_t locus) |
static unsigned | get_relative_location_for_stmt (tree fn, gimple *stmt) |
static int | match_with_target (cgraph_node *n, gimple *stmt, function_instance *inlined_fn, cgraph_node *orig_callee) |
static void | dump_stmt (gimple *stmt, count_info *info, function_instance *inlined_fn, inline_stack &stack) |
void | mark_expr_locations (function_instance *f, tree t, cgraph_node *node, hash_set< const count_info * > &counts) |
static void | walk_block (tree fn, function_instance *s, tree block) |
static void | fake_read_autofdo_module_profile () |
static void | read_profile (void) |
static bool | afdo_indirect_call (gcall *stmt, const icall_target_map &map, bool transform, cgraph_edge *indirect_edge) |
static bool | afdo_vpt (gcall *gs, const icall_target_map &map, bool transform, cgraph_edge *indirect_edge) |
static bool | is_bb_annotated (const basic_block bb, const bb_set &annotated) |
static void | set_bb_annotated (basic_block bb, bb_set *annotated) |
static void | update_count_by_afdo_count (profile_count *count, gcov_type c) |
static void | update_count_by_afdo_count (profile_count *count, profile_count c) |
static bool | afdo_set_bb_count (basic_block bb, hash_set< basic_block > &zero_bbs) |
static void | afdo_find_equiv_class (bb_set *annotated_bb) |
static bool | afdo_propagate_edge (bool is_succ, bb_set *annotated_bb) |
static void | afdo_propagate_circuit (const bb_set &annotated_bb) |
static void | afdo_propagate (bb_set *annotated_bb) |
static int | cmp (const void *a, const void *b) |
static void | add_scale (vec< scale > *scales, profile_count annotated, profile_count orig) |
static void | scale_bbs (const vec< basic_block > &bbs, sreal scale) |
void | afdo_adjust_guessed_profile (bb_set *annotated_bb) |
static void | afdo_calculate_branch_prob (bb_set *annotated_bb) |
static void | afdo_annotate_cfg (void) |
static unsigned int | auto_profile (void) |
Variables | |
static string_table * | afdo_string_table |
static autofdo_source_profile * | afdo_source_profile |
static gcov_summary * | afdo_profile_info |
static gcov_type | afdo_count_scale = 1 |
typedef std::set<basic_block> autofdo::bb_set |
typedef std::map<unsigned, gcov_type> autofdo::icall_target_map |
Map from function name's index in string_table to target's execution count.
typedef auto_vec<decl_lineno, 20> autofdo::inline_stack |
Represent an inline stack. vector[0] is the leaf node.
typedef hash_map<int_hash <int, -1, -2>, int> autofdo::name_index_map |
typedef hash_set<int_hash <int, -1, -2> > autofdo::name_index_set |
Set and map used to translate name indexes.
typedef std::set<gimple *> autofdo::stmt_set |
Set of gimple stmts. Used to track if the stmt has already been promoted to direct call.
typedef auto_vec<char *> autofdo::string_vector |
String array that stores function names.
|
static |
Add scale ORIG/ANNOTATED to SCALES.
References profile_count::dump(), dump_file, profile_count::force_nonzero(), profile_count::guessed_local(), PRId64, profile_count::to_sreal_scale(), and profile_count::value().
Referenced by afdo_adjust_guessed_profile().
void autofdo::afdo_adjust_guessed_profile | ( | bb_set * | annotated_bb | ) |
In case given basic block was fully optimized out, AutoFDO will have no data about it. In this case try to preserve static profile. Identify connected components (in undirected form of CFG) which has no annotations at all. Look at thir boundaries and try to determine scaling factor and scale.
References add_scale(), AFDO_EINFO, b, cfun, cmp(), basic_block_def::count, dump_file, FOR_ALL_BB_FN, gcc_checking_assert, profile_count::guessed_local(), basic_block_def::index, profile_count::initialized_p(), is_bb_annotated(), last_basic_block_for_fn, profile_count::max(), max_count, n_basic_blocks_for_fn, PRId64, scale_bbs(), profile_count::to_sreal_scale(), and profile_count::zero().
Referenced by afdo_calculate_branch_prob().
|
static |
Annotate auto profile to the control flow graph.
References AFDO, profile_count::afdo(), afdo_calculate_branch_prob(), afdo_count_scale, afdo_set_bb_count(), afdo_source_profile, calculate_dominance_info(), CDI_DOMINATORS, CDI_POST_DOMINATORS, cfun, basic_block_def::count, cgraph_node::count, count, current_function_decl, profile_count::dump(), dump_file, ENTRY_BLOCK_PTR_FOR_FN, EXIT_BLOCK_PTR_FOR_FN, FOR_ALL_BB_FN, FOR_EACH_BB_FN, free_dominance_info(), gcc_assert, cgraph_node::get(), gimple_value_profile_transformations(), profile_count::global0afdo(), GUESSED_LOCAL, basic_block_def::index, profile_count::ipa_p(), is_bb_annotated(), hash_set< KeyId, Lazy, Traits >::is_empty(), loop_optimizer_finalize(), loop_optimizer_init(), profile_count::nonzero_p(), NULL, PROFILE_READ, profile_status_for_fn, profile_count::quality(), set_bb_annotated(), TODO_update_ssa, profile_count::uninitialized(), update_count_by_afdo_count(), update_max_bb_count(), update_ssa(), and profile_count::zero().
Referenced by auto_profile().
|
static |
Propagate counts on control flow graph and calculate branch probabilities.
References profile_count::afdo(), profile_probability::afdo(), afdo_adjust_guessed_profile(), AFDO_EINFO, afdo_find_equiv_class(), afdo_propagate(), profile_probability::always(), basic_block_def::aux, cfun, FOR_ALL_BB_FN, FOR_EACH_BB_FN, FOR_EACH_EDGE, gcc_assert, GUESSED, is_bb_annotated(), profile_probability::never(), profile_count::nonzero_p(), NULL, basic_block_def::succs, and profile_count::zero().
Referenced by afdo_annotate_cfg().
|
static |
BB1 and BB2 are in an equivalent class iff: 1. BB1 dominates BB2. 2. BB2 post-dominates BB1. 3. BB1 and BB2 are in the same loop nest. This function finds the equivalent class for each basic block, and stores a pointer to the first BB in its equivalent class. Meanwhile, set bb counts for the same equivalent class to be idenical. Update ANNOTATED_BB for the first BB in its equivalent class.
References basic_block_def::aux, CDI_DOMINATORS, CDI_POST_DOMINATORS, cfun, basic_block_def::count, dominated_by_p(), dump_file, FOR_ALL_BB_FN, get_dominated_by(), basic_block_def::index, is_bb_annotated(), basic_block_def::loop_father, NULL, set_bb_annotated(), and update_count_by_afdo_count().
Referenced by afdo_calculate_branch_prob().
|
static |
From AutoFDO profiles, find values inside STMT for that we want to measure histograms for indirect-call optimization. This function is actually served for 2 purposes: * before annotation, we need to mark histogram, promote and inline * after annotation, we just need to mark, and let follow-up logic to decide if it needs to promote and inline.
References afdo_count_scale, afdo_string_table, cfun, count, histogram_value_t::counters, symtab_node::decl, symtab_node::definition, dump_file, cgraph_node::get_for_asmname(), get_identifier(), gimple_add_histogram_value(), gimple_alloc_histogram_value(), gimple_bb(), gimple_call_fn(), gimple_call_fndecl(), gimple_call_internal_p(), HIST_TYPE_INDIR_CALL, histogram_value_t::hvalue, cgraph_edge::make_speculative(), map, histogram_value_t::n_counters, NULL, NULL_TREE, print_generic_expr(), print_gimple_stmt(), cgraph_node::profile_id, and TDF_SLIM.
Referenced by afdo_vpt().
|
static |
Propagate the basic block count and edge count on the control flow graph. We do the propagation iteratively until stablize.
References afdo_propagate_circuit(), afdo_propagate_edge(), basic_block_def::aux, cfun, changed, basic_block_def::count, profile_count::dump(), dump_file, FOR_ALL_BB_FN, i, basic_block_def::index, is_bb_annotated(), set_bb_annotated(), and update_count_by_afdo_count().
Referenced by afdo_calculate_branch_prob().
|
static |
Special propagation for circuit expressions. Because GCC translates control flow into data flow for circuit expressions. E.g. BB1: if (a && b) BB2 else BB3 will be translated into: BB1: if (a) goto BB.t1 else goto BB.t3 BB.t1: if (b) goto BB.t2 else goto BB.t3 BB.t2: goto BB.t3 BB.t3: tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2) if (tmp) goto BB2 else goto BB3 In this case, we need to propagate through PHI to determine the edge count of BB1->BB.t1, BB.t1->BB.t2.
References AFDO_EINFO, cfun, dyn_cast(), FOR_ALL_BB_FN, FOR_EACH_EDGE, gimple_assign_rhs1(), gimple_assign_single_p(), gimple_cond_code(), gimple_cond_lhs(), gimple_cond_rhs(), gimple_phi_arg_def(), gimple_phi_arg_edge(), gimple_phi_num_args(), i, integer_onep(), integer_zerop(), is_bb_annotated(), last_nondebug_stmt(), SSA_NAME_DEF_STMT, basic_block_def::succs, TREE_CODE, TREE_CONSTANT, and profile_count::zero().
Referenced by afdo_propagate().
If a basic block's count is known, and only one of its in/out edges' count is unknown, its count can be calculated. Meanwhile, if all of the in/out edges' counts are known, then the basic block's unknown count can also be calculated. Also, if a block has a single predecessor or successor, the block's count can be propagated to that predecessor or successor. IS_SUCC is true if out edges of a basic blocks are examined. Update ANNOTATED_BB accordingly. Return TRUE if any basic block/edge count is changed.
References profile_count::afdo(), AFDO_EINFO, cfun, changed, basic_block_def::count, profile_count::dump(), dump_file, FOR_EACH_BB_FN, FOR_EACH_EDGE, gcc_assert, basic_block_def::index, is_bb_annotated(), profile_count::nonzero_p(), NULL, basic_block_def::preds, set_bb_annotated(), single_pred_edge(), single_pred_p(), single_succ_edge(), single_succ_p(), basic_block_def::succs, update_count_by_afdo_count(), and profile_count::zero().
Referenced by afdo_propagate().
|
static |
For a given BB, set its execution count. Attach value profile if a stmt is not in PROMOTED, because we only want to promote an indirect call once. Return TRUE if BB is annotated.
References hash_set< KeyId, Lazy, Traits >::add(), afdo_count_scale, afdo_source_profile, afdo_vpt(), autofdo::count_info::count, basic_block_def::count, dump_file, dyn_cast(), FOR_EACH_EDGE, gimple_clobber_p(), gimple_phi_arg_location_from_edge(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_start_phis(), gsi_stmt(), basic_block_def::index, max_count, NULL, PRId64, print_gimple_stmt(), PRIu64, basic_block_def::succs, autofdo::count_info::targets, TDF_SLIM, and update_count_by_afdo_count().
Referenced by afdo_annotate_cfg().
|
static |
From AutoFDO profiles, find values inside STMT for that we want to measure histograms and adds them to list VALUES.
References afdo_indirect_call(), and map.
Referenced by afdo_set_bb_count(), and afdo_vpt_for_early_inline().
|
static |
Use AutoFDO profile to annoate the control flow graph. Return the todo flag.
References afdo_annotate_cfg(), afdo_profile_info, afdo_source_profile, BUILTINS_LOCATION, CDI_DOMINATORS, CDI_POST_DOMINATORS, compute_function_frequency(), symtab_node::decl, DECL_SOURCE_LOCATION, DECL_STRUCT_FUNCTION, FINISHED, FOR_EACH_FUNCTION, free_dominance_info(), gimple_has_body_p(), init_node_map(), pop_cfun(), profile_info, push_cfun(), cgraph_edge::rebuild_edges(), and symtab.
|
static |
|
static |
Dump LOC to F.
Referenced by autofdo::function_instance::dump(), dump_inline_stack(), autofdo::function_instance::dump_inline_stack(), autofdo::function_instance::match(), and walk_block().
|
static |
Dump STACK to F.
References dump_afdo_loc(), and raw_symbol_name().
Referenced by autofdo::autofdo_source_profile::get_callsite_total_count(), and autofdo::autofdo_source_profile::update_inlined_ind_target().
|
static |
References autofdo::count_info::count, dump_file, gcc_checking_assert, print_gimple_stmt(), PRIu64, and TDF_SLIM.
Referenced by autofdo::function_instance::match().
|
static |
Module profile is only used by LIPO. Here we simply ignore it.
References gcc_assert, and gcov_read_unsigned().
Referenced by read_profile().
|
static |
Return the combined location, which is a 32bit integer in which higher 16 bits stores the line offset of LOC to the start lineno of DECL, The lower 16 bits stores the discriminator.
References DECL_SOURCE_LINE, DECL_SOURCE_LOCATION, get_discriminator_from_loc(), inform(), LOCATION_LINE, and warning_at().
Referenced by get_inline_stack(), get_relative_location_for_locus(), and autofdo::function_instance::match().
Return the function decl of a given lexical BLOCK.
References BLOCK_ABSTRACT_ORIGIN, inlined_function_outer_scope_p(), and NULL_TREE.
Referenced by get_inline_stack(), and get_relative_location_for_locus().
|
static |
Store inline stack for STMT in STACK.
References BLOCK_SOURCE_LOCATION, BLOCK_SUPERCONTEXT, current_function_decl, get_combined_location(), get_function_decl_from_block(), LOCATION_BLOCK, LOCATION_LOCUS, TREE_CODE, and UNKNOWN_LOCATION.
Referenced by get_inline_stack_in_node().
|
static |
Same as get_inline_stack for a given node which may be an inline clone. If NODE is NULL, assume current_function_decl.
References cgraph_edge::call_stmt, cgraph_edge::caller, cgraph_node::callers, symtab_node::decl, get_inline_stack(), gimple_location(), cgraph_node::inlined_to, and NULL.
Referenced by autofdo::autofdo_source_profile::get_callsite_total_count(), autofdo::autofdo_source_profile::get_count_info(), mark_expr_locations(), autofdo::function_instance::match(), autofdo::autofdo_source_profile::remove_icall_target(), and autofdo::autofdo_source_profile::update_inlined_ind_target().
Helper functions.
Return the original name of NAME: strip the suffix that starts with '.' for names that are generetad after auto-profile pass. This is to match profiled names with the names in the IR at this stage. Note that we only have to strip sufix and not in the middle. Caller is responsible for freeing RET.
References get_original_name(), i, and NULL.
Referenced by get_original_name(), and autofdo::autofdo_source_profile::offline_external_functions().
Return combined location of LOCUS within BLOCK that is in function FN. This is a 32bit integer in which higher 16 bits stores the line offset of LOC to the start lineno of DECL, The lower 16 bits stores the discriminator.
References BLOCK_SUPERCONTEXT, get_combined_location(), get_function_decl_from_block(), inlined_function_outer_scope_p(), LOCATION_LOCUS, TREE_CODE, and UNKNOWN_LOCATION.
Referenced by get_relative_location_for_stmt(), and walk_block().
Return combined location of STMT in function FN.
References get_relative_location_for_locus(), gimple_location(), and LOCATION_BLOCK.
Referenced by autofdo::function_instance::find_icall_target_map(), and autofdo::function_instance::remove_icall_target().
|
static |
References AFDO, basic_block_def::count, gcc_checking_assert, profile_count::nonzero_p(), and profile_count::quality().
Referenced by afdo_adjust_guessed_profile(), afdo_annotate_cfg(), afdo_calculate_branch_prob(), afdo_find_equiv_class(), afdo_propagate(), afdo_propagate_circuit(), and afdo_propagate_edge().
void autofdo::mark_expr_locations | ( | function_instance * | f, |
tree | t, | ||
cgraph_node * | node, | ||
hash_set< const count_info * > & | counts ) |
Mark expr locations as used.
References hash_set< KeyId, Lazy, Traits >::add(), EXPR_LOCATION, get_inline_stack_in_node(), handled_component_p(), autofdo::function_instance::lookup_count(), and TREE_OPERAND.
Referenced by autofdo::function_instance::match().
|
static |
Try to check if inlined_fn can correspond to a call of function N. Return non-zero if it correspons and 2 if renaming was done.
References afdo_string_table, symtab_node::decl, symtab_node::definition, dump_file, lang_hooks::dwarf_name, gimple_location(), i, inform(), autofdo::function_instance::name(), raw_symbol_name(), autofdo::function_instance::set_name(), cgraph_node::ultimate_alias_target(), and warning().
Referenced by autofdo::function_instance::match().
|
static |
Return assembler name as in symbol table and DW_AT_linkage_name.
References sorry(), and user_label_prefix.
Referenced by dump_inline_stack(), autofdo::autofdo_source_profile::get_callsite_total_count(), autofdo::function_instance::get_function_instance_by_decl(), autofdo::autofdo_source_profile::get_function_instance_by_inline_stack(), autofdo::string_table::get_index_by_decl(), match_with_target(), autofdo::autofdo_source_profile::offline_external_functions(), raw_symbol_name(), and walk_block().
|
static |
Convenience wrapper that looks up assembler name.
References DECL_ASSEMBLER_NAME, IDENTIFIER_POINTER, and raw_symbol_name().
|
static |
Read data from profile data file.
References afdo_source_profile, afdo_string_table, AUTO_PROFILE_VERSION, autofdo::autofdo_source_profile::create(), error(), fake_read_autofdo_module_profile(), GCOV_DATA_MAGIC, gcov_open(), gcov_read_unsigned(), and NULL.
Referenced by read_autofdo_file().
|
static |
Scale counts of all basic blocks in BBS by SCALE and convert them to IPA quality.
References profile_count::afdo(), b, profile_count::dump(), dump_file, profile_count::nonzero_p(), and profile_count::zero().
Referenced by afdo_adjust_guessed_profile().
|
static |
References AFDO, basic_block_def::count, gcc_checking_assert, profile_count::nonzero_p(), and profile_count::quality().
Referenced by afdo_annotate_cfg(), afdo_find_equiv_class(), afdo_propagate(), and afdo_propagate_edge().
|
static |
Update COUNT by known autofdo count C.
References profile_count::afdo(), count, profile_count::from_gcov_type(), GUESSED, GUESSED_LOCAL, and profile_count::zero().
Referenced by afdo_annotate_cfg(), afdo_find_equiv_class(), afdo_propagate(), afdo_propagate_edge(), and afdo_set_bb_count().
|
static |
Update COUNT by known autofdo count C.
References count.
|
static |
Walk scope block BLOCK and mark all inlined functions as realized.
References BLOCK_ABSTRACT_ORIGIN, BLOCK_CHAIN, BLOCK_SOURCE_LOCATION, BLOCK_SUBBLOCKS, BLOCK_SUPERCONTEXT, dump_afdo_loc(), dump_file, get_relative_location_for_locus(), inlined_function_outer_scope_p(), raw_symbol_name(), and walk_block().
Referenced by autofdo::autofdo_source_profile::offline_unrealized_inlines(), and walk_block().
|
static |
Scaling factor for afdo data. Compared to normal profile AFDO profile counts are much lower, depending on sampling frequency. We scale data up to reudce effects of roundoff errors.
Referenced by afdo_annotate_cfg(), afdo_indirect_call(), afdo_set_bb_count(), autofdo::function_instance::find_icall_target_map(), autofdo::autofdo_source_profile::get_callsite_total_count(), autofdo::autofdo_source_profile::read(), and autofdo::autofdo_source_profile::update_inlined_ind_target().
|
static |
gcov_summary structure to store the profile_info.
Referenced by auto_profile(), autofdo::autofdo_source_profile::read(), read_autofdo_file(), and autofdo::function_instance::read_function_instance().
|
static |
Store the AutoFDO source profile.
Referenced by afdo_annotate_cfg(), afdo_callsite_hot_enough_for_early_inline(), afdo_set_bb_count(), afdo_vpt_for_early_inline(), auto_profile(), end_auto_profile(), autofdo::function_instance::offline(), autofdo::autofdo_source_profile::read(), read_profile(), and remove_afdo_speculative_target().
|
static |
Store the strings read from the profile data file.
Referenced by afdo_indirect_call(), autofdo::function_instance::dump(), autofdo::function_instance::dump_inline_stack(), end_auto_profile(), autofdo::function_instance::find_icall_target_map(), autofdo::autofdo_source_profile::get_callsite_total_count(), autofdo::function_instance::get_cgraph_node(), autofdo::autofdo_source_profile::get_function_instance_by_decl(), autofdo::function_instance::get_function_instance_by_decl(), autofdo::autofdo_source_profile::get_function_instance_by_inline_stack(), autofdo::function_instance::match(), match_with_target(), autofdo::autofdo_source_profile::offline_external_functions(), autofdo::autofdo_source_profile::offline_unrealized_inlines(), autofdo::autofdo_source_profile::read(), and read_profile().