auto-profile.cc File Reference

#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
#include "predict.h"
#include "alloc-pool.h"
#include "tree-pass.h"
#include "ssa.h"
#include "cgraph.h"
#include "gcov-io.h"
#include "diagnostic-core.h"
#include "profile.h"
#include "langhooks.h"
#include "context.h"
#include "pass_manager.h"
#include "cfgloop.h"
#include "tree-cfg.h"
#include "tree-cfgcleanup.h"
#include "tree-into-ssa.h"
#include "gimple-iterator.h"
#include "value-prof.h"
#include "symbol-summary.h"
#include "sreal.h"
#include "ipa-cp.h"
#include "ipa-prop.h"
#include "ipa-fnsummary.h"
#include "ipa-inline.h"
#include "tree-inline.h"
#include "auto-profile.h"
#include "tree-pretty-print.h"
#include "gimple-pretty-print.h"
#include "output.h"

Include dependency graph for auto-profile.cc:

Data Structures
class	autofdo::edge_info
struct	autofdo::decl_lineno
class	autofdo::count_info
struct	autofdo::string_compare
struct	autofdo::summary_info
struct	autofdo::summary_info::detailed_summary
class	autofdo::string_table
class	autofdo::function_instance_descriptor
class	autofdo::function_instance
class	autofdo::autofdo_source_profile
struct	autofdo::scale

Namespaces
namespace	autofdo

Macros
#define	INCLUDE_MAP
#define	INCLUDE_SET
#define	DEFAULT_AUTO_PROFILE_FILE   "fbdata.afdo"
#define	AFDO_EINFO(e)

Typedefs
typedef auto_vec< decl_lineno, 20 >	autofdo::inline_stack
typedef auto_vec< const char * >	autofdo::string_vector
typedef std::map< unsigned, gcov_type >	autofdo::icall_target_map
typedef std::set< gimple * >	autofdo::stmt_set
typedef hash_set< int_hash< int, -1, -2 > >	autofdo::name_index_set
typedef hash_map< int_hash< int, -1, -2 >, int >	autofdo::name_index_map
typedef std::set< basic_block >	autofdo::bb_set

Functions
bool	maybe_hot_afdo_count_p (profile_count count)
static bool	stmt_loc_used_by_debug_info (gimple *stmt)
static char *	autofdo::get_original_name (const char *name, bool alloc=true)
static unsigned	autofdo::get_combined_location (location_t loc, tree decl)
static tree	autofdo::get_function_decl_from_block (tree block)
static void	autofdo::dump_afdo_loc (FILE *f, unsigned loc)
static const char *	autofdo::raw_symbol_name (const char *asmname)
static const char *	autofdo::raw_symbol_name (tree decl)
static void	autofdo::dump_inline_stack (FILE *f, inline_stack *stack)
static void	autofdo::get_inline_stack (location_t locus, inline_stack *stack, tree fn=current_function_decl)
static void	autofdo::get_inline_stack_in_node (location_t locus, inline_stack *stack, cgraph_node *node)
static unsigned	autofdo::get_relative_location_for_locus (tree fn, tree block, location_t locus)
static unsigned	autofdo::get_relative_location_for_stmt (tree fn, gimple *stmt)
static const char *	autofdo::get_normalized_path (const char *path, bool from_gcov=false)
static int	autofdo::match_with_target (cgraph_node *n, gimple *stmt, function_instance *inlined_fn, cgraph_node *orig_callee)
static void	autofdo::dump_stmt (gimple *stmt, count_info *info, function_instance *inlined_fn, inline_stack &stack)
void	autofdo::mark_expr_locations (function_instance *f, tree t, cgraph_node *node, hash_set< const count_info * > &counts)
static void	autofdo::walk_block (tree fn, function_instance *s, tree block)
static void	autofdo::fake_read_autofdo_module_profile ()
static void	autofdo::read_profile (void)
static bool	autofdo::afdo_indirect_call (gcall *stmt, const icall_target_map &map, bool transform, cgraph_edge *indirect_edge)
static bool	autofdo::afdo_vpt (gcall *gs, const icall_target_map &map, bool transform, cgraph_edge *indirect_edge)
static bool	autofdo::is_bb_annotated (const basic_block bb, const bb_set &annotated)
static void	autofdo::set_bb_annotated (basic_block bb, bb_set *annotated)
static void	autofdo::update_count_by_afdo_count (profile_count *count, gcov_type c)
static void	autofdo::update_count_by_afdo_count (profile_count *count, profile_count c)
static gcov_type	autofdo::afdo_unscaled_edge_count (edge e)
static bool	autofdo::afdo_set_bb_count (basic_block bb, hash_set< basic_block > &zero_bbs)
static void	autofdo::afdo_find_equiv_class (bb_set *annotated_bb)
static bool	autofdo::afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
static void	autofdo::afdo_propagate_circuit (const bb_set &annotated_bb)
static void	autofdo::afdo_propagate (bb_set *annotated_bb)
static int	autofdo::cmp (const void *a, const void *b)
static void	autofdo::add_scale (vec< scale > *scales, profile_count annotated, profile_count orig)
static void	autofdo::scale_bbs (const vec< basic_block > &bbs, sreal scale)
sreal	autofdo::determine_scale (vec< scale > *scales, profile_count max_count, profile_count max_count_in_fn)
bool	autofdo::scale_bb_profile ()
void	autofdo::afdo_adjust_guessed_profile (bb_set *annotated_bb)
static void	autofdo::afdo_calculate_branch_prob (bb_set *annotated_bb)
static void	autofdo::afdo_annotate_cfg (void)
static unsigned int	autofdo::auto_profile (void)
void	read_autofdo_file (void)
void	end_auto_profile (void)
bool	afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edgeedge)
bool	afdo_vpt_for_early_inline (cgraph_node *node)
void	remove_afdo_speculative_target (cgraph_edge *e)
simple_ipa_opt_pass *	make_pass_ipa_auto_profile (gcc::context *ctxt)
simple_ipa_opt_pass *	make_pass_ipa_auto_profile_offline (gcc::context *ctxt)

Variables
gcov_type	afdo_hot_bb_threshold = -1
static summary_info *	autofdo::afdo_summary_info
static string_table *	autofdo::afdo_string_table
static autofdo_source_profile *	autofdo::afdo_source_profile
static gcov_summary *	autofdo::afdo_profile_info
static std::map< gcov_type, int >	autofdo::timestamp_info_map
static gcov_type	autofdo::afdo_count_scale = 1

Macro Definition Documentation

AFDO_EINFO

#define AFDO_EINFO

(

e

)

Value:

((class edge_info *) e->aux)

Intermediate edge info used when propagating AutoFDO profile information.
We can't edge->count() directly since it's computed from edge's probability
while probability is yet not decided during propagation.   

Referenced by autofdo::afdo_adjust_guessed_profile(), autofdo::afdo_calculate_branch_prob(), autofdo::afdo_propagate_circuit(), and autofdo::afdo_propagate_edge().

DEFAULT_AUTO_PROFILE_FILE

#define DEFAULT_AUTO_PROFILE_FILE   "fbdata.afdo"

The following routines implement AutoFDO optimization.

  This optimization uses sampling profiles to annotate basic block counts
  and uses heuristics to estimate branch probabilities.

  There are three phases in AutoFDO:

  Phase 1: At startup.
    Read profile from the profile data file.
    The following info is read from the profile datafile:
       * string_table: a map between function name and its index.
       * autofdo_source_profile: a map from function_instance name to
         function_instance. This is represented as a forest of
         function_instances.
       * WorkingSet: a histogram of how many instructions are covered for a
         given percentage of total cycles. This is describing the binary
         level information (not source level). This info is used to help
         decide if we want aggressive optimizations that could increase
         code footprint (e.g. loop unroll etc.)
    A function instance is an instance of function that could either be a
    standalone symbol, or a clone of a function that is inlined into another
    function.

  Phase 2: In afdo_offline pass.
    Remove function instances from other translation units
    and offline all cross-translation unit inlining done during train
    run compilation.  This is necessary to not lose profiles with
    LTO train run.

  Phase 3: During early optimization.
    AFDO inline + value profile transformation.
    This happens during early optimization.
    During early inlining AFDO inliner is executed which
    uses autofdo_source_profile to find if a callsite is:
       * inlined in the profiled binary.
       * callee body is hot in the profiling run.
    If both condition satisfies, early inline will inline the callsite
    regardless of the code growth.

    Performing this early has benefit of doing early optimizations
    before read IPA passes and getting more "context sensitivity" of
    the profile read.  Profile of inlined functions may differ
    significantly from one inline instance to another and from the
    offline version.

    This is controlled by -fauto-profile-inlining and is independent
    of -fearly-inlining.

  Phase 4: In AFDO pass.
    Offline all functions that has been inlined in the
    train run but were not inlined in early inlining nor AFDO
    inline.

  Phase 5: In AFDO pass.
    Annotate control flow graph.
       * Annotate basic block count
       * Estimate branch probability
       * Use earlier static profile to fill in the gaps
         if AFDO profile is ambiguous

  After the above 5 phases, all profile is readily annotated on the GCC IR.
  AutoFDO tries to reuse all FDO infrastructure as much as possible to make
  use of the profile. E.g. it uses existing mechanism to calculate the basic
  block/edge frequency, as well as the cgraph node/edge count.

Referenced by read_autofdo_file().

INCLUDE_MAP

#define INCLUDE_MAP

Read and annotate call graph profile from the auto profile data file.
   Copyright (C) 2014-2026 Free Software Foundation, Inc.
   Contributed by Dehao Chen (dehao@google.com)

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.   

INCLUDE_SET

#define INCLUDE_SET

Function Documentation

afdo_callsite_hot_enough_for_early_inline()

bool afdo_callsite_hot_enough_for_early_inline

(

struct cgraph_edge *

edge

)

Returns TRUE if EDGE is hot enough to be inlined early.   

References profile_count::afdo(), afdo_callsite_hot_enough_for_early_inline(), autofdo::afdo_source_profile, cgraph_node::count, profile_count::dump(), dump_file, profile_count::from_gcov_type(), autofdo::autofdo_source_profile::get_callsite_total_count(), and maybe_hot_afdo_count_p().

Referenced by afdo_callsite_hot_enough_for_early_inline(), and inline_functions_by_afdo().

afdo_vpt_for_early_inline()

bool afdo_vpt_for_early_inline

(

cgraph_node *

node

)

Do indirect call promotion during early inlining to make the
IR match the profiled binary before actual annotation.

This is needed because an indirect call might have been promoted
and inlined in the profiled binary.  If we do not promote and
inline these indirect calls before annotation, the profile for
these promoted functions will be lost.

e.g. foo() --indirect_call--> bar()
In profiled binary, the callsite is promoted and inlined, making
the profile look like:

foo: {
  loc_foo_1: count_1
  bar@loc_foo_2: {
    loc_bar_1: count_2
    loc_bar_2: count_3
  }
}

Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
If we perform annotation on it, the profile inside bar@loc_foo2
will be wasted.

To avoid this, we promote loc_foo_2 and inline the promoted bar
function before annotation, so the profile inside bar@loc_foo2
will be useful.   

Try to turn indirect calls into speculative calls for early inlining.   

References autofdo::afdo_source_profile, autofdo::afdo_vpt(), afdo_vpt_for_early_inline(), autofdo::count_info::count, symtab_node::decl, autofdo::autofdo_source_profile::get_count_info(), gimple_bb(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), MAX, NULL, stmt_loc_used_by_debug_info(), and autofdo::count_info::targets.

Referenced by afdo_vpt_for_early_inline(), and inline_functions_by_afdo().

end_auto_profile()

void end_auto_profile

(

void

)

Free the resources.   

References autofdo::afdo_source_profile, autofdo::afdo_string_table, autofdo::afdo_summary_info, end_auto_profile(), NULL, and profile_info.

Referenced by compile_file(), and end_auto_profile().

make_pass_ipa_auto_profile()

simple_ipa_opt_pass * make_pass_ipa_auto_profile

(

gcc::context *

ctxt

)

References make_pass_ipa_auto_profile().

Referenced by make_pass_ipa_auto_profile(), and make_pass_sprintf_length().

make_pass_ipa_auto_profile_offline()

simple_ipa_opt_pass * make_pass_ipa_auto_profile_offline

(

gcc::context *

ctxt

)

References make_pass_ipa_auto_profile_offline().

Referenced by make_pass_ipa_auto_profile_offline(), and make_pass_sprintf_length().

maybe_hot_afdo_count_p()

bool maybe_hot_afdo_count_p

(

profile_count

count

)

Return true if COUNT is possibly hot.   

References afdo_hot_bb_threshold, gcc_checking_assert, and maybe_hot_afdo_count_p().

Referenced by afdo_callsite_hot_enough_for_early_inline(), end_branch_prob(), and maybe_hot_afdo_count_p().

read_autofdo_file()

void read_autofdo_file

(

void

)

Read the profile from the profile data file.   

Read, process, finalize AutoFDO data structures.   

References autofdo::afdo_profile_info, gcov_summary::cutoff, DEFAULT_AUTO_PROFILE_FILE, NULL, read_autofdo_file(), autofdo::read_profile(), gcov_summary::runs, and gcov_summary::sum_max.

Referenced by read_autofdo_file().

remove_afdo_speculative_target()

void remove_afdo_speculative_target

(

cgraph_edge *

e

)

If speculation used during early inline, remove the target
so we do not speculate the indirect edge again during afdo pass.   

If speculation was early inlined, remove it from profile data so we
do not repeat it later.   

References autofdo::afdo_source_profile, remove_afdo_speculative_target(), and autofdo::autofdo_source_profile::remove_icall_target().

Referenced by inline_functions_by_afdo(), and remove_afdo_speculative_target().

stmt_loc_used_by_debug_info()

bool stmt_loc_used_by_debug_info ( gimple * stmt )

static

Return true if location of STMT may be expressed by debug info.   

References gimple_clobber_p(), gimple_debug_begin_stmt_p(), is_gimple_debug(), and stmt_loc_used_by_debug_info().

Referenced by autofdo::afdo_set_bb_count(), afdo_vpt_for_early_inline(), autofdo::autofdo_source_profile::get_count_info(), and stmt_loc_used_by_debug_info().

Variable Documentation

afdo_hot_bb_threshold

gcov_type afdo_hot_bb_threshold = -1

profile counts determined by AFDO smaller than afdo_hot_bb_threshold are
considered cols.   

Referenced by maybe_hot_afdo_count_p(), and autofdo::autofdo_source_profile::read().