Line data Source code
1 : /* Read and annotate call graph profile from the auto profile data file.
2 : Copyright (C) 2014-2026 Free Software Foundation, Inc.
3 : Contributed by Dehao Chen (dehao@google.com)
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #define INCLUDE_MAP
23 : #define INCLUDE_SET
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "predict.h"
30 : #include "alloc-pool.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "cgraph.h"
34 : #include "gcov-io.h"
35 : #include "diagnostic-core.h"
36 : #include "profile.h"
37 : #include "langhooks.h"
38 : #include "context.h"
39 : #include "pass_manager.h"
40 : #include "cfgloop.h"
41 : #include "tree-cfg.h"
42 : #include "tree-cfgcleanup.h"
43 : #include "tree-into-ssa.h"
44 : #include "gimple-iterator.h"
45 : #include "value-prof.h"
46 : #include "symbol-summary.h"
47 : #include "sreal.h"
48 : #include "ipa-cp.h"
49 : #include "ipa-prop.h"
50 : #include "ipa-fnsummary.h"
51 : #include "ipa-inline.h"
52 : #include "tree-inline.h"
53 : #include "auto-profile.h"
54 : #include "tree-pretty-print.h"
55 : #include "gimple-pretty-print.h"
56 : #include "output.h"
57 :
58 : /* The following routines implement AutoFDO optimization.
59 :
60 : This optimization uses sampling profiles to annotate basic block counts
61 : and uses heuristics to estimate branch probabilities.
62 :
63 : There are three phases in AutoFDO:
64 :
65 : Phase 1: At startup.
66 : Read profile from the profile data file.
67 : The following info is read from the profile datafile:
68 : * string_table: a map between function name and its index.
69 : * autofdo_source_profile: a map from function_instance name to
70 : function_instance. This is represented as a forest of
71 : function_instances.
72 : * WorkingSet: a histogram of how many instructions are covered for a
73 : given percentage of total cycles. This is describing the binary
74 : level information (not source level). This info is used to help
75 : decide if we want aggressive optimizations that could increase
76 : code footprint (e.g. loop unroll etc.)
77 : A function instance is an instance of function that could either be a
78 : standalone symbol, or a clone of a function that is inlined into another
79 : function.
80 :
81 : Phase 2: In afdo_offline pass.
82 : Remove function instances from other translation units
83 : and offline all cross-translation unit inlining done during train
84 : run compilation. This is necessary to not lose profiles with
85 : LTO train run.
86 :
87 : Phase 3: During early optimization.
88 : AFDO inline + value profile transformation.
89 : This happens during early optimization.
90 : During early inlining AFDO inliner is executed which
91 : uses autofdo_source_profile to find if a callsite is:
92 : * inlined in the profiled binary.
93 : * callee body is hot in the profiling run.
94 : If both condition satisfies, early inline will inline the callsite
95 : regardless of the code growth.
96 :
97 : Performing this early has benefit of doing early optimizations
98 : before read IPA passes and getting more "context sensitivity" of
99 : the profile read. Profile of inlined functions may differ
100 : significantly from one inline instance to another and from the
101 : offline version.
102 :
103 : This is controlled by -fauto-profile-inlining and is independent
104 : of -fearly-inlining.
105 :
106 : Phase 4: In AFDO pass.
107 : Offline all functions that has been inlined in the
108 : train run but were not inlined in early inlining nor AFDO
109 : inline.
110 :
111 : Phase 5: In AFDO pass.
112 : Annotate control flow graph.
113 : * Annotate basic block count
114 : * Estimate branch probability
115 : * Use earlier static profile to fill in the gaps
116 : if AFDO profile is ambiguous
117 :
118 : After the above 5 phases, all profile is readily annotated on the GCC IR.
119 : AutoFDO tries to reuse all FDO infrastructure as much as possible to make
120 : use of the profile. E.g. it uses existing mechanism to calculate the basic
121 : block/edge frequency, as well as the cgraph node/edge count.
122 : */
123 :
124 : #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
125 : #define AUTO_PROFILE_VERSION 3
126 :
127 : /* profile counts determined by AFDO smaller than afdo_hot_bb_threshold are
128 : considered cols. */
129 : gcov_type afdo_hot_bb_threshold = -1;
130 :
131 : /* Return true if COUNT is possibly hot. */
132 : bool
133 0 : maybe_hot_afdo_count_p (profile_count count)
134 : {
135 0 : gcc_checking_assert (count.ipa ().initialized_p ());
136 0 : return count.ipa ().to_gcov_type () >= afdo_hot_bb_threshold;
137 : }
138 :
139 : /* Return true if location of STMT may be expressed by debug info. */
140 :
141 : static bool
142 0 : stmt_loc_used_by_debug_info (gimple *stmt)
143 : {
144 : /* Only inline_entry and gimple_bind's locations
145 : are not output into debug output. */
146 0 : if (is_gimple_debug (stmt))
147 0 : return gimple_debug_begin_stmt_p (stmt);
148 0 : if (gimple_code (stmt) == GIMPLE_LABEL
149 : || gimple_code (stmt) == GIMPLE_NOP
150 : || gimple_code (stmt) == GIMPLE_PREDICT)
151 : return false;
152 0 : if (gimple_clobber_p (stmt))
153 0 : return false;
154 : return true;
155 : }
156 :
157 : namespace autofdo
158 : {
159 :
160 : /* Intermediate edge info used when propagating AutoFDO profile information.
161 : We can't edge->count() directly since it's computed from edge's probability
162 : while probability is yet not decided during propagation. */
163 : #define AFDO_EINFO(e) ((class edge_info *) e->aux)
164 : class edge_info
165 : {
166 : public:
167 0 : edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
168 0 : bool is_annotated () const { return annotated_; }
169 0 : void set_annotated () { annotated_ = true; }
170 0 : profile_count get_count () const { return count_; }
171 0 : void set_count (profile_count count) { count_ = count; }
172 : private:
173 : profile_count count_;
174 : bool annotated_;
175 : };
176 :
177 : /* Represent a source location: (function_decl, lineno). */
178 : struct decl_lineno
179 : {
180 : tree decl;
181 : /* Relative locations stored in auto-profile. */
182 : unsigned int afdo_loc;
183 : /* Actual location afdo_loc was computed from used to output diagnostics. */
184 : location_t location;
185 : };
186 :
187 : /* Represent an inline stack. vector[0] is the leaf node. */
188 : typedef auto_vec<decl_lineno, 20> inline_stack;
189 :
190 : /* String array that stores function names. */
191 : typedef auto_vec<const char *> string_vector;
192 :
193 : /* Map from function name's index in string_table to target's
194 : execution count. */
195 : typedef std::map<unsigned, gcov_type> icall_target_map;
196 :
197 : /* Set of gimple stmts. Used to track if the stmt has already been promoted
198 : to direct call. */
199 : typedef std::set<gimple *> stmt_set;
200 :
201 : /* Set and map used to translate name indexes. */
202 : typedef hash_set<int_hash <int, -1, -2>> name_index_set;
203 : typedef hash_map<int_hash <int, -1, -2>, int> name_index_map;
204 :
205 : /* Represent count info of an inline stack. */
206 0 : class count_info
207 : {
208 : public:
209 : /* Sampled count of the inline stack. */
210 : gcov_type count;
211 :
212 : /* Map from indirect call target to its sample count. */
213 : icall_target_map targets;
214 :
215 : /* Whether this inline stack is already used in annotation.
216 :
217 : Each inline stack should only be used to annotate IR once.
218 : This will be enforced when instruction-level discriminator
219 : is supported. */
220 : };
221 :
222 : /* operator< for "const char *". */
223 : struct string_compare
224 : {
225 0 : bool operator()(const char *a, const char *b) const
226 : {
227 0 : return strcmp (a, b) < 0;
228 : }
229 : };
230 :
231 : /* Store the summary information for the profile. */
232 : struct summary_info
233 : {
234 : /* There are currently 16 hard-coded percentiles in the GCOV format. */
235 : static constexpr unsigned NUM_PERCENTILES = 16;
236 :
237 : /* The detailed summary is a histogram-based calculation of the minimum
238 : execution count required to belong to a certain set of percentile of
239 : counts. */
240 : struct detailed_summary
241 : {
242 : /* The percentile that this represents (multiplied by 1,000,000). */
243 : uint32_t cutoff;
244 : /* The minimum execution count required to belong to this percentile. */
245 : uint64_t min_count;
246 : /* The number of samples which belong to this percentile. */
247 : uint64_t num_counts;
248 : };
249 :
250 : /* The sum of execution counts of all samples. */
251 : uint64_t total_count;
252 : /* The maximum individual count. */
253 : uint64_t max_count;
254 : /* The maximum head count across all functions. */
255 : uint64_t max_function_count;
256 : /* The number of lines that have samples. */
257 : uint64_t num_counts;
258 : /* The number of functions that have samples. */
259 : uint64_t num_functions;
260 : /* The percentile threshold information. */
261 : detailed_summary detailed_summaries[NUM_PERCENTILES];
262 :
263 : /* Read profile. Return TRUE on success. */
264 : bool read ();
265 :
266 : /* Get the minimum count required for percentile CUTOFF. */
267 : uint64_t get_threshold_count (uint32_t cutoff);
268 : };
269 :
270 : /* Store a string array, indexed by string position in the array. */
271 : class string_table
272 : {
273 : public:
274 : static const int unknown_filename = -1;
275 :
276 0 : string_table ()
277 0 : {}
278 :
279 : ~string_table ();
280 :
281 : /* For a given string, returns its index. */
282 : int get_index (const char *name) const;
283 :
284 : /* For a given decl, returns the index of the decl name. */
285 : int get_index_by_decl (tree decl) const;
286 :
287 : /* For a given index, returns the symbol name. */
288 : const char *get_symbol_name (int index) const;
289 :
290 : /* For a given index, returns the filename. */
291 : const char *get_filename (int index) const;
292 :
293 : /* For a given symbol name index, returns the filename index. */
294 : int get_filename_by_symbol (int index) const;
295 :
296 : /* For a given function name, returns the filename index. */
297 : int get_filename_by_symbol (const char *name) const;
298 :
299 : /* For a given filename, returns the index. */
300 : int get_filename_index (const char *name) const;
301 :
302 : /* Get the original name and file name index for a node. This will return the
303 : name from the current TU if there are multiple symbols that map to
304 : NAME. */
305 : std::pair<const char *, int> get_original_name (const char *name) const;
306 :
307 : /* Read profile, return TRUE on success. */
308 : bool read ();
309 :
310 : /* Return number of entries. */
311 0 : size_t num_entries () { return symbol_names_.length (); }
312 :
313 : /* Add new symbol name STRING (with an associated file name FILENAME_IDX) and
314 : return its index. */
315 : int add_symbol_name (const char *string, int filename_idx);
316 :
317 : /* Add new filename and return its index (returning the same if it already
318 : exists). */
319 : int add_filename (const char *name);
320 :
321 : /* Return cgraph node corresponding to given name index. */
322 : cgraph_node *get_cgraph_node (int);
323 :
324 : const string_vector& filenames () { return filenames_; }
325 : private:
326 : typedef std::map<const char *, unsigned, string_compare> string_index_map;
327 : typedef std::map<const char *, auto_vec<unsigned>, string_compare>
328 : clashing_name_map;
329 : typedef std::map<const char *, char *, string_compare> string_string_map;
330 :
331 : string_vector symbol_names_;
332 : string_vector filenames_;
333 :
334 : string_index_map symbol_name_map_;
335 : string_index_map filename_map_;
336 : string_index_map symbol_to_filename_map_;
337 :
338 : string_string_map original_names_map_;
339 : clashing_name_map clashing_names_map_;
340 : };
341 :
342 : /* Descriptor for a function_instance which can be used to disambiguate it from
343 : other instances. This consists of the symbol name and the file name indices
344 : from string_table. */
345 :
346 : class function_instance_descriptor
347 : {
348 : /* The string_table index for the file name. */
349 : unsigned file_name_;
350 : /* The string_table index for the function name. */
351 : unsigned symbol_name_;
352 :
353 : public:
354 0 : unsigned file_name () const { return file_name_; }
355 0 : unsigned symbol_name () const { return symbol_name_; }
356 :
357 0 : function_instance_descriptor (unsigned file_name, unsigned symbol_name)
358 0 : : file_name_ (file_name), symbol_name_ (symbol_name)
359 : {}
360 :
361 0 : function_instance_descriptor (int file_name, int symbol_name)
362 0 : : file_name_ (file_name), symbol_name_ (symbol_name)
363 : {}
364 :
365 0 : void set_symbol_name (unsigned new_name) { symbol_name_ = new_name; }
366 :
367 0 : bool operator< (const function_instance_descriptor &other) const
368 : {
369 0 : return file_name_ < other.file_name_
370 0 : || (file_name_ == other.file_name_
371 0 : && symbol_name_ < other.symbol_name_);
372 : }
373 : };
374 :
375 : /* Profile of a function instance:
376 : 1. total_count of the function.
377 : 2. head_count (entry basic block count) of the function (only valid when
378 : function is a top-level function_instance, i.e. it is the original copy
379 : instead of the inlined copy).
380 : 3. map from source location (decl_lineno) to profile (count_info).
381 : 4. map from callsite to callee function_instance. */
382 :
383 : class function_instance
384 : {
385 : public:
386 : typedef auto_vec<function_instance *> function_instance_stack;
387 :
388 : /* Read the profile and return a function_instance with head count as
389 : HEAD_COUNT. Recursively read callsites to create nested function_instances
390 : too. STACK is used to track the recursive creation process. */
391 : static function_instance *
392 : read_function_instance (function_instance_stack *stack, bool toplevel = true);
393 :
394 : /* Recursively deallocate all callsites (nested function_instances). */
395 : ~function_instance ();
396 :
397 : /* Accessors. */
398 0 : int symbol_name () const { return descriptor_.symbol_name (); }
399 0 : int file_name () const { return descriptor_.file_name (); }
400 0 : void set_symbol_name (int index) { descriptor_.set_symbol_name (index); }
401 0 : function_instance_descriptor get_descriptor () const { return descriptor_; }
402 :
403 : gcov_type
404 0 : total_count () const
405 : {
406 0 : return total_count_;
407 : }
408 :
409 : /* Return head count or -1 if unknown. */
410 : gcov_type
411 0 : head_count () const
412 : {
413 0 : return head_count_;
414 : }
415 :
416 : gcov_type
417 0 : timestamp () const
418 : {
419 0 : return timestamp_;
420 : }
421 :
422 0 : void set_timestamp (gcov_type timestamp) { timestamp_ = timestamp; }
423 :
424 : /* Propagate timestamp from top-level function_instance to
425 : inlined instances. */
426 : void prop_timestamp ();
427 :
428 : /* Traverse callsites of the current function_instance to find one at the
429 : location of LINENO and callee name represented in DECL.
430 : LOCATION should match LINENO and is used to output diagnostics. */
431 : function_instance *get_function_instance_by_decl (unsigned lineno,
432 : tree decl,
433 : location_t location) const;
434 :
435 : /* Merge profile of clones. Note that cloning hasn't been performed when
436 : we annotate the CFG (at this stage). */
437 : void merge (function_instance *other,
438 : vec <function_instance *> &new_functions);
439 :
440 : /* Look for inline instances that was not realized and
441 : remove them while possibly merging them to offline variants. */
442 : void offline_if_not_realized (vec <function_instance *> &new_functions);
443 :
444 : /* Match function instance with gimple body. */
445 : bool match (cgraph_node *node, vec <function_instance *> &new_functions,
446 : name_index_map &to_symbol_name);
447 :
448 : /* Offline all inlined functions with name in SEEN.
449 : If new toplevel functions are created, add them to NEW_FUNCTIONS. */
450 : void offline_if_in_set (name_index_set &seen,
451 : vec <function_instance *> &new_functions);
452 :
453 : /* Walk inlined functions and if their name is not in SEEN
454 : remove it. */
455 :
456 : void remove_external_functions (name_index_set &seen,
457 : name_index_map &to_symbol_name,
458 : vec <function_instance *> &new_functions);
459 :
460 : /* Store the profile info for LOC in INFO. Return TRUE if profile info
461 : is found. */
462 : bool get_count_info (location_t loc, count_info *info) const;
463 :
464 : /* Read the inlined indirect call target profile for STMT in FN and store it
465 : in MAP, return the total count for all inlined indirect calls. */
466 : gcov_type find_icall_target_map (tree fn, gcall *stmt,
467 : icall_target_map *map) const;
468 :
469 : /* Remove inlined indirect call target profile for STMT in FN. */
470 : void remove_icall_target (tree fn, gcall *stmt);
471 :
472 : /* Mark LOC as annotated. */
473 : void mark_annotated (location_t loc);
474 :
475 : void dump (FILE *f, int indent = 0, bool nested = false) const;
476 :
477 : void dump_inline_stack (FILE *f) const;
478 :
479 : DEBUG_FUNCTION void debug () const;
480 :
481 : /* Mark function as removed from indir target list. */
482 : void
483 0 : remove_icall_target ()
484 : {
485 0 : removed_icall_target_ = true;
486 : }
487 :
488 : /* Return true if function is removed from indir target list. */
489 : bool
490 0 : removed_icall_target ()
491 : {
492 0 : return removed_icall_target_;
493 : }
494 :
495 : /* Set inlined_to pointer. */
496 : void
497 0 : set_inlined_to (function_instance *inlined_to)
498 : {
499 0 : gcc_checking_assert (inlined_to != this);
500 0 : inlined_to_ = inlined_to;
501 0 : }
502 :
503 : /* Return pointer to the function instance this function is inlined
504 : to or NULL if it is outer instance. */
505 : function_instance *
506 0 : inlined_to () const
507 : {
508 0 : return inlined_to_;
509 : }
510 :
511 : /* Mark function as realized. */
512 : void
513 0 : set_realized ()
514 : {
515 0 : realized_ = true;
516 0 : }
517 :
518 : /* Return true if function is realized. */
519 : bool
520 0 : realized_p ()
521 : {
522 0 : return realized_;
523 : }
524 :
525 : /* Mark function as in_worklist. */
526 : void
527 0 : set_in_worklist ()
528 : {
529 0 : gcc_checking_assert (!inlined_to_ && !in_worklist_p ());
530 0 : in_worklist_ = true;
531 0 : }
532 :
533 : void
534 0 : clear_in_worklist ()
535 : {
536 0 : gcc_checking_assert (!inlined_to_ && in_worklist_p ());
537 0 : in_worklist_ = false;
538 0 : }
539 :
540 :
541 : /* Return true if function is in_worklist. */
542 : bool
543 0 : in_worklist_p ()
544 : {
545 0 : return in_worklist_;
546 : }
547 :
548 : /* Return corresponding cgraph node. */
549 : cgraph_node *get_cgraph_node ();
550 :
551 : void
552 0 : set_location (location_t l)
553 : {
554 0 : gcc_checking_assert (location_ == UNKNOWN_LOCATION);
555 0 : location_= l;
556 0 : }
557 :
558 : location_t
559 0 : get_location ()
560 : {
561 0 : return location_;
562 : }
563 :
564 : void
565 0 : set_call_location (location_t l)
566 : {
567 0 : gcc_checking_assert (call_location_ == UNKNOWN_LOCATION
568 : && l != UNKNOWN_LOCATION);
569 0 : call_location_= l;
570 0 : }
571 :
572 : location_t
573 0 : get_call_location ()
574 : {
575 0 : return call_location_;
576 : }
577 :
578 : /* Lookup count and warn about duplicates. */
579 : count_info *lookup_count (location_t loc, inline_stack &stack,
580 : cgraph_node *node);
581 : private:
582 : /* Callsite, represented as (decl_lineno, callee_function_name_index). */
583 : typedef std::pair<unsigned, unsigned> callsite;
584 :
585 : /* Map from callsite to callee function_instance. */
586 : typedef std::map<callsite, function_instance *> callsite_map;
587 :
588 0 : function_instance (unsigned symbol_name, unsigned file_name,
589 : gcov_type head_count)
590 0 : : descriptor_ (file_name, symbol_name), total_count_ (0),
591 0 : head_count_ (head_count), timestamp_ (0),
592 0 : removed_icall_target_ (false), realized_ (false), in_worklist_ (false),
593 0 : inlined_to_ (NULL), location_ (UNKNOWN_LOCATION),
594 0 : call_location_ (UNKNOWN_LOCATION)
595 : {
596 : }
597 :
598 : /* Map from source location (decl_lineno) to profile (count_info). */
599 : typedef std::map<unsigned, count_info> position_count_map;
600 :
601 : /* The indices into the string table identifying the function_instance. */
602 : function_instance_descriptor descriptor_;
603 :
604 : /* Total sample count. */
605 : gcov_type total_count_;
606 :
607 : /* Entry BB's sample count. */
608 : gcov_type head_count_;
609 :
610 : /* perf timestamp associated with first execution of function, which is
611 : used to compute node->tp_first_run. */
612 : gcov_type timestamp_;
613 :
614 : /* Map from callsite location to callee function_instance. */
615 : callsite_map callsites;
616 :
617 : /* Map from source location to count_info. */
618 : position_count_map pos_counts;
619 :
620 : /* True if function was removed from indir target list. */
621 : bool removed_icall_target_;
622 :
623 : /* True if function exists in IL. I.e. for toplevel instance we
624 : have corresponding symbol and for inline instance we inlined
625 : to it. */
626 : bool realized_;
627 :
628 : /* True if function is in worklist for merging/offlining. */
629 : bool in_worklist_;
630 :
631 : /* Pointer to outer function instance or NULL if this
632 : is a toplevel one. */
633 : function_instance *inlined_to_;
634 :
635 : /* Location of function and its call (in case it is inlined). */
636 : location_t location_, call_location_;
637 :
638 : /* Turn inline instance to offline. */
639 : static bool offline (function_instance *fn,
640 : vec <function_instance *> &new_functions);
641 :
642 : /* Helper routine for prop_timestamp. */
643 : void prop_timestamp_1 (gcov_type timestamp);
644 : };
645 :
646 : /* Profile for all functions. */
647 : class autofdo_source_profile
648 : {
649 : public:
650 : static autofdo_source_profile *
651 0 : create ()
652 : {
653 0 : autofdo_source_profile *map = new autofdo_source_profile ();
654 :
655 0 : if (map->read ())
656 : return map;
657 0 : delete map;
658 0 : return NULL;
659 : }
660 :
661 : ~autofdo_source_profile ();
662 :
663 : /* For a given DECL, returns the top-level function_instance. */
664 : function_instance *get_function_instance_by_decl (tree decl, const char * = NULL) const;
665 :
666 : /* For a given DESCRIPTOR, return the matching instance if found. */
667 : function_instance *
668 : get_function_instance_by_descriptor (function_instance_descriptor) const;
669 :
670 : void add_function_instance (function_instance *);
671 :
672 : /* Find count_info for a given gimple STMT. If found, store the count_info
673 : in INFO and return true; otherwise return false.
674 : NODE can be used to specify particular inline clone. */
675 : bool get_count_info (gimple *stmt, count_info *info,
676 : cgraph_node *node = NULL) const;
677 :
678 : /* Find count_info for a given gimple location GIMPLE_LOC. If found,
679 : store the count_info in INFO and return true; otherwise return false.
680 : NODE can be used to specify particular inline clone. */
681 : bool get_count_info (location_t gimple_loc, count_info *info,
682 : cgraph_node *node = NULL) const;
683 :
684 : /* Find total count of the callee of EDGE. */
685 : gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;
686 :
687 : /* Update value profile INFO for STMT within NODE from the inlined indirect
688 : callsite. Return true if INFO is updated. */
689 : bool update_inlined_ind_target (gcall *stmt, count_info *info,
690 : cgraph_node *node);
691 :
692 : void remove_icall_target (cgraph_edge *e);
693 :
694 : /* Offline all functions not defined in the current translation unit. */
695 : void offline_external_functions ();
696 :
697 : void offline_unrealized_inlines ();
698 :
699 : private:
700 : /* Map from pair of function_instance filename and symbol name (in
701 : string_table) to function_instance. */
702 : typedef std::map<function_instance_descriptor, function_instance *>
703 : name_function_instance_map;
704 :
705 0 : autofdo_source_profile () {}
706 :
707 : /* Read AutoFDO profile and returns TRUE on success. */
708 : bool read ();
709 :
710 : /* Return the function_instance in the profile that correspond to the
711 : inline STACK. */
712 : function_instance *
713 : get_function_instance_by_inline_stack (const inline_stack &stack) const;
714 :
715 : /* Find the matching function instance which has DESCRIPTOR as its
716 : descriptor. If not found, also try checking if an instance exists with the
717 : same name which has no associated filename. */
718 : name_function_instance_map::const_iterator find_iter_for_function_instance (
719 : function_instance_descriptor descriptor) const;
720 :
721 : /* Similar to the above, but return a pointer to the instance instead of an
722 : iterator. */
723 : function_instance *
724 : find_function_instance (function_instance_descriptor descriptor) const;
725 :
726 : /* Remove a function instance from the map. Returns true if the entry was
727 : actually deleted. */
728 : bool remove_function_instance (function_instance *inst);
729 :
730 : name_function_instance_map map_;
731 :
732 : auto_vec <function_instance *> duplicate_functions_;
733 : };
734 :
735 : /* Store the summary information from the GCOV file. */
736 : static summary_info *afdo_summary_info;
737 :
738 : /* Store the strings read from the profile data file. */
739 : static string_table *afdo_string_table;
740 :
741 : /* Store the AutoFDO source profile. */
742 : static autofdo_source_profile *afdo_source_profile;
743 :
744 : /* gcov_summary structure to store the profile_info. */
745 : static gcov_summary *afdo_profile_info;
746 :
747 : /* Map from timestamp -> <name, tp_first_run>.
748 :
749 : The purpose of this map is to map 64-bit timestamp values to (1..N) sorted
750 : by ascending order of timestamps and assign that to node->tp_first_run,
751 : since we don't need the full 64-bit range. */
752 : static std::map<gcov_type, int> timestamp_info_map;
753 :
754 : /* Scaling factor for afdo data. Compared to normal profile
755 : AFDO profile counts are much lower, depending on sampling
756 : frequency. We scale data up to reduce effects of roundoff
757 : errors. */
758 :
759 : static gcov_type afdo_count_scale = 1;
760 :
761 : /* Helper functions. */
762 :
763 : /* Return the original name of NAME: strip the suffix that starts
764 : with '.' for names that are generated after auto-profile pass.
765 : This is to match profiled names with the names in the IR at this stage.
766 : Note that we only have to strip suffix and not in the middle.
767 : Caller is responsible for freeing RET. */
768 :
769 : static char *
770 0 : get_original_name (const char *name, bool alloc = true)
771 : {
772 0 : char *ret = alloc ? xstrdup (name) : const_cast<char *> (name);
773 0 : char *last_dot = strrchr (ret, '.');
774 0 : if (last_dot == NULL)
775 : return ret;
776 0 : bool only_digits = true;
777 : char *ptr = last_dot;
778 0 : while (*(++ptr) != 0)
779 0 : if (*ptr < '0' || *ptr > '9')
780 : {
781 : only_digits = false;
782 : break;
783 : }
784 0 : if (only_digits)
785 0 : *last_dot = 0;
786 0 : char *next_dot = strrchr (ret, '.');
787 : /* if nested function such as foo.0, return foo.0 */
788 0 : if (next_dot == NULL)
789 : {
790 0 : *last_dot = '.';
791 0 : return ret;
792 : }
793 : /* Suffixes of clones that compiler generates after auto-profile. */
794 0 : const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"};
795 0 : for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i)
796 : {
797 0 : int len = strlen (suffixes[i]);
798 0 : if (len == last_dot - next_dot - 1
799 0 : && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
800 : {
801 0 : *next_dot = 0;
802 0 : return get_original_name (ret, false);
803 : }
804 : }
805 : /* Otherwise, it is for clones such as .omp_fn.N that was done before
806 : auto-profile and should be kept as it is. */
807 0 : *last_dot = '.';
808 0 : return ret;
809 : }
810 :
811 : /* Return the combined location, which is a 32bit integer in which
812 : higher 16 bits stores the line offset of LOC to the start lineno
813 : of DECL, The lower 16 bits stores the discriminator. */
814 :
815 : static unsigned
816 0 : get_combined_location (location_t loc, tree decl)
817 : {
818 0 : bool warned = false;
819 : /* TODO: allow more bits for line and less bits for discriminator. */
820 0 : if ((LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) >= (1<<15)
821 0 : || (LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) <= -(1<<15))
822 0 : warned = warning_at (loc, OPT_Wauto_profile,
823 : "auto-profile cannot encode offset %i "
824 : "that exceeds 16 bytes",
825 0 : LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl));
826 0 : if (warned)
827 0 : inform (DECL_SOURCE_LOCATION (decl), "location offset is related to");
828 0 : if ((unsigned)get_discriminator_from_loc (loc) >= (1u << 16))
829 0 : warning_at (loc, OPT_Wauto_profile,
830 : "auto-profile cannot encode discriminators "
831 : "that exceeds 16 bytes");
832 0 : return ((unsigned)(LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16)
833 0 : | get_discriminator_from_loc (loc);
834 : }
835 :
836 : /* Return the function decl of a given lexical BLOCK. */
837 :
838 : static tree
839 0 : get_function_decl_from_block (tree block)
840 : {
841 0 : if (!inlined_function_outer_scope_p (block))
842 : return NULL_TREE;
843 :
844 0 : return BLOCK_ABSTRACT_ORIGIN (block);
845 : }
846 :
847 : /* Dump LOC to F. */
848 :
849 : static void
850 0 : dump_afdo_loc (FILE *f, unsigned loc)
851 : {
852 0 : if (loc & 65535)
853 0 : fprintf (f, "%i.%i", loc >> 16, loc & 65535);
854 : else
855 0 : fprintf (f, "%i", loc >> 16);
856 0 : }
857 :
858 : /* Return assembler name as in symbol table and DW_AT_linkage_name. */
859 :
860 : static const char *
861 0 : raw_symbol_name (const char *asmname)
862 : {
863 : /* If we start supporting user_label_prefixes, add_linkage_attr will also
864 : need to be fixed. */
865 0 : if (strlen (user_label_prefix))
866 0 : sorry ("auto-profile is not supported for targets with user label prefix");
867 0 : return asmname + (asmname[0] == '*');
868 : }
869 :
870 : /* Convenience wrapper that looks up assembler name. */
871 :
872 : static const char *
873 0 : raw_symbol_name (tree decl)
874 : {
875 0 : return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
876 : }
877 :
878 : /* Dump STACK to F. */
879 :
880 : static void
881 0 : dump_inline_stack (FILE *f, inline_stack *stack)
882 : {
883 0 : bool first = true;
884 0 : for (decl_lineno &p : *stack)
885 : {
886 0 : fprintf (f, "%s%s:",
887 : first ? "" : "; ",
888 : raw_symbol_name (p.decl));
889 0 : dump_afdo_loc (f, p.afdo_loc);
890 0 : first = false;
891 : }
892 0 : fprintf (f, "\n");
893 0 : }
894 :
895 : /* Store inline stack for STMT in STACK. */
896 :
897 : static void
898 0 : get_inline_stack (location_t locus, inline_stack *stack,
899 : tree fn = current_function_decl)
900 : {
901 0 : if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
902 : return;
903 :
904 0 : tree block = LOCATION_BLOCK (locus);
905 0 : if (block && TREE_CODE (block) == BLOCK)
906 : {
907 0 : for (block = BLOCK_SUPERCONTEXT (block);
908 0 : block && (TREE_CODE (block) == BLOCK);
909 0 : block = BLOCK_SUPERCONTEXT (block))
910 : {
911 0 : location_t tmp_locus = BLOCK_SOURCE_LOCATION (block);
912 0 : if (LOCATION_LOCUS (tmp_locus) == UNKNOWN_LOCATION)
913 0 : continue;
914 :
915 0 : tree decl = get_function_decl_from_block (block);
916 0 : stack->safe_push (
917 0 : {decl, get_combined_location (locus, decl), locus});
918 0 : locus = tmp_locus;
919 : }
920 : }
921 0 : stack->safe_push ({fn, get_combined_location (locus, fn), locus});
922 : }
923 :
924 : /* Same as get_inline_stack for a given node which may be
925 : an inline clone. If NODE is NULL, assume current_function_decl. */
926 : static void
927 0 : get_inline_stack_in_node (location_t locus, inline_stack *stack,
928 : cgraph_node *node)
929 : {
930 0 : if (!node)
931 0 : return get_inline_stack (locus, stack);
932 0 : do
933 : {
934 0 : get_inline_stack (locus, stack, node->decl);
935 : /* If caller is inlined, continue building stack. */
936 0 : if (!node->inlined_to)
937 : node = NULL;
938 : else
939 : {
940 0 : locus = gimple_location (node->callers->call_stmt);
941 0 : node = node->callers->caller;
942 : }
943 : }
944 0 : while (node);
945 : }
946 :
947 : /* Return combined location of LOCUS within BLOCK that is in
948 : function FN.
949 :
950 : This is a 32bit integer in which higher 16 bits stores the line offset of
951 : LOC to the start lineno of DECL, The lower 16 bits stores the
952 : discriminator. */
953 :
954 : static unsigned
955 0 : get_relative_location_for_locus (tree fn, tree block, location_t locus)
956 : {
957 0 : if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
958 : return -1;
959 :
960 0 : for (; block && (TREE_CODE (block) == BLOCK);
961 0 : block = BLOCK_SUPERCONTEXT (block))
962 0 : if (inlined_function_outer_scope_p (block))
963 0 : return get_combined_location (locus,
964 0 : get_function_decl_from_block (block));
965 0 : return get_combined_location (locus, fn);
966 : }
967 :
968 : /* Return combined location of STMT in function FN. */
969 :
970 : static unsigned
971 0 : get_relative_location_for_stmt (tree fn, gimple *stmt)
972 : {
973 0 : return get_relative_location_for_locus
974 0 : (fn, LOCATION_BLOCK (gimple_location (stmt)),
975 0 : gimple_location (stmt));
976 : }
977 :
978 : /* Return either the basename or the realpath for a given path based on
979 : PARAM_PROFILE_FUNC_INTERNAL_ID. */
980 :
981 : static const char *
982 0 : get_normalized_path (const char *path, bool from_gcov = false)
983 : {
984 0 : if (param_profile_func_internal_id == 1)
985 : /* The GCOV will already contain the entire path. It doesn't need to be
986 : normalized with lrealpath (). */
987 0 : return from_gcov ? path : lrealpath (path);
988 0 : return lbasename (path);
989 : }
990 :
991 : /* Member functions for summary_info. */
992 :
993 : bool
994 0 : summary_info::read ()
995 : {
996 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_SUMMARY)
997 : return false;
998 :
999 0 : total_count = gcov_read_counter ();
1000 0 : max_count = gcov_read_counter ();
1001 0 : max_function_count = gcov_read_counter ();
1002 0 : num_counts = gcov_read_counter ();
1003 0 : num_functions = gcov_read_counter ();
1004 0 : uint64_t num_detailed_summaries = gcov_read_counter ();
1005 0 : gcc_checking_assert (num_detailed_summaries == NUM_PERCENTILES);
1006 0 : for (uint64_t i = 0; i < num_detailed_summaries; i++)
1007 : {
1008 0 : detailed_summaries[i].cutoff = gcov_read_unsigned ();
1009 0 : detailed_summaries[i].min_count = gcov_read_counter ();
1010 0 : detailed_summaries[i].num_counts = gcov_read_counter ();
1011 : }
1012 :
1013 0 : return !gcov_is_error ();
1014 : }
1015 :
1016 : /* Get the minimum count required for percentile CUTOFF. */
1017 :
1018 : uint64_t
1019 0 : summary_info::get_threshold_count (uint32_t cutoff)
1020 : {
1021 : /* The cutoffs stored in the GCOV are fractions multiplied by 1,000,000. */
1022 0 : gcc_checking_assert (cutoff <= 1'000'000);
1023 0 : unsigned idx = 0;
1024 : /* Find the first cutoff at least as high as CUTOFF. */
1025 0 : for (; idx < NUM_PERCENTILES; idx++)
1026 0 : if (detailed_summaries[idx].cutoff >= cutoff)
1027 : break;
1028 0 : idx = std::min (NUM_PERCENTILES - 1, idx);
1029 0 : return detailed_summaries[idx].min_count;
1030 : }
1031 :
1032 : /* Member functions for string_table. */
1033 :
1034 : /* Deconstructor. */
1035 :
1036 0 : string_table::~string_table ()
1037 : {
1038 0 : for (unsigned i = 0; i < symbol_names_.length (); i++)
1039 0 : free (const_cast<char *> (symbol_names_[i]));
1040 0 : for (unsigned i = 0; i < filenames_.length (); i++)
1041 0 : free (const_cast<char *> (filenames_[i]));
1042 0 : for (auto it = original_names_map_.begin (); it != original_names_map_.end ();
1043 0 : it++)
1044 0 : free (it->second);
1045 0 : }
1046 :
1047 :
1048 : /* Return the index of a given function NAME. Return -1 if NAME is not
1049 : found in string table. */
1050 :
1051 : int
1052 0 : string_table::get_index (const char *name) const
1053 : {
1054 0 : if (name == NULL)
1055 : return -1;
1056 0 : string_index_map::const_iterator iter = symbol_name_map_.find (name);
1057 0 : if (iter == symbol_name_map_.end ())
1058 : return -1;
1059 :
1060 0 : return iter->second;
1061 : }
1062 :
1063 : /* Return the index of a given function DECL. Return -1 if DECL is not
1064 : found in string table. */
1065 :
1066 : int
1067 0 : string_table::get_index_by_decl (tree decl) const
1068 : {
1069 0 : const char *name = raw_symbol_name (decl);
1070 0 : int ret = get_index (name);
1071 0 : if (ret != -1)
1072 : return ret;
1073 0 : if (DECL_FROM_INLINE (decl))
1074 0 : return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl));
1075 :
1076 : return -1;
1077 : }
1078 :
1079 : /* Return the function name of a given INDEX. */
1080 :
1081 : const char *
1082 0 : string_table::get_symbol_name (int index) const
1083 : {
1084 0 : if (index <= 0 || index >= (int) symbol_names_.length ())
1085 0 : fatal_error (UNKNOWN_LOCATION,
1086 : "auto-profile contains invalid symbol name index %d", index);
1087 :
1088 0 : return symbol_names_[index];
1089 : }
1090 :
1091 : /* For a given index, returns the string. */
1092 :
1093 : const char *
1094 0 : string_table::get_filename (int index) const
1095 : {
1096 : /* There may not be any file name for some functions, ignore them. */
1097 0 : if (index == string_table::unknown_filename)
1098 : return "<unknown>";
1099 :
1100 0 : if (index < 0 || index >= (int) filenames_.length ())
1101 0 : fatal_error (UNKNOWN_LOCATION,
1102 : "auto-profile contains invalid filename index %d", index);
1103 :
1104 0 : return filenames_[index];
1105 : }
1106 :
1107 : /* For a given symbol name index, returns the filename index. */
1108 :
1109 : int
1110 0 : string_table::get_filename_by_symbol (int index) const
1111 : {
1112 0 : return get_filename_by_symbol (get_symbol_name (index));
1113 : }
1114 :
1115 : /* For a given function name, returns the filename index. */
1116 :
1117 : int
1118 0 : string_table::get_filename_by_symbol (const char *name) const
1119 : {
1120 0 : auto it = symbol_to_filename_map_.find (name);
1121 0 : if (it != symbol_to_filename_map_.end () && it->second < filenames_.length ())
1122 0 : return it->second;
1123 : return string_table::unknown_filename;
1124 : }
1125 :
1126 : /* For a given filename, returns the index. */
1127 :
1128 : int
1129 0 : string_table::get_filename_index (const char *name) const
1130 : {
1131 0 : auto iter = filename_map_.find (name);
1132 0 : return iter == filename_map_.end () ? string_table::unknown_filename
1133 0 : : iter->second;
1134 : }
1135 :
1136 : /* Get the original name and file name index for a node. This will return the
1137 : name from the current TU if there are multiple symbols that map to
1138 : NAME. */
1139 :
1140 : std::pair<const char *, int>
1141 0 : string_table::get_original_name (const char *name) const
1142 : {
1143 : /* Check if the un-prefixed name differs from the actual name. */
1144 0 : auto stripped = original_names_map_.find (name);
1145 :
1146 : /* The original name for the symbol is its name, i.e. there are no
1147 : suffixes. */
1148 0 : if (stripped == original_names_map_.end ())
1149 0 : return {name, get_filename_by_symbol (name)};
1150 :
1151 : /* Figure out if a clash exists. */
1152 0 : auto clash = clashing_names_map_.find (stripped->second);
1153 0 : gcc_checking_assert (clash != clashing_names_map_.end ());
1154 :
1155 : /* Try to find a function from the current TU. */
1156 0 : gcc_checking_assert (clash->second.length () >= 1);
1157 0 : symtab_node *n
1158 0 : = cgraph_node::get_for_asmname (get_identifier (stripped->second));
1159 0 : if (n && is_a<cgraph_node *> (n))
1160 0 : for (cgraph_node *cn = dyn_cast<cgraph_node *> (n); cn;)
1161 : {
1162 : /* Check if there is a symbol in the current TU that has the same name
1163 : as in the GCOV. */
1164 0 : for (auto name : clash->second)
1165 : {
1166 0 : int filename_idx = get_filename_by_symbol (name);
1167 0 : if (cn->definition && cn->has_gimple_body_p ()
1168 0 : && !strcmp (get_normalized_path (DECL_SOURCE_FILE (cn->decl)),
1169 : get_filename (filename_idx)))
1170 0 : return {stripped->second, filename_idx};
1171 : }
1172 0 : cn = dyn_cast<cgraph_node *> (cn->next_sharing_asm_name);
1173 : }
1174 :
1175 : /* No match found. Just stick to the current symbol and return the stripped
1176 : name. */
1177 0 : return {stripped->second, get_filename_by_symbol (name)};
1178 : }
1179 :
1180 : /* Add new symbol name STRING (with an associated file name FILENAME_IDX) and
1181 : return its index. */
1182 :
1183 : int
1184 0 : string_table::add_symbol_name (const char *string, int filename_idx)
1185 : {
1186 0 : gcc_checking_assert (
1187 : filename_idx == string_table::unknown_filename
1188 : || (filename_idx >= 0 && filename_idx < (int) filenames_.length ()));
1189 0 : symbol_names_.safe_push (string);
1190 0 : symbol_name_map_[symbol_names_.last ()] = symbol_names_.length () - 1;
1191 0 : symbol_to_filename_map_[symbol_names_.last ()] = filename_idx;
1192 0 : return symbol_names_.length () - 1;
1193 : }
1194 :
1195 : /* Add new filename and return its index (returning the same if it already
1196 : exists). */
1197 :
1198 : int
1199 0 : string_table::add_filename (const char *name)
1200 : {
1201 0 : auto it = filename_map_.find (name);
1202 0 : if (it != filename_map_.end ())
1203 0 : return it->second;
1204 0 : filenames_.safe_push (xstrdup (name));
1205 0 : return filenames_.length () - 1;
1206 : }
1207 :
1208 : /* Read the string table. Return TRUE if reading is successful. */
1209 :
1210 : bool
1211 0 : string_table::read ()
1212 : {
1213 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES)
1214 : return false;
1215 : /* Skip the length of the section. */
1216 0 : gcov_read_unsigned ();
1217 : /* Read in the file name table. */
1218 0 : unsigned file_num = gcov_read_unsigned ();
1219 0 : filenames_.reserve (file_num);
1220 0 : for (unsigned i = 0; i < file_num; i++)
1221 : {
1222 0 : const char *filename = gcov_read_string ();
1223 0 : filenames_.quick_push (xstrdup (get_normalized_path (filename, true)));
1224 0 : filename_map_[filenames_.last ()] = i;
1225 0 : free (const_cast<char *> (filename));
1226 0 : if (gcov_is_error ())
1227 : return false;
1228 : }
1229 : /* Read in the function name -> file name table. */
1230 0 : unsigned string_num = gcov_read_unsigned ();
1231 0 : symbol_names_.reserve (string_num);
1232 0 : for (unsigned i = 0; i < string_num; i++)
1233 : {
1234 0 : symbol_names_.quick_push (const_cast<char *> (gcov_read_string ()));
1235 0 : symbol_name_map_[symbol_names_.last ()] = i;
1236 0 : unsigned filename_idx = gcov_read_unsigned ();
1237 0 : symbol_to_filename_map_[symbol_names_.last ()] = filename_idx;
1238 0 : char *original = const_cast<char *> (
1239 0 : autofdo::get_original_name (symbol_names_.last ()));
1240 0 : if (strcmp (original, symbol_names_.last ()))
1241 : {
1242 : /* Take ownership of ORIGINAL. */
1243 0 : original_names_map_[symbol_names_.last ()] = original;
1244 0 : clashing_names_map_[original].safe_push (i);
1245 : /* It is possible that a public symbol with the stripped name exists.
1246 : If it does exist, add it as well. */
1247 0 : auto publik = symbol_name_map_.find (original);
1248 0 : if (publik != symbol_name_map_.end ()
1249 0 : && clashing_names_map_.find (publik->first)
1250 0 : == clashing_names_map_.end ())
1251 0 : clashing_names_map_[publik->first].safe_push (publik->second);
1252 : }
1253 : else
1254 : /* There are no suffixes to remove. */
1255 0 : free (original);
1256 :
1257 0 : if (gcov_is_error ())
1258 0 : return false;
1259 : }
1260 0 : return true;
1261 : }
1262 :
1263 : /* Return cgraph node corresponding to given NAME_INDEX,
1264 : NULL if unavailable. */
1265 : cgraph_node *
1266 0 : string_table::get_cgraph_node (int name_index)
1267 : {
1268 0 : const char *sname = get_symbol_name (name_index);
1269 :
1270 0 : symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname));
1271 0 : for (;n; n = n->next_sharing_asm_name)
1272 0 : if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
1273 0 : if (cn->definition && cn->has_gimple_body_p ())
1274 : return cn;
1275 : return NULL;
1276 : }
1277 :
1278 : /* Return corresponding cgraph node. */
1279 :
1280 : cgraph_node *
1281 0 : function_instance::get_cgraph_node ()
1282 : {
1283 0 : return afdo_string_table->get_cgraph_node (symbol_name ());
1284 : }
1285 :
1286 : /* Member functions for function_instance. */
1287 :
1288 0 : function_instance::~function_instance ()
1289 : {
1290 0 : gcc_assert (!in_worklist_p ());
1291 0 : for (callsite_map::iterator iter = callsites.begin ();
1292 0 : iter != callsites.end (); ++iter)
1293 0 : delete iter->second;
1294 0 : }
1295 :
1296 : /* Propagate timestamp TS of function_instance to inlined instances if it's
1297 : not already set. */
1298 :
1299 : void
1300 0 : function_instance::prop_timestamp_1 (gcov_type ts)
1301 : {
1302 0 : if (!timestamp () && total_count () > 0)
1303 0 : set_timestamp (ts);
1304 0 : for (auto it = callsites.begin (); it != callsites.end (); ++it)
1305 0 : it->second->prop_timestamp_1 (ts);
1306 0 : }
1307 :
1308 : void
1309 0 : function_instance::prop_timestamp (void)
1310 : {
1311 0 : prop_timestamp_1 (timestamp ());
1312 0 : }
1313 :
1314 : /* Traverse callsites of the current function_instance to find one at the
1315 : location of LINENO and callee name represented in DECL. */
1316 :
1317 : function_instance *
1318 0 : function_instance::get_function_instance_by_decl (unsigned lineno,
1319 : tree decl,
1320 : location_t location) const
1321 : {
1322 0 : int func_name_idx = afdo_string_table->get_index_by_decl (decl);
1323 0 : if (func_name_idx != -1)
1324 : {
1325 0 : callsite_map::const_iterator ret
1326 0 : = callsites.find (std::make_pair (lineno, func_name_idx));
1327 0 : if (ret != callsites.end ())
1328 0 : return ret->second;
1329 : }
1330 0 : if (DECL_FROM_INLINE (decl))
1331 : {
1332 0 : function_instance
1333 0 : *ret = get_function_instance_by_decl (lineno,
1334 0 : DECL_ABSTRACT_ORIGIN (decl),
1335 : location);
1336 0 : return ret;
1337 : }
1338 0 : if (dump_enabled_p ())
1339 : {
1340 0 : for (auto const &iter : callsites)
1341 0 : if (iter.first.first == lineno)
1342 0 : dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
1343 0 : dump_user_location_t::from_location_t (location),
1344 : "auto-profile has mismatched function name %s"
1345 : " instead of %s at loc %i:%i",
1346 : afdo_string_table->get_symbol_name (
1347 0 : iter.first.second),
1348 : raw_symbol_name (decl), lineno >> 16,
1349 : lineno & 65535);
1350 : }
1351 :
1352 : return NULL;
1353 : }
1354 :
1355 : /* Merge profile of OTHER to THIS. Note that cloning hasn't been performed
1356 : when we annotate the CFG (at this stage). */
1357 :
1358 : void
1359 0 : function_instance::merge (function_instance *other,
1360 : vec <function_instance *> &new_functions)
1361 : {
1362 : /* Do not merge to itself and only merge functions of same name. */
1363 0 : gcc_checking_assert (other != this
1364 : && other->symbol_name () == symbol_name ());
1365 :
1366 0 : if (file_name () != other->file_name ())
1367 : return;
1368 :
1369 0 : total_count_ += other->total_count_;
1370 0 : if (other->total_count () && total_count () && other->head_count () == -1)
1371 0 : head_count_ = -1;
1372 0 : else if (head_count_ != -1)
1373 0 : head_count_ += other->head_count_;
1374 :
1375 : /* While merging timestamps, set the one that occurs earlier. */
1376 0 : if (other->timestamp () < timestamp ())
1377 0 : set_timestamp (other->timestamp ());
1378 :
1379 : bool changed = true;
1380 :
1381 0 : while (changed)
1382 : {
1383 0 : changed = false;
1384 : /* If both function instances agree on particular inlined function,
1385 : merge profiles. Otherwise offline the instance. */
1386 0 : for (callsite_map::const_iterator iter = other->callsites.begin ();
1387 0 : iter != other->callsites.end ();)
1388 0 : if (callsites.count (iter->first) == 0)
1389 : {
1390 0 : function_instance *f = iter->second;
1391 0 : if (dump_file)
1392 : {
1393 0 : fprintf (dump_file, " Mismatch in inlined functions;"
1394 : " offlining in merge source:");
1395 0 : f->dump_inline_stack (dump_file);
1396 0 : fprintf (dump_file, "\n");
1397 : }
1398 : /* We already merged outer part of the function accounting
1399 : the inlined call; compensate. */
1400 0 : for (function_instance *s = this; s; s = s->inlined_to ())
1401 : {
1402 0 : s->total_count_ -= f->total_count ();
1403 0 : gcc_checking_assert (s->total_count_ >= 0);
1404 : }
1405 0 : other->callsites.erase (iter);
1406 0 : function_instance::offline (f, new_functions);
1407 : /* Start from beginning as merging might have offlined
1408 : some functions in the case of recursive inlining. */
1409 0 : iter = other->callsites.begin ();
1410 : }
1411 : else
1412 0 : ++iter;
1413 0 : for (callsite_map::const_iterator iter = callsites.begin ();
1414 0 : iter != callsites.end ();)
1415 0 : if (other->callsites.count (iter->first) == 0)
1416 : {
1417 0 : function_instance *f = iter->second;
1418 0 : if (dump_file)
1419 : {
1420 0 : fprintf (dump_file, " Mismatch in inlined functions;"
1421 : " offlining in merge destination:");
1422 0 : f->dump_inline_stack (dump_file);
1423 0 : fprintf (dump_file, "\n");
1424 : }
1425 0 : callsites.erase (iter);
1426 0 : function_instance::offline (f, new_functions);
1427 0 : iter = callsites.begin ();
1428 0 : changed = true;
1429 : }
1430 : else
1431 0 : ++iter;
1432 : }
1433 0 : for (callsite_map::const_iterator iter = other->callsites.begin ();
1434 0 : iter != other->callsites.end (); ++iter)
1435 : {
1436 0 : if (dump_file)
1437 : {
1438 0 : fprintf (dump_file, " Merging profile for inlined function\n"
1439 : " from: ");
1440 0 : iter->second->dump_inline_stack (dump_file);
1441 0 : fprintf (dump_file, " total:%" PRIu64 "\n to : ",
1442 0 : (int64_t)iter->second->total_count ());
1443 0 : callsites[iter->first]->dump_inline_stack (dump_file);
1444 0 : fprintf (dump_file, " total:%" PRIu64 "\n",
1445 0 : (int64_t)callsites[iter->first]->total_count ());
1446 : }
1447 :
1448 0 : callsites[iter->first]->merge (iter->second, new_functions);
1449 : }
1450 :
1451 0 : for (position_count_map::const_iterator iter = other->pos_counts.begin ();
1452 0 : iter != other->pos_counts.end (); ++iter)
1453 0 : if (pos_counts.count (iter->first) == 0)
1454 0 : pos_counts[iter->first] = iter->second;
1455 : else
1456 : {
1457 0 : pos_counts[iter->first].count += iter->second.count;
1458 0 : for (icall_target_map::const_iterator titer
1459 0 : = iter->second.targets.begin ();
1460 0 : titer != iter->second.targets.end (); ++titer)
1461 0 : if (pos_counts[iter->first].targets.count (titer->first) == 0)
1462 0 : pos_counts[iter->first].targets[titer->first]
1463 0 : = titer->second;
1464 : else
1465 0 : pos_counts[iter->first].targets[titer->first]
1466 0 : += titer->second;
1467 : }
1468 : }
1469 :
1470 : /* Make inline function FN offline.
1471 : If toplevel function of same name already exists, then merge profiles.
1472 : Otherwise turn FN toplevel. Return true if new toplevel function
1473 : was introduced.
1474 : If new toplevel functions are created and NEW_FUNCTIONS != NULL,
1475 : add them to NEW_FUNCTIONS.
1476 :
1477 : TODO: When offlining indirect call we lose information about the
1478 : call target. It should be possible to add it into
1479 : targets histogram. */
1480 :
1481 : bool
1482 0 : function_instance::offline (function_instance *fn,
1483 : vec <function_instance *> &new_functions)
1484 : {
1485 0 : gcc_checking_assert (fn->inlined_to ());
1486 0 : for (function_instance *s = fn->inlined_to (); s; s = s->inlined_to ())
1487 : {
1488 0 : s->total_count_ -= fn->total_count ();
1489 0 : gcc_checking_assert (s->total_count_ >= 0);
1490 : }
1491 0 : function_instance *to
1492 0 : = afdo_source_profile->get_function_instance_by_descriptor (
1493 : fn->get_descriptor ());
1494 0 : fn->set_inlined_to (NULL);
1495 : /* If there is offline function of same name, we need to merge profile.
1496 : Delay this by adding function to a worklist so we do not run into
1497 : problem with recursive inlining. */
1498 0 : if (to)
1499 : {
1500 0 : if (fn->in_worklist_p ())
1501 : return false;
1502 0 : fn->set_in_worklist ();
1503 0 : new_functions.safe_push (fn);
1504 0 : if (dump_file)
1505 : {
1506 0 : fprintf (dump_file, " Recoding duplicate: ");
1507 0 : to->dump_inline_stack (dump_file);
1508 0 : fprintf (dump_file, "\n");
1509 : }
1510 0 : return true;
1511 : }
1512 0 : if (dump_file)
1513 : {
1514 0 : fprintf (dump_file, " Added as offline instance: ");
1515 0 : fn->dump_inline_stack (dump_file);
1516 0 : fprintf (dump_file, "\n");
1517 : }
1518 0 : if (fn->total_count ())
1519 0 : fn->head_count_ = -1;
1520 0 : afdo_source_profile->add_function_instance (fn);
1521 0 : fn->set_in_worklist ();
1522 0 : new_functions.safe_push (fn);
1523 0 : return true;
1524 : }
1525 :
1526 : /* Offline all inlined functions with name in SEEN.
1527 : If new toplevel functions are created, add them to NEW_FUNCTIONS. */
1528 :
1529 : void
1530 0 : function_instance::offline_if_in_set (name_index_set &seen,
1531 : vec <function_instance *> &new_functions)
1532 : {
1533 0 : for (callsite_map::const_iterator iter = callsites.begin ();
1534 0 : iter != callsites.end ();)
1535 0 : if (seen.contains (iter->first.second))
1536 : {
1537 0 : function_instance *f = iter->second;
1538 0 : if (dump_file)
1539 : {
1540 0 : fprintf (dump_file, "Offlining function inlined to other module: ");
1541 0 : f->dump_inline_stack (dump_file);
1542 0 : fprintf (dump_file, "\n");
1543 : }
1544 0 : iter = callsites.erase (iter);
1545 0 : function_instance::offline (f, new_functions);
1546 : /* Start from beginning as merging might have offlined
1547 : some functions in the case of recursive inlining. */
1548 0 : iter = callsites.begin ();
1549 : }
1550 : else
1551 : {
1552 0 : iter->second->offline_if_in_set (seen, new_functions);
1553 0 : ++iter;
1554 : }
1555 0 : }
1556 :
1557 : /* Try to check if inlined_fn can correspond to a call of function N.
1558 : Return non-zero if it corresponds and 2 if renaming was done. */
1559 :
1560 : static int
1561 0 : match_with_target (cgraph_node *n,
1562 : gimple *stmt,
1563 : function_instance *inlined_fn,
1564 : cgraph_node *orig_callee)
1565 : {
1566 0 : cgraph_node *callee = orig_callee->ultimate_alias_target ();
1567 0 : const char *symbol_name = raw_symbol_name (callee->decl);
1568 0 : const char *name
1569 0 : = afdo_string_table->get_symbol_name (inlined_fn->symbol_name ());
1570 0 : if (strcmp (name, symbol_name))
1571 : {
1572 0 : int i;
1573 0 : bool in_suffix = false;
1574 0 : for (i = 0; i; i++)
1575 : {
1576 : if (name[i] != symbol_name[i])
1577 : break;
1578 : if (name[i] == '.')
1579 : in_suffix = true;
1580 : }
1581 : /* Accept dwarf names and stripped suffixes. */
1582 0 : if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0),
1583 : afdo_string_table->get_symbol_name (
1584 : inlined_fn->symbol_name ()))
1585 0 : || (!name[i] && symbol_name[i] == '.') || in_suffix)
1586 : {
1587 0 : int index = afdo_string_table->get_index (symbol_name);
1588 0 : if (index == -1)
1589 0 : index = afdo_string_table->add_symbol_name (
1590 0 : xstrdup (symbol_name),
1591 : afdo_string_table->add_filename (
1592 0 : get_normalized_path (DECL_SOURCE_FILE (callee->decl))));
1593 0 : if (dump_file)
1594 0 : fprintf (dump_file,
1595 : " Renaming inlined call target %s to %s\n",
1596 : name, symbol_name);
1597 0 : inlined_fn->set_symbol_name (index);
1598 0 : return 2;
1599 : }
1600 : /* Only warn about declarations. It is possible that the function
1601 : is declared as alias in other module and we inlined cross-module. */
1602 0 : if (callee->definition
1603 0 : && warning (OPT_Wauto_profile,
1604 : "auto-profile of %q+F contains inlined "
1605 : "function with symbol name %s instead of symbol name %s",
1606 : n->decl, name, symbol_name))
1607 0 : inform (gimple_location (stmt), "corresponding call");
1608 0 : return 0;
1609 : }
1610 : return 1;
1611 : }
1612 :
1613 : static void
1614 0 : dump_stmt (gimple *stmt, count_info *info, function_instance *inlined_fn,
1615 : inline_stack &stack)
1616 : {
1617 0 : if (dump_file)
1618 : {
1619 0 : fprintf (dump_file, " ");
1620 0 : if (!stack.length ())
1621 0 : fprintf (dump_file, " ");
1622 : else
1623 : {
1624 0 : gcc_checking_assert (stack.length () == 1);
1625 0 : fprintf (dump_file, "%5i", stack[0].afdo_loc >> 16);
1626 0 : if (stack[0].afdo_loc & 65535)
1627 0 : fprintf (dump_file, ".%-5i", stack[0].afdo_loc & 65535);
1628 : else
1629 0 : fprintf (dump_file, " ");
1630 0 : if (info)
1631 0 : fprintf (dump_file, "%9" PRIu64 " ", (int64_t)info->count);
1632 0 : else if (inlined_fn)
1633 0 : fprintf (dump_file, " inlined ");
1634 : else
1635 0 : fprintf (dump_file, " no info ");
1636 : }
1637 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1638 : }
1639 0 : }
1640 :
1641 : /* Lookup count and warn about duplicates. */
1642 : count_info *
1643 0 : function_instance::lookup_count (location_t loc, inline_stack &stack,
1644 : cgraph_node *node)
1645 : {
1646 0 : gcc_checking_assert (stack.length () < 2);
1647 0 : if (stack.length ())
1648 : {
1649 0 : int c = pos_counts.count (stack[0].afdo_loc);
1650 0 : if (c > 1
1651 : && warning (OPT_Wauto_profile,
1652 : "duplicated count information"
1653 : " in auto-profile of %q+F"
1654 : " with relative location %i discriminator %i",
1655 : node->decl, stack[0].afdo_loc >> 16,
1656 : stack[0].afdo_loc & 65535))
1657 : inform (loc, "corresponding source location");
1658 0 : if (c)
1659 0 : return &pos_counts[stack[0].afdo_loc];
1660 : }
1661 : return NULL;
1662 : }
1663 :
1664 : /* Mark expr locations as used. */
1665 : void
1666 0 : mark_expr_locations (function_instance *f, tree t, cgraph_node *node,
1667 : hash_set<const count_info *> &counts)
1668 : {
1669 0 : inline_stack stack;
1670 0 : return;
1671 : if (!t)
1672 : return;
1673 : do
1674 : {
1675 : get_inline_stack_in_node (EXPR_LOCATION (t), &stack, node);
1676 : /* FIXME: EXPR_LOCATION does not always originate from current
1677 : function. */
1678 : if (stack.length () > 1)
1679 : break;
1680 : count_info *info = f->lookup_count (EXPR_LOCATION (t), stack, node);
1681 : if (info)
1682 : counts.add (info);
1683 : if (handled_component_p (t))
1684 : t = TREE_OPERAND (t, 0);
1685 : else
1686 : break;
1687 : }
1688 : while (true);
1689 0 : }
1690 :
1691 : /* Match function instance with gimple body.
1692 : Report mismatches, attempt to fix them if possible and remove data we will
1693 : not use.
1694 :
1695 : Set location and call_location so we can output diagnostics and know what
1696 : functions was already matched. */
1697 :
1698 : bool
1699 0 : function_instance::match (cgraph_node *node,
1700 : vec <function_instance *> &new_functions,
1701 : name_index_map &to_symbol_name)
1702 : {
1703 0 : if (get_location () != UNKNOWN_LOCATION)
1704 : return false;
1705 0 : set_location (DECL_SOURCE_LOCATION (node->decl));
1706 0 : if (dump_file)
1707 : {
1708 0 : fprintf (dump_file,
1709 : "\nMatching gimple function %s with auto profile: ",
1710 : node->dump_name ());
1711 0 : dump_inline_stack (dump_file);
1712 0 : fprintf (dump_file, "\n");
1713 : }
1714 0 : basic_block bb;
1715 : /* Sets used to track if entires in auto-profile are useful. */
1716 0 : hash_set<const count_info *> counts;
1717 0 : hash_set<const count_info *> targets;
1718 0 : hash_set<const function_instance *> functions;
1719 0 : hash_set<const function_instance *> functions_to_offline;
1720 :
1721 : /* We try to fill in lost disciminator if there is unique call
1722 : with given line number. This map is used to record them. */
1723 0 : hash_map<int_hash <int, -1, -2>,auto_vec <gcall *>> lineno_to_call;
1724 0 : bool lineno_to_call_computed = false;
1725 :
1726 0 : for (tree arg = DECL_ARGUMENTS (node->decl); arg; arg = DECL_CHAIN (arg))
1727 : {
1728 0 : inline_stack stack;
1729 :
1730 0 : get_inline_stack_in_node (DECL_SOURCE_LOCATION (arg), &stack, node);
1731 0 : count_info *info = lookup_count (DECL_SOURCE_LOCATION (arg), stack, node);
1732 0 : if (stack.length () && dump_file)
1733 : {
1734 0 : gcc_checking_assert (stack.length () == 1);
1735 0 : fprintf (dump_file, "%5i", stack[0].afdo_loc >> 16);
1736 0 : if (stack[0].afdo_loc & 65535)
1737 0 : fprintf (dump_file, " .%-5i arg", stack[0].afdo_loc & 65535);
1738 : else
1739 0 : fprintf (dump_file, " arg ");
1740 0 : print_generic_expr (dump_file, arg);
1741 0 : fprintf (dump_file, "\n");
1742 : }
1743 0 : if (info)
1744 0 : counts.add (info);
1745 0 : }
1746 0 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1747 : {
1748 0 : if (dump_file)
1749 0 : fprintf (dump_file, " basic block %i\n", bb->index);
1750 0 : for (gphi_iterator gpi = gsi_start_phis (bb);
1751 0 : !gsi_end_p (gpi);
1752 0 : gsi_next (&gpi))
1753 : {
1754 0 : gphi *phi = gpi.phi ();
1755 0 : inline_stack stack;
1756 :
1757 : /* We do not assign discriminators to PHI nodes.
1758 : In case we every start using them, we wil need to
1759 : update tree-cfg.cc::assign_discriminators. */
1760 0 : gcc_assert (gimple_location (phi) == UNKNOWN_LOCATION);
1761 0 : get_inline_stack_in_node (gimple_location (phi), &stack, node);
1762 0 : count_info *info = lookup_count (gimple_location (phi), stack, node);
1763 0 : gcc_assert (!info);
1764 0 : dump_stmt (phi, info, NULL, stack);
1765 0 : counts.add (info);
1766 0 : for (edge e : bb->succs)
1767 : {
1768 0 : location_t phi_loc
1769 0 : = gimple_phi_arg_location_from_edge (phi, e);
1770 0 : inline_stack stack;
1771 0 : get_inline_stack_in_node (phi_loc, &stack, node);
1772 0 : count_info *info = lookup_count (phi_loc, stack, node);
1773 0 : if (info)
1774 0 : counts.add (info);
1775 0 : gcc_checking_assert (stack.length () < 2);
1776 0 : mark_expr_locations (this,
1777 : gimple_phi_arg_def_from_edge (phi, e),
1778 : node, counts);
1779 0 : }
1780 0 : }
1781 : /* TODO: goto locuses are not used for BB annotation. */
1782 0 : for (edge e : bb->succs)
1783 : {
1784 0 : inline_stack stack;
1785 0 : get_inline_stack_in_node (e->goto_locus, &stack, node);
1786 0 : count_info *info = lookup_count (e->goto_locus, stack, node);
1787 0 : if (info)
1788 0 : counts.add (info);
1789 0 : }
1790 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
1791 0 : !gsi_end_p (gsi); gsi_next (&gsi))
1792 : {
1793 0 : inline_stack stack;
1794 0 : gimple *stmt = gsi_stmt (gsi);
1795 0 : get_inline_stack_in_node (gimple_location (stmt), &stack, node);
1796 :
1797 0 : count_info *info = lookup_count (gimple_location (stmt), stack, node);
1798 0 : if (info)
1799 0 : counts.add (info);
1800 0 : for (unsigned int op = 0; op < gimple_num_ops (stmt); op++)
1801 0 : mark_expr_locations (this, gimple_op (stmt, op), node, counts);
1802 0 : if (gimple_code (stmt) == GIMPLE_CALL)
1803 : {
1804 0 : function_instance *inlined_fn = NULL;
1805 0 : function_instance *inlined_fn_nodisc = NULL;
1806 : /* Lookup callsite. */
1807 0 : if (stack.length ())
1808 : {
1809 0 : int c = 0;
1810 0 : int cnodis = 0;
1811 0 : for (auto const &iter : callsites)
1812 0 : if (iter.first.first == stack[0].afdo_loc)
1813 : {
1814 0 : if (!c)
1815 0 : inlined_fn = iter.second;
1816 0 : c++;
1817 : }
1818 : /* Discriminators are sometimes lost; try to find the
1819 : call without discriminator info. */
1820 0 : else if (iter.first.first == (stack[0].afdo_loc & ~65535))
1821 : {
1822 0 : if (!cnodis)
1823 0 : inlined_fn_nodisc = iter.second;
1824 0 : cnodis++;
1825 : }
1826 0 : if ((c > 1 || (!c && cnodis > 1))
1827 0 : && warning (OPT_Wauto_profile,
1828 : "duplicated callsite in auto-profile of %q+F"
1829 : " with relative location %i,"
1830 : " discriminator %i",
1831 0 : node->decl, stack[0].afdo_loc >> 16,
1832 0 : stack[0].afdo_loc & 65535))
1833 0 : inform (gimple_location (stmt), "corresponding call");
1834 0 : if (inlined_fn && info && info->targets.size ()
1835 0 : && warning (OPT_Wauto_profile,
1836 : "both call targets and inline callsite"
1837 : " information is present in auto-profile"
1838 : " of function %q+F with relative location"
1839 : " %i, discriminator %i",
1840 0 : node->decl, stack[0].afdo_loc >> 16,
1841 0 : stack[0].afdo_loc & 65535))
1842 0 : inform (gimple_location (stmt), "corresponding call");
1843 0 : tree callee = gimple_call_fndecl (stmt);
1844 0 : cgraph_node *callee_node;
1845 0 : unsigned int loc = stack[0].afdo_loc;
1846 0 : bool lost_discriminator = false;
1847 0 : if (!inlined_fn && inlined_fn_nodisc)
1848 : {
1849 0 : if (!lineno_to_call_computed)
1850 : {
1851 0 : basic_block bb2;
1852 0 : FOR_EACH_BB_FN (bb2,
1853 : DECL_STRUCT_FUNCTION (node->decl))
1854 0 : for (gimple_stmt_iterator gsi2
1855 0 : = gsi_start_bb (bb2);
1856 0 : !gsi_end_p (gsi2); gsi_next (&gsi2))
1857 0 : if (gcall *call
1858 0 : = dyn_cast <gcall *> (gsi_stmt (gsi2)))
1859 : {
1860 0 : inline_stack stack2;
1861 0 : get_inline_stack_in_node
1862 0 : (gimple_location (call),
1863 : &stack2, node);
1864 0 : if (stack2.length ())
1865 0 : lineno_to_call.get_or_insert
1866 0 : (stack2[0].afdo_loc >> 16).safe_push (call);
1867 0 : }
1868 : lineno_to_call_computed = true;
1869 : }
1870 : /* If we can determine lost discriminator uniquely,
1871 : use it. */
1872 0 : if (lineno_to_call.get
1873 0 : (stack[0].afdo_loc >> 16)->length () == 1)
1874 : {
1875 0 : if (warning (OPT_Wauto_profile,
1876 : "auto-profile of %q+F seem to contain"
1877 : " lost discriminator %i for"
1878 : " call of %s at relative location %i",
1879 : node->decl, loc & 65535,
1880 : afdo_string_table->get_symbol_name (
1881 : inlined_fn_nodisc->symbol_name ()),
1882 : loc >> 16))
1883 0 : inform (gimple_location (stmt),
1884 : "corresponding call");
1885 0 : inlined_fn = inlined_fn_nodisc;
1886 0 : if (dump_file)
1887 0 : fprintf (dump_file, " Lost discriminator %i\n",
1888 : loc & 65535);
1889 0 : loc = loc & ~65535;
1890 : }
1891 : lost_discriminator = true;
1892 : }
1893 0 : if (callee && (callee_node = cgraph_node::get (callee)))
1894 : {
1895 0 : if (inlined_fn)
1896 : {
1897 0 : int old_name = inlined_fn->symbol_name ();
1898 0 : int r = match_with_target (node, stmt, inlined_fn,
1899 : callee_node);
1900 0 : if (r == 2)
1901 : {
1902 0 : auto iter = callsites.find ({loc, old_name});
1903 0 : gcc_checking_assert (
1904 : old_name != inlined_fn->symbol_name ()
1905 : && iter != callsites.end ()
1906 : && iter->second == inlined_fn);
1907 0 : callsite key2 = {stack[0].afdo_loc,
1908 0 : inlined_fn->symbol_name ()};
1909 0 : callsites.erase (iter);
1910 0 : callsites[key2] = inlined_fn;
1911 : }
1912 0 : if (r)
1913 0 : functions.add (inlined_fn);
1914 : else
1915 0 : functions_to_offline.add (inlined_fn);
1916 : }
1917 :
1918 0 : if (info && info->targets.size () > 1)
1919 0 : warning_at (gimple_location (stmt), OPT_Wauto_profile,
1920 : "auto-profile of %q+F contains multiple"
1921 : " targets for a direct call with relative"
1922 : " location %i, discriminator %i",
1923 0 : node->decl, stack[0].afdo_loc >> 16,
1924 0 : stack[0].afdo_loc & 65535);
1925 : /* We do not need target profile for direct calls. */
1926 0 : if (info)
1927 0 : info->targets.clear ();
1928 : }
1929 : else
1930 : {
1931 0 : if (inlined_fn
1932 0 : && inlined_fn->get_call_location ()
1933 : != UNKNOWN_LOCATION)
1934 : {
1935 0 : if (warning (OPT_Wauto_profile,
1936 : "function contains two calls of the same"
1937 : " relative location +%i,"
1938 : " discriminator %i,"
1939 : " that leads to lost auto-profile",
1940 : loc >> 16,
1941 : loc & 65535))
1942 : {
1943 0 : inform (gimple_location (stmt),
1944 : "location of the first call");
1945 0 : inform (inlined_fn->get_call_location (),
1946 : "location of the second call");
1947 : }
1948 0 : if (dump_file)
1949 0 : fprintf (dump_file,
1950 : " Duplicated call location\n");
1951 0 : inlined_fn = NULL;
1952 : }
1953 0 : if (inlined_fn)
1954 : {
1955 0 : inlined_fn->set_call_location
1956 0 : (gimple_location (stmt));
1957 : /* Do renaming if needed so we can look up
1958 : cgraph node and recurse into inlined function. */
1959 0 : int *newn
1960 0 : = to_symbol_name.get (inlined_fn->symbol_name ());
1961 0 : gcc_checking_assert (
1962 : !newn || *newn != inlined_fn->symbol_name ());
1963 0 : if (newn || lost_discriminator)
1964 : {
1965 0 : auto iter = callsites.find (
1966 0 : {loc, inlined_fn->symbol_name ()});
1967 0 : gcc_checking_assert (iter != callsites.end ()
1968 : && iter->second
1969 : == inlined_fn);
1970 0 : callsite key2
1971 0 : = {stack[0].afdo_loc,
1972 0 : newn ? *newn : inlined_fn->symbol_name ()};
1973 0 : callsites.erase (iter);
1974 0 : callsites[key2] = inlined_fn;
1975 0 : inlined_fn->set_symbol_name (
1976 0 : newn ? *newn : inlined_fn->symbol_name ());
1977 : }
1978 0 : functions.add (inlined_fn);
1979 : }
1980 0 : if (info)
1981 0 : targets.add (info);
1982 : }
1983 : }
1984 0 : dump_stmt (stmt, info, inlined_fn, stack);
1985 : }
1986 : else
1987 0 : dump_stmt (stmt, info, NULL, stack);
1988 0 : }
1989 : }
1990 0 : bool warned = false;
1991 0 : for (auto &iter : pos_counts)
1992 0 : if (iter.second.targets.size ()
1993 0 : && counts.contains (&iter.second)
1994 0 : && !targets.contains (&iter.second))
1995 : {
1996 0 : if (!warned)
1997 0 : warned = warning_at
1998 0 : (DECL_SOURCE_LOCATION (node->decl),
1999 0 : OPT_Wauto_profile,
2000 : "auto-profile of %q+F contains indirect call targets"
2001 : " not associated with an indirect call statement",
2002 : node->decl);
2003 0 : if (warned)
2004 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2005 : "count %" PRIu64
2006 : " with relative location +%i, discriminator %i",
2007 0 : iter.second.count, iter.first >> 16, iter.first & 65535);
2008 0 : if (dump_file)
2009 : {
2010 0 : fprintf (dump_file, "Removing targets of ");
2011 0 : dump_afdo_loc (dump_file, iter.first);
2012 0 : fprintf (dump_file, "\n");
2013 : }
2014 0 : iter.second.targets.clear ();
2015 : }
2016 0 : warned = false;
2017 : /* Profile sometimes contains extra location for start or end of function
2018 : (prologue, epilogue).
2019 : TODO: If present, perhaps it can be used to determine entry block
2020 : and exit block counts. */
2021 0 : unsigned int end_location = get_combined_location
2022 0 : (DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl);
2023 0 : unsigned int start_location = get_combined_location
2024 0 : (DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl);
2025 : /* When outputting code to builtins location we use line number 0.
2026 : create_gcov is stupid and happily computes offsets across files.
2027 : Silently ignore it. */
2028 0 : unsigned int zero_location
2029 0 : = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16;
2030 0 : for (position_count_map::const_iterator iter = pos_counts.begin ();
2031 0 : iter != pos_counts.end ();)
2032 0 : if (!counts.contains (&iter->second))
2033 : {
2034 0 : if (iter->first != end_location
2035 : && iter->first != start_location
2036 : && (iter->first & 65535) != zero_location
2037 : && iter->first
2038 : /* FIXME: dwarf5 does not represent inline stack of debug
2039 : statements and consequently create_gcov is sometimes
2040 : mixing up statements from other functions. Do not warn
2041 : user about this until this problem is solved.
2042 : We still write info into dump file. */
2043 : && 0)
2044 : {
2045 : if (!warned)
2046 : warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
2047 : OPT_Wauto_profile,
2048 : "auto-profile of %q+F contains extra statements",
2049 : node->decl);
2050 : if (warned)
2051 : inform (DECL_SOURCE_LOCATION (node->decl),
2052 : "count %" PRIu64 " with relative location +%i,"
2053 : " discriminator %i",
2054 : iter->second.count, iter->first >> 16,
2055 : iter->first & 65535);
2056 : if ((iter->first >> 16) > (end_location >> 16) && warned)
2057 : inform (DECL_SOURCE_LOCATION (node->decl),
2058 : "location is after end of function");
2059 : }
2060 0 : if (dump_file)
2061 : {
2062 0 : fprintf (dump_file, "Removing unmatched count ");
2063 0 : dump_afdo_loc (dump_file, iter->first);
2064 0 : fprintf (dump_file, ":%" PRIu64, iter->second.count);
2065 0 : for (auto &titer : iter->second.targets)
2066 0 : fprintf (dump_file, " %s:%" PRIu64,
2067 0 : afdo_string_table->get_symbol_name (titer.first),
2068 0 : (int64_t) titer.second);
2069 0 : fprintf (dump_file, "\n");
2070 : }
2071 0 : iter = pos_counts.erase (iter);
2072 : }
2073 : else
2074 0 : iter++;
2075 0 : warned = false;
2076 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2077 0 : iter != callsites.end ();)
2078 0 : if (!functions.contains (iter->second))
2079 : {
2080 0 : function_instance *f = iter->second;
2081 : /* If we did not see the corresponding statement, warn. */
2082 0 : if (!functions_to_offline.contains (iter->second))
2083 : {
2084 0 : if (!warned)
2085 0 : warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
2086 0 : OPT_Wauto_profile,
2087 : "auto-profile of %q+F contains"
2088 : " extra callsites",
2089 : node->decl);
2090 0 : if (warned)
2091 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2092 : "call of %s with total count %" PRId64
2093 : ", relative location +%i, discriminator %i",
2094 0 : afdo_string_table->get_symbol_name (iter->first.second),
2095 0 : iter->second->total_count (), iter->first.first >> 16,
2096 0 : iter->first.first & 65535);
2097 0 : if ((iter->first.first >> 16) > (end_location >> 16) && warned)
2098 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2099 : "location is after end of function");
2100 0 : if (dump_file)
2101 : {
2102 0 : fprintf (dump_file,
2103 : "Offlining inline with no corresponding gimple stmt ");
2104 0 : f->dump_inline_stack (dump_file);
2105 0 : fprintf (dump_file, "\n");
2106 : }
2107 : }
2108 0 : else if (dump_file)
2109 : {
2110 0 : fprintf (dump_file,
2111 : "Offlining mismatched inline ");
2112 0 : f->dump_inline_stack (dump_file);
2113 0 : fprintf (dump_file, "\n");
2114 : }
2115 0 : callsites.erase (iter);
2116 0 : offline (f, new_functions);
2117 0 : iter = callsites.begin ();
2118 : }
2119 : else
2120 0 : iter++;
2121 0 : for (auto &iter : callsites)
2122 0 : if (cgraph_node *n = iter.second->get_cgraph_node ())
2123 0 : iter.second->match (n, new_functions, to_symbol_name);
2124 0 : return true;
2125 0 : }
2126 :
2127 : /* Walk inlined functions and if their name is not in SEEN
2128 : remove it. Also rename function names as given by
2129 : to_symbol_name map. */
2130 :
2131 : void
2132 0 : function_instance::remove_external_functions
2133 : (name_index_set &seen,
2134 : name_index_map &to_symbol_name,
2135 : vec <function_instance *> &new_functions)
2136 : {
2137 0 : auto_vec <callsite, 20> to_rename;
2138 :
2139 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2140 0 : iter != callsites.end ();)
2141 0 : if (!seen.contains (iter->first.second))
2142 : {
2143 0 : function_instance *f = iter->second;
2144 0 : if (dump_file)
2145 : {
2146 0 : fprintf (dump_file, " Removing external inline: ");
2147 0 : f->dump_inline_stack (dump_file);
2148 0 : fprintf (dump_file, "\n");
2149 : }
2150 0 : iter = callsites.erase (iter);
2151 0 : f->set_inlined_to (NULL);
2152 0 : f->offline_if_in_set (seen, new_functions);
2153 0 : delete f;
2154 : }
2155 : else
2156 : {
2157 0 : gcc_checking_assert ((int) iter->first.second
2158 : == iter->second->symbol_name ());
2159 0 : int *newn = iter->second->get_call_location () == UNKNOWN_LOCATION
2160 0 : ? to_symbol_name.get (iter->first.second)
2161 : : NULL;
2162 0 : if (newn)
2163 : {
2164 0 : gcc_checking_assert (iter->second->inlined_to ());
2165 0 : to_rename.safe_push (iter->first);
2166 : }
2167 0 : iter->second->remove_external_functions
2168 0 : (seen, to_symbol_name, new_functions);
2169 0 : ++iter;
2170 : }
2171 0 : for (auto &key : to_rename)
2172 : {
2173 0 : auto iter = callsites.find (key);
2174 0 : callsite key2 = key;
2175 0 : key2.second = *to_symbol_name.get (key.second);
2176 0 : iter->second->set_symbol_name (key2.second);
2177 0 : callsites.erase (iter);
2178 0 : callsites[key2] = iter->second;
2179 : }
2180 0 : auto_vec <int, 20> target_to_rename;
2181 0 : for (auto &iter : pos_counts)
2182 : {
2183 0 : for (auto const &titer : iter.second.targets)
2184 : {
2185 0 : int *ren = to_symbol_name.get (titer.first);
2186 0 : if (ren)
2187 0 : target_to_rename.safe_push (titer.first);
2188 : }
2189 0 : while (target_to_rename.length ())
2190 : {
2191 0 : int key = target_to_rename.pop ();
2192 0 : int key2 = *to_symbol_name.get (key);
2193 0 : auto i = iter.second.targets.find (key);
2194 0 : if (iter.second.targets.count (key2) == 0)
2195 0 : iter.second.targets[key2] = i->second;
2196 : else
2197 0 : iter.second.targets[key2] += i->second;
2198 0 : iter.second.targets.erase (i);
2199 : }
2200 : }
2201 0 : }
2202 :
2203 : /* Look for inline instances that was not realized and
2204 : remove them while possibly merging them to offline variants. */
2205 :
2206 : void
2207 0 : function_instance::offline_if_not_realized
2208 : (vec <function_instance *> &new_functions)
2209 : {
2210 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2211 0 : iter != callsites.end ();)
2212 0 : if (!iter->second->realized_p ())
2213 : {
2214 0 : function_instance *f = iter->second;
2215 0 : if (dump_file)
2216 : {
2217 0 : fprintf (dump_file, "Offlining unrealized inline ");
2218 0 : f->dump_inline_stack (dump_file);
2219 0 : fprintf (dump_file, "\n");
2220 : }
2221 0 : iter = callsites.erase (iter);
2222 0 : offline (f, new_functions);
2223 : }
2224 : else
2225 : {
2226 0 : iter->second->offline_if_not_realized (new_functions);
2227 0 : ++iter;
2228 : }
2229 0 : }
2230 :
2231 : /* Dump instance to F indented by INDENT. */
2232 :
2233 : void
2234 0 : function_instance::dump (FILE *f, int indent, bool nested) const
2235 : {
2236 0 : if (!nested)
2237 0 : fprintf (f, "%*s%s total:%" PRIu64 " head:%" PRId64 "\n", indent, "",
2238 : afdo_string_table->get_symbol_name (symbol_name ()),
2239 0 : (int64_t) total_count (), (int64_t) head_count ());
2240 : else
2241 0 : fprintf (f, " total:%" PRIu64 "\n", (int64_t)total_count ());
2242 0 : for (auto const &iter : pos_counts)
2243 : {
2244 0 : fprintf (f, "%*s", indent + 2, "");
2245 0 : dump_afdo_loc (f, iter.first);
2246 0 : fprintf (f, ": %" PRIu64, (int64_t)iter.second.count);
2247 :
2248 0 : for (auto const &titer : iter.second.targets)
2249 0 : fprintf (f, " %s:%" PRIu64,
2250 0 : afdo_string_table->get_symbol_name (titer.first),
2251 0 : (int64_t) titer.second);
2252 0 : fprintf (f,"\n");
2253 : }
2254 0 : for (auto const &iter : callsites)
2255 : {
2256 0 : fprintf (f, "%*s", indent + 2, "");
2257 0 : dump_afdo_loc (f, iter.first.first);
2258 0 : fprintf (f, ": %s",
2259 0 : afdo_string_table->get_symbol_name (iter.first.second));
2260 0 : iter.second->dump (f, indent + 2, true);
2261 0 : gcc_checking_assert ((int) iter.first.second
2262 : == iter.second->symbol_name ());
2263 : }
2264 0 : }
2265 :
2266 : /* Dump inline path. */
2267 :
2268 : void
2269 0 : function_instance::dump_inline_stack (FILE *f) const
2270 : {
2271 0 : auto_vec <callsite, 20> stack;
2272 0 : const function_instance *p = this, *s = inlined_to ();
2273 0 : while (s)
2274 : {
2275 0 : bool found = false;
2276 0 : for (callsite_map::const_iterator iter = s->callsites.begin ();
2277 0 : iter != s->callsites.end (); ++iter)
2278 0 : if (iter->second == p)
2279 : {
2280 0 : gcc_checking_assert (
2281 : !found && (int) iter->first.second == p->symbol_name ());
2282 0 : stack.safe_push ({iter->first.first, s->symbol_name ()});
2283 0 : found = true;
2284 : }
2285 0 : gcc_checking_assert (found);
2286 0 : p = s;
2287 0 : s = s->inlined_to ();
2288 : }
2289 0 : for (callsite &s: stack)
2290 : {
2291 0 : fprintf (f, "%s:", afdo_string_table->get_symbol_name (s.second));
2292 0 : dump_afdo_loc (f, s.first);
2293 0 : fprintf (f, " ");
2294 : }
2295 0 : fprintf (f, "%s", afdo_string_table->get_symbol_name (symbol_name ()));
2296 0 : }
2297 :
2298 : /* Dump instance to stderr. */
2299 :
2300 : void
2301 0 : function_instance::debug () const
2302 : {
2303 0 : dump (stderr);
2304 0 : }
2305 :
2306 : /* Return profile info for LOC in INFO. */
2307 :
2308 : bool
2309 0 : function_instance::get_count_info (location_t loc, count_info *info) const
2310 : {
2311 0 : position_count_map::const_iterator iter = pos_counts.find (loc);
2312 0 : if (iter == pos_counts.end ())
2313 : return false;
2314 0 : *info = iter->second;
2315 0 : return true;
2316 : }
2317 :
2318 : /* Read the inlined indirect call target profile for STMT and store it in
2319 : MAP, return the total count for all inlined indirect calls. */
2320 :
2321 : gcov_type
2322 0 : function_instance::find_icall_target_map (tree fn, gcall *stmt,
2323 : icall_target_map *map) const
2324 : {
2325 0 : gcov_type ret = 0;
2326 0 : unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);
2327 :
2328 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2329 0 : iter != callsites.end (); ++iter)
2330 : {
2331 0 : unsigned callee = iter->second->symbol_name ();
2332 : /* Check if callsite location match the stmt. */
2333 0 : if (iter->first.first != stmt_offset
2334 0 : || iter->second->removed_icall_target ())
2335 0 : continue;
2336 0 : struct cgraph_node *node = cgraph_node::get_for_asmname (
2337 : get_identifier (afdo_string_table->get_symbol_name (callee)));
2338 0 : if (node == NULL)
2339 0 : continue;
2340 0 : (*map)[callee] = iter->second->total_count () * afdo_count_scale;
2341 0 : ret += iter->second->total_count () * afdo_count_scale;
2342 : }
2343 0 : return ret;
2344 : }
2345 :
2346 : /* Remove the inlined indirect call target profile for STMT. */
2347 :
2348 : void
2349 0 : function_instance::remove_icall_target (tree fn, gcall *stmt)
2350 : {
2351 0 : unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);
2352 0 : int n = 0;
2353 :
2354 0 : for (auto iter : callsites)
2355 0 : if (iter.first.first == stmt_offset)
2356 : {
2357 0 : iter.second->remove_icall_target ();
2358 0 : n++;
2359 : }
2360 : /* TODO: If we add support for multiple targets, we may want to
2361 : remove only those we succesfully inlined. */
2362 0 : gcc_assert (n);
2363 0 : }
2364 :
2365 : /* Offline all functions not defined in the current unit.
2366 : We will not be able to early inline them.
2367 : Doing so early will get VPT decisions more realistic. */
2368 :
2369 : void
2370 0 : autofdo_source_profile::offline_external_functions ()
2371 : {
2372 : /* First check all available definitions and mark their names as
2373 : visible. */
2374 0 : cgraph_node *node;
2375 0 : name_index_set seen;
2376 0 : name_index_map to_symbol_name;
2377 0 : size_t last_name;
2378 :
2379 : /* Add renames erasing suffixes produced by late clones, such as
2380 : .isra, .ipcp. */
2381 0 : for (size_t i = 1; i < afdo_string_table->num_entries (); i++)
2382 : {
2383 0 : const char *n1 = afdo_string_table->get_symbol_name (i);
2384 0 : std::pair<const char *, int> name_filename
2385 0 : = afdo_string_table->get_original_name (n1);
2386 0 : const char *n2 = name_filename.first;
2387 0 : if (!strcmp (n1, n2))
2388 : {
2389 : /* Watch for duplicate entries.
2390 : This seems to happen in practice and may be useful to distinguish
2391 : multiple static symbols of the same name, but we do not realy
2392 : have a way to differentiate them in get_symbol_name lookup. */
2393 0 : int index = afdo_string_table->get_index (n1);
2394 0 : if (index != (int)i)
2395 : {
2396 0 : if (dump_file)
2397 0 : fprintf (dump_file,
2398 : "string table in auto-profile contains"
2399 : " duplicated name %s\n", n1);
2400 0 : to_symbol_name.put (i, index);
2401 : }
2402 0 : continue;
2403 0 : }
2404 0 : if (dump_file)
2405 0 : fprintf (dump_file, "Adding rename removing clone suffixes %s -> %s\n",
2406 : n1, n2);
2407 0 : int index = afdo_string_table->get_index (n2);
2408 0 : if (index == -1)
2409 0 : index = afdo_string_table->add_symbol_name (xstrdup (n2),
2410 : name_filename.second);
2411 0 : to_symbol_name.put (i, index);
2412 : }
2413 0 : last_name = afdo_string_table->num_entries ();
2414 0 : FOR_EACH_DEFINED_FUNCTION (node)
2415 : {
2416 0 : const char *name = raw_symbol_name (node->decl);
2417 0 : const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0);
2418 0 : int index = afdo_string_table->get_index (name);
2419 :
2420 : /* Inline function may be identified by its dwarf names;
2421 : rename them to symbol names. With LTO dwarf names are
2422 : lost in free_lange_data. */
2423 0 : if (strcmp (name, dwarf_name))
2424 : {
2425 0 : int index2 = afdo_string_table->get_index (dwarf_name);
2426 0 : if (index2 != -1)
2427 : {
2428 0 : if (index == -1)
2429 0 : index = afdo_string_table->add_symbol_name (
2430 0 : xstrdup (name),
2431 : afdo_string_table->add_filename (
2432 0 : get_normalized_path (DECL_SOURCE_FILE (node->decl))));
2433 0 : if (dump_file)
2434 : {
2435 0 : fprintf (dump_file, "Adding dwarf->symbol rename %s -> %s\n",
2436 : afdo_string_table->get_symbol_name (index2), name);
2437 0 : if (to_symbol_name.get (index2))
2438 0 : fprintf (dump_file, "Dwarf name is not unique");
2439 : }
2440 0 : to_symbol_name.put (index2, index);
2441 0 : seen.add (index2);
2442 : }
2443 : }
2444 0 : if (index != -1)
2445 : {
2446 0 : if (dump_file)
2447 0 : fprintf (dump_file, "%s is defined in node %s\n",
2448 : afdo_string_table->get_symbol_name (index),
2449 : node->dump_name ());
2450 0 : seen.add (index);
2451 : }
2452 : else
2453 : {
2454 0 : if (dump_file)
2455 : {
2456 0 : if (dwarf_name && strcmp (dwarf_name, name))
2457 0 : fprintf (dump_file,
2458 : "Node %s not in auto profile (%s neither %s)\n",
2459 : node->dump_name (),
2460 : name,
2461 : dwarf_name);
2462 : else
2463 0 : fprintf (dump_file,
2464 : "Node %s (symbol %s) not in auto profile\n",
2465 : node->dump_name (),
2466 : name);
2467 : }
2468 : }
2469 : }
2470 :
2471 0 : for (auto iter : to_symbol_name)
2472 : {
2473 : /* In case dwarf name was duplicated and later renamed,
2474 : handle both. No more than one hop should be needed. */
2475 0 : int *newn = to_symbol_name.get (iter.second);
2476 0 : if (newn)
2477 0 : iter.second = *newn;
2478 0 : gcc_checking_assert (!to_symbol_name.get (iter.second));
2479 0 : if (seen.contains (iter.second))
2480 0 : seen.add (iter.first);
2481 : }
2482 :
2483 : /* Now process all toplevel (offline) function instances.
2484 :
2485 : If instance has no definition in this translation unit,
2486 : first offline all inlined functions which are defined here
2487 : (so we do not lose profile due to cross-module inlining
2488 : done by link-time optimizers).
2489 :
2490 : If instance has a definition, look into all inlined functions
2491 : and remove external ones (result of cross-module inlining).
2492 :
2493 : TODO: after early-inlining we ought to offline all functions
2494 : that were not inlined. */
2495 0 : vec <function_instance *>&fns = duplicate_functions_;
2496 0 : auto_vec <function_instance *, 20>fns2;
2497 : /* Populate worklist with all functions to process. Processing
2498 : may introduce new functions by offlining. */
2499 0 : for (auto &function : map_)
2500 : {
2501 0 : function.second->set_in_worklist ();
2502 0 : fns.safe_push (function.second);
2503 : }
2504 :
2505 : /* There are two worklists. First all functions needs to be matched
2506 : with gimple body and only then we want to do merging, since matching
2507 : should be done on unmodified profile and merging works better if
2508 : mismatches are already resolved both in source and destination. */
2509 0 : while (fns.length () || fns2.length ())
2510 : {
2511 : /* In case renaming introduced new name, keep seen up to date. */
2512 0 : for (; last_name < afdo_string_table->num_entries (); last_name++)
2513 : {
2514 0 : const char *name = afdo_string_table->get_symbol_name (last_name);
2515 0 : symtab_node *n
2516 0 : = afdo_string_table->get_cgraph_node (last_name);
2517 0 : if (dump_file)
2518 0 : fprintf (dump_file, "New name %s %s\n", name,
2519 : n ? "wth corresponding definition"
2520 : : "with no corresponding definition");
2521 0 : if (n)
2522 0 : seen.add (last_name);
2523 : }
2524 0 : if (fns.length ())
2525 : {
2526 0 : function_instance *f = fns.pop ();
2527 0 : if (f->get_location () == UNKNOWN_LOCATION)
2528 : {
2529 0 : int index = f->symbol_name ();
2530 0 : int *newn = to_symbol_name.get (index);
2531 0 : if (newn)
2532 : {
2533 0 : if (find_function_instance (f->get_descriptor ()) == f)
2534 0 : remove_function_instance (f);
2535 0 : f->set_symbol_name (*newn);
2536 0 : if (!find_function_instance (f->get_descriptor ()))
2537 0 : add_function_instance (f);
2538 : }
2539 0 : if (cgraph_node *n = f->get_cgraph_node ())
2540 : {
2541 0 : gcc_checking_assert (seen.contains (f->symbol_name ()));
2542 0 : f->match (n, fns, to_symbol_name);
2543 : }
2544 : }
2545 0 : fns2.safe_push (f);
2546 : }
2547 : else
2548 : {
2549 0 : function_instance *f = fns2.pop ();
2550 0 : int index = f->symbol_name ();
2551 0 : gcc_checking_assert (f->in_worklist_p ());
2552 :
2553 : /* If map has different function_instance of same name, then
2554 : this is a duplicated entry which needs to be merged. */
2555 0 : function_instance *index_inst
2556 0 : = find_function_instance (f->get_descriptor ());
2557 0 : if (index_inst && index_inst != f)
2558 : {
2559 0 : if (dump_file)
2560 : {
2561 0 : fprintf (dump_file, "Merging duplicate instance: ");
2562 0 : f->dump_inline_stack (dump_file);
2563 0 : fprintf (dump_file, "\n");
2564 : }
2565 0 : index_inst->merge (f, fns);
2566 0 : gcc_checking_assert (!f->inlined_to ());
2567 0 : f->clear_in_worklist ();
2568 0 : delete f;
2569 : }
2570 : /* If name was not seen in the symbol table, remove it. */
2571 0 : else if (!seen.contains (index))
2572 : {
2573 0 : f->offline_if_in_set (seen, fns);
2574 0 : f->clear_in_worklist ();
2575 0 : if (dump_file)
2576 0 : fprintf (dump_file, "Removing external %s\n",
2577 : afdo_string_table->get_symbol_name (
2578 : f->symbol_name ()));
2579 0 : if (index_inst == f)
2580 0 : remove_function_instance (f);
2581 0 : delete f;
2582 : }
2583 : /* If this is offline function instance seen in this
2584 : translation unit offline external inlines and possibly
2585 : rename from dwarf name. */
2586 : else
2587 : {
2588 0 : f->remove_external_functions (seen, to_symbol_name, fns);
2589 0 : f->clear_in_worklist ();
2590 : }
2591 : }
2592 : }
2593 0 : if (dump_file)
2594 0 : for (auto const &function : map_)
2595 : {
2596 0 : seen.contains (function.second->symbol_name ());
2597 0 : function.second->dump (dump_file);
2598 : }
2599 0 : }
2600 :
2601 : /* Walk scope block BLOCK and mark all inlined functions as realized. */
2602 :
2603 : static void
2604 0 : walk_block (tree fn, function_instance *s, tree block)
2605 : {
2606 0 : if (inlined_function_outer_scope_p (block))
2607 : {
2608 0 : unsigned loc = get_relative_location_for_locus
2609 0 : (fn, BLOCK_SUPERCONTEXT (block),
2610 0 : BLOCK_SOURCE_LOCATION (block));
2611 0 : function_instance *ns
2612 : = s->get_function_instance_by_decl
2613 0 : (loc, BLOCK_ABSTRACT_ORIGIN (block),
2614 0 : BLOCK_SOURCE_LOCATION (block));
2615 0 : if (!ns)
2616 : {
2617 0 : if (dump_file)
2618 : {
2619 0 : fprintf (dump_file, " Failed to find inlined instance:");
2620 0 : s->dump_inline_stack (dump_file);
2621 0 : fprintf (dump_file, ":");
2622 0 : dump_afdo_loc (dump_file, loc);
2623 0 : fprintf (dump_file, " %s\n",
2624 0 : raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block)));
2625 : }
2626 0 : return;
2627 : }
2628 0 : s = ns;
2629 0 : if (dump_file)
2630 : {
2631 0 : fprintf (dump_file, " Marking realized inline: ");
2632 0 : s->dump_inline_stack (dump_file);
2633 0 : fprintf (dump_file, "\n");
2634 : }
2635 0 : s->set_realized ();
2636 : }
2637 0 : for (tree t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2638 0 : walk_block (fn, s, t);
2639 : }
2640 :
2641 : /* Offline all inline functions that are not marked as realized.
2642 : This will merge their profile into offline versions where available.
2643 : Also remove all functions we will no longer use. */
2644 :
2645 : void
2646 0 : autofdo_source_profile::offline_unrealized_inlines ()
2647 : {
2648 0 : auto_vec <function_instance *>fns;
2649 : /* Populate worklist with all functions to process. Processing
2650 : may introduce new functions by offlining. */
2651 0 : for (auto const &function : map_)
2652 : {
2653 0 : fns.safe_push (function.second);
2654 0 : function.second->set_in_worklist ();
2655 : }
2656 0 : while (fns.length ())
2657 : {
2658 0 : function_instance *f = fns.pop ();
2659 0 : int index = f->symbol_name ();
2660 0 : function_instance *index_inst
2661 0 : = find_function_instance (f->get_descriptor ());
2662 0 : bool in_map = index_inst != nullptr;
2663 0 : if (in_map)
2664 0 : if (cgraph_node *n = f->get_cgraph_node ())
2665 : {
2666 0 : if (dump_file)
2667 0 : fprintf (dump_file, "Marking realized %s\n",
2668 : afdo_string_table->get_symbol_name (index));
2669 0 : f->set_realized ();
2670 0 : if (DECL_INITIAL (n->decl)
2671 0 : && DECL_INITIAL (n->decl) != error_mark_node)
2672 0 : walk_block (n->decl, f, DECL_INITIAL (n->decl));
2673 : }
2674 0 : f->offline_if_not_realized (fns);
2675 0 : gcc_checking_assert ((in_map || !f->realized_p ())
2676 : && f->in_worklist_p ());
2677 :
2678 : /* If this is duplicated instance, merge it into one in map. */
2679 0 : if (in_map && index_inst != f)
2680 : {
2681 0 : if (dump_file)
2682 : {
2683 0 : fprintf (dump_file, "Merging duplicate instance: ");
2684 0 : f->dump_inline_stack (dump_file);
2685 0 : fprintf (dump_file, "\n");
2686 : }
2687 0 : index_inst->merge (f, fns);
2688 0 : f->clear_in_worklist ();
2689 0 : gcc_checking_assert (!f->inlined_to ());
2690 0 : delete f;
2691 : }
2692 : /* If function is not in symbol table, remove it. */
2693 0 : else if (!f->realized_p ())
2694 : {
2695 0 : if (dump_file)
2696 0 : fprintf (dump_file, "Removing optimized out function %s\n",
2697 : afdo_string_table->get_symbol_name (f->symbol_name ()));
2698 0 : if (in_map)
2699 0 : remove_function_instance (index_inst);
2700 0 : f->clear_in_worklist ();
2701 0 : delete f;
2702 : }
2703 : else
2704 0 : f->clear_in_worklist ();
2705 : }
2706 0 : if (dump_file)
2707 0 : for (auto const &function : map_)
2708 0 : function.second->dump (dump_file);
2709 0 : }
2710 :
2711 : /* Read the profile and create a function_instance with head count as
2712 : HEAD_COUNT. Recursively read callsites to create nested function_instances
2713 : too. STACK is used to track the recursive creation process. */
2714 :
2715 : /* function instance profile format:
2716 :
2717 : ENTRY_COUNT: 8 bytes
2718 : TIMESTAMP: 8 bytes (only for toplevel symbols)
2719 : NAME_INDEX: 4 bytes
2720 : NUM_POS_COUNTS: 4 bytes
2721 : NUM_CALLSITES: 4 byte
2722 : POS_COUNT_1:
2723 : POS_1_OFFSET: 4 bytes
2724 : NUM_TARGETS: 4 bytes
2725 : COUNT: 8 bytes
2726 : TARGET_1:
2727 : VALUE_PROFILE_TYPE: 4 bytes
2728 : TARGET_IDX: 8 bytes
2729 : COUNT: 8 bytes
2730 : TARGET_2
2731 : ...
2732 : TARGET_n
2733 : POS_COUNT_2
2734 : ...
2735 : POS_COUNT_N
2736 : CALLSITE_1:
2737 : CALLSITE_1_OFFSET: 4 bytes
2738 : FUNCTION_INSTANCE_PROFILE (nested)
2739 : CALLSITE_2
2740 : ...
2741 : CALLSITE_n. */
2742 :
2743 : function_instance *
2744 0 : function_instance::read_function_instance (function_instance_stack *stack,
2745 : bool toplevel)
2746 : {
2747 0 : gcov_type_unsigned timestamp = 0;
2748 0 : gcov_type head_count = -1;
2749 0 : if (toplevel)
2750 : {
2751 0 : head_count = gcov_read_counter ();
2752 0 : timestamp = (gcov_type_unsigned) gcov_read_counter ();
2753 : }
2754 0 : unsigned name = gcov_read_unsigned ();
2755 0 : unsigned num_pos_counts = gcov_read_unsigned ();
2756 0 : unsigned num_callsites = gcov_read_unsigned ();
2757 0 : function_instance *s
2758 : = new function_instance (name,
2759 0 : afdo_string_table->get_filename_by_symbol (name),
2760 0 : head_count);
2761 0 : if (timestamp > 0)
2762 0 : s->set_timestamp (timestamp);
2763 0 : if (!stack->is_empty ())
2764 0 : s->set_inlined_to (stack->last ());
2765 0 : stack->safe_push (s);
2766 :
2767 0 : for (unsigned i = 0; i < num_pos_counts; i++)
2768 : {
2769 0 : unsigned offset = gcov_read_unsigned ();
2770 0 : unsigned num_targets = gcov_read_unsigned ();
2771 0 : gcov_type count = gcov_read_counter ();
2772 0 : s->pos_counts[offset].count = count;
2773 :
2774 0 : for (unsigned j = 0; j < stack->length (); j++)
2775 0 : (*stack)[j]->total_count_ += count;
2776 0 : for (unsigned j = 0; j < num_targets; j++)
2777 : {
2778 : /* Only indirect call target histogram is supported now. */
2779 0 : gcov_read_unsigned ();
2780 0 : gcov_type target_idx = gcov_read_counter ();
2781 0 : s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
2782 : }
2783 : }
2784 0 : for (unsigned i = 0; i < num_callsites; i++)
2785 : {
2786 0 : unsigned offset = gcov_read_unsigned ();
2787 0 : function_instance *callee_function_instance
2788 0 : = read_function_instance (stack, false);
2789 0 : s->callsites[std::make_pair (offset,
2790 0 : callee_function_instance->symbol_name ())]
2791 0 : = callee_function_instance;
2792 : }
2793 0 : stack->pop ();
2794 0 : return s;
2795 : }
2796 :
2797 : /* Member functions for autofdo_source_profile. */
2798 :
2799 0 : autofdo_source_profile::~autofdo_source_profile ()
2800 : {
2801 0 : for (name_function_instance_map::const_iterator iter = map_.begin ();
2802 0 : iter != map_.end (); ++iter)
2803 0 : delete iter->second;
2804 0 : }
2805 :
2806 : /* For a given DECL, returns the top-level function_instance. */
2807 :
2808 : function_instance *
2809 0 : autofdo_source_profile::get_function_instance_by_decl (tree decl, const char *filename) const
2810 : {
2811 0 : if (!filename)
2812 0 : filename = get_normalized_path (DECL_SOURCE_FILE (decl));
2813 0 : int index = afdo_string_table->get_index_by_decl (decl);
2814 0 : if (index == -1)
2815 : return NULL;
2816 :
2817 0 : function_instance_descriptor descriptor (
2818 0 : afdo_string_table->get_filename_index (filename), index);
2819 0 : return find_function_instance (descriptor);
2820 : }
2821 :
2822 : /* For a given DESCRIPTOR, return the matching instance if found. */
2823 :
2824 : function_instance *
2825 0 : autofdo_source_profile::get_function_instance_by_descriptor (
2826 : function_instance_descriptor descriptor) const
2827 : {
2828 0 : return find_function_instance (descriptor);
2829 : }
2830 :
2831 : /* Add function instance FN. */
2832 :
2833 : void
2834 0 : autofdo_source_profile::add_function_instance (function_instance *fn)
2835 : {
2836 0 : gcc_checking_assert (map_.find (fn->get_descriptor ()) == map_.end ());
2837 0 : map_[fn->get_descriptor ()] = fn;
2838 0 : }
2839 :
2840 : /* Find count_info for a given gimple STMT. If found, store the count_info
2841 : in INFO and return true; otherwise return false. */
2842 :
2843 : bool
2844 0 : autofdo_source_profile::get_count_info (gimple *stmt, count_info *info,
2845 : cgraph_node *node) const
2846 : {
2847 0 : gcc_checking_assert (stmt_loc_used_by_debug_info (stmt));
2848 0 : return get_count_info (gimple_location (stmt), info, node);
2849 : }
2850 :
2851 : bool
2852 0 : autofdo_source_profile::get_count_info (location_t gimple_loc,
2853 : count_info *info,
2854 : cgraph_node *node) const
2855 : {
2856 0 : if (LOCATION_LOCUS (gimple_loc) == cfun->function_end_locus)
2857 : return false;
2858 :
2859 0 : inline_stack stack;
2860 0 : get_inline_stack_in_node (gimple_loc, &stack, node);
2861 0 : if (stack.length () == 0)
2862 : return false;
2863 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2864 0 : if (s == NULL)
2865 : return false;
2866 0 : return s->get_count_info (stack[0].afdo_loc, info);
2867 0 : }
2868 :
2869 : /* Update value profile INFO for STMT from the inlined indirect callsite.
2870 : Return true if INFO is updated. */
2871 :
2872 : bool
2873 0 : autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
2874 : count_info *info,
2875 : cgraph_node *node)
2876 : {
2877 0 : if (dump_file)
2878 : {
2879 0 : fprintf (dump_file, "Checking indirect call -> direct call ");
2880 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2881 : }
2882 :
2883 0 : if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
2884 : {
2885 0 : if (dump_file)
2886 0 : fprintf (dump_file, " bad locus (function end)\n");
2887 0 : return false;
2888 : }
2889 :
2890 0 : count_info old_info;
2891 0 : get_count_info (stmt, &old_info, node);
2892 0 : gcov_type total = 0;
2893 0 : for (icall_target_map::const_iterator iter = old_info.targets.begin ();
2894 0 : iter != old_info.targets.end (); ++iter)
2895 0 : total += iter->second;
2896 0 : total *= afdo_count_scale;
2897 :
2898 : /* Program behavior changed, original promoted (and inlined) target is not
2899 : hot any more. Will avoid promote the original target.
2900 :
2901 : To check if original promoted target is still hot, we check the total
2902 : count of the unpromoted targets (stored in TOTAL). If a callsite count
2903 : (stored in INFO) is smaller than half of the total count, the original
2904 : promoted target is considered not hot any more. */
2905 0 : if (info->count < total / 2)
2906 : {
2907 0 : if (dump_file)
2908 0 : fprintf (dump_file, " not hot anymore %ld < %ld",
2909 : (long)info->count,
2910 : (long)total /2);
2911 0 : return false;
2912 : }
2913 :
2914 0 : inline_stack stack;
2915 0 : get_inline_stack_in_node (gimple_location (stmt), &stack, node);
2916 0 : if (stack.length () == 0)
2917 : {
2918 0 : if (dump_file)
2919 0 : fprintf (dump_file, " no inline stack\n");
2920 0 : return false;
2921 : }
2922 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2923 0 : if (s == NULL)
2924 : {
2925 0 : if (dump_file)
2926 : {
2927 0 : fprintf (dump_file, " function not found in inline stack:");
2928 0 : dump_inline_stack (dump_file, &stack);
2929 : }
2930 0 : return false;
2931 : }
2932 0 : icall_target_map map;
2933 0 : if (s->find_icall_target_map (node ? node->decl
2934 : : current_function_decl,
2935 : stmt, &map) == 0)
2936 : {
2937 0 : if (dump_file)
2938 : {
2939 0 : fprintf (dump_file, " no target map for stack: ");
2940 0 : dump_inline_stack (dump_file, &stack);
2941 : }
2942 0 : return false;
2943 : }
2944 0 : for (icall_target_map::const_iterator iter = map.begin ();
2945 0 : iter != map.end (); ++iter)
2946 0 : info->targets[iter->first] = iter->second;
2947 0 : if (dump_file)
2948 : {
2949 0 : fprintf (dump_file, " looks good; stack:");
2950 0 : dump_inline_stack (dump_file, &stack);
2951 : }
2952 : return true;
2953 0 : }
2954 :
2955 : void
2956 0 : autofdo_source_profile::remove_icall_target (cgraph_edge *e)
2957 : {
2958 0 : autofdo::inline_stack stack;
2959 0 : autofdo::get_inline_stack_in_node (gimple_location (e->call_stmt),
2960 : &stack, e->caller);
2961 0 : autofdo::function_instance *s
2962 0 : = get_function_instance_by_inline_stack (stack);
2963 0 : s->remove_icall_target (e->caller->decl, e->call_stmt);
2964 0 : }
2965 :
2966 : /* Find total count of the callee of EDGE. */
2967 :
2968 : gcov_type
2969 0 : autofdo_source_profile::get_callsite_total_count (
2970 : struct cgraph_edge *edge) const
2971 : {
2972 0 : inline_stack stack;
2973 0 : stack.safe_push ({edge->callee->decl, 0, UNKNOWN_LOCATION});
2974 :
2975 0 : get_inline_stack_in_node (gimple_location (edge->call_stmt), &stack,
2976 : edge->caller);
2977 0 : if (dump_file)
2978 : {
2979 0 : if (!edge->caller->inlined_to)
2980 0 : fprintf (dump_file, "Looking up afdo profile for call %s -> %s stack:",
2981 0 : edge->caller->dump_name (), edge->callee->dump_name ());
2982 : else
2983 0 : fprintf (dump_file, "Looking up afdo profile for call %s -> %s transitively %s stack:",
2984 0 : edge->caller->dump_name (), edge->callee->dump_name (),
2985 : edge->caller->inlined_to->dump_name ());
2986 0 : dump_inline_stack (dump_file, &stack);
2987 : }
2988 :
2989 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2990 0 : if (s == NULL)
2991 : {
2992 0 : if (dump_file)
2993 0 : fprintf (dump_file, "No function instance found\n");
2994 0 : return 0;
2995 : }
2996 0 : if (afdo_string_table->get_index_by_decl (edge->callee->decl)
2997 0 : != s->symbol_name ())
2998 : {
2999 0 : if (dump_file)
3000 0 : fprintf (dump_file, "Mismatched name of callee %s and profile %s\n",
3001 0 : raw_symbol_name (edge->callee->decl),
3002 : afdo_string_table->get_symbol_name (s->symbol_name ()));
3003 0 : return 0;
3004 : }
3005 :
3006 0 : return s->total_count () * afdo_count_scale;
3007 0 : }
3008 :
3009 : /* Read AutoFDO profile and returns TRUE on success. */
3010 :
3011 : /* source profile format:
3012 :
3013 : GCOV_TAG_AFDO_FUNCTION: 4 bytes
3014 : LENGTH: 4 bytes
3015 : NUM_FUNCTIONS: 4 bytes
3016 : FUNCTION_INSTANCE_1
3017 : FUNCTION_INSTANCE_2
3018 : ...
3019 : FUNCTION_INSTANCE_N. */
3020 :
3021 : bool
3022 0 : autofdo_source_profile::read ()
3023 : {
3024 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION)
3025 : {
3026 0 : inform (UNKNOWN_LOCATION, "Not expected TAG.");
3027 0 : return false;
3028 : }
3029 :
3030 0 : gcc_checking_assert (!afdo_source_profile);
3031 0 : afdo_source_profile = this;
3032 :
3033 : /* Skip the length of the section. */
3034 0 : gcov_read_unsigned ();
3035 :
3036 : /* Read in the function/callsite profile, and store it in local
3037 : data structure. */
3038 0 : unsigned function_num = gcov_read_unsigned ();
3039 0 : for (unsigned i = 0; i < function_num; i++)
3040 : {
3041 0 : function_instance::function_instance_stack stack;
3042 0 : function_instance *s
3043 0 : = function_instance::read_function_instance (&stack);
3044 :
3045 0 : if (find_function_instance (s->get_descriptor ()) == nullptr)
3046 0 : add_function_instance (s);
3047 : else
3048 0 : fatal_error (UNKNOWN_LOCATION,
3049 : "auto-profile contains duplicated function instance %s",
3050 : afdo_string_table->get_symbol_name (s->symbol_name ()));
3051 0 : s->prop_timestamp ();
3052 0 : timestamp_info_map.insert({s->timestamp (), 0});
3053 0 : }
3054 :
3055 : /* timestamp_info_map is std::map with timestamp as key,
3056 : so it's already sorted in ascending order wrt timestamps.
3057 : This loop maps function with lowest timestamp to 1, and so on.
3058 : In afdo_annotate_cfg, node->tp_first_run is then set to corresponding
3059 : tp_first_run value. */
3060 :
3061 0 : int tp_first_run = 1;
3062 0 : for (auto &p : timestamp_info_map)
3063 0 : p.second = tp_first_run++;
3064 :
3065 0 : afdo_profile_info->sum_max = afdo_summary_info->max_count;
3066 : /* Scale up the profile, but leave some bits in case some counts gets
3067 : bigger than sum_max eventually. */
3068 0 : if (afdo_profile_info->sum_max)
3069 0 : afdo_count_scale
3070 0 : = MAX (((gcov_type)1 << (profile_count::n_bits - 10))
3071 : / afdo_profile_info->sum_max, 1);
3072 0 : afdo_profile_info->cutoff *= afdo_count_scale;
3073 : /* Derive the hot count threshold from the profile summary. */
3074 0 : afdo_hot_bb_threshold = afdo_summary_info->get_threshold_count (
3075 0 : param_hot_bb_count_ws_permille * 1000)
3076 0 : * afdo_count_scale;
3077 0 : set_hot_bb_threshold (afdo_hot_bb_threshold);
3078 0 : if (dump_file)
3079 0 : fprintf (dump_file,
3080 : "Max count in profile %" PRIu64 "\n"
3081 : "Setting scale %" PRIu64 "\n"
3082 : "Scaled max count %" PRIu64 "\n"
3083 : "Cutoff %" PRIu64 "\n"
3084 : "Unscaled hot count threshold %" PRIu64 "\n"
3085 : "Hot count threshold %" PRIu64 "\n\n",
3086 : (int64_t) afdo_profile_info->sum_max, (int64_t) afdo_count_scale,
3087 0 : (int64_t) (afdo_profile_info->sum_max * afdo_count_scale),
3088 0 : (int64_t) afdo_profile_info->cutoff,
3089 0 : (int64_t) afdo_summary_info->get_threshold_count (
3090 0 : param_hot_bb_count_ws_permille * 1000),
3091 : (int64_t) afdo_hot_bb_threshold);
3092 0 : afdo_profile_info->sum_max *= afdo_count_scale;
3093 0 : return true;
3094 : }
3095 :
3096 : /* Return the function_instance in the profile that correspond to the
3097 : inline STACK. */
3098 :
3099 : function_instance *
3100 0 : autofdo_source_profile::get_function_instance_by_inline_stack (
3101 : const inline_stack &stack) const
3102 : {
3103 0 : function_instance_descriptor descriptor (
3104 : afdo_string_table->get_filename_index (
3105 0 : get_normalized_path (DECL_SOURCE_FILE (stack[stack.length () - 1].decl))),
3106 0 : afdo_string_table->get_index_by_decl (stack[stack.length () - 1].decl));
3107 0 : function_instance *s = find_function_instance (descriptor);
3108 :
3109 0 : if (s == NULL)
3110 : {
3111 0 : if (dump_file)
3112 0 : fprintf (dump_file, "No offline instance for %s\n",
3113 0 : raw_symbol_name (stack[stack.length () - 1].decl));
3114 0 : return NULL;
3115 : }
3116 :
3117 0 : for (unsigned i = stack.length () - 1; i > 0; i--)
3118 : {
3119 0 : s = s->get_function_instance_by_decl (stack[i].afdo_loc,
3120 0 : stack[i - 1].decl,
3121 0 : stack[i].location);
3122 0 : if (s == NULL)
3123 : {
3124 : /* afdo inliner extends the stack by last entry with unknown
3125 : location while checking if function was inlined during train run.
3126 : We do not want to print diagnostics about every function
3127 : which is not inlined. */
3128 : if (s && dump_enabled_p () && stack[i].location != UNKNOWN_LOCATION)
3129 : dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
3130 : dump_user_location_t::from_location_t
3131 : (stack[i].location),
3132 : "auto-profile has no inlined function instance "
3133 : "for inlined call of %s at relative "
3134 : " location +%i, discriminator %i\n",
3135 : raw_symbol_name (stack[i - 1].decl),
3136 : stack[i].afdo_loc >> 16,
3137 : stack[i].afdo_loc & 65535);
3138 : return NULL;
3139 : }
3140 : }
3141 : return s;
3142 : }
3143 :
3144 : /* Find the matching function instance which has DESCRIPTOR as its
3145 : descriptor. If not found, also try checking if an instance exists with the
3146 : same name which has no associated filename. */
3147 :
3148 : autofdo_source_profile::name_function_instance_map::const_iterator
3149 0 : autofdo_source_profile::find_iter_for_function_instance (
3150 : function_instance_descriptor descriptor) const
3151 : {
3152 0 : auto it = map_.find (descriptor);
3153 :
3154 : /* Try searching for the symbol not having a filename if it isn't found. */
3155 0 : if (it == map_.end ())
3156 0 : it = map_.find (
3157 0 : function_instance_descriptor (string_table::unknown_filename,
3158 0 : (int) descriptor.symbol_name ()));
3159 0 : return it;
3160 : }
3161 :
3162 : /* Similar to the above, but return a pointer to the instance instead of an
3163 : iterator. */
3164 :
3165 : function_instance *
3166 0 : autofdo_source_profile::find_function_instance (
3167 : function_instance_descriptor descriptor) const
3168 : {
3169 0 : auto it = find_iter_for_function_instance (descriptor);
3170 0 : return it == map_.end () ? NULL : it->second;
3171 : }
3172 :
3173 : /* Remove a function instance from the map. Returns true if the entry was
3174 : actually deleted. */
3175 :
3176 : bool
3177 0 : autofdo_source_profile::remove_function_instance (function_instance *inst)
3178 : {
3179 0 : auto iter = find_iter_for_function_instance (inst->get_descriptor ());
3180 0 : if (iter != map_.end ())
3181 : {
3182 0 : map_.erase (iter);
3183 0 : return true;
3184 : }
3185 : return false;
3186 : }
3187 :
3188 : /* Module profile is only used by LIPO. Here we simply ignore it. */
3189 :
3190 : static void
3191 0 : fake_read_autofdo_module_profile ()
3192 : {
3193 : /* Read in the module info. */
3194 0 : gcov_read_unsigned ();
3195 :
3196 : /* Skip the length of the section. */
3197 0 : gcov_read_unsigned ();
3198 :
3199 : /* Read in the file name table. */
3200 0 : unsigned total_module_num = gcov_read_unsigned ();
3201 0 : gcc_assert (total_module_num == 0);
3202 0 : }
3203 :
3204 : /* Read data from profile data file. */
3205 :
3206 : static void
3207 0 : read_profile (void)
3208 : {
3209 0 : if (gcov_open (auto_profile_file, 1) == 0)
3210 : {
3211 0 : error ("cannot open profile file %s", auto_profile_file);
3212 0 : return;
3213 : }
3214 :
3215 0 : if (gcov_read_unsigned () != GCOV_DATA_MAGIC)
3216 : {
3217 0 : error ("AutoFDO profile magic number does not match");
3218 0 : return;
3219 : }
3220 :
3221 : /* Skip the version number. */
3222 0 : unsigned version = gcov_read_unsigned ();
3223 0 : if (version != AUTO_PROFILE_VERSION)
3224 : {
3225 0 : error ("AutoFDO profile version %u does not match %u",
3226 : version, AUTO_PROFILE_VERSION);
3227 0 : return;
3228 : }
3229 :
3230 : /* Skip the empty integer. */
3231 0 : gcov_read_unsigned ();
3232 :
3233 : /* summary_info. */
3234 0 : afdo_summary_info = new summary_info ();
3235 0 : if (!afdo_summary_info->read ())
3236 : {
3237 0 : error ("cannot read summary information from %s", auto_profile_file);
3238 0 : return;
3239 : }
3240 :
3241 : /* string_table. */
3242 0 : afdo_string_table = new string_table ();
3243 0 : if (!afdo_string_table->read ())
3244 : {
3245 0 : error ("cannot read string table from %s", auto_profile_file);
3246 0 : return;
3247 : }
3248 :
3249 : /* autofdo_source_profile. */
3250 0 : afdo_source_profile = autofdo_source_profile::create ();
3251 0 : if (afdo_source_profile == NULL
3252 0 : || gcov_is_error ())
3253 : {
3254 0 : error ("cannot read function profile from %s", auto_profile_file);
3255 0 : delete afdo_source_profile;
3256 0 : afdo_source_profile = NULL;
3257 0 : return;
3258 : }
3259 :
3260 : /* autofdo_module_profile. */
3261 0 : fake_read_autofdo_module_profile ();
3262 0 : if (gcov_is_error ())
3263 : {
3264 0 : error ("cannot read module profile from %s", auto_profile_file);
3265 0 : return;
3266 : }
3267 : }
3268 :
3269 : /* From AutoFDO profiles, find values inside STMT for that we want to measure
3270 : histograms for indirect-call optimization.
3271 :
3272 : This function is actually served for 2 purposes:
3273 : * before annotation, we need to mark histogram, promote and inline
3274 : * after annotation, we just need to mark, and let follow-up logic to
3275 : decide if it needs to promote and inline. */
3276 :
3277 : static bool
3278 0 : afdo_indirect_call (gcall *stmt, const icall_target_map &map,
3279 : bool transform, cgraph_edge *indirect_edge)
3280 : {
3281 0 : tree callee;
3282 :
3283 0 : if (map.size () == 0)
3284 : {
3285 0 : if (dump_file)
3286 0 : fprintf (dump_file, "No targets found\n");
3287 0 : return false;
3288 : }
3289 0 : if (!stmt)
3290 : {
3291 0 : if (dump_file)
3292 0 : fprintf (dump_file, "No call statement\n");
3293 0 : return false;
3294 : }
3295 0 : if (gimple_call_internal_p (stmt))
3296 : {
3297 0 : if (dump_file)
3298 0 : fprintf (dump_file, "Internal call\n");
3299 0 : return false;
3300 : }
3301 0 : if (gimple_call_fndecl (stmt) != NULL_TREE)
3302 : {
3303 0 : if (dump_file)
3304 0 : fprintf (dump_file, "Call is already direct\n");
3305 0 : return false;
3306 : }
3307 :
3308 0 : gcov_type total = 0;
3309 0 : icall_target_map::const_iterator max_iter = map.end ();
3310 :
3311 0 : for (icall_target_map::const_iterator iter = map.begin ();
3312 0 : iter != map.end (); ++iter)
3313 : {
3314 0 : total += iter->second;
3315 0 : if (max_iter == map.end () || max_iter->second < iter->second)
3316 : max_iter = iter;
3317 : }
3318 0 : total *= afdo_count_scale;
3319 0 : struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
3320 0 : get_identifier (afdo_string_table->get_symbol_name (max_iter->first)));
3321 0 : if (direct_call == NULL)
3322 : {
3323 0 : if (dump_file)
3324 0 : fprintf (dump_file, "Failed to find cgraph node for %s\n",
3325 0 : afdo_string_table->get_symbol_name (max_iter->first));
3326 0 : return false;
3327 : }
3328 :
3329 0 : callee = gimple_call_fn (stmt);
3330 :
3331 0 : if (!transform)
3332 : {
3333 0 : if (!direct_call->profile_id)
3334 : {
3335 0 : if (dump_file)
3336 0 : fprintf (dump_file, "No profile id\n");
3337 0 : return false;
3338 : }
3339 0 : histogram_value hist = gimple_alloc_histogram_value (
3340 : cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
3341 0 : hist->n_counters = 4;
3342 0 : hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
3343 0 : gimple_add_histogram_value (cfun, stmt, hist);
3344 :
3345 : /* Total counter */
3346 0 : hist->hvalue.counters[0] = total;
3347 : /* Number of value/counter pairs */
3348 0 : hist->hvalue.counters[1] = 1;
3349 : /* Value */
3350 0 : hist->hvalue.counters[2] = direct_call->profile_id;
3351 : /* Counter */
3352 0 : hist->hvalue.counters[3] = max_iter->second * afdo_count_scale;
3353 :
3354 0 : if (!direct_call->profile_id)
3355 : {
3356 0 : if (dump_file)
3357 0 : fprintf (dump_file, "Histogram attached\n");
3358 0 : return false;
3359 : }
3360 : return false;
3361 : }
3362 :
3363 0 : if (dump_file)
3364 : {
3365 0 : fprintf (dump_file, "Indirect call -> direct call ");
3366 0 : print_generic_expr (dump_file, callee, TDF_SLIM);
3367 0 : fprintf (dump_file, " => ");
3368 0 : print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
3369 : }
3370 :
3371 0 : if (!direct_call->definition)
3372 : {
3373 0 : if (dump_file)
3374 0 : fprintf (dump_file, " no definition available\n");
3375 0 : return false;
3376 : }
3377 :
3378 0 : if (dump_file)
3379 : {
3380 0 : fprintf (dump_file, " transformation on insn ");
3381 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
3382 0 : fprintf (dump_file, "\n");
3383 : }
3384 :
3385 0 : indirect_edge->make_speculative
3386 0 : (direct_call,
3387 0 : gimple_bb (stmt)->count.apply_scale (99, 100));
3388 0 : return true;
3389 : }
3390 :
3391 : /* From AutoFDO profiles, find values inside STMT for that we want to measure
3392 : histograms and adds them to list VALUES. */
3393 :
3394 : static bool
3395 0 : afdo_vpt (gcall *gs, const icall_target_map &map,
3396 : bool transform, cgraph_edge *indirect_edge)
3397 : {
3398 0 : return afdo_indirect_call (gs, map, transform, indirect_edge);
3399 : }
3400 :
3401 : typedef std::set<basic_block> bb_set;
3402 :
3403 : static bool
3404 0 : is_bb_annotated (const basic_block bb, const bb_set &annotated)
3405 : {
3406 0 : if (annotated.find (bb) != annotated.end ())
3407 : {
3408 0 : gcc_checking_assert (bb->count.quality () == AFDO
3409 : || !bb->count.nonzero_p ());
3410 : return true;
3411 : }
3412 0 : gcc_checking_assert (bb->count.quality () != AFDO
3413 : || !bb->count.nonzero_p ());
3414 : return false;
3415 : }
3416 :
3417 : static void
3418 0 : set_bb_annotated (basic_block bb, bb_set *annotated)
3419 : {
3420 0 : gcc_checking_assert (bb->count.quality () == AFDO
3421 : || !bb->count.nonzero_p ());
3422 0 : annotated->insert (bb);
3423 0 : }
3424 :
3425 : /* Update COUNT by known autofdo count C. */
3426 : static void
3427 0 : update_count_by_afdo_count (profile_count *count, gcov_type c)
3428 : {
3429 0 : if (c)
3430 0 : *count = profile_count::from_gcov_type (c).afdo ();
3431 : /* In case we have guessed profile which is already zero, preserve
3432 : quality info. */
3433 0 : else if (count->nonzero_p ()
3434 0 : || count->quality () == GUESSED
3435 0 : || count->quality () == GUESSED_LOCAL)
3436 0 : *count = profile_count::zero ().afdo ();
3437 0 : }
3438 :
3439 : /* Update COUNT by known autofdo count C. */
3440 : static void
3441 0 : update_count_by_afdo_count (profile_count *count, profile_count c)
3442 : {
3443 0 : if (c.nonzero_p ())
3444 0 : *count = c;
3445 : /* In case we have guessed profile which is already zero, preserve
3446 : quality info. */
3447 0 : else if (count->nonzero_p ()
3448 0 : || count->quality () < c.quality ())
3449 0 : *count = c;
3450 0 : }
3451 :
3452 : /* Try to determine unscaled count of edge E.
3453 : Return -1 if nothing is known. */
3454 :
3455 : static gcov_type
3456 0 : afdo_unscaled_edge_count (edge e)
3457 : {
3458 0 : gcov_type max_count = -1;
3459 0 : basic_block bb_succ = e->dest;
3460 0 : count_info info;
3461 0 : if (afdo_source_profile->get_count_info (e->goto_locus, &info))
3462 : {
3463 0 : if (info.count > max_count)
3464 : max_count = info.count;
3465 0 : if (dump_file && info.count)
3466 : {
3467 0 : fprintf (dump_file,
3468 : " goto location of edge %i->%i with count %" PRIu64"\n",
3469 0 : e->src->index, e->dest->index, (int64_t)info.count);
3470 : }
3471 : }
3472 0 : for (gphi_iterator gpi = gsi_start_phis (bb_succ);
3473 0 : !gsi_end_p (gpi); gsi_next (&gpi))
3474 : {
3475 0 : gphi *phi = gpi.phi ();
3476 0 : location_t phi_loc
3477 0 : = gimple_phi_arg_location_from_edge (phi, e);
3478 0 : if (afdo_source_profile->get_count_info (phi_loc, &info))
3479 : {
3480 0 : if (info.count > max_count)
3481 : max_count = info.count;
3482 0 : if (dump_file && info.count)
3483 : {
3484 0 : fprintf (dump_file,
3485 : " phi op of edge %i->%i with count %" PRIu64": ",
3486 0 : e->src->index, e->dest->index, (int64_t)info.count);
3487 0 : print_gimple_stmt (dump_file, phi, 0, TDF_SLIM);
3488 : }
3489 : }
3490 : }
3491 0 : return max_count;
3492 0 : }
3493 :
3494 : /* For a given BB, set its execution count. Attach value profile if a stmt
3495 : is not in PROMOTED, because we only want to promote an indirect call once.
3496 : Return TRUE if BB is annotated. */
3497 :
3498 : static bool
3499 0 : afdo_set_bb_count (basic_block bb, hash_set <basic_block> &zero_bbs)
3500 : {
3501 0 : gimple_stmt_iterator gsi;
3502 0 : gcov_type max_count = -1;
3503 0 : if (dump_file)
3504 0 : fprintf (dump_file, " Looking up AFDO count of bb %i\n", bb->index);
3505 :
3506 0 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3507 : {
3508 0 : count_info info;
3509 0 : gimple *stmt = gsi_stmt (gsi);
3510 0 : if (!stmt_loc_used_by_debug_info (stmt))
3511 0 : continue;
3512 0 : if (afdo_source_profile->get_count_info (stmt, &info))
3513 : {
3514 0 : if (info.count > max_count)
3515 : max_count = info.count;
3516 0 : if (dump_file)
3517 : {
3518 0 : fprintf (dump_file, " count %" PRIu64 " in stmt: ",
3519 : (int64_t)info.count);
3520 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
3521 : }
3522 0 : gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
3523 : /* TODO; if inlined early and indirect call was not optimized out,
3524 : we will end up speculating again. Early inliner should remove
3525 : all targets for edges it speculated into safely. */
3526 0 : if (call
3527 0 : && info.targets.size () > 0)
3528 0 : afdo_vpt (call, info.targets, false, NULL);
3529 : }
3530 0 : }
3531 :
3532 0 : if (max_count == -1 && single_succ_p (bb))
3533 0 : max_count = afdo_unscaled_edge_count (single_succ_edge (bb));
3534 :
3535 0 : if (max_count == -1)
3536 : return false;
3537 :
3538 0 : if (max_count)
3539 : {
3540 0 : update_count_by_afdo_count (&bb->count, max_count * afdo_count_scale);
3541 0 : if (dump_file)
3542 0 : fprintf (dump_file,
3543 : " Annotated bb %i with count %" PRId64
3544 : ", scaled to %" PRId64 "\n",
3545 : bb->index, (int64_t)max_count,
3546 0 : (int64_t)(max_count * afdo_count_scale));
3547 0 : return true;
3548 : }
3549 : else
3550 : {
3551 0 : if (dump_file)
3552 0 : fprintf (dump_file,
3553 : " bb %i has statements with 0 count\n", bb->index);
3554 0 : zero_bbs.add (bb);
3555 : }
3556 0 : return false;
3557 : }
3558 :
3559 : /* BB1 and BB2 are in an equivalent class iff:
3560 : 1. BB1 dominates BB2.
3561 : 2. BB2 post-dominates BB1.
3562 : 3. BB1 and BB2 are in the same loop nest.
3563 : This function finds the equivalent class for each basic block, and
3564 : stores a pointer to the first BB in its equivalent class. Meanwhile,
3565 : set bb counts for the same equivalent class to be idenical. Update
3566 : ANNOTATED_BB for the first BB in its equivalent class. */
3567 :
3568 : static void
3569 0 : afdo_find_equiv_class (bb_set *annotated_bb)
3570 : {
3571 0 : basic_block bb;
3572 :
3573 0 : FOR_ALL_BB_FN (bb, cfun)
3574 0 : bb->aux = NULL;
3575 :
3576 0 : FOR_ALL_BB_FN (bb, cfun)
3577 : {
3578 0 : if (bb->aux != NULL)
3579 0 : continue;
3580 0 : bb->aux = bb;
3581 0 : for (basic_block bb1 : get_dominated_by (CDI_DOMINATORS, bb))
3582 0 : if (bb1->aux == NULL && dominated_by_p (CDI_POST_DOMINATORS, bb, bb1)
3583 0 : && bb1->loop_father == bb->loop_father)
3584 : {
3585 0 : bb1->aux = bb;
3586 0 : if (is_bb_annotated (bb1, *annotated_bb)
3587 0 : && (!is_bb_annotated (bb, *annotated_bb)
3588 0 : || bb1->count > bb->count))
3589 : {
3590 0 : if (dump_file)
3591 : {
3592 0 : fprintf (dump_file,
3593 : " Copying count of bb %i to bb %i; count is:",
3594 : bb1->index,
3595 : bb->index);
3596 0 : bb1->count.dump (dump_file);
3597 0 : fprintf (dump_file, "\n");
3598 : }
3599 0 : update_count_by_afdo_count (&bb->count, bb1->count);
3600 0 : set_bb_annotated (bb, annotated_bb);
3601 : }
3602 0 : }
3603 :
3604 0 : for (basic_block bb1 : get_dominated_by (CDI_POST_DOMINATORS, bb))
3605 0 : if (bb1->aux == NULL && dominated_by_p (CDI_DOMINATORS, bb, bb1)
3606 0 : && bb1->loop_father == bb->loop_father)
3607 : {
3608 0 : bb1->aux = bb;
3609 0 : if (is_bb_annotated (bb1, *annotated_bb)
3610 0 : && (!is_bb_annotated (bb, *annotated_bb)
3611 0 : || bb1->count > bb->count))
3612 : {
3613 0 : if (dump_file)
3614 : {
3615 0 : fprintf (dump_file,
3616 : " Copying count of bb %i to bb %i; count is:",
3617 : bb1->index,
3618 : bb->index);
3619 0 : bb1->count.dump (dump_file);
3620 0 : fprintf (dump_file, "\n");
3621 : }
3622 0 : update_count_by_afdo_count (&bb->count, bb1->count);
3623 0 : set_bb_annotated (bb, annotated_bb);
3624 : }
3625 0 : }
3626 : }
3627 0 : }
3628 :
3629 : /* If a basic block's count is known, and only one of its in/out edges' count
3630 : is unknown, its count can be calculated. Meanwhile, if all of the in/out
3631 : edges' counts are known, then the basic block's unknown count can also be
3632 : calculated. Also, if a block has a single predecessor or successor, the block's
3633 : count can be propagated to that predecessor or successor.
3634 : IS_SUCC is true if out edges of a basic blocks are examined.
3635 : Update ANNOTATED_BB accordingly.
3636 : Return TRUE if any basic block/edge count is changed. */
3637 :
3638 : static bool
3639 0 : afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
3640 : {
3641 0 : basic_block bb;
3642 0 : bool changed = false;
3643 :
3644 0 : FOR_EACH_BB_FN (bb, cfun)
3645 : {
3646 0 : edge e, unknown_edge = NULL;
3647 0 : edge_iterator ei;
3648 0 : int num_unknown_edges = 0;
3649 0 : int num_edges = 0;
3650 0 : profile_count total_known_count = profile_count::zero ().afdo ();
3651 :
3652 0 : FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
3653 : {
3654 0 : gcc_assert (AFDO_EINFO (e) != NULL);
3655 0 : if (! AFDO_EINFO (e)->is_annotated ())
3656 0 : num_unknown_edges++, unknown_edge = e;
3657 : else
3658 0 : total_known_count += AFDO_EINFO (e)->get_count ();
3659 0 : num_edges++;
3660 : }
3661 0 : if (dump_file)
3662 : {
3663 0 : fprintf (dump_file, "bb %i %s propagating %s edges %i, "
3664 : "unknown edges %i, known count ",
3665 : bb->index,
3666 0 : is_bb_annotated (bb, *annotated_bb) ? "(annotated)" : "",
3667 : is_succ ? "successors" : "predecessors", num_edges,
3668 : num_unknown_edges);
3669 0 : total_known_count.dump (dump_file);
3670 0 : fprintf (dump_file, " bb count ");
3671 0 : bb->count.dump (dump_file);
3672 0 : fprintf (dump_file, "\n");
3673 : }
3674 :
3675 : /* Be careful not to annotate block with no successor in special cases. */
3676 0 : if (num_unknown_edges == 0 && num_edges
3677 0 : && !is_bb_annotated (bb, *annotated_bb))
3678 : {
3679 0 : if (dump_file)
3680 : {
3681 0 : fprintf (dump_file, " Annotating bb %i with count ", bb->index);
3682 0 : total_known_count.dump (dump_file);
3683 0 : fprintf (dump_file, "\n");
3684 : }
3685 0 : update_count_by_afdo_count (&bb->count, total_known_count);
3686 0 : set_bb_annotated (bb, annotated_bb);
3687 0 : changed = true;
3688 : }
3689 0 : else if (is_bb_annotated (bb, *annotated_bb)
3690 : /* We do not want to consider 0 (afdo) > 0 (precise) */
3691 0 : && total_known_count.nonzero_p ()
3692 0 : && bb->count < total_known_count)
3693 : {
3694 0 : if (dump_file)
3695 : {
3696 0 : fprintf (dump_file, " Increasing bb %i count from ",
3697 : bb->index);
3698 0 : bb->count.dump (dump_file);
3699 0 : fprintf (dump_file, " to ");
3700 0 : total_known_count.dump (dump_file);
3701 0 : fprintf (dump_file, " hoping to mitigate afdo inconsistency\n");
3702 : }
3703 0 : bb->count = total_known_count;
3704 0 : changed = true;
3705 : }
3706 0 : else if (num_unknown_edges == 1 && is_bb_annotated (bb, *annotated_bb))
3707 : {
3708 0 : if (bb->count > total_known_count)
3709 : {
3710 0 : profile_count new_count = bb->count - total_known_count;
3711 0 : AFDO_EINFO (unknown_edge)->set_count (new_count);
3712 : }
3713 : else
3714 0 : AFDO_EINFO (unknown_edge)->set_count
3715 0 : (profile_count::zero ().afdo ());
3716 0 : if (dump_file)
3717 : {
3718 0 : fprintf (dump_file, " Annotated edge %i->%i with count ",
3719 0 : unknown_edge->src->index, unknown_edge->dest->index);
3720 0 : AFDO_EINFO (unknown_edge)->get_count ().dump (dump_file);
3721 0 : fprintf (dump_file, "\n");
3722 : }
3723 0 : AFDO_EINFO (unknown_edge)->set_annotated ();
3724 0 : changed = true;
3725 : }
3726 0 : else if (num_unknown_edges > 1
3727 0 : && is_bb_annotated (bb, *annotated_bb)
3728 0 : && (total_known_count >= bb->count || !bb->count.nonzero_p ()))
3729 : {
3730 0 : FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
3731 : {
3732 0 : gcc_assert (AFDO_EINFO (e) != NULL);
3733 0 : if (! AFDO_EINFO (e)->is_annotated ())
3734 : {
3735 0 : AFDO_EINFO (e)->set_count
3736 0 : (profile_count::zero ().afdo ());
3737 0 : AFDO_EINFO (e)->set_annotated ();
3738 0 : if (dump_file)
3739 : {
3740 0 : fprintf (dump_file, " Annotated edge %i->%i with count ",
3741 0 : e->src->index, e->dest->index);
3742 0 : AFDO_EINFO (unknown_edge)->get_count ().dump (dump_file);
3743 0 : fprintf (dump_file, "\n");
3744 : }
3745 : }
3746 : }
3747 : }
3748 0 : else if (num_unknown_edges == 0
3749 0 : && is_bb_annotated (bb, *annotated_bb)
3750 0 : && (is_succ ? single_succ_p (bb) : single_pred_p (bb)))
3751 : {
3752 0 : edge e = is_succ ? single_succ_edge (bb) : single_pred_edge (bb);
3753 0 : if (AFDO_EINFO (e)->is_annotated ()
3754 0 : && AFDO_EINFO (e)->get_count () < bb->count)
3755 : {
3756 0 : if (dump_file)
3757 : {
3758 0 : fprintf (dump_file, " Increasing edge %i->%i count from ",
3759 0 : e->src->index, e->dest->index);
3760 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
3761 0 : fprintf (dump_file, " to ");
3762 0 : bb->count.dump (dump_file);
3763 0 : fprintf (dump_file, " hoping to mitigate afdo inconsistency\n");
3764 : }
3765 0 : AFDO_EINFO (e)->set_count (bb->count);
3766 0 : changed = true;
3767 : }
3768 : }
3769 : }
3770 0 : return changed;
3771 : }
3772 :
3773 : /* Special propagation for circuit expressions. Because GCC translates
3774 : control flow into data flow for circuit expressions. E.g.
3775 : BB1:
3776 : if (a && b)
3777 : BB2
3778 : else
3779 : BB3
3780 :
3781 : will be translated into:
3782 :
3783 : BB1:
3784 : if (a)
3785 : goto BB.t1
3786 : else
3787 : goto BB.t3
3788 : BB.t1:
3789 : if (b)
3790 : goto BB.t2
3791 : else
3792 : goto BB.t3
3793 : BB.t2:
3794 : goto BB.t3
3795 : BB.t3:
3796 : tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
3797 : if (tmp)
3798 : goto BB2
3799 : else
3800 : goto BB3
3801 :
3802 : In this case, we need to propagate through PHI to determine the edge
3803 : count of BB1->BB.t1, BB.t1->BB.t2. */
3804 :
3805 : static void
3806 0 : afdo_propagate_circuit (const bb_set &annotated_bb)
3807 : {
3808 0 : basic_block bb;
3809 0 : FOR_ALL_BB_FN (bb, cfun)
3810 : {
3811 0 : gimple *def_stmt;
3812 0 : tree cmp_rhs, cmp_lhs;
3813 0 : gimple *cmp_stmt = last_nondebug_stmt (bb);
3814 0 : edge e;
3815 0 : edge_iterator ei;
3816 :
3817 0 : if (!cmp_stmt || gimple_code (cmp_stmt) != GIMPLE_COND)
3818 0 : continue;
3819 0 : cmp_rhs = gimple_cond_rhs (cmp_stmt);
3820 0 : cmp_lhs = gimple_cond_lhs (cmp_stmt);
3821 0 : if (!TREE_CONSTANT (cmp_rhs)
3822 0 : || !(integer_zerop (cmp_rhs) || integer_onep (cmp_rhs)))
3823 0 : continue;
3824 0 : if (TREE_CODE (cmp_lhs) != SSA_NAME)
3825 0 : continue;
3826 0 : if (!is_bb_annotated (bb, annotated_bb))
3827 0 : continue;
3828 0 : def_stmt = SSA_NAME_DEF_STMT (cmp_lhs);
3829 0 : while (def_stmt && gimple_code (def_stmt) == GIMPLE_ASSIGN
3830 0 : && gimple_assign_single_p (def_stmt)
3831 0 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
3832 0 : def_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
3833 0 : if (!def_stmt)
3834 0 : continue;
3835 0 : gphi *phi_stmt = dyn_cast <gphi *> (def_stmt);
3836 0 : if (!phi_stmt)
3837 0 : continue;
3838 0 : FOR_EACH_EDGE (e, ei, bb->succs)
3839 : {
3840 0 : unsigned i, total = 0;
3841 0 : edge only_one;
3842 0 : bool check_value_one = (((integer_onep (cmp_rhs))
3843 0 : ^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
3844 0 : ^ ((e->flags & EDGE_TRUE_VALUE) != 0));
3845 0 : if (! AFDO_EINFO (e)->is_annotated ())
3846 0 : continue;
3847 0 : for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
3848 : {
3849 0 : tree val = gimple_phi_arg_def (phi_stmt, i);
3850 0 : edge ep = gimple_phi_arg_edge (phi_stmt, i);
3851 :
3852 0 : if (!TREE_CONSTANT (val)
3853 0 : || !(integer_zerop (val) || integer_onep (val)))
3854 0 : continue;
3855 0 : if (check_value_one ^ integer_onep (val))
3856 0 : continue;
3857 0 : total++;
3858 0 : only_one = ep;
3859 0 : if (! (AFDO_EINFO (e)->get_count ()).nonzero_p ()
3860 0 : && ! AFDO_EINFO (ep)->is_annotated ())
3861 : {
3862 0 : AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ());
3863 0 : AFDO_EINFO (ep)->set_annotated ();
3864 : }
3865 : }
3866 0 : if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ())
3867 : {
3868 0 : AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ());
3869 0 : AFDO_EINFO (only_one)->set_annotated ();
3870 : }
3871 : }
3872 : }
3873 0 : }
3874 :
3875 : /* Propagate the basic block count and edge count on the control flow
3876 : graph. We do the propagation iteratively until stabilize. */
3877 :
3878 : static void
3879 0 : afdo_propagate (bb_set *annotated_bb)
3880 : {
3881 0 : bool changed = true;
3882 0 : int i = 0;
3883 :
3884 0 : basic_block bb;
3885 0 : FOR_ALL_BB_FN (bb, cfun)
3886 0 : if (!is_bb_annotated (bb, *annotated_bb)
3887 0 : && is_bb_annotated ((basic_block)bb->aux, *annotated_bb))
3888 : {
3889 0 : update_count_by_afdo_count (&bb->count, ((basic_block)bb->aux)->count);
3890 0 : set_bb_annotated (bb, annotated_bb);
3891 0 : if (dump_file)
3892 : {
3893 0 : fprintf (dump_file,
3894 : " Copying count of bb %i to bb %i; count is:",
3895 0 : ((basic_block)bb->aux)->index,
3896 : bb->index);
3897 0 : bb->count.dump (dump_file);
3898 0 : fprintf (dump_file, "\n");
3899 : }
3900 : }
3901 :
3902 0 : while (changed && i++ < 100)
3903 : {
3904 0 : changed = false;
3905 :
3906 0 : if (afdo_propagate_edge (true, annotated_bb))
3907 : changed = true;
3908 0 : if (afdo_propagate_edge (false, annotated_bb))
3909 0 : changed = true;
3910 0 : afdo_propagate_circuit (*annotated_bb);
3911 : }
3912 0 : if (dump_file)
3913 0 : fprintf (dump_file, "Propagation took %i iterations %s\n",
3914 : i, changed ? "; iteration limit reached\n" : "");
3915 0 : }
3916 :
3917 : /* qsort comparator of sreals. */
3918 : static int
3919 0 : cmp (const void *a, const void *b)
3920 : {
3921 0 : if (*(const sreal *)a < *(const sreal *)b)
3922 : return 1;
3923 0 : if (*(const sreal *)a > *(const sreal *)b)
3924 0 : return -1;
3925 : return 0;
3926 : }
3927 :
3928 : /* To scale a connected component of graph we collect desired scales of
3929 : basic blocks on the boundary and then compute a robust average. */
3930 :
3931 : struct scale
3932 : {
3933 : /* Scale desired. */
3934 : sreal scale;
3935 : /* Weight for averaging computed from execution count of the edge
3936 : scale originates from. */
3937 : uint64_t weight;
3938 : };
3939 :
3940 : /* Add scale ORIG/ANNOTATED to SCALES. */
3941 :
3942 : static void
3943 0 : add_scale (vec <scale> *scales, profile_count annotated, profile_count orig)
3944 : {
3945 0 : if (dump_file)
3946 : {
3947 0 : orig.dump (dump_file);
3948 0 : fprintf (dump_file, " should be ");
3949 0 : annotated.dump (dump_file);
3950 0 : fprintf (dump_file, "\n");
3951 : }
3952 0 : if (orig.nonzero_p ())
3953 : {
3954 0 : sreal scale
3955 0 : = annotated.guessed_local ()
3956 0 : .to_sreal_scale (orig);
3957 0 : if (dump_file)
3958 0 : fprintf (dump_file, " adding scale %.16f, weight %" PRId64 "\n",
3959 0 : scale.to_double (), annotated.value () + 1);
3960 0 : scales->safe_push ({scale, annotated.value () + 1});
3961 : }
3962 0 : }
3963 :
3964 : /* Scale counts of all basic blocks in BBS by SCALE and convert them to
3965 : IPA quality. */
3966 :
3967 : static void
3968 0 : scale_bbs (const vec <basic_block> &bbs, sreal scale)
3969 : {
3970 0 : if (dump_file)
3971 0 : fprintf (dump_file, " Scaling by %.16f\n", scale.to_double ());
3972 0 : for (basic_block b : bbs)
3973 0 : if (!(b->count == profile_count::zero ())
3974 0 : && b->count.initialized_p ())
3975 : {
3976 0 : profile_count o = b->count;
3977 0 : b->count = b->count.force_guessed () * scale;
3978 :
3979 : /* If we scaled to 0, make it auto-fdo since that is treated
3980 : less agressively. */
3981 0 : if (!b->count.nonzero_p () && o.nonzero_p ())
3982 0 : b->count = profile_count::zero ().afdo ();
3983 0 : if (dump_file)
3984 : {
3985 0 : fprintf (dump_file, " bb %i count updated ", b->index);
3986 0 : o.dump (dump_file);
3987 0 : fprintf (dump_file, " -> ");
3988 0 : b->count.dump (dump_file);
3989 0 : fprintf (dump_file, "\n");
3990 : }
3991 : }
3992 0 : }
3993 :
3994 : /* Determine scaling factor by taking robust average of SCALES
3995 : and taking into account limits.
3996 : MAX_COUNT is maximal guessed count to be scaled while MAC_COUNT_IN_FN
3997 : is maximal count in function determined by auto-fdo. */
3998 :
3999 : sreal
4000 0 : determine_scale (vec <scale> *scales, profile_count max_count,
4001 : profile_count max_count_in_fn)
4002 : {
4003 0 : scales->qsort (cmp);
4004 :
4005 0 : uint64_t overall_weight = 0;
4006 0 : for (scale &e : *scales)
4007 0 : overall_weight += e.weight;
4008 :
4009 0 : uint64_t cummulated = 0, weight_sum = 0;
4010 0 : sreal scale_sum = 0;
4011 0 : for (scale &e : *scales)
4012 : {
4013 0 : uint64_t prev = cummulated;
4014 0 : cummulated += e.weight;
4015 0 : if (cummulated >= overall_weight / 4
4016 0 : && prev <= 3 * overall_weight / 4)
4017 : {
4018 0 : scale_sum += e.scale * e.weight;
4019 0 : weight_sum += e.weight;
4020 0 : if (dump_file)
4021 0 : fprintf (dump_file, " accounting scale %.16f, weight %" PRId64 "\n",
4022 : e.scale.to_double (), e.weight);
4023 : }
4024 0 : else if (dump_file)
4025 0 : fprintf (dump_file, " ignoring scale %.16f, weight %" PRId64 "\n",
4026 : e.scale.to_double (), e.weight);
4027 : }
4028 0 : sreal scale = scale_sum / (sreal)weight_sum;
4029 :
4030 : /* Avoid scaled regions to have very large counts.
4031 : Otherwise they may dominate ipa-profile's histogram computing cutoff
4032 : of hot basic blocks. */
4033 0 : if (max_count * scale > max_count_in_fn.guessed_local ().apply_scale (128, 1))
4034 : {
4035 0 : if (dump_file)
4036 : {
4037 0 : fprintf (dump_file, "Scaling by %.16f produces max count ",
4038 : scale.to_double ());
4039 0 : (max_count * scale).dump (dump_file);
4040 0 : fprintf (dump_file, " that exceeds max count in fn ");
4041 0 : max_count_in_fn.dump (dump_file);
4042 0 : fprintf (dump_file, "; capping\n");
4043 : }
4044 0 : scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count);
4045 : }
4046 0 : return scale;
4047 : }
4048 :
4049 : /* Scale profile of the whole function to approximately match auto-profile. */
4050 :
4051 : bool
4052 0 : scale_bb_profile ()
4053 : {
4054 0 : const function_instance *s
4055 : = afdo_source_profile->get_function_instance_by_decl
4056 0 : (current_function_decl);
4057 :
4058 : /* In the first pass only store non-zero counts. */
4059 0 : gcov_type head_count = s->head_count () * autofdo::afdo_count_scale;
4060 0 : hash_set <basic_block> zero_bbs;
4061 0 : auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
4062 0 : auto_vec <scale, 20> scales;
4063 0 : basic_block bb;
4064 0 : profile_count max_count = profile_count::zero ();
4065 0 : profile_count max_count_in_fn = profile_count::zero ();
4066 0 : bbs.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun));
4067 0 : bbs.quick_push (EXIT_BLOCK_PTR_FOR_FN (cfun));
4068 0 : if (head_count > 0)
4069 : {
4070 0 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4071 0 : max_count = entry_count;
4072 0 : update_count_by_afdo_count (&entry_count, head_count);
4073 0 : max_count_in_fn = entry_count;
4074 0 : add_scale (&scales, entry_count, ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
4075 : }
4076 0 : FOR_EACH_BB_FN (bb, cfun)
4077 : {
4078 0 : profile_count cnt = bb->count;
4079 0 : bbs.safe_push (bb);
4080 0 : max_count = profile_count::max_prefer_initialized (max_count, cnt);
4081 0 : if (afdo_set_bb_count (bb, zero_bbs))
4082 : {
4083 0 : std::swap (cnt, bb->count);
4084 0 : max_count_in_fn
4085 0 : = profile_count::max_prefer_initialized (max_count_in_fn, cnt);
4086 0 : add_scale (&scales, cnt, bb->count);
4087 : }
4088 : }
4089 0 : if (scales.length ())
4090 : {
4091 0 : sreal scale = determine_scale (&scales, max_count, max_count_in_fn);
4092 0 : scale_bbs (bbs, scale);
4093 0 : return true;
4094 : }
4095 : return false;
4096 0 : }
4097 :
4098 : /* In case given basic block was fully optimized out, AutoFDO
4099 : will have no data about it. In this case try to preserve static profile.
4100 : Identify connected components (in undirected form of CFG) which has
4101 : no annotations at all. Look at thir boundaries and try to determine
4102 : scaling factor and scale. */
4103 :
4104 : void
4105 0 : afdo_adjust_guessed_profile (bb_set *annotated_bb)
4106 : {
4107 : /* Basic blocks of connected component currently processed. */
4108 0 : auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
4109 : /* Scale factors found. */
4110 0 : auto_vec <scale, 20> scales;
4111 0 : auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun));
4112 :
4113 0 : basic_block seed_bb;
4114 0 : unsigned int component_id = 1;
4115 :
4116 : /* Map from basic block to its component.
4117 : 0 is used for univisited BBs,
4118 : 1 means that BB is annotated,
4119 : >=2 is an id of the component BB belongs to. */
4120 0 : auto_vec <unsigned int, 20> component;
4121 0 : component.safe_grow (last_basic_block_for_fn (cfun));
4122 0 : profile_count max_count_in_fn = profile_count::zero ();
4123 0 : FOR_ALL_BB_FN (seed_bb, cfun)
4124 0 : if (is_bb_annotated (seed_bb, *annotated_bb))
4125 : {
4126 0 : component[seed_bb->index] = 1;
4127 0 : max_count_in_fn
4128 0 : = profile_count::max_prefer_initialized (max_count_in_fn, seed_bb->count);
4129 : }
4130 : else
4131 0 : component[seed_bb->index] = 0;
4132 0 : FOR_ALL_BB_FN (seed_bb, cfun)
4133 0 : if (!component[seed_bb->index])
4134 : {
4135 0 : stack.quick_push (seed_bb);
4136 0 : component_id++;
4137 0 : bbs.truncate (0);
4138 0 : scales.truncate (0);
4139 0 : component[seed_bb->index] = component_id;
4140 0 : profile_count max_count = profile_count::zero ();
4141 :
4142 : /* Identify connected component starting in BB. */
4143 0 : if (dump_file)
4144 0 : fprintf (dump_file, "Starting connected component in bb %i\n",
4145 : seed_bb->index);
4146 0 : do
4147 : {
4148 0 : basic_block b = stack.pop ();
4149 :
4150 0 : bbs.quick_push (b);
4151 0 : max_count = profile_count::max_prefer_initialized (max_count, b->count);
4152 :
4153 0 : for (edge e: b->preds)
4154 0 : if (!component[e->src->index])
4155 : {
4156 0 : stack.quick_push (e->src);
4157 0 : component[e->src->index] = component_id;
4158 : }
4159 0 : for (edge e: b->succs)
4160 0 : if (!component[e->dest->index])
4161 : {
4162 0 : stack.quick_push (e->dest);
4163 0 : component[e->dest->index] = component_id;
4164 : }
4165 : }
4166 0 : while (!stack.is_empty ());
4167 :
4168 : /* If all blocks in components has 0 count, we do not need
4169 : to scale, only we must convert to IPA quality. */
4170 0 : if (!max_count.nonzero_p ())
4171 : {
4172 0 : if (dump_file)
4173 0 : fprintf (dump_file, " All counts are 0; scale = 1\n");
4174 0 : scale_bbs (bbs, 1);
4175 0 : continue;
4176 : }
4177 :
4178 : /* Now visit the component and try to figure out its desired
4179 : frequency. */
4180 0 : for (basic_block b : bbs)
4181 : {
4182 0 : if (dump_file)
4183 : {
4184 0 : fprintf (dump_file, " visiting bb %i with count ", b->index);
4185 0 : b->count.dump (dump_file);
4186 0 : fprintf (dump_file, "\n");
4187 : }
4188 0 : if (!b->count.nonzero_p ())
4189 0 : continue;
4190 : /* Sum of counts of annotated edges into B. */
4191 0 : profile_count annotated_count = profile_count::zero ();
4192 : /* Sum of counts of edges into B with source in current
4193 : component. */
4194 0 : profile_count current_component_count = profile_count::zero ();
4195 0 : bool boundary = false;
4196 :
4197 0 : for (edge e: b->preds)
4198 0 : if (AFDO_EINFO (e)->is_annotated ())
4199 : {
4200 0 : if (dump_file)
4201 : {
4202 0 : fprintf (dump_file, " Annotated pred edge to %i "
4203 0 : "with count ", e->src->index);
4204 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
4205 0 : fprintf (dump_file, "\n");
4206 : }
4207 0 : boundary = true;
4208 0 : annotated_count += AFDO_EINFO (e)->get_count ();
4209 : }
4210 : /* If source is anotated, combine with static
4211 : probability prediction.
4212 : TODO: We can do better in case some of edges out are
4213 : annotated and distribute only remaining count out of BB. */
4214 0 : else if (is_bb_annotated (e->src, *annotated_bb))
4215 : {
4216 0 : boundary = true;
4217 0 : if (dump_file)
4218 : {
4219 0 : fprintf (dump_file, " Annotated predecessor %i "
4220 : "with count ", e->src->index);
4221 0 : e->src->count.dump (dump_file);
4222 0 : fprintf (dump_file, " edge count using static profile ");
4223 0 : e->count ().dump (dump_file);
4224 0 : fprintf (dump_file, "\n");
4225 : }
4226 0 : annotated_count += e->count ();
4227 : }
4228 : else
4229 : {
4230 0 : current_component_count += e->count ();
4231 0 : gcc_checking_assert (component[e->src->index] == component_id);
4232 : }
4233 0 : if (boundary && current_component_count.initialized_p ())
4234 : {
4235 0 : if (dump_file)
4236 0 : fprintf (dump_file, " bb %i in count ", b->index);
4237 0 : add_scale (&scales,
4238 : annotated_count,
4239 : b->count - current_component_count);
4240 : }
4241 0 : for (edge e: b->succs)
4242 0 : if (AFDO_EINFO (e)->is_annotated ())
4243 : {
4244 0 : if (dump_file)
4245 0 : fprintf (dump_file, " edge %i->%i count ",
4246 0 : b->index, e->dest->index);
4247 0 : add_scale (&scales, AFDO_EINFO (e)->get_count (), e->count ());
4248 : }
4249 0 : else if (is_bb_annotated (e->dest, *annotated_bb))
4250 : {
4251 0 : profile_count annotated_count = e->dest->count;
4252 0 : profile_count out_count = profile_count::zero ();
4253 0 : bool ok = true;
4254 :
4255 0 : for (edge e2: e->dest->preds)
4256 0 : if (AFDO_EINFO (e2)->is_annotated ())
4257 0 : annotated_count -= AFDO_EINFO (e2)->get_count ();
4258 0 : else if (component[e2->src->index] == component_id)
4259 0 : out_count += e2->count ();
4260 0 : else if (is_bb_annotated (e2->src, *annotated_bb))
4261 0 : annotated_count -= e2->count ();
4262 0 : else if (e2->probability.nonzero_p ())
4263 : {
4264 : ok = false;
4265 : break;
4266 : }
4267 0 : if (!ok)
4268 0 : continue;
4269 0 : if (dump_file)
4270 0 : fprintf (dump_file,
4271 : " edge %i->%i has annotated successor; count ",
4272 0 : b->index, e->dest->index);
4273 0 : add_scale (&scales, annotated_count, e->count ());
4274 : }
4275 :
4276 : }
4277 :
4278 : /* If we failed to find annotated entry or exit edge,
4279 : look for exit edges and scale profile so the dest
4280 : BB get all flow it needs. This is imprecise because
4281 : the edge is not annotated and thus BB has more than
4282 : one such predecessor. */
4283 0 : if (!scales.length ())
4284 0 : for (basic_block b : bbs)
4285 0 : if (b->count.nonzero_p ())
4286 0 : for (edge e: b->succs)
4287 0 : if (is_bb_annotated (e->dest, *annotated_bb))
4288 : {
4289 0 : profile_count annotated_count = e->dest->count;
4290 0 : for (edge e2: e->dest->preds)
4291 0 : if (AFDO_EINFO (e2)->is_annotated ())
4292 0 : annotated_count -= AFDO_EINFO (e2)->get_count ();
4293 0 : if (dump_file)
4294 0 : fprintf (dump_file,
4295 : " edge %i->%i has annotated successor;"
4296 : " upper bound count ",
4297 0 : b->index, e->dest->index);
4298 0 : add_scale (&scales, annotated_count, e->count ());
4299 : }
4300 0 : if (!scales.length ())
4301 : {
4302 0 : if (dump_file)
4303 0 : fprintf (dump_file,
4304 : " Can not determine count from the boundary; giving up\n");
4305 0 : continue;
4306 : }
4307 0 : gcc_checking_assert (scales.length ());
4308 0 : sreal scale = determine_scale (&scales, max_count, max_count_in_fn);
4309 0 : scale_bbs (bbs, scale);
4310 : }
4311 0 : }
4312 :
4313 : /* Propagate counts on control flow graph and calculate branch
4314 : probabilities. */
4315 :
4316 : static void
4317 0 : afdo_calculate_branch_prob (bb_set *annotated_bb)
4318 : {
4319 0 : edge e;
4320 0 : edge_iterator ei;
4321 0 : basic_block bb;
4322 :
4323 0 : FOR_ALL_BB_FN (bb, cfun)
4324 : {
4325 0 : gcc_assert (bb->aux == NULL);
4326 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4327 : {
4328 0 : gcc_assert (e->aux == NULL);
4329 0 : e->aux = new edge_info ();
4330 0 : gcov_type c = afdo_unscaled_edge_count (e);
4331 0 : if (c == 0 && e->count () == profile_count::zero ())
4332 : {
4333 0 : AFDO_EINFO (e)->set_count (profile_count::zero ());
4334 0 : if (dump_file)
4335 0 : fprintf (dump_file,
4336 : " Annotating edge %i->%i with count 0;"
4337 : " static profile aggress",
4338 0 : e->src->index, e->dest->index);
4339 : }
4340 0 : else if (c > 0)
4341 : {
4342 0 : AFDO_EINFO (e)->set_count
4343 0 : (profile_count::from_gcov_type
4344 0 : (c * autofdo::afdo_count_scale).afdo ());
4345 0 : if (dump_file)
4346 : {
4347 0 : fprintf (dump_file,
4348 : " Annotating edge %i->%i with count ",
4349 0 : e->src->index, e->dest->index);
4350 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
4351 0 : fprintf (dump_file, "\n");
4352 : }
4353 : }
4354 : }
4355 : }
4356 :
4357 0 : afdo_find_equiv_class (annotated_bb);
4358 0 : afdo_propagate (annotated_bb);
4359 :
4360 0 : FOR_EACH_BB_FN (bb, cfun)
4361 0 : if (is_bb_annotated (bb, *annotated_bb))
4362 : {
4363 0 : bool all_known = true;
4364 0 : profile_count total_count = profile_count::zero ().afdo ();
4365 :
4366 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4367 : {
4368 0 : gcc_assert (AFDO_EINFO (e) != NULL);
4369 0 : if (! AFDO_EINFO (e)->is_annotated ())
4370 : {
4371 : /* If by static profile this edge never happens,
4372 : still propagate the rest. */
4373 0 : if (e->probability.nonzero_p ())
4374 : {
4375 : all_known = false;
4376 : break;
4377 : }
4378 : }
4379 : else
4380 0 : total_count += AFDO_EINFO (e)->get_count ();
4381 : }
4382 0 : if (!all_known || !total_count.nonzero_p ())
4383 0 : continue;
4384 0 : if (dump_file)
4385 : {
4386 0 : fprintf (dump_file, "Total count of bb %i is ", bb->index);
4387 0 : total_count.dump (dump_file);
4388 0 : fprintf (dump_file, "\n");
4389 : }
4390 :
4391 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4392 0 : if (AFDO_EINFO (e)->is_annotated ())
4393 : {
4394 0 : profile_count cnt = AFDO_EINFO (e)->get_count ();
4395 : /* If probability is 1, preserve reliable static prediction
4396 : (This is, for example the case of single fallthru edge
4397 : or single fallthru plus unlikely EH edge.) */
4398 0 : if (cnt == total_count
4399 0 : && e->probability == profile_probability::always ())
4400 : ;
4401 0 : else if (cnt.nonzero_p ())
4402 0 : e->probability
4403 0 : = cnt.probability_in (total_count);
4404 : /* If probability is zero, preserve reliable static
4405 : prediction. */
4406 0 : else if (e->probability.nonzero_p ()
4407 0 : || e->probability.quality () == GUESSED)
4408 0 : e->probability = profile_probability::never ().afdo ();
4409 0 : if (dump_file)
4410 : {
4411 0 : fprintf (dump_file, " probability of edge %i->%i"
4412 : " with count ",
4413 0 : e->src->index, e->dest->index);
4414 0 : cnt.dump (dump_file);
4415 0 : fprintf (dump_file, " set to ");
4416 0 : e->probability.dump (dump_file);
4417 0 : fprintf (dump_file, "\n");
4418 : }
4419 : }
4420 : }
4421 0 : afdo_adjust_guessed_profile (annotated_bb);
4422 0 : FOR_ALL_BB_FN (bb, cfun)
4423 : {
4424 0 : bb->aux = NULL;
4425 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4426 0 : if (AFDO_EINFO (e) != NULL)
4427 : {
4428 0 : delete AFDO_EINFO (e);
4429 0 : e->aux = NULL;
4430 : }
4431 : }
4432 0 : }
4433 :
4434 : /* Annotate auto profile to the control flow graph. */
4435 :
4436 : static void
4437 0 : afdo_annotate_cfg (void)
4438 : {
4439 0 : basic_block bb;
4440 0 : bb_set annotated_bb;
4441 0 : const function_instance *s
4442 0 : = afdo_source_profile->get_function_instance_by_decl (
4443 : current_function_decl);
4444 :
4445 : /* FIXME: This is a workaround for sourcefile tracking, if afdo_string_table
4446 : ends up with empty filename or incorrect filename for the function and
4447 : should be removed once issues with sourcefile tracking get fixed. */
4448 0 : if (s == NULL)
4449 0 : for (unsigned i = 0; i < afdo_string_table->filenames ().length (); i++)
4450 : {
4451 0 : s = afdo_source_profile->get_function_instance_by_decl (current_function_decl, afdo_string_table->filenames()[i]);
4452 0 : if (s)
4453 : break;
4454 : }
4455 :
4456 0 : if (s == NULL)
4457 : {
4458 0 : if (dump_file)
4459 0 : fprintf (dump_file, "No afdo profile for %s\n",
4460 0 : cgraph_node::get (current_function_decl)->dump_name ());
4461 : /* create_gcov only dumps symbols with some samples in them.
4462 : This means that we get nonempty zero_bbs only if some
4463 : nonzero counts in profile were not matched with statements. */
4464 0 : if (!flag_profile_partial_training
4465 0 : && !param_auto_profile_reorder_only)
4466 : {
4467 0 : FOR_ALL_BB_FN (bb, cfun)
4468 0 : if (bb->count.quality () == GUESSED_LOCAL)
4469 0 : bb->count = bb->count.global0afdo ();
4470 0 : update_max_bb_count ();
4471 : }
4472 0 : return;
4473 : }
4474 :
4475 0 : auto ts_it = timestamp_info_map.find (s->timestamp ());
4476 0 : if (ts_it != timestamp_info_map.end ())
4477 : {
4478 0 : cgraph_node *node = cgraph_node::get (current_function_decl);
4479 0 : node->tp_first_run = ts_it->second;
4480 :
4481 0 : if (dump_file)
4482 0 : fprintf (dump_file, "Setting %s->tp_first_run to %d\n",
4483 : node->asm_name (), node->tp_first_run);
4484 : }
4485 :
4486 0 : if (param_auto_profile_reorder_only)
4487 : return;
4488 :
4489 0 : calculate_dominance_info (CDI_POST_DOMINATORS);
4490 0 : calculate_dominance_info (CDI_DOMINATORS);
4491 0 : loop_optimizer_init (0);
4492 :
4493 0 : if (dump_file)
4494 : {
4495 0 : fprintf (dump_file, "\n\nAnnotating BB profile of %s\n",
4496 0 : cgraph_node::get (current_function_decl)->dump_name ());
4497 0 : fprintf (dump_file, "\n");
4498 0 : s->dump (dump_file);
4499 0 : fprintf (dump_file, "\n");
4500 : }
4501 0 : bool profile_found = false;
4502 0 : hash_set <basic_block> zero_bbs;
4503 0 : gcov_type head_count = s->head_count () * autofdo::afdo_count_scale;
4504 :
4505 0 : if (!param_auto_profile_bbs)
4506 : {
4507 0 : if (scale_bb_profile ())
4508 : return;
4509 : }
4510 : else
4511 : {
4512 : /* In the first pass only store non-zero counts. */
4513 0 : profile_found = head_count > 0;
4514 0 : FOR_EACH_BB_FN (bb, cfun)
4515 : {
4516 0 : if (afdo_set_bb_count (bb, zero_bbs))
4517 : {
4518 0 : if (bb->count.quality () == AFDO)
4519 : {
4520 0 : gcc_assert (bb->count.nonzero_p ());
4521 : profile_found = true;
4522 : }
4523 0 : set_bb_annotated (bb, &annotated_bb);
4524 : }
4525 : }
4526 : }
4527 : /* Exit without clobbering static profile if there was no
4528 : non-zero count. */
4529 0 : if (!profile_found)
4530 : {
4531 : /* create_gcov only dumps symbols with some samples in them.
4532 : This means that we get nonempty zero_bbs only if some
4533 : nonzero counts in profile were not matched with statements.
4534 : ??? We can adjust create_gcov to also recordinfo
4535 : about function with no samples. Then we can distinguish
4536 : between lost profiles which should be kept local and
4537 : real functions with 0 samples during train run. */
4538 0 : if (zero_bbs.is_empty ())
4539 : {
4540 0 : if (dump_file)
4541 0 : fprintf (dump_file, "No afdo samples found"
4542 : "; Setting global count to afdo0\n");
4543 : }
4544 : else
4545 : {
4546 0 : if (dump_file)
4547 0 : fprintf (dump_file, "Setting global count to afdo0\n");
4548 : }
4549 0 : if (!flag_profile_partial_training)
4550 : {
4551 0 : FOR_ALL_BB_FN (bb, cfun)
4552 0 : if (bb->count.quality () == GUESSED_LOCAL)
4553 0 : bb->count = bb->count.global0afdo ();
4554 0 : update_max_bb_count ();
4555 : }
4556 :
4557 0 : loop_optimizer_finalize ();
4558 0 : free_dominance_info (CDI_DOMINATORS);
4559 0 : free_dominance_info (CDI_POST_DOMINATORS);
4560 0 : return;
4561 : }
4562 : /* We try to preserve static profile for BBs with 0
4563 : afdo samples, but if even static profile agrees with 0,
4564 : consider it final so propagation works better. */
4565 0 : for (basic_block bb : zero_bbs)
4566 0 : if (!bb->count.nonzero_p ())
4567 : {
4568 0 : update_count_by_afdo_count (&bb->count, 0);
4569 0 : set_bb_annotated (bb, &annotated_bb);
4570 0 : if (dump_file)
4571 : {
4572 0 : fprintf (dump_file, " Annotating bb %i with count ", bb->index);
4573 0 : bb->count.dump (dump_file);
4574 0 : fprintf (dump_file,
4575 : " (has 0 count in both static and afdo profile)\n");
4576 : }
4577 : }
4578 :
4579 : /* Update profile. */
4580 0 : if (head_count > 0)
4581 : {
4582 0 : update_count_by_afdo_count (&ENTRY_BLOCK_PTR_FOR_FN (cfun)->count,
4583 : head_count);
4584 0 : set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun), &annotated_bb);
4585 0 : if (!is_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb, annotated_bb)
4586 0 : || ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
4587 0 : > ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count)
4588 : {
4589 0 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count
4590 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4591 0 : set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb,
4592 : &annotated_bb);
4593 : }
4594 0 : if (!is_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun), annotated_bb)
4595 0 : || ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
4596 0 : > EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count)
4597 : {
4598 0 : EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count
4599 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4600 0 : set_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb, &annotated_bb);
4601 : }
4602 : }
4603 :
4604 : /* Calculate, propagate count and probability information on CFG. */
4605 0 : afdo_calculate_branch_prob (&annotated_bb);
4606 :
4607 : /* If we failed to turn some of original guessed profile to global,
4608 : set basic blocks uninitialized. */
4609 0 : FOR_ALL_BB_FN (bb, cfun)
4610 0 : if (!bb->count.ipa_p ())
4611 : {
4612 : /* We skip annotating entry profile if it is 0
4613 : in hope to be able to determine it better from the
4614 : static profile.
4615 :
4616 : Now we know we can not derive it from other info,
4617 : so set it since it is better than UNKNOWN. */
4618 0 : if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
4619 0 : bb->count = profile_count::zero ().afdo ();
4620 : else
4621 0 : bb->count = profile_count::uninitialized ();
4622 0 : if (dump_file)
4623 0 : fprintf (dump_file, " Unknown count of bb %i\n", bb->index);
4624 0 : cfun->cfg->full_profile = false;
4625 : }
4626 :
4627 0 : cgraph_node::get (current_function_decl)->count
4628 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4629 0 : update_max_bb_count ();
4630 0 : profile_status_for_fn (cfun) = PROFILE_READ;
4631 0 : if (flag_value_profile_transformations)
4632 : {
4633 0 : gimple_value_profile_transformations ();
4634 0 : free_dominance_info (CDI_DOMINATORS);
4635 0 : free_dominance_info (CDI_POST_DOMINATORS);
4636 0 : update_ssa (TODO_update_ssa);
4637 : }
4638 :
4639 0 : loop_optimizer_finalize ();
4640 0 : free_dominance_info (CDI_DOMINATORS);
4641 0 : free_dominance_info (CDI_POST_DOMINATORS);
4642 0 : }
4643 :
4644 : /* Use AutoFDO profile to annotate the control flow graph.
4645 : Return the todo flag. */
4646 :
4647 : static unsigned int
4648 0 : auto_profile (void)
4649 : {
4650 0 : struct cgraph_node *node;
4651 :
4652 0 : if (symtab->state == FINISHED || !afdo_source_profile)
4653 : return 0;
4654 :
4655 0 : init_node_map (true);
4656 0 : profile_info = autofdo::afdo_profile_info;
4657 0 : afdo_source_profile->offline_unrealized_inlines ();
4658 :
4659 0 : FOR_EACH_FUNCTION (node)
4660 : {
4661 0 : if (!gimple_has_body_p (node->decl))
4662 0 : continue;
4663 :
4664 : /* Don't profile functions produced for builtin stuff. */
4665 0 : if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
4666 0 : continue;
4667 :
4668 0 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4669 :
4670 : /* Local pure-const may imply need to fixup the cfg.
4671 : This is similar to what is done in tree-profile.cc. */
4672 0 : if ((execute_fixup_cfg () & TODO_cleanup_cfg))
4673 0 : cleanup_tree_cfg ();
4674 :
4675 0 : autofdo::afdo_annotate_cfg ();
4676 0 : compute_function_frequency ();
4677 :
4678 0 : free_dominance_info (CDI_DOMINATORS);
4679 0 : free_dominance_info (CDI_POST_DOMINATORS);
4680 0 : cgraph_edge::rebuild_edges ();
4681 0 : pop_cfun ();
4682 : }
4683 :
4684 : return 0;
4685 : }
4686 : } /* namespace autofdo. */
4687 :
4688 : /* Read the profile from the profile data file. */
4689 :
4690 : void
4691 0 : read_autofdo_file (void)
4692 : {
4693 0 : if (auto_profile_file == NULL)
4694 0 : auto_profile_file = DEFAULT_AUTO_PROFILE_FILE;
4695 :
4696 0 : autofdo::afdo_profile_info = XNEW (gcov_summary);
4697 0 : autofdo::afdo_profile_info->runs = 1;
4698 0 : autofdo::afdo_profile_info->sum_max = 0;
4699 0 : autofdo::afdo_profile_info->cutoff = 1;
4700 :
4701 : /* Read the profile from the profile file. */
4702 0 : autofdo::read_profile ();
4703 0 : }
4704 :
4705 : /* Free the resources. */
4706 :
4707 : void
4708 0 : end_auto_profile (void)
4709 : {
4710 0 : delete autofdo::afdo_source_profile;
4711 0 : delete autofdo::afdo_string_table;
4712 0 : delete autofdo::afdo_summary_info;
4713 0 : profile_info = NULL;
4714 0 : }
4715 :
4716 : /* Returns TRUE if EDGE is hot enough to be inlined early. */
4717 :
4718 : bool
4719 0 : afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge)
4720 : {
4721 0 : gcov_type count
4722 0 : = autofdo::afdo_source_profile->get_callsite_total_count (edge);
4723 :
4724 0 : if (count > 0)
4725 : {
4726 0 : bool is_hot;
4727 0 : profile_count pcount = profile_count::from_gcov_type (count).afdo ();
4728 0 : is_hot = maybe_hot_afdo_count_p (pcount);
4729 0 : if (dump_file)
4730 : {
4731 0 : fprintf (dump_file, "Call %s -> %s has %s afdo profile count ",
4732 0 : edge->caller->dump_name (), edge->callee->dump_name (),
4733 : is_hot ? "hot" : "cold");
4734 0 : pcount.dump (dump_file);
4735 0 : fprintf (dump_file, "\n");
4736 : }
4737 0 : return is_hot;
4738 : }
4739 :
4740 : return false;
4741 : }
4742 :
4743 : /* Do indirect call promotion during early inlining to make the
4744 : IR match the profiled binary before actual annotation.
4745 :
4746 : This is needed because an indirect call might have been promoted
4747 : and inlined in the profiled binary. If we do not promote and
4748 : inline these indirect calls before annotation, the profile for
4749 : these promoted functions will be lost.
4750 :
4751 : e.g. foo() --indirect_call--> bar()
4752 : In profiled binary, the callsite is promoted and inlined, making
4753 : the profile look like:
4754 :
4755 : foo: {
4756 : loc_foo_1: count_1
4757 : bar@loc_foo_2: {
4758 : loc_bar_1: count_2
4759 : loc_bar_2: count_3
4760 : }
4761 : }
4762 :
4763 : Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
4764 : If we perform annotation on it, the profile inside bar@loc_foo2
4765 : will be wasted.
4766 :
4767 : To avoid this, we promote loc_foo_2 and inline the promoted bar
4768 : function before annotation, so the profile inside bar@loc_foo2
4769 : will be useful. */
4770 :
4771 : bool
4772 0 : afdo_vpt_for_early_inline (cgraph_node *node)
4773 : {
4774 0 : if (!node->indirect_calls)
4775 : return false;
4776 0 : bool changed = false;
4777 0 : cgraph_node *outer = node->inlined_to ? node->inlined_to : node;
4778 0 : if (autofdo::afdo_source_profile->get_function_instance_by_decl
4779 0 : (outer->decl) == NULL)
4780 : return false;
4781 0 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
4782 : {
4783 0 : gcov_type bb_count = 0;
4784 0 : autofdo::count_info info;
4785 0 : basic_block bb = gimple_bb (e->call_stmt);
4786 :
4787 : /* TODO: This is quadratic; cache the value. */
4788 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
4789 0 : !gsi_end_p (gsi); gsi_next (&gsi))
4790 : {
4791 0 : gimple *stmt = gsi_stmt (gsi);
4792 0 : if (!stmt_loc_used_by_debug_info (stmt))
4793 0 : continue;
4794 0 : autofdo::count_info info;
4795 0 : if (autofdo::afdo_source_profile->get_count_info (stmt, &info, node))
4796 0 : bb_count = MAX (bb_count, info.count);
4797 0 : }
4798 0 : autofdo::afdo_source_profile->get_count_info (e->call_stmt, &info, node);
4799 0 : info.count = bb_count;
4800 0 : if (!autofdo::afdo_source_profile->update_inlined_ind_target
4801 0 : (e->call_stmt, &info, node))
4802 0 : continue;
4803 0 : changed |= autofdo::afdo_vpt (e->call_stmt, info.targets, true, e);
4804 0 : }
4805 : return changed;
4806 : }
4807 :
4808 : /* If speculation used during early inline, remove the target
4809 : so we do not speculate the indirect edge again during afdo pass. */
4810 :
4811 : void
4812 0 : remove_afdo_speculative_target (cgraph_edge *e)
4813 : {
4814 0 : autofdo::afdo_source_profile->remove_icall_target (e);
4815 0 : }
4816 :
4817 : namespace
4818 : {
4819 :
4820 : const pass_data pass_data_ipa_auto_profile = {
4821 : SIMPLE_IPA_PASS, "afdo", /* name */
4822 : OPTGROUP_NONE, /* optinfo_flags */
4823 : TV_IPA_AUTOFDO, /* tv_id */
4824 : 0, /* properties_required */
4825 : 0, /* properties_provided */
4826 : 0, /* properties_destroyed */
4827 : 0, /* todo_flags_start */
4828 : 0, /* todo_flags_finish */
4829 : };
4830 :
4831 : class pass_ipa_auto_profile : public simple_ipa_opt_pass
4832 : {
4833 : public:
4834 285722 : pass_ipa_auto_profile (gcc::context *ctxt)
4835 571444 : : simple_ipa_opt_pass (pass_data_ipa_auto_profile, ctxt)
4836 : {
4837 : }
4838 :
4839 : /* opt_pass methods: */
4840 : bool
4841 229960 : gate (function *) final override
4842 : {
4843 229960 : return flag_auto_profile;
4844 : }
4845 : unsigned int
4846 0 : execute (function *) final override
4847 : {
4848 0 : return autofdo::auto_profile ();
4849 : }
4850 : }; // class pass_ipa_auto_profile
4851 :
4852 : } // anon namespace
4853 :
4854 : simple_ipa_opt_pass *
4855 285722 : make_pass_ipa_auto_profile (gcc::context *ctxt)
4856 : {
4857 285722 : return new pass_ipa_auto_profile (ctxt);
4858 : }
4859 :
4860 : namespace
4861 : {
4862 :
4863 : const pass_data pass_data_ipa_auto_profile_offline = {
4864 : SIMPLE_IPA_PASS, "afdo_offline", /* name */
4865 : OPTGROUP_NONE, /* optinfo_flags */
4866 : TV_IPA_AUTOFDO_OFFLINE, /* tv_id */
4867 : 0, /* properties_required */
4868 : 0, /* properties_provided */
4869 : 0, /* properties_destroyed */
4870 : 0, /* todo_flags_start */
4871 : 0, /* todo_flags_finish */
4872 : };
4873 :
4874 : class pass_ipa_auto_profile_offline : public simple_ipa_opt_pass
4875 : {
4876 : public:
4877 285722 : pass_ipa_auto_profile_offline (gcc::context *ctxt)
4878 571444 : : simple_ipa_opt_pass (pass_data_ipa_auto_profile_offline, ctxt)
4879 : {
4880 : }
4881 :
4882 : /* opt_pass methods: */
4883 : bool
4884 229960 : gate (function *) final override
4885 : {
4886 229960 : return flag_auto_profile;
4887 : }
4888 : unsigned int
4889 0 : execute (function *) final override
4890 : {
4891 0 : read_autofdo_file ();
4892 0 : if (autofdo::afdo_source_profile)
4893 0 : autofdo::afdo_source_profile->offline_external_functions ();
4894 0 : return 0;
4895 : }
4896 : }; // class pass_ipa_auto_profile
4897 :
4898 : } // anon namespace
4899 :
4900 : simple_ipa_opt_pass *
4901 285722 : make_pass_ipa_auto_profile_offline (gcc::context *ctxt)
4902 : {
4903 285722 : return new pass_ipa_auto_profile_offline (ctxt);
4904 : }
|
