Line data Source code
1 : /* Read and annotate call graph profile from the auto profile data file.
2 : Copyright (C) 2014-2026 Free Software Foundation, Inc.
3 : Contributed by Dehao Chen (dehao@google.com)
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #include "config.h"
22 : #define INCLUDE_MAP
23 : #define INCLUDE_SET
24 : #include "system.h"
25 : #include "coretypes.h"
26 : #include "backend.h"
27 : #include "tree.h"
28 : #include "gimple.h"
29 : #include "predict.h"
30 : #include "alloc-pool.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "cgraph.h"
34 : #include "gcov-io.h"
35 : #include "diagnostic-core.h"
36 : #include "profile.h"
37 : #include "langhooks.h"
38 : #include "context.h"
39 : #include "pass_manager.h"
40 : #include "cfgloop.h"
41 : #include "tree-cfg.h"
42 : #include "tree-cfgcleanup.h"
43 : #include "tree-into-ssa.h"
44 : #include "gimple-iterator.h"
45 : #include "value-prof.h"
46 : #include "symbol-summary.h"
47 : #include "sreal.h"
48 : #include "ipa-cp.h"
49 : #include "ipa-prop.h"
50 : #include "ipa-fnsummary.h"
51 : #include "ipa-inline.h"
52 : #include "tree-inline.h"
53 : #include "auto-profile.h"
54 : #include "tree-pretty-print.h"
55 : #include "gimple-pretty-print.h"
56 : #include "output.h"
57 :
58 : /* The following routines implement AutoFDO optimization.
59 :
60 : This optimization uses sampling profiles to annotate basic block counts
61 : and uses heuristics to estimate branch probabilities.
62 :
63 : There are three phases in AutoFDO:
64 :
65 : Phase 1: At startup.
66 : Read profile from the profile data file.
67 : The following info is read from the profile datafile:
68 : * string_table: a map between function name and its index.
69 : * autofdo_source_profile: a map from function_instance name to
70 : function_instance. This is represented as a forest of
71 : function_instances.
72 : * WorkingSet: a histogram of how many instructions are covered for a
73 : given percentage of total cycles. This is describing the binary
74 : level information (not source level). This info is used to help
75 : decide if we want aggressive optimizations that could increase
76 : code footprint (e.g. loop unroll etc.)
77 : A function instance is an instance of function that could either be a
78 : standalone symbol, or a clone of a function that is inlined into another
79 : function.
80 :
81 : Phase 2: In afdo_offline pass.
82 : Remove function instances from other translation units
83 : and offline all cross-translation unit inlining done during train
84 : run compilation. This is necessary to not lose profiles with
85 : LTO train run.
86 :
87 : Phase 3: During early optimization.
88 : AFDO inline + value profile transformation.
89 : This happens during early optimization.
90 : During early inlining AFDO inliner is executed which
91 : uses autofdo_source_profile to find if a callsite is:
92 : * inlined in the profiled binary.
93 : * callee body is hot in the profiling run.
94 : If both condition satisfies, early inline will inline the callsite
95 : regardless of the code growth.
96 :
97 : Performing this early has benefit of doing early optimizations
98 : before read IPA passes and getting more "context sensitivity" of
99 : the profile read. Profile of inlined functions may differ
100 : significantly from one inline instance to another and from the
101 : offline version.
102 :
103 : This is controlled by -fauto-profile-inlining and is independent
104 : of -fearly-inlining.
105 :
106 : Phase 4: In AFDO pass.
107 : Offline all functions that has been inlined in the
108 : train run but were not inlined in early inlining nor AFDO
109 : inline.
110 :
111 : Phase 5: In AFDO pass.
112 : Annotate control flow graph.
113 : * Annotate basic block count
114 : * Estimate branch probability
115 : * Use earlier static profile to fill in the gaps
116 : if AFDO profile is ambiguous
117 :
118 : After the above 5 phases, all profile is readily annotated on the GCC IR.
119 : AutoFDO tries to reuse all FDO infrastructure as much as possible to make
120 : use of the profile. E.g. it uses existing mechanism to calculate the basic
121 : block/edge frequency, as well as the cgraph node/edge count.
122 : */
123 :
124 : #define DEFAULT_AUTO_PROFILE_FILE "fbdata.afdo"
125 :
126 : /* profile counts determined by AFDO smaller than afdo_hot_bb_threshold are
127 : considered cols. */
128 : gcov_type afdo_hot_bb_threshold = -1;
129 :
130 : /* Return true if COUNT is possibly hot. */
131 : bool
132 0 : maybe_hot_afdo_count_p (profile_count count)
133 : {
134 0 : gcc_checking_assert (count.ipa ().initialized_p ());
135 0 : return count.ipa ().to_gcov_type () >= afdo_hot_bb_threshold;
136 : }
137 :
138 : /* Return true if location of STMT may be expressed by debug info. */
139 :
140 : static bool
141 0 : stmt_loc_used_by_debug_info (gimple *stmt)
142 : {
143 : /* Only inline_entry and gimple_bind's locations
144 : are not output into debug output. */
145 0 : if (is_gimple_debug (stmt))
146 0 : return gimple_debug_begin_stmt_p (stmt);
147 0 : if (gimple_code (stmt) == GIMPLE_LABEL
148 : || gimple_code (stmt) == GIMPLE_NOP
149 : || gimple_code (stmt) == GIMPLE_PREDICT)
150 : return false;
151 0 : if (gimple_clobber_p (stmt))
152 0 : return false;
153 : return true;
154 : }
155 :
156 : namespace autofdo
157 : {
158 :
159 : /* Intermediate edge info used when propagating AutoFDO profile information.
160 : We can't edge->count() directly since it's computed from edge's probability
161 : while probability is yet not decided during propagation. */
162 : #define AFDO_EINFO(e) ((class edge_info *) e->aux)
163 : class edge_info
164 : {
165 : public:
166 0 : edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {}
167 0 : bool is_annotated () const { return annotated_; }
168 0 : void set_annotated () { annotated_ = true; }
169 0 : profile_count get_count () const { return count_; }
170 0 : void set_count (profile_count count) { count_ = count; }
171 : private:
172 : profile_count count_;
173 : bool annotated_;
174 : };
175 :
176 : /* Represent a source location: (function_decl, lineno). */
177 : struct decl_lineno
178 : {
179 : tree decl;
180 : /* Relative locations stored in auto-profile. */
181 : unsigned int afdo_loc;
182 : /* Actual location afdo_loc was computed from used to output diagnostics. */
183 : location_t location;
184 : };
185 :
186 : /* Represent an inline stack. vector[0] is the leaf node. */
187 : typedef auto_vec<decl_lineno, 20> inline_stack;
188 :
189 : /* String array that stores function names. */
190 : typedef auto_vec<const char *> string_vector;
191 :
192 : /* Map from function name's index in string_table to target's
193 : execution count. */
194 : typedef std::map<unsigned, gcov_type> icall_target_map;
195 :
196 : /* Set of gimple stmts. Used to track if the stmt has already been promoted
197 : to direct call. */
198 : typedef std::set<gimple *> stmt_set;
199 :
200 : /* Set and map used to translate name indexes. */
201 : typedef hash_set<int_hash <int, -1, -2>> name_index_set;
202 : typedef hash_map<int_hash <int, -1, -2>, int> name_index_map;
203 :
204 : /* Represent count info of an inline stack. */
205 0 : class count_info
206 : {
207 : public:
208 : /* Sampled count of the inline stack. */
209 : gcov_type count;
210 :
211 : /* Map from indirect call target to its sample count. */
212 : icall_target_map targets;
213 :
214 : /* Whether this inline stack is already used in annotation.
215 :
216 : Each inline stack should only be used to annotate IR once.
217 : This will be enforced when instruction-level discriminator
218 : is supported. */
219 : };
220 :
221 : /* operator< for "const char *". */
222 : struct string_compare
223 : {
224 0 : bool operator()(const char *a, const char *b) const
225 : {
226 0 : return strcmp (a, b) < 0;
227 : }
228 : };
229 :
230 : /* Store the summary information for the profile. */
231 : struct summary_info
232 : {
233 : /* There are currently 16 hard-coded percentiles in the GCOV format. */
234 : static constexpr unsigned NUM_PERCENTILES = 16;
235 :
236 : /* The detailed summary is a histogram-based calculation of the minimum
237 : execution count required to belong to a certain set of percentile of
238 : counts. */
239 : struct detailed_summary
240 : {
241 : /* The percentile that this represents (multiplied by 1,000,000). */
242 : uint32_t cutoff;
243 : /* The minimum execution count required to belong to this percentile. */
244 : uint64_t min_count;
245 : /* The number of samples which belong to this percentile. */
246 : uint64_t num_counts;
247 : };
248 :
249 : /* The sum of execution counts of all samples. */
250 : uint64_t total_count;
251 : /* The maximum individual count. */
252 : uint64_t max_count;
253 : /* The maximum head count across all functions. */
254 : uint64_t max_function_count;
255 : /* The number of lines that have samples. */
256 : uint64_t num_counts;
257 : /* The number of functions that have samples. */
258 : uint64_t num_functions;
259 : /* The percentile threshold information. */
260 : detailed_summary detailed_summaries[NUM_PERCENTILES];
261 :
262 : /* Read profile. Return TRUE on success. */
263 : bool read ();
264 :
265 : /* Get the minimum count required for percentile CUTOFF. */
266 : uint64_t get_threshold_count (uint32_t cutoff);
267 : };
268 :
269 : /* Store a string array, indexed by string position in the array. */
270 : class string_table
271 : {
272 : public:
273 : static const int unknown_filename = -1;
274 :
275 0 : string_table ()
276 0 : {}
277 :
278 : ~string_table ();
279 :
280 : /* For a given string, returns its index. */
281 : int get_index (const char *name) const;
282 :
283 : /* For a given decl, returns the index of the decl name. */
284 : int get_index_by_decl (tree decl) const;
285 :
286 : /* For a given index, returns the symbol name. */
287 : const char *get_symbol_name (int index) const;
288 :
289 : /* For a given index, returns the filename. */
290 : const char *get_filename (int index) const;
291 :
292 : /* For a given symbol name index, returns the filename index. */
293 : int get_filename_by_symbol (int index) const;
294 :
295 : /* For a given function name, returns the filename index. */
296 : int get_filename_by_symbol (const char *name) const;
297 :
298 : /* For a given filename, returns the index. */
299 : int get_filename_index (const char *name) const;
300 :
301 : /* Get the original name and file name index for a node. This will return the
302 : name from the current TU if there are multiple symbols that map to
303 : NAME. */
304 : std::pair<const char *, int> get_original_name (const char *name) const;
305 :
306 : /* Read profile, return TRUE on success. */
307 : bool read ();
308 :
309 : /* Return number of entries. */
310 0 : size_t num_entries () { return symbol_names_.length (); }
311 :
312 : /* Add new symbol name STRING (with an associated file name FILENAME_IDX) and
313 : return its index. */
314 : int add_symbol_name (const char *string, int filename_idx);
315 :
316 : /* Add new filename and return its index (returning the same if it already
317 : exists). */
318 : int add_filename (const char *name);
319 :
320 : /* Return cgraph node corresponding to given name index. */
321 : cgraph_node *get_cgraph_node (int);
322 :
323 : const string_vector& filenames () { return filenames_; }
324 : private:
325 : typedef std::map<const char *, unsigned, string_compare> string_index_map;
326 : typedef std::map<const char *, auto_vec<unsigned>, string_compare>
327 : clashing_name_map;
328 : typedef std::map<const char *, char *, string_compare> string_string_map;
329 :
330 : string_vector symbol_names_;
331 : string_vector filenames_;
332 :
333 : string_index_map symbol_name_map_;
334 : string_index_map filename_map_;
335 : string_index_map symbol_to_filename_map_;
336 :
337 : string_string_map original_names_map_;
338 : clashing_name_map clashing_names_map_;
339 : };
340 :
341 : /* Descriptor for a function_instance which can be used to disambiguate it from
342 : other instances. This consists of the symbol name and the file name indices
343 : from string_table. */
344 :
345 : class function_instance_descriptor
346 : {
347 : /* The string_table index for the file name. */
348 : unsigned file_name_;
349 : /* The string_table index for the function name. */
350 : unsigned symbol_name_;
351 :
352 : public:
353 0 : unsigned file_name () const { return file_name_; }
354 0 : unsigned symbol_name () const { return symbol_name_; }
355 :
356 0 : function_instance_descriptor (unsigned file_name, unsigned symbol_name)
357 0 : : file_name_ (file_name), symbol_name_ (symbol_name)
358 : {}
359 :
360 0 : function_instance_descriptor (int file_name, int symbol_name)
361 0 : : file_name_ (file_name), symbol_name_ (symbol_name)
362 : {}
363 :
364 0 : void set_symbol_name (unsigned new_name) { symbol_name_ = new_name; }
365 :
366 0 : bool operator< (const function_instance_descriptor &other) const
367 : {
368 0 : return file_name_ < other.file_name_
369 0 : || (file_name_ == other.file_name_
370 0 : && symbol_name_ < other.symbol_name_);
371 : }
372 : };
373 :
374 : /* Profile of a function instance:
375 : 1. total_count of the function.
376 : 2. head_count (entry basic block count) of the function (only valid when
377 : function is a top-level function_instance, i.e. it is the original copy
378 : instead of the inlined copy).
379 : 3. map from source location (decl_lineno) to profile (count_info).
380 : 4. map from callsite to callee function_instance. */
381 :
382 : class function_instance
383 : {
384 : public:
385 : typedef auto_vec<function_instance *> function_instance_stack;
386 :
387 : /* Read the profile and return a function_instance with head count as
388 : HEAD_COUNT. Recursively read callsites to create nested function_instances
389 : too. STACK is used to track the recursive creation process. */
390 : static function_instance *
391 : read_function_instance (function_instance_stack *stack, bool toplevel = true);
392 :
393 : /* Recursively deallocate all callsites (nested function_instances). */
394 : ~function_instance ();
395 :
396 : /* Accessors. */
397 0 : int symbol_name () const { return descriptor_.symbol_name (); }
398 0 : int file_name () const { return descriptor_.file_name (); }
399 0 : void set_symbol_name (int index) { descriptor_.set_symbol_name (index); }
400 0 : function_instance_descriptor get_descriptor () const { return descriptor_; }
401 :
402 : gcov_type
403 0 : total_count () const
404 : {
405 0 : return total_count_;
406 : }
407 :
408 : /* Return head count or -1 if unknown. */
409 : gcov_type
410 0 : head_count () const
411 : {
412 0 : return head_count_;
413 : }
414 :
415 : gcov_type
416 0 : timestamp () const
417 : {
418 0 : return timestamp_;
419 : }
420 :
421 0 : void set_timestamp (gcov_type timestamp) { timestamp_ = timestamp; }
422 :
423 : /* Propagate timestamp from top-level function_instance to
424 : inlined instances. */
425 : void prop_timestamp ();
426 :
427 : /* Traverse callsites of the current function_instance to find one at the
428 : location of LINENO and callee name represented in DECL.
429 : LOCATION should match LINENO and is used to output diagnostics. */
430 : function_instance *get_function_instance_by_decl (unsigned lineno,
431 : tree decl,
432 : location_t location) const;
433 :
434 : /* Merge profile of clones. Note that cloning hasn't been performed when
435 : we annotate the CFG (at this stage). */
436 : void merge (function_instance *other,
437 : vec <function_instance *> &new_functions);
438 :
439 : /* Look for inline instances that was not realized and
440 : remove them while possibly merging them to offline variants. */
441 : void offline_if_not_realized (vec <function_instance *> &new_functions);
442 :
443 : /* Match function instance with gimple body. */
444 : bool match (cgraph_node *node, vec <function_instance *> &new_functions,
445 : name_index_map &to_symbol_name);
446 :
447 : /* Offline all inlined functions with name in SEEN.
448 : If new toplevel functions are created, add them to NEW_FUNCTIONS. */
449 : void offline_if_in_set (name_index_set &seen,
450 : vec <function_instance *> &new_functions);
451 :
452 : /* Walk inlined functions and if their name is not in SEEN
453 : remove it. */
454 :
455 : void remove_external_functions (name_index_set &seen,
456 : name_index_map &to_symbol_name,
457 : vec <function_instance *> &new_functions);
458 :
459 : /* Store the profile info for LOC in INFO. Return TRUE if profile info
460 : is found. */
461 : bool get_count_info (location_t loc, count_info *info) const;
462 :
463 : /* Read the inlined indirect call target profile for STMT in FN and store it
464 : in MAP, return the total count for all inlined indirect calls. */
465 : gcov_type find_icall_target_map (tree fn, gcall *stmt,
466 : icall_target_map *map) const;
467 :
468 : /* Remove inlined indirect call target profile for STMT in FN. */
469 : void remove_icall_target (tree fn, gcall *stmt);
470 :
471 : /* Mark LOC as annotated. */
472 : void mark_annotated (location_t loc);
473 :
474 : void dump (FILE *f, int indent = 0, bool nested = false) const;
475 :
476 : void dump_inline_stack (FILE *f) const;
477 :
478 : DEBUG_FUNCTION void debug () const;
479 :
480 : /* Mark function as removed from indir target list. */
481 : void
482 0 : remove_icall_target ()
483 : {
484 0 : removed_icall_target_ = true;
485 : }
486 :
487 : /* Return true if function is removed from indir target list. */
488 : bool
489 0 : removed_icall_target ()
490 : {
491 0 : return removed_icall_target_;
492 : }
493 :
494 : /* Set inlined_to pointer. */
495 : void
496 0 : set_inlined_to (function_instance *inlined_to)
497 : {
498 0 : gcc_checking_assert (inlined_to != this);
499 0 : inlined_to_ = inlined_to;
500 0 : }
501 :
502 : /* Return pointer to the function instance this function is inlined
503 : to or NULL if it is outer instance. */
504 : function_instance *
505 0 : inlined_to () const
506 : {
507 0 : return inlined_to_;
508 : }
509 :
510 : /* Mark function as realized. */
511 : void
512 0 : set_realized ()
513 : {
514 0 : realized_ = true;
515 : }
516 :
517 : /* Return true if function is realized. */
518 : bool
519 0 : realized_p ()
520 : {
521 0 : return realized_;
522 : }
523 :
524 : /* Mark function as in_worklist. */
525 : void
526 0 : set_in_worklist ()
527 : {
528 0 : gcc_checking_assert (!inlined_to_ && !in_worklist_p ());
529 0 : in_worklist_ = true;
530 0 : }
531 :
532 : void
533 0 : clear_in_worklist ()
534 : {
535 0 : gcc_checking_assert (!inlined_to_ && in_worklist_p ());
536 0 : in_worklist_ = false;
537 0 : }
538 :
539 :
540 : /* Return true if function is in_worklist. */
541 : bool
542 0 : in_worklist_p ()
543 : {
544 0 : return in_worklist_;
545 : }
546 :
547 : /* Return corresponding cgraph node. */
548 : cgraph_node *get_cgraph_node ();
549 :
550 : void
551 0 : set_location (location_t l)
552 : {
553 0 : gcc_checking_assert (location_ == UNKNOWN_LOCATION);
554 0 : location_= l;
555 0 : }
556 :
557 : location_t
558 0 : get_location ()
559 : {
560 0 : return location_;
561 : }
562 :
563 : void
564 0 : set_call_location (location_t l)
565 : {
566 0 : gcc_checking_assert (call_location_ == UNKNOWN_LOCATION
567 : && l != UNKNOWN_LOCATION);
568 0 : call_location_= l;
569 0 : }
570 :
571 : location_t
572 0 : get_call_location ()
573 : {
574 0 : return call_location_;
575 : }
576 :
577 : /* Lookup count and warn about duplicates. */
578 : count_info *lookup_count (location_t loc, inline_stack &stack,
579 : cgraph_node *node);
580 : private:
581 : /* Callsite, represented as (decl_lineno, callee_function_name_index). */
582 : typedef std::pair<unsigned, unsigned> callsite;
583 :
584 : /* Map from callsite to callee function_instance. */
585 : typedef std::map<callsite, function_instance *> callsite_map;
586 :
587 0 : function_instance (unsigned symbol_name, unsigned file_name,
588 : gcov_type head_count)
589 0 : : descriptor_ (file_name, symbol_name), total_count_ (0),
590 0 : head_count_ (head_count), timestamp_ (0),
591 0 : removed_icall_target_ (false), realized_ (false), in_worklist_ (false),
592 0 : inlined_to_ (NULL), location_ (UNKNOWN_LOCATION),
593 0 : call_location_ (UNKNOWN_LOCATION)
594 : {
595 : }
596 :
597 : /* Map from source location (decl_lineno) to profile (count_info). */
598 : typedef std::map<unsigned, count_info> position_count_map;
599 :
600 : /* The indices into the string table identifying the function_instance. */
601 : function_instance_descriptor descriptor_;
602 :
603 : /* Total sample count. */
604 : gcov_type total_count_;
605 :
606 : /* Entry BB's sample count. */
607 : gcov_type head_count_;
608 :
609 : /* perf timestamp associated with first execution of function, which is
610 : used to compute node->tp_first_run. */
611 : gcov_type timestamp_;
612 :
613 : /* Map from callsite location to callee function_instance. */
614 : callsite_map callsites;
615 :
616 : /* Map from source location to count_info. */
617 : position_count_map pos_counts;
618 :
619 : /* True if function was removed from indir target list. */
620 : bool removed_icall_target_;
621 :
622 : /* True if function exists in IL. I.e. for toplevel instance we
623 : have corresponding symbol and for inline instance we inlined
624 : to it. */
625 : bool realized_;
626 :
627 : /* True if function is in worklist for merging/offlining. */
628 : bool in_worklist_;
629 :
630 : /* Pointer to outer function instance or NULL if this
631 : is a toplevel one. */
632 : function_instance *inlined_to_;
633 :
634 : /* Location of function and its call (in case it is inlined). */
635 : location_t location_, call_location_;
636 :
637 : /* Turn inline instance to offline. */
638 : static bool offline (function_instance *fn,
639 : vec <function_instance *> &new_functions);
640 :
641 : /* Helper routine for prop_timestamp. */
642 : void prop_timestamp_1 (gcov_type timestamp);
643 : };
644 :
645 : /* Profile for all functions. */
646 : class autofdo_source_profile
647 : {
648 : public:
649 : static autofdo_source_profile *
650 0 : create ()
651 : {
652 0 : autofdo_source_profile *map = new autofdo_source_profile ();
653 :
654 0 : if (map->read ())
655 : return map;
656 0 : delete map;
657 : return NULL;
658 : }
659 :
660 : ~autofdo_source_profile ();
661 :
662 : /* For a given DECL, returns the top-level function_instance. */
663 : function_instance *get_function_instance_by_decl (tree decl, const char * = NULL) const;
664 :
665 : /* For a given DESCRIPTOR, return the matching instance if found. */
666 : function_instance *
667 : get_function_instance_by_descriptor (function_instance_descriptor) const;
668 :
669 : void add_function_instance (function_instance *);
670 :
671 : /* Find count_info for a given gimple STMT. If found, store the count_info
672 : in INFO and return true; otherwise return false.
673 : NODE can be used to specify particular inline clone. */
674 : bool get_count_info (gimple *stmt, count_info *info,
675 : cgraph_node *node = NULL) const;
676 :
677 : /* Find count_info for a given gimple location GIMPLE_LOC. If found,
678 : store the count_info in INFO and return true; otherwise return false.
679 : NODE can be used to specify particular inline clone. */
680 : bool get_count_info (location_t gimple_loc, count_info *info,
681 : cgraph_node *node = NULL) const;
682 :
683 : /* Find total count of the callee of EDGE. */
684 : gcov_type get_callsite_total_count (struct cgraph_edge *edge) const;
685 :
686 : /* Update value profile INFO for STMT within NODE from the inlined indirect
687 : callsite. Return true if INFO is updated. */
688 : bool update_inlined_ind_target (gcall *stmt, count_info *info,
689 : cgraph_node *node);
690 :
691 : void remove_icall_target (cgraph_edge *e);
692 :
693 : /* Offline all functions not defined in the current translation unit. */
694 : void offline_external_functions ();
695 :
696 : void offline_unrealized_inlines ();
697 :
698 : private:
699 : /* Map from pair of function_instance filename and symbol name (in
700 : string_table) to function_instance. */
701 : typedef std::map<function_instance_descriptor, function_instance *>
702 : name_function_instance_map;
703 :
704 0 : autofdo_source_profile () {}
705 :
706 : /* Read AutoFDO profile and returns TRUE on success. */
707 : bool read ();
708 :
709 : /* Return the function_instance in the profile that correspond to the
710 : inline STACK. */
711 : function_instance *
712 : get_function_instance_by_inline_stack (const inline_stack &stack) const;
713 :
714 : /* Find the matching function instance which has DESCRIPTOR as its
715 : descriptor. If not found, also try checking if an instance exists with the
716 : same name which has no associated filename. */
717 : name_function_instance_map::const_iterator find_iter_for_function_instance (
718 : function_instance_descriptor descriptor) const;
719 :
720 : /* Similar to the above, but return a pointer to the instance instead of an
721 : iterator. */
722 : function_instance *
723 : find_function_instance (function_instance_descriptor descriptor) const;
724 :
725 : /* Remove a function instance from the map. Returns true if the entry was
726 : actually deleted. */
727 : bool remove_function_instance (function_instance *inst);
728 :
729 : name_function_instance_map map_;
730 :
731 : auto_vec <function_instance *> duplicate_functions_;
732 : };
733 :
734 : /* Store the summary information from the GCOV file. */
735 : static summary_info *afdo_summary_info;
736 :
737 : /* Store the strings read from the profile data file. */
738 : static string_table *afdo_string_table;
739 :
740 : /* Store the AutoFDO source profile. */
741 : static autofdo_source_profile *afdo_source_profile;
742 :
743 : /* gcov_summary structure to store the profile_info. */
744 : static gcov_summary *afdo_profile_info;
745 :
746 : /* Map from timestamp -> <name, tp_first_run>.
747 :
748 : The purpose of this map is to map 64-bit timestamp values to (1..N) sorted
749 : by ascending order of timestamps and assign that to node->tp_first_run,
750 : since we don't need the full 64-bit range. */
751 : static std::map<gcov_type, int> timestamp_info_map;
752 :
753 : /* Scaling factor for afdo data. Compared to normal profile
754 : AFDO profile counts are much lower, depending on sampling
755 : frequency. We scale data up to reduce effects of roundoff
756 : errors. */
757 :
758 : static gcov_type afdo_count_scale = 1;
759 :
760 : /* Helper functions. */
761 :
762 : /* Return the original name of NAME: strip the suffix that starts
763 : with '.' for names that are generated after auto-profile pass.
764 : This is to match profiled names with the names in the IR at this stage.
765 : Note that we only have to strip suffix and not in the middle.
766 : Caller is responsible for freeing RET. */
767 :
768 : static char *
769 0 : get_original_name (const char *name, bool alloc = true)
770 : {
771 0 : char *ret = alloc ? xstrdup (name) : const_cast<char *> (name);
772 0 : char *last_dot = strrchr (ret, '.');
773 0 : if (last_dot == NULL)
774 : return ret;
775 0 : bool only_digits = true;
776 : char *ptr = last_dot;
777 0 : while (*(++ptr) != 0)
778 0 : if (*ptr < '0' || *ptr > '9')
779 : {
780 : only_digits = false;
781 : break;
782 : }
783 0 : if (only_digits)
784 0 : *last_dot = 0;
785 0 : char *next_dot = strrchr (ret, '.');
786 : /* if nested function such as foo.0, return foo.0 */
787 0 : if (next_dot == NULL)
788 : {
789 0 : *last_dot = '.';
790 0 : return ret;
791 : }
792 : /* Suffixes of clones that compiler generates after auto-profile. */
793 0 : const char *suffixes[] = {"isra", "constprop", "lto_priv", "part", "cold"};
794 0 : for (unsigned i = 0; i < sizeof (suffixes) / sizeof (const char *); ++i)
795 : {
796 0 : int len = strlen (suffixes[i]);
797 0 : if (len == last_dot - next_dot - 1
798 0 : && strncmp (next_dot + 1, suffixes[i], strlen (suffixes[i])) == 0)
799 : {
800 0 : *next_dot = 0;
801 0 : return get_original_name (ret, false);
802 : }
803 : }
804 : /* Otherwise, it is for clones such as .omp_fn.N that was done before
805 : auto-profile and should be kept as it is. */
806 0 : *last_dot = '.';
807 0 : return ret;
808 : }
809 :
810 : /* Return the combined location, which is a 32bit integer in which
811 : higher 16 bits stores the line offset of LOC to the start lineno
812 : of DECL, The lower 16 bits stores the discriminator. */
813 :
814 : static unsigned
815 0 : get_combined_location (location_t loc, tree decl)
816 : {
817 0 : bool warned = false;
818 : /* TODO: allow more bits for line and less bits for discriminator. */
819 0 : if ((LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) >= (1<<15)
820 0 : || (LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) <= -(1<<15))
821 0 : warned = warning_at (loc, OPT_Wauto_profile,
822 : "auto-profile cannot encode offset %i "
823 : "that exceeds 16 bytes",
824 0 : LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl));
825 0 : if (warned)
826 0 : inform (DECL_SOURCE_LOCATION (decl), "location offset is related to");
827 0 : if ((unsigned)get_discriminator_from_loc (loc) >= (1u << 16))
828 0 : warning_at (loc, OPT_Wauto_profile,
829 : "auto-profile cannot encode discriminators "
830 : "that exceeds 16 bytes");
831 0 : return ((unsigned)(LOCATION_LINE (loc) - DECL_SOURCE_LINE (decl)) << 16)
832 0 : | get_discriminator_from_loc (loc);
833 : }
834 :
835 : /* Return the function decl of a given lexical BLOCK. */
836 :
837 : static tree
838 0 : get_function_decl_from_block (tree block)
839 : {
840 0 : if (!inlined_function_outer_scope_p (block))
841 : return NULL_TREE;
842 :
843 0 : return BLOCK_ABSTRACT_ORIGIN (block);
844 : }
845 :
846 : /* Dump LOC to F. */
847 :
848 : static void
849 0 : dump_afdo_loc (FILE *f, unsigned loc)
850 : {
851 0 : if (loc & 65535)
852 0 : fprintf (f, "%i.%i", loc >> 16, loc & 65535);
853 : else
854 0 : fprintf (f, "%i", loc >> 16);
855 0 : }
856 :
857 : /* Return assembler name as in symbol table and DW_AT_linkage_name. */
858 :
859 : static const char *
860 0 : raw_symbol_name (const char *asmname)
861 : {
862 : /* If we start supporting user_label_prefixes, add_linkage_attr will also
863 : need to be fixed. */
864 0 : if (strlen (user_label_prefix))
865 0 : sorry ("auto-profile is not supported for targets with user label prefix");
866 0 : return asmname + (asmname[0] == '*');
867 : }
868 :
869 : /* Convenience wrapper that looks up assembler name. */
870 :
871 : static const char *
872 0 : raw_symbol_name (tree decl)
873 : {
874 0 : return raw_symbol_name (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)));
875 : }
876 :
877 : /* Dump STACK to F. */
878 :
879 : static void
880 0 : dump_inline_stack (FILE *f, inline_stack *stack)
881 : {
882 0 : bool first = true;
883 0 : for (decl_lineno &p : *stack)
884 : {
885 0 : fprintf (f, "%s%s:",
886 : first ? "" : "; ",
887 : raw_symbol_name (p.decl));
888 0 : dump_afdo_loc (f, p.afdo_loc);
889 0 : first = false;
890 : }
891 0 : fprintf (f, "\n");
892 0 : }
893 :
894 : /* Store inline stack for STMT in STACK. */
895 :
896 : static void
897 0 : get_inline_stack (location_t locus, inline_stack *stack,
898 : tree fn = current_function_decl)
899 : {
900 0 : if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
901 : return;
902 :
903 0 : tree block = LOCATION_BLOCK (locus);
904 0 : if (block && TREE_CODE (block) == BLOCK)
905 : {
906 0 : for (block = BLOCK_SUPERCONTEXT (block);
907 0 : block && (TREE_CODE (block) == BLOCK);
908 0 : block = BLOCK_SUPERCONTEXT (block))
909 : {
910 0 : location_t tmp_locus = BLOCK_SOURCE_LOCATION (block);
911 0 : if (LOCATION_LOCUS (tmp_locus) == UNKNOWN_LOCATION)
912 0 : continue;
913 :
914 0 : tree decl = get_function_decl_from_block (block);
915 0 : stack->safe_push (
916 0 : {decl, get_combined_location (locus, decl), locus});
917 0 : locus = tmp_locus;
918 : }
919 : }
920 0 : stack->safe_push ({fn, get_combined_location (locus, fn), locus});
921 : }
922 :
923 : /* Same as get_inline_stack for a given node which may be
924 : an inline clone. If NODE is NULL, assume current_function_decl. */
925 : static void
926 0 : get_inline_stack_in_node (location_t locus, inline_stack *stack,
927 : cgraph_node *node)
928 : {
929 0 : if (!node)
930 0 : return get_inline_stack (locus, stack);
931 0 : do
932 : {
933 0 : get_inline_stack (locus, stack, node->decl);
934 : /* If caller is inlined, continue building stack. */
935 0 : if (!node->inlined_to)
936 : node = NULL;
937 : else
938 : {
939 0 : locus = gimple_location (node->callers->call_stmt);
940 0 : node = node->callers->caller;
941 : }
942 : }
943 0 : while (node);
944 : }
945 :
946 : /* Return combined location of LOCUS within BLOCK that is in
947 : function FN.
948 :
949 : This is a 32bit integer in which higher 16 bits stores the line offset of
950 : LOC to the start lineno of DECL, The lower 16 bits stores the
951 : discriminator. */
952 :
953 : static unsigned
954 0 : get_relative_location_for_locus (tree fn, tree block, location_t locus)
955 : {
956 0 : if (LOCATION_LOCUS (locus) == UNKNOWN_LOCATION)
957 : return -1;
958 :
959 0 : for (; block && (TREE_CODE (block) == BLOCK);
960 0 : block = BLOCK_SUPERCONTEXT (block))
961 0 : if (inlined_function_outer_scope_p (block))
962 0 : return get_combined_location (locus,
963 0 : get_function_decl_from_block (block));
964 0 : return get_combined_location (locus, fn);
965 : }
966 :
967 : /* Return combined location of STMT in function FN. */
968 :
969 : static unsigned
970 0 : get_relative_location_for_stmt (tree fn, gimple *stmt)
971 : {
972 0 : return get_relative_location_for_locus
973 0 : (fn, LOCATION_BLOCK (gimple_location (stmt)),
974 0 : gimple_location (stmt));
975 : }
976 :
977 : /* Return either the basename or the realpath for a given path based on
978 : PARAM_PROFILE_FUNC_INTERNAL_ID. */
979 :
980 : static const char *
981 0 : get_normalized_path (const char *path, bool from_gcov = false)
982 : {
983 0 : if (param_profile_func_internal_id == 1)
984 : /* The GCOV will already contain the entire path. It doesn't need to be
985 : normalized with lrealpath (). */
986 0 : return from_gcov ? path : lrealpath (path);
987 0 : return lbasename (path);
988 : }
989 :
990 : /* Member functions for summary_info. */
991 :
992 : bool
993 0 : summary_info::read ()
994 : {
995 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_SUMMARY)
996 : return false;
997 :
998 0 : total_count = gcov_read_counter ();
999 0 : max_count = gcov_read_counter ();
1000 0 : max_function_count = gcov_read_counter ();
1001 0 : num_counts = gcov_read_counter ();
1002 0 : num_functions = gcov_read_counter ();
1003 0 : uint64_t num_detailed_summaries = gcov_read_counter ();
1004 0 : gcc_checking_assert (num_detailed_summaries == NUM_PERCENTILES);
1005 0 : for (uint64_t i = 0; i < num_detailed_summaries; i++)
1006 : {
1007 0 : detailed_summaries[i].cutoff = gcov_read_unsigned ();
1008 0 : detailed_summaries[i].min_count = gcov_read_counter ();
1009 0 : detailed_summaries[i].num_counts = gcov_read_counter ();
1010 : }
1011 :
1012 0 : return !gcov_is_error ();
1013 : }
1014 :
1015 : /* Get the minimum count required for percentile CUTOFF. */
1016 :
1017 : uint64_t
1018 0 : summary_info::get_threshold_count (uint32_t cutoff)
1019 : {
1020 : /* The cutoffs stored in the GCOV are fractions multiplied by 1,000,000. */
1021 0 : gcc_checking_assert (cutoff <= 1'000'000);
1022 0 : unsigned idx = 0;
1023 : /* Find the first cutoff at least as high as CUTOFF. */
1024 0 : for (; idx < NUM_PERCENTILES; idx++)
1025 0 : if (detailed_summaries[idx].cutoff >= cutoff)
1026 : break;
1027 0 : idx = std::min (NUM_PERCENTILES - 1, idx);
1028 0 : return detailed_summaries[idx].min_count;
1029 : }
1030 :
1031 : /* Member functions for string_table. */
1032 :
1033 : /* Deconstructor. */
1034 :
1035 0 : string_table::~string_table ()
1036 : {
1037 0 : for (unsigned i = 0; i < symbol_names_.length (); i++)
1038 0 : free (const_cast<char *> (symbol_names_[i]));
1039 0 : for (unsigned i = 0; i < filenames_.length (); i++)
1040 0 : free (const_cast<char *> (filenames_[i]));
1041 0 : for (auto it = original_names_map_.begin (); it != original_names_map_.end ();
1042 0 : it++)
1043 0 : free (it->second);
1044 0 : }
1045 :
1046 :
1047 : /* Return the index of a given function NAME. Return -1 if NAME is not
1048 : found in string table. */
1049 :
1050 : int
1051 0 : string_table::get_index (const char *name) const
1052 : {
1053 0 : if (name == NULL)
1054 : return -1;
1055 0 : string_index_map::const_iterator iter = symbol_name_map_.find (name);
1056 0 : if (iter == symbol_name_map_.end ())
1057 : return -1;
1058 :
1059 0 : return iter->second;
1060 : }
1061 :
1062 : /* Return the index of a given function DECL. Return -1 if DECL is not
1063 : found in string table. */
1064 :
1065 : int
1066 0 : string_table::get_index_by_decl (tree decl) const
1067 : {
1068 0 : const char *name = raw_symbol_name (decl);
1069 0 : int ret = get_index (name);
1070 0 : if (ret != -1)
1071 : return ret;
1072 0 : if (DECL_FROM_INLINE (decl))
1073 0 : return get_index_by_decl (DECL_ABSTRACT_ORIGIN (decl));
1074 :
1075 : return -1;
1076 : }
1077 :
1078 : /* Return the function name of a given INDEX. */
1079 :
1080 : const char *
1081 0 : string_table::get_symbol_name (int index) const
1082 : {
1083 0 : if (index <= 0 || index >= (int) symbol_names_.length ())
1084 0 : fatal_error (UNKNOWN_LOCATION,
1085 : "auto-profile contains invalid symbol name index %d", index);
1086 :
1087 0 : return symbol_names_[index];
1088 : }
1089 :
1090 : /* For a given index, returns the string. */
1091 :
1092 : const char *
1093 0 : string_table::get_filename (int index) const
1094 : {
1095 : /* There may not be any file name for some functions, ignore them. */
1096 0 : if (index == string_table::unknown_filename)
1097 : return "<unknown>";
1098 :
1099 0 : if (index < 0 || index >= (int) filenames_.length ())
1100 0 : fatal_error (UNKNOWN_LOCATION,
1101 : "auto-profile contains invalid filename index %d", index);
1102 :
1103 0 : return filenames_[index];
1104 : }
1105 :
1106 : /* For a given symbol name index, returns the filename index. */
1107 :
1108 : int
1109 0 : string_table::get_filename_by_symbol (int index) const
1110 : {
1111 0 : return get_filename_by_symbol (get_symbol_name (index));
1112 : }
1113 :
1114 : /* For a given function name, returns the filename index. */
1115 :
1116 : int
1117 0 : string_table::get_filename_by_symbol (const char *name) const
1118 : {
1119 0 : auto it = symbol_to_filename_map_.find (name);
1120 0 : if (it != symbol_to_filename_map_.end () && it->second < filenames_.length ())
1121 0 : return it->second;
1122 : return string_table::unknown_filename;
1123 : }
1124 :
1125 : /* For a given filename, returns the index. */
1126 :
1127 : int
1128 0 : string_table::get_filename_index (const char *name) const
1129 : {
1130 0 : auto iter = filename_map_.find (name);
1131 0 : return iter == filename_map_.end () ? string_table::unknown_filename
1132 0 : : iter->second;
1133 : }
1134 :
1135 : /* Get the original name and file name index for a node. This will return the
1136 : name from the current TU if there are multiple symbols that map to
1137 : NAME. */
1138 :
1139 : std::pair<const char *, int>
1140 0 : string_table::get_original_name (const char *name) const
1141 : {
1142 : /* Check if the un-prefixed name differs from the actual name. */
1143 0 : auto stripped = original_names_map_.find (name);
1144 :
1145 : /* The original name for the symbol is its name, i.e. there are no
1146 : suffixes. */
1147 0 : if (stripped == original_names_map_.end ())
1148 0 : return {name, get_filename_by_symbol (name)};
1149 :
1150 : /* Figure out if a clash exists. */
1151 0 : auto clash = clashing_names_map_.find (stripped->second);
1152 0 : gcc_checking_assert (clash != clashing_names_map_.end ());
1153 :
1154 : /* Try to find a function from the current TU. */
1155 0 : gcc_checking_assert (clash->second.length () >= 1);
1156 0 : symtab_node *n
1157 0 : = cgraph_node::get_for_asmname (get_identifier (stripped->second));
1158 0 : if (n && is_a<cgraph_node *> (n))
1159 0 : for (cgraph_node *cn = dyn_cast<cgraph_node *> (n); cn;)
1160 : {
1161 : /* Check if there is a symbol in the current TU that has the same name
1162 : as in the GCOV. */
1163 0 : for (auto name : clash->second)
1164 : {
1165 0 : int filename_idx = get_filename_by_symbol (name);
1166 0 : if (cn->definition && cn->has_gimple_body_p ()
1167 0 : && !strcmp (get_normalized_path (DECL_SOURCE_FILE (cn->decl)),
1168 : get_filename (filename_idx)))
1169 0 : return {stripped->second, filename_idx};
1170 : }
1171 0 : cn = dyn_cast<cgraph_node *> (cn->next_sharing_asm_name);
1172 : }
1173 :
1174 : /* No match found. Just stick to the current symbol and return the stripped
1175 : name. */
1176 0 : return {stripped->second, get_filename_by_symbol (name)};
1177 : }
1178 :
1179 : /* Add new symbol name STRING (with an associated file name FILENAME_IDX) and
1180 : return its index. */
1181 :
1182 : int
1183 0 : string_table::add_symbol_name (const char *string, int filename_idx)
1184 : {
1185 0 : gcc_checking_assert (
1186 : filename_idx == string_table::unknown_filename
1187 : || (filename_idx >= 0 && filename_idx < (int) filenames_.length ()));
1188 0 : symbol_names_.safe_push (string);
1189 0 : symbol_name_map_[symbol_names_.last ()] = symbol_names_.length () - 1;
1190 0 : symbol_to_filename_map_[symbol_names_.last ()] = filename_idx;
1191 0 : return symbol_names_.length () - 1;
1192 : }
1193 :
1194 : /* Add new filename and return its index (returning the same if it already
1195 : exists). */
1196 :
1197 : int
1198 0 : string_table::add_filename (const char *name)
1199 : {
1200 0 : auto it = filename_map_.find (name);
1201 0 : if (it != filename_map_.end ())
1202 0 : return it->second;
1203 0 : filenames_.safe_push (xstrdup (name));
1204 0 : return filenames_.length () - 1;
1205 : }
1206 :
1207 : /* Read the string table. Return TRUE if reading is successful. */
1208 :
1209 : bool
1210 0 : string_table::read ()
1211 : {
1212 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_FILE_NAMES)
1213 : return false;
1214 : /* Skip the length of the section. */
1215 0 : gcov_read_unsigned ();
1216 : /* Read in the file name table. */
1217 0 : unsigned file_num = gcov_read_unsigned ();
1218 0 : filenames_.reserve (file_num);
1219 0 : for (unsigned i = 0; i < file_num; i++)
1220 : {
1221 0 : const char *filename = gcov_read_string ();
1222 0 : filenames_.quick_push (xstrdup (get_normalized_path (filename, true)));
1223 0 : filename_map_[filenames_.last ()] = i;
1224 0 : free (const_cast<char *> (filename));
1225 0 : if (gcov_is_error ())
1226 : return false;
1227 : }
1228 : /* Read in the function name -> file name table. */
1229 0 : unsigned string_num = gcov_read_unsigned ();
1230 0 : symbol_names_.reserve (string_num);
1231 0 : for (unsigned i = 0; i < string_num; i++)
1232 : {
1233 0 : symbol_names_.quick_push (const_cast<char *> (gcov_read_string ()));
1234 0 : symbol_name_map_[symbol_names_.last ()] = i;
1235 0 : unsigned filename_idx = gcov_read_unsigned ();
1236 0 : symbol_to_filename_map_[symbol_names_.last ()] = filename_idx;
1237 0 : char *original = const_cast<char *> (
1238 0 : autofdo::get_original_name (symbol_names_.last ()));
1239 0 : if (strcmp (original, symbol_names_.last ()))
1240 : {
1241 : /* Take ownership of ORIGINAL. */
1242 0 : original_names_map_[symbol_names_.last ()] = original;
1243 0 : clashing_names_map_[original].safe_push (i);
1244 : /* It is possible that a public symbol with the stripped name exists.
1245 : If it does exist, add it as well. */
1246 0 : auto publik = symbol_name_map_.find (original);
1247 0 : if (publik != symbol_name_map_.end ()
1248 0 : && clashing_names_map_.find (publik->first)
1249 0 : == clashing_names_map_.end ())
1250 0 : clashing_names_map_[publik->first].safe_push (publik->second);
1251 : }
1252 : else
1253 : /* There are no suffixes to remove. */
1254 0 : free (original);
1255 :
1256 0 : if (gcov_is_error ())
1257 0 : return false;
1258 : }
1259 0 : return true;
1260 : }
1261 :
1262 : /* Return cgraph node corresponding to given NAME_INDEX,
1263 : NULL if unavailable. */
1264 : cgraph_node *
1265 0 : string_table::get_cgraph_node (int name_index)
1266 : {
1267 0 : const char *sname = get_symbol_name (name_index);
1268 :
1269 0 : symtab_node *n = cgraph_node::get_for_asmname (get_identifier (sname));
1270 0 : for (;n; n = n->next_sharing_asm_name)
1271 0 : if (cgraph_node *cn = dyn_cast <cgraph_node *> (n))
1272 0 : if (cn->definition && cn->has_gimple_body_p ())
1273 : return cn;
1274 : return NULL;
1275 : }
1276 :
1277 : /* Return corresponding cgraph node. */
1278 :
1279 : cgraph_node *
1280 0 : function_instance::get_cgraph_node ()
1281 : {
1282 0 : return afdo_string_table->get_cgraph_node (symbol_name ());
1283 : }
1284 :
1285 : /* Member functions for function_instance. */
1286 :
1287 0 : function_instance::~function_instance ()
1288 : {
1289 0 : gcc_assert (!in_worklist_p ());
1290 0 : for (callsite_map::iterator iter = callsites.begin ();
1291 0 : iter != callsites.end (); ++iter)
1292 0 : delete iter->second;
1293 0 : }
1294 :
1295 : /* Propagate timestamp TS of function_instance to inlined instances if it's
1296 : not already set. */
1297 :
1298 : void
1299 0 : function_instance::prop_timestamp_1 (gcov_type ts)
1300 : {
1301 0 : if (!timestamp () && total_count () > 0)
1302 0 : set_timestamp (ts);
1303 0 : for (auto it = callsites.begin (); it != callsites.end (); ++it)
1304 0 : it->second->prop_timestamp_1 (ts);
1305 0 : }
1306 :
1307 : void
1308 0 : function_instance::prop_timestamp (void)
1309 : {
1310 0 : prop_timestamp_1 (timestamp ());
1311 0 : }
1312 :
1313 : /* Traverse callsites of the current function_instance to find one at the
1314 : location of LINENO and callee name represented in DECL. */
1315 :
1316 : function_instance *
1317 0 : function_instance::get_function_instance_by_decl (unsigned lineno,
1318 : tree decl,
1319 : location_t location) const
1320 : {
1321 0 : int func_name_idx = afdo_string_table->get_index_by_decl (decl);
1322 0 : if (func_name_idx != -1)
1323 : {
1324 0 : callsite_map::const_iterator ret
1325 0 : = callsites.find (std::make_pair (lineno, func_name_idx));
1326 0 : if (ret != callsites.end ())
1327 0 : return ret->second;
1328 : }
1329 0 : if (DECL_FROM_INLINE (decl))
1330 : {
1331 0 : function_instance
1332 0 : *ret = get_function_instance_by_decl (lineno,
1333 0 : DECL_ABSTRACT_ORIGIN (decl),
1334 : location);
1335 0 : return ret;
1336 : }
1337 0 : if (dump_enabled_p ())
1338 : {
1339 0 : for (auto const &iter : callsites)
1340 0 : if (iter.first.first == lineno)
1341 0 : dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
1342 0 : dump_user_location_t::from_location_t (location),
1343 : "auto-profile has mismatched function name %s"
1344 : " instead of %s at loc %i:%i",
1345 : afdo_string_table->get_symbol_name (
1346 0 : iter.first.second),
1347 : raw_symbol_name (decl), lineno >> 16,
1348 : lineno & 65535);
1349 : }
1350 :
1351 : return NULL;
1352 : }
1353 :
1354 : /* Merge profile of OTHER to THIS. Note that cloning hasn't been performed
1355 : when we annotate the CFG (at this stage). */
1356 :
1357 : void
1358 0 : function_instance::merge (function_instance *other,
1359 : vec <function_instance *> &new_functions)
1360 : {
1361 : /* Do not merge to itself and only merge functions of same name. */
1362 0 : gcc_checking_assert (other != this
1363 : && other->symbol_name () == symbol_name ());
1364 :
1365 0 : if (file_name () != other->file_name ())
1366 : return;
1367 :
1368 0 : total_count_ += other->total_count_;
1369 0 : if (other->total_count () && total_count () && other->head_count () == -1)
1370 0 : head_count_ = -1;
1371 0 : else if (head_count_ != -1)
1372 0 : head_count_ += other->head_count_;
1373 :
1374 : /* While merging timestamps, set the one that occurs earlier. */
1375 0 : if (timestamp () == 0
1376 0 : || (other->timestamp () > 0
1377 0 : && other->timestamp () < timestamp ()))
1378 0 : set_timestamp (other->timestamp ());
1379 :
1380 0 : bool changed = true;
1381 :
1382 0 : while (changed)
1383 : {
1384 0 : changed = false;
1385 : /* If both function instances agree on particular inlined function,
1386 : merge profiles. Otherwise offline the instance. */
1387 0 : for (callsite_map::const_iterator iter = other->callsites.begin ();
1388 0 : iter != other->callsites.end ();)
1389 0 : if (callsites.count (iter->first) == 0)
1390 : {
1391 0 : function_instance *f = iter->second;
1392 0 : if (dump_file)
1393 : {
1394 0 : fprintf (dump_file, " Mismatch in inlined functions;"
1395 : " offlining in merge source:");
1396 0 : f->dump_inline_stack (dump_file);
1397 0 : fprintf (dump_file, "\n");
1398 : }
1399 : /* We already merged outer part of the function accounting
1400 : the inlined call; compensate. */
1401 0 : for (function_instance *s = this; s; s = s->inlined_to ())
1402 : {
1403 0 : s->total_count_ -= f->total_count ();
1404 0 : gcc_checking_assert (s->total_count_ >= 0);
1405 : }
1406 0 : other->callsites.erase (iter);
1407 0 : function_instance::offline (f, new_functions);
1408 : /* Start from beginning as merging might have offlined
1409 : some functions in the case of recursive inlining. */
1410 0 : iter = other->callsites.begin ();
1411 : }
1412 : else
1413 0 : ++iter;
1414 0 : for (callsite_map::const_iterator iter = callsites.begin ();
1415 0 : iter != callsites.end ();)
1416 0 : if (other->callsites.count (iter->first) == 0)
1417 : {
1418 0 : function_instance *f = iter->second;
1419 0 : if (dump_file)
1420 : {
1421 0 : fprintf (dump_file, " Mismatch in inlined functions;"
1422 : " offlining in merge destination:");
1423 0 : f->dump_inline_stack (dump_file);
1424 0 : fprintf (dump_file, "\n");
1425 : }
1426 0 : callsites.erase (iter);
1427 0 : function_instance::offline (f, new_functions);
1428 0 : iter = callsites.begin ();
1429 0 : changed = true;
1430 : }
1431 : else
1432 0 : ++iter;
1433 : }
1434 0 : for (callsite_map::const_iterator iter = other->callsites.begin ();
1435 0 : iter != other->callsites.end (); ++iter)
1436 : {
1437 0 : if (dump_file)
1438 : {
1439 0 : fprintf (dump_file, " Merging profile for inlined function\n"
1440 : " from: ");
1441 0 : iter->second->dump_inline_stack (dump_file);
1442 0 : fprintf (dump_file, " total:%" PRIu64 "\n to : ",
1443 0 : (int64_t)iter->second->total_count ());
1444 0 : callsites[iter->first]->dump_inline_stack (dump_file);
1445 0 : fprintf (dump_file, " total:%" PRIu64 "\n",
1446 0 : (int64_t)callsites[iter->first]->total_count ());
1447 : }
1448 :
1449 0 : callsites[iter->first]->merge (iter->second, new_functions);
1450 : }
1451 :
1452 0 : for (position_count_map::const_iterator iter = other->pos_counts.begin ();
1453 0 : iter != other->pos_counts.end (); ++iter)
1454 0 : if (pos_counts.count (iter->first) == 0)
1455 0 : pos_counts[iter->first] = iter->second;
1456 : else
1457 : {
1458 0 : pos_counts[iter->first].count += iter->second.count;
1459 0 : for (icall_target_map::const_iterator titer
1460 0 : = iter->second.targets.begin ();
1461 0 : titer != iter->second.targets.end (); ++titer)
1462 0 : if (pos_counts[iter->first].targets.count (titer->first) == 0)
1463 0 : pos_counts[iter->first].targets[titer->first]
1464 0 : = titer->second;
1465 : else
1466 0 : pos_counts[iter->first].targets[titer->first]
1467 0 : += titer->second;
1468 : }
1469 : }
1470 :
1471 : /* Make inline function FN offline.
1472 : If toplevel function of same name already exists, then merge profiles.
1473 : Otherwise turn FN toplevel. Return true if new toplevel function
1474 : was introduced.
1475 : If new toplevel functions are created and NEW_FUNCTIONS != NULL,
1476 : add them to NEW_FUNCTIONS.
1477 :
1478 : TODO: When offlining indirect call we lose information about the
1479 : call target. It should be possible to add it into
1480 : targets histogram. */
1481 :
1482 : bool
1483 0 : function_instance::offline (function_instance *fn,
1484 : vec <function_instance *> &new_functions)
1485 : {
1486 0 : gcc_checking_assert (fn->inlined_to ());
1487 0 : for (function_instance *s = fn->inlined_to (); s; s = s->inlined_to ())
1488 : {
1489 0 : s->total_count_ -= fn->total_count ();
1490 0 : gcc_checking_assert (s->total_count_ >= 0);
1491 : }
1492 0 : function_instance *to
1493 0 : = afdo_source_profile->get_function_instance_by_descriptor (
1494 : fn->get_descriptor ());
1495 0 : fn->set_inlined_to (NULL);
1496 : /* If there is offline function of same name, we need to merge profile.
1497 : Delay this by adding function to a worklist so we do not run into
1498 : problem with recursive inlining. */
1499 0 : if (to)
1500 : {
1501 0 : if (fn->in_worklist_p ())
1502 : return false;
1503 0 : fn->set_in_worklist ();
1504 0 : new_functions.safe_push (fn);
1505 0 : if (dump_file)
1506 : {
1507 0 : fprintf (dump_file, " Recoding duplicate: ");
1508 0 : to->dump_inline_stack (dump_file);
1509 0 : fprintf (dump_file, "\n");
1510 : }
1511 : return true;
1512 : }
1513 0 : if (dump_file)
1514 : {
1515 0 : fprintf (dump_file, " Added as offline instance: ");
1516 0 : fn->dump_inline_stack (dump_file);
1517 0 : fprintf (dump_file, "\n");
1518 : }
1519 0 : if (fn->total_count ())
1520 0 : fn->head_count_ = -1;
1521 0 : afdo_source_profile->add_function_instance (fn);
1522 0 : fn->set_in_worklist ();
1523 0 : new_functions.safe_push (fn);
1524 0 : return true;
1525 : }
1526 :
1527 : /* Offline all inlined functions with name in SEEN.
1528 : If new toplevel functions are created, add them to NEW_FUNCTIONS. */
1529 :
1530 : void
1531 0 : function_instance::offline_if_in_set (name_index_set &seen,
1532 : vec <function_instance *> &new_functions)
1533 : {
1534 0 : for (callsite_map::const_iterator iter = callsites.begin ();
1535 0 : iter != callsites.end ();)
1536 0 : if (seen.contains (iter->first.second))
1537 : {
1538 0 : function_instance *f = iter->second;
1539 0 : if (dump_file)
1540 : {
1541 0 : fprintf (dump_file, "Offlining function inlined to other module: ");
1542 0 : f->dump_inline_stack (dump_file);
1543 0 : fprintf (dump_file, "\n");
1544 : }
1545 0 : iter = callsites.erase (iter);
1546 0 : function_instance::offline (f, new_functions);
1547 : /* Start from beginning as merging might have offlined
1548 : some functions in the case of recursive inlining. */
1549 0 : iter = callsites.begin ();
1550 : }
1551 : else
1552 : {
1553 0 : iter->second->offline_if_in_set (seen, new_functions);
1554 0 : ++iter;
1555 : }
1556 0 : }
1557 :
1558 : /* Try to check if inlined_fn can correspond to a call of function N.
1559 : Return non-zero if it corresponds and 2 if renaming was done. */
1560 :
1561 : static int
1562 0 : match_with_target (cgraph_node *n,
1563 : gimple *stmt,
1564 : function_instance *inlined_fn,
1565 : cgraph_node *orig_callee)
1566 : {
1567 0 : cgraph_node *callee = orig_callee->ultimate_alias_target ();
1568 0 : const char *symbol_name = raw_symbol_name (callee->decl);
1569 0 : const char *name
1570 0 : = afdo_string_table->get_symbol_name (inlined_fn->symbol_name ());
1571 0 : if (strcmp (name, symbol_name))
1572 : {
1573 0 : int i;
1574 0 : bool in_suffix = false;
1575 0 : for (i = 0; i; i++)
1576 : {
1577 : if (name[i] != symbol_name[i])
1578 : break;
1579 : if (name[i] == '.')
1580 : in_suffix = true;
1581 : }
1582 : /* Accept dwarf names and stripped suffixes. */
1583 0 : if (!strcmp (lang_hooks.dwarf_name (callee->decl, 0),
1584 : afdo_string_table->get_symbol_name (
1585 : inlined_fn->symbol_name ()))
1586 0 : || (!name[i] && symbol_name[i] == '.') || in_suffix)
1587 : {
1588 0 : int index = afdo_string_table->get_index (symbol_name);
1589 0 : if (index == -1)
1590 0 : index = afdo_string_table->add_symbol_name (
1591 0 : xstrdup (symbol_name),
1592 : afdo_string_table->add_filename (
1593 0 : get_normalized_path (DECL_SOURCE_FILE (callee->decl))));
1594 0 : if (dump_file)
1595 0 : fprintf (dump_file,
1596 : " Renaming inlined call target %s to %s\n",
1597 : name, symbol_name);
1598 0 : inlined_fn->set_symbol_name (index);
1599 0 : return 2;
1600 : }
1601 : /* Only warn about declarations. It is possible that the function
1602 : is declared as alias in other module and we inlined cross-module. */
1603 0 : if (callee->definition
1604 0 : && warning (OPT_Wauto_profile,
1605 : "auto-profile of %q+F contains inlined "
1606 : "function with symbol name %s instead of symbol name %s",
1607 : n->decl, name, symbol_name))
1608 0 : inform (gimple_location (stmt), "corresponding call");
1609 : return 0;
1610 : }
1611 : return 1;
1612 : }
1613 :
1614 : static void
1615 0 : dump_stmt (gimple *stmt, count_info *info, function_instance *inlined_fn,
1616 : inline_stack &stack)
1617 : {
1618 0 : if (dump_file)
1619 : {
1620 0 : fprintf (dump_file, " ");
1621 0 : if (!stack.length ())
1622 0 : fprintf (dump_file, " ");
1623 : else
1624 : {
1625 0 : gcc_checking_assert (stack.length () == 1);
1626 0 : fprintf (dump_file, "%5i", stack[0].afdo_loc >> 16);
1627 0 : if (stack[0].afdo_loc & 65535)
1628 0 : fprintf (dump_file, ".%-5i", stack[0].afdo_loc & 65535);
1629 : else
1630 0 : fprintf (dump_file, " ");
1631 0 : if (info)
1632 0 : fprintf (dump_file, "%9" PRIu64 " ", (int64_t)info->count);
1633 0 : else if (inlined_fn)
1634 0 : fprintf (dump_file, " inlined ");
1635 : else
1636 0 : fprintf (dump_file, " no info ");
1637 : }
1638 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
1639 : }
1640 0 : }
1641 :
1642 : /* Lookup count and warn about duplicates. */
1643 : count_info *
1644 0 : function_instance::lookup_count (location_t loc, inline_stack &stack,
1645 : cgraph_node *node)
1646 : {
1647 0 : gcc_checking_assert (stack.length () < 2);
1648 0 : if (stack.length ())
1649 : {
1650 0 : int c = pos_counts.count (stack[0].afdo_loc);
1651 0 : if (c > 1
1652 : && warning (OPT_Wauto_profile,
1653 : "duplicated count information"
1654 : " in auto-profile of %q+F"
1655 : " with relative location %i discriminator %i",
1656 : node->decl, stack[0].afdo_loc >> 16,
1657 : stack[0].afdo_loc & 65535))
1658 : inform (loc, "corresponding source location");
1659 0 : if (c)
1660 0 : return &pos_counts[stack[0].afdo_loc];
1661 : }
1662 : return NULL;
1663 : }
1664 :
1665 : /* Mark expr locations as used. */
1666 : void
1667 0 : mark_expr_locations (function_instance *f, tree t, cgraph_node *node,
1668 : hash_set<const count_info *> &counts)
1669 : {
1670 0 : inline_stack stack;
1671 0 : return;
1672 : if (!t)
1673 : return;
1674 : do
1675 : {
1676 : get_inline_stack_in_node (EXPR_LOCATION (t), &stack, node);
1677 : /* FIXME: EXPR_LOCATION does not always originate from current
1678 : function. */
1679 : if (stack.length () > 1)
1680 : break;
1681 : count_info *info = f->lookup_count (EXPR_LOCATION (t), stack, node);
1682 : if (info)
1683 : counts.add (info);
1684 : if (handled_component_p (t))
1685 : t = TREE_OPERAND (t, 0);
1686 : else
1687 : break;
1688 : }
1689 : while (true);
1690 0 : }
1691 :
1692 : /* Match function instance with gimple body.
1693 : Report mismatches, attempt to fix them if possible and remove data we will
1694 : not use.
1695 :
1696 : Set location and call_location so we can output diagnostics and know what
1697 : functions was already matched. */
1698 :
1699 : bool
1700 0 : function_instance::match (cgraph_node *node,
1701 : vec <function_instance *> &new_functions,
1702 : name_index_map &to_symbol_name)
1703 : {
1704 0 : if (get_location () != UNKNOWN_LOCATION)
1705 : return false;
1706 0 : set_location (DECL_SOURCE_LOCATION (node->decl));
1707 0 : if (dump_file)
1708 : {
1709 0 : fprintf (dump_file,
1710 : "\nMatching gimple function %s with auto profile: ",
1711 : node->dump_name ());
1712 0 : dump_inline_stack (dump_file);
1713 0 : fprintf (dump_file, "\n");
1714 : }
1715 0 : basic_block bb;
1716 : /* Sets used to track if entries in auto-profile are useful. */
1717 0 : hash_set<const count_info *> counts;
1718 0 : hash_set<const count_info *> targets;
1719 0 : hash_set<const function_instance *> functions;
1720 0 : hash_set<const function_instance *> functions_to_offline;
1721 :
1722 : /* We try to fill in lost disciminator if there is unique call
1723 : with given line number. This map is used to record them. */
1724 0 : hash_map<int_hash <int, -1, -2>,auto_vec <gcall *>> lineno_to_call;
1725 0 : bool lineno_to_call_computed = false;
1726 :
1727 0 : for (tree arg = DECL_ARGUMENTS (node->decl); arg; arg = DECL_CHAIN (arg))
1728 : {
1729 0 : inline_stack stack;
1730 :
1731 0 : get_inline_stack_in_node (DECL_SOURCE_LOCATION (arg), &stack, node);
1732 0 : count_info *info = lookup_count (DECL_SOURCE_LOCATION (arg), stack, node);
1733 0 : if (stack.length () && dump_file)
1734 : {
1735 0 : gcc_checking_assert (stack.length () == 1);
1736 0 : fprintf (dump_file, "%5i", stack[0].afdo_loc >> 16);
1737 0 : if (stack[0].afdo_loc & 65535)
1738 0 : fprintf (dump_file, " .%-5i arg", stack[0].afdo_loc & 65535);
1739 : else
1740 0 : fprintf (dump_file, " arg ");
1741 0 : print_generic_expr (dump_file, arg);
1742 0 : fprintf (dump_file, "\n");
1743 : }
1744 0 : if (info)
1745 0 : counts.add (info);
1746 0 : }
1747 0 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
1748 : {
1749 0 : if (dump_file)
1750 0 : fprintf (dump_file, " basic block %i\n", bb->index);
1751 0 : for (gphi_iterator gpi = gsi_start_phis (bb);
1752 0 : !gsi_end_p (gpi);
1753 0 : gsi_next (&gpi))
1754 : {
1755 0 : gphi *phi = gpi.phi ();
1756 0 : inline_stack stack;
1757 :
1758 : /* We do not assign discriminators to PHI nodes.
1759 : In case we every start using them, we will need to
1760 : update tree-cfg.cc::assign_discriminators. */
1761 0 : gcc_assert (gimple_location (phi) == UNKNOWN_LOCATION);
1762 0 : get_inline_stack_in_node (gimple_location (phi), &stack, node);
1763 0 : count_info *info = lookup_count (gimple_location (phi), stack, node);
1764 0 : gcc_assert (!info);
1765 0 : dump_stmt (phi, info, NULL, stack);
1766 0 : counts.add (info);
1767 : /* PHI arguments are indexed by incoming (predecessor) edges. */
1768 0 : for (edge e : bb->preds)
1769 : {
1770 0 : location_t phi_loc
1771 0 : = gimple_phi_arg_location_from_edge (phi, e);
1772 0 : inline_stack stack;
1773 0 : get_inline_stack_in_node (phi_loc, &stack, node);
1774 0 : count_info *info = lookup_count (phi_loc, stack, node);
1775 0 : if (info)
1776 0 : counts.add (info);
1777 0 : gcc_checking_assert (stack.length () < 2);
1778 0 : mark_expr_locations (this,
1779 : gimple_phi_arg_def_from_edge (phi, e),
1780 : node, counts);
1781 0 : }
1782 0 : }
1783 : /* TODO: goto locuses are not used for BB annotation. */
1784 0 : for (edge e : bb->succs)
1785 : {
1786 0 : inline_stack stack;
1787 0 : get_inline_stack_in_node (e->goto_locus, &stack, node);
1788 0 : count_info *info = lookup_count (e->goto_locus, stack, node);
1789 0 : if (info)
1790 0 : counts.add (info);
1791 0 : }
1792 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
1793 0 : !gsi_end_p (gsi); gsi_next (&gsi))
1794 : {
1795 0 : inline_stack stack;
1796 0 : gimple *stmt = gsi_stmt (gsi);
1797 0 : get_inline_stack_in_node (gimple_location (stmt), &stack, node);
1798 :
1799 0 : count_info *info = lookup_count (gimple_location (stmt), stack, node);
1800 0 : if (info)
1801 0 : counts.add (info);
1802 0 : for (unsigned int op = 0; op < gimple_num_ops (stmt); op++)
1803 0 : mark_expr_locations (this, gimple_op (stmt, op), node, counts);
1804 0 : if (gimple_code (stmt) == GIMPLE_CALL)
1805 : {
1806 0 : function_instance *inlined_fn = NULL;
1807 0 : function_instance *inlined_fn_nodisc = NULL;
1808 : /* Lookup callsite. */
1809 0 : if (stack.length ())
1810 : {
1811 0 : int c = 0;
1812 0 : int cnodis = 0;
1813 0 : for (auto const &iter : callsites)
1814 0 : if (iter.first.first == stack[0].afdo_loc)
1815 : {
1816 0 : if (!c)
1817 0 : inlined_fn = iter.second;
1818 0 : c++;
1819 : }
1820 : /* Discriminators are sometimes lost; try to find the
1821 : call without discriminator info. */
1822 0 : else if (iter.first.first == (stack[0].afdo_loc & ~65535))
1823 : {
1824 0 : if (!cnodis)
1825 0 : inlined_fn_nodisc = iter.second;
1826 0 : cnodis++;
1827 : }
1828 0 : if ((c > 1 || (!c && cnodis > 1))
1829 0 : && warning (OPT_Wauto_profile,
1830 : "duplicated callsite in auto-profile of %q+F"
1831 : " with relative location %i,"
1832 : " discriminator %i",
1833 0 : node->decl, stack[0].afdo_loc >> 16,
1834 0 : stack[0].afdo_loc & 65535))
1835 0 : inform (gimple_location (stmt), "corresponding call");
1836 0 : if (inlined_fn && info && info->targets.size ()
1837 0 : && warning (OPT_Wauto_profile,
1838 : "both call targets and inline callsite"
1839 : " information is present in auto-profile"
1840 : " of function %q+F with relative location"
1841 : " %i, discriminator %i",
1842 0 : node->decl, stack[0].afdo_loc >> 16,
1843 0 : stack[0].afdo_loc & 65535))
1844 0 : inform (gimple_location (stmt), "corresponding call");
1845 0 : tree callee = gimple_call_fndecl (stmt);
1846 0 : cgraph_node *callee_node;
1847 0 : unsigned int loc = stack[0].afdo_loc;
1848 0 : bool lost_discriminator = false;
1849 0 : if (!inlined_fn && inlined_fn_nodisc)
1850 : {
1851 0 : if (!lineno_to_call_computed)
1852 : {
1853 0 : basic_block bb2;
1854 0 : FOR_EACH_BB_FN (bb2,
1855 : DECL_STRUCT_FUNCTION (node->decl))
1856 0 : for (gimple_stmt_iterator gsi2
1857 0 : = gsi_start_bb (bb2);
1858 0 : !gsi_end_p (gsi2); gsi_next (&gsi2))
1859 0 : if (gcall *call
1860 0 : = dyn_cast <gcall *> (gsi_stmt (gsi2)))
1861 : {
1862 0 : inline_stack stack2;
1863 0 : get_inline_stack_in_node
1864 0 : (gimple_location (call),
1865 : &stack2, node);
1866 0 : if (stack2.length ())
1867 0 : lineno_to_call.get_or_insert
1868 0 : (stack2[0].afdo_loc >> 16).safe_push (call);
1869 0 : }
1870 : lineno_to_call_computed = true;
1871 : }
1872 : /* If we can determine lost discriminator uniquely,
1873 : use it. */
1874 0 : if (lineno_to_call.get
1875 0 : (stack[0].afdo_loc >> 16)->length () == 1)
1876 : {
1877 0 : if (warning (OPT_Wauto_profile,
1878 : "auto-profile of %q+F seem to contain"
1879 : " lost discriminator %i for"
1880 : " call of %s at relative location %i",
1881 : node->decl, loc & 65535,
1882 : afdo_string_table->get_symbol_name (
1883 : inlined_fn_nodisc->symbol_name ()),
1884 : loc >> 16))
1885 0 : inform (gimple_location (stmt),
1886 : "corresponding call");
1887 0 : inlined_fn = inlined_fn_nodisc;
1888 0 : if (dump_file)
1889 0 : fprintf (dump_file, " Lost discriminator %i\n",
1890 : loc & 65535);
1891 0 : loc = loc & ~65535;
1892 : }
1893 : lost_discriminator = true;
1894 : }
1895 0 : if (callee && (callee_node = cgraph_node::get (callee)))
1896 : {
1897 0 : if (inlined_fn)
1898 : {
1899 0 : int old_name = inlined_fn->symbol_name ();
1900 0 : int r = match_with_target (node, stmt, inlined_fn,
1901 : callee_node);
1902 0 : if (r == 2)
1903 : {
1904 0 : auto iter = callsites.find ({loc, old_name});
1905 0 : gcc_checking_assert (
1906 : old_name != inlined_fn->symbol_name ()
1907 : && iter != callsites.end ()
1908 : && iter->second == inlined_fn);
1909 0 : callsite key2 = {stack[0].afdo_loc,
1910 0 : inlined_fn->symbol_name ()};
1911 0 : callsites.erase (iter);
1912 0 : callsites[key2] = inlined_fn;
1913 : }
1914 0 : if (r)
1915 0 : functions.add (inlined_fn);
1916 : else
1917 0 : functions_to_offline.add (inlined_fn);
1918 : }
1919 :
1920 0 : if (info && info->targets.size () > 1)
1921 0 : warning_at (gimple_location (stmt), OPT_Wauto_profile,
1922 : "auto-profile of %q+F contains multiple"
1923 : " targets for a direct call with relative"
1924 : " location %i, discriminator %i",
1925 0 : node->decl, stack[0].afdo_loc >> 16,
1926 0 : stack[0].afdo_loc & 65535);
1927 : /* We do not need target profile for direct calls. */
1928 0 : if (info)
1929 0 : info->targets.clear ();
1930 : }
1931 : else
1932 : {
1933 0 : if (inlined_fn
1934 0 : && inlined_fn->get_call_location ()
1935 : != UNKNOWN_LOCATION)
1936 : {
1937 0 : if (warning (OPT_Wauto_profile,
1938 : "function contains two calls of the same"
1939 : " relative location +%i,"
1940 : " discriminator %i,"
1941 : " that leads to lost auto-profile",
1942 : loc >> 16,
1943 : loc & 65535))
1944 : {
1945 0 : inform (gimple_location (stmt),
1946 : "location of the first call");
1947 0 : inform (inlined_fn->get_call_location (),
1948 : "location of the second call");
1949 : }
1950 0 : if (dump_file)
1951 0 : fprintf (dump_file,
1952 : " Duplicated call location\n");
1953 0 : inlined_fn = NULL;
1954 : }
1955 0 : if (inlined_fn)
1956 : {
1957 0 : inlined_fn->set_call_location
1958 0 : (gimple_location (stmt));
1959 : /* Do renaming if needed so we can look up
1960 : cgraph node and recurse into inlined function. */
1961 0 : int *newn
1962 0 : = to_symbol_name.get (inlined_fn->symbol_name ());
1963 0 : gcc_checking_assert (
1964 : !newn || *newn != inlined_fn->symbol_name ());
1965 0 : if (newn || lost_discriminator)
1966 : {
1967 0 : auto iter = callsites.find (
1968 0 : {loc, inlined_fn->symbol_name ()});
1969 0 : gcc_checking_assert (iter != callsites.end ()
1970 : && iter->second
1971 : == inlined_fn);
1972 0 : callsite key2
1973 0 : = {stack[0].afdo_loc,
1974 0 : newn ? *newn : inlined_fn->symbol_name ()};
1975 0 : callsites.erase (iter);
1976 0 : callsites[key2] = inlined_fn;
1977 0 : inlined_fn->set_symbol_name (
1978 0 : newn ? *newn : inlined_fn->symbol_name ());
1979 : }
1980 0 : functions.add (inlined_fn);
1981 : }
1982 0 : if (info)
1983 0 : targets.add (info);
1984 : }
1985 : }
1986 0 : dump_stmt (stmt, info, inlined_fn, stack);
1987 : }
1988 : else
1989 0 : dump_stmt (stmt, info, NULL, stack);
1990 0 : }
1991 : }
1992 0 : bool warned = false;
1993 0 : for (auto &iter : pos_counts)
1994 0 : if (iter.second.targets.size ()
1995 0 : && counts.contains (&iter.second)
1996 0 : && !targets.contains (&iter.second))
1997 : {
1998 0 : if (!warned)
1999 0 : warned = warning_at
2000 0 : (DECL_SOURCE_LOCATION (node->decl),
2001 0 : OPT_Wauto_profile,
2002 : "auto-profile of %q+F contains indirect call targets"
2003 : " not associated with an indirect call statement",
2004 : node->decl);
2005 0 : if (warned)
2006 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2007 : "count %" PRIu64
2008 : " with relative location +%i, discriminator %i",
2009 0 : iter.second.count, iter.first >> 16, iter.first & 65535);
2010 0 : if (dump_file)
2011 : {
2012 0 : fprintf (dump_file, "Removing targets of ");
2013 0 : dump_afdo_loc (dump_file, iter.first);
2014 0 : fprintf (dump_file, "\n");
2015 : }
2016 0 : iter.second.targets.clear ();
2017 : }
2018 0 : warned = false;
2019 : /* Profile sometimes contains extra location for start or end of function
2020 : (prologue, epilogue).
2021 : TODO: If present, perhaps it can be used to determine entry block
2022 : and exit block counts. */
2023 0 : unsigned int end_location = get_combined_location
2024 0 : (DECL_STRUCT_FUNCTION (node->decl)->function_end_locus, node->decl);
2025 0 : unsigned int start_location = get_combined_location
2026 0 : (DECL_STRUCT_FUNCTION (node->decl)->function_start_locus, node->decl);
2027 : /* When outputting code to builtins location we use line number 0.
2028 : create_gcov is stupid and happily computes offsets across files.
2029 : Silently ignore it. */
2030 0 : unsigned int zero_location
2031 0 : = ((unsigned)(1-DECL_SOURCE_LINE (node->decl))) << 16;
2032 0 : for (position_count_map::const_iterator iter = pos_counts.begin ();
2033 0 : iter != pos_counts.end ();)
2034 0 : if (!counts.contains (&iter->second))
2035 : {
2036 0 : if (iter->first != end_location
2037 : && iter->first != start_location
2038 : && (iter->first & 65535) != zero_location
2039 : && iter->first
2040 : /* FIXME: dwarf5 does not represent inline stack of debug
2041 : statements and consequently create_gcov is sometimes
2042 : mixing up statements from other functions. Do not warn
2043 : user about this until this problem is solved.
2044 : We still write info into dump file. */
2045 : && 0)
2046 : {
2047 : if (!warned)
2048 : warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
2049 : OPT_Wauto_profile,
2050 : "auto-profile of %q+F contains extra statements",
2051 : node->decl);
2052 : if (warned)
2053 : inform (DECL_SOURCE_LOCATION (node->decl),
2054 : "count %" PRIu64 " with relative location +%i,"
2055 : " discriminator %i",
2056 : iter->second.count, iter->first >> 16,
2057 : iter->first & 65535);
2058 : if ((iter->first >> 16) > (end_location >> 16) && warned)
2059 : inform (DECL_SOURCE_LOCATION (node->decl),
2060 : "location is after end of function");
2061 : }
2062 0 : if (dump_file)
2063 : {
2064 0 : fprintf (dump_file, "Removing unmatched count ");
2065 0 : dump_afdo_loc (dump_file, iter->first);
2066 0 : fprintf (dump_file, ":%" PRIu64, iter->second.count);
2067 0 : for (auto &titer : iter->second.targets)
2068 0 : fprintf (dump_file, " %s:%" PRIu64,
2069 0 : afdo_string_table->get_symbol_name (titer.first),
2070 0 : (int64_t) titer.second);
2071 0 : fprintf (dump_file, "\n");
2072 : }
2073 0 : iter = pos_counts.erase (iter);
2074 : }
2075 : else
2076 0 : iter++;
2077 0 : warned = false;
2078 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2079 0 : iter != callsites.end ();)
2080 0 : if (!functions.contains (iter->second))
2081 : {
2082 0 : function_instance *f = iter->second;
2083 : /* If we did not see the corresponding statement, warn. */
2084 0 : if (!functions_to_offline.contains (iter->second))
2085 : {
2086 0 : if (!warned)
2087 0 : warned = warning_at (DECL_SOURCE_LOCATION (node->decl),
2088 0 : OPT_Wauto_profile,
2089 : "auto-profile of %q+F contains"
2090 : " extra callsites",
2091 : node->decl);
2092 0 : if (warned)
2093 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2094 : "call of %s with total count %" PRId64
2095 : ", relative location +%i, discriminator %i",
2096 0 : afdo_string_table->get_symbol_name (iter->first.second),
2097 0 : iter->second->total_count (), iter->first.first >> 16,
2098 0 : iter->first.first & 65535);
2099 0 : if ((iter->first.first >> 16) > (end_location >> 16) && warned)
2100 0 : inform (DECL_SOURCE_LOCATION (node->decl),
2101 : "location is after end of function");
2102 0 : if (dump_file)
2103 : {
2104 0 : fprintf (dump_file,
2105 : "Offlining inline with no corresponding gimple stmt ");
2106 0 : f->dump_inline_stack (dump_file);
2107 0 : fprintf (dump_file, "\n");
2108 : }
2109 : }
2110 0 : else if (dump_file)
2111 : {
2112 0 : fprintf (dump_file,
2113 : "Offlining mismatched inline ");
2114 0 : f->dump_inline_stack (dump_file);
2115 0 : fprintf (dump_file, "\n");
2116 : }
2117 0 : callsites.erase (iter);
2118 0 : offline (f, new_functions);
2119 0 : iter = callsites.begin ();
2120 : }
2121 : else
2122 0 : iter++;
2123 0 : for (auto &iter : callsites)
2124 0 : if (cgraph_node *n = iter.second->get_cgraph_node ())
2125 0 : iter.second->match (n, new_functions, to_symbol_name);
2126 0 : return true;
2127 0 : }
2128 :
2129 : /* Walk inlined functions and if their name is not in SEEN
2130 : remove it. Also rename function names as given by
2131 : to_symbol_name map. */
2132 :
2133 : void
2134 0 : function_instance::remove_external_functions
2135 : (name_index_set &seen,
2136 : name_index_map &to_symbol_name,
2137 : vec <function_instance *> &new_functions)
2138 : {
2139 0 : auto_vec <callsite, 20> to_rename;
2140 :
2141 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2142 0 : iter != callsites.end ();)
2143 0 : if (!seen.contains (iter->first.second))
2144 : {
2145 0 : function_instance *f = iter->second;
2146 0 : if (dump_file)
2147 : {
2148 0 : fprintf (dump_file, " Removing external inline: ");
2149 0 : f->dump_inline_stack (dump_file);
2150 0 : fprintf (dump_file, "\n");
2151 : }
2152 0 : iter = callsites.erase (iter);
2153 0 : f->set_inlined_to (NULL);
2154 0 : f->offline_if_in_set (seen, new_functions);
2155 0 : delete f;
2156 : }
2157 : else
2158 : {
2159 0 : gcc_checking_assert ((int) iter->first.second
2160 : == iter->second->symbol_name ());
2161 0 : int *newn = iter->second->get_call_location () == UNKNOWN_LOCATION
2162 0 : ? to_symbol_name.get (iter->first.second)
2163 : : NULL;
2164 0 : if (newn)
2165 : {
2166 0 : gcc_checking_assert (iter->second->inlined_to ());
2167 0 : to_rename.safe_push (iter->first);
2168 : }
2169 0 : iter->second->remove_external_functions
2170 0 : (seen, to_symbol_name, new_functions);
2171 0 : ++iter;
2172 : }
2173 0 : for (auto &key : to_rename)
2174 : {
2175 0 : auto iter = callsites.find (key);
2176 0 : callsite key2 = key;
2177 0 : key2.second = *to_symbol_name.get (key.second);
2178 0 : iter->second->set_symbol_name (key2.second);
2179 0 : callsites.erase (iter);
2180 0 : callsites[key2] = iter->second;
2181 : }
2182 0 : auto_vec <int, 20> target_to_rename;
2183 0 : for (auto &iter : pos_counts)
2184 : {
2185 0 : for (auto const &titer : iter.second.targets)
2186 : {
2187 0 : int *ren = to_symbol_name.get (titer.first);
2188 0 : if (ren)
2189 0 : target_to_rename.safe_push (titer.first);
2190 : }
2191 0 : while (target_to_rename.length ())
2192 : {
2193 0 : int key = target_to_rename.pop ();
2194 0 : int key2 = *to_symbol_name.get (key);
2195 0 : auto i = iter.second.targets.find (key);
2196 0 : if (iter.second.targets.count (key2) == 0)
2197 0 : iter.second.targets[key2] = i->second;
2198 : else
2199 0 : iter.second.targets[key2] += i->second;
2200 0 : iter.second.targets.erase (i);
2201 : }
2202 : }
2203 0 : }
2204 :
2205 : /* Look for inline instances that was not realized and
2206 : remove them while possibly merging them to offline variants. */
2207 :
2208 : void
2209 0 : function_instance::offline_if_not_realized
2210 : (vec <function_instance *> &new_functions)
2211 : {
2212 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2213 0 : iter != callsites.end ();)
2214 0 : if (!iter->second->realized_p ())
2215 : {
2216 0 : function_instance *f = iter->second;
2217 0 : if (dump_file)
2218 : {
2219 0 : fprintf (dump_file, "Offlining unrealized inline ");
2220 0 : f->dump_inline_stack (dump_file);
2221 0 : fprintf (dump_file, "\n");
2222 : }
2223 0 : iter = callsites.erase (iter);
2224 0 : offline (f, new_functions);
2225 : }
2226 : else
2227 : {
2228 0 : iter->second->offline_if_not_realized (new_functions);
2229 0 : ++iter;
2230 : }
2231 0 : }
2232 :
2233 : /* Dump instance to F indented by INDENT. */
2234 :
2235 : void
2236 0 : function_instance::dump (FILE *f, int indent, bool nested) const
2237 : {
2238 0 : if (!nested)
2239 0 : fprintf (f, "%*s%s total:%" PRIu64 " head:%" PRId64 "\n", indent, "",
2240 : afdo_string_table->get_symbol_name (symbol_name ()),
2241 0 : (int64_t) total_count (), (int64_t) head_count ());
2242 : else
2243 0 : fprintf (f, " total:%" PRIu64 "\n", (int64_t)total_count ());
2244 0 : for (auto const &iter : pos_counts)
2245 : {
2246 0 : fprintf (f, "%*s", indent + 2, "");
2247 0 : dump_afdo_loc (f, iter.first);
2248 0 : fprintf (f, ": %" PRIu64, (int64_t)iter.second.count);
2249 :
2250 0 : for (auto const &titer : iter.second.targets)
2251 0 : fprintf (f, " %s:%" PRIu64,
2252 0 : afdo_string_table->get_symbol_name (titer.first),
2253 0 : (int64_t) titer.second);
2254 0 : fprintf (f,"\n");
2255 : }
2256 0 : for (auto const &iter : callsites)
2257 : {
2258 0 : fprintf (f, "%*s", indent + 2, "");
2259 0 : dump_afdo_loc (f, iter.first.first);
2260 0 : fprintf (f, ": %s",
2261 0 : afdo_string_table->get_symbol_name (iter.first.second));
2262 0 : iter.second->dump (f, indent + 2, true);
2263 0 : gcc_checking_assert ((int) iter.first.second
2264 : == iter.second->symbol_name ());
2265 : }
2266 0 : }
2267 :
2268 : /* Dump inline path. */
2269 :
2270 : void
2271 0 : function_instance::dump_inline_stack (FILE *f) const
2272 : {
2273 0 : auto_vec <callsite, 20> stack;
2274 0 : const function_instance *p = this, *s = inlined_to ();
2275 0 : while (s)
2276 : {
2277 0 : bool found = false;
2278 0 : for (callsite_map::const_iterator iter = s->callsites.begin ();
2279 0 : iter != s->callsites.end (); ++iter)
2280 0 : if (iter->second == p)
2281 : {
2282 0 : gcc_checking_assert (
2283 : !found && (int) iter->first.second == p->symbol_name ());
2284 0 : stack.safe_push ({iter->first.first, s->symbol_name ()});
2285 0 : found = true;
2286 : }
2287 0 : gcc_checking_assert (found);
2288 0 : p = s;
2289 0 : s = s->inlined_to ();
2290 : }
2291 0 : for (callsite &s: stack)
2292 : {
2293 0 : fprintf (f, "%s:", afdo_string_table->get_symbol_name (s.second));
2294 0 : dump_afdo_loc (f, s.first);
2295 0 : fprintf (f, " ");
2296 : }
2297 0 : fprintf (f, "%s", afdo_string_table->get_symbol_name (symbol_name ()));
2298 0 : }
2299 :
2300 : /* Dump instance to stderr. */
2301 :
2302 : void
2303 0 : function_instance::debug () const
2304 : {
2305 0 : dump (stderr);
2306 0 : }
2307 :
2308 : /* Return profile info for LOC in INFO. */
2309 :
2310 : bool
2311 0 : function_instance::get_count_info (location_t loc, count_info *info) const
2312 : {
2313 0 : position_count_map::const_iterator iter = pos_counts.find (loc);
2314 0 : if (iter == pos_counts.end ())
2315 : return false;
2316 0 : *info = iter->second;
2317 0 : return true;
2318 : }
2319 :
2320 : /* Read the inlined indirect call target profile for STMT and store it in
2321 : MAP, return the total count for all inlined indirect calls. */
2322 :
2323 : gcov_type
2324 0 : function_instance::find_icall_target_map (tree fn, gcall *stmt,
2325 : icall_target_map *map) const
2326 : {
2327 0 : gcov_type ret = 0;
2328 0 : unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);
2329 :
2330 0 : for (callsite_map::const_iterator iter = callsites.begin ();
2331 0 : iter != callsites.end (); ++iter)
2332 : {
2333 0 : unsigned callee = iter->second->symbol_name ();
2334 : /* Check if callsite location match the stmt. */
2335 0 : if (iter->first.first != stmt_offset
2336 0 : || iter->second->removed_icall_target ())
2337 0 : continue;
2338 0 : struct cgraph_node *node = cgraph_node::get_for_asmname (
2339 : get_identifier (afdo_string_table->get_symbol_name (callee)));
2340 0 : if (node == NULL)
2341 0 : continue;
2342 0 : (*map)[callee] = iter->second->total_count () * afdo_count_scale;
2343 0 : ret += iter->second->total_count () * afdo_count_scale;
2344 : }
2345 0 : return ret;
2346 : }
2347 :
2348 : /* Remove the inlined indirect call target profile for STMT. */
2349 :
2350 : void
2351 0 : function_instance::remove_icall_target (tree fn, gcall *stmt)
2352 : {
2353 0 : unsigned stmt_offset = get_relative_location_for_stmt (fn, stmt);
2354 0 : int n = 0;
2355 :
2356 0 : for (auto iter : callsites)
2357 0 : if (iter.first.first == stmt_offset)
2358 : {
2359 0 : iter.second->remove_icall_target ();
2360 0 : n++;
2361 : }
2362 : /* TODO: If we add support for multiple targets, we may want to
2363 : remove only those we successfully inlined. */
2364 0 : gcc_assert (n);
2365 0 : }
2366 :
2367 : /* Offline all functions not defined in the current unit.
2368 : We will not be able to early inline them.
2369 : Doing so early will get VPT decisions more realistic. */
2370 :
2371 : void
2372 0 : autofdo_source_profile::offline_external_functions ()
2373 : {
2374 : /* First check all available definitions and mark their names as
2375 : visible. */
2376 0 : cgraph_node *node;
2377 0 : name_index_set seen;
2378 0 : name_index_map to_symbol_name;
2379 0 : size_t last_name;
2380 :
2381 : /* Add renames erasing suffixes produced by late clones, such as
2382 : .isra, .ipcp. */
2383 0 : for (size_t i = 1; i < afdo_string_table->num_entries (); i++)
2384 : {
2385 0 : const char *n1 = afdo_string_table->get_symbol_name (i);
2386 0 : std::pair<const char *, int> name_filename
2387 0 : = afdo_string_table->get_original_name (n1);
2388 0 : const char *n2 = name_filename.first;
2389 0 : if (!strcmp (n1, n2))
2390 : {
2391 : /* Watch for duplicate entries.
2392 : This seems to happen in practice and may be useful to distinguish
2393 : multiple static symbols of the same name, but we do not really
2394 : have a way to differentiate them in get_symbol_name lookup. */
2395 0 : int index = afdo_string_table->get_index (n1);
2396 0 : if (index != (int)i)
2397 : {
2398 0 : if (dump_file)
2399 0 : fprintf (dump_file,
2400 : "string table in auto-profile contains"
2401 : " duplicated name %s\n", n1);
2402 0 : to_symbol_name.put (i, index);
2403 : }
2404 0 : continue;
2405 0 : }
2406 0 : if (dump_file)
2407 0 : fprintf (dump_file, "Adding rename removing clone suffixes %s -> %s\n",
2408 : n1, n2);
2409 0 : int index = afdo_string_table->get_index (n2);
2410 0 : if (index == -1)
2411 0 : index = afdo_string_table->add_symbol_name (xstrdup (n2),
2412 : name_filename.second);
2413 0 : to_symbol_name.put (i, index);
2414 : }
2415 0 : last_name = afdo_string_table->num_entries ();
2416 0 : FOR_EACH_DEFINED_FUNCTION (node)
2417 : {
2418 0 : const char *name = raw_symbol_name (node->decl);
2419 0 : const char *dwarf_name = lang_hooks.dwarf_name (node->decl, 0);
2420 0 : int index = afdo_string_table->get_index (name);
2421 :
2422 : /* Inline function may be identified by its dwarf names;
2423 : rename them to symbol names. With LTO dwarf names are
2424 : lost in free_lange_data. */
2425 0 : if (strcmp (name, dwarf_name))
2426 : {
2427 0 : int index2 = afdo_string_table->get_index (dwarf_name);
2428 0 : if (index2 != -1)
2429 : {
2430 0 : if (index == -1)
2431 0 : index = afdo_string_table->add_symbol_name (
2432 0 : xstrdup (name),
2433 : afdo_string_table->add_filename (
2434 0 : get_normalized_path (DECL_SOURCE_FILE (node->decl))));
2435 0 : if (dump_file)
2436 : {
2437 0 : fprintf (dump_file, "Adding dwarf->symbol rename %s -> %s\n",
2438 : afdo_string_table->get_symbol_name (index2), name);
2439 0 : if (to_symbol_name.get (index2))
2440 0 : fprintf (dump_file, "Dwarf name is not unique");
2441 : }
2442 0 : to_symbol_name.put (index2, index);
2443 0 : seen.add (index2);
2444 : }
2445 : }
2446 0 : if (index != -1)
2447 : {
2448 0 : if (dump_file)
2449 0 : fprintf (dump_file, "%s is defined in node %s\n",
2450 : afdo_string_table->get_symbol_name (index),
2451 : node->dump_name ());
2452 0 : seen.add (index);
2453 : }
2454 : else
2455 : {
2456 0 : if (dump_file)
2457 : {
2458 0 : if (dwarf_name && strcmp (dwarf_name, name))
2459 0 : fprintf (dump_file,
2460 : "Node %s not in auto profile (%s neither %s)\n",
2461 : node->dump_name (),
2462 : name,
2463 : dwarf_name);
2464 : else
2465 0 : fprintf (dump_file,
2466 : "Node %s (symbol %s) not in auto profile\n",
2467 : node->dump_name (),
2468 : name);
2469 : }
2470 : }
2471 : }
2472 :
2473 0 : for (auto iter : to_symbol_name)
2474 : {
2475 : /* In case dwarf name was duplicated and later renamed,
2476 : handle both. No more than one hop should be needed. */
2477 0 : int *newn = to_symbol_name.get (iter.second);
2478 0 : if (newn)
2479 0 : iter.second = *newn;
2480 0 : gcc_checking_assert (!to_symbol_name.get (iter.second));
2481 0 : if (seen.contains (iter.second))
2482 0 : seen.add (iter.first);
2483 : }
2484 :
2485 : /* Now process all toplevel (offline) function instances.
2486 :
2487 : If instance has no definition in this translation unit,
2488 : first offline all inlined functions which are defined here
2489 : (so we do not lose profile due to cross-module inlining
2490 : done by link-time optimizers).
2491 :
2492 : If instance has a definition, look into all inlined functions
2493 : and remove external ones (result of cross-module inlining).
2494 :
2495 : TODO: after early-inlining we ought to offline all functions
2496 : that were not inlined. */
2497 0 : vec <function_instance *>&fns = duplicate_functions_;
2498 0 : auto_vec <function_instance *, 20>fns2;
2499 : /* Populate worklist with all functions to process. Processing
2500 : may introduce new functions by offlining. */
2501 0 : for (auto &function : map_)
2502 : {
2503 0 : function.second->set_in_worklist ();
2504 0 : fns.safe_push (function.second);
2505 : }
2506 :
2507 : /* There are two worklists. First all functions needs to be matched
2508 : with gimple body and only then we want to do merging, since matching
2509 : should be done on unmodified profile and merging works better if
2510 : mismatches are already resolved both in source and destination. */
2511 0 : while (fns.length () || fns2.length ())
2512 : {
2513 : /* In case renaming introduced new name, keep seen up to date. */
2514 0 : for (; last_name < afdo_string_table->num_entries (); last_name++)
2515 : {
2516 0 : const char *name = afdo_string_table->get_symbol_name (last_name);
2517 0 : symtab_node *n
2518 0 : = afdo_string_table->get_cgraph_node (last_name);
2519 0 : if (dump_file)
2520 0 : fprintf (dump_file, "New name %s %s\n", name,
2521 : n ? "with corresponding definition"
2522 : : "with no corresponding definition");
2523 0 : if (n)
2524 0 : seen.add (last_name);
2525 : }
2526 0 : if (fns.length ())
2527 : {
2528 0 : function_instance *f = fns.pop ();
2529 0 : if (f->get_location () == UNKNOWN_LOCATION)
2530 : {
2531 0 : int index = f->symbol_name ();
2532 0 : int *newn = to_symbol_name.get (index);
2533 0 : if (newn)
2534 : {
2535 0 : if (find_function_instance (f->get_descriptor ()) == f)
2536 0 : remove_function_instance (f);
2537 0 : f->set_symbol_name (*newn);
2538 0 : if (!find_function_instance (f->get_descriptor ()))
2539 0 : add_function_instance (f);
2540 : }
2541 0 : if (cgraph_node *n = f->get_cgraph_node ())
2542 : {
2543 0 : gcc_checking_assert (seen.contains (f->symbol_name ()));
2544 0 : f->match (n, fns, to_symbol_name);
2545 : }
2546 : }
2547 0 : fns2.safe_push (f);
2548 : }
2549 : else
2550 : {
2551 0 : function_instance *f = fns2.pop ();
2552 0 : int index = f->symbol_name ();
2553 0 : gcc_checking_assert (f->in_worklist_p ());
2554 :
2555 : /* If map has different function_instance of same name, then
2556 : this is a duplicated entry which needs to be merged. */
2557 0 : function_instance *index_inst
2558 0 : = find_function_instance (f->get_descriptor ());
2559 0 : if (index_inst && index_inst != f)
2560 : {
2561 0 : if (dump_file)
2562 : {
2563 0 : fprintf (dump_file, "Merging duplicate instance: ");
2564 0 : f->dump_inline_stack (dump_file);
2565 0 : fprintf (dump_file, "\n");
2566 : }
2567 0 : index_inst->merge (f, fns);
2568 0 : gcc_checking_assert (!f->inlined_to ());
2569 0 : f->clear_in_worklist ();
2570 0 : delete f;
2571 : }
2572 : /* If name was not seen in the symbol table, remove it. */
2573 0 : else if (!seen.contains (index))
2574 : {
2575 0 : f->offline_if_in_set (seen, fns);
2576 0 : f->clear_in_worklist ();
2577 0 : if (dump_file)
2578 0 : fprintf (dump_file, "Removing external %s\n",
2579 : afdo_string_table->get_symbol_name (
2580 : f->symbol_name ()));
2581 0 : if (index_inst == f)
2582 0 : remove_function_instance (f);
2583 0 : delete f;
2584 : }
2585 : /* If this is offline function instance seen in this
2586 : translation unit offline external inlines and possibly
2587 : rename from dwarf name. */
2588 : else
2589 : {
2590 0 : f->remove_external_functions (seen, to_symbol_name, fns);
2591 0 : f->clear_in_worklist ();
2592 : }
2593 : }
2594 : }
2595 0 : if (dump_file)
2596 0 : for (auto const &function : map_)
2597 : {
2598 0 : seen.contains (function.second->symbol_name ());
2599 0 : function.second->dump (dump_file);
2600 : }
2601 0 : }
2602 :
2603 : /* Walk scope block BLOCK and mark all inlined functions as realized. */
2604 :
2605 : static void
2606 0 : walk_block (tree fn, function_instance *s, tree block)
2607 : {
2608 0 : if (inlined_function_outer_scope_p (block))
2609 : {
2610 0 : unsigned loc = get_relative_location_for_locus
2611 0 : (fn, BLOCK_SUPERCONTEXT (block),
2612 0 : BLOCK_SOURCE_LOCATION (block));
2613 0 : function_instance *ns
2614 : = s->get_function_instance_by_decl
2615 0 : (loc, BLOCK_ABSTRACT_ORIGIN (block),
2616 0 : BLOCK_SOURCE_LOCATION (block));
2617 0 : if (!ns)
2618 : {
2619 0 : if (dump_file)
2620 : {
2621 0 : fprintf (dump_file, " Failed to find inlined instance:");
2622 0 : s->dump_inline_stack (dump_file);
2623 0 : fprintf (dump_file, ":");
2624 0 : dump_afdo_loc (dump_file, loc);
2625 0 : fprintf (dump_file, " %s\n",
2626 0 : raw_symbol_name (BLOCK_ABSTRACT_ORIGIN (block)));
2627 : }
2628 : return;
2629 : }
2630 0 : s = ns;
2631 0 : if (dump_file)
2632 : {
2633 0 : fprintf (dump_file, " Marking realized inline: ");
2634 0 : s->dump_inline_stack (dump_file);
2635 0 : fprintf (dump_file, "\n");
2636 : }
2637 0 : s->set_realized ();
2638 : }
2639 0 : for (tree t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2640 0 : walk_block (fn, s, t);
2641 : }
2642 :
2643 : /* Offline all inline functions that are not marked as realized.
2644 : This will merge their profile into offline versions where available.
2645 : Also remove all functions we will no longer use. */
2646 :
2647 : void
2648 0 : autofdo_source_profile::offline_unrealized_inlines ()
2649 : {
2650 0 : auto_vec <function_instance *>fns;
2651 : /* Populate worklist with all functions to process. Processing
2652 : may introduce new functions by offlining. */
2653 0 : for (auto const &function : map_)
2654 : {
2655 0 : fns.safe_push (function.second);
2656 0 : function.second->set_in_worklist ();
2657 : }
2658 0 : while (fns.length ())
2659 : {
2660 0 : function_instance *f = fns.pop ();
2661 0 : int index = f->symbol_name ();
2662 0 : function_instance *index_inst
2663 0 : = find_function_instance (f->get_descriptor ());
2664 0 : bool in_map = index_inst != nullptr;
2665 0 : if (in_map)
2666 0 : if (cgraph_node *n = f->get_cgraph_node ())
2667 : {
2668 0 : if (dump_file)
2669 0 : fprintf (dump_file, "Marking realized %s\n",
2670 : afdo_string_table->get_symbol_name (index));
2671 0 : f->set_realized ();
2672 0 : if (DECL_INITIAL (n->decl)
2673 0 : && DECL_INITIAL (n->decl) != error_mark_node)
2674 0 : walk_block (n->decl, f, DECL_INITIAL (n->decl));
2675 : }
2676 0 : f->offline_if_not_realized (fns);
2677 0 : gcc_checking_assert ((in_map || !f->realized_p ())
2678 : && f->in_worklist_p ());
2679 :
2680 : /* If this is duplicated instance, merge it into one in map. */
2681 0 : if (in_map && index_inst != f)
2682 : {
2683 0 : if (dump_file)
2684 : {
2685 0 : fprintf (dump_file, "Merging duplicate instance: ");
2686 0 : f->dump_inline_stack (dump_file);
2687 0 : fprintf (dump_file, "\n");
2688 : }
2689 0 : index_inst->merge (f, fns);
2690 0 : f->clear_in_worklist ();
2691 0 : gcc_checking_assert (!f->inlined_to ());
2692 0 : delete f;
2693 : }
2694 : /* If function is not in symbol table, remove it. */
2695 0 : else if (!f->realized_p ())
2696 : {
2697 0 : if (dump_file)
2698 0 : fprintf (dump_file, "Removing optimized out function %s\n",
2699 : afdo_string_table->get_symbol_name (f->symbol_name ()));
2700 0 : if (in_map)
2701 0 : remove_function_instance (index_inst);
2702 0 : f->clear_in_worklist ();
2703 0 : delete f;
2704 : }
2705 : else
2706 0 : f->clear_in_worklist ();
2707 : }
2708 0 : if (dump_file)
2709 0 : for (auto const &function : map_)
2710 0 : function.second->dump (dump_file);
2711 0 : }
2712 :
2713 : /* Read the profile and create a function_instance with head count as
2714 : HEAD_COUNT. Recursively read callsites to create nested function_instances
2715 : too. STACK is used to track the recursive creation process. */
2716 :
2717 : /* function instance profile format:
2718 :
2719 : ENTRY_COUNT: 8 bytes
2720 : TIMESTAMP: 8 bytes (only for toplevel symbols)
2721 : NAME_INDEX: 4 bytes
2722 : NUM_POS_COUNTS: 4 bytes
2723 : NUM_CALLSITES: 4 byte
2724 : POS_COUNT_1:
2725 : POS_1_OFFSET: 4 bytes
2726 : NUM_TARGETS: 4 bytes
2727 : COUNT: 8 bytes
2728 : TARGET_1:
2729 : VALUE_PROFILE_TYPE: 4 bytes
2730 : TARGET_IDX: 8 bytes
2731 : COUNT: 8 bytes
2732 : TARGET_2
2733 : ...
2734 : TARGET_n
2735 : POS_COUNT_2
2736 : ...
2737 : POS_COUNT_N
2738 : CALLSITE_1:
2739 : CALLSITE_1_OFFSET: 4 bytes
2740 : FUNCTION_INSTANCE_PROFILE (nested)
2741 : CALLSITE_2
2742 : ...
2743 : CALLSITE_n. */
2744 :
2745 : function_instance *
2746 0 : function_instance::read_function_instance (function_instance_stack *stack,
2747 : bool toplevel)
2748 : {
2749 0 : gcov_type_unsigned timestamp = 0;
2750 0 : gcov_type head_count = -1;
2751 0 : if (toplevel)
2752 : {
2753 0 : head_count = gcov_read_counter ();
2754 0 : timestamp = (gcov_type_unsigned) gcov_read_counter ();
2755 : }
2756 0 : unsigned name = gcov_read_unsigned ();
2757 0 : unsigned num_pos_counts = gcov_read_unsigned ();
2758 0 : unsigned num_callsites = gcov_read_unsigned ();
2759 0 : function_instance *s
2760 : = new function_instance (name,
2761 0 : afdo_string_table->get_filename_by_symbol (name),
2762 0 : head_count);
2763 0 : if (timestamp > 0)
2764 0 : s->set_timestamp (timestamp);
2765 0 : if (!stack->is_empty ())
2766 0 : s->set_inlined_to (stack->last ());
2767 0 : stack->safe_push (s);
2768 :
2769 0 : for (unsigned i = 0; i < num_pos_counts; i++)
2770 : {
2771 0 : unsigned offset = gcov_read_unsigned ();
2772 0 : unsigned num_targets = gcov_read_unsigned ();
2773 0 : gcov_type count = gcov_read_counter ();
2774 0 : s->pos_counts[offset].count = count;
2775 :
2776 0 : for (unsigned j = 0; j < stack->length (); j++)
2777 0 : (*stack)[j]->total_count_ += count;
2778 0 : for (unsigned j = 0; j < num_targets; j++)
2779 : {
2780 : /* Only indirect call target histogram is supported now. */
2781 0 : gcov_read_unsigned ();
2782 0 : gcov_type target_idx = gcov_read_counter ();
2783 0 : s->pos_counts[offset].targets[target_idx] = gcov_read_counter ();
2784 : }
2785 : }
2786 0 : for (unsigned i = 0; i < num_callsites; i++)
2787 : {
2788 0 : unsigned offset = gcov_read_unsigned ();
2789 0 : function_instance *callee_function_instance
2790 0 : = read_function_instance (stack, false);
2791 0 : s->callsites[std::make_pair (offset,
2792 0 : callee_function_instance->symbol_name ())]
2793 0 : = callee_function_instance;
2794 : }
2795 0 : stack->pop ();
2796 0 : return s;
2797 : }
2798 :
2799 : /* Member functions for autofdo_source_profile. */
2800 :
2801 0 : autofdo_source_profile::~autofdo_source_profile ()
2802 : {
2803 0 : for (name_function_instance_map::const_iterator iter = map_.begin ();
2804 0 : iter != map_.end (); ++iter)
2805 0 : delete iter->second;
2806 0 : }
2807 :
2808 : /* For a given DECL, returns the top-level function_instance. */
2809 :
2810 : function_instance *
2811 0 : autofdo_source_profile::get_function_instance_by_decl (tree decl, const char *filename) const
2812 : {
2813 0 : if (!filename)
2814 0 : filename = get_normalized_path (DECL_SOURCE_FILE (decl));
2815 0 : int index = afdo_string_table->get_index_by_decl (decl);
2816 0 : if (index == -1)
2817 : return NULL;
2818 :
2819 0 : function_instance_descriptor descriptor (
2820 0 : afdo_string_table->get_filename_index (filename), index);
2821 0 : return find_function_instance (descriptor);
2822 : }
2823 :
2824 : /* For a given DESCRIPTOR, return the matching instance if found. */
2825 :
2826 : function_instance *
2827 0 : autofdo_source_profile::get_function_instance_by_descriptor (
2828 : function_instance_descriptor descriptor) const
2829 : {
2830 0 : return find_function_instance (descriptor);
2831 : }
2832 :
2833 : /* Add function instance FN. */
2834 :
2835 : void
2836 0 : autofdo_source_profile::add_function_instance (function_instance *fn)
2837 : {
2838 0 : gcc_checking_assert (map_.find (fn->get_descriptor ()) == map_.end ());
2839 0 : map_[fn->get_descriptor ()] = fn;
2840 0 : }
2841 :
2842 : /* Find count_info for a given gimple STMT. If found, store the count_info
2843 : in INFO and return true; otherwise return false. */
2844 :
2845 : bool
2846 0 : autofdo_source_profile::get_count_info (gimple *stmt, count_info *info,
2847 : cgraph_node *node) const
2848 : {
2849 0 : gcc_checking_assert (stmt_loc_used_by_debug_info (stmt));
2850 0 : return get_count_info (gimple_location (stmt), info, node);
2851 : }
2852 :
2853 : bool
2854 0 : autofdo_source_profile::get_count_info (location_t gimple_loc,
2855 : count_info *info,
2856 : cgraph_node *node) const
2857 : {
2858 0 : if (LOCATION_LOCUS (gimple_loc) == cfun->function_end_locus)
2859 : return false;
2860 :
2861 0 : inline_stack stack;
2862 0 : get_inline_stack_in_node (gimple_loc, &stack, node);
2863 0 : if (stack.length () == 0)
2864 : return false;
2865 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2866 0 : if (s == NULL)
2867 : return false;
2868 0 : return s->get_count_info (stack[0].afdo_loc, info);
2869 0 : }
2870 :
2871 : /* Update value profile INFO for STMT from the inlined indirect callsite.
2872 : Return true if INFO is updated. */
2873 :
2874 : bool
2875 0 : autofdo_source_profile::update_inlined_ind_target (gcall *stmt,
2876 : count_info *info,
2877 : cgraph_node *node)
2878 : {
2879 0 : if (dump_file)
2880 : {
2881 0 : fprintf (dump_file, "Checking indirect call -> direct call ");
2882 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
2883 : }
2884 :
2885 0 : if (LOCATION_LOCUS (gimple_location (stmt)) == cfun->function_end_locus)
2886 : {
2887 0 : if (dump_file)
2888 0 : fprintf (dump_file, " bad locus (function end)\n");
2889 : return false;
2890 : }
2891 :
2892 0 : count_info old_info;
2893 0 : get_count_info (stmt, &old_info, node);
2894 0 : gcov_type total = 0;
2895 0 : for (icall_target_map::const_iterator iter = old_info.targets.begin ();
2896 0 : iter != old_info.targets.end (); ++iter)
2897 0 : total += iter->second;
2898 0 : total *= afdo_count_scale;
2899 :
2900 : /* Program behavior changed, original promoted (and inlined) target is not
2901 : hot any more. Will avoid promote the original target.
2902 :
2903 : To check if original promoted target is still hot, we check the total
2904 : count of the unpromoted targets (stored in TOTAL). If a callsite count
2905 : (stored in INFO) is smaller than half of the total count, the original
2906 : promoted target is considered not hot any more. */
2907 0 : if (info->count < total / 2)
2908 : {
2909 0 : if (dump_file)
2910 0 : fprintf (dump_file, " not hot anymore %ld < %ld",
2911 : (long)info->count,
2912 : (long)total /2);
2913 : return false;
2914 : }
2915 :
2916 0 : inline_stack stack;
2917 0 : get_inline_stack_in_node (gimple_location (stmt), &stack, node);
2918 0 : if (stack.length () == 0)
2919 : {
2920 0 : if (dump_file)
2921 0 : fprintf (dump_file, " no inline stack\n");
2922 : return false;
2923 : }
2924 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2925 0 : if (s == NULL)
2926 : {
2927 0 : if (dump_file)
2928 : {
2929 0 : fprintf (dump_file, " function not found in inline stack:");
2930 0 : dump_inline_stack (dump_file, &stack);
2931 : }
2932 : return false;
2933 : }
2934 0 : icall_target_map map;
2935 0 : if (s->find_icall_target_map (node ? node->decl
2936 : : current_function_decl,
2937 : stmt, &map) == 0)
2938 : {
2939 0 : if (dump_file)
2940 : {
2941 0 : fprintf (dump_file, " no target map for stack: ");
2942 0 : dump_inline_stack (dump_file, &stack);
2943 : }
2944 : return false;
2945 : }
2946 0 : for (icall_target_map::const_iterator iter = map.begin ();
2947 0 : iter != map.end (); ++iter)
2948 0 : info->targets[iter->first] = iter->second;
2949 0 : if (dump_file)
2950 : {
2951 0 : fprintf (dump_file, " looks good; stack:");
2952 0 : dump_inline_stack (dump_file, &stack);
2953 : }
2954 : return true;
2955 0 : }
2956 :
2957 : void
2958 0 : autofdo_source_profile::remove_icall_target (cgraph_edge *e)
2959 : {
2960 0 : autofdo::inline_stack stack;
2961 0 : autofdo::get_inline_stack_in_node (gimple_location (e->call_stmt),
2962 : &stack, e->caller);
2963 0 : autofdo::function_instance *s
2964 0 : = get_function_instance_by_inline_stack (stack);
2965 0 : s->remove_icall_target (e->caller->decl, e->call_stmt);
2966 0 : }
2967 :
2968 : /* Find total count of the callee of EDGE. */
2969 :
2970 : gcov_type
2971 0 : autofdo_source_profile::get_callsite_total_count (
2972 : struct cgraph_edge *edge) const
2973 : {
2974 0 : inline_stack stack;
2975 0 : stack.safe_push ({edge->callee->decl, 0, UNKNOWN_LOCATION});
2976 :
2977 0 : get_inline_stack_in_node (gimple_location (edge->call_stmt), &stack,
2978 : edge->caller);
2979 0 : if (dump_file)
2980 : {
2981 0 : if (!edge->caller->inlined_to)
2982 0 : fprintf (dump_file, "Looking up afdo profile for call %s -> %s stack:",
2983 0 : edge->caller->dump_name (), edge->callee->dump_name ());
2984 : else
2985 0 : fprintf (dump_file, "Looking up afdo profile for call %s -> %s transitively %s stack:",
2986 0 : edge->caller->dump_name (), edge->callee->dump_name (),
2987 : edge->caller->inlined_to->dump_name ());
2988 0 : dump_inline_stack (dump_file, &stack);
2989 : }
2990 :
2991 0 : function_instance *s = get_function_instance_by_inline_stack (stack);
2992 0 : if (s == NULL)
2993 : {
2994 0 : if (dump_file)
2995 0 : fprintf (dump_file, "No function instance found\n");
2996 : return 0;
2997 : }
2998 0 : if (afdo_string_table->get_index_by_decl (edge->callee->decl)
2999 0 : != s->symbol_name ())
3000 : {
3001 0 : if (dump_file)
3002 0 : fprintf (dump_file, "Mismatched name of callee %s and profile %s\n",
3003 0 : raw_symbol_name (edge->callee->decl),
3004 : afdo_string_table->get_symbol_name (s->symbol_name ()));
3005 : return 0;
3006 : }
3007 :
3008 0 : return s->total_count () * afdo_count_scale;
3009 0 : }
3010 :
3011 : /* Read AutoFDO profile and returns TRUE on success. */
3012 :
3013 : /* source profile format:
3014 :
3015 : GCOV_TAG_AFDO_FUNCTION: 4 bytes
3016 : LENGTH: 4 bytes
3017 : NUM_FUNCTIONS: 4 bytes
3018 : FUNCTION_INSTANCE_1
3019 : FUNCTION_INSTANCE_2
3020 : ...
3021 : FUNCTION_INSTANCE_N. */
3022 :
3023 : bool
3024 0 : autofdo_source_profile::read ()
3025 : {
3026 0 : if (gcov_read_unsigned () != GCOV_TAG_AFDO_FUNCTION)
3027 : {
3028 0 : inform (UNKNOWN_LOCATION, "Not expected TAG.");
3029 0 : return false;
3030 : }
3031 :
3032 0 : gcc_checking_assert (!afdo_source_profile);
3033 0 : afdo_source_profile = this;
3034 :
3035 : /* Skip the length of the section. */
3036 0 : gcov_read_unsigned ();
3037 :
3038 : /* Read in the function/callsite profile, and store it in local
3039 : data structure. */
3040 0 : unsigned function_num = gcov_read_unsigned ();
3041 0 : for (unsigned i = 0; i < function_num; i++)
3042 : {
3043 0 : function_instance::function_instance_stack stack;
3044 0 : function_instance *s
3045 0 : = function_instance::read_function_instance (&stack);
3046 :
3047 0 : if (find_function_instance (s->get_descriptor ()) == nullptr)
3048 0 : add_function_instance (s);
3049 : else
3050 0 : fatal_error (UNKNOWN_LOCATION,
3051 : "auto-profile contains duplicated function instance %s",
3052 : afdo_string_table->get_symbol_name (s->symbol_name ()));
3053 0 : s->prop_timestamp ();
3054 0 : timestamp_info_map.insert({s->timestamp (), 0});
3055 0 : }
3056 :
3057 : /* timestamp_info_map is std::map with timestamp as key,
3058 : so it's already sorted in ascending order wrt timestamps.
3059 : This loop maps function with lowest timestamp to 1, and so on.
3060 : In afdo_annotate_cfg, node->tp_first_run is then set to corresponding
3061 : tp_first_run value. */
3062 :
3063 0 : int tp_first_run = 1;
3064 0 : for (auto &p : timestamp_info_map)
3065 0 : p.second = tp_first_run++;
3066 :
3067 0 : afdo_profile_info->sum_max = afdo_summary_info->max_count;
3068 : /* Scale up the profile, but leave some bits in case some counts gets
3069 : bigger than sum_max eventually. */
3070 0 : if (afdo_profile_info->sum_max)
3071 0 : afdo_count_scale
3072 0 : = MAX (((gcov_type)1 << (profile_count::n_bits - 10))
3073 : / afdo_profile_info->sum_max, 1);
3074 0 : afdo_profile_info->cutoff *= afdo_count_scale;
3075 : /* Derive the hot count threshold from the profile summary. */
3076 0 : afdo_hot_bb_threshold = afdo_summary_info->get_threshold_count (
3077 0 : param_hot_bb_count_ws_permille * 1000)
3078 0 : * afdo_count_scale;
3079 0 : set_hot_bb_threshold (afdo_hot_bb_threshold);
3080 0 : if (dump_file)
3081 0 : fprintf (dump_file,
3082 : "Max count in profile %" PRIu64 "\n"
3083 : "Setting scale %" PRIu64 "\n"
3084 : "Scaled max count %" PRIu64 "\n"
3085 : "Cutoff %" PRIu64 "\n"
3086 : "Unscaled hot count threshold %" PRIu64 "\n"
3087 : "Hot count threshold %" PRIu64 "\n\n",
3088 : (int64_t) afdo_profile_info->sum_max, (int64_t) afdo_count_scale,
3089 0 : (int64_t) (afdo_profile_info->sum_max * afdo_count_scale),
3090 0 : (int64_t) afdo_profile_info->cutoff,
3091 0 : (int64_t) afdo_summary_info->get_threshold_count (
3092 0 : param_hot_bb_count_ws_permille * 1000),
3093 : (int64_t) afdo_hot_bb_threshold);
3094 0 : afdo_profile_info->sum_max *= afdo_count_scale;
3095 0 : return true;
3096 : }
3097 :
3098 : /* Return the function_instance in the profile that correspond to the
3099 : inline STACK. */
3100 :
3101 : function_instance *
3102 0 : autofdo_source_profile::get_function_instance_by_inline_stack (
3103 : const inline_stack &stack) const
3104 : {
3105 0 : function_instance_descriptor descriptor (
3106 : afdo_string_table->get_filename_index (
3107 0 : get_normalized_path (DECL_SOURCE_FILE (stack[stack.length () - 1].decl))),
3108 0 : afdo_string_table->get_index_by_decl (stack[stack.length () - 1].decl));
3109 0 : function_instance *s = find_function_instance (descriptor);
3110 :
3111 0 : if (s == NULL)
3112 : {
3113 0 : if (dump_file)
3114 0 : fprintf (dump_file, "No offline instance for %s\n",
3115 0 : raw_symbol_name (stack[stack.length () - 1].decl));
3116 : return NULL;
3117 : }
3118 :
3119 0 : for (unsigned i = stack.length () - 1; i > 0; i--)
3120 : {
3121 0 : s = s->get_function_instance_by_decl (stack[i].afdo_loc,
3122 0 : stack[i - 1].decl,
3123 0 : stack[i].location);
3124 0 : if (s == NULL)
3125 : {
3126 : /* afdo inliner extends the stack by last entry with unknown
3127 : location while checking if function was inlined during train run.
3128 : We do not want to print diagnostics about every function
3129 : which is not inlined. */
3130 : if (s && dump_enabled_p () && stack[i].location != UNKNOWN_LOCATION)
3131 : dump_printf_loc (MSG_NOTE | MSG_PRIORITY_INTERNALS,
3132 : dump_user_location_t::from_location_t
3133 : (stack[i].location),
3134 : "auto-profile has no inlined function instance "
3135 : "for inlined call of %s at relative "
3136 : " location +%i, discriminator %i\n",
3137 : raw_symbol_name (stack[i - 1].decl),
3138 : stack[i].afdo_loc >> 16,
3139 : stack[i].afdo_loc & 65535);
3140 : return NULL;
3141 : }
3142 : }
3143 : return s;
3144 : }
3145 :
3146 : /* Find the matching function instance which has DESCRIPTOR as its
3147 : descriptor. If not found, also try checking if an instance exists with the
3148 : same name which has no associated filename. */
3149 :
3150 : autofdo_source_profile::name_function_instance_map::const_iterator
3151 0 : autofdo_source_profile::find_iter_for_function_instance (
3152 : function_instance_descriptor descriptor) const
3153 : {
3154 0 : auto it = map_.find (descriptor);
3155 :
3156 : /* Try searching for the symbol not having a filename if it isn't found. */
3157 0 : if (it == map_.end ())
3158 0 : it = map_.find (
3159 0 : function_instance_descriptor (string_table::unknown_filename,
3160 0 : (int) descriptor.symbol_name ()));
3161 0 : return it;
3162 : }
3163 :
3164 : /* Similar to the above, but return a pointer to the instance instead of an
3165 : iterator. */
3166 :
3167 : function_instance *
3168 0 : autofdo_source_profile::find_function_instance (
3169 : function_instance_descriptor descriptor) const
3170 : {
3171 0 : auto it = find_iter_for_function_instance (descriptor);
3172 0 : return it == map_.end () ? NULL : it->second;
3173 : }
3174 :
3175 : /* Remove a function instance from the map. Returns true if the entry was
3176 : actually deleted. */
3177 :
3178 : bool
3179 0 : autofdo_source_profile::remove_function_instance (function_instance *inst)
3180 : {
3181 0 : auto iter = find_iter_for_function_instance (inst->get_descriptor ());
3182 0 : if (iter != map_.end ())
3183 : {
3184 0 : map_.erase (iter);
3185 0 : return true;
3186 : }
3187 : return false;
3188 : }
3189 :
3190 : /* Module profile is only used by LIPO. Here we simply ignore it. */
3191 :
3192 : static void
3193 0 : fake_read_autofdo_module_profile ()
3194 : {
3195 : /* Read in the module info. */
3196 0 : gcov_read_unsigned ();
3197 :
3198 : /* Skip the length of the section. */
3199 0 : gcov_read_unsigned ();
3200 :
3201 : /* Read in the file name table. */
3202 0 : unsigned total_module_num = gcov_read_unsigned ();
3203 0 : gcc_assert (total_module_num == 0);
3204 0 : }
3205 :
3206 : /* Read data from profile data file. */
3207 :
3208 : static void
3209 0 : read_profile (void)
3210 : {
3211 0 : if (gcov_open (auto_profile_file, 1) == 0)
3212 : {
3213 0 : error ("cannot open profile file %s", auto_profile_file);
3214 0 : return;
3215 : }
3216 :
3217 0 : if (gcov_read_unsigned () != GCOV_DATA_MAGIC)
3218 : {
3219 0 : error ("AutoFDO profile magic number does not match");
3220 0 : return;
3221 : }
3222 :
3223 : /* Skip the version number. */
3224 0 : unsigned version = gcov_read_unsigned ();
3225 0 : if (version != AUTO_PROFILE_VERSION)
3226 : {
3227 0 : error ("AutoFDO profile version %u does not match %u",
3228 : version, AUTO_PROFILE_VERSION);
3229 0 : return;
3230 : }
3231 :
3232 : /* Skip the empty integer. */
3233 0 : gcov_read_unsigned ();
3234 :
3235 : /* summary_info. */
3236 0 : afdo_summary_info = new summary_info ();
3237 0 : if (!afdo_summary_info->read ())
3238 : {
3239 0 : error ("cannot read summary information from %s", auto_profile_file);
3240 0 : return;
3241 : }
3242 :
3243 : /* string_table. */
3244 0 : afdo_string_table = new string_table ();
3245 0 : if (!afdo_string_table->read ())
3246 : {
3247 0 : error ("cannot read string table from %s", auto_profile_file);
3248 0 : return;
3249 : }
3250 :
3251 : /* autofdo_source_profile. */
3252 0 : afdo_source_profile = autofdo_source_profile::create ();
3253 0 : if (afdo_source_profile == NULL
3254 0 : || gcov_is_error ())
3255 : {
3256 0 : error ("cannot read function profile from %s", auto_profile_file);
3257 0 : delete afdo_source_profile;
3258 0 : afdo_source_profile = NULL;
3259 0 : return;
3260 : }
3261 :
3262 : /* autofdo_module_profile. */
3263 0 : fake_read_autofdo_module_profile ();
3264 0 : if (gcov_is_error ())
3265 : {
3266 0 : error ("cannot read module profile from %s", auto_profile_file);
3267 0 : return;
3268 : }
3269 : }
3270 :
3271 : /* From AutoFDO profiles, find values inside STMT for that we want to measure
3272 : histograms for indirect-call optimization.
3273 :
3274 : This function is actually served for 2 purposes:
3275 : * before annotation, we need to mark histogram, promote and inline
3276 : * after annotation, we just need to mark, and let follow-up logic to
3277 : decide if it needs to promote and inline. */
3278 :
3279 : static bool
3280 0 : afdo_indirect_call (gcall *stmt, const icall_target_map &map,
3281 : bool transform, cgraph_edge *indirect_edge)
3282 : {
3283 0 : tree callee;
3284 :
3285 0 : if (map.size () == 0)
3286 : {
3287 0 : if (dump_file)
3288 0 : fprintf (dump_file, "No targets found\n");
3289 : return false;
3290 : }
3291 0 : if (!stmt)
3292 : {
3293 0 : if (dump_file)
3294 0 : fprintf (dump_file, "No call statement\n");
3295 : return false;
3296 : }
3297 0 : if (gimple_call_internal_p (stmt))
3298 : {
3299 0 : if (dump_file)
3300 0 : fprintf (dump_file, "Internal call\n");
3301 : return false;
3302 : }
3303 0 : if (gimple_call_fndecl (stmt) != NULL_TREE)
3304 : {
3305 0 : if (dump_file)
3306 0 : fprintf (dump_file, "Call is already direct\n");
3307 : return false;
3308 : }
3309 :
3310 0 : gcov_type total = 0;
3311 0 : icall_target_map::const_iterator max_iter = map.end ();
3312 :
3313 0 : for (icall_target_map::const_iterator iter = map.begin ();
3314 0 : iter != map.end (); ++iter)
3315 : {
3316 0 : total += iter->second;
3317 0 : if (max_iter == map.end () || max_iter->second < iter->second)
3318 : max_iter = iter;
3319 : }
3320 0 : total *= afdo_count_scale;
3321 0 : struct cgraph_node *direct_call = cgraph_node::get_for_asmname (
3322 0 : get_identifier (afdo_string_table->get_symbol_name (max_iter->first)));
3323 0 : if (direct_call == NULL)
3324 : {
3325 0 : if (dump_file)
3326 0 : fprintf (dump_file, "Failed to find cgraph node for %s\n",
3327 0 : afdo_string_table->get_symbol_name (max_iter->first));
3328 : return false;
3329 : }
3330 :
3331 0 : callee = gimple_call_fn (stmt);
3332 :
3333 0 : if (!transform)
3334 : {
3335 0 : if (!direct_call->profile_id)
3336 : {
3337 0 : if (dump_file)
3338 0 : fprintf (dump_file, "No profile id\n");
3339 : return false;
3340 : }
3341 0 : histogram_value hist = gimple_alloc_histogram_value (
3342 : cfun, HIST_TYPE_INDIR_CALL, stmt, callee);
3343 0 : hist->n_counters = 4;
3344 0 : hist->hvalue.counters = XNEWVEC (gcov_type, hist->n_counters);
3345 0 : gimple_add_histogram_value (cfun, stmt, hist);
3346 :
3347 : /* Total counter */
3348 0 : hist->hvalue.counters[0] = total;
3349 : /* Number of value/counter pairs */
3350 0 : hist->hvalue.counters[1] = 1;
3351 : /* Value */
3352 0 : hist->hvalue.counters[2] = direct_call->profile_id;
3353 : /* Counter */
3354 0 : hist->hvalue.counters[3] = max_iter->second * afdo_count_scale;
3355 :
3356 0 : if (!direct_call->profile_id)
3357 : {
3358 0 : if (dump_file)
3359 0 : fprintf (dump_file, "Histogram attached\n");
3360 : return false;
3361 : }
3362 : return false;
3363 : }
3364 :
3365 0 : if (dump_file)
3366 : {
3367 0 : fprintf (dump_file, "Indirect call -> direct call ");
3368 0 : print_generic_expr (dump_file, callee, TDF_SLIM);
3369 0 : fprintf (dump_file, " => ");
3370 0 : print_generic_expr (dump_file, direct_call->decl, TDF_SLIM);
3371 : }
3372 :
3373 0 : if (!direct_call->definition)
3374 : {
3375 0 : if (dump_file)
3376 0 : fprintf (dump_file, " no definition available\n");
3377 : return false;
3378 : }
3379 :
3380 0 : if (dump_file)
3381 : {
3382 0 : fprintf (dump_file, " transformation on insn ");
3383 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
3384 0 : fprintf (dump_file, "\n");
3385 : }
3386 :
3387 0 : indirect_edge->make_speculative
3388 0 : (direct_call,
3389 0 : gimple_bb (stmt)->count.apply_scale (99, 100));
3390 0 : return true;
3391 : }
3392 :
3393 : /* From AutoFDO profiles, find values inside STMT for that we want to measure
3394 : histograms and adds them to list VALUES. */
3395 :
3396 : static bool
3397 0 : afdo_vpt (gcall *gs, const icall_target_map &map,
3398 : bool transform, cgraph_edge *indirect_edge)
3399 : {
3400 0 : return afdo_indirect_call (gs, map, transform, indirect_edge);
3401 : }
3402 :
3403 : typedef std::set<basic_block> bb_set;
3404 :
3405 : static bool
3406 0 : is_bb_annotated (const basic_block bb, const bb_set &annotated)
3407 : {
3408 0 : if (annotated.find (bb) != annotated.end ())
3409 : {
3410 0 : gcc_checking_assert (bb->count.quality () == AFDO
3411 : || !bb->count.nonzero_p ());
3412 : return true;
3413 : }
3414 0 : gcc_checking_assert (bb->count.quality () != AFDO
3415 : || !bb->count.nonzero_p ());
3416 : return false;
3417 : }
3418 :
3419 : static void
3420 0 : set_bb_annotated (basic_block bb, bb_set *annotated)
3421 : {
3422 0 : gcc_checking_assert (bb->count.quality () == AFDO
3423 : || !bb->count.nonzero_p ());
3424 0 : annotated->insert (bb);
3425 0 : }
3426 :
3427 : /* Update COUNT by known autofdo count C. */
3428 : static void
3429 0 : update_count_by_afdo_count (profile_count *count, gcov_type c)
3430 : {
3431 0 : if (c)
3432 0 : *count = profile_count::from_gcov_type (c).afdo ();
3433 : /* In case we have guessed profile which is already zero, preserve
3434 : quality info. */
3435 0 : else if (count->nonzero_p ()
3436 0 : || count->quality () == GUESSED
3437 0 : || count->quality () == GUESSED_LOCAL)
3438 0 : *count = profile_count::zero ().afdo ();
3439 0 : }
3440 :
3441 : /* Update COUNT by known autofdo count C. */
3442 : static void
3443 0 : update_count_by_afdo_count (profile_count *count, profile_count c)
3444 : {
3445 0 : if (c.nonzero_p ())
3446 0 : *count = c;
3447 : /* In case we have guessed profile which is already zero, preserve
3448 : quality info. */
3449 0 : else if (count->nonzero_p ()
3450 0 : || count->quality () < c.quality ())
3451 0 : *count = c;
3452 0 : }
3453 :
3454 : /* Try to determine unscaled count of edge E.
3455 : Return -1 if nothing is known. */
3456 :
3457 : static gcov_type
3458 0 : afdo_unscaled_edge_count (edge e)
3459 : {
3460 0 : gcov_type max_count = -1;
3461 0 : basic_block bb_succ = e->dest;
3462 0 : count_info info;
3463 0 : if (afdo_source_profile->get_count_info (e->goto_locus, &info))
3464 : {
3465 0 : if (info.count > max_count)
3466 : max_count = info.count;
3467 0 : if (dump_file && info.count)
3468 : {
3469 0 : fprintf (dump_file,
3470 : " goto location of edge %i->%i with count %" PRIu64"\n",
3471 0 : e->src->index, e->dest->index, (int64_t)info.count);
3472 : }
3473 : }
3474 0 : for (gphi_iterator gpi = gsi_start_phis (bb_succ);
3475 0 : !gsi_end_p (gpi); gsi_next (&gpi))
3476 : {
3477 0 : gphi *phi = gpi.phi ();
3478 0 : location_t phi_loc
3479 0 : = gimple_phi_arg_location_from_edge (phi, e);
3480 0 : if (afdo_source_profile->get_count_info (phi_loc, &info))
3481 : {
3482 0 : if (info.count > max_count)
3483 : max_count = info.count;
3484 0 : if (dump_file && info.count)
3485 : {
3486 0 : fprintf (dump_file,
3487 : " phi op of edge %i->%i with count %" PRIu64": ",
3488 0 : e->src->index, e->dest->index, (int64_t)info.count);
3489 0 : print_gimple_stmt (dump_file, phi, 0, TDF_SLIM);
3490 : }
3491 : }
3492 : }
3493 0 : return max_count;
3494 0 : }
3495 :
3496 : /* For a given BB, set its execution count. Attach value profile if a stmt
3497 : is not in PROMOTED, because we only want to promote an indirect call once.
3498 : Return TRUE if BB is annotated. */
3499 :
3500 : static bool
3501 0 : afdo_set_bb_count (basic_block bb, hash_set <basic_block> &zero_bbs)
3502 : {
3503 0 : gimple_stmt_iterator gsi;
3504 0 : gcov_type max_count = -1;
3505 0 : if (dump_file)
3506 0 : fprintf (dump_file, " Looking up AFDO count of bb %i\n", bb->index);
3507 :
3508 0 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3509 : {
3510 0 : count_info info;
3511 0 : gimple *stmt = gsi_stmt (gsi);
3512 0 : if (!stmt_loc_used_by_debug_info (stmt))
3513 0 : continue;
3514 0 : if (afdo_source_profile->get_count_info (stmt, &info))
3515 : {
3516 0 : if (info.count > max_count)
3517 : max_count = info.count;
3518 0 : if (dump_file)
3519 : {
3520 0 : fprintf (dump_file, " count %" PRIu64 " in stmt: ",
3521 : (int64_t)info.count);
3522 0 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
3523 : }
3524 0 : gcall *call = dyn_cast <gcall *> (gsi_stmt (gsi));
3525 : /* TODO; if inlined early and indirect call was not optimized out,
3526 : we will end up speculating again. Early inliner should remove
3527 : all targets for edges it speculated into safely. */
3528 0 : if (call
3529 0 : && info.targets.size () > 0)
3530 0 : afdo_vpt (call, info.targets, false, NULL);
3531 : }
3532 0 : }
3533 :
3534 0 : if (max_count == -1 && single_succ_p (bb))
3535 0 : max_count = afdo_unscaled_edge_count (single_succ_edge (bb));
3536 :
3537 0 : if (max_count == -1)
3538 : return false;
3539 :
3540 0 : if (max_count)
3541 : {
3542 0 : update_count_by_afdo_count (&bb->count, max_count * afdo_count_scale);
3543 0 : if (dump_file)
3544 0 : fprintf (dump_file,
3545 : " Annotated bb %i with count %" PRId64
3546 : ", scaled to %" PRId64 "\n",
3547 : bb->index, (int64_t)max_count,
3548 0 : (int64_t)(max_count * afdo_count_scale));
3549 : return true;
3550 : }
3551 : else
3552 : {
3553 0 : if (dump_file)
3554 0 : fprintf (dump_file,
3555 : " bb %i has statements with 0 count\n", bb->index);
3556 0 : zero_bbs.add (bb);
3557 : }
3558 0 : return false;
3559 : }
3560 :
3561 : /* BB1 and BB2 are in an equivalent class iff:
3562 : 1. BB1 dominates BB2.
3563 : 2. BB2 post-dominates BB1.
3564 : 3. BB1 and BB2 are in the same loop nest.
3565 : This function finds the equivalent class for each basic block, and
3566 : stores a pointer to the first BB in its equivalent class. Meanwhile,
3567 : set bb counts for the same equivalent class to be identical. Update
3568 : ANNOTATED_BB for the first BB in its equivalent class. */
3569 :
3570 : static void
3571 0 : afdo_find_equiv_class (bb_set *annotated_bb)
3572 : {
3573 0 : basic_block bb;
3574 :
3575 0 : FOR_ALL_BB_FN (bb, cfun)
3576 0 : bb->aux = NULL;
3577 :
3578 0 : FOR_ALL_BB_FN (bb, cfun)
3579 : {
3580 0 : if (bb->aux != NULL)
3581 0 : continue;
3582 0 : bb->aux = bb;
3583 0 : for (basic_block bb1 : get_dominated_by (CDI_DOMINATORS, bb))
3584 0 : if (bb1->aux == NULL && dominated_by_p (CDI_POST_DOMINATORS, bb, bb1)
3585 0 : && bb1->loop_father == bb->loop_father)
3586 : {
3587 0 : bb1->aux = bb;
3588 0 : if (is_bb_annotated (bb1, *annotated_bb)
3589 0 : && (!is_bb_annotated (bb, *annotated_bb)
3590 0 : || bb1->count > bb->count))
3591 : {
3592 0 : if (dump_file)
3593 : {
3594 0 : fprintf (dump_file,
3595 : " Copying count of bb %i to bb %i; count is:",
3596 : bb1->index,
3597 : bb->index);
3598 0 : bb1->count.dump (dump_file);
3599 0 : fprintf (dump_file, "\n");
3600 : }
3601 0 : update_count_by_afdo_count (&bb->count, bb1->count);
3602 0 : set_bb_annotated (bb, annotated_bb);
3603 : }
3604 0 : }
3605 :
3606 0 : for (basic_block bb1 : get_dominated_by (CDI_POST_DOMINATORS, bb))
3607 0 : if (bb1->aux == NULL && dominated_by_p (CDI_DOMINATORS, bb, bb1)
3608 0 : && bb1->loop_father == bb->loop_father)
3609 : {
3610 0 : bb1->aux = bb;
3611 0 : if (is_bb_annotated (bb1, *annotated_bb)
3612 0 : && (!is_bb_annotated (bb, *annotated_bb)
3613 0 : || bb1->count > bb->count))
3614 : {
3615 0 : if (dump_file)
3616 : {
3617 0 : fprintf (dump_file,
3618 : " Copying count of bb %i to bb %i; count is:",
3619 : bb1->index,
3620 : bb->index);
3621 0 : bb1->count.dump (dump_file);
3622 0 : fprintf (dump_file, "\n");
3623 : }
3624 0 : update_count_by_afdo_count (&bb->count, bb1->count);
3625 0 : set_bb_annotated (bb, annotated_bb);
3626 : }
3627 0 : }
3628 : }
3629 0 : }
3630 :
3631 : /* If a basic block's count is known, and only one of its in/out edges' count
3632 : is unknown, its count can be calculated. Meanwhile, if all of the in/out
3633 : edges' counts are known, then the basic block's unknown count can also be
3634 : calculated. Also, if a block has a single predecessor or successor, the block's
3635 : count can be propagated to that predecessor or successor.
3636 : IS_SUCC is true if out edges of a basic blocks are examined.
3637 : Update ANNOTATED_BB accordingly.
3638 : Return TRUE if any basic block/edge count is changed. */
3639 :
3640 : static bool
3641 0 : afdo_propagate_edge (bool is_succ, bb_set *annotated_bb)
3642 : {
3643 0 : basic_block bb;
3644 0 : bool changed = false;
3645 :
3646 0 : FOR_EACH_BB_FN (bb, cfun)
3647 : {
3648 0 : edge e, unknown_edge = NULL;
3649 0 : edge_iterator ei;
3650 0 : int num_unknown_edges = 0;
3651 0 : int num_edges = 0;
3652 0 : profile_count total_known_count = profile_count::zero ().afdo ();
3653 :
3654 0 : FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
3655 : {
3656 0 : gcc_assert (AFDO_EINFO (e) != NULL);
3657 0 : if (! AFDO_EINFO (e)->is_annotated ())
3658 0 : num_unknown_edges++, unknown_edge = e;
3659 : else
3660 0 : total_known_count += AFDO_EINFO (e)->get_count ();
3661 0 : num_edges++;
3662 : }
3663 0 : if (dump_file)
3664 : {
3665 0 : fprintf (dump_file, "bb %i %s propagating %s edges %i, "
3666 : "unknown edges %i, known count ",
3667 : bb->index,
3668 0 : is_bb_annotated (bb, *annotated_bb) ? "(annotated)" : "",
3669 : is_succ ? "successors" : "predecessors", num_edges,
3670 : num_unknown_edges);
3671 0 : total_known_count.dump (dump_file);
3672 0 : fprintf (dump_file, " bb count ");
3673 0 : bb->count.dump (dump_file);
3674 0 : fprintf (dump_file, "\n");
3675 : }
3676 :
3677 : /* Be careful not to annotate block with no successor in special cases. */
3678 0 : if (num_unknown_edges == 0 && num_edges
3679 0 : && !is_bb_annotated (bb, *annotated_bb))
3680 : {
3681 0 : if (dump_file)
3682 : {
3683 0 : fprintf (dump_file, " Annotating bb %i with count ", bb->index);
3684 0 : total_known_count.dump (dump_file);
3685 0 : fprintf (dump_file, "\n");
3686 : }
3687 0 : update_count_by_afdo_count (&bb->count, total_known_count);
3688 0 : set_bb_annotated (bb, annotated_bb);
3689 0 : changed = true;
3690 : }
3691 0 : else if (is_bb_annotated (bb, *annotated_bb)
3692 : /* We do not want to consider 0 (afdo) > 0 (precise) */
3693 0 : && total_known_count.nonzero_p ()
3694 0 : && bb->count < total_known_count)
3695 : {
3696 0 : if (dump_file)
3697 : {
3698 0 : fprintf (dump_file, " Increasing bb %i count from ",
3699 : bb->index);
3700 0 : bb->count.dump (dump_file);
3701 0 : fprintf (dump_file, " to ");
3702 0 : total_known_count.dump (dump_file);
3703 0 : fprintf (dump_file, " hoping to mitigate afdo inconsistency\n");
3704 : }
3705 0 : bb->count = total_known_count;
3706 0 : changed = true;
3707 : }
3708 0 : else if (num_unknown_edges == 1 && is_bb_annotated (bb, *annotated_bb))
3709 : {
3710 0 : if (bb->count > total_known_count)
3711 : {
3712 0 : profile_count new_count = bb->count - total_known_count;
3713 0 : AFDO_EINFO (unknown_edge)->set_count (new_count);
3714 : }
3715 : else
3716 0 : AFDO_EINFO (unknown_edge)->set_count
3717 0 : (profile_count::zero ().afdo ());
3718 0 : if (dump_file)
3719 : {
3720 0 : fprintf (dump_file, " Annotated edge %i->%i with count ",
3721 0 : unknown_edge->src->index, unknown_edge->dest->index);
3722 0 : AFDO_EINFO (unknown_edge)->get_count ().dump (dump_file);
3723 0 : fprintf (dump_file, "\n");
3724 : }
3725 0 : AFDO_EINFO (unknown_edge)->set_annotated ();
3726 0 : changed = true;
3727 : }
3728 0 : else if (num_unknown_edges > 1
3729 0 : && is_bb_annotated (bb, *annotated_bb)
3730 0 : && (total_known_count >= bb->count || !bb->count.nonzero_p ()))
3731 : {
3732 0 : FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds)
3733 : {
3734 0 : gcc_assert (AFDO_EINFO (e) != NULL);
3735 0 : if (! AFDO_EINFO (e)->is_annotated ())
3736 : {
3737 0 : AFDO_EINFO (e)->set_count
3738 0 : (profile_count::zero ().afdo ());
3739 0 : AFDO_EINFO (e)->set_annotated ();
3740 0 : if (dump_file)
3741 : {
3742 0 : fprintf (dump_file, " Annotated edge %i->%i with count ",
3743 0 : e->src->index, e->dest->index);
3744 0 : AFDO_EINFO (unknown_edge)->get_count ().dump (dump_file);
3745 0 : fprintf (dump_file, "\n");
3746 : }
3747 : }
3748 : }
3749 : }
3750 0 : else if (num_unknown_edges == 0
3751 0 : && is_bb_annotated (bb, *annotated_bb)
3752 0 : && (is_succ ? single_succ_p (bb) : single_pred_p (bb)))
3753 : {
3754 0 : edge e = is_succ ? single_succ_edge (bb) : single_pred_edge (bb);
3755 0 : if (AFDO_EINFO (e)->is_annotated ()
3756 0 : && AFDO_EINFO (e)->get_count () < bb->count)
3757 : {
3758 0 : if (dump_file)
3759 : {
3760 0 : fprintf (dump_file, " Increasing edge %i->%i count from ",
3761 0 : e->src->index, e->dest->index);
3762 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
3763 0 : fprintf (dump_file, " to ");
3764 0 : bb->count.dump (dump_file);
3765 0 : fprintf (dump_file, " hoping to mitigate afdo inconsistency\n");
3766 : }
3767 0 : AFDO_EINFO (e)->set_count (bb->count);
3768 0 : changed = true;
3769 : }
3770 : }
3771 : }
3772 0 : return changed;
3773 : }
3774 :
3775 : /* Special propagation for circuit expressions. Because GCC translates
3776 : control flow into data flow for circuit expressions. E.g.
3777 : BB1:
3778 : if (a && b)
3779 : BB2
3780 : else
3781 : BB3
3782 :
3783 : will be translated into:
3784 :
3785 : BB1:
3786 : if (a)
3787 : goto BB.t1
3788 : else
3789 : goto BB.t3
3790 : BB.t1:
3791 : if (b)
3792 : goto BB.t2
3793 : else
3794 : goto BB.t3
3795 : BB.t2:
3796 : goto BB.t3
3797 : BB.t3:
3798 : tmp = PHI (0 (BB1), 0 (BB.t1), 1 (BB.t2)
3799 : if (tmp)
3800 : goto BB2
3801 : else
3802 : goto BB3
3803 :
3804 : In this case, we need to propagate through PHI to determine the edge
3805 : count of BB1->BB.t1, BB.t1->BB.t2. */
3806 :
3807 : static void
3808 0 : afdo_propagate_circuit (const bb_set &annotated_bb)
3809 : {
3810 0 : basic_block bb;
3811 0 : FOR_ALL_BB_FN (bb, cfun)
3812 : {
3813 0 : gimple *def_stmt;
3814 0 : tree cmp_rhs, cmp_lhs;
3815 0 : gimple *cmp_stmt = last_nondebug_stmt (bb);
3816 0 : edge e;
3817 0 : edge_iterator ei;
3818 :
3819 0 : if (!cmp_stmt || gimple_code (cmp_stmt) != GIMPLE_COND)
3820 0 : continue;
3821 0 : cmp_rhs = gimple_cond_rhs (cmp_stmt);
3822 0 : cmp_lhs = gimple_cond_lhs (cmp_stmt);
3823 0 : if (!TREE_CONSTANT (cmp_rhs)
3824 0 : || !(integer_zerop (cmp_rhs) || integer_onep (cmp_rhs)))
3825 0 : continue;
3826 0 : if (TREE_CODE (cmp_lhs) != SSA_NAME)
3827 0 : continue;
3828 0 : if (!is_bb_annotated (bb, annotated_bb))
3829 0 : continue;
3830 0 : def_stmt = SSA_NAME_DEF_STMT (cmp_lhs);
3831 0 : while (def_stmt && gimple_code (def_stmt) == GIMPLE_ASSIGN
3832 0 : && gimple_assign_single_p (def_stmt)
3833 0 : && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
3834 0 : def_stmt = SSA_NAME_DEF_STMT (gimple_assign_rhs1 (def_stmt));
3835 0 : if (!def_stmt)
3836 0 : continue;
3837 0 : gphi *phi_stmt = dyn_cast <gphi *> (def_stmt);
3838 0 : if (!phi_stmt)
3839 0 : continue;
3840 0 : FOR_EACH_EDGE (e, ei, bb->succs)
3841 : {
3842 0 : unsigned i, total = 0;
3843 0 : edge only_one;
3844 0 : bool check_value_one = (((integer_onep (cmp_rhs))
3845 0 : ^ (gimple_cond_code (cmp_stmt) == EQ_EXPR))
3846 0 : ^ ((e->flags & EDGE_TRUE_VALUE) != 0));
3847 0 : if (! AFDO_EINFO (e)->is_annotated ())
3848 0 : continue;
3849 0 : for (i = 0; i < gimple_phi_num_args (phi_stmt); i++)
3850 : {
3851 0 : tree val = gimple_phi_arg_def (phi_stmt, i);
3852 0 : edge ep = gimple_phi_arg_edge (phi_stmt, i);
3853 :
3854 0 : if (!TREE_CONSTANT (val)
3855 0 : || !(integer_zerop (val) || integer_onep (val)))
3856 0 : continue;
3857 0 : if (check_value_one ^ integer_onep (val))
3858 0 : continue;
3859 0 : total++;
3860 0 : only_one = ep;
3861 0 : if (! (AFDO_EINFO (e)->get_count ()).nonzero_p ()
3862 0 : && ! AFDO_EINFO (ep)->is_annotated ())
3863 : {
3864 0 : AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ());
3865 0 : AFDO_EINFO (ep)->set_annotated ();
3866 : }
3867 : }
3868 0 : if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ())
3869 : {
3870 0 : AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ());
3871 0 : AFDO_EINFO (only_one)->set_annotated ();
3872 : }
3873 : }
3874 : }
3875 0 : }
3876 :
3877 : /* Propagate the basic block count and edge count on the control flow
3878 : graph. We do the propagation iteratively until stabilize. */
3879 :
3880 : static void
3881 0 : afdo_propagate (bb_set *annotated_bb)
3882 : {
3883 0 : bool changed = true;
3884 0 : int i = 0;
3885 :
3886 0 : basic_block bb;
3887 0 : FOR_ALL_BB_FN (bb, cfun)
3888 0 : if (!is_bb_annotated (bb, *annotated_bb)
3889 0 : && is_bb_annotated ((basic_block)bb->aux, *annotated_bb))
3890 : {
3891 0 : update_count_by_afdo_count (&bb->count, ((basic_block)bb->aux)->count);
3892 0 : set_bb_annotated (bb, annotated_bb);
3893 0 : if (dump_file)
3894 : {
3895 0 : fprintf (dump_file,
3896 : " Copying count of bb %i to bb %i; count is:",
3897 0 : ((basic_block)bb->aux)->index,
3898 : bb->index);
3899 0 : bb->count.dump (dump_file);
3900 0 : fprintf (dump_file, "\n");
3901 : }
3902 : }
3903 :
3904 0 : while (changed && i++ < 100)
3905 : {
3906 0 : changed = false;
3907 :
3908 0 : if (afdo_propagate_edge (true, annotated_bb))
3909 : changed = true;
3910 0 : if (afdo_propagate_edge (false, annotated_bb))
3911 0 : changed = true;
3912 0 : afdo_propagate_circuit (*annotated_bb);
3913 : }
3914 0 : if (dump_file)
3915 0 : fprintf (dump_file, "Propagation took %i iterations %s\n",
3916 : i, changed ? "; iteration limit reached\n" : "");
3917 0 : }
3918 :
3919 : /* qsort comparator of sreals. */
3920 : static int
3921 0 : cmp (const void *a, const void *b)
3922 : {
3923 0 : if (*(const sreal *)a < *(const sreal *)b)
3924 : return 1;
3925 0 : if (*(const sreal *)a > *(const sreal *)b)
3926 0 : return -1;
3927 : return 0;
3928 : }
3929 :
3930 : /* To scale a connected component of graph we collect desired scales of
3931 : basic blocks on the boundary and then compute a robust average. */
3932 :
3933 : struct scale
3934 : {
3935 : /* Scale desired. */
3936 : sreal scale;
3937 : /* Weight for averaging computed from execution count of the edge
3938 : scale originates from. */
3939 : uint64_t weight;
3940 : };
3941 :
3942 : /* Add scale ORIG/ANNOTATED to SCALES. */
3943 :
3944 : static void
3945 0 : add_scale (vec <scale> *scales, profile_count annotated, profile_count orig)
3946 : {
3947 0 : if (dump_file)
3948 : {
3949 0 : orig.dump (dump_file);
3950 0 : fprintf (dump_file, " should be ");
3951 0 : annotated.dump (dump_file);
3952 0 : fprintf (dump_file, "\n");
3953 : }
3954 0 : if (orig.nonzero_p ())
3955 : {
3956 0 : sreal scale
3957 0 : = annotated.guessed_local ()
3958 0 : .to_sreal_scale (orig);
3959 0 : if (dump_file)
3960 0 : fprintf (dump_file, " adding scale %.16f, weight %" PRId64 "\n",
3961 0 : scale.to_double (), annotated.value () + 1);
3962 0 : scales->safe_push ({scale, annotated.value () + 1});
3963 : }
3964 0 : }
3965 :
3966 : /* Scale counts of all basic blocks in BBS by SCALE and convert them to
3967 : IPA quality. */
3968 :
3969 : static void
3970 0 : scale_bbs (const vec <basic_block> &bbs, sreal scale)
3971 : {
3972 0 : if (dump_file)
3973 0 : fprintf (dump_file, " Scaling by %.16f\n", scale.to_double ());
3974 0 : for (basic_block b : bbs)
3975 0 : if (!(b->count == profile_count::zero ())
3976 0 : && b->count.initialized_p ())
3977 : {
3978 0 : profile_count o = b->count;
3979 0 : b->count = b->count.force_guessed () * scale;
3980 :
3981 : /* If we scaled to 0, make it auto-fdo since that is treated
3982 : less aggressively. */
3983 0 : if (!b->count.nonzero_p () && o.nonzero_p ())
3984 0 : b->count = profile_count::zero ().afdo ();
3985 0 : if (dump_file)
3986 : {
3987 0 : fprintf (dump_file, " bb %i count updated ", b->index);
3988 0 : o.dump (dump_file);
3989 0 : fprintf (dump_file, " -> ");
3990 0 : b->count.dump (dump_file);
3991 0 : fprintf (dump_file, "\n");
3992 : }
3993 : }
3994 0 : }
3995 :
3996 : /* Determine scaling factor by taking robust average of SCALES
3997 : and taking into account limits.
3998 : MAX_COUNT is maximal guessed count to be scaled while MAC_COUNT_IN_FN
3999 : is maximal count in function determined by auto-fdo. */
4000 :
4001 : sreal
4002 0 : determine_scale (vec <scale> *scales, profile_count max_count,
4003 : profile_count max_count_in_fn)
4004 : {
4005 0 : scales->qsort (cmp);
4006 :
4007 0 : uint64_t overall_weight = 0;
4008 0 : for (scale &e : *scales)
4009 0 : overall_weight += e.weight;
4010 :
4011 0 : uint64_t cummulated = 0, weight_sum = 0;
4012 0 : sreal scale_sum = 0;
4013 0 : for (scale &e : *scales)
4014 : {
4015 0 : uint64_t prev = cummulated;
4016 0 : cummulated += e.weight;
4017 0 : if (cummulated >= overall_weight / 4
4018 0 : && prev <= 3 * overall_weight / 4)
4019 : {
4020 0 : scale_sum += e.scale * e.weight;
4021 0 : weight_sum += e.weight;
4022 0 : if (dump_file)
4023 0 : fprintf (dump_file, " accounting scale %.16f, weight %" PRId64 "\n",
4024 : e.scale.to_double (), e.weight);
4025 : }
4026 0 : else if (dump_file)
4027 0 : fprintf (dump_file, " ignoring scale %.16f, weight %" PRId64 "\n",
4028 : e.scale.to_double (), e.weight);
4029 : }
4030 0 : sreal scale = scale_sum / (sreal)weight_sum;
4031 :
4032 : /* Avoid scaled regions to have very large counts.
4033 : Otherwise they may dominate ipa-profile's histogram computing cutoff
4034 : of hot basic blocks. */
4035 0 : if (max_count * scale > max_count_in_fn.guessed_local ().apply_scale (128, 1))
4036 : {
4037 0 : if (dump_file)
4038 : {
4039 0 : fprintf (dump_file, "Scaling by %.16f produces max count ",
4040 : scale.to_double ());
4041 0 : (max_count * scale).dump (dump_file);
4042 0 : fprintf (dump_file, " that exceeds max count in fn ");
4043 0 : max_count_in_fn.dump (dump_file);
4044 0 : fprintf (dump_file, "; capping\n");
4045 : }
4046 0 : scale = max_count_in_fn.guessed_local ().to_sreal_scale (max_count);
4047 : }
4048 0 : return scale;
4049 : }
4050 :
4051 : /* Scale profile of the whole function to approximately match auto-profile. */
4052 :
4053 : bool
4054 0 : scale_bb_profile ()
4055 : {
4056 0 : const function_instance *s
4057 : = afdo_source_profile->get_function_instance_by_decl
4058 0 : (current_function_decl);
4059 :
4060 : /* In the first pass only store non-zero counts. */
4061 0 : gcov_type head_count = s->head_count () * autofdo::afdo_count_scale;
4062 0 : hash_set <basic_block> zero_bbs;
4063 0 : auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
4064 0 : auto_vec <scale, 20> scales;
4065 0 : basic_block bb;
4066 0 : profile_count max_count = profile_count::zero ();
4067 0 : profile_count max_count_in_fn = profile_count::zero ();
4068 0 : bbs.quick_push (ENTRY_BLOCK_PTR_FOR_FN (cfun));
4069 0 : bbs.quick_push (EXIT_BLOCK_PTR_FOR_FN (cfun));
4070 0 : if (head_count > 0)
4071 : {
4072 0 : profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4073 0 : max_count = entry_count;
4074 0 : update_count_by_afdo_count (&entry_count, head_count);
4075 0 : max_count_in_fn = entry_count;
4076 0 : add_scale (&scales, entry_count, ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
4077 : }
4078 0 : FOR_EACH_BB_FN (bb, cfun)
4079 : {
4080 0 : profile_count cnt = bb->count;
4081 0 : bbs.safe_push (bb);
4082 0 : max_count = profile_count::max_prefer_initialized (max_count, cnt);
4083 0 : if (afdo_set_bb_count (bb, zero_bbs))
4084 : {
4085 0 : std::swap (cnt, bb->count);
4086 0 : max_count_in_fn
4087 0 : = profile_count::max_prefer_initialized (max_count_in_fn, cnt);
4088 0 : add_scale (&scales, cnt, bb->count);
4089 : }
4090 : }
4091 0 : if (scales.length ())
4092 : {
4093 0 : sreal scale = determine_scale (&scales, max_count, max_count_in_fn);
4094 0 : scale_bbs (bbs, scale);
4095 0 : return true;
4096 : }
4097 : return false;
4098 0 : }
4099 :
4100 : /* In case given basic block was fully optimized out, AutoFDO
4101 : will have no data about it. In this case try to preserve static profile.
4102 : Identify connected components (in undirected form of CFG) which has
4103 : no annotations at all. Look at thir boundaries and try to determine
4104 : scaling factor and scale. */
4105 :
4106 : void
4107 0 : afdo_adjust_guessed_profile (bb_set *annotated_bb)
4108 : {
4109 : /* Basic blocks of connected component currently processed. */
4110 0 : auto_vec <basic_block, 20> bbs (n_basic_blocks_for_fn (cfun));
4111 : /* Scale factors found. */
4112 0 : auto_vec <scale, 20> scales;
4113 0 : auto_vec <basic_block, 20> stack (n_basic_blocks_for_fn (cfun));
4114 :
4115 0 : basic_block seed_bb;
4116 0 : unsigned int component_id = 1;
4117 :
4118 : /* Map from basic block to its component.
4119 : 0 is used for univisited BBs,
4120 : 1 means that BB is annotated,
4121 : >=2 is an id of the component BB belongs to. */
4122 0 : auto_vec <unsigned int, 20> component;
4123 0 : component.safe_grow (last_basic_block_for_fn (cfun));
4124 0 : profile_count max_count_in_fn = profile_count::zero ();
4125 0 : FOR_ALL_BB_FN (seed_bb, cfun)
4126 0 : if (is_bb_annotated (seed_bb, *annotated_bb))
4127 : {
4128 0 : component[seed_bb->index] = 1;
4129 0 : max_count_in_fn
4130 0 : = profile_count::max_prefer_initialized (max_count_in_fn, seed_bb->count);
4131 : }
4132 : else
4133 0 : component[seed_bb->index] = 0;
4134 0 : FOR_ALL_BB_FN (seed_bb, cfun)
4135 0 : if (!component[seed_bb->index])
4136 : {
4137 0 : stack.quick_push (seed_bb);
4138 0 : component_id++;
4139 0 : bbs.truncate (0);
4140 0 : scales.truncate (0);
4141 0 : component[seed_bb->index] = component_id;
4142 0 : profile_count max_count = profile_count::zero ();
4143 :
4144 : /* Identify connected component starting in BB. */
4145 0 : if (dump_file)
4146 0 : fprintf (dump_file, "Starting connected component in bb %i\n",
4147 : seed_bb->index);
4148 0 : do
4149 : {
4150 0 : basic_block b = stack.pop ();
4151 :
4152 0 : bbs.quick_push (b);
4153 0 : max_count = profile_count::max_prefer_initialized (max_count, b->count);
4154 :
4155 0 : for (edge e: b->preds)
4156 0 : if (!component[e->src->index])
4157 : {
4158 0 : stack.quick_push (e->src);
4159 0 : component[e->src->index] = component_id;
4160 : }
4161 0 : for (edge e: b->succs)
4162 0 : if (!component[e->dest->index])
4163 : {
4164 0 : stack.quick_push (e->dest);
4165 0 : component[e->dest->index] = component_id;
4166 : }
4167 : }
4168 0 : while (!stack.is_empty ());
4169 :
4170 : /* If all blocks in components has 0 count, we do not need
4171 : to scale, only we must convert to IPA quality. */
4172 0 : if (!max_count.nonzero_p ())
4173 : {
4174 0 : if (dump_file)
4175 0 : fprintf (dump_file, " All counts are 0; scale = 1\n");
4176 0 : scale_bbs (bbs, 1);
4177 0 : continue;
4178 : }
4179 :
4180 : /* Now visit the component and try to figure out its desired
4181 : frequency. */
4182 0 : for (basic_block b : bbs)
4183 : {
4184 0 : if (dump_file)
4185 : {
4186 0 : fprintf (dump_file, " visiting bb %i with count ", b->index);
4187 0 : b->count.dump (dump_file);
4188 0 : fprintf (dump_file, "\n");
4189 : }
4190 0 : if (!b->count.nonzero_p ())
4191 0 : continue;
4192 : /* Sum of counts of annotated edges into B. */
4193 0 : profile_count annotated_count = profile_count::zero ();
4194 : /* Sum of counts of edges into B with source in current
4195 : component. */
4196 0 : profile_count current_component_count = profile_count::zero ();
4197 0 : bool boundary = false;
4198 :
4199 0 : for (edge e: b->preds)
4200 0 : if (AFDO_EINFO (e)->is_annotated ())
4201 : {
4202 0 : if (dump_file)
4203 : {
4204 0 : fprintf (dump_file, " Annotated pred edge to %i "
4205 0 : "with count ", e->src->index);
4206 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
4207 0 : fprintf (dump_file, "\n");
4208 : }
4209 0 : boundary = true;
4210 0 : annotated_count += AFDO_EINFO (e)->get_count ();
4211 : }
4212 : /* If source is annotated, combine with static
4213 : probability prediction.
4214 : TODO: We can do better in case some of edges out are
4215 : annotated and distribute only remaining count out of BB. */
4216 0 : else if (is_bb_annotated (e->src, *annotated_bb))
4217 : {
4218 0 : boundary = true;
4219 0 : if (dump_file)
4220 : {
4221 0 : fprintf (dump_file, " Annotated predecessor %i "
4222 : "with count ", e->src->index);
4223 0 : e->src->count.dump (dump_file);
4224 0 : fprintf (dump_file, " edge count using static profile ");
4225 0 : e->count ().dump (dump_file);
4226 0 : fprintf (dump_file, "\n");
4227 : }
4228 0 : annotated_count += e->count ();
4229 : }
4230 : else
4231 : {
4232 0 : current_component_count += e->count ();
4233 0 : gcc_checking_assert (component[e->src->index] == component_id);
4234 : }
4235 0 : if (boundary && current_component_count.initialized_p ())
4236 : {
4237 0 : if (dump_file)
4238 0 : fprintf (dump_file, " bb %i in count ", b->index);
4239 0 : add_scale (&scales,
4240 : annotated_count,
4241 : b->count - current_component_count);
4242 : }
4243 0 : for (edge e: b->succs)
4244 0 : if (AFDO_EINFO (e)->is_annotated ())
4245 : {
4246 0 : if (dump_file)
4247 0 : fprintf (dump_file, " edge %i->%i count ",
4248 0 : b->index, e->dest->index);
4249 0 : add_scale (&scales, AFDO_EINFO (e)->get_count (), e->count ());
4250 : }
4251 0 : else if (is_bb_annotated (e->dest, *annotated_bb))
4252 : {
4253 0 : profile_count annotated_count = e->dest->count;
4254 0 : profile_count out_count = profile_count::zero ();
4255 0 : bool ok = true;
4256 :
4257 0 : for (edge e2: e->dest->preds)
4258 0 : if (AFDO_EINFO (e2)->is_annotated ())
4259 0 : annotated_count -= AFDO_EINFO (e2)->get_count ();
4260 0 : else if (component[e2->src->index] == component_id)
4261 0 : out_count += e2->count ();
4262 0 : else if (is_bb_annotated (e2->src, *annotated_bb))
4263 0 : annotated_count -= e2->count ();
4264 0 : else if (e2->probability.nonzero_p ())
4265 : {
4266 : ok = false;
4267 : break;
4268 : }
4269 0 : if (!ok)
4270 0 : continue;
4271 0 : if (dump_file)
4272 0 : fprintf (dump_file,
4273 : " edge %i->%i has annotated successor; count ",
4274 0 : b->index, e->dest->index);
4275 0 : add_scale (&scales, annotated_count, e->count ());
4276 : }
4277 :
4278 : }
4279 :
4280 : /* If we failed to find annotated entry or exit edge,
4281 : look for exit edges and scale profile so the dest
4282 : BB get all flow it needs. This is imprecise because
4283 : the edge is not annotated and thus BB has more than
4284 : one such predecessor. */
4285 0 : if (!scales.length ())
4286 0 : for (basic_block b : bbs)
4287 0 : if (b->count.nonzero_p ())
4288 0 : for (edge e: b->succs)
4289 0 : if (is_bb_annotated (e->dest, *annotated_bb))
4290 : {
4291 0 : profile_count annotated_count = e->dest->count;
4292 0 : for (edge e2: e->dest->preds)
4293 0 : if (AFDO_EINFO (e2)->is_annotated ())
4294 0 : annotated_count -= AFDO_EINFO (e2)->get_count ();
4295 0 : if (dump_file)
4296 0 : fprintf (dump_file,
4297 : " edge %i->%i has annotated successor;"
4298 : " upper bound count ",
4299 0 : b->index, e->dest->index);
4300 0 : add_scale (&scales, annotated_count, e->count ());
4301 : }
4302 0 : if (!scales.length ())
4303 : {
4304 0 : if (dump_file)
4305 0 : fprintf (dump_file,
4306 : " Can not determine count from the boundary; giving up\n");
4307 0 : continue;
4308 : }
4309 0 : gcc_checking_assert (scales.length ());
4310 0 : sreal scale = determine_scale (&scales, max_count, max_count_in_fn);
4311 0 : scale_bbs (bbs, scale);
4312 : }
4313 0 : }
4314 :
4315 : /* Propagate counts on control flow graph and calculate branch
4316 : probabilities. */
4317 :
4318 : static void
4319 0 : afdo_calculate_branch_prob (bb_set *annotated_bb)
4320 : {
4321 0 : edge e;
4322 0 : edge_iterator ei;
4323 0 : basic_block bb;
4324 :
4325 0 : FOR_ALL_BB_FN (bb, cfun)
4326 : {
4327 0 : gcc_assert (bb->aux == NULL);
4328 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4329 : {
4330 0 : gcc_assert (e->aux == NULL);
4331 0 : e->aux = new edge_info ();
4332 0 : gcov_type c = afdo_unscaled_edge_count (e);
4333 0 : if (c == 0 && e->count () == profile_count::zero ())
4334 : {
4335 0 : AFDO_EINFO (e)->set_count (profile_count::zero ());
4336 0 : if (dump_file)
4337 0 : fprintf (dump_file,
4338 : " Annotating edge %i->%i with count 0;"
4339 : " static profile aggress",
4340 0 : e->src->index, e->dest->index);
4341 : }
4342 0 : else if (c > 0)
4343 : {
4344 0 : AFDO_EINFO (e)->set_count
4345 0 : (profile_count::from_gcov_type
4346 0 : (c * autofdo::afdo_count_scale).afdo ());
4347 0 : if (dump_file)
4348 : {
4349 0 : fprintf (dump_file,
4350 : " Annotating edge %i->%i with count ",
4351 0 : e->src->index, e->dest->index);
4352 0 : AFDO_EINFO (e)->get_count ().dump (dump_file);
4353 0 : fprintf (dump_file, "\n");
4354 : }
4355 : }
4356 : }
4357 : }
4358 :
4359 0 : afdo_find_equiv_class (annotated_bb);
4360 0 : afdo_propagate (annotated_bb);
4361 :
4362 0 : FOR_EACH_BB_FN (bb, cfun)
4363 0 : if (is_bb_annotated (bb, *annotated_bb))
4364 : {
4365 0 : bool all_known = true;
4366 0 : profile_count total_count = profile_count::zero ().afdo ();
4367 :
4368 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4369 : {
4370 0 : gcc_assert (AFDO_EINFO (e) != NULL);
4371 0 : if (! AFDO_EINFO (e)->is_annotated ())
4372 : {
4373 : /* If by static profile this edge never happens,
4374 : still propagate the rest. */
4375 0 : if (e->probability.nonzero_p ())
4376 : {
4377 : all_known = false;
4378 : break;
4379 : }
4380 : }
4381 : else
4382 0 : total_count += AFDO_EINFO (e)->get_count ();
4383 : }
4384 0 : if (!all_known || !total_count.nonzero_p ())
4385 0 : continue;
4386 0 : if (dump_file)
4387 : {
4388 0 : fprintf (dump_file, "Total count of bb %i is ", bb->index);
4389 0 : total_count.dump (dump_file);
4390 0 : fprintf (dump_file, "\n");
4391 : }
4392 :
4393 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4394 0 : if (AFDO_EINFO (e)->is_annotated ())
4395 : {
4396 0 : profile_count cnt = AFDO_EINFO (e)->get_count ();
4397 : /* If probability is 1, preserve reliable static prediction
4398 : (This is, for example the case of single fallthru edge
4399 : or single fallthru plus unlikely EH edge.) */
4400 0 : if (cnt == total_count
4401 0 : && e->probability == profile_probability::always ())
4402 : ;
4403 0 : else if (cnt.nonzero_p ())
4404 0 : e->probability
4405 0 : = cnt.probability_in (total_count);
4406 : /* If probability is zero, preserve reliable static
4407 : prediction. */
4408 0 : else if (e->probability.nonzero_p ()
4409 0 : || e->probability.quality () == GUESSED)
4410 0 : e->probability = profile_probability::never ().afdo ();
4411 0 : if (dump_file)
4412 : {
4413 0 : fprintf (dump_file, " probability of edge %i->%i"
4414 : " with count ",
4415 0 : e->src->index, e->dest->index);
4416 0 : cnt.dump (dump_file);
4417 0 : fprintf (dump_file, " set to ");
4418 0 : e->probability.dump (dump_file);
4419 0 : fprintf (dump_file, "\n");
4420 : }
4421 : }
4422 : }
4423 0 : afdo_adjust_guessed_profile (annotated_bb);
4424 : /* Avoid scaling with a zero entry count during IPA profile merging. */
4425 0 : basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
4426 0 : if (!entry->count.nonzero_p ())
4427 0 : entry->count = profile_count::from_gcov_type
4428 0 : (MAX ((gcov_type) 1, autofdo::afdo_count_scale / 2)).afdo ();
4429 0 : FOR_ALL_BB_FN (bb, cfun)
4430 : {
4431 0 : bb->aux = NULL;
4432 0 : FOR_EACH_EDGE (e, ei, bb->succs)
4433 0 : if (AFDO_EINFO (e) != NULL)
4434 : {
4435 0 : delete AFDO_EINFO (e);
4436 0 : e->aux = NULL;
4437 : }
4438 : }
4439 0 : }
4440 :
4441 : /* Annotate auto profile to the control flow graph. */
4442 :
4443 : static void
4444 0 : afdo_annotate_cfg (void)
4445 : {
4446 0 : basic_block bb;
4447 0 : bb_set annotated_bb;
4448 0 : const function_instance *s
4449 0 : = afdo_source_profile->get_function_instance_by_decl (
4450 : current_function_decl);
4451 :
4452 : /* FIXME: This is a workaround for sourcefile tracking, if afdo_string_table
4453 : ends up with empty filename or incorrect filename for the function and
4454 : should be removed once issues with sourcefile tracking get fixed. */
4455 0 : if (s == NULL)
4456 0 : for (unsigned i = 0; i < afdo_string_table->filenames ().length (); i++)
4457 : {
4458 0 : s = afdo_source_profile->get_function_instance_by_decl (current_function_decl, afdo_string_table->filenames()[i]);
4459 0 : if (s)
4460 : break;
4461 : }
4462 :
4463 0 : if (s == NULL)
4464 : {
4465 0 : if (dump_file)
4466 0 : fprintf (dump_file, "No afdo profile for %s\n",
4467 0 : cgraph_node::get (current_function_decl)->dump_name ());
4468 : /* create_gcov only dumps symbols with some samples in them.
4469 : This means that we get nonempty zero_bbs only if some
4470 : nonzero counts in profile were not matched with statements. */
4471 0 : if (!flag_profile_partial_training
4472 0 : && !param_auto_profile_reorder_only)
4473 : {
4474 0 : FOR_ALL_BB_FN (bb, cfun)
4475 0 : if (bb->count.quality () == GUESSED_LOCAL)
4476 0 : bb->count = bb->count.global0afdo ();
4477 0 : update_max_bb_count ();
4478 : }
4479 : return;
4480 : }
4481 :
4482 0 : auto ts_it = timestamp_info_map.find (s->timestamp ());
4483 0 : if (ts_it != timestamp_info_map.end ())
4484 : {
4485 0 : cgraph_node *node = cgraph_node::get (current_function_decl);
4486 0 : node->tp_first_run = ts_it->second;
4487 :
4488 0 : if (dump_file)
4489 0 : fprintf (dump_file, "Setting %s->tp_first_run to %d\n",
4490 : node->asm_name (), node->tp_first_run);
4491 : }
4492 :
4493 0 : if (param_auto_profile_reorder_only)
4494 : return;
4495 :
4496 0 : calculate_dominance_info (CDI_POST_DOMINATORS);
4497 0 : calculate_dominance_info (CDI_DOMINATORS);
4498 0 : loop_optimizer_init (0);
4499 :
4500 0 : if (dump_file)
4501 : {
4502 0 : fprintf (dump_file, "\n\nAnnotating BB profile of %s\n",
4503 0 : cgraph_node::get (current_function_decl)->dump_name ());
4504 0 : fprintf (dump_file, "\n");
4505 0 : s->dump (dump_file);
4506 0 : fprintf (dump_file, "\n");
4507 : }
4508 0 : bool profile_found = false;
4509 0 : hash_set <basic_block> zero_bbs;
4510 0 : gcov_type head_count = s->head_count () * autofdo::afdo_count_scale;
4511 :
4512 0 : if (!param_auto_profile_bbs)
4513 : {
4514 0 : if (scale_bb_profile ())
4515 : return;
4516 : }
4517 : else
4518 : {
4519 : /* In the first pass only store non-zero counts. */
4520 0 : profile_found = head_count > 0;
4521 0 : FOR_EACH_BB_FN (bb, cfun)
4522 : {
4523 0 : if (afdo_set_bb_count (bb, zero_bbs))
4524 : {
4525 0 : if (bb->count.quality () == AFDO)
4526 : {
4527 0 : gcc_assert (bb->count.nonzero_p ());
4528 : profile_found = true;
4529 : }
4530 0 : set_bb_annotated (bb, &annotated_bb);
4531 : }
4532 : }
4533 : }
4534 : /* Exit without clobbering static profile if there was no
4535 : non-zero count. */
4536 0 : if (!profile_found)
4537 : {
4538 : /* create_gcov only dumps symbols with some samples in them.
4539 : This means that we get nonempty zero_bbs only if some
4540 : nonzero counts in profile were not matched with statements.
4541 : ??? We can adjust create_gcov to also recordinfo
4542 : about function with no samples. Then we can distinguish
4543 : between lost profiles which should be kept local and
4544 : real functions with 0 samples during train run. */
4545 0 : if (zero_bbs.is_empty ())
4546 : {
4547 0 : if (dump_file)
4548 0 : fprintf (dump_file, "No afdo samples found"
4549 : "; Setting global count to afdo0\n");
4550 : }
4551 : else
4552 : {
4553 0 : if (dump_file)
4554 0 : fprintf (dump_file, "Setting global count to afdo0\n");
4555 : }
4556 0 : if (!flag_profile_partial_training)
4557 : {
4558 0 : FOR_ALL_BB_FN (bb, cfun)
4559 0 : if (bb->count.quality () == GUESSED_LOCAL)
4560 0 : bb->count = bb->count.global0afdo ();
4561 0 : update_max_bb_count ();
4562 : }
4563 :
4564 0 : loop_optimizer_finalize ();
4565 0 : free_dominance_info (CDI_DOMINATORS);
4566 0 : free_dominance_info (CDI_POST_DOMINATORS);
4567 0 : return;
4568 : }
4569 : /* We try to preserve static profile for BBs with 0
4570 : afdo samples, but if even static profile agrees with 0,
4571 : consider it final so propagation works better. */
4572 0 : for (basic_block bb : zero_bbs)
4573 0 : if (!bb->count.nonzero_p ())
4574 : {
4575 0 : update_count_by_afdo_count (&bb->count, 0);
4576 0 : set_bb_annotated (bb, &annotated_bb);
4577 0 : if (dump_file)
4578 : {
4579 0 : fprintf (dump_file, " Annotating bb %i with count ", bb->index);
4580 0 : bb->count.dump (dump_file);
4581 0 : fprintf (dump_file,
4582 : " (has 0 count in both static and afdo profile)\n");
4583 : }
4584 : }
4585 :
4586 : /* Update profile. */
4587 0 : if (head_count > 0)
4588 : {
4589 0 : update_count_by_afdo_count (&ENTRY_BLOCK_PTR_FOR_FN (cfun)->count,
4590 : head_count);
4591 0 : set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun), &annotated_bb);
4592 0 : if (!is_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb, annotated_bb)
4593 0 : || ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
4594 0 : > ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count)
4595 : {
4596 0 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count
4597 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4598 0 : set_bb_annotated (ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb,
4599 : &annotated_bb);
4600 : }
4601 0 : if (!is_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun), annotated_bb)
4602 0 : || ENTRY_BLOCK_PTR_FOR_FN (cfun)->count
4603 0 : > EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count)
4604 : {
4605 0 : EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb->count
4606 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4607 0 : set_bb_annotated (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb, &annotated_bb);
4608 : }
4609 : }
4610 :
4611 : /* Calculate, propagate count and probability information on CFG. */
4612 0 : afdo_calculate_branch_prob (&annotated_bb);
4613 :
4614 : /* If we failed to turn some of original guessed profile to global,
4615 : set basic blocks uninitialized. */
4616 0 : FOR_ALL_BB_FN (bb, cfun)
4617 0 : if (!bb->count.ipa_p ())
4618 : {
4619 : /* We skip annotating entry profile if it is 0
4620 : in hope to be able to determine it better from the
4621 : static profile.
4622 :
4623 : Now we know we can not derive it from other info,
4624 : so set it since it is better than UNKNOWN. */
4625 0 : if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
4626 0 : bb->count = profile_count::zero ().afdo ();
4627 : else
4628 0 : bb->count = profile_count::uninitialized ();
4629 0 : if (dump_file)
4630 0 : fprintf (dump_file, " Unknown count of bb %i\n", bb->index);
4631 0 : cfun->cfg->full_profile = false;
4632 : }
4633 :
4634 0 : cgraph_node::get (current_function_decl)->count
4635 0 : = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
4636 0 : update_max_bb_count ();
4637 0 : profile_status_for_fn (cfun) = PROFILE_READ;
4638 0 : if (flag_value_profile_transformations)
4639 : {
4640 0 : gimple_value_profile_transformations ();
4641 0 : free_dominance_info (CDI_DOMINATORS);
4642 0 : free_dominance_info (CDI_POST_DOMINATORS);
4643 0 : update_ssa (TODO_update_ssa);
4644 : }
4645 :
4646 0 : loop_optimizer_finalize ();
4647 0 : free_dominance_info (CDI_DOMINATORS);
4648 0 : free_dominance_info (CDI_POST_DOMINATORS);
4649 0 : }
4650 :
4651 : /* Use AutoFDO profile to annotate the control flow graph.
4652 : Return the todo flag. */
4653 :
4654 : static unsigned int
4655 0 : auto_profile (void)
4656 : {
4657 0 : struct cgraph_node *node;
4658 :
4659 0 : if (symtab->state == FINISHED || !afdo_source_profile)
4660 : return 0;
4661 :
4662 0 : init_node_map (true);
4663 0 : profile_info = autofdo::afdo_profile_info;
4664 0 : afdo_source_profile->offline_unrealized_inlines ();
4665 :
4666 0 : FOR_EACH_FUNCTION (node)
4667 : {
4668 0 : if (!gimple_has_body_p (node->decl))
4669 0 : continue;
4670 :
4671 : /* Don't profile functions produced for builtin stuff. */
4672 0 : if (DECL_SOURCE_LOCATION (node->decl) == BUILTINS_LOCATION)
4673 0 : continue;
4674 :
4675 0 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4676 :
4677 : /* Local pure-const may imply need to fixup the cfg.
4678 : This is similar to what is done in tree-profile.cc. */
4679 0 : if ((execute_fixup_cfg () & TODO_cleanup_cfg))
4680 0 : cleanup_tree_cfg ();
4681 :
4682 0 : autofdo::afdo_annotate_cfg ();
4683 0 : compute_function_frequency ();
4684 :
4685 0 : free_dominance_info (CDI_DOMINATORS);
4686 0 : free_dominance_info (CDI_POST_DOMINATORS);
4687 0 : cgraph_edge::rebuild_edges ();
4688 0 : pop_cfun ();
4689 : }
4690 :
4691 : return 0;
4692 : }
4693 : } /* namespace autofdo. */
4694 :
4695 : /* Read the profile from the profile data file. */
4696 :
4697 : void
4698 0 : read_autofdo_file (void)
4699 : {
4700 0 : if (auto_profile_file == NULL)
4701 0 : auto_profile_file = DEFAULT_AUTO_PROFILE_FILE;
4702 :
4703 0 : autofdo::afdo_profile_info = XNEW (gcov_summary);
4704 0 : autofdo::afdo_profile_info->runs = 1;
4705 0 : autofdo::afdo_profile_info->sum_max = 0;
4706 0 : autofdo::afdo_profile_info->cutoff = 1;
4707 :
4708 : /* Read the profile from the profile file. */
4709 0 : autofdo::read_profile ();
4710 0 : }
4711 :
4712 : /* Free the resources. */
4713 :
4714 : void
4715 0 : end_auto_profile (void)
4716 : {
4717 0 : delete autofdo::afdo_source_profile;
4718 0 : delete autofdo::afdo_string_table;
4719 0 : delete autofdo::afdo_summary_info;
4720 0 : profile_info = NULL;
4721 0 : }
4722 :
4723 : /* Returns TRUE if EDGE is hot enough to be inlined early. */
4724 :
4725 : bool
4726 0 : afdo_callsite_hot_enough_for_early_inline (struct cgraph_edge *edge)
4727 : {
4728 0 : gcov_type count
4729 0 : = autofdo::afdo_source_profile->get_callsite_total_count (edge);
4730 :
4731 0 : if (count > 0)
4732 : {
4733 0 : bool is_hot;
4734 0 : profile_count pcount = profile_count::from_gcov_type (count).afdo ();
4735 0 : is_hot = maybe_hot_afdo_count_p (pcount);
4736 0 : if (dump_file)
4737 : {
4738 0 : fprintf (dump_file, "Call %s -> %s has %s afdo profile count ",
4739 0 : edge->caller->dump_name (), edge->callee->dump_name (),
4740 : is_hot ? "hot" : "cold");
4741 0 : pcount.dump (dump_file);
4742 0 : fprintf (dump_file, "\n");
4743 : }
4744 0 : return is_hot;
4745 : }
4746 :
4747 : return false;
4748 : }
4749 :
4750 : /* Do indirect call promotion during early inlining to make the
4751 : IR match the profiled binary before actual annotation.
4752 :
4753 : This is needed because an indirect call might have been promoted
4754 : and inlined in the profiled binary. If we do not promote and
4755 : inline these indirect calls before annotation, the profile for
4756 : these promoted functions will be lost.
4757 :
4758 : e.g. foo() --indirect_call--> bar()
4759 : In profiled binary, the callsite is promoted and inlined, making
4760 : the profile look like:
4761 :
4762 : foo: {
4763 : loc_foo_1: count_1
4764 : bar@loc_foo_2: {
4765 : loc_bar_1: count_2
4766 : loc_bar_2: count_3
4767 : }
4768 : }
4769 :
4770 : Before AutoFDO pass, loc_foo_2 is not promoted thus not inlined.
4771 : If we perform annotation on it, the profile inside bar@loc_foo2
4772 : will be wasted.
4773 :
4774 : To avoid this, we promote loc_foo_2 and inline the promoted bar
4775 : function before annotation, so the profile inside bar@loc_foo2
4776 : will be useful. */
4777 :
4778 : bool
4779 0 : afdo_vpt_for_early_inline (cgraph_node *node)
4780 : {
4781 0 : if (!node->indirect_calls)
4782 : return false;
4783 0 : bool changed = false;
4784 0 : cgraph_node *outer = node->inlined_to ? node->inlined_to : node;
4785 0 : if (autofdo::afdo_source_profile->get_function_instance_by_decl
4786 0 : (outer->decl) == NULL)
4787 : return false;
4788 0 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
4789 : {
4790 0 : gcov_type bb_count = 0;
4791 0 : autofdo::count_info info;
4792 0 : basic_block bb = gimple_bb (e->call_stmt);
4793 :
4794 : /* TODO: This is quadratic; cache the value. */
4795 0 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
4796 0 : !gsi_end_p (gsi); gsi_next (&gsi))
4797 : {
4798 0 : gimple *stmt = gsi_stmt (gsi);
4799 0 : if (!stmt_loc_used_by_debug_info (stmt))
4800 0 : continue;
4801 0 : autofdo::count_info info;
4802 0 : if (autofdo::afdo_source_profile->get_count_info (stmt, &info, node))
4803 0 : bb_count = MAX (bb_count, info.count);
4804 0 : }
4805 0 : autofdo::afdo_source_profile->get_count_info (e->call_stmt, &info, node);
4806 0 : info.count = bb_count;
4807 0 : if (!autofdo::afdo_source_profile->update_inlined_ind_target
4808 0 : (e->call_stmt, &info, node))
4809 0 : continue;
4810 0 : changed |= autofdo::afdo_vpt (e->call_stmt, info.targets, true, e);
4811 0 : }
4812 : return changed;
4813 : }
4814 :
4815 : /* If speculation used during early inline, remove the target
4816 : so we do not speculate the indirect edge again during afdo pass. */
4817 :
4818 : void
4819 0 : remove_afdo_speculative_target (cgraph_edge *e)
4820 : {
4821 0 : autofdo::afdo_source_profile->remove_icall_target (e);
4822 0 : }
4823 :
4824 : namespace
4825 : {
4826 :
4827 : const pass_data pass_data_ipa_auto_profile = {
4828 : SIMPLE_IPA_PASS, "afdo", /* name */
4829 : OPTGROUP_NONE, /* optinfo_flags */
4830 : TV_IPA_AUTOFDO, /* tv_id */
4831 : 0, /* properties_required */
4832 : 0, /* properties_provided */
4833 : 0, /* properties_destroyed */
4834 : 0, /* todo_flags_start */
4835 : 0, /* todo_flags_finish */
4836 : };
4837 :
4838 : class pass_ipa_auto_profile : public simple_ipa_opt_pass
4839 : {
4840 : public:
4841 294196 : pass_ipa_auto_profile (gcc::context *ctxt)
4842 588392 : : simple_ipa_opt_pass (pass_data_ipa_auto_profile, ctxt)
4843 : {
4844 : }
4845 :
4846 : /* opt_pass methods: */
4847 : bool
4848 237067 : gate (function *) final override
4849 : {
4850 237067 : return flag_auto_profile;
4851 : }
4852 : unsigned int
4853 0 : execute (function *) final override
4854 : {
4855 0 : return autofdo::auto_profile ();
4856 : }
4857 : }; // class pass_ipa_auto_profile
4858 :
4859 : } // anon namespace
4860 :
4861 : simple_ipa_opt_pass *
4862 294196 : make_pass_ipa_auto_profile (gcc::context *ctxt)
4863 : {
4864 294196 : return new pass_ipa_auto_profile (ctxt);
4865 : }
4866 :
4867 : namespace
4868 : {
4869 :
4870 : const pass_data pass_data_ipa_auto_profile_offline = {
4871 : SIMPLE_IPA_PASS, "afdo_offline", /* name */
4872 : OPTGROUP_NONE, /* optinfo_flags */
4873 : TV_IPA_AUTOFDO_OFFLINE, /* tv_id */
4874 : 0, /* properties_required */
4875 : 0, /* properties_provided */
4876 : 0, /* properties_destroyed */
4877 : 0, /* todo_flags_start */
4878 : 0, /* todo_flags_finish */
4879 : };
4880 :
4881 : class pass_ipa_auto_profile_offline : public simple_ipa_opt_pass
4882 : {
4883 : public:
4884 294196 : pass_ipa_auto_profile_offline (gcc::context *ctxt)
4885 588392 : : simple_ipa_opt_pass (pass_data_ipa_auto_profile_offline, ctxt)
4886 : {
4887 : }
4888 :
4889 : /* opt_pass methods: */
4890 : bool
4891 237067 : gate (function *) final override
4892 : {
4893 237067 : return flag_auto_profile;
4894 : }
4895 : unsigned int
4896 0 : execute (function *) final override
4897 : {
4898 0 : read_autofdo_file ();
4899 0 : if (autofdo::afdo_source_profile)
4900 0 : autofdo::afdo_source_profile->offline_external_functions ();
4901 0 : return 0;
4902 : }
4903 : }; // class pass_ipa_auto_profile
4904 :
4905 : } // anon namespace
4906 :
4907 : simple_ipa_opt_pass *
4908 294196 : make_pass_ipa_auto_profile_offline (gcc::context *ctxt)
4909 : {
4910 294196 : return new pass_ipa_auto_profile_offline (ctxt);
4911 : }
|
