Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2363738344 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2363738344 : uint64_t tmp;
88 2363738344 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2363738344 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2361293362 : *res = tmp / c;
92 2361293362 : return true;
93 : }
94 2444982 : if (c == 1)
95 : {
96 39645 : *res = (uint64_t) -1;
97 39645 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2405337 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greater then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5577563841 : set_quality (profile_quality quality)
169 : {
170 1166063749 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1166063749 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1164316115 : }
174 :
175 : public:
176 4413300241 : profile_probability (): m_val (uninitialized_probability)
177 2419533925 : { set_quality (GUESSED); }
178 :
179 441 : profile_probability (uint32_t val, profile_quality quality):
180 441 : m_val (val)
181 441 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 775968325 : static profile_probability never ()
185 : {
186 775968325 : profile_probability ret;
187 775968325 : ret.m_val = 0;
188 775968325 : ret.set_quality (PRECISE);
189 465325887 : return ret;
190 : }
191 :
192 5383164 : static profile_probability guessed_never ()
193 : {
194 5383164 : profile_probability ret;
195 5383164 : ret.m_val = 0;
196 5383164 : ret.set_quality (GUESSED);
197 5383146 : return ret;
198 : }
199 :
200 5366196 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10732392 : profile_probability r = guessed_always () / 2000;
204 5366196 : r.m_val--;
205 5366196 : return r;
206 : }
207 :
208 88378 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 176756 : profile_probability r = guessed_always () / 5;
212 88378 : r.m_val--;
213 88378 : return r;
214 : }
215 :
216 746272 : static profile_probability even ()
217 : {
218 1492544 : return guessed_always () / 2;
219 : }
220 :
221 347593 : static profile_probability very_likely ()
222 : {
223 347593 : return always () - very_unlikely ();
224 : }
225 :
226 33615 : static profile_probability likely ()
227 : {
228 33615 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 5695 : bool nonzero_p () const
232 : {
233 5695 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6510590 : static profile_probability guessed_always ()
237 : {
238 6510590 : profile_probability ret;
239 6510590 : ret.m_val = max_probability;
240 6510590 : ret.set_quality (GUESSED);
241 6291404 : return ret;
242 : }
243 :
244 569540700 : static profile_probability always ()
245 : {
246 568786159 : profile_probability ret;
247 569540700 : ret.m_val = max_probability;
248 569540700 : ret.set_quality (PRECISE);
249 389750140 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 963343674 : static profile_probability uninitialized ()
255 : {
256 963343674 : profile_probability c;
257 963343674 : c.m_val = uninitialized_probability;
258 963343674 : c.set_quality (GUESSED);
259 949646585 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5825593884 : bool initialized_p () const
264 : {
265 4067637209 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1622022 : bool reliable_p () const
270 : {
271 133515426 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3224532 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3224532 : profile_probability ret;
279 3224532 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3224532 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3224532 : ret.set_quality (GUESSED);
282 3224532 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 43 : profile_probability adjusted () const
287 : {
288 43 : profile_probability ret = *this;
289 43 : if (!initialized_p ())
290 0 : return *this;
291 43 : ret.set_quality (ADJUSTED);
292 43 : return ret;
293 : }
294 :
295 1146644075 : int to_reg_br_prob_base () const
296 : {
297 1146644075 : gcc_checking_assert (initialized_p ());
298 1146644075 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 504332233 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 504332233 : profile_probability ret;
305 504332233 : ret.m_val = ((unsigned int)v) / 8;
306 504332233 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 16198117 : return ret;
308 : }
309 :
310 474619554 : int to_reg_br_prob_note () const
311 : {
312 474619554 : gcc_checking_assert (initialized_p ());
313 474619554 : int ret = m_val * 8 + m_adjusted_quality;
314 474619554 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 474619554 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3223 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3223 : profile_probability ret;
323 3223 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3223 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3223 : uint64_t tmp;
329 3223 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3223 : gcc_checking_assert (tmp <= max_probability);
331 3223 : ret.m_val = tmp;
332 : }
333 3223 : ret.set_quality (PRECISE);
334 3223 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1684756309 : bool operator== (const profile_probability &other) const
339 : {
340 2374699255 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6798934 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6798934 : if (other == never ())
346 35356 : return *this;
347 6763578 : if (*this == never ())
348 23939 : return other;
349 6739639 : if (!initialized_p () || !other.initialized_p ())
350 5534581 : return uninitialized ();
351 :
352 1205058 : profile_probability ret;
353 1205058 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1205058 : ret.set_quality (MIN (quality (), other.quality ()));
355 1205058 : return ret;
356 : }
357 :
358 6652854 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6652854 : if (other == never ())
361 : return *this;
362 5326678 : if (*this == never ())
363 : {
364 911879 : *this = other;
365 911879 : return *this;
366 : }
367 4414799 : if (!initialized_p () || !other.initialized_p ())
368 2976948 : return *this = uninitialized ();
369 : else
370 : {
371 1437851 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 8617857 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1437851 : return *this;
375 : }
376 :
377 35017330 : profile_probability operator- (const profile_probability &other) const
378 : {
379 70034660 : if (*this == never ()
380 35017330 : || other == never ())
381 1914012 : return *this;
382 33103318 : if (!initialized_p () || !other.initialized_p ())
383 2193265 : return uninitialized ();
384 30910053 : profile_probability ret;
385 30910053 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30910053 : ret.set_quality (MIN (quality (), other.quality ()));
387 30910053 : return ret;
388 : }
389 :
390 11864624 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 23729248 : if (*this == never ()
393 11864624 : || other == never ())
394 : return *this;
395 9451393 : if (!initialized_p () || !other.initialized_p ())
396 2142529 : return *this = uninitialized ();
397 : else
398 : {
399 7308864 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 43853041 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 7308864 : return *this;
403 : }
404 :
405 1252469 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2504938 : if (*this == never ()
408 1252469 : || other == never ())
409 125778 : return never ();
410 1126691 : if (!initialized_p () || !other.initialized_p ())
411 58559 : return uninitialized ();
412 1068132 : profile_probability ret;
413 1068132 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 1068132 : ret.set_quality (MIN (quality (), other.quality ()));
415 1068132 : return ret;
416 : }
417 :
418 130536 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 261072 : if (*this == never ()
421 130536 : || other == never ())
422 444 : return *this = never ();
423 130092 : if (!initialized_p () || !other.initialized_p ())
424 17 : return *this = uninitialized ();
425 : else
426 : {
427 130075 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 130075 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 130075 : return *this;
431 : }
432 :
433 1797108 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1797108 : if (*this == never ())
436 102658 : return never ();
437 1694450 : if (!initialized_p () || !other.initialized_p ())
438 228497 : return uninitialized ();
439 1465953 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1465953 : if (m_val >= other.m_val)
442 : {
443 52206 : ret.m_val = max_probability;
444 52206 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 52206 : return ret;
447 : }
448 1413747 : else if (!m_val)
449 66357 : ret.m_val = 0;
450 : else
451 : {
452 1347390 : gcc_checking_assert (other.m_val);
453 1347390 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2816766 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1413747 : return ret;
457 : }
458 :
459 294929 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 294929 : if (*this == never ())
462 2149 : return *this = never ();
463 292780 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 292780 : if (m_val > other.m_val)
470 : {
471 516 : m_val = max_probability;
472 516 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 516 : return *this;
475 : }
476 292264 : else if (!m_val)
477 : ;
478 : else
479 : {
480 290110 : gcc_checking_assert (other.m_val);
481 290110 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 292264 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 292264 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 301706 : profile_probability split (const profile_probability &cprob)
504 : {
505 301706 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 301706 : if (!(*this == always ()))
512 292187 : *this = (*this - ret) / ret.invert ();
513 301706 : return ret;
514 : }
515 :
516 : /* Return VAL scaled by this probability. Round initialized probabilities
517 : to the nearest integer, with halfway values away from zero. Treat an
518 : uninitialized probability as one half and truncate toward zero. */
519 278826 : gcov_type apply (gcov_type val) const
520 : {
521 278826 : if (*this == uninitialized ())
522 73 : return val / 2;
523 :
524 : /* A unit probability leaves VAL unchanged. Return it directly because
525 : the magnitude of the minimum gcov_type value is one greater than the
526 : maximum gcov_type value. */
527 278753 : if (m_val == max_probability)
528 : return val;
529 :
530 : /* Convert to unsigned before negating so that the minimum gcov_type
531 : value has a representable magnitude. Scale the magnitude with
532 : overflow-safe arithmetic, then restore the sign. */
533 277276 : gcov_type_unsigned magnitude
534 277292 : = val < 0 ? -(gcov_type_unsigned) val : (gcov_type_unsigned) val;
535 277292 : uint64_t scaled;
536 277292 : bool scaled_p
537 277292 : = safe_scale_64bit (magnitude, m_val, max_probability, &scaled);
538 : /* The scaled result fits in uint64_t. With the unit case handled above,
539 : it also fits in the nonnegative range of gcov_type. */
540 277292 : gcc_checking_assert (scaled_p);
541 277292 : gcc_checking_assert
542 : (scaled <= (gcov_type_unsigned) INTTYPE_MAXIMUM (gcov_type));
543 :
544 277292 : return val < 0 ? -(gcov_type) scaled : (gcov_type) scaled;
545 : }
546 :
547 : /* Return 1-*THIS. */
548 21818136 : profile_probability invert () const
549 : {
550 31752180 : return always() - *this;
551 : }
552 :
553 : /* Return THIS with quality dropped to GUESSED. */
554 889014 : profile_probability guessed () const
555 : {
556 889014 : profile_probability ret = *this;
557 889014 : ret.set_quality (GUESSED);
558 888978 : return ret;
559 : }
560 :
561 : /* Return THIS with quality dropped to AFDO. */
562 : profile_probability afdo () const
563 : {
564 : profile_probability ret = *this;
565 : ret.set_quality (AFDO);
566 : return ret;
567 : }
568 :
569 : /* Return *THIS * NUM / DEN. */
570 14808452 : profile_probability apply_scale (int64_t num, int64_t den) const
571 : {
572 14808452 : if (*this == never ())
573 16379 : return *this;
574 14792073 : if (!initialized_p ())
575 5946052 : return uninitialized ();
576 8846021 : profile_probability ret;
577 8846021 : uint64_t tmp;
578 8846021 : safe_scale_64bit (m_val, num, den, &tmp);
579 8846021 : ret.m_val = MIN (tmp, max_probability);
580 8846021 : ret.set_quality (MIN (quality (), ADJUSTED));
581 8846021 : return ret;
582 : }
583 :
584 : /* Return *THIS * NUM / DEN. */
585 14231 : profile_probability apply_scale (profile_probability num,
586 : profile_probability den) const
587 : {
588 14231 : if (*this == never ())
589 306 : return *this;
590 13925 : if (num == never ())
591 0 : return num;
592 13925 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
593 0 : return uninitialized ();
594 13925 : if (num == den)
595 0 : return *this;
596 13925 : gcc_checking_assert (den.m_val);
597 :
598 13925 : profile_probability ret;
599 13925 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
600 : max_probability);
601 13925 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
602 : num.quality ()), den.quality ()));
603 13925 : return ret;
604 : }
605 :
606 : /* Return true when the probability of edge is reliable.
607 :
608 : The profile guessing code is good at predicting branch outcome (i.e.
609 : taken/not taken), that is predicted right slightly over 75% of time.
610 : It is however notoriously poor on predicting the probability itself.
611 : In general the profile appear a lot flatter (with probabilities closer
612 : to 50%) than the reality so it is bad idea to use it to drive optimization
613 : such as those disabling dynamic branch prediction for well predictable
614 : branches.
615 :
616 : There are two exceptions - edges leading to noreturn edges and edges
617 : predicted by number of iterations heuristics are predicted well. This macro
618 : should be able to distinguish those, but at the moment it simply check for
619 : noreturn heuristic that is only one giving probability over 99% or below
620 : 1%. In future we might want to propagate reliability information across the
621 : CFG if we find this information useful on multiple places. */
622 0 : bool probably_reliable_p () const
623 : {
624 0 : if (quality () >= ADJUSTED)
625 : return true;
626 0 : if (!initialized_p ())
627 : return false;
628 0 : return m_val < max_probability / 100
629 0 : || m_val > max_probability - max_probability / 100;
630 : }
631 :
632 : /* Return false if profile_probability is bogus. */
633 4483543398 : bool verify () const
634 : {
635 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
636 4483543398 : if (m_val == uninitialized_probability)
637 966455729 : return quality () == GUESSED;
638 3517087669 : else if (quality () < GUESSED)
639 : return false;
640 3517087669 : return m_val <= max_probability;
641 : }
642 :
643 : /* Comparisons are three-state and conservative. False is returned if
644 : the inequality cannot be decided. */
645 9184720 : bool operator< (const profile_probability &other) const
646 : {
647 9184720 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
648 : }
649 :
650 18010185 : bool operator> (const profile_probability &other) const
651 : {
652 18010185 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
653 : }
654 :
655 5528302 : bool operator<= (const profile_probability &other) const
656 : {
657 5528302 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
658 : }
659 :
660 966631 : bool operator>= (const profile_probability &other) const
661 : {
662 966631 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
663 : }
664 :
665 17 : profile_probability operator* (int64_t num) const
666 : {
667 17 : return apply_scale (num, 1);
668 : }
669 :
670 : profile_probability operator*= (int64_t num)
671 : {
672 : *this = apply_scale (num, 1);
673 : return *this;
674 : }
675 :
676 14137529 : profile_probability operator/ (int64_t den) const
677 : {
678 14137529 : return apply_scale (1, den);
679 : }
680 :
681 247438 : profile_probability operator/= (int64_t den)
682 : {
683 247438 : *this = apply_scale (1, den);
684 247438 : return *this;
685 : }
686 :
687 : /* Compute n-th power. */
688 : profile_probability pow (int) const;
689 :
690 : /* Compute sware root. */
691 : profile_probability sqrt () const;
692 :
693 : /* Get the value of the probability. */
694 0 : uint32_t value () const { return m_val; }
695 :
696 : /* Get the quality of the probability. */
697 6237039484 : enum profile_quality quality () const
698 : {
699 6017076037 : return (profile_quality) (m_adjusted_quality
700 6017098127 : ? m_adjusted_quality + min_quality - 1
701 51680108 : : UNINITIALIZED_PROFILE);
702 : }
703 :
704 : /* Output THIS to F. */
705 : void dump (FILE *f) const;
706 :
707 : /* Output THIS to BUFFER. */
708 : void dump (char *buffer) const;
709 :
710 : /* Print THIS to stderr. */
711 : void debug () const;
712 :
713 : /* Return true if THIS is known to differ significantly from OTHER. */
714 : bool differs_from_p (profile_probability other) const;
715 :
716 : /* Return if difference is greater than 50%. */
717 : bool differs_lot_from_p (profile_probability other) const;
718 :
719 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
720 : happens with COUNT2 probability. Return probability that either *THIS or
721 : OTHER happens. */
722 : profile_probability combine_with_count (profile_count count1,
723 : profile_probability other,
724 : profile_count count2) const;
725 :
726 : /* Return probability as sreal. */
727 : sreal to_sreal () const;
728 : /* LTO streaming support. */
729 : static profile_probability stream_in (class lto_input_block *);
730 : void stream_out (struct output_block *);
731 : void stream_out (struct lto_output_stream *);
732 : };
733 :
734 : /* Main data type to hold profile counters in GCC. Profile counts originate
735 : either from profile feedback, static profile estimation or both. We do not
736 : perform whole program profile propagation and thus profile estimation
737 : counters are often local to function, while counters from profile feedback
738 : (or special cases of profile estimation) can be used inter-procedurally.
739 :
740 : There are 3 basic types
741 : 1) local counters which are result of intra-procedural static profile
742 : estimation.
743 : 2) ipa counters which are result of profile feedback or special case
744 : of static profile estimation (such as in function main).
745 : 3) counters which counts as 0 inter-procedurally (because given function
746 : was never run in train feedback) but they hold local static profile
747 : estimate.
748 :
749 : Counters of type 1 and 3 cannot be mixed with counters of different type
750 : within operation (because whole function should use one type of counter)
751 : with exception that global zero mix in most operations where outcome is
752 : well defined.
753 :
754 : To take local counter and use it inter-procedurally use ipa member function
755 : which strips information irrelevant at the inter-procedural level.
756 :
757 : Counters are 61bit integers representing number of executions during the
758 : train run or normalized frequency within the function.
759 :
760 : As the profile is maintained during the compilation, many adjustments are
761 : made. Not all transformations can be made precisely, most importantly
762 : when code is being duplicated. It also may happen that part of CFG has
763 : profile counts known while other do not - for example when LTO optimizing
764 : partly profiled program or when profile was lost due to COMDAT merging.
765 :
766 : For this reason profile_count tracks more information than
767 : just unsigned integer and it is also ready for profile mismatches.
768 : The API of this data type represent operations that are natural
769 : on profile counts - sum, difference and operation with scales and
770 : probabilities. All operations are safe by never getting negative counts
771 : and they do end up in uninitialized scale if any of the parameters is
772 : uninitialized.
773 :
774 : All comparisons that are three state and handling of probabilities. Thus
775 : a < b is not equal to !(a >= b).
776 :
777 : The following pre-defined counts are available:
778 :
779 : profile_count::zero () for code that is known to execute zero times at
780 : runtime (this can be detected statically i.e. for paths leading to
781 : abort ();
782 : profile_count::one () for code that is known to execute once (such as
783 : main () function
784 : profile_count::uninitialized () for unknown execution count.
785 :
786 : */
787 :
788 : struct GTY(()) profile_count
789 : {
790 : public:
791 : /* Use 60bit to hold basic block counters. Should be at least
792 : 64bit. Although a counter cannot be negative, we use a signed
793 : type to hold various extra stages. */
794 :
795 : static const int n_bits = 60;
796 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
797 : private:
798 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
799 :
800 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
801 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
802 : incorrectly detects the alignment of a structure where the only
803 : 64-bit aligned object is a bit-field. We force the alignment of
804 : the entire field to mitigate this. */
805 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
806 : #else
807 : #define UINT64_BIT_FIELD_ALIGN
808 : #endif
809 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
810 : #undef UINT64_BIT_FIELD_ALIGN
811 : enum profile_quality m_quality : 4;
812 : public:
813 :
814 : /* Return true if both values can meaningfully appear in single function
815 : body. We have either all counters in function local or global, otherwise
816 : operations between them are not really defined well. */
817 5043557045 : bool compatible_p (const profile_count other) const
818 : {
819 9260201420 : if (!initialized_p () || !other.initialized_p ())
820 : return true;
821 16688815282 : if (*this == zero ()
822 161655890 : || other == zero ())
823 : return true;
824 : /* Do not allow nonzero global profile together with local guesses
825 : that are globally0. */
826 4050054377 : if (ipa ().nonzero_p ()
827 258208 : && !(other.ipa () == other))
828 0 : return false;
829 4050054377 : if (other.ipa ().nonzero_p ()
830 267935 : && !(ipa () == *this))
831 0 : return false;
832 :
833 4050054377 : return ipa_p () == other.ipa_p ();
834 : }
835 :
836 : /* Used for counters which are expected to be never executed. */
837 18766421928 : static profile_count zero ()
838 : {
839 14695068381 : return from_gcov_type (0);
840 : }
841 :
842 : static profile_count adjusted_zero ()
843 : {
844 : profile_count c;
845 : c.m_val = 0;
846 : c.m_quality = ADJUSTED;
847 : return c;
848 : }
849 :
850 : static profile_count afdo_zero ()
851 : {
852 : profile_count c;
853 : c.m_val = 0;
854 : c.m_quality = AFDO;
855 : return c;
856 : }
857 :
858 : static profile_count guessed_zero ()
859 : {
860 : profile_count c;
861 : c.m_val = 0;
862 : c.m_quality = GUESSED;
863 : return c;
864 : }
865 :
866 54353 : static profile_count one ()
867 : {
868 54155 : return from_gcov_type (1);
869 : }
870 :
871 : /* Value of counters which has not been initialized. Either because
872 : initialization did not happen yet or because profile is unknown. */
873 113864833 : static profile_count uninitialized ()
874 : {
875 113864833 : profile_count c;
876 113864833 : c.m_val = uninitialized_count;
877 113864833 : c.m_quality = GUESSED_LOCAL;
878 113864833 : return c;
879 : }
880 :
881 : /* Conversion to gcov_type is lossy. */
882 42255178 : gcov_type to_gcov_type () const
883 : {
884 0 : gcc_checking_assert (initialized_p ());
885 42255178 : return m_val;
886 : }
887 :
888 : /* Return true if value has been initialized. */
889 33250957153 : bool initialized_p () const
890 : {
891 11179198335 : return m_val != uninitialized_count;
892 : }
893 :
894 : /* Return true if value can be trusted. */
895 27551220 : bool reliable_p () const
896 : {
897 27550938 : return m_quality >= ADJUSTED;
898 : }
899 :
900 : /* Return true if value can be operated inter-procedurally. */
901 9146901194 : bool ipa_p () const
902 : {
903 5082767911 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
904 : }
905 :
906 : /* Return true if quality of profile is precise. */
907 60280371 : bool precise_p () const
908 : {
909 60280371 : return m_quality == PRECISE;
910 : }
911 :
912 : /* Get the value of the count. */
913 0 : uint64_t value () const { return m_val; }
914 :
915 : /* Get the quality of the count. */
916 100011826 : enum profile_quality quality () const { return m_quality; }
917 :
918 : /* When merging basic blocks, the two different profile counts are unified.
919 : Return true if this can be done without losing info about profile.
920 : The only case we care about here is when first BB contains something
921 : that makes it terminate in a way not visible in CFG. */
922 3410080 : bool ok_for_merging (profile_count other) const
923 : {
924 3410080 : if (m_quality < ADJUSTED
925 18161 : || other.m_quality < ADJUSTED)
926 : return true;
927 16954 : return !(other < *this);
928 : }
929 :
930 : /* When merging two BBs with different counts, pick common count that looks
931 : most representative. */
932 18129113 : profile_count merge (profile_count other) const
933 : {
934 915047 : if (*this == other || !other.initialized_p ()
935 914751 : || m_quality > other.m_quality)
936 17215820 : return *this;
937 913293 : if (other.m_quality > m_quality
938 913293 : || other > *this)
939 0 : return other;
940 913293 : return *this;
941 : }
942 :
943 : /* Basic operations. */
944 20274940492 : bool operator== (const profile_count &other) const
945 : {
946 15443194154 : return m_val == other.m_val && m_quality == other.m_quality;
947 : }
948 :
949 3984738 : profile_count operator+ (const profile_count &other) const
950 : {
951 3984738 : if (other == zero ())
952 2553 : return *this;
953 3982185 : if (*this == zero ())
954 240456 : return other;
955 3741729 : if (!initialized_p () || !other.initialized_p ())
956 6488 : return uninitialized ();
957 :
958 3735241 : profile_count ret;
959 3735241 : gcc_checking_assert (compatible_p (other));
960 3735241 : uint64_t ret_val = m_val + other.m_val;
961 3735241 : ret.m_val = MIN (ret_val, max_count);
962 3735241 : ret.m_quality = MIN (m_quality, other.m_quality);
963 3735241 : return ret;
964 : }
965 :
966 75523155 : profile_count &operator+= (const profile_count &other)
967 : {
968 75523155 : if (other == zero ())
969 5665724 : return *this;
970 69857431 : if (*this == zero ())
971 : {
972 42854052 : *this = other;
973 42854052 : return *this;
974 : }
975 27003379 : if (!initialized_p () || !other.initialized_p ())
976 356622 : return *this = uninitialized ();
977 : else
978 : {
979 26646757 : gcc_checking_assert (compatible_p (other));
980 26646757 : uint64_t ret_val = m_val + other.m_val;
981 26646757 : m_val = MIN (ret_val, max_count);
982 26646757 : m_quality = MIN (m_quality, other.m_quality);
983 : }
984 26646757 : return *this;
985 : }
986 :
987 21426237 : profile_count operator- (const profile_count &other) const
988 : {
989 22999958 : if (*this == zero () || other == zero ())
990 1559662 : return *this;
991 19866575 : if (!initialized_p () || !other.initialized_p ())
992 9184548 : return uninitialized ();
993 10682027 : gcc_checking_assert (compatible_p (other));
994 10682027 : profile_count ret;
995 10682027 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
996 10682027 : ret.m_quality = MIN (m_quality, other.m_quality);
997 10682027 : return ret;
998 : }
999 :
1000 9358209 : profile_count &operator-= (const profile_count &other)
1001 : {
1002 10110661 : if (*this == zero () || other == zero ())
1003 : return *this;
1004 9062705 : if (!initialized_p () || !other.initialized_p ())
1005 390916 : return *this = uninitialized ();
1006 : else
1007 : {
1008 8671789 : gcc_checking_assert (compatible_p (other));
1009 8671789 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
1010 8671789 : m_quality = MIN (m_quality, other.m_quality);
1011 : }
1012 8671789 : return *this;
1013 : }
1014 :
1015 : /* Return false if profile_count is bogus. */
1016 3425010847 : bool verify () const
1017 : {
1018 3425010847 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1019 3425010847 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1020 : }
1021 :
1022 : /* Comparisons are three-state and conservative. False is returned if
1023 : the inequality cannot be decided. */
1024 1093087660 : bool operator< (const profile_count &other) const
1025 : {
1026 1093087660 : if (!initialized_p () || !other.initialized_p ())
1027 : return false;
1028 1088166698 : if (*this == zero ())
1029 4474278 : return !(other == zero ());
1030 1085929559 : if (other == zero ())
1031 911 : return false;
1032 1085928648 : gcc_checking_assert (compatible_p (other));
1033 1085928648 : return m_val < other.m_val;
1034 : }
1035 :
1036 39577339 : bool operator> (const profile_count &other) const
1037 : {
1038 39577339 : if (!initialized_p () || !other.initialized_p ())
1039 : return false;
1040 34366153 : if (*this == zero ())
1041 227715 : return false;
1042 34138438 : if (other == zero ())
1043 1564742 : return !(*this == zero ());
1044 33356067 : gcc_checking_assert (compatible_p (other));
1045 33356067 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1046 : }
1047 :
1048 : bool operator< (const gcov_type other) const
1049 : {
1050 : gcc_checking_assert (ipa_p ());
1051 : gcc_checking_assert (other >= 0);
1052 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1053 : }
1054 :
1055 12300 : bool operator> (const gcov_type other) const
1056 : {
1057 12300 : gcc_checking_assert (ipa_p ());
1058 12300 : gcc_checking_assert (other >= 0);
1059 12300 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1060 : }
1061 :
1062 1234717 : bool operator<= (const profile_count &other) const
1063 : {
1064 1234717 : if (!initialized_p () || !other.initialized_p ())
1065 : return false;
1066 1234717 : if (*this == zero ())
1067 5038 : return true;
1068 1229679 : if (other == zero ())
1069 0 : return (*this == zero ());
1070 1229679 : gcc_checking_assert (compatible_p (other));
1071 1229679 : return m_val <= other.m_val;
1072 : }
1073 :
1074 1435912 : bool operator>= (const profile_count &other) const
1075 : {
1076 1435912 : if (!initialized_p () || !other.initialized_p ())
1077 : return false;
1078 1434695 : if (other == zero ())
1079 3044 : return true;
1080 1431651 : if (*this == zero ())
1081 24326 : return (other == zero ());
1082 1419488 : gcc_checking_assert (compatible_p (other));
1083 1419488 : return m_val >= other.m_val;
1084 : }
1085 :
1086 97705 : bool operator<= (const gcov_type other) const
1087 : {
1088 97705 : gcc_checking_assert (ipa_p ());
1089 97705 : gcc_checking_assert (other >= 0);
1090 97705 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1091 : }
1092 :
1093 63362 : bool operator>= (const gcov_type other) const
1094 : {
1095 63362 : gcc_checking_assert (ipa_p ());
1096 63362 : gcc_checking_assert (other >= 0);
1097 63362 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1098 : }
1099 :
1100 908827412 : profile_count operator* (int64_t num) const
1101 : {
1102 908749387 : return apply_scale (num, 1);
1103 : }
1104 :
1105 : profile_count operator*= (int64_t num)
1106 : {
1107 : *this = apply_scale (num, 1);
1108 : return *this;
1109 : }
1110 :
1111 : profile_count operator* (const sreal &num) const;
1112 : profile_count operator*= (const sreal &num);
1113 :
1114 2420828 : profile_count operator/ (int64_t den) const
1115 : {
1116 2420828 : return apply_scale (1, den);
1117 : }
1118 :
1119 4 : profile_count operator/= (int64_t den)
1120 : {
1121 0 : *this = apply_scale (1, den);
1122 4 : return *this;
1123 : }
1124 :
1125 : /* Return true when value is not zero and can be used for scaling.
1126 : This is different from *this > 0 because that requires counter to
1127 : be IPA. */
1128 8300615083 : bool nonzero_p () const
1129 : {
1130 8300614633 : return initialized_p () && m_val != 0;
1131 : }
1132 :
1133 : /* Make counter forcibly nonzero. */
1134 : profile_count force_nonzero () const;
1135 :
1136 :
1137 : /* Return maximum of A and B. If one of values is uninitialized return the
1138 : other. */
1139 :
1140 : static profile_count
1141 83837629 : max_prefer_initialized (const profile_count a, const profile_count b)
1142 : {
1143 83837629 : if (!a.initialized_p ())
1144 21044909 : return b;
1145 62792720 : if (!b.initialized_p ())
1146 2296818 : return a;
1147 60495902 : profile_count ret;
1148 60495902 : gcc_checking_assert (a.compatible_p (b));
1149 60495902 : ret.m_val = MAX (a.m_val, b.m_val);
1150 60495902 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1151 60495902 : return ret;
1152 : }
1153 :
1154 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1155 : accordingly. */
1156 : profile_count apply_probability (int prob) const
1157 : {
1158 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1159 : if (m_val == 0)
1160 : return *this;
1161 : if (!initialized_p ())
1162 : return uninitialized ();
1163 : profile_count ret;
1164 : uint64_t tmp;
1165 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1166 : ret.m_val = tmp;
1167 : ret.m_quality = MIN (m_quality, ADJUSTED);
1168 : return ret;
1169 : }
1170 :
1171 : /* Scale counter according to PROB. */
1172 402896099 : profile_count apply_probability (profile_probability prob) const
1173 : {
1174 415082529 : if (*this == zero () || prob == profile_probability::always ())
1175 130630534 : return *this;
1176 272265565 : if (prob == profile_probability::never ())
1177 20897403 : return zero ();
1178 251368162 : if (!initialized_p () || !prob.initialized_p ())
1179 31442012 : return uninitialized ();
1180 219926150 : profile_count ret;
1181 219926150 : uint64_t tmp;
1182 219926150 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1183 : &tmp);
1184 219926150 : ret.m_val = tmp;
1185 219948240 : ret.m_quality = MIN (m_quality, prob.quality ());
1186 219926150 : return ret;
1187 : }
1188 :
1189 : /* Return *THIS * NUM / DEN. */
1190 1036418649 : profile_count apply_scale (int64_t num, int64_t den) const
1191 : {
1192 1036418649 : if (m_val == 0)
1193 24040246 : return *this;
1194 1012378403 : if (!initialized_p ())
1195 4295106 : return uninitialized ();
1196 1008083297 : profile_count ret;
1197 1008083297 : uint64_t tmp;
1198 :
1199 1008083297 : gcc_checking_assert (num >= 0 && den > 0);
1200 1008083297 : safe_scale_64bit (m_val, num, den, &tmp);
1201 1008083297 : ret.m_val = MIN (tmp, max_count);
1202 1008083297 : ret.m_quality = MIN (m_quality, ADJUSTED);
1203 1008083297 : return ret;
1204 : }
1205 :
1206 31310002 : profile_count apply_scale (profile_count num, profile_count den) const
1207 : {
1208 31310002 : if (*this == zero ())
1209 2831176 : return *this;
1210 28478826 : if (num == zero ())
1211 95125 : return num;
1212 28383701 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1213 7959295 : return uninitialized ();
1214 20424406 : if (num == den)
1215 5967915 : return *this;
1216 14456491 : gcc_checking_assert (den.m_val);
1217 :
1218 14456491 : profile_count ret;
1219 14456491 : uint64_t val;
1220 14456491 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1221 14456491 : ret.m_val = MIN (val, max_count);
1222 14456491 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1223 : num.m_quality), den.m_quality);
1224 : /* Be sure that ret is not local if num is global.
1225 : Also ensure that ret is not global0 when num is global. */
1226 14456491 : if (num.ipa_p ())
1227 : {
1228 : /* This is common case of AFDO scaling when we upgrade
1229 : GLOBAL0_AFDO function to AFDO. Be sure that result
1230 : is AFDO and not GUESSED (which is unnecessarily low). */
1231 2409 : if (num.m_quality == AFDO
1232 0 : && (ret.m_quality != GUESSED
1233 0 : && ret.m_quality != GUESSED_LOCAL))
1234 0 : ret.m_quality = AFDO;
1235 : else
1236 4410 : ret.m_quality = MAX (ret.m_quality,
1237 : num == num.ipa () ? GUESSED : num.m_quality);
1238 : }
1239 14456491 : return ret;
1240 : }
1241 :
1242 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1243 18610876 : profile_count guessed_local () const
1244 : {
1245 18610876 : profile_count ret = *this;
1246 18610876 : if (!initialized_p ())
1247 0 : return *this;
1248 18610876 : ret.m_quality = GUESSED_LOCAL;
1249 18610876 : return ret;
1250 : }
1251 :
1252 : /* We know that profile is globally 0 but keep local profile if present. */
1253 827 : profile_count global0 () const
1254 : {
1255 827 : profile_count ret = *this;
1256 827 : if (!initialized_p ())
1257 0 : return *this;
1258 827 : ret.m_quality = GUESSED_GLOBAL0;
1259 827 : return ret;
1260 : }
1261 :
1262 : /* We know that profile is globally afdo 0 but keep local profile
1263 : if present. */
1264 0 : profile_count global0afdo () const
1265 : {
1266 0 : profile_count ret = *this;
1267 0 : if (!initialized_p ())
1268 0 : return *this;
1269 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1270 0 : return ret;
1271 : }
1272 :
1273 : /* We know that profile is globally adjusted 0 but keep local profile
1274 : if present. */
1275 63 : profile_count global0adjusted () const
1276 : {
1277 63 : profile_count ret = *this;
1278 63 : if (!initialized_p ())
1279 0 : return *this;
1280 63 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1281 63 : return ret;
1282 : }
1283 :
1284 : /* Return THIS with quality dropped to GUESSED. */
1285 397 : profile_count guessed () const
1286 : {
1287 397 : profile_count ret = *this;
1288 397 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1289 397 : return ret;
1290 : }
1291 :
1292 : /* Return THIS with quality GUESSED. */
1293 0 : profile_count force_guessed () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : gcc_checking_assert (initialized_p ());
1297 0 : ret.m_quality = GUESSED;
1298 0 : return ret;
1299 : }
1300 :
1301 : /* Return variant of profile count which is always safe to compare
1302 : across functions. */
1303 9234062105 : profile_count ipa () const
1304 : {
1305 9234062105 : if (m_quality > GUESSED_GLOBAL0)
1306 14809706 : return *this;
1307 9219252399 : if (m_quality == GUESSED_GLOBAL0)
1308 238974 : return zero ();
1309 9219013425 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1310 23313 : return adjusted_zero ();
1311 9218990112 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1312 0 : return afdo_zero ();
1313 9218990112 : return uninitialized ();
1314 : }
1315 :
1316 : /* Return THIS with quality dropped to AFDO. */
1317 0 : profile_count afdo () const
1318 : {
1319 0 : profile_count ret = *this;
1320 0 : ret.m_quality = AFDO;
1321 0 : return ret;
1322 : }
1323 :
1324 : /* Return probability of event with counter THIS within event with counter
1325 : OVERALL. */
1326 1111868238 : profile_probability probability_in (const profile_count overall) const
1327 : {
1328 1111886397 : if (*this == zero ()
1329 18159 : && !(overall == zero ()))
1330 2378 : return profile_probability::never ();
1331 1111865316 : if (!initialized_p () || !overall.initialized_p ()
1332 2223731176 : || !overall.m_val)
1333 52491 : return profile_probability::uninitialized ();
1334 1326947547 : if (*this == overall && m_quality == PRECISE)
1335 24464 : return profile_probability::always ();
1336 1111788905 : profile_probability ret;
1337 1111788905 : gcc_checking_assert (compatible_p (overall));
1338 :
1339 1111788905 : if (overall.m_val < m_val)
1340 : {
1341 154655 : ret.m_val = profile_probability::max_probability;
1342 154655 : ret.set_quality (GUESSED);
1343 154655 : return ret;
1344 : }
1345 : else
1346 : {
1347 1111634250 : gcc_checking_assert (overall.m_val);
1348 1111634250 : uint64_t tmp;
1349 1111634250 : safe_scale_64bit (m_val, profile_probability::max_probability,
1350 : overall.m_val, &tmp);
1351 1111634250 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1352 1111634250 : ret.m_val = tmp;
1353 : }
1354 1111634250 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1355 : GUESSED), ADJUSTED));
1356 1111634250 : return ret;
1357 : }
1358 :
1359 : /* Return true if profile count is very large, so we risk overflows
1360 : with loop transformations. */
1361 : bool
1362 1064548 : very_large_p ()
1363 : {
1364 1064548 : if (!initialized_p ())
1365 : return false;
1366 1064548 : return m_val > max_count / 65536;
1367 : }
1368 :
1369 : int to_frequency (struct function *fun) const;
1370 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1371 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1372 :
1373 : /* Output THIS to F. */
1374 : void dump (FILE *f, struct function *fun = NULL) const;
1375 :
1376 : /* Print THIS to stderr. */
1377 : void debug () const;
1378 :
1379 : /* Return true if THIS is known to differ significantly from OTHER. */
1380 : bool differs_from_p (profile_count other) const;
1381 :
1382 : /* We want to scale profile across function boundary from NUM to DEN.
1383 : Take care of the side case when NUM and DEN are zeros of incompatible
1384 : kinds. */
1385 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1386 :
1387 : /* THIS is a count of bb which is known to be executed IPA times.
1388 : Combine this information into bb counter. This means returning IPA
1389 : if it is nonzero, not changing anything if IPA is uninitialized
1390 : and if IPA is zero, turning THIS into corresponding local profile with
1391 : global0. */
1392 : profile_count combine_with_ipa_count (profile_count ipa);
1393 :
1394 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1395 : profile_count combine_with_ipa_count_within
1396 : (profile_count ipa, profile_count ipa2);
1397 :
1398 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1399 : Conversions back and forth are used to read the coverage and get it
1400 : into internal representation. */
1401 : static profile_count from_gcov_type (gcov_type v,
1402 : profile_quality quality = PRECISE);
1403 :
1404 : /* LTO streaming support. */
1405 : static profile_count stream_in (class lto_input_block *);
1406 : void stream_out (struct output_block *);
1407 : void stream_out (struct lto_output_stream *);
1408 : };
1409 : #endif
|
