Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2346758706 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2346758706 : uint64_t tmp;
88 2346758706 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2346758706 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2344304072 : *res = tmp / c;
92 2344304072 : return true;
93 : }
94 2454634 : if (c == 1)
95 : {
96 39413 : *res = (uint64_t) -1;
97 39413 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2415221 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greater then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5523635595 : set_quality (profile_quality quality)
169 : {
170 1155396026 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1155396026 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1153637880 : }
174 :
175 : public:
176 4370050248 : profile_probability (): m_val (uninitialized_probability)
177 2392109354 : { set_quality (GUESSED); }
178 :
179 441 : profile_probability (uint32_t val, profile_quality quality):
180 441 : m_val (val)
181 441 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 770929605 : static profile_probability never ()
185 : {
186 770929605 : profile_probability ret;
187 770929605 : ret.m_val = 0;
188 770929605 : ret.set_quality (PRECISE);
189 461113421 : return ret;
190 : }
191 :
192 5339669 : static profile_probability guessed_never ()
193 : {
194 5339669 : profile_probability ret;
195 5339669 : ret.m_val = 0;
196 5339669 : ret.set_quality (GUESSED);
197 5339651 : return ret;
198 : }
199 :
200 5377322 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10754644 : profile_probability r = guessed_always () / 2000;
204 5377322 : r.m_val--;
205 5377322 : return r;
206 : }
207 :
208 89423 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 178846 : profile_probability r = guessed_always () / 5;
212 89423 : r.m_val--;
213 89423 : return r;
214 : }
215 :
216 738801 : static profile_probability even ()
217 : {
218 1477602 : return guessed_always () / 2;
219 : }
220 :
221 349890 : static profile_probability very_likely ()
222 : {
223 349890 : return always () - very_unlikely ();
224 : }
225 :
226 33998 : static profile_probability likely ()
227 : {
228 33998 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 5912 : bool nonzero_p () const
232 : {
233 5912 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6516779 : static profile_probability guessed_always ()
237 : {
238 6516779 : profile_probability ret;
239 6516779 : ret.m_val = max_probability;
240 6516779 : ret.set_quality (GUESSED);
241 6294577 : return ret;
242 : }
243 :
244 567800509 : static profile_probability always ()
245 : {
246 567036603 : profile_probability ret;
247 567800509 : ret.m_val = max_probability;
248 567800509 : ret.set_quality (PRECISE);
249 386646250 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 948429037 : static profile_probability uninitialized ()
255 : {
256 948429037 : profile_probability c;
257 948429037 : c.m_val = uninitialized_probability;
258 948429037 : c.set_quality (GUESSED);
259 934750348 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5766945415 : bool initialized_p () const
264 : {
265 4023083889 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1597996 : bool reliable_p () const
270 : {
271 132078315 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3216601 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3216601 : profile_probability ret;
279 3216601 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3216601 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3216601 : ret.set_quality (GUESSED);
282 3216601 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 43 : profile_probability adjusted () const
287 : {
288 43 : profile_probability ret = *this;
289 43 : if (!initialized_p ())
290 0 : return *this;
291 43 : ret.set_quality (ADJUSTED);
292 43 : return ret;
293 : }
294 :
295 1135931462 : int to_reg_br_prob_base () const
296 : {
297 1135931462 : gcc_checking_assert (initialized_p ());
298 1135931462 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 500596478 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 500596478 : profile_probability ret;
305 500596478 : ret.m_val = ((unsigned int)v) / 8;
306 500596478 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 16095646 : return ret;
308 : }
309 :
310 471071009 : int to_reg_br_prob_note () const
311 : {
312 471071009 : gcc_checking_assert (initialized_p ());
313 471071009 : int ret = m_val * 8 + m_adjusted_quality;
314 471071009 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 471071009 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3225 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3225 : profile_probability ret;
323 3225 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3225 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3225 : uint64_t tmp;
329 3225 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3225 : gcc_checking_assert (tmp <= max_probability);
331 3225 : ret.m_val = tmp;
332 : }
333 3225 : ret.set_quality (PRECISE);
334 3225 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1674260054 : bool operator== (const profile_probability &other) const
339 : {
340 2358412632 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6743447 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6743447 : if (other == never ())
346 35290 : return *this;
347 6708157 : if (*this == never ())
348 23942 : return other;
349 6684215 : if (!initialized_p () || !other.initialized_p ())
350 5491363 : return uninitialized ();
351 :
352 1192852 : profile_probability ret;
353 1192852 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1192852 : ret.set_quality (MIN (quality (), other.quality ()));
355 1192852 : return ret;
356 : }
357 :
358 6621803 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6621803 : if (other == never ())
361 : return *this;
362 5312496 : if (*this == never ())
363 : {
364 916085 : *this = other;
365 916085 : return *this;
366 : }
367 4396411 : if (!initialized_p () || !other.initialized_p ())
368 2977995 : return *this = uninitialized ();
369 : else
370 : {
371 1418416 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 8501247 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1418416 : return *this;
375 : }
376 :
377 34848161 : profile_probability operator- (const profile_probability &other) const
378 : {
379 69696322 : if (*this == never ()
380 34848161 : || other == never ())
381 1923854 : return *this;
382 32924307 : if (!initialized_p () || !other.initialized_p ())
383 2199666 : return uninitialized ();
384 30724641 : profile_probability ret;
385 30724641 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30724641 : ret.set_quality (MIN (quality (), other.quality ()));
387 30724641 : return ret;
388 : }
389 :
390 11772270 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 23544540 : if (*this == never ()
393 11772270 : || other == never ())
394 : return *this;
395 9389359 : if (!initialized_p () || !other.initialized_p ())
396 2144265 : return *this = uninitialized ();
397 : else
398 : {
399 7245094 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 43470421 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 7245094 : return *this;
403 : }
404 :
405 1259226 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2518452 : if (*this == never ()
408 1259226 : || other == never ())
409 125587 : return never ();
410 1133639 : if (!initialized_p () || !other.initialized_p ())
411 58650 : return uninitialized ();
412 1074989 : profile_probability ret;
413 1074989 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 1074989 : ret.set_quality (MIN (quality (), other.quality ()));
415 1074989 : return ret;
416 : }
417 :
418 128351 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 256702 : if (*this == never ()
421 128351 : || other == never ())
422 460 : return *this = never ();
423 127891 : if (!initialized_p () || !other.initialized_p ())
424 30 : return *this = uninitialized ();
425 : else
426 : {
427 127861 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 127861 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 127861 : return *this;
431 : }
432 :
433 1790260 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1790260 : if (*this == never ())
436 102755 : return never ();
437 1687505 : if (!initialized_p () || !other.initialized_p ())
438 229013 : return uninitialized ();
439 1458492 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1458492 : if (m_val >= other.m_val)
442 : {
443 51605 : ret.m_val = max_probability;
444 51605 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 51605 : return ret;
447 : }
448 1406887 : else if (!m_val)
449 65643 : ret.m_val = 0;
450 : else
451 : {
452 1341244 : gcc_checking_assert (other.m_val);
453 1341244 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2802785 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1406887 : return ret;
457 : }
458 :
459 296344 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 296344 : if (*this == never ())
462 2192 : return *this = never ();
463 294152 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 294152 : if (m_val > other.m_val)
470 : {
471 497 : m_val = max_probability;
472 497 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 497 : return *this;
475 : }
476 293655 : else if (!m_val)
477 : ;
478 : else
479 : {
480 291498 : gcc_checking_assert (other.m_val);
481 291498 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 293655 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 293655 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 302728 : profile_probability split (const profile_probability &cprob)
504 : {
505 302728 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 302728 : if (!(*this == always ()))
512 293255 : *this = (*this - ret) / ret.invert ();
513 302728 : return ret;
514 : }
515 :
516 : /* Return VAL scaled by this probability. Round initialized probabilities
517 : to the nearest integer, with halfway values away from zero. Treat an
518 : uninitialized probability as one half and truncate toward zero. */
519 276803 : gcov_type apply (gcov_type val) const
520 : {
521 276803 : if (*this == uninitialized ())
522 78 : return val / 2;
523 :
524 : /* A unit probability leaves VAL unchanged. Return it directly because
525 : the magnitude of the minimum gcov_type value is one greater than the
526 : maximum gcov_type value. */
527 276725 : if (m_val == max_probability)
528 : return val;
529 :
530 : /* Convert to unsigned before negating so that the minimum gcov_type
531 : value has a representable magnitude. Scale the magnitude with
532 : overflow-safe arithmetic, then restore the sign. */
533 273253 : gcov_type_unsigned magnitude
534 273269 : = val < 0 ? -(gcov_type_unsigned) val : (gcov_type_unsigned) val;
535 273269 : uint64_t scaled;
536 273269 : bool scaled_p
537 273269 : = safe_scale_64bit (magnitude, m_val, max_probability, &scaled);
538 : /* The scaled result fits in uint64_t. With the unit case handled above,
539 : it also fits in the nonnegative range of gcov_type. */
540 273269 : gcc_checking_assert (scaled_p);
541 273269 : gcc_checking_assert
542 : (scaled <= (gcov_type_unsigned) INTTYPE_MAXIMUM (gcov_type));
543 :
544 273269 : return val < 0 ? -(gcov_type) scaled : (gcov_type) scaled;
545 : }
546 :
547 : /* Return 1-*THIS. */
548 21701674 : profile_probability invert () const
549 : {
550 31596908 : return always() - *this;
551 : }
552 :
553 : /* Return THIS with quality dropped to GUESSED. */
554 882788 : profile_probability guessed () const
555 : {
556 882788 : profile_probability ret = *this;
557 882788 : ret.set_quality (GUESSED);
558 882752 : return ret;
559 : }
560 :
561 : /* Return THIS with quality dropped to AFDO. */
562 : profile_probability afdo () const
563 : {
564 : profile_probability ret = *this;
565 : ret.set_quality (AFDO);
566 : return ret;
567 : }
568 :
569 : /* Return *THIS * NUM / DEN. */
570 14767626 : profile_probability apply_scale (int64_t num, int64_t den) const
571 : {
572 14767626 : if (*this == never ())
573 16728 : return *this;
574 14750898 : if (!initialized_p ())
575 5908512 : return uninitialized ();
576 8842386 : profile_probability ret;
577 8842386 : uint64_t tmp;
578 8842386 : safe_scale_64bit (m_val, num, den, &tmp);
579 8842386 : ret.m_val = MIN (tmp, max_probability);
580 8842386 : ret.set_quality (MIN (quality (), ADJUSTED));
581 8842386 : return ret;
582 : }
583 :
584 : /* Return *THIS * NUM / DEN. */
585 14406 : profile_probability apply_scale (profile_probability num,
586 : profile_probability den) const
587 : {
588 14406 : if (*this == never ())
589 306 : return *this;
590 14100 : if (num == never ())
591 0 : return num;
592 14100 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
593 0 : return uninitialized ();
594 14100 : if (num == den)
595 0 : return *this;
596 14100 : gcc_checking_assert (den.m_val);
597 :
598 14100 : profile_probability ret;
599 14100 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
600 : max_probability);
601 14100 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
602 : num.quality ()), den.quality ()));
603 14100 : return ret;
604 : }
605 :
606 : /* Return true when the probability of edge is reliable.
607 :
608 : The profile guessing code is good at predicting branch outcome (i.e.
609 : taken/not taken), that is predicted right slightly over 75% of time.
610 : It is however notoriously poor on predicting the probability itself.
611 : In general the profile appear a lot flatter (with probabilities closer
612 : to 50%) than the reality so it is bad idea to use it to drive optimization
613 : such as those disabling dynamic branch prediction for well predictable
614 : branches.
615 :
616 : There are two exceptions - edges leading to noreturn edges and edges
617 : predicted by number of iterations heuristics are predicted well. This macro
618 : should be able to distinguish those, but at the moment it simply check for
619 : noreturn heuristic that is only one giving probability over 99% or below
620 : 1%. In future we might want to propagate reliability information across the
621 : CFG if we find this information useful on multiple places. */
622 0 : bool probably_reliable_p () const
623 : {
624 0 : if (quality () >= ADJUSTED)
625 : return true;
626 0 : if (!initialized_p ())
627 : return false;
628 0 : return m_val < max_probability / 100
629 0 : || m_val > max_probability - max_probability / 100;
630 : }
631 :
632 : /* Return false if profile_probability is bogus. */
633 4440453550 : bool verify () const
634 : {
635 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
636 4440453550 : if (m_val == uninitialized_probability)
637 963926980 : return quality () == GUESSED;
638 3476526570 : else if (quality () < GUESSED)
639 : return false;
640 3476526570 : return m_val <= max_probability;
641 : }
642 :
643 : /* Comparisons are three-state and conservative. False is returned if
644 : the inequality cannot be decided. */
645 9071309 : bool operator< (const profile_probability &other) const
646 : {
647 9071309 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
648 : }
649 :
650 17879350 : bool operator> (const profile_probability &other) const
651 : {
652 17879350 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
653 : }
654 :
655 5544542 : bool operator<= (const profile_probability &other) const
656 : {
657 5544542 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
658 : }
659 :
660 972031 : bool operator>= (const profile_probability &other) const
661 : {
662 972031 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
663 : }
664 :
665 17 : profile_probability operator* (int64_t num) const
666 : {
667 17 : return apply_scale (num, 1);
668 : }
669 :
670 : profile_probability operator*= (int64_t num)
671 : {
672 : *this = apply_scale (num, 1);
673 : return *this;
674 : }
675 :
676 14091695 : profile_probability operator/ (int64_t den) const
677 : {
678 14091695 : return apply_scale (1, den);
679 : }
680 :
681 247902 : profile_probability operator/= (int64_t den)
682 : {
683 247902 : *this = apply_scale (1, den);
684 247902 : return *this;
685 : }
686 :
687 : /* Compute n-th power. */
688 : profile_probability pow (int) const;
689 :
690 : /* Compute sware root. */
691 : profile_probability sqrt () const;
692 :
693 : /* Get the value of the probability. */
694 0 : uint32_t value () const { return m_val; }
695 :
696 : /* Get the quality of the probability. */
697 6180976811 : enum profile_quality quality () const
698 : {
699 5961514177 : return (profile_quality) (m_adjusted_quality
700 5961536425 : ? m_adjusted_quality + min_quality - 1
701 51396683 : : UNINITIALIZED_PROFILE);
702 : }
703 :
704 : /* Output THIS to F. */
705 : void dump (FILE *f) const;
706 :
707 : /* Output THIS to BUFFER. */
708 : void dump (char *buffer) const;
709 :
710 : /* Print THIS to stderr. */
711 : void debug () const;
712 :
713 : /* Return true if THIS is known to differ significantly from OTHER. */
714 : bool differs_from_p (profile_probability other) const;
715 :
716 : /* Return if difference is greater than 50%. */
717 : bool differs_lot_from_p (profile_probability other) const;
718 :
719 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
720 : happens with COUNT2 probability. Return probability that either *THIS or
721 : OTHER happens. */
722 : profile_probability combine_with_count (profile_count count1,
723 : profile_probability other,
724 : profile_count count2) const;
725 :
726 : /* Return probability as sreal. */
727 : sreal to_sreal () const;
728 : /* LTO streaming support. */
729 : static profile_probability stream_in (class lto_input_block *);
730 : void stream_out (struct output_block *);
731 : void stream_out (struct lto_output_stream *);
732 : };
733 :
734 : /* Main data type to hold profile counters in GCC. Profile counts originate
735 : either from profile feedback, static profile estimation or both. We do not
736 : perform whole program profile propagation and thus profile estimation
737 : counters are often local to function, while counters from profile feedback
738 : (or special cases of profile estimation) can be used inter-procedurally.
739 :
740 : There are 3 basic types
741 : 1) local counters which are result of intra-procedural static profile
742 : estimation.
743 : 2) ipa counters which are result of profile feedback or special case
744 : of static profile estimation (such as in function main).
745 : 3) counters which counts as 0 inter-procedurally (because given function
746 : was never run in train feedback) but they hold local static profile
747 : estimate.
748 :
749 : Counters of type 1 and 3 cannot be mixed with counters of different type
750 : within operation (because whole function should use one type of counter)
751 : with exception that global zero mix in most operations where outcome is
752 : well defined.
753 :
754 : To take local counter and use it inter-procedurally use ipa member function
755 : which strips information irrelevant at the inter-procedural level.
756 :
757 : Counters are 61bit integers representing number of executions during the
758 : train run or normalized frequency within the function.
759 :
760 : As the profile is maintained during the compilation, many adjustments are
761 : made. Not all transformations can be made precisely, most importantly
762 : when code is being duplicated. It also may happen that part of CFG has
763 : profile counts known while other do not - for example when LTO optimizing
764 : partly profiled program or when profile was lost due to COMDAT merging.
765 :
766 : For this reason profile_count tracks more information than
767 : just unsigned integer and it is also ready for profile mismatches.
768 : The API of this data type represent operations that are natural
769 : on profile counts - sum, difference and operation with scales and
770 : probabilities. All operations are safe by never getting negative counts
771 : and they do end up in uninitialized scale if any of the parameters is
772 : uninitialized.
773 :
774 : All comparisons that are three state and handling of probabilities. Thus
775 : a < b is not equal to !(a >= b).
776 :
777 : The following pre-defined counts are available:
778 :
779 : profile_count::zero () for code that is known to execute zero times at
780 : runtime (this can be detected statically i.e. for paths leading to
781 : abort ();
782 : profile_count::one () for code that is known to execute once (such as
783 : main () function
784 : profile_count::uninitialized () for unknown execution count.
785 :
786 : */
787 :
788 : struct GTY(()) profile_count
789 : {
790 : public:
791 : /* Use 60bit to hold basic block counters. Should be at least
792 : 64bit. Although a counter cannot be negative, we use a signed
793 : type to hold various extra stages. */
794 :
795 : static const int n_bits = 60;
796 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
797 : private:
798 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
799 :
800 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
801 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
802 : incorrectly detects the alignment of a structure where the only
803 : 64-bit aligned object is a bit-field. We force the alignment of
804 : the entire field to mitigate this. */
805 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
806 : #else
807 : #define UINT64_BIT_FIELD_ALIGN
808 : #endif
809 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
810 : #undef UINT64_BIT_FIELD_ALIGN
811 : enum profile_quality m_quality : 4;
812 : public:
813 :
814 : /* Return true if both values can meaningfully appear in single function
815 : body. We have either all counters in function local or global, otherwise
816 : operations between them are not really defined well. */
817 4986425141 : bool compatible_p (const profile_count other) const
818 : {
819 9151196660 : if (!initialized_p () || !other.initialized_p ())
820 : return true;
821 16486211308 : if (*this == zero ()
822 157506633 : || other == zero ())
823 : return true;
824 : /* Do not allow nonzero global profile together with local guesses
825 : that are globally0. */
826 4002548896 : if (ipa ().nonzero_p ()
827 258643 : && !(other.ipa () == other))
828 0 : return false;
829 4002548896 : if (other.ipa ().nonzero_p ()
830 268370 : && !(ipa () == *this))
831 0 : return false;
832 :
833 4002548896 : return ipa_p () == other.ipa_p ();
834 : }
835 :
836 : /* Used for counters which are expected to be never executed. */
837 18578434814 : static profile_count zero ()
838 : {
839 14535676432 : return from_gcov_type (0);
840 : }
841 :
842 : static profile_count adjusted_zero ()
843 : {
844 : profile_count c;
845 : c.m_val = 0;
846 : c.m_quality = ADJUSTED;
847 : return c;
848 : }
849 :
850 : static profile_count afdo_zero ()
851 : {
852 : profile_count c;
853 : c.m_val = 0;
854 : c.m_quality = AFDO;
855 : return c;
856 : }
857 :
858 : static profile_count guessed_zero ()
859 : {
860 : profile_count c;
861 : c.m_val = 0;
862 : c.m_quality = GUESSED;
863 : return c;
864 : }
865 :
866 53226 : static profile_count one ()
867 : {
868 53028 : return from_gcov_type (1);
869 : }
870 :
871 : /* Value of counters which has not been initialized. Either because
872 : initialization did not happen yet or because profile is unknown. */
873 112897143 : static profile_count uninitialized ()
874 : {
875 112897143 : profile_count c;
876 112897143 : c.m_val = uninitialized_count;
877 112897143 : c.m_quality = GUESSED_LOCAL;
878 112897143 : return c;
879 : }
880 :
881 : /* Conversion to gcov_type is lossy. */
882 42658685 : gcov_type to_gcov_type () const
883 : {
884 0 : gcc_checking_assert (initialized_p ());
885 42658685 : return m_val;
886 : }
887 :
888 : /* Return true if value has been initialized. */
889 32869608417 : bool initialized_p () const
890 : {
891 11052373049 : return m_val != uninitialized_count;
892 : }
893 :
894 : /* Return true if value can be trusted. */
895 27247029 : bool reliable_p () const
896 : {
897 27246747 : return m_quality >= ADJUSTED;
898 : }
899 :
900 : /* Return true if value can be operated inter-procedurally. */
901 9043575467 : bool ipa_p () const
902 : {
903 5027516613 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
904 : }
905 :
906 : /* Return true if quality of profile is precise. */
907 59452374 : bool precise_p () const
908 : {
909 59452374 : return m_quality == PRECISE;
910 : }
911 :
912 : /* Get the value of the count. */
913 0 : uint64_t value () const { return m_val; }
914 :
915 : /* Get the quality of the count. */
916 98321829 : enum profile_quality quality () const { return m_quality; }
917 :
918 : /* When merging basic blocks, the two different profile counts are unified.
919 : Return true if this can be done without losing info about profile.
920 : The only case we care about here is when first BB contains something
921 : that makes it terminate in a way not visible in CFG. */
922 3379910 : bool ok_for_merging (profile_count other) const
923 : {
924 3379910 : if (m_quality < ADJUSTED
925 17126 : || other.m_quality < ADJUSTED)
926 : return true;
927 15909 : return !(other < *this);
928 : }
929 :
930 : /* When merging two BBs with different counts, pick common count that looks
931 : most representative. */
932 17789462 : profile_count merge (profile_count other) const
933 : {
934 907417 : if (*this == other || !other.initialized_p ()
935 907121 : || m_quality > other.m_quality)
936 16883810 : return *this;
937 905652 : if (other.m_quality > m_quality
938 905652 : || other > *this)
939 0 : return other;
940 905652 : return *this;
941 : }
942 :
943 : /* Basic operations. */
944 20058755880 : bool operator== (const profile_count &other) const
945 : {
946 15265366141 : return m_val == other.m_val && m_quality == other.m_quality;
947 : }
948 :
949 3941533 : profile_count operator+ (const profile_count &other) const
950 : {
951 3941533 : if (other == zero ())
952 2581 : return *this;
953 3938952 : if (*this == zero ())
954 241571 : return other;
955 3697381 : if (!initialized_p () || !other.initialized_p ())
956 6177 : return uninitialized ();
957 :
958 3691204 : profile_count ret;
959 3691204 : gcc_checking_assert (compatible_p (other));
960 3691204 : uint64_t ret_val = m_val + other.m_val;
961 3691204 : ret.m_val = MIN (ret_val, max_count);
962 3691204 : ret.m_quality = MIN (m_quality, other.m_quality);
963 3691204 : return ret;
964 : }
965 :
966 74924745 : profile_count &operator+= (const profile_count &other)
967 : {
968 74924745 : if (other == zero ())
969 5551152 : return *this;
970 69373593 : if (*this == zero ())
971 : {
972 42603775 : *this = other;
973 42603775 : return *this;
974 : }
975 26769818 : if (!initialized_p () || !other.initialized_p ())
976 365455 : return *this = uninitialized ();
977 : else
978 : {
979 26404363 : gcc_checking_assert (compatible_p (other));
980 26404363 : uint64_t ret_val = m_val + other.m_val;
981 26404363 : m_val = MIN (ret_val, max_count);
982 26404363 : m_quality = MIN (m_quality, other.m_quality);
983 : }
984 26404363 : return *this;
985 : }
986 :
987 20992361 : profile_count operator- (const profile_count &other) const
988 : {
989 22479348 : if (*this == zero () || other == zero ())
990 1472922 : return *this;
991 19519439 : if (!initialized_p () || !other.initialized_p ())
992 8811562 : return uninitialized ();
993 10707877 : gcc_checking_assert (compatible_p (other));
994 10707877 : profile_count ret;
995 10707877 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
996 10707877 : ret.m_quality = MIN (m_quality, other.m_quality);
997 10707877 : return ret;
998 : }
999 :
1000 9295111 : profile_count &operator-= (const profile_count &other)
1001 : {
1002 10053239 : if (*this == zero () || other == zero ())
1003 : return *this;
1004 8997106 : if (!initialized_p () || !other.initialized_p ())
1005 392463 : return *this = uninitialized ();
1006 : else
1007 : {
1008 8604643 : gcc_checking_assert (compatible_p (other));
1009 8604643 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
1010 8604643 : m_quality = MIN (m_quality, other.m_quality);
1011 : }
1012 8604643 : return *this;
1013 : }
1014 :
1015 : /* Return false if profile_count is bogus. */
1016 3391754589 : bool verify () const
1017 : {
1018 3391754589 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1019 3391754589 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1020 : }
1021 :
1022 : /* Comparisons are three-state and conservative. False is returned if
1023 : the inequality cannot be decided. */
1024 1087286343 : bool operator< (const profile_count &other) const
1025 : {
1026 1087286343 : if (!initialized_p () || !other.initialized_p ())
1027 : return false;
1028 1082377024 : if (*this == zero ())
1029 4392636 : return !(other == zero ());
1030 1080180706 : if (other == zero ())
1031 921 : return false;
1032 1080179785 : gcc_checking_assert (compatible_p (other));
1033 1080179785 : return m_val < other.m_val;
1034 : }
1035 :
1036 39139165 : bool operator> (const profile_count &other) const
1037 : {
1038 39139165 : if (!initialized_p () || !other.initialized_p ())
1039 : return false;
1040 34039742 : if (*this == zero ())
1041 229161 : return false;
1042 33810581 : if (other == zero ())
1043 1562730 : return !(*this == zero ());
1044 33029216 : gcc_checking_assert (compatible_p (other));
1045 33029216 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1046 : }
1047 :
1048 : bool operator< (const gcov_type other) const
1049 : {
1050 : gcc_checking_assert (ipa_p ());
1051 : gcc_checking_assert (other >= 0);
1052 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1053 : }
1054 :
1055 12715 : bool operator> (const gcov_type other) const
1056 : {
1057 12715 : gcc_checking_assert (ipa_p ());
1058 12715 : gcc_checking_assert (other >= 0);
1059 12715 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1060 : }
1061 :
1062 1224663 : bool operator<= (const profile_count &other) const
1063 : {
1064 1224663 : if (!initialized_p () || !other.initialized_p ())
1065 : return false;
1066 1224663 : if (*this == zero ())
1067 5084 : return true;
1068 1219579 : if (other == zero ())
1069 0 : return (*this == zero ());
1070 1219579 : gcc_checking_assert (compatible_p (other));
1071 1219579 : return m_val <= other.m_val;
1072 : }
1073 :
1074 1427214 : bool operator>= (const profile_count &other) const
1075 : {
1076 1427214 : if (!initialized_p () || !other.initialized_p ())
1077 : return false;
1078 1425997 : if (other == zero ())
1079 2972 : return true;
1080 1423025 : if (*this == zero ())
1081 24184 : return (other == zero ());
1082 1410933 : gcc_checking_assert (compatible_p (other));
1083 1410933 : return m_val >= other.m_val;
1084 : }
1085 :
1086 97765 : bool operator<= (const gcov_type other) const
1087 : {
1088 97765 : gcc_checking_assert (ipa_p ());
1089 97765 : gcc_checking_assert (other >= 0);
1090 97765 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1091 : }
1092 :
1093 63364 : bool operator>= (const gcov_type other) const
1094 : {
1095 63364 : gcc_checking_assert (ipa_p ());
1096 63364 : gcc_checking_assert (other >= 0);
1097 63364 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1098 : }
1099 :
1100 902063022 : profile_count operator* (int64_t num) const
1101 : {
1102 901984201 : return apply_scale (num, 1);
1103 : }
1104 :
1105 : profile_count operator*= (int64_t num)
1106 : {
1107 : *this = apply_scale (num, 1);
1108 : return *this;
1109 : }
1110 :
1111 : profile_count operator* (const sreal &num) const;
1112 : profile_count operator*= (const sreal &num);
1113 :
1114 2404462 : profile_count operator/ (int64_t den) const
1115 : {
1116 2404462 : return apply_scale (1, den);
1117 : }
1118 :
1119 4 : profile_count operator/= (int64_t den)
1120 : {
1121 0 : *this = apply_scale (1, den);
1122 4 : return *this;
1123 : }
1124 :
1125 : /* Return true when value is not zero and can be used for scaling.
1126 : This is different from *this > 0 because that requires counter to
1127 : be IPA. */
1128 8202086574 : bool nonzero_p () const
1129 : {
1130 8202086124 : return initialized_p () && m_val != 0;
1131 : }
1132 :
1133 : /* Make counter forcibly nonzero. */
1134 : profile_count force_nonzero () const;
1135 :
1136 :
1137 : /* Return maximum of A and B. If one of values is uninitialized return the
1138 : other. */
1139 :
1140 : static profile_count
1141 82199112 : max_prefer_initialized (const profile_count a, const profile_count b)
1142 : {
1143 82199112 : if (!a.initialized_p ())
1144 20727814 : return b;
1145 61471298 : if (!b.initialized_p ())
1146 2169075 : return a;
1147 59302223 : profile_count ret;
1148 59302223 : gcc_checking_assert (a.compatible_p (b));
1149 59302223 : ret.m_val = MAX (a.m_val, b.m_val);
1150 59302223 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1151 59302223 : return ret;
1152 : }
1153 :
1154 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1155 : accordingly. */
1156 : profile_count apply_probability (int prob) const
1157 : {
1158 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1159 : if (m_val == 0)
1160 : return *this;
1161 : if (!initialized_p ())
1162 : return uninitialized ();
1163 : profile_count ret;
1164 : uint64_t tmp;
1165 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1166 : ret.m_val = tmp;
1167 : ret.m_quality = MIN (m_quality, ADJUSTED);
1168 : return ret;
1169 : }
1170 :
1171 : /* Scale counter according to PROB. */
1172 401275882 : profile_count apply_probability (profile_probability prob) const
1173 : {
1174 413552866 : if (*this == zero () || prob == profile_probability::always ())
1175 129650464 : return *this;
1176 271625418 : if (prob == profile_probability::never ())
1177 20630281 : return zero ();
1178 250995137 : if (!initialized_p () || !prob.initialized_p ())
1179 31569978 : return uninitialized ();
1180 219425159 : profile_count ret;
1181 219425159 : uint64_t tmp;
1182 219425159 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1183 : &tmp);
1184 219425159 : ret.m_val = tmp;
1185 219447407 : ret.m_quality = MIN (m_quality, prob.quality ());
1186 219425159 : return ret;
1187 : }
1188 :
1189 : /* Return *THIS * NUM / DEN. */
1190 1030815689 : profile_count apply_scale (int64_t num, int64_t den) const
1191 : {
1192 1030815689 : if (m_val == 0)
1193 24057047 : return *this;
1194 1006758642 : if (!initialized_p ())
1195 4183283 : return uninitialized ();
1196 1002575359 : profile_count ret;
1197 1002575359 : uint64_t tmp;
1198 :
1199 1002575359 : gcc_checking_assert (num >= 0 && den > 0);
1200 1002575359 : safe_scale_64bit (m_val, num, den, &tmp);
1201 1002575359 : ret.m_val = MIN (tmp, max_count);
1202 1002575359 : ret.m_quality = MIN (m_quality, ADJUSTED);
1203 1002575359 : return ret;
1204 : }
1205 :
1206 30300404 : profile_count apply_scale (profile_count num, profile_count den) const
1207 : {
1208 30300404 : if (*this == zero ())
1209 2678616 : return *this;
1210 27621788 : if (num == zero ())
1211 90525 : return num;
1212 27531263 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1213 7760384 : return uninitialized ();
1214 19770879 : if (num == den)
1215 5883123 : return *this;
1216 13887756 : gcc_checking_assert (den.m_val);
1217 :
1218 13887756 : profile_count ret;
1219 13887756 : uint64_t val;
1220 13887756 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1221 13887756 : ret.m_val = MIN (val, max_count);
1222 13887756 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1223 : num.m_quality), den.m_quality);
1224 : /* Be sure that ret is not local if num is global.
1225 : Also ensure that ret is not global0 when num is global. */
1226 13887756 : if (num.ipa_p ())
1227 : {
1228 : /* This is common case of AFDO scaling when we upgrade
1229 : GLOBAL0_AFDO function to AFDO. Be sure that result
1230 : is AFDO and not GUESSED (which is unnecessarily low). */
1231 2409 : if (num.m_quality == AFDO
1232 0 : && (ret.m_quality != GUESSED
1233 0 : && ret.m_quality != GUESSED_LOCAL))
1234 0 : ret.m_quality = AFDO;
1235 : else
1236 4410 : ret.m_quality = MAX (ret.m_quality,
1237 : num == num.ipa () ? GUESSED : num.m_quality);
1238 : }
1239 13887756 : return ret;
1240 : }
1241 :
1242 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1243 18332108 : profile_count guessed_local () const
1244 : {
1245 18332108 : profile_count ret = *this;
1246 18332108 : if (!initialized_p ())
1247 0 : return *this;
1248 18332108 : ret.m_quality = GUESSED_LOCAL;
1249 18332108 : return ret;
1250 : }
1251 :
1252 : /* We know that profile is globally 0 but keep local profile if present. */
1253 827 : profile_count global0 () const
1254 : {
1255 827 : profile_count ret = *this;
1256 827 : if (!initialized_p ())
1257 0 : return *this;
1258 827 : ret.m_quality = GUESSED_GLOBAL0;
1259 827 : return ret;
1260 : }
1261 :
1262 : /* We know that profile is globally afdo 0 but keep local profile
1263 : if present. */
1264 0 : profile_count global0afdo () const
1265 : {
1266 0 : profile_count ret = *this;
1267 0 : if (!initialized_p ())
1268 0 : return *this;
1269 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1270 0 : return ret;
1271 : }
1272 :
1273 : /* We know that profile is globally adjusted 0 but keep local profile
1274 : if present. */
1275 63 : profile_count global0adjusted () const
1276 : {
1277 63 : profile_count ret = *this;
1278 63 : if (!initialized_p ())
1279 0 : return *this;
1280 63 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1281 63 : return ret;
1282 : }
1283 :
1284 : /* Return THIS with quality dropped to GUESSED. */
1285 397 : profile_count guessed () const
1286 : {
1287 397 : profile_count ret = *this;
1288 397 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1289 397 : return ret;
1290 : }
1291 :
1292 : /* Return THIS with quality GUESSED. */
1293 0 : profile_count force_guessed () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : gcc_checking_assert (initialized_p ());
1297 0 : ret.m_quality = GUESSED;
1298 0 : return ret;
1299 : }
1300 :
1301 : /* Return variant of profile count which is always safe to compare
1302 : across functions. */
1303 9125946616 : profile_count ipa () const
1304 : {
1305 9125946616 : if (m_quality > GUESSED_GLOBAL0)
1306 14307002 : return *this;
1307 9111639614 : if (m_quality == GUESSED_GLOBAL0)
1308 239148 : return zero ();
1309 9111400466 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1310 23313 : return adjusted_zero ();
1311 9111377153 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1312 0 : return afdo_zero ();
1313 9111377153 : return uninitialized ();
1314 : }
1315 :
1316 : /* Return THIS with quality dropped to AFDO. */
1317 0 : profile_count afdo () const
1318 : {
1319 0 : profile_count ret = *this;
1320 0 : ret.m_quality = AFDO;
1321 0 : return ret;
1322 : }
1323 :
1324 : /* Return probability of event with counter THIS within event with counter
1325 : OVERALL. */
1326 1101473605 : profile_probability probability_in (const profile_count overall) const
1327 : {
1328 1101491326 : if (*this == zero ()
1329 17721 : && !(overall == zero ()))
1330 2381 : return profile_probability::never ();
1331 1101470680 : if (!initialized_p () || !overall.initialized_p ()
1332 2202941904 : || !overall.m_val)
1333 49276 : return profile_probability::uninitialized ();
1334 1316234500 : if (*this == overall && m_quality == PRECISE)
1335 24442 : return profile_probability::always ();
1336 1101397506 : profile_probability ret;
1337 1101397506 : gcc_checking_assert (compatible_p (overall));
1338 :
1339 1101397506 : if (overall.m_val < m_val)
1340 : {
1341 155856 : ret.m_val = profile_probability::max_probability;
1342 155856 : ret.set_quality (GUESSED);
1343 155856 : return ret;
1344 : }
1345 : else
1346 : {
1347 1101241650 : gcc_checking_assert (overall.m_val);
1348 1101241650 : uint64_t tmp;
1349 1101241650 : safe_scale_64bit (m_val, profile_probability::max_probability,
1350 : overall.m_val, &tmp);
1351 1101241650 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1352 1101241650 : ret.m_val = tmp;
1353 : }
1354 1101241650 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1355 : GUESSED), ADJUSTED));
1356 1101241650 : return ret;
1357 : }
1358 :
1359 : /* Return true if profile count is very large, so we risk overflows
1360 : with loop transformations. */
1361 : bool
1362 1058957 : very_large_p ()
1363 : {
1364 1058957 : if (!initialized_p ())
1365 : return false;
1366 1058957 : return m_val > max_count / 65536;
1367 : }
1368 :
1369 : int to_frequency (struct function *fun) const;
1370 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1371 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1372 :
1373 : /* Output THIS to F. */
1374 : void dump (FILE *f, struct function *fun = NULL) const;
1375 :
1376 : /* Print THIS to stderr. */
1377 : void debug () const;
1378 :
1379 : /* Return true if THIS is known to differ significantly from OTHER. */
1380 : bool differs_from_p (profile_count other) const;
1381 :
1382 : /* We want to scale profile across function boundary from NUM to DEN.
1383 : Take care of the side case when NUM and DEN are zeros of incompatible
1384 : kinds. */
1385 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1386 :
1387 : /* THIS is a count of bb which is known to be executed IPA times.
1388 : Combine this information into bb counter. This means returning IPA
1389 : if it is nonzero, not changing anything if IPA is uninitialized
1390 : and if IPA is zero, turning THIS into corresponding local profile with
1391 : global0. */
1392 : profile_count combine_with_ipa_count (profile_count ipa);
1393 :
1394 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1395 : profile_count combine_with_ipa_count_within
1396 : (profile_count ipa, profile_count ipa2);
1397 :
1398 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1399 : Conversions back and forth are used to read the coverage and get it
1400 : into internal representation. */
1401 : static profile_count from_gcov_type (gcov_type v,
1402 : profile_quality quality = PRECISE);
1403 :
1404 : /* LTO streaming support. */
1405 : static profile_count stream_in (class lto_input_block *);
1406 : void stream_out (struct output_block *);
1407 : void stream_out (struct lto_output_stream *);
1408 : };
1409 : #endif
|