Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2350981226 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2350981226 : uint64_t tmp;
88 2350981226 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2350981226 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2348577762 : *res = tmp / c;
92 2348577762 : return true;
93 : }
94 2403464 : if (c == 1)
95 : {
96 40212 : *res = (uint64_t) -1;
97 40212 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2363252 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greater then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5522973319 : set_quality (profile_quality quality)
169 : {
170 1156246528 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1156246528 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1154516691 : }
174 :
175 : public:
176 4368509094 : profile_probability (): m_val (uninitialized_probability)
177 2394245562 : { set_quality (GUESSED); }
178 :
179 441 : profile_probability (uint32_t val, profile_quality quality):
180 441 : m_val (val)
181 441 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 768574236 : static profile_probability never ()
185 : {
186 768574236 : profile_probability ret;
187 768574236 : ret.m_val = 0;
188 768574236 : ret.set_quality (PRECISE);
189 460732380 : return ret;
190 : }
191 :
192 5341378 : static profile_probability guessed_never ()
193 : {
194 5341378 : profile_probability ret;
195 5341378 : ret.m_val = 0;
196 5341378 : ret.set_quality (GUESSED);
197 5341360 : return ret;
198 : }
199 :
200 5331611 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10663222 : profile_probability r = guessed_always () / 2000;
204 5331611 : r.m_val--;
205 5331611 : return r;
206 : }
207 :
208 88275 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 176550 : profile_probability r = guessed_always () / 5;
212 88275 : r.m_val--;
213 88275 : return r;
214 : }
215 :
216 741137 : static profile_probability even ()
217 : {
218 1482274 : return guessed_always () / 2;
219 : }
220 :
221 347524 : static profile_probability very_likely ()
222 : {
223 347524 : return always () - very_unlikely ();
224 : }
225 :
226 33575 : static profile_probability likely ()
227 : {
228 33575 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 2410 : bool nonzero_p () const
232 : {
233 2410 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6469486 : static profile_probability guessed_always ()
237 : {
238 6469486 : profile_probability ret;
239 6469486 : ret.m_val = max_probability;
240 6469486 : ret.set_quality (GUESSED);
241 6250241 : return ret;
242 : }
243 :
244 564104569 : static profile_probability always ()
245 : {
246 563352187 : profile_probability ret;
247 564104569 : ret.m_val = max_probability;
248 564104569 : ret.set_quality (PRECISE);
249 384956009 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 950709157 : static profile_probability uninitialized ()
255 : {
256 950709157 : profile_probability c;
257 950709157 : c.m_val = uninitialized_probability;
258 950709157 : c.set_quality (GUESSED);
259 937055971 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5762292533 : bool initialized_p () const
264 : {
265 4018311978 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1577324 : bool reliable_p () const
270 : {
271 132125387 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3210465 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3210465 : profile_probability ret;
279 3210465 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3210465 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3210465 : ret.set_quality (GUESSED);
282 3210465 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 43 : profile_probability adjusted () const
287 : {
288 43 : profile_probability ret = *this;
289 43 : if (!initialized_p ())
290 0 : return *this;
291 43 : ret.set_quality (ADJUSTED);
292 43 : return ret;
293 : }
294 :
295 1136905506 : int to_reg_br_prob_base () const
296 : {
297 1136905506 : gcc_checking_assert (initialized_p ());
298 1136905506 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 500426799 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 500426799 : profile_probability ret;
305 500426799 : ret.m_val = ((unsigned int)v) / 8;
306 500426799 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 16082432 : return ret;
308 : }
309 :
310 470924136 : int to_reg_br_prob_note () const
311 : {
312 470924136 : gcc_checking_assert (initialized_p ());
313 470924136 : int ret = m_val * 8 + m_adjusted_quality;
314 470924136 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 470924136 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3219 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3219 : profile_probability ret;
323 3219 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3219 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3219 : uint64_t tmp;
329 3219 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3219 : gcc_checking_assert (tmp <= max_probability);
331 3219 : ret.m_val = tmp;
332 : }
333 3219 : ret.set_quality (PRECISE);
334 3219 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1669648200 : bool operator== (const profile_probability &other) const
339 : {
340 2353179991 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6759989 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6759989 : if (other == never ())
346 34697 : return *this;
347 6725292 : if (*this == never ())
348 23854 : return other;
349 6701438 : if (!initialized_p () || !other.initialized_p ())
350 5492738 : return uninitialized ();
351 :
352 1208700 : profile_probability ret;
353 1208700 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1208700 : ret.set_quality (MIN (quality (), other.quality ()));
355 1208700 : return ret;
356 : }
357 :
358 6597642 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6597642 : if (other == never ())
361 : return *this;
362 5280298 : if (*this == never ())
363 : {
364 905697 : *this = other;
365 905697 : return *this;
366 : }
367 4374601 : if (!initialized_p () || !other.initialized_p ())
368 2968055 : return *this = uninitialized ();
369 : else
370 : {
371 1406546 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 8430291 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1406546 : return *this;
375 : }
376 :
377 34777692 : profile_probability operator- (const profile_probability &other) const
378 : {
379 69555384 : if (*this == never ()
380 34777692 : || other == never ())
381 1901138 : return *this;
382 32876554 : if (!initialized_p () || !other.initialized_p ())
383 2192471 : return uninitialized ();
384 30684083 : profile_probability ret;
385 30684083 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30684083 : ret.set_quality (MIN (quality (), other.quality ()));
387 30684083 : return ret;
388 : }
389 :
390 11728655 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 23457310 : if (*this == never ()
393 11728655 : || other == never ())
394 : return *this;
395 9371387 : if (!initialized_p () || !other.initialized_p ())
396 2142656 : return *this = uninitialized ();
397 : else
398 : {
399 7228731 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 43372247 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 7228731 : return *this;
403 : }
404 :
405 1254230 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2508460 : if (*this == never ()
408 1254230 : || other == never ())
409 125083 : return never ();
410 1129147 : if (!initialized_p () || !other.initialized_p ())
411 58552 : return uninitialized ();
412 1070595 : profile_probability ret;
413 1070595 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 1070595 : ret.set_quality (MIN (quality (), other.quality ()));
415 1070595 : return ret;
416 : }
417 :
418 138017 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 276034 : if (*this == never ()
421 138017 : || other == never ())
422 438 : return *this = never ();
423 137579 : if (!initialized_p () || !other.initialized_p ())
424 12 : return *this = uninitialized ();
425 : else
426 : {
427 137567 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 137567 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 137567 : return *this;
431 : }
432 :
433 1781100 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1781100 : if (*this == never ())
436 102548 : return never ();
437 1678552 : if (!initialized_p () || !other.initialized_p ())
438 228399 : return uninitialized ();
439 1450153 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1450153 : if (m_val >= other.m_val)
442 : {
443 50743 : ret.m_val = max_probability;
444 50743 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 50743 : return ret;
447 : }
448 1399410 : else if (!m_val)
449 65734 : ret.m_val = 0;
450 : else
451 : {
452 1333676 : gcc_checking_assert (other.m_val);
453 1333676 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2788599 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1399410 : return ret;
457 : }
458 :
459 288543 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 288543 : if (*this == never ())
462 2005 : return *this = never ();
463 286538 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 286538 : if (m_val > other.m_val)
470 : {
471 455 : m_val = max_probability;
472 455 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 455 : return *this;
475 : }
476 286083 : else if (!m_val)
477 : ;
478 : else
479 : {
480 283951 : gcc_checking_assert (other.m_val);
481 283951 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 286083 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 286083 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 301683 : profile_probability split (const profile_probability &cprob)
504 : {
505 301683 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 301683 : if (!(*this == always ()))
512 292162 : *this = (*this - ret) / ret.invert ();
513 301683 : return ret;
514 : }
515 :
516 : /* Return VAL scaled by this probability. Round initialized probabilities
517 : to the nearest integer, with halfway values away from zero. Treat an
518 : uninitialized probability as one half and truncate toward zero. */
519 277249 : gcov_type apply (gcov_type val) const
520 : {
521 277249 : if (*this == uninitialized ())
522 70 : return val / 2;
523 :
524 : /* A unit probability leaves VAL unchanged. Return it directly because
525 : the magnitude of the minimum gcov_type value is one greater than the
526 : maximum gcov_type value. */
527 277179 : if (m_val == max_probability)
528 : return val;
529 :
530 : /* Convert to unsigned before negating so that the minimum gcov_type
531 : value has a representable magnitude. Scale the magnitude with
532 : overflow-safe arithmetic, then restore the sign. */
533 275514 : gcov_type_unsigned magnitude
534 275530 : = val < 0 ? -(gcov_type_unsigned) val : (gcov_type_unsigned) val;
535 275530 : uint64_t scaled;
536 275530 : bool scaled_p
537 275530 : = safe_scale_64bit (magnitude, m_val, max_probability, &scaled);
538 : /* The scaled result fits in uint64_t. With the unit case handled above,
539 : it also fits in the nonnegative range of gcov_type. */
540 275530 : gcc_checking_assert (scaled_p);
541 275530 : gcc_checking_assert
542 : (scaled <= (gcov_type_unsigned) INTTYPE_MAXIMUM (gcov_type));
543 :
544 275530 : return val < 0 ? -(gcov_type) scaled : (gcov_type) scaled;
545 : }
546 :
547 : /* Return 1-*THIS. */
548 21672735 : profile_probability invert () const
549 : {
550 31530214 : return always() - *this;
551 : }
552 :
553 : /* Return THIS with quality dropped to GUESSED. */
554 871805 : profile_probability guessed () const
555 : {
556 871805 : profile_probability ret = *this;
557 871805 : ret.set_quality (GUESSED);
558 871769 : return ret;
559 : }
560 :
561 : /* Return THIS with quality dropped to AFDO. */
562 : profile_probability afdo () const
563 : {
564 : profile_probability ret = *this;
565 : ret.set_quality (AFDO);
566 : return ret;
567 : }
568 :
569 : /* Return *THIS * NUM / DEN. */
570 14696721 : profile_probability apply_scale (int64_t num, int64_t den) const
571 : {
572 14696721 : if (*this == never ())
573 16161 : return *this;
574 14680560 : if (!initialized_p ())
575 5904494 : return uninitialized ();
576 8776066 : profile_probability ret;
577 8776066 : uint64_t tmp;
578 8776066 : safe_scale_64bit (m_val, num, den, &tmp);
579 8776066 : ret.m_val = MIN (tmp, max_probability);
580 8776066 : ret.set_quality (MIN (quality (), ADJUSTED));
581 8776066 : return ret;
582 : }
583 :
584 : /* Return *THIS * NUM / DEN. */
585 14098 : profile_probability apply_scale (profile_probability num,
586 : profile_probability den) const
587 : {
588 14098 : if (*this == never ())
589 306 : return *this;
590 13792 : if (num == never ())
591 0 : return num;
592 13792 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
593 0 : return uninitialized ();
594 13792 : if (num == den)
595 0 : return *this;
596 13792 : gcc_checking_assert (den.m_val);
597 :
598 13792 : profile_probability ret;
599 13792 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
600 : max_probability);
601 13792 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
602 : num.quality ()), den.quality ()));
603 13792 : return ret;
604 : }
605 :
606 : /* Return true when the probability of edge is reliable.
607 :
608 : The profile guessing code is good at predicting branch outcome (i.e.
609 : taken/not taken), that is predicted right slightly over 75% of time.
610 : It is however notoriously poor on predicting the probability itself.
611 : In general the profile appear a lot flatter (with probabilities closer
612 : to 50%) than the reality so it is bad idea to use it to drive optimization
613 : such as those disabling dynamic branch prediction for well predictable
614 : branches.
615 :
616 : There are two exceptions - edges leading to noreturn edges and edges
617 : predicted by number of iterations heuristics are predicted well. This macro
618 : should be able to distinguish those, but at the moment it simply check for
619 : noreturn heuristic that is only one giving probability over 99% or below
620 : 1%. In future we might want to propagate reliability information across the
621 : CFG if we find this information useful on multiple places. */
622 0 : bool probably_reliable_p () const
623 : {
624 0 : if (quality () >= ADJUSTED)
625 : return true;
626 0 : if (!initialized_p ())
627 : return false;
628 0 : return m_val < max_probability / 100
629 0 : || m_val > max_probability - max_probability / 100;
630 : }
631 :
632 : /* Return false if profile_probability is bogus. */
633 4425804149 : bool verify () const
634 : {
635 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
636 4425804149 : if (m_val == uninitialized_probability)
637 959281865 : return quality () == GUESSED;
638 3466522284 : else if (quality () < GUESSED)
639 : return false;
640 3466522284 : return m_val <= max_probability;
641 : }
642 :
643 : /* Comparisons are three-state and conservative. False is returned if
644 : the inequality cannot be decided. */
645 9108724 : bool operator< (const profile_probability &other) const
646 : {
647 9108724 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
648 : }
649 :
650 17864564 : bool operator> (const profile_probability &other) const
651 : {
652 17864564 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
653 : }
654 :
655 5490811 : bool operator<= (const profile_probability &other) const
656 : {
657 5490811 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
658 : }
659 :
660 960788 : bool operator>= (const profile_probability &other) const
661 : {
662 960788 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
663 : }
664 :
665 17 : profile_probability operator* (int64_t num) const
666 : {
667 17 : return apply_scale (num, 1);
668 : }
669 :
670 : profile_probability operator*= (int64_t num)
671 : {
672 : *this = apply_scale (num, 1);
673 : return *this;
674 : }
675 :
676 14033001 : profile_probability operator/ (int64_t den) const
677 : {
678 14033001 : return apply_scale (1, den);
679 : }
680 :
681 247044 : profile_probability operator/= (int64_t den)
682 : {
683 247044 : *this = apply_scale (1, den);
684 247044 : return *this;
685 : }
686 :
687 : /* Compute n-th power. */
688 : profile_probability pow (int) const;
689 :
690 : /* Compute sware root. */
691 : profile_probability sqrt () const;
692 :
693 : /* Get the value of the probability. */
694 0 : uint32_t value () const { return m_val; }
695 :
696 : /* Get the quality of the probability. */
697 6163288778 : enum profile_quality quality () const
698 : {
699 5945291318 : return (profile_quality) (m_adjusted_quality
700 5945314605 : ? m_adjusted_quality + min_quality - 1
701 51289293 : : UNINITIALIZED_PROFILE);
702 : }
703 :
704 : /* Output THIS to F. */
705 : void dump (FILE *f) const;
706 :
707 : /* Output THIS to BUFFER. */
708 : void dump (char *buffer) const;
709 :
710 : /* Print THIS to stderr. */
711 : void debug () const;
712 :
713 : /* Return true if THIS is known to differ significantly from OTHER. */
714 : bool differs_from_p (profile_probability other) const;
715 :
716 : /* Return if difference is greater than 50%. */
717 : bool differs_lot_from_p (profile_probability other) const;
718 :
719 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
720 : happens with COUNT2 probability. Return probability that either *THIS or
721 : OTHER happens. */
722 : profile_probability combine_with_count (profile_count count1,
723 : profile_probability other,
724 : profile_count count2) const;
725 :
726 : /* Return probability as sreal. */
727 : sreal to_sreal () const;
728 : /* LTO streaming support. */
729 : static profile_probability stream_in (class lto_input_block *);
730 : void stream_out (struct output_block *);
731 : void stream_out (struct lto_output_stream *);
732 : };
733 :
734 : /* Main data type to hold profile counters in GCC. Profile counts originate
735 : either from profile feedback, static profile estimation or both. We do not
736 : perform whole program profile propagation and thus profile estimation
737 : counters are often local to function, while counters from profile feedback
738 : (or special cases of profile estimation) can be used inter-procedurally.
739 :
740 : There are 3 basic types
741 : 1) local counters which are result of intra-procedural static profile
742 : estimation.
743 : 2) ipa counters which are result of profile feedback or special case
744 : of static profile estimation (such as in function main).
745 : 3) counters which counts as 0 inter-procedurally (because given function
746 : was never run in train feedback) but they hold local static profile
747 : estimate.
748 :
749 : Counters of type 1 and 3 cannot be mixed with counters of different type
750 : within operation (because whole function should use one type of counter)
751 : with exception that global zero mix in most operations where outcome is
752 : well defined.
753 :
754 : To take local counter and use it inter-procedurally use ipa member function
755 : which strips information irrelevant at the inter-procedural level.
756 :
757 : Counters are 61bit integers representing number of executions during the
758 : train run or normalized frequency within the function.
759 :
760 : As the profile is maintained during the compilation, many adjustments are
761 : made. Not all transformations can be made precisely, most importantly
762 : when code is being duplicated. It also may happen that part of CFG has
763 : profile counts known while other do not - for example when LTO optimizing
764 : partly profiled program or when profile was lost due to COMDAT merging.
765 :
766 : For this reason profile_count tracks more information than
767 : just unsigned integer and it is also ready for profile mismatches.
768 : The API of this data type represent operations that are natural
769 : on profile counts - sum, difference and operation with scales and
770 : probabilities. All operations are safe by never getting negative counts
771 : and they do end up in uninitialized scale if any of the parameters is
772 : uninitialized.
773 :
774 : All comparisons that are three state and handling of probabilities. Thus
775 : a < b is not equal to !(a >= b).
776 :
777 : The following pre-defined counts are available:
778 :
779 : profile_count::zero () for code that is known to execute zero times at
780 : runtime (this can be detected statically i.e. for paths leading to
781 : abort ();
782 : profile_count::one () for code that is known to execute once (such as
783 : main () function
784 : profile_count::uninitialized () for unknown execution count.
785 :
786 : */
787 :
788 : struct GTY(()) profile_count
789 : {
790 : public:
791 : /* Use 60bit to hold basic block counters. Should be at least
792 : 64bit. Although a counter cannot be negative, we use a signed
793 : type to hold various extra stages. */
794 :
795 : static const int n_bits = 60;
796 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
797 : private:
798 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
799 :
800 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
801 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
802 : incorrectly detects the alignment of a structure where the only
803 : 64-bit aligned object is a bit-field. We force the alignment of
804 : the entire field to mitigate this. */
805 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
806 : #else
807 : #define UINT64_BIT_FIELD_ALIGN
808 : #endif
809 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
810 : #undef UINT64_BIT_FIELD_ALIGN
811 : enum profile_quality m_quality : 4;
812 : public:
813 :
814 : /* Return true if both values can meaningfully appear in single function
815 : body. We have either all counters in function local or global, otherwise
816 : operations between them are not really defined well. */
817 4996470501 : bool compatible_p (const profile_count other) const
818 : {
819 9173839135 : if (!initialized_p () || !other.initialized_p ())
820 : return true;
821 16538811907 : if (*this == zero ()
822 154921657 : || other == zero ())
823 : return true;
824 : /* Do not allow nonzero global profile together with local guesses
825 : that are globally0. */
826 4017640244 : if (ipa ().nonzero_p ()
827 253301 : && !(other.ipa () == other))
828 0 : return false;
829 4017640244 : if (other.ipa ().nonzero_p ()
830 262491 : && !(ipa () == *this))
831 0 : return false;
832 :
833 4017640244 : return ipa_p () == other.ipa_p ();
834 : }
835 :
836 : /* Used for counters which are expected to be never executed. */
837 18632763544 : static profile_count zero ()
838 : {
839 14581548845 : return from_gcov_type (0);
840 : }
841 :
842 : static profile_count adjusted_zero ()
843 : {
844 : profile_count c;
845 : c.m_val = 0;
846 : c.m_quality = ADJUSTED;
847 : return c;
848 : }
849 :
850 : static profile_count afdo_zero ()
851 : {
852 : profile_count c;
853 : c.m_val = 0;
854 : c.m_quality = AFDO;
855 : return c;
856 : }
857 :
858 : static profile_count guessed_zero ()
859 : {
860 : profile_count c;
861 : c.m_val = 0;
862 : c.m_quality = GUESSED;
863 : return c;
864 : }
865 :
866 53813 : static profile_count one ()
867 : {
868 53615 : return from_gcov_type (1);
869 : }
870 :
871 : /* Value of counters which has not been initialized. Either because
872 : initialization did not happen yet or because profile is unknown. */
873 113457912 : static profile_count uninitialized ()
874 : {
875 113457912 : profile_count c;
876 113457912 : c.m_val = uninitialized_count;
877 113457912 : c.m_quality = GUESSED_LOCAL;
878 113457912 : return c;
879 : }
880 :
881 : /* Conversion to gcov_type is lossy. */
882 42343999 : gcov_type to_gcov_type () const
883 : {
884 0 : gcc_checking_assert (initialized_p ());
885 42343999 : return m_val;
886 : }
887 :
888 : /* Return true if value has been initialized. */
889 32972978714 : bool initialized_p () const
890 : {
891 11076591839 : return m_val != uninitialized_count;
892 : }
893 :
894 : /* Return true if value can be trusted. */
895 27075779 : bool reliable_p () const
896 : {
897 27075500 : return m_quality >= ADJUSTED;
898 : }
899 :
900 : /* Return true if value can be operated inter-procedurally. */
901 9079916314 : bool ipa_p () const
902 : {
903 5048467632 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
904 : }
905 :
906 : /* Return true if quality of profile is precise. */
907 59745467 : bool precise_p () const
908 : {
909 59745467 : return m_quality == PRECISE;
910 : }
911 :
912 : /* Get the value of the count. */
913 0 : uint64_t value () const { return m_val; }
914 :
915 : /* Get the quality of the count. */
916 98808733 : enum profile_quality quality () const { return m_quality; }
917 :
918 : /* When merging basic blocks, the two different profile counts are unified.
919 : Return true if this can be done without losing info about profile.
920 : The only case we care about here is when first BB contains something
921 : that makes it terminate in a way not visible in CFG. */
922 3373951 : bool ok_for_merging (profile_count other) const
923 : {
924 3373951 : if (m_quality < ADJUSTED
925 17319 : || other.m_quality < ADJUSTED)
926 : return true;
927 16306 : return !(other < *this);
928 : }
929 :
930 : /* When merging two BBs with different counts, pick common count that looks
931 : most representative. */
932 17868480 : profile_count merge (profile_count other) const
933 : {
934 905831 : if (*this == other || !other.initialized_p ()
935 905535 : || m_quality > other.m_quality)
936 16964264 : return *this;
937 904216 : if (other.m_quality > m_quality
938 904216 : || other > *this)
939 0 : return other;
940 904216 : return *this;
941 : }
942 :
943 : /* Basic operations. */
944 20124486263 : bool operator== (const profile_count &other) const
945 : {
946 15321801555 : return m_val == other.m_val && m_quality == other.m_quality;
947 : }
948 :
949 3944439 : profile_count operator+ (const profile_count &other) const
950 : {
951 3944439 : if (other == zero ())
952 2505 : return *this;
953 3941934 : if (*this == zero ())
954 239622 : return other;
955 3702312 : if (!initialized_p () || !other.initialized_p ())
956 6422 : return uninitialized ();
957 :
958 3695890 : profile_count ret;
959 3695890 : gcc_checking_assert (compatible_p (other));
960 3695890 : uint64_t ret_val = m_val + other.m_val;
961 3695890 : ret.m_val = MIN (ret_val, max_count);
962 3695890 : ret.m_quality = MIN (m_quality, other.m_quality);
963 3695890 : return ret;
964 : }
965 :
966 74784589 : profile_count &operator+= (const profile_count &other)
967 : {
968 74784589 : if (other == zero ())
969 5492213 : return *this;
970 69292376 : if (*this == zero ())
971 : {
972 42438420 : *this = other;
973 42438420 : return *this;
974 : }
975 26853956 : if (!initialized_p () || !other.initialized_p ())
976 353286 : return *this = uninitialized ();
977 : else
978 : {
979 26500670 : gcc_checking_assert (compatible_p (other));
980 26500670 : uint64_t ret_val = m_val + other.m_val;
981 26500670 : m_val = MIN (ret_val, max_count);
982 26500670 : m_quality = MIN (m_quality, other.m_quality);
983 : }
984 26500670 : return *this;
985 : }
986 :
987 20998012 : profile_count operator- (const profile_count &other) const
988 : {
989 22520389 : if (*this == zero () || other == zero ())
990 1508139 : return *this;
991 19489873 : if (!initialized_p () || !other.initialized_p ())
992 8919523 : return uninitialized ();
993 10570350 : gcc_checking_assert (compatible_p (other));
994 10570350 : profile_count ret;
995 10570350 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
996 10570350 : ret.m_quality = MIN (m_quality, other.m_quality);
997 10570350 : return ret;
998 : }
999 :
1000 9199088 : profile_count &operator-= (const profile_count &other)
1001 : {
1002 9947410 : if (*this == zero () || other == zero ())
1003 : return *this;
1004 8907266 : if (!initialized_p () || !other.initialized_p ())
1005 390761 : return *this = uninitialized ();
1006 : else
1007 : {
1008 8516505 : gcc_checking_assert (compatible_p (other));
1009 8516505 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
1010 8516505 : m_quality = MIN (m_quality, other.m_quality);
1011 : }
1012 8516505 : return *this;
1013 : }
1014 :
1015 : /* Return false if profile_count is bogus. */
1016 3382170221 : bool verify () const
1017 : {
1018 3382170221 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1019 3382170221 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1020 : }
1021 :
1022 : /* Comparisons are three-state and conservative. False is returned if
1023 : the inequality cannot be decided. */
1024 1092084551 : bool operator< (const profile_count &other) const
1025 : {
1026 1092084551 : if (!initialized_p () || !other.initialized_p ())
1027 : return false;
1028 1087201063 : if (*this == zero ())
1029 4330946 : return !(other == zero ());
1030 1085035590 : if (other == zero ())
1031 720 : return false;
1032 1085034870 : gcc_checking_assert (compatible_p (other));
1033 1085034870 : return m_val < other.m_val;
1034 : }
1035 :
1036 39168698 : bool operator> (const profile_count &other) const
1037 : {
1038 39168698 : if (!initialized_p () || !other.initialized_p ())
1039 : return false;
1040 34056479 : if (*this == zero ())
1041 224327 : return false;
1042 33832152 : if (other == zero ())
1043 1545052 : return !(*this == zero ());
1044 33059626 : gcc_checking_assert (compatible_p (other));
1045 33059626 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1046 : }
1047 :
1048 : bool operator< (const gcov_type other) const
1049 : {
1050 : gcc_checking_assert (ipa_p ());
1051 : gcc_checking_assert (other >= 0);
1052 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1053 : }
1054 :
1055 12102 : bool operator> (const gcov_type other) const
1056 : {
1057 12102 : gcc_checking_assert (ipa_p ());
1058 12102 : gcc_checking_assert (other >= 0);
1059 12102 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1060 : }
1061 :
1062 1221434 : bool operator<= (const profile_count &other) const
1063 : {
1064 1221434 : if (!initialized_p () || !other.initialized_p ())
1065 : return false;
1066 1221434 : if (*this == zero ())
1067 4232 : return true;
1068 1217202 : if (other == zero ())
1069 0 : return (*this == zero ());
1070 1217202 : gcc_checking_assert (compatible_p (other));
1071 1217202 : return m_val <= other.m_val;
1072 : }
1073 :
1074 1422927 : bool operator>= (const profile_count &other) const
1075 : {
1076 1422927 : if (!initialized_p () || !other.initialized_p ())
1077 : return false;
1078 1421799 : if (other == zero ())
1079 3028 : return true;
1080 1418771 : if (*this == zero ())
1081 23786 : return (other == zero ());
1082 1406878 : gcc_checking_assert (compatible_p (other));
1083 1406878 : return m_val >= other.m_val;
1084 : }
1085 :
1086 100958 : bool operator<= (const gcov_type other) const
1087 : {
1088 100958 : gcc_checking_assert (ipa_p ());
1089 100958 : gcc_checking_assert (other >= 0);
1090 100958 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1091 : }
1092 :
1093 64295 : bool operator>= (const gcov_type other) const
1094 : {
1095 64295 : gcc_checking_assert (ipa_p ());
1096 64295 : gcc_checking_assert (other >= 0);
1097 64295 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1098 : }
1099 :
1100 908619347 : profile_count operator* (int64_t num) const
1101 : {
1102 908542481 : return apply_scale (num, 1);
1103 : }
1104 :
1105 : profile_count operator*= (int64_t num)
1106 : {
1107 : *this = apply_scale (num, 1);
1108 : return *this;
1109 : }
1110 :
1111 : profile_count operator* (const sreal &num) const;
1112 : profile_count operator*= (const sreal &num);
1113 :
1114 2404713 : profile_count operator/ (int64_t den) const
1115 : {
1116 2404713 : return apply_scale (1, den);
1117 : }
1118 :
1119 8 : profile_count operator/= (int64_t den)
1120 : {
1121 0 : *this = apply_scale (1, den);
1122 8 : return *this;
1123 : }
1124 :
1125 : /* Return true when value is not zero and can be used for scaling.
1126 : This is different from *this > 0 because that requires counter to
1127 : be IPA. */
1128 8232420662 : bool nonzero_p () const
1129 : {
1130 8232420202 : return initialized_p () && m_val != 0;
1131 : }
1132 :
1133 : /* Make counter forcibly nonzero. */
1134 : profile_count force_nonzero () const;
1135 :
1136 :
1137 : /* Return maximum of A and B. If one of values is uninitialized return the
1138 : other. */
1139 :
1140 : static profile_count
1141 82539002 : max_prefer_initialized (const profile_count a, const profile_count b)
1142 : {
1143 82539002 : if (!a.initialized_p ())
1144 20711095 : return b;
1145 61827907 : if (!b.initialized_p ())
1146 2171970 : return a;
1147 59655937 : profile_count ret;
1148 59655937 : gcc_checking_assert (a.compatible_p (b));
1149 59655937 : ret.m_val = MAX (a.m_val, b.m_val);
1150 59655937 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1151 59655937 : return ret;
1152 : }
1153 :
1154 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1155 : accordingly. */
1156 : profile_count apply_probability (int prob) const
1157 : {
1158 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1159 : if (m_val == 0)
1160 : return *this;
1161 : if (!initialized_p ())
1162 : return uninitialized ();
1163 : profile_count ret;
1164 : uint64_t tmp;
1165 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1166 : ret.m_val = tmp;
1167 : ret.m_quality = MIN (m_quality, ADJUSTED);
1168 : return ret;
1169 : }
1170 :
1171 : /* Scale counter according to PROB. */
1172 399029548 : profile_count apply_probability (profile_probability prob) const
1173 : {
1174 411282693 : if (*this == zero () || prob == profile_probability::always ())
1175 129242685 : return *this;
1176 269786863 : if (prob == profile_probability::never ())
1177 20574431 : return zero ();
1178 249212432 : if (!initialized_p () || !prob.initialized_p ())
1179 31253311 : return uninitialized ();
1180 217959121 : profile_count ret;
1181 217959121 : uint64_t tmp;
1182 217959121 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1183 : &tmp);
1184 217959121 : ret.m_val = tmp;
1185 217982408 : ret.m_quality = MIN (m_quality, prob.quality ());
1186 217959121 : return ret;
1187 : }
1188 :
1189 : /* Return *THIS * NUM / DEN. */
1190 1035947179 : profile_count apply_scale (int64_t num, int64_t den) const
1191 : {
1192 1035947179 : if (m_val == 0)
1193 24703843 : return *this;
1194 1011243336 : if (!initialized_p ())
1195 4200326 : return uninitialized ();
1196 1007043010 : profile_count ret;
1197 1007043010 : uint64_t tmp;
1198 :
1199 1007043010 : gcc_checking_assert (num >= 0 && den > 0);
1200 1007043010 : safe_scale_64bit (m_val, num, den, &tmp);
1201 1007043010 : ret.m_val = MIN (tmp, max_count);
1202 1007043010 : ret.m_quality = MIN (m_quality, ADJUSTED);
1203 1007043010 : return ret;
1204 : }
1205 :
1206 30592914 : profile_count apply_scale (profile_count num, profile_count den) const
1207 : {
1208 30592914 : if (*this == zero ())
1209 2708595 : return *this;
1210 27884319 : if (num == zero ())
1211 94074 : return num;
1212 27790245 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1213 7762163 : return uninitialized ();
1214 20028082 : if (num == den)
1215 5860457 : return *this;
1216 14167625 : gcc_checking_assert (den.m_val);
1217 :
1218 14167625 : profile_count ret;
1219 14167625 : uint64_t val;
1220 14167625 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1221 14167625 : ret.m_val = MIN (val, max_count);
1222 14167625 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1223 : num.m_quality), den.m_quality);
1224 : /* Be sure that ret is not local if num is global.
1225 : Also ensure that ret is not global0 when num is global. */
1226 14167625 : if (num.ipa_p ())
1227 : {
1228 : /* This is common case of AFDO scaling when we upgrade
1229 : GLOBAL0_AFDO function to AFDO. Be sure that result
1230 : is AFDO and not GUESSED (which is unnecessarily low). */
1231 1872 : if (num.m_quality == AFDO
1232 0 : && (ret.m_quality != GUESSED
1233 0 : && ret.m_quality != GUESSED_LOCAL))
1234 0 : ret.m_quality = AFDO;
1235 : else
1236 3441 : ret.m_quality = MAX (ret.m_quality,
1237 : num == num.ipa () ? GUESSED : num.m_quality);
1238 : }
1239 14167625 : return ret;
1240 : }
1241 :
1242 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1243 18393450 : profile_count guessed_local () const
1244 : {
1245 18393450 : profile_count ret = *this;
1246 18393450 : if (!initialized_p ())
1247 0 : return *this;
1248 18393450 : ret.m_quality = GUESSED_LOCAL;
1249 18393450 : return ret;
1250 : }
1251 :
1252 : /* We know that profile is globally 0 but keep local profile if present. */
1253 794 : profile_count global0 () const
1254 : {
1255 794 : profile_count ret = *this;
1256 794 : if (!initialized_p ())
1257 0 : return *this;
1258 794 : ret.m_quality = GUESSED_GLOBAL0;
1259 794 : return ret;
1260 : }
1261 :
1262 : /* We know that profile is globally afdo 0 but keep local profile
1263 : if present. */
1264 0 : profile_count global0afdo () const
1265 : {
1266 0 : profile_count ret = *this;
1267 0 : if (!initialized_p ())
1268 0 : return *this;
1269 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1270 0 : return ret;
1271 : }
1272 :
1273 : /* We know that profile is globally adjusted 0 but keep local profile
1274 : if present. */
1275 64 : profile_count global0adjusted () const
1276 : {
1277 64 : profile_count ret = *this;
1278 64 : if (!initialized_p ())
1279 0 : return *this;
1280 64 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1281 64 : return ret;
1282 : }
1283 :
1284 : /* Return THIS with quality dropped to GUESSED. */
1285 383 : profile_count guessed () const
1286 : {
1287 383 : profile_count ret = *this;
1288 383 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1289 383 : return ret;
1290 : }
1291 :
1292 : /* Return THIS with quality GUESSED. */
1293 0 : profile_count force_guessed () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : gcc_checking_assert (initialized_p ());
1297 0 : ret.m_quality = GUESSED;
1298 0 : return ret;
1299 : }
1300 :
1301 : /* Return variant of profile count which is always safe to compare
1302 : across functions. */
1303 9164638614 : profile_count ipa () const
1304 : {
1305 9164638614 : if (m_quality > GUESSED_GLOBAL0)
1306 14168938 : return *this;
1307 9150469676 : if (m_quality == GUESSED_GLOBAL0)
1308 211921 : return zero ();
1309 9150257755 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1310 17449 : return adjusted_zero ();
1311 9150240306 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1312 0 : return afdo_zero ();
1313 9150240306 : return uninitialized ();
1314 : }
1315 :
1316 : /* Return THIS with quality dropped to AFDO. */
1317 0 : profile_count afdo () const
1318 : {
1319 0 : profile_count ret = *this;
1320 0 : ret.m_quality = AFDO;
1321 0 : return ret;
1322 : }
1323 :
1324 : /* Return probability of event with counter THIS within event with counter
1325 : OVERALL. */
1326 1102489223 : profile_probability probability_in (const profile_count overall) const
1327 : {
1328 1102506920 : if (*this == zero ()
1329 17697 : && !(overall == zero ()))
1330 2192 : return profile_probability::never ();
1331 1102486499 : if (!initialized_p () || !overall.initialized_p ()
1332 2204973530 : || !overall.m_val)
1333 51572 : return profile_probability::uninitialized ();
1334 1316668745 : if (*this == overall && m_quality == PRECISE)
1335 30529 : return profile_probability::always ();
1336 1102404930 : profile_probability ret;
1337 1102404930 : gcc_checking_assert (compatible_p (overall));
1338 :
1339 1102404930 : if (overall.m_val < m_val)
1340 : {
1341 154085 : ret.m_val = profile_probability::max_probability;
1342 154085 : ret.set_quality (GUESSED);
1343 154085 : return ret;
1344 : }
1345 : else
1346 : {
1347 1102250845 : gcc_checking_assert (overall.m_val);
1348 1102250845 : uint64_t tmp;
1349 1102250845 : safe_scale_64bit (m_val, profile_probability::max_probability,
1350 : overall.m_val, &tmp);
1351 1102250845 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1352 1102250845 : ret.m_val = tmp;
1353 : }
1354 1102250845 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1355 : GUESSED), ADJUSTED));
1356 1102250845 : return ret;
1357 : }
1358 :
1359 : /* Return true if profile count is very large, so we risk overflows
1360 : with loop transformations. */
1361 : bool
1362 1059054 : very_large_p ()
1363 : {
1364 1059054 : if (!initialized_p ())
1365 : return false;
1366 1059054 : return m_val > max_count / 65536;
1367 : }
1368 :
1369 : int to_frequency (struct function *fun) const;
1370 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1371 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1372 :
1373 : /* Output THIS to F. */
1374 : void dump (FILE *f, struct function *fun = NULL) const;
1375 :
1376 : /* Print THIS to stderr. */
1377 : void debug () const;
1378 :
1379 : /* Return true if THIS is known to differ significantly from OTHER. */
1380 : bool differs_from_p (profile_count other) const;
1381 :
1382 : /* We want to scale profile across function boundary from NUM to DEN.
1383 : Take care of the side case when NUM and DEN are zeros of incompatible
1384 : kinds. */
1385 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1386 :
1387 : /* THIS is a count of bb which is known to be executed IPA times.
1388 : Combine this information into bb counter. This means returning IPA
1389 : if it is nonzero, not changing anything if IPA is uninitialized
1390 : and if IPA is zero, turning THIS into corresponding local profile with
1391 : global0. */
1392 : profile_count combine_with_ipa_count (profile_count ipa);
1393 :
1394 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1395 : profile_count combine_with_ipa_count_within
1396 : (profile_count ipa, profile_count ipa2);
1397 :
1398 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1399 : Conversions back and forth are used to read the coverage and get it
1400 : into internal representation. */
1401 : static profile_count from_gcov_type (gcov_type v,
1402 : profile_quality quality = PRECISE);
1403 :
1404 : /* LTO streaming support. */
1405 : static profile_count stream_in (class lto_input_block *);
1406 : void stream_out (struct output_block *);
1407 : void stream_out (struct lto_output_stream *);
1408 : };
1409 : #endif
|
