Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2321974314 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2321974314 : uint64_t tmp;
88 2321974314 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2321974314 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2319676770 : *res = tmp / c;
92 2319676770 : return true;
93 : }
94 2297544 : if (c == 1)
95 : {
96 38790 : *res = (uint64_t) -1;
97 38790 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2258754 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greated then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5358754721 : set_quality (profile_quality quality)
169 : {
170 1127002046 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1127002046 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1125280728 : }
174 :
175 : public:
176 4233473993 : profile_probability (): m_val (uninitialized_probability)
177 2321036114 : { set_quality (GUESSED); }
178 :
179 384 : profile_probability (uint32_t val, profile_quality quality):
180 384 : m_val (val)
181 384 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 742911683 : static profile_probability never ()
185 : {
186 742911683 : profile_probability ret;
187 742911683 : ret.m_val = 0;
188 742911683 : ret.set_quality (PRECISE);
189 442710396 : return ret;
190 : }
191 :
192 5275630 : static profile_probability guessed_never ()
193 : {
194 5275630 : profile_probability ret;
195 5275630 : ret.m_val = 0;
196 5275630 : ret.set_quality (GUESSED);
197 5275630 : return ret;
198 : }
199 :
200 5200037 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10400074 : profile_probability r = guessed_always () / 2000;
204 5200037 : r.m_val--;
205 5200037 : return r;
206 : }
207 :
208 85056 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 170112 : profile_probability r = guessed_always () / 5;
212 85056 : r.m_val--;
213 85056 : return r;
214 : }
215 :
216 731193 : static profile_probability even ()
217 : {
218 1462386 : return guessed_always () / 2;
219 : }
220 :
221 340374 : static profile_probability very_likely ()
222 : {
223 340374 : return always () - very_unlikely ();
224 : }
225 :
226 32706 : static profile_probability likely ()
227 : {
228 32706 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 1932 : bool nonzero_p () const
232 : {
233 1932 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6325761 : static profile_probability guessed_always ()
237 : {
238 6325761 : profile_probability ret;
239 6325761 : ret.m_val = max_probability;
240 6325761 : ret.set_quality (GUESSED);
241 6130307 : return ret;
242 : }
243 :
244 549229203 : static profile_probability always ()
245 : {
246 549229203 : profile_probability ret;
247 549229203 : ret.m_val = max_probability;
248 549229203 : ret.set_quality (PRECISE);
249 371928243 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 910201532 : static profile_probability uninitialized ()
255 : {
256 910201532 : profile_probability c;
257 910201532 : c.m_val = uninitialized_probability;
258 910201532 : c.set_quality (GUESSED);
259 896688653 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5617510003 : bool initialized_p () const
264 : {
265 3912877634 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1544543 : bool reliable_p () const
270 : {
271 123931597 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3171190 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3171190 : profile_probability ret;
279 3171190 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3171190 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3171190 : ret.set_quality (GUESSED);
282 3171190 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 42 : profile_probability adjusted () const
287 : {
288 42 : profile_probability ret = *this;
289 42 : if (!initialized_p ())
290 0 : return *this;
291 42 : ret.set_quality (ADJUSTED);
292 42 : return ret;
293 : }
294 :
295 1108262217 : int to_reg_br_prob_base () const
296 : {
297 1108262217 : gcc_checking_assert (initialized_p ());
298 1108262217 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 494207863 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 494207863 : profile_probability ret;
305 494207863 : ret.m_val = ((unsigned int)v) / 8;
306 494207863 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 15901759 : return ret;
308 : }
309 :
310 465023010 : int to_reg_br_prob_note () const
311 : {
312 465023010 : gcc_checking_assert (initialized_p ());
313 465023010 : int ret = m_val * 8 + m_adjusted_quality;
314 465023010 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 465023010 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3092 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3092 : profile_probability ret;
323 3092 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3092 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3092 : uint64_t tmp;
329 3092 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3092 : gcc_checking_assert (tmp <= max_probability);
331 3092 : ret.m_val = tmp;
332 : }
333 3092 : ret.set_quality (PRECISE);
334 3092 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1624800898 : bool operator== (const profile_probability &other) const
339 : {
340 2294921062 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6655287 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6655287 : if (other == never ())
346 31711 : return *this;
347 6623576 : if (*this == never ())
348 23530 : return other;
349 6600046 : if (!initialized_p () || !other.initialized_p ())
350 5425902 : return uninitialized ();
351 :
352 1174144 : profile_probability ret;
353 1174144 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1174144 : ret.set_quality (MIN (quality (), other.quality ()));
355 1174144 : return ret;
356 : }
357 :
358 6353288 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6353288 : if (other == never ())
361 : return *this;
362 5109467 : if (*this == never ())
363 : {
364 873515 : *this = other;
365 873515 : return *this;
366 : }
367 4235952 : if (!initialized_p () || !other.initialized_p ())
368 2941904 : return *this = uninitialized ();
369 : else
370 : {
371 1294048 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 1294048 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1294048 : return *this;
375 : }
376 :
377 34100227 : profile_probability operator- (const profile_probability &other) const
378 : {
379 68200454 : if (*this == never ()
380 34100227 : || other == never ())
381 1856303 : return *this;
382 32243924 : if (!initialized_p () || !other.initialized_p ())
383 2165118 : return uninitialized ();
384 30078806 : profile_probability ret;
385 30078806 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30078806 : ret.set_quality (MIN (quality (), other.quality ()));
387 30078806 : return ret;
388 : }
389 :
390 11402059 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 22804118 : if (*this == never ()
393 11402059 : || other == never ())
394 2258691 : return *this;
395 9143368 : if (!initialized_p () || !other.initialized_p ())
396 2134668 : return *this = uninitialized ();
397 : else
398 : {
399 7008700 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 7008700 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 7008700 : return *this;
403 : }
404 :
405 1162965 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2325930 : if (*this == never ()
408 1162965 : || other == never ())
409 121236 : return never ();
410 1041729 : if (!initialized_p () || !other.initialized_p ())
411 58151 : return uninitialized ();
412 983578 : profile_probability ret;
413 983578 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 983578 : ret.set_quality (MIN (quality (), other.quality ()));
415 983578 : return ret;
416 : }
417 :
418 133779 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 267558 : if (*this == never ()
421 133779 : || other == never ())
422 417 : return *this = never ();
423 133362 : if (!initialized_p () || !other.initialized_p ())
424 9 : return *this = uninitialized ();
425 : else
426 : {
427 133353 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 248680 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 133353 : return *this;
431 : }
432 :
433 1771059 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1771059 : if (*this == never ())
436 103240 : return never ();
437 1667819 : if (!initialized_p () || !other.initialized_p ())
438 225559 : return uninitialized ();
439 1442260 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1442260 : if (m_val >= other.m_val)
442 : {
443 47499 : ret.m_val = max_probability;
444 47499 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 47499 : return ret;
447 : }
448 1394761 : else if (!m_val)
449 61811 : ret.m_val = 0;
450 : else
451 : {
452 1332950 : gcc_checking_assert (other.m_val);
453 1332950 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2781500 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1394761 : return ret;
457 : }
458 :
459 317502 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 317502 : if (*this == never ())
462 1521 : return *this = never ();
463 315981 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 315981 : if (m_val > other.m_val)
470 : {
471 695 : m_val = max_probability;
472 695 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 695 : return *this;
475 : }
476 315286 : else if (!m_val)
477 : ;
478 : else
479 : {
480 312130 : gcc_checking_assert (other.m_val);
481 312130 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 576998 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 315286 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 281439 : profile_probability split (const profile_probability &cprob)
504 : {
505 281439 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 281439 : if (!(*this == always ()))
512 273114 : *this = (*this - ret) / ret.invert ();
513 281439 : return ret;
514 : }
515 :
516 257382 : gcov_type apply (gcov_type val) const
517 : {
518 257382 : if (*this == uninitialized ())
519 54 : return val / 2;
520 257328 : return RDIV (val * m_val, max_probability);
521 : }
522 :
523 : /* Return 1-*THIS. */
524 21285476 : profile_probability invert () const
525 : {
526 30849292 : return always() - *this;
527 : }
528 :
529 : /* Return THIS with quality dropped to GUESSED. */
530 833575 : profile_probability guessed () const
531 : {
532 833575 : profile_probability ret = *this;
533 833575 : ret.set_quality (GUESSED);
534 833575 : return ret;
535 : }
536 :
537 : /* Return THIS with quality dropped to AFDO. */
538 : profile_probability afdo () const
539 : {
540 : profile_probability ret = *this;
541 : ret.set_quality (AFDO);
542 : return ret;
543 : }
544 :
545 : /* Return *THIS * NUM / DEN. */
546 14405592 : profile_probability apply_scale (int64_t num, int64_t den) const
547 : {
548 14405592 : if (*this == never ())
549 15882 : return *this;
550 14389710 : if (!initialized_p ())
551 5838742 : return uninitialized ();
552 8550968 : profile_probability ret;
553 8550968 : uint64_t tmp;
554 8550968 : safe_scale_64bit (m_val, num, den, &tmp);
555 8550968 : ret.m_val = MIN (tmp, max_probability);
556 8550968 : ret.set_quality (MIN (quality (), ADJUSTED));
557 8550968 : return ret;
558 : }
559 :
560 : /* Return *THIS * NUM / DEN. */
561 11158 : profile_probability apply_scale (profile_probability num,
562 : profile_probability den) const
563 : {
564 11158 : if (*this == never ())
565 56 : return *this;
566 11102 : if (num == never ())
567 0 : return num;
568 11102 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
569 0 : return uninitialized ();
570 11102 : if (num == den)
571 0 : return *this;
572 11102 : gcc_checking_assert (den.m_val);
573 :
574 11102 : profile_probability ret;
575 11102 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
576 : max_probability);
577 11102 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
578 : num.quality ()), den.quality ()));
579 11102 : return ret;
580 : }
581 :
582 : /* Return true when the probability of edge is reliable.
583 :
584 : The profile guessing code is good at predicting branch outcome (i.e.
585 : taken/not taken), that is predicted right slightly over 75% of time.
586 : It is however notoriously poor on predicting the probability itself.
587 : In general the profile appear a lot flatter (with probabilities closer
588 : to 50%) than the reality so it is bad idea to use it to drive optimization
589 : such as those disabling dynamic branch prediction for well predictable
590 : branches.
591 :
592 : There are two exceptions - edges leading to noreturn edges and edges
593 : predicted by number of iterations heuristics are predicted well. This macro
594 : should be able to distinguish those, but at the moment it simply check for
595 : noreturn heuristic that is only one giving probability over 99% or below
596 : 1%. In future we might want to propagate reliability information across the
597 : CFG if we find this information useful on multiple places. */
598 0 : bool probably_reliable_p () const
599 : {
600 0 : if (quality () >= ADJUSTED)
601 : return true;
602 0 : if (!initialized_p ())
603 : return false;
604 0 : return m_val < max_probability / 100
605 0 : || m_val > max_probability - max_probability / 100;
606 : }
607 :
608 : /* Return false if profile_probability is bogus. */
609 4307258579 : bool verify () const
610 : {
611 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
612 4307258579 : if (m_val == uninitialized_probability)
613 931921081 : return quality () == GUESSED;
614 3375337498 : else if (quality () < GUESSED)
615 : return false;
616 3375337498 : return m_val <= max_probability;
617 : }
618 :
619 : /* Comparisons are three-state and conservative. False is returned if
620 : the inequality cannot be decided. */
621 8128397 : bool operator< (const profile_probability &other) const
622 : {
623 8128397 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
624 : }
625 :
626 17600332 : bool operator> (const profile_probability &other) const
627 : {
628 17600332 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
629 : }
630 :
631 5344190 : bool operator<= (const profile_probability &other) const
632 : {
633 5344190 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
634 : }
635 :
636 977278 : bool operator>= (const profile_probability &other) const
637 : {
638 977278 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
639 : }
640 :
641 17 : profile_probability operator* (int64_t num) const
642 : {
643 17 : return apply_scale (num, 1);
644 : }
645 :
646 : profile_probability operator*= (int64_t num)
647 : {
648 : *this = apply_scale (num, 1);
649 : return *this;
650 : }
651 :
652 13779017 : profile_probability operator/ (int64_t den) const
653 : {
654 13779017 : return apply_scale (1, den);
655 : }
656 :
657 241705 : profile_probability operator/= (int64_t den)
658 : {
659 241705 : *this = apply_scale (1, den);
660 241705 : return *this;
661 : }
662 :
663 : /* Compute n-th power. */
664 : profile_probability pow (int) const;
665 :
666 : /* Compute sware root. */
667 : profile_probability sqrt () const;
668 :
669 : /* Get the value of the probability. */
670 0 : uint32_t value () const { return m_val; }
671 :
672 : /* Get the quality of the probability. */
673 6009053729 : enum profile_quality quality () const
674 : {
675 5796398822 : return (profile_quality) (m_adjusted_quality
676 5796420647 : ? m_adjusted_quality + min_quality - 1
677 : : UNINITIALIZED_PROFILE);
678 : }
679 :
680 : /* Output THIS to F. */
681 : void dump (FILE *f) const;
682 :
683 : /* Output THIS to BUFFER. */
684 : void dump (char *buffer) const;
685 :
686 : /* Print THIS to stderr. */
687 : void debug () const;
688 :
689 : /* Return true if THIS is known to differ significantly from OTHER. */
690 : bool differs_from_p (profile_probability other) const;
691 :
692 : /* Return if difference is greater than 50%. */
693 : bool differs_lot_from_p (profile_probability other) const;
694 :
695 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
696 : happens with COUNT2 probability. Return probability that either *THIS or
697 : OTHER happens. */
698 : profile_probability combine_with_count (profile_count count1,
699 : profile_probability other,
700 : profile_count count2) const;
701 :
702 : /* Return probability as sreal. */
703 : sreal to_sreal () const;
704 : /* LTO streaming support. */
705 : static profile_probability stream_in (class lto_input_block *);
706 : void stream_out (struct output_block *);
707 : void stream_out (struct lto_output_stream *);
708 : };
709 :
710 : /* Main data type to hold profile counters in GCC. Profile counts originate
711 : either from profile feedback, static profile estimation or both. We do not
712 : perform whole program profile propagation and thus profile estimation
713 : counters are often local to function, while counters from profile feedback
714 : (or special cases of profile estimation) can be used inter-procedurally.
715 :
716 : There are 3 basic types
717 : 1) local counters which are result of intra-procedural static profile
718 : estimation.
719 : 2) ipa counters which are result of profile feedback or special case
720 : of static profile estimation (such as in function main).
721 : 3) counters which counts as 0 inter-procedurally (because given function
722 : was never run in train feedback) but they hold local static profile
723 : estimate.
724 :
725 : Counters of type 1 and 3 cannot be mixed with counters of different type
726 : within operation (because whole function should use one type of counter)
727 : with exception that global zero mix in most operations where outcome is
728 : well defined.
729 :
730 : To take local counter and use it inter-procedurally use ipa member function
731 : which strips information irrelevant at the inter-procedural level.
732 :
733 : Counters are 61bit integers representing number of executions during the
734 : train run or normalized frequency within the function.
735 :
736 : As the profile is maintained during the compilation, many adjustments are
737 : made. Not all transformations can be made precisely, most importantly
738 : when code is being duplicated. It also may happen that part of CFG has
739 : profile counts known while other do not - for example when LTO optimizing
740 : partly profiled program or when profile was lost due to COMDAT merging.
741 :
742 : For this reason profile_count tracks more information than
743 : just unsigned integer and it is also ready for profile mismatches.
744 : The API of this data type represent operations that are natural
745 : on profile counts - sum, difference and operation with scales and
746 : probabilities. All operations are safe by never getting negative counts
747 : and they do end up in uninitialized scale if any of the parameters is
748 : uninitialized.
749 :
750 : All comparisons that are three state and handling of probabilities. Thus
751 : a < b is not equal to !(a >= b).
752 :
753 : The following pre-defined counts are available:
754 :
755 : profile_count::zero () for code that is known to execute zero times at
756 : runtime (this can be detected statically i.e. for paths leading to
757 : abort ();
758 : profile_count::one () for code that is known to execute once (such as
759 : main () function
760 : profile_count::uninitialized () for unknown execution count.
761 :
762 : */
763 :
764 : struct GTY(()) profile_count
765 : {
766 : public:
767 : /* Use 60bit to hold basic block counters. Should be at least
768 : 64bit. Although a counter cannot be negative, we use a signed
769 : type to hold various extra stages. */
770 :
771 : static const int n_bits = 60;
772 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
773 : private:
774 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
775 :
776 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
777 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
778 : incorrectly detects the alignment of a structure where the only
779 : 64-bit aligned object is a bit-field. We force the alignment of
780 : the entire field to mitigate this. */
781 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
782 : #else
783 : #define UINT64_BIT_FIELD_ALIGN
784 : #endif
785 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
786 : #undef UINT64_BIT_FIELD_ALIGN
787 : enum profile_quality m_quality : 4;
788 : public:
789 :
790 : /* Return true if both values can meaningfully appear in single function
791 : body. We have either all counters in function local or global, otherwise
792 : operations between them are not really defined well. */
793 4885510826 : bool compatible_p (const profile_count other) const
794 : {
795 8982783451 : if (!initialized_p () || !other.initialized_p ())
796 : return true;
797 16227045886 : if (*this == zero ()
798 8041053582 : || other == zero ())
799 148319965 : return true;
800 : /* Do not allow nonzero global profile together with local guesses
801 : that are globally0. */
802 3944676187 : if (ipa ().nonzero_p ()
803 244157 : && !(other.ipa () == other))
804 0 : return false;
805 3944676187 : if (other.ipa ().nonzero_p ()
806 253563 : && !(ipa () == *this))
807 0 : return false;
808 :
809 3944676187 : return ipa_p () == other.ipa_p ();
810 : }
811 :
812 : /* Used for counters which are expected to be never executed. */
813 18357106190 : static profile_count zero ()
814 : {
815 14365604178 : return from_gcov_type (0);
816 : }
817 :
818 10545 : static profile_count adjusted_zero ()
819 : {
820 10545 : profile_count c;
821 10545 : c.m_val = 0;
822 10545 : c.m_quality = ADJUSTED;
823 10545 : return c;
824 : }
825 :
826 0 : static profile_count afdo_zero ()
827 : {
828 0 : profile_count c;
829 0 : c.m_val = 0;
830 0 : c.m_quality = AFDO;
831 0 : return c;
832 : }
833 :
834 : static profile_count guessed_zero ()
835 : {
836 : profile_count c;
837 : c.m_val = 0;
838 : c.m_quality = GUESSED;
839 : return c;
840 : }
841 :
842 50518 : static profile_count one ()
843 : {
844 50305 : return from_gcov_type (1);
845 : }
846 :
847 : /* Value of counters which has not been initialized. Either because
848 : initialization did not happen yet or because profile is unknown. */
849 9167742386 : static profile_count uninitialized ()
850 : {
851 9167742386 : profile_count c;
852 9167742386 : c.m_val = uninitialized_count;
853 9167742386 : c.m_quality = GUESSED_LOCAL;
854 9136038383 : return c;
855 : }
856 :
857 : /* Conversion to gcov_type is lossy. */
858 42384739 : gcov_type to_gcov_type () const
859 : {
860 0 : gcc_checking_assert (initialized_p ());
861 42384739 : return m_val;
862 : }
863 :
864 : /* Return true if value has been initialized. */
865 32366895749 : bool initialized_p () const
866 : {
867 10847458198 : return m_val != uninitialized_count;
868 : }
869 :
870 : /* Return true if value can be trusted. */
871 26295908 : bool reliable_p () const
872 : {
873 26295639 : return m_quality >= ADJUSTED;
874 : }
875 :
876 : /* Return true if value can be operated inter-procedurally. */
877 8936751496 : bool ipa_p () const
878 : {
879 4978890510 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
880 : }
881 :
882 : /* Return true if quality of profile is precise. */
883 55144182 : bool precise_p () const
884 : {
885 55144182 : return m_quality == PRECISE;
886 : }
887 :
888 : /* Get the value of the count. */
889 0 : uint64_t value () const { return m_val; }
890 :
891 : /* Get the quality of the count. */
892 96327393 : enum profile_quality quality () const { return m_quality; }
893 :
894 : /* When merging basic blocks, the two different profile counts are unified.
895 : Return true if this can be done without losing info about profile.
896 : The only case we care about here is when first BB contains something
897 : that makes it terminate in a way not visible in CFG. */
898 3214412 : bool ok_for_merging (profile_count other) const
899 : {
900 3214412 : if (m_quality < ADJUSTED
901 16083 : || other.m_quality < ADJUSTED)
902 : return true;
903 15147 : return !(other < *this);
904 : }
905 :
906 : /* When merging two BBs with different counts, pick common count that looks
907 : most representative. */
908 16828743 : profile_count merge (profile_count other) const
909 : {
910 833685 : if (*this == other || !other.initialized_p ()
911 833400 : || m_quality > other.m_quality)
912 15996571 : return *this;
913 832172 : if (other.m_quality > m_quality
914 832172 : || other > *this)
915 0 : return other;
916 832172 : return *this;
917 : }
918 :
919 : /* Basic operations. */
920 19806257399 : bool operator== (const profile_count &other) const
921 : {
922 15068092994 : return m_val == other.m_val && m_quality == other.m_quality;
923 : }
924 :
925 3806749 : profile_count operator+ (const profile_count &other) const
926 : {
927 3806749 : if (other == zero ())
928 2406 : return *this;
929 3804343 : if (*this == zero ())
930 210318 : return other;
931 3594025 : if (!initialized_p () || !other.initialized_p ())
932 6035 : return uninitialized ();
933 :
934 3587990 : profile_count ret;
935 3587990 : gcc_checking_assert (compatible_p (other));
936 3587990 : uint64_t ret_val = m_val + other.m_val;
937 3587990 : ret.m_val = MIN (ret_val, max_count);
938 3587990 : ret.m_quality = MIN (m_quality, other.m_quality);
939 3587990 : return ret;
940 : }
941 :
942 73529416 : profile_count &operator+= (const profile_count &other)
943 : {
944 73529416 : if (other == zero ())
945 5643691 : return *this;
946 67885725 : if (*this == zero ())
947 : {
948 41649664 : *this = other;
949 41649664 : return *this;
950 : }
951 26236061 : if (!initialized_p () || !other.initialized_p ())
952 350792 : return *this = uninitialized ();
953 : else
954 : {
955 25885269 : gcc_checking_assert (compatible_p (other));
956 25885269 : uint64_t ret_val = m_val + other.m_val;
957 25885269 : m_val = MIN (ret_val, max_count);
958 25885269 : m_quality = MIN (m_quality, other.m_quality);
959 : }
960 25885269 : return *this;
961 : }
962 :
963 20235089 : profile_count operator- (const profile_count &other) const
964 : {
965 20252331 : if (*this == zero () || other == zero ())
966 1376780 : return *this;
967 18858309 : if (!initialized_p () || !other.initialized_p ())
968 8322203 : return uninitialized ();
969 10536106 : gcc_checking_assert (compatible_p (other));
970 10536106 : profile_count ret;
971 10536106 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
972 10536106 : ret.m_quality = MIN (m_quality, other.m_quality);
973 10536106 : return ret;
974 : }
975 :
976 9021152 : profile_count &operator-= (const profile_count &other)
977 : {
978 9471093 : if (*this == zero () || other == zero ())
979 287099 : return *this;
980 8734053 : if (!initialized_p () || !other.initialized_p ())
981 350941 : return *this = uninitialized ();
982 : else
983 : {
984 8383112 : gcc_checking_assert (compatible_p (other));
985 8383112 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
986 8383112 : m_quality = MIN (m_quality, other.m_quality);
987 : }
988 8383112 : return *this;
989 : }
990 :
991 : /* Return false if profile_count is bogus. */
992 3287118418 : bool verify () const
993 : {
994 3287118418 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
995 3287118418 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
996 : }
997 :
998 : /* Comparisons are three-state and conservative. False is returned if
999 : the inequality cannot be decided. */
1000 1095784398 : bool operator< (const profile_count &other) const
1001 : {
1002 1095784398 : if (!initialized_p () || !other.initialized_p ())
1003 : return false;
1004 1091017174 : if (*this == zero ())
1005 4136104 : return !(other == zero ());
1006 1088949122 : if (other == zero ())
1007 873 : return false;
1008 1088948249 : gcc_checking_assert (compatible_p (other));
1009 1088948249 : return m_val < other.m_val;
1010 : }
1011 :
1012 38158980 : bool operator> (const profile_count &other) const
1013 : {
1014 38158980 : if (!initialized_p () || !other.initialized_p ())
1015 : return false;
1016 33307768 : if (*this == zero ())
1017 221417 : return false;
1018 33086351 : if (other == zero ())
1019 1513256 : return !(*this == zero ());
1020 32329723 : gcc_checking_assert (compatible_p (other));
1021 32329723 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1022 : }
1023 :
1024 : bool operator< (const gcov_type other) const
1025 : {
1026 : gcc_checking_assert (ipa_p ());
1027 : gcc_checking_assert (other >= 0);
1028 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1029 : }
1030 :
1031 7599 : bool operator> (const gcov_type other) const
1032 : {
1033 7599 : gcc_checking_assert (ipa_p ());
1034 7599 : gcc_checking_assert (other >= 0);
1035 7599 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1036 : }
1037 :
1038 1178775 : bool operator<= (const profile_count &other) const
1039 : {
1040 1178775 : if (!initialized_p () || !other.initialized_p ())
1041 : return false;
1042 1178775 : if (*this == zero ())
1043 3926 : return true;
1044 1174849 : if (other == zero ())
1045 0 : return (*this == zero ());
1046 1174849 : gcc_checking_assert (compatible_p (other));
1047 1174849 : return m_val <= other.m_val;
1048 : }
1049 :
1050 1405461 : bool operator>= (const profile_count &other) const
1051 : {
1052 1405461 : if (!initialized_p () || !other.initialized_p ())
1053 : return false;
1054 1404356 : if (other == zero ())
1055 2825 : return true;
1056 1401531 : if (*this == zero ())
1057 20020 : return (other == zero ());
1058 1391521 : gcc_checking_assert (compatible_p (other));
1059 1391521 : return m_val >= other.m_val;
1060 : }
1061 :
1062 97441 : bool operator<= (const gcov_type other) const
1063 : {
1064 97441 : gcc_checking_assert (ipa_p ());
1065 97441 : gcc_checking_assert (other >= 0);
1066 97441 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1067 : }
1068 :
1069 60535 : bool operator>= (const gcov_type other) const
1070 : {
1071 60535 : gcc_checking_assert (ipa_p ());
1072 60535 : gcc_checking_assert (other >= 0);
1073 60535 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1074 : }
1075 :
1076 915526050 : profile_count operator* (int64_t num) const
1077 : {
1078 915442639 : return apply_scale (num, 1);
1079 : }
1080 :
1081 : profile_count operator*= (int64_t num)
1082 : {
1083 : *this = apply_scale (num, 1);
1084 : return *this;
1085 : }
1086 :
1087 : profile_count operator* (const sreal &num) const;
1088 : profile_count operator*= (const sreal &num);
1089 :
1090 2357287 : profile_count operator/ (int64_t den) const
1091 : {
1092 2357287 : return apply_scale (1, den);
1093 : }
1094 :
1095 8 : profile_count operator/= (int64_t den)
1096 : {
1097 0 : *this = apply_scale (1, den);
1098 8 : return *this;
1099 : }
1100 :
1101 : /* Return true when value is not zero and can be used for scaling.
1102 : This is different from *this > 0 because that requires counter to
1103 : be IPA. */
1104 8087623027 : bool nonzero_p () const
1105 : {
1106 8087622372 : return initialized_p () && m_val != 0;
1107 : }
1108 :
1109 : /* Make counter forcibly nonzero. */
1110 : profile_count force_nonzero () const;
1111 :
1112 :
1113 : /* Return maximum of A and B. If one of values is uninitialized return the
1114 : other. */
1115 :
1116 : static profile_count
1117 79025560 : max_prefer_initialized (const profile_count a, const profile_count b)
1118 : {
1119 79025560 : if (!a.initialized_p ())
1120 19621936 : return b;
1121 59403624 : if (!b.initialized_p ())
1122 1907303 : return a;
1123 57496321 : profile_count ret;
1124 57496321 : gcc_checking_assert (a.compatible_p (b));
1125 57496321 : ret.m_val = MAX (a.m_val, b.m_val);
1126 57496321 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1127 57496321 : return ret;
1128 : }
1129 :
1130 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1131 : accordingly. */
1132 : profile_count apply_probability (int prob) const
1133 : {
1134 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1135 : if (m_val == 0)
1136 : return *this;
1137 : if (!initialized_p ())
1138 : return uninitialized ();
1139 : profile_count ret;
1140 : uint64_t tmp;
1141 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1142 : ret.m_val = tmp;
1143 : ret.m_quality = MIN (m_quality, ADJUSTED);
1144 : return ret;
1145 : }
1146 :
1147 : /* Scale counter according to PROB. */
1148 387825797 : profile_count apply_probability (profile_probability prob) const
1149 : {
1150 396651464 : if (*this == zero () || prob == profile_probability::always ())
1151 124764743 : return *this;
1152 263061054 : if (prob == profile_probability::never ())
1153 20214598 : return zero ();
1154 242846456 : if (!initialized_p () || !prob.initialized_p ())
1155 30213374 : return uninitialized ();
1156 212633082 : profile_count ret;
1157 212633082 : uint64_t tmp;
1158 212633082 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1159 : &tmp);
1160 212633082 : ret.m_val = tmp;
1161 212654907 : ret.m_quality = MIN (m_quality, prob.quality ());
1162 212633082 : return ret;
1163 : }
1164 :
1165 : /* Return *THIS * NUM / DEN. */
1166 1040911155 : profile_count apply_scale (int64_t num, int64_t den) const
1167 : {
1168 1040911155 : if (m_val == 0)
1169 24498678 : return *this;
1170 1016412477 : if (!initialized_p ())
1171 3951975 : return uninitialized ();
1172 1012460502 : profile_count ret;
1173 1012460502 : uint64_t tmp;
1174 :
1175 1012460502 : gcc_checking_assert (num >= 0 && den > 0);
1176 1012460502 : safe_scale_64bit (m_val, num, den, &tmp);
1177 1012460502 : ret.m_val = MIN (tmp, max_count);
1178 1012460502 : ret.m_quality = MIN (m_quality, ADJUSTED);
1179 1012460502 : return ret;
1180 : }
1181 :
1182 28833772 : profile_count apply_scale (profile_count num, profile_count den) const
1183 : {
1184 28833772 : if (*this == zero ())
1185 2482764 : return *this;
1186 26351008 : if (num == zero ())
1187 86606 : return num;
1188 26264402 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1189 7119659 : return uninitialized ();
1190 19144743 : if (num == den)
1191 5608548 : return *this;
1192 13536195 : gcc_checking_assert (den.m_val);
1193 :
1194 13536195 : profile_count ret;
1195 13536195 : uint64_t val;
1196 13536195 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1197 13536195 : ret.m_val = MIN (val, max_count);
1198 13536195 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1199 : num.m_quality), den.m_quality);
1200 : /* Be sure that ret is not local if num is global.
1201 : Also ensure that ret is not global0 when num is global. */
1202 13536195 : if (num.ipa_p ())
1203 : {
1204 : /* This is common case of AFDO scaling when we upgrade
1205 : GLOBAL0_AFDO function to AFDO. Be sure that result
1206 : is AFDO and not GUESSED (which is unnecesarily low). */
1207 1817 : if (num.m_quality == AFDO
1208 0 : && (ret.m_quality != GUESSED
1209 0 : && ret.m_quality != GUESSED_LOCAL))
1210 0 : ret.m_quality = AFDO;
1211 : else
1212 3665 : ret.m_quality = MAX (ret.m_quality,
1213 : num == num.ipa () ? GUESSED : num.m_quality);
1214 : }
1215 13536195 : return ret;
1216 : }
1217 :
1218 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1219 17926296 : profile_count guessed_local () const
1220 : {
1221 17926296 : profile_count ret = *this;
1222 17926296 : if (!initialized_p ())
1223 0 : return *this;
1224 17926296 : ret.m_quality = GUESSED_LOCAL;
1225 17926296 : return ret;
1226 : }
1227 :
1228 : /* We know that profile is globally 0 but keep local profile if present. */
1229 792 : profile_count global0 () const
1230 : {
1231 792 : profile_count ret = *this;
1232 792 : if (!initialized_p ())
1233 0 : return *this;
1234 792 : ret.m_quality = GUESSED_GLOBAL0;
1235 792 : return ret;
1236 : }
1237 :
1238 : /* We know that profile is globally afdo 0 but keep local profile
1239 : if present. */
1240 0 : profile_count global0afdo () const
1241 : {
1242 0 : profile_count ret = *this;
1243 0 : if (!initialized_p ())
1244 0 : return *this;
1245 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1246 0 : return ret;
1247 : }
1248 :
1249 : /* We know that profile is globally adjusted 0 but keep local profile
1250 : if present. */
1251 36 : profile_count global0adjusted () const
1252 : {
1253 36 : profile_count ret = *this;
1254 36 : if (!initialized_p ())
1255 0 : return *this;
1256 36 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1257 36 : return ret;
1258 : }
1259 :
1260 : /* Return THIS with quality dropped to GUESSED. */
1261 403 : profile_count guessed () const
1262 : {
1263 403 : profile_count ret = *this;
1264 403 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1265 403 : return ret;
1266 : }
1267 :
1268 : /* Return THIS with quality GUESSED. */
1269 0 : profile_count force_guessed () const
1270 : {
1271 0 : profile_count ret = *this;
1272 0 : gcc_checking_assert (initialized_p ());
1273 0 : ret.m_quality = GUESSED;
1274 0 : return ret;
1275 : }
1276 :
1277 : /* Return variant of profile count which is always safe to compare
1278 : across functions. */
1279 9024707891 : profile_count ipa () const
1280 : {
1281 9024707891 : if (m_quality > GUESSED_GLOBAL0)
1282 14304934 : return *this;
1283 9010402957 : if (m_quality == GUESSED_GLOBAL0)
1284 211928 : return zero ();
1285 9010191029 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1286 10545 : return adjusted_zero ();
1287 9010180484 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1288 0 : return afdo_zero ();
1289 9010180484 : return uninitialized ();
1290 : }
1291 :
1292 : /* Return THIS with quality dropped to AFDO. */
1293 0 : profile_count afdo () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : ret.m_quality = AFDO;
1297 0 : return ret;
1298 : }
1299 :
1300 : /* Return probability of event with counter THIS within event with counter
1301 : OVERALL. */
1302 1074549622 : profile_probability probability_in (const profile_count overall) const
1303 : {
1304 1074566630 : if (*this == zero ()
1305 17008 : && !(overall == zero ()))
1306 1556 : return profile_probability::never ();
1307 1074547505 : if (!initialized_p () || !overall.initialized_p ()
1308 2149095571 : || !overall.m_val)
1309 63747 : return profile_probability::uninitialized ();
1310 1284297774 : if (*this == overall && m_quality == PRECISE)
1311 27681 : return profile_probability::always ();
1312 1074456638 : profile_probability ret;
1313 1074456638 : gcc_checking_assert (compatible_p (overall));
1314 :
1315 1074456638 : if (overall.m_val < m_val)
1316 : {
1317 171835 : ret.m_val = profile_probability::max_probability;
1318 171835 : ret.set_quality (GUESSED);
1319 171835 : return ret;
1320 : }
1321 : else
1322 : {
1323 1074284803 : gcc_checking_assert (overall.m_val);
1324 1074284803 : uint64_t tmp;
1325 1074284803 : safe_scale_64bit (m_val, profile_probability::max_probability,
1326 : overall.m_val, &tmp);
1327 1074284803 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1328 1074284803 : ret.m_val = tmp;
1329 : }
1330 1074284803 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1331 : GUESSED), ADJUSTED));
1332 1074284803 : return ret;
1333 : }
1334 :
1335 : /* Return true if profile count is very large, so we risk overflows
1336 : with loop transformations. */
1337 : bool
1338 1034833 : very_large_p ()
1339 : {
1340 1034833 : if (!initialized_p ())
1341 : return false;
1342 1034833 : return m_val > max_count / 65536;
1343 : }
1344 :
1345 : int to_frequency (struct function *fun) const;
1346 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1347 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1348 :
1349 : /* Output THIS to F. */
1350 : void dump (FILE *f, struct function *fun = NULL) const;
1351 :
1352 : /* Print THIS to stderr. */
1353 : void debug () const;
1354 :
1355 : /* Return true if THIS is known to differ significantly from OTHER. */
1356 : bool differs_from_p (profile_count other) const;
1357 :
1358 : /* We want to scale profile across function boundary from NUM to DEN.
1359 : Take care of the side case when NUM and DEN are zeros of incompatible
1360 : kinds. */
1361 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1362 :
1363 : /* THIS is a count of bb which is known to be executed IPA times.
1364 : Combine this information into bb counter. This means returning IPA
1365 : if it is nonzero, not changing anything if IPA is uninitialized
1366 : and if IPA is zero, turning THIS into corresponding local profile with
1367 : global0. */
1368 : profile_count combine_with_ipa_count (profile_count ipa);
1369 :
1370 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1371 : profile_count combine_with_ipa_count_within
1372 : (profile_count ipa, profile_count ipa2);
1373 :
1374 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1375 : Conversions back and forth are used to read the coverage and get it
1376 : into internal representation. */
1377 : static profile_count from_gcov_type (gcov_type v,
1378 : profile_quality quality = PRECISE);
1379 :
1380 : /* LTO streaming support. */
1381 : static profile_count stream_in (class lto_input_block *);
1382 : void stream_out (struct output_block *);
1383 : void stream_out (struct lto_output_stream *);
1384 : };
1385 : #endif
|
