Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2332715966 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2332715966 : uint64_t tmp;
88 2332715966 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2332715966 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2330436905 : *res = tmp / c;
92 2330436905 : return true;
93 : }
94 2279061 : if (c == 1)
95 : {
96 35466 : *res = (uint64_t) -1;
97 35466 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2243595 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greated then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5373651126 : set_quality (profile_quality quality)
169 : {
170 1133747746 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1133747746 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1132024773 : }
174 :
175 : public:
176 4241626353 : profile_probability (): m_val (uninitialized_probability)
177 2328475154 : { set_quality (GUESSED); }
178 :
179 384 : profile_probability (uint32_t val, profile_quality quality):
180 384 : m_val (val)
181 384 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 743725800 : static profile_probability never ()
185 : {
186 743725800 : profile_probability ret;
187 743725800 : ret.m_val = 0;
188 743725800 : ret.set_quality (PRECISE);
189 442349430 : return ret;
190 : }
191 :
192 5301552 : static profile_probability guessed_never ()
193 : {
194 5301552 : profile_probability ret;
195 5301552 : ret.m_val = 0;
196 5301552 : ret.set_quality (GUESSED);
197 5301552 : return ret;
198 : }
199 :
200 5209954 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10419908 : profile_probability r = guessed_always () / 2000;
204 5209954 : r.m_val--;
205 5209954 : return r;
206 : }
207 :
208 84846 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 169692 : profile_probability r = guessed_always () / 5;
212 84846 : r.m_val--;
213 84846 : return r;
214 : }
215 :
216 731062 : static profile_probability even ()
217 : {
218 1462124 : return guessed_always () / 2;
219 : }
220 :
221 339083 : static profile_probability very_likely ()
222 : {
223 339083 : return always () - very_unlikely ();
224 : }
225 :
226 32559 : static profile_probability likely ()
227 : {
228 32559 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 2164 : bool nonzero_p () const
232 : {
233 2164 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6335247 : static profile_probability guessed_always ()
237 : {
238 6335247 : profile_probability ret;
239 6335247 : ret.m_val = max_probability;
240 6335247 : ret.set_quality (GUESSED);
241 6140947 : return ret;
242 : }
243 :
244 551049743 : static profile_probability always ()
245 : {
246 551049743 : profile_probability ret;
247 551049743 : ret.m_val = max_probability;
248 551049743 : ret.set_quality (PRECISE);
249 373621753 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 908271616 : static profile_probability uninitialized ()
255 : {
256 908271616 : profile_probability c;
257 908271616 : c.m_val = uninitialized_probability;
258 908271616 : c.set_quality (GUESSED);
259 894740013 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5642019635 : bool initialized_p () const
264 : {
265 3929024042 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1533816 : bool reliable_p () const
270 : {
271 124184690 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3179483 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3179483 : profile_probability ret;
279 3179483 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3179483 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3179483 : ret.set_quality (GUESSED);
282 3179483 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 42 : profile_probability adjusted () const
287 : {
288 42 : profile_probability ret = *this;
289 42 : if (!initialized_p ())
290 0 : return *this;
291 42 : ret.set_quality (ADJUSTED);
292 42 : return ret;
293 : }
294 :
295 1114930689 : int to_reg_br_prob_base () const
296 : {
297 1114930689 : gcc_checking_assert (initialized_p ());
298 1114930689 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 495677679 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 495677679 : profile_probability ret;
305 495677679 : ret.m_val = ((unsigned int)v) / 8;
306 495677679 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 15946682 : return ret;
308 : }
309 :
310 466412380 : int to_reg_br_prob_note () const
311 : {
312 466412380 : gcc_checking_assert (initialized_p ());
313 466412380 : int ret = m_val * 8 + m_adjusted_quality;
314 466412380 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 466412380 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3094 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3094 : profile_probability ret;
323 3094 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3094 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3094 : uint64_t tmp;
329 3094 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3094 : gcc_checking_assert (tmp <= max_probability);
331 3094 : ret.m_val = tmp;
332 : }
333 3094 : ret.set_quality (PRECISE);
334 3094 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1628522982 : bool operator== (const profile_probability &other) const
339 : {
340 2299877687 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6686070 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6686070 : if (other == never ())
346 32058 : return *this;
347 6654012 : if (*this == never ())
348 23522 : return other;
349 6630490 : if (!initialized_p () || !other.initialized_p ())
350 5451569 : return uninitialized ();
351 :
352 1178921 : profile_probability ret;
353 1178921 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1178921 : ret.set_quality (MIN (quality (), other.quality ()));
355 1178921 : return ret;
356 : }
357 :
358 6363379 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6363379 : if (other == never ())
361 : return *this;
362 5132168 : if (*this == never ())
363 : {
364 875371 : *this = other;
365 875371 : return *this;
366 : }
367 4256797 : if (!initialized_p () || !other.initialized_p ())
368 2956548 : return *this = uninitialized ();
369 : else
370 : {
371 1300249 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 1300249 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1300249 : return *this;
375 : }
376 :
377 34190696 : profile_probability operator- (const profile_probability &other) const
378 : {
379 68381392 : if (*this == never ()
380 34190696 : || other == never ())
381 1849844 : return *this;
382 32340852 : if (!initialized_p () || !other.initialized_p ())
383 2160270 : return uninitialized ();
384 30180582 : profile_probability ret;
385 30180582 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30180582 : ret.set_quality (MIN (quality (), other.quality ()));
387 30180582 : return ret;
388 : }
389 :
390 11374324 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 22748648 : if (*this == never ()
393 11374324 : || other == never ())
394 2245455 : return *this;
395 9128869 : if (!initialized_p () || !other.initialized_p ())
396 2133808 : return *this = uninitialized ();
397 : else
398 : {
399 6995061 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 6995061 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 6995061 : return *this;
403 : }
404 :
405 1159424 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2318848 : if (*this == never ()
408 1159424 : || other == never ())
409 121226 : return never ();
410 1038198 : if (!initialized_p () || !other.initialized_p ())
411 58137 : return uninitialized ();
412 980061 : profile_probability ret;
413 980061 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 980061 : ret.set_quality (MIN (quality (), other.quality ()));
415 980061 : return ret;
416 : }
417 :
418 141569 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 283138 : if (*this == never ()
421 141569 : || other == never ())
422 395 : return *this = never ();
423 141174 : if (!initialized_p () || !other.initialized_p ())
424 9 : return *this = uninitialized ();
425 : else
426 : {
427 141165 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 264379 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 141165 : return *this;
431 : }
432 :
433 1779424 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1779424 : if (*this == never ())
436 104795 : return never ();
437 1674629 : if (!initialized_p () || !other.initialized_p ())
438 225261 : return uninitialized ();
439 1449368 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1449368 : if (m_val >= other.m_val)
442 : {
443 43914 : ret.m_val = max_probability;
444 43914 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 43914 : return ret;
447 : }
448 1405454 : else if (!m_val)
449 61774 : ret.m_val = 0;
450 : else
451 : {
452 1343680 : gcc_checking_assert (other.m_val);
453 1343680 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2802822 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1405454 : return ret;
457 : }
458 :
459 337913 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 337913 : if (*this == never ())
462 1503 : return *this = never ();
463 336410 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 336410 : if (m_val > other.m_val)
470 : {
471 686 : m_val = max_probability;
472 686 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 686 : return *this;
475 : }
476 335724 : else if (!m_val)
477 : ;
478 : else
479 : {
480 332309 : gcc_checking_assert (other.m_val);
481 332309 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 613753 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 335724 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 281830 : profile_probability split (const profile_probability &cprob)
504 : {
505 281830 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 281830 : if (!(*this == always ()))
512 273476 : *this = (*this - ret) / ret.invert ();
513 281830 : return ret;
514 : }
515 :
516 262638 : gcov_type apply (gcov_type val) const
517 : {
518 262638 : if (*this == uninitialized ())
519 54 : return val / 2;
520 262584 : return RDIV (val * m_val, max_probability);
521 : }
522 :
523 : /* Return 1-*THIS. */
524 21338716 : profile_probability invert () const
525 : {
526 30927910 : return always() - *this;
527 : }
528 :
529 : /* Return THIS with quality dropped to GUESSED. */
530 840695 : profile_probability guessed () const
531 : {
532 840695 : profile_probability ret = *this;
533 840695 : ret.set_quality (GUESSED);
534 840695 : return ret;
535 : }
536 :
537 : /* Return THIS with quality dropped to AFDO. */
538 : profile_probability afdo () const
539 : {
540 : profile_probability ret = *this;
541 : ret.set_quality (AFDO);
542 : return ret;
543 : }
544 :
545 : /* Return *THIS * NUM / DEN. */
546 14468350 : profile_probability apply_scale (int64_t num, int64_t den) const
547 : {
548 14468350 : if (*this == never ())
549 16067 : return *this;
550 14452283 : if (!initialized_p ())
551 5863342 : return uninitialized ();
552 8588941 : profile_probability ret;
553 8588941 : uint64_t tmp;
554 8588941 : safe_scale_64bit (m_val, num, den, &tmp);
555 8588941 : ret.m_val = MIN (tmp, max_probability);
556 8588941 : ret.set_quality (MIN (quality (), ADJUSTED));
557 8588941 : return ret;
558 : }
559 :
560 : /* Return *THIS * NUM / DEN. */
561 11622 : profile_probability apply_scale (profile_probability num,
562 : profile_probability den) const
563 : {
564 11622 : if (*this == never ())
565 56 : return *this;
566 11566 : if (num == never ())
567 0 : return num;
568 11566 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
569 0 : return uninitialized ();
570 11566 : if (num == den)
571 0 : return *this;
572 11566 : gcc_checking_assert (den.m_val);
573 :
574 11566 : profile_probability ret;
575 11566 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
576 : max_probability);
577 11566 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
578 : num.quality ()), den.quality ()));
579 11566 : return ret;
580 : }
581 :
582 : /* Return true when the probability of edge is reliable.
583 :
584 : The profile guessing code is good at predicting branch outcome (i.e.
585 : taken/not taken), that is predicted right slightly over 75% of time.
586 : It is however notoriously poor on predicting the probability itself.
587 : In general the profile appear a lot flatter (with probabilities closer
588 : to 50%) than the reality so it is bad idea to use it to drive optimization
589 : such as those disabling dynamic branch prediction for well predictable
590 : branches.
591 :
592 : There are two exceptions - edges leading to noreturn edges and edges
593 : predicted by number of iterations heuristics are predicted well. This macro
594 : should be able to distinguish those, but at the moment it simply check for
595 : noreturn heuristic that is only one giving probability over 99% or below
596 : 1%. In future we might want to propagate reliability information across the
597 : CFG if we find this information useful on multiple places. */
598 0 : bool probably_reliable_p () const
599 : {
600 0 : if (quality () >= ADJUSTED)
601 : return true;
602 0 : if (!initialized_p ())
603 : return false;
604 0 : return m_val < max_probability / 100
605 0 : || m_val > max_probability - max_probability / 100;
606 : }
607 :
608 : /* Return false if profile_probability is bogus. */
609 4315785422 : bool verify () const
610 : {
611 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
612 4315785422 : if (m_val == uninitialized_probability)
613 931360371 : return quality () == GUESSED;
614 3384425051 : else if (quality () < GUESSED)
615 : return false;
616 3384425051 : return m_val <= max_probability;
617 : }
618 :
619 : /* Comparisons are three-state and conservative. False is returned if
620 : the inequality cannot be decided. */
621 8209524 : bool operator< (const profile_probability &other) const
622 : {
623 8209524 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
624 : }
625 :
626 17689893 : bool operator> (const profile_probability &other) const
627 : {
628 17689893 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
629 : }
630 :
631 5350799 : bool operator<= (const profile_probability &other) const
632 : {
633 5350799 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
634 : }
635 :
636 981646 : bool operator>= (const profile_probability &other) const
637 : {
638 981646 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
639 : }
640 :
641 17 : profile_probability operator* (int64_t num) const
642 : {
643 17 : return apply_scale (num, 1);
644 : }
645 :
646 : profile_probability operator*= (int64_t num)
647 : {
648 : *this = apply_scale (num, 1);
649 : return *this;
650 : }
651 :
652 13829510 : profile_probability operator/ (int64_t den) const
653 : {
654 13829510 : return apply_scale (1, den);
655 : }
656 :
657 242249 : profile_probability operator/= (int64_t den)
658 : {
659 242249 : *this = apply_scale (1, den);
660 242249 : return *this;
661 : }
662 :
663 : /* Compute n-th power. */
664 : profile_probability pow (int) const;
665 :
666 : /* Compute sware root. */
667 : profile_probability sqrt () const;
668 :
669 : /* Get the value of the probability. */
670 0 : uint32_t value () const { return m_val; }
671 :
672 : /* Get the quality of the probability. */
673 6021848366 : enum profile_quality quality () const
674 : {
675 5807911691 : return (profile_quality) (m_adjusted_quality
676 5807933513 : ? m_adjusted_quality + min_quality - 1
677 : : UNINITIALIZED_PROFILE);
678 : }
679 :
680 : /* Output THIS to F. */
681 : void dump (FILE *f) const;
682 :
683 : /* Output THIS to BUFFER. */
684 : void dump (char *buffer) const;
685 :
686 : /* Print THIS to stderr. */
687 : void debug () const;
688 :
689 : /* Return true if THIS is known to differ significantly from OTHER. */
690 : bool differs_from_p (profile_probability other) const;
691 :
692 : /* Return if difference is greater than 50%. */
693 : bool differs_lot_from_p (profile_probability other) const;
694 :
695 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
696 : happens with COUNT2 probability. Return probability that either *THIS or
697 : OTHER happens. */
698 : profile_probability combine_with_count (profile_count count1,
699 : profile_probability other,
700 : profile_count count2) const;
701 :
702 : /* Return probability as sreal. */
703 : sreal to_sreal () const;
704 : /* LTO streaming support. */
705 : static profile_probability stream_in (class lto_input_block *);
706 : void stream_out (struct output_block *);
707 : void stream_out (struct lto_output_stream *);
708 : };
709 :
710 : /* Main data type to hold profile counters in GCC. Profile counts originate
711 : either from profile feedback, static profile estimation or both. We do not
712 : perform whole program profile propagation and thus profile estimation
713 : counters are often local to function, while counters from profile feedback
714 : (or special cases of profile estimation) can be used inter-procedurally.
715 :
716 : There are 3 basic types
717 : 1) local counters which are result of intra-procedural static profile
718 : estimation.
719 : 2) ipa counters which are result of profile feedback or special case
720 : of static profile estimation (such as in function main).
721 : 3) counters which counts as 0 inter-procedurally (because given function
722 : was never run in train feedback) but they hold local static profile
723 : estimate.
724 :
725 : Counters of type 1 and 3 cannot be mixed with counters of different type
726 : within operation (because whole function should use one type of counter)
727 : with exception that global zero mix in most operations where outcome is
728 : well defined.
729 :
730 : To take local counter and use it inter-procedurally use ipa member function
731 : which strips information irrelevant at the inter-procedural level.
732 :
733 : Counters are 61bit integers representing number of executions during the
734 : train run or normalized frequency within the function.
735 :
736 : As the profile is maintained during the compilation, many adjustments are
737 : made. Not all transformations can be made precisely, most importantly
738 : when code is being duplicated. It also may happen that part of CFG has
739 : profile counts known while other do not - for example when LTO optimizing
740 : partly profiled program or when profile was lost due to COMDAT merging.
741 :
742 : For this reason profile_count tracks more information than
743 : just unsigned integer and it is also ready for profile mismatches.
744 : The API of this data type represent operations that are natural
745 : on profile counts - sum, difference and operation with scales and
746 : probabilities. All operations are safe by never getting negative counts
747 : and they do end up in uninitialized scale if any of the parameters is
748 : uninitialized.
749 :
750 : All comparisons that are three state and handling of probabilities. Thus
751 : a < b is not equal to !(a >= b).
752 :
753 : The following pre-defined counts are available:
754 :
755 : profile_count::zero () for code that is known to execute zero times at
756 : runtime (this can be detected statically i.e. for paths leading to
757 : abort ();
758 : profile_count::one () for code that is known to execute once (such as
759 : main () function
760 : profile_count::uninitialized () for unknown execution count.
761 :
762 : */
763 :
764 : struct GTY(()) profile_count
765 : {
766 : public:
767 : /* Use 60bit to hold basic block counters. Should be at least
768 : 64bit. Although a counter cannot be negative, we use a signed
769 : type to hold various extra stages. */
770 :
771 : static const int n_bits = 60;
772 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
773 : private:
774 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
775 :
776 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
777 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
778 : incorrectly detects the alignment of a structure where the only
779 : 64-bit aligned object is a bit-field. We force the alignment of
780 : the entire field to mitigate this. */
781 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
782 : #else
783 : #define UINT64_BIT_FIELD_ALIGN
784 : #endif
785 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
786 : #undef UINT64_BIT_FIELD_ALIGN
787 : enum profile_quality m_quality : 4;
788 : public:
789 :
790 : /* Return true if both values can meaningfully appear in single function
791 : body. We have either all counters in function local or global, otherwise
792 : operations between them are not really defined well. */
793 4886466778 : bool compatible_p (const profile_count other) const
794 : {
795 8988197872 : if (!initialized_p () || !other.initialized_p ())
796 : return true;
797 16246280644 : if (*this == zero ()
798 8051164298 : || other == zero ())
799 147264881 : return true;
800 : /* Do not allow nonzero global profile together with local guesses
801 : that are globally0. */
802 3950293292 : if (ipa ().nonzero_p ()
803 244358 : && !(other.ipa () == other))
804 0 : return false;
805 3950293292 : if (other.ipa ().nonzero_p ()
806 253622 : && !(ipa () == *this))
807 0 : return false;
808 :
809 3950293292 : return ipa_p () == other.ipa_p ();
810 : }
811 :
812 : /* Used for counters which are expected to be never executed. */
813 18379034851 : static profile_count zero ()
814 : {
815 14373474559 : return from_gcov_type (0);
816 : }
817 :
818 10541 : static profile_count adjusted_zero ()
819 : {
820 10541 : profile_count c;
821 10541 : c.m_val = 0;
822 10541 : c.m_quality = ADJUSTED;
823 10541 : return c;
824 : }
825 :
826 0 : static profile_count afdo_zero ()
827 : {
828 0 : profile_count c;
829 0 : c.m_val = 0;
830 0 : c.m_quality = AFDO;
831 0 : return c;
832 : }
833 :
834 : static profile_count guessed_zero ()
835 : {
836 : profile_count c;
837 : c.m_val = 0;
838 : c.m_quality = GUESSED;
839 : return c;
840 : }
841 :
842 47637 : static profile_count one ()
843 : {
844 47424 : return from_gcov_type (1);
845 : }
846 :
847 : /* Value of counters which has not been initialized. Either because
848 : initialization did not happen yet or because profile is unknown. */
849 9187119973 : static profile_count uninitialized ()
850 : {
851 9187119973 : profile_count c;
852 9187119973 : c.m_val = uninitialized_count;
853 9187119973 : c.m_quality = GUESSED_LOCAL;
854 9156004583 : return c;
855 : }
856 :
857 : /* Conversion to gcov_type is lossy. */
858 42296392 : gcov_type to_gcov_type () const
859 : {
860 0 : gcc_checking_assert (initialized_p ());
861 42296392 : return m_val;
862 : }
863 :
864 : /* Return true if value has been initialized. */
865 32403814508 : bool initialized_p () const
866 : {
867 10855873626 : return m_val != uninitialized_count;
868 : }
869 :
870 : /* Return true if value can be trusted. */
871 26181050 : bool reliable_p () const
872 : {
873 26180781 : return m_quality >= ADJUSTED;
874 : }
875 :
876 : /* Return true if value can be operated inter-procedurally. */
877 8950018966 : bool ipa_p () const
878 : {
879 4986765117 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
880 : }
881 :
882 : /* Return true if quality of profile is precise. */
883 55565372 : bool precise_p () const
884 : {
885 55565372 : return m_quality == PRECISE;
886 : }
887 :
888 : /* Get the value of the count. */
889 0 : uint64_t value () const { return m_val; }
890 :
891 : /* Get the quality of the count. */
892 95306793 : enum profile_quality quality () const { return m_quality; }
893 :
894 : /* When merging basic blocks, the two different profile counts are unified.
895 : Return true if this can be done without losing info about profile.
896 : The only case we care about here is when first BB contains something
897 : that makes it terminate in a way not visible in CFG. */
898 3217129 : bool ok_for_merging (profile_count other) const
899 : {
900 3217129 : if (m_quality < ADJUSTED
901 15853 : || other.m_quality < ADJUSTED)
902 : return true;
903 14885 : return !(other < *this);
904 : }
905 :
906 : /* When merging two BBs with different counts, pick common count that looks
907 : most representative. */
908 16607904 : profile_count merge (profile_count other) const
909 : {
910 826569 : if (*this == other || !other.initialized_p ()
911 826284 : || m_quality > other.m_quality)
912 15782826 : return *this;
913 825078 : if (other.m_quality > m_quality
914 825078 : || other > *this)
915 0 : return other;
916 825078 : return *this;
917 : }
918 :
919 : /* Basic operations. */
920 19822023079 : bool operator== (const profile_count &other) const
921 : {
922 15061867514 : return m_val == other.m_val && m_quality == other.m_quality;
923 : }
924 :
925 3819057 : profile_count operator+ (const profile_count &other) const
926 : {
927 3819057 : if (other == zero ())
928 2392 : return *this;
929 3816665 : if (*this == zero ())
930 208824 : return other;
931 3607841 : if (!initialized_p () || !other.initialized_p ())
932 6345 : return uninitialized ();
933 :
934 3601496 : profile_count ret;
935 3601496 : gcc_checking_assert (compatible_p (other));
936 3601496 : uint64_t ret_val = m_val + other.m_val;
937 3601496 : ret.m_val = MIN (ret_val, max_count);
938 3601496 : ret.m_quality = MIN (m_quality, other.m_quality);
939 3601496 : return ret;
940 : }
941 :
942 73962972 : profile_count &operator+= (const profile_count &other)
943 : {
944 73962972 : if (other == zero ())
945 5571245 : return *this;
946 68391727 : if (*this == zero ())
947 : {
948 41814160 : *this = other;
949 41814160 : return *this;
950 : }
951 26577567 : if (!initialized_p () || !other.initialized_p ())
952 350959 : return *this = uninitialized ();
953 : else
954 : {
955 26226608 : gcc_checking_assert (compatible_p (other));
956 26226608 : uint64_t ret_val = m_val + other.m_val;
957 26226608 : m_val = MIN (ret_val, max_count);
958 26226608 : m_quality = MIN (m_quality, other.m_quality);
959 : }
960 26226608 : return *this;
961 : }
962 :
963 19969554 : profile_count operator- (const profile_count &other) const
964 : {
965 19986529 : if (*this == zero () || other == zero ())
966 1332841 : return *this;
967 18636713 : if (!initialized_p () || !other.initialized_p ())
968 8040866 : return uninitialized ();
969 10595847 : gcc_checking_assert (compatible_p (other));
970 10595847 : profile_count ret;
971 10595847 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
972 10595847 : ret.m_quality = MIN (m_quality, other.m_quality);
973 10595847 : return ret;
974 : }
975 :
976 9063590 : profile_count &operator-= (const profile_count &other)
977 : {
978 9510843 : if (*this == zero () || other == zero ())
979 281867 : return *this;
980 8781723 : if (!initialized_p () || !other.initialized_p ())
981 349511 : return *this = uninitialized ();
982 : else
983 : {
984 8432212 : gcc_checking_assert (compatible_p (other));
985 8432212 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
986 8432212 : m_quality = MIN (m_quality, other.m_quality);
987 : }
988 8432212 : return *this;
989 : }
990 :
991 : /* Return false if profile_count is bogus. */
992 3289579952 : bool verify () const
993 : {
994 3289579952 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
995 3289579952 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
996 : }
997 :
998 : /* Comparisons are three-state and conservative. False is returned if
999 : the inequality cannot be decided. */
1000 1098754845 : bool operator< (const profile_count &other) const
1001 : {
1002 1098754845 : if (!initialized_p () || !other.initialized_p ())
1003 : return false;
1004 1093998359 : if (*this == zero ())
1005 4098144 : return !(other == zero ());
1006 1091949287 : if (other == zero ())
1007 875 : return false;
1008 1091948412 : gcc_checking_assert (compatible_p (other));
1009 1091948412 : return m_val < other.m_val;
1010 : }
1011 :
1012 38461576 : bool operator> (const profile_count &other) const
1013 : {
1014 38461576 : if (!initialized_p () || !other.initialized_p ())
1015 : return false;
1016 33661729 : if (*this == zero ())
1017 221885 : return false;
1018 33439844 : if (other == zero ())
1019 1516426 : return !(*this == zero ());
1020 32681631 : gcc_checking_assert (compatible_p (other));
1021 32681631 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1022 : }
1023 :
1024 : bool operator< (const gcov_type other) const
1025 : {
1026 : gcc_checking_assert (ipa_p ());
1027 : gcc_checking_assert (other >= 0);
1028 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1029 : }
1030 :
1031 7587 : bool operator> (const gcov_type other) const
1032 : {
1033 7587 : gcc_checking_assert (ipa_p ());
1034 7587 : gcc_checking_assert (other >= 0);
1035 7587 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1036 : }
1037 :
1038 1172339 : bool operator<= (const profile_count &other) const
1039 : {
1040 1172339 : if (!initialized_p () || !other.initialized_p ())
1041 : return false;
1042 1172339 : if (*this == zero ())
1043 4015 : return true;
1044 1168324 : if (other == zero ())
1045 0 : return (*this == zero ());
1046 1168324 : gcc_checking_assert (compatible_p (other));
1047 1168324 : return m_val <= other.m_val;
1048 : }
1049 :
1050 1419853 : bool operator>= (const profile_count &other) const
1051 : {
1052 1419853 : if (!initialized_p () || !other.initialized_p ())
1053 : return false;
1054 1418748 : if (other == zero ())
1055 2746 : return true;
1056 1416002 : if (*this == zero ())
1057 20682 : return (other == zero ());
1058 1405661 : gcc_checking_assert (compatible_p (other));
1059 1405661 : return m_val >= other.m_val;
1060 : }
1061 :
1062 97326 : bool operator<= (const gcov_type other) const
1063 : {
1064 97326 : gcc_checking_assert (ipa_p ());
1065 97326 : gcc_checking_assert (other >= 0);
1066 97326 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1067 : }
1068 :
1069 60406 : bool operator>= (const gcov_type other) const
1070 : {
1071 60406 : gcc_checking_assert (ipa_p ());
1072 60406 : gcc_checking_assert (other >= 0);
1073 60406 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1074 : }
1075 :
1076 918625278 : profile_count operator* (int64_t num) const
1077 : {
1078 918543102 : return apply_scale (num, 1);
1079 : }
1080 :
1081 : profile_count operator*= (int64_t num)
1082 : {
1083 : *this = apply_scale (num, 1);
1084 : return *this;
1085 : }
1086 :
1087 : profile_count operator* (const sreal &num) const;
1088 : profile_count operator*= (const sreal &num);
1089 :
1090 2351253 : profile_count operator/ (int64_t den) const
1091 : {
1092 2351253 : return apply_scale (1, den);
1093 : }
1094 :
1095 8 : profile_count operator/= (int64_t den)
1096 : {
1097 0 : *this = apply_scale (1, den);
1098 8 : return *this;
1099 : }
1100 :
1101 : /* Return true when value is not zero and can be used for scaling.
1102 : This is different from *this > 0 because that requires counter to
1103 : be IPA. */
1104 8097315873 : bool nonzero_p () const
1105 : {
1106 8097315226 : return initialized_p () && m_val != 0;
1107 : }
1108 :
1109 : /* Make counter forcibly nonzero. */
1110 : profile_count force_nonzero () const;
1111 :
1112 :
1113 : /* Return maximum of A and B. If one of values is uninitialized return the
1114 : other. */
1115 :
1116 : static profile_count
1117 78254386 : max_prefer_initialized (const profile_count a, const profile_count b)
1118 : {
1119 78254386 : if (!a.initialized_p ())
1120 19482693 : return b;
1121 58771693 : if (!b.initialized_p ())
1122 1868217 : return a;
1123 56903476 : profile_count ret;
1124 56903476 : gcc_checking_assert (a.compatible_p (b));
1125 56903476 : ret.m_val = MAX (a.m_val, b.m_val);
1126 56903476 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1127 56903476 : return ret;
1128 : }
1129 :
1130 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1131 : accordingly. */
1132 : profile_count apply_probability (int prob) const
1133 : {
1134 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1135 : if (m_val == 0)
1136 : return *this;
1137 : if (!initialized_p ())
1138 : return uninitialized ();
1139 : profile_count ret;
1140 : uint64_t tmp;
1141 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1142 : ret.m_val = tmp;
1143 : ret.m_quality = MIN (m_quality, ADJUSTED);
1144 : return ret;
1145 : }
1146 :
1147 : /* Scale counter according to PROB. */
1148 389249592 : profile_count apply_probability (profile_probability prob) const
1149 : {
1150 398000883 : if (*this == zero () || prob == profile_probability::always ())
1151 125048511 : return *this;
1152 264201081 : if (prob == profile_probability::never ())
1153 20109879 : return zero ();
1154 244091202 : if (!initialized_p () || !prob.initialized_p ())
1155 30176349 : return uninitialized ();
1156 213914853 : profile_count ret;
1157 213914853 : uint64_t tmp;
1158 213914853 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1159 : &tmp);
1160 213914853 : ret.m_val = tmp;
1161 213936675 : ret.m_quality = MIN (m_quality, prob.quality ());
1162 213914853 : return ret;
1163 : }
1164 :
1165 : /* Return *THIS * NUM / DEN. */
1166 1043661727 : profile_count apply_scale (int64_t num, int64_t den) const
1167 : {
1168 1043661727 : if (m_val == 0)
1169 24233687 : return *this;
1170 1019428040 : if (!initialized_p ())
1171 3900679 : return uninitialized ();
1172 1015527361 : profile_count ret;
1173 1015527361 : uint64_t tmp;
1174 :
1175 1015527361 : gcc_checking_assert (num >= 0 && den > 0);
1176 1015527361 : safe_scale_64bit (m_val, num, den, &tmp);
1177 1015527361 : ret.m_val = MIN (tmp, max_count);
1178 1015527361 : ret.m_quality = MIN (m_quality, ADJUSTED);
1179 1015527361 : return ret;
1180 : }
1181 :
1182 28150676 : profile_count apply_scale (profile_count num, profile_count den) const
1183 : {
1184 28150676 : if (*this == zero ())
1185 2393881 : return *this;
1186 25756795 : if (num == zero ())
1187 83594 : return num;
1188 25673201 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1189 6940021 : return uninitialized ();
1190 18733180 : if (num == den)
1191 5421418 : return *this;
1192 13311762 : gcc_checking_assert (den.m_val);
1193 :
1194 13311762 : profile_count ret;
1195 13311762 : uint64_t val;
1196 13311762 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1197 13311762 : ret.m_val = MIN (val, max_count);
1198 13311762 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1199 : num.m_quality), den.m_quality);
1200 : /* Be sure that ret is not local if num is global.
1201 : Also ensure that ret is not global0 when num is global. */
1202 13311762 : if (num.ipa_p ())
1203 : {
1204 : /* This is common case of AFDO scaling when we upgrade
1205 : GLOBAL0_AFDO function to AFDO. Be sure that result
1206 : is AFDO and not GUESSED (which is unnecesarily low). */
1207 1817 : if (num.m_quality == AFDO
1208 0 : && (ret.m_quality != GUESSED
1209 0 : && ret.m_quality != GUESSED_LOCAL))
1210 0 : ret.m_quality = AFDO;
1211 : else
1212 3665 : ret.m_quality = MAX (ret.m_quality,
1213 : num == num.ipa () ? GUESSED : num.m_quality);
1214 : }
1215 13311762 : return ret;
1216 : }
1217 :
1218 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1219 17843282 : profile_count guessed_local () const
1220 : {
1221 17843282 : profile_count ret = *this;
1222 17843282 : if (!initialized_p ())
1223 0 : return *this;
1224 17843282 : ret.m_quality = GUESSED_LOCAL;
1225 17843282 : return ret;
1226 : }
1227 :
1228 : /* We know that profile is globally 0 but keep local profile if present. */
1229 789 : profile_count global0 () const
1230 : {
1231 789 : profile_count ret = *this;
1232 789 : if (!initialized_p ())
1233 0 : return *this;
1234 789 : ret.m_quality = GUESSED_GLOBAL0;
1235 789 : return ret;
1236 : }
1237 :
1238 : /* We know that profile is globally afdo 0 but keep local profile
1239 : if present. */
1240 0 : profile_count global0afdo () const
1241 : {
1242 0 : profile_count ret = *this;
1243 0 : if (!initialized_p ())
1244 0 : return *this;
1245 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1246 0 : return ret;
1247 : }
1248 :
1249 : /* We know that profile is globally adjusted 0 but keep local profile
1250 : if present. */
1251 36 : profile_count global0adjusted () const
1252 : {
1253 36 : profile_count ret = *this;
1254 36 : if (!initialized_p ())
1255 0 : return *this;
1256 36 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1257 36 : return ret;
1258 : }
1259 :
1260 : /* Return THIS with quality dropped to GUESSED. */
1261 402 : profile_count guessed () const
1262 : {
1263 402 : profile_count ret = *this;
1264 402 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1265 402 : return ret;
1266 : }
1267 :
1268 : /* Return THIS with quality GUESSED. */
1269 0 : profile_count force_guessed () const
1270 : {
1271 0 : profile_count ret = *this;
1272 0 : gcc_checking_assert (initialized_p ());
1273 0 : ret.m_quality = GUESSED;
1274 0 : return ret;
1275 : }
1276 :
1277 : /* Return variant of profile count which is always safe to compare
1278 : across functions. */
1279 9035995687 : profile_count ipa () const
1280 : {
1281 9035995687 : if (m_quality > GUESSED_GLOBAL0)
1282 13789835 : return *this;
1283 9022205852 : if (m_quality == GUESSED_GLOBAL0)
1284 211391 : return zero ();
1285 9021994461 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1286 10541 : return adjusted_zero ();
1287 9021983920 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1288 0 : return afdo_zero ();
1289 9021983920 : return uninitialized ();
1290 : }
1291 :
1292 : /* Return THIS with quality dropped to AFDO. */
1293 0 : profile_count afdo () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : ret.m_quality = AFDO;
1297 0 : return ret;
1298 : }
1299 :
1300 : /* Return probability of event with counter THIS within event with counter
1301 : OVERALL. */
1302 1081123546 : profile_probability probability_in (const profile_count overall) const
1303 : {
1304 1081141344 : if (*this == zero ()
1305 17798 : && !(overall == zero ()))
1306 2159 : return profile_probability::never ();
1307 1081120826 : if (!initialized_p () || !overall.initialized_p ()
1308 2162242213 : || !overall.m_val)
1309 63434 : return profile_probability::uninitialized ();
1310 1289870472 : if (*this == overall && m_quality == PRECISE)
1311 27690 : return profile_probability::always ();
1312 1081030263 : profile_probability ret;
1313 1081030263 : gcc_checking_assert (compatible_p (overall));
1314 :
1315 1081030263 : if (overall.m_val < m_val)
1316 : {
1317 170792 : ret.m_val = profile_probability::max_probability;
1318 170792 : ret.set_quality (GUESSED);
1319 170792 : return ret;
1320 : }
1321 : else
1322 : {
1323 1080859471 : gcc_checking_assert (overall.m_val);
1324 1080859471 : uint64_t tmp;
1325 1080859471 : safe_scale_64bit (m_val, profile_probability::max_probability,
1326 : overall.m_val, &tmp);
1327 1080859471 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1328 1080859471 : ret.m_val = tmp;
1329 : }
1330 1080859471 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1331 : GUESSED), ADJUSTED));
1332 1080859471 : return ret;
1333 : }
1334 :
1335 : /* Return true if profile count is very large, so we risk overflows
1336 : with loop transformations. */
1337 : bool
1338 1029186 : very_large_p ()
1339 : {
1340 1029186 : if (!initialized_p ())
1341 : return false;
1342 1029186 : return m_val > max_count / 65536;
1343 : }
1344 :
1345 : int to_frequency (struct function *fun) const;
1346 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1347 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1348 :
1349 : /* Output THIS to F. */
1350 : void dump (FILE *f, struct function *fun = NULL) const;
1351 :
1352 : /* Print THIS to stderr. */
1353 : void debug () const;
1354 :
1355 : /* Return true if THIS is known to differ significantly from OTHER. */
1356 : bool differs_from_p (profile_count other) const;
1357 :
1358 : /* We want to scale profile across function boundary from NUM to DEN.
1359 : Take care of the side case when NUM and DEN are zeros of incompatible
1360 : kinds. */
1361 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1362 :
1363 : /* THIS is a count of bb which is known to be executed IPA times.
1364 : Combine this information into bb counter. This means returning IPA
1365 : if it is nonzero, not changing anything if IPA is uninitialized
1366 : and if IPA is zero, turning THIS into corresponding local profile with
1367 : global0. */
1368 : profile_count combine_with_ipa_count (profile_count ipa);
1369 :
1370 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1371 : profile_count combine_with_ipa_count_within
1372 : (profile_count ipa, profile_count ipa2);
1373 :
1374 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1375 : Conversions back and forth are used to read the coverage and get it
1376 : into internal representation. */
1377 : static profile_count from_gcov_type (gcov_type v,
1378 : profile_quality quality = PRECISE);
1379 :
1380 : /* LTO streaming support. */
1381 : static profile_count stream_in (class lto_input_block *);
1382 : void stream_out (struct output_block *);
1383 : void stream_out (struct lto_output_stream *);
1384 : };
1385 : #endif
|
