Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2291056213 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2291056213 : uint64_t tmp;
88 2291056213 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2291056213 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2288761779 : *res = tmp / c;
92 2288761779 : return true;
93 : }
94 2294434 : if (c == 1)
95 : {
96 37549 : *res = (uint64_t) -1;
97 37549 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2256885 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greater then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5297048870 : set_quality (profile_quality quality)
169 : {
170 1119588887 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1119588887 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1117879794 : }
174 :
175 : public:
176 4179169076 : profile_probability (): m_val (uninitialized_probability)
177 2292173671 : { set_quality (GUESSED); }
178 :
179 392 : profile_probability (uint32_t val, profile_quality quality):
180 392 : m_val (val)
181 392 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 731308831 : static profile_probability never ()
185 : {
186 731308831 : profile_probability ret;
187 731308831 : ret.m_val = 0;
188 731308831 : ret.set_quality (PRECISE);
189 436647149 : return ret;
190 : }
191 :
192 5221607 : static profile_probability guessed_never ()
193 : {
194 5221607 : profile_probability ret;
195 5221607 : ret.m_val = 0;
196 5221607 : ret.set_quality (GUESSED);
197 5221607 : return ret;
198 : }
199 :
200 5181541 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10363082 : profile_probability r = guessed_always () / 2000;
204 5181541 : r.m_val--;
205 5181541 : return r;
206 : }
207 :
208 85855 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 171710 : profile_probability r = guessed_always () / 5;
212 85855 : r.m_val--;
213 85855 : return r;
214 : }
215 :
216 721525 : static profile_probability even ()
217 : {
218 1443050 : return guessed_always () / 2;
219 : }
220 :
221 341979 : static profile_probability very_likely ()
222 : {
223 341979 : return always () - very_unlikely ();
224 : }
225 :
226 33410 : static profile_probability likely ()
227 : {
228 33410 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 1799 : bool nonzero_p () const
232 : {
233 1799 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6297995 : static profile_probability guessed_always ()
237 : {
238 6297995 : profile_probability ret;
239 6297995 : ret.m_val = max_probability;
240 6297995 : ret.set_quality (GUESSED);
241 6100914 : return ret;
242 : }
243 :
244 541278073 : static profile_probability always ()
245 : {
246 541278073 : profile_probability ret;
247 541278073 : ret.m_val = max_probability;
248 541278073 : ret.set_quality (PRECISE);
249 366175484 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 896698498 : static profile_probability uninitialized ()
255 : {
256 896698498 : profile_probability c;
257 896698498 : c.m_val = uninitialized_probability;
258 896698498 : c.set_quality (GUESSED);
259 883198846 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5554683282 : bool initialized_p () const
264 : {
265 3866810939 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1518308 : bool reliable_p () const
270 : {
271 121001560 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3142528 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3142528 : profile_probability ret;
279 3142528 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3142528 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3142528 : ret.set_quality (GUESSED);
282 3142528 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 42 : profile_probability adjusted () const
287 : {
288 42 : profile_probability ret = *this;
289 42 : if (!initialized_p ())
290 0 : return *this;
291 42 : ret.set_quality (ADJUSTED);
292 42 : return ret;
293 : }
294 :
295 1100979758 : int to_reg_br_prob_base () const
296 : {
297 1100979758 : gcc_checking_assert (initialized_p ());
298 1100979758 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 486757385 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 486757385 : profile_probability ret;
305 486757385 : ret.m_val = ((unsigned int)v) / 8;
306 486757385 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 15685682 : return ret;
308 : }
309 :
310 457964470 : int to_reg_br_prob_note () const
311 : {
312 457964470 : gcc_checking_assert (initialized_p ());
313 457964470 : int ret = m_val * 8 + m_adjusted_quality;
314 457964470 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 457964470 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3209 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3209 : profile_probability ret;
323 3209 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3209 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3209 : uint64_t tmp;
329 3209 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3209 : gcc_checking_assert (tmp <= max_probability);
331 3209 : ret.m_val = tmp;
332 : }
333 3209 : ret.set_quality (PRECISE);
334 3209 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1599970674 : bool operator== (const profile_probability &other) const
339 : {
340 2261389698 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6577035 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6577035 : if (other == never ())
346 31515 : return *this;
347 6545520 : if (*this == never ())
348 23543 : return other;
349 6521977 : if (!initialized_p () || !other.initialized_p ())
350 5373005 : return uninitialized ();
351 :
352 1148972 : profile_probability ret;
353 1148972 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1148972 : ret.set_quality (MIN (quality (), other.quality ()));
355 1148972 : return ret;
356 : }
357 :
358 6342166 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6342166 : if (other == never ())
361 : return *this;
362 5116081 : if (*this == never ())
363 : {
364 882525 : *this = other;
365 882525 : return *this;
366 : }
367 4233556 : if (!initialized_p () || !other.initialized_p ())
368 2944309 : return *this = uninitialized ();
369 : else
370 : {
371 1289247 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 1289247 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1289247 : return *this;
375 : }
376 :
377 33611741 : profile_probability operator- (const profile_probability &other) const
378 : {
379 67223482 : if (*this == never ()
380 33611741 : || other == never ())
381 1856351 : return *this;
382 31755390 : if (!initialized_p () || !other.initialized_p ())
383 2181711 : return uninitialized ();
384 29573679 : profile_probability ret;
385 29573679 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 29573679 : ret.set_quality (MIN (quality (), other.quality ()));
387 29573679 : return ret;
388 : }
389 :
390 11343969 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 22687938 : if (*this == never ()
393 11343969 : || other == never ())
394 2242270 : return *this;
395 9101699 : if (!initialized_p () || !other.initialized_p ())
396 2138386 : return *this = uninitialized ();
397 : else
398 : {
399 6963313 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 6963313 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 6963313 : return *this;
403 : }
404 :
405 1181901 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2363802 : if (*this == never ()
408 1181901 : || other == never ())
409 120832 : return never ();
410 1061069 : if (!initialized_p () || !other.initialized_p ())
411 58469 : return uninitialized ();
412 1002600 : profile_probability ret;
413 1002600 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 1002600 : ret.set_quality (MIN (quality (), other.quality ()));
415 1002600 : return ret;
416 : }
417 :
418 132939 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 265878 : if (*this == never ()
421 132939 : || other == never ())
422 469 : return *this = never ();
423 132470 : if (!initialized_p () || !other.initialized_p ())
424 9 : return *this = uninitialized ();
425 : else
426 : {
427 132461 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 247123 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 132461 : return *this;
431 : }
432 :
433 1710025 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1710025 : if (*this == never ())
436 103122 : return never ();
437 1606903 : if (!initialized_p () || !other.initialized_p ())
438 227038 : return uninitialized ();
439 1379865 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1379865 : if (m_val >= other.m_val)
442 : {
443 45047 : ret.m_val = max_probability;
444 45047 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 45047 : return ret;
447 : }
448 1334818 : else if (!m_val)
449 63600 : ret.m_val = 0;
450 : else
451 : {
452 1271218 : gcc_checking_assert (other.m_val);
453 1271218 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2659975 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1334818 : return ret;
457 : }
458 :
459 316756 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 316756 : if (*this == never ())
462 1745 : return *this = never ();
463 315011 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 315011 : if (m_val > other.m_val)
470 : {
471 720 : m_val = max_probability;
472 720 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 720 : return *this;
475 : }
476 314291 : else if (!m_val)
477 : ;
478 : else
479 : {
480 311099 : gcc_checking_assert (other.m_val);
481 311099 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 565973 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 314291 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 282149 : profile_probability split (const profile_probability &cprob)
504 : {
505 282149 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 282149 : if (!(*this == always ()))
512 273882 : *this = (*this - ret) / ret.invert ();
513 282149 : return ret;
514 : }
515 :
516 265157 : gcov_type apply (gcov_type val) const
517 : {
518 265157 : if (*this == uninitialized ())
519 55 : return val / 2;
520 265102 : return RDIV (val * m_val, max_probability);
521 : }
522 :
523 : /* Return 1-*THIS. */
524 20988712 : profile_probability invert () const
525 : {
526 30456413 : return always() - *this;
527 : }
528 :
529 : /* Return THIS with quality dropped to GUESSED. */
530 822802 : profile_probability guessed () const
531 : {
532 822802 : profile_probability ret = *this;
533 822802 : ret.set_quality (GUESSED);
534 822766 : return ret;
535 : }
536 :
537 : /* Return THIS with quality dropped to AFDO. */
538 : profile_probability afdo () const
539 : {
540 : profile_probability ret = *this;
541 : ret.set_quality (AFDO);
542 : return ret;
543 : }
544 :
545 : /* Return *THIS * NUM / DEN. */
546 14306195 : profile_probability apply_scale (int64_t num, int64_t den) const
547 : {
548 14306195 : if (*this == never ())
549 15967 : return *this;
550 14290228 : if (!initialized_p ())
551 5782347 : return uninitialized ();
552 8507881 : profile_probability ret;
553 8507881 : uint64_t tmp;
554 8507881 : safe_scale_64bit (m_val, num, den, &tmp);
555 8507881 : ret.m_val = MIN (tmp, max_probability);
556 8507881 : ret.set_quality (MIN (quality (), ADJUSTED));
557 8507881 : return ret;
558 : }
559 :
560 : /* Return *THIS * NUM / DEN. */
561 10389 : profile_probability apply_scale (profile_probability num,
562 : profile_probability den) const
563 : {
564 10389 : if (*this == never ())
565 199 : return *this;
566 10190 : if (num == never ())
567 0 : return num;
568 10190 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
569 0 : return uninitialized ();
570 10190 : if (num == den)
571 0 : return *this;
572 10190 : gcc_checking_assert (den.m_val);
573 :
574 10190 : profile_probability ret;
575 10190 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
576 : max_probability);
577 10190 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
578 : num.quality ()), den.quality ()));
579 10190 : return ret;
580 : }
581 :
582 : /* Return true when the probability of edge is reliable.
583 :
584 : The profile guessing code is good at predicting branch outcome (i.e.
585 : taken/not taken), that is predicted right slightly over 75% of time.
586 : It is however notoriously poor on predicting the probability itself.
587 : In general the profile appear a lot flatter (with probabilities closer
588 : to 50%) than the reality so it is bad idea to use it to drive optimization
589 : such as those disabling dynamic branch prediction for well predictable
590 : branches.
591 :
592 : There are two exceptions - edges leading to noreturn edges and edges
593 : predicted by number of iterations heuristics are predicted well. This macro
594 : should be able to distinguish those, but at the moment it simply check for
595 : noreturn heuristic that is only one giving probability over 99% or below
596 : 1%. In future we might want to propagate reliability information across the
597 : CFG if we find this information useful on multiple places. */
598 0 : bool probably_reliable_p () const
599 : {
600 0 : if (quality () >= ADJUSTED)
601 : return true;
602 0 : if (!initialized_p ())
603 : return false;
604 0 : return m_val < max_probability / 100
605 0 : || m_val > max_probability - max_probability / 100;
606 : }
607 :
608 : /* Return false if profile_probability is bogus. */
609 4266911909 : bool verify () const
610 : {
611 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
612 4266911909 : if (m_val == uninitialized_probability)
613 935487231 : return quality () == GUESSED;
614 3331424678 : else if (quality () < GUESSED)
615 : return false;
616 3331424678 : return m_val <= max_probability;
617 : }
618 :
619 : /* Comparisons are three-state and conservative. False is returned if
620 : the inequality cannot be decided. */
621 7997397 : bool operator< (const profile_probability &other) const
622 : {
623 7997397 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
624 : }
625 :
626 17340788 : bool operator> (const profile_probability &other) const
627 : {
628 17340788 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
629 : }
630 :
631 5333795 : bool operator<= (const profile_probability &other) const
632 : {
633 5333795 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
634 : }
635 :
636 969203 : bool operator>= (const profile_probability &other) const
637 : {
638 969203 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
639 : }
640 :
641 17 : profile_probability operator* (int64_t num) const
642 : {
643 17 : return apply_scale (num, 1);
644 : }
645 :
646 : profile_probability operator*= (int64_t num)
647 : {
648 : *this = apply_scale (num, 1);
649 : return *this;
650 : }
651 :
652 13672944 : profile_probability operator/ (int64_t den) const
653 : {
654 13672944 : return apply_scale (1, den);
655 : }
656 :
657 243403 : profile_probability operator/= (int64_t den)
658 : {
659 243403 : *this = apply_scale (1, den);
660 243403 : return *this;
661 : }
662 :
663 : /* Compute n-th power. */
664 : profile_probability pow (int) const;
665 :
666 : /* Compute sware root. */
667 : profile_probability sqrt () const;
668 :
669 : /* Get the value of the probability. */
670 0 : uint32_t value () const { return m_val; }
671 :
672 : /* Get the quality of the probability. */
673 5944271623 : enum profile_quality quality () const
674 : {
675 5736502230 : return (profile_quality) (m_adjusted_quality
676 5736524638 : ? m_adjusted_quality + min_quality - 1
677 : : UNINITIALIZED_PROFILE);
678 : }
679 :
680 : /* Output THIS to F. */
681 : void dump (FILE *f) const;
682 :
683 : /* Output THIS to BUFFER. */
684 : void dump (char *buffer) const;
685 :
686 : /* Print THIS to stderr. */
687 : void debug () const;
688 :
689 : /* Return true if THIS is known to differ significantly from OTHER. */
690 : bool differs_from_p (profile_probability other) const;
691 :
692 : /* Return if difference is greater than 50%. */
693 : bool differs_lot_from_p (profile_probability other) const;
694 :
695 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
696 : happens with COUNT2 probability. Return probability that either *THIS or
697 : OTHER happens. */
698 : profile_probability combine_with_count (profile_count count1,
699 : profile_probability other,
700 : profile_count count2) const;
701 :
702 : /* Return probability as sreal. */
703 : sreal to_sreal () const;
704 : /* LTO streaming support. */
705 : static profile_probability stream_in (class lto_input_block *);
706 : void stream_out (struct output_block *);
707 : void stream_out (struct lto_output_stream *);
708 : };
709 :
710 : /* Main data type to hold profile counters in GCC. Profile counts originate
711 : either from profile feedback, static profile estimation or both. We do not
712 : perform whole program profile propagation and thus profile estimation
713 : counters are often local to function, while counters from profile feedback
714 : (or special cases of profile estimation) can be used inter-procedurally.
715 :
716 : There are 3 basic types
717 : 1) local counters which are result of intra-procedural static profile
718 : estimation.
719 : 2) ipa counters which are result of profile feedback or special case
720 : of static profile estimation (such as in function main).
721 : 3) counters which counts as 0 inter-procedurally (because given function
722 : was never run in train feedback) but they hold local static profile
723 : estimate.
724 :
725 : Counters of type 1 and 3 cannot be mixed with counters of different type
726 : within operation (because whole function should use one type of counter)
727 : with exception that global zero mix in most operations where outcome is
728 : well defined.
729 :
730 : To take local counter and use it inter-procedurally use ipa member function
731 : which strips information irrelevant at the inter-procedural level.
732 :
733 : Counters are 61bit integers representing number of executions during the
734 : train run or normalized frequency within the function.
735 :
736 : As the profile is maintained during the compilation, many adjustments are
737 : made. Not all transformations can be made precisely, most importantly
738 : when code is being duplicated. It also may happen that part of CFG has
739 : profile counts known while other do not - for example when LTO optimizing
740 : partly profiled program or when profile was lost due to COMDAT merging.
741 :
742 : For this reason profile_count tracks more information than
743 : just unsigned integer and it is also ready for profile mismatches.
744 : The API of this data type represent operations that are natural
745 : on profile counts - sum, difference and operation with scales and
746 : probabilities. All operations are safe by never getting negative counts
747 : and they do end up in uninitialized scale if any of the parameters is
748 : uninitialized.
749 :
750 : All comparisons that are three state and handling of probabilities. Thus
751 : a < b is not equal to !(a >= b).
752 :
753 : The following pre-defined counts are available:
754 :
755 : profile_count::zero () for code that is known to execute zero times at
756 : runtime (this can be detected statically i.e. for paths leading to
757 : abort ();
758 : profile_count::one () for code that is known to execute once (such as
759 : main () function
760 : profile_count::uninitialized () for unknown execution count.
761 :
762 : */
763 :
764 : struct GTY(()) profile_count
765 : {
766 : public:
767 : /* Use 60bit to hold basic block counters. Should be at least
768 : 64bit. Although a counter cannot be negative, we use a signed
769 : type to hold various extra stages. */
770 :
771 : static const int n_bits = 60;
772 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
773 : private:
774 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
775 :
776 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
777 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
778 : incorrectly detects the alignment of a structure where the only
779 : 64-bit aligned object is a bit-field. We force the alignment of
780 : the entire field to mitigate this. */
781 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
782 : #else
783 : #define UINT64_BIT_FIELD_ALIGN
784 : #endif
785 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
786 : #undef UINT64_BIT_FIELD_ALIGN
787 : enum profile_quality m_quality : 4;
788 : public:
789 :
790 : /* Return true if both values can meaningfully appear in single function
791 : body. We have either all counters in function local or global, otherwise
792 : operations between them are not really defined well. */
793 4828639778 : bool compatible_p (const profile_count other) const
794 : {
795 8868084698 : if (!initialized_p () || !other.initialized_p ())
796 : return true;
797 15999845993 : if (*this == zero ()
798 7929297141 : || other == zero ())
799 144432753 : return true;
800 : /* Do not allow nonzero global profile together with local guesses
801 : that are globally0. */
802 3890841673 : if (ipa ().nonzero_p ()
803 256416 : && !(other.ipa () == other))
804 0 : return false;
805 3890841673 : if (other.ipa ().nonzero_p ()
806 265663 : && !(ipa () == *this))
807 0 : return false;
808 :
809 3890841673 : return ipa_p () == other.ipa_p ();
810 : }
811 :
812 : /* Used for counters which are expected to be never executed. */
813 18092386115 : static profile_count zero ()
814 : {
815 14155922004 : return from_gcov_type (0);
816 : }
817 :
818 10521 : static profile_count adjusted_zero ()
819 : {
820 10521 : profile_count c;
821 10521 : c.m_val = 0;
822 10521 : c.m_quality = ADJUSTED;
823 10521 : return c;
824 : }
825 :
826 0 : static profile_count afdo_zero ()
827 : {
828 0 : profile_count c;
829 0 : c.m_val = 0;
830 0 : c.m_quality = AFDO;
831 0 : return c;
832 : }
833 :
834 : static profile_count guessed_zero ()
835 : {
836 : profile_count c;
837 : c.m_val = 0;
838 : c.m_quality = GUESSED;
839 : return c;
840 : }
841 :
842 50117 : static profile_count one ()
843 : {
844 49895 : return from_gcov_type (1);
845 : }
846 :
847 : /* Value of counters which has not been initialized. Either because
848 : initialization did not happen yet or because profile is unknown. */
849 9033112448 : static profile_count uninitialized ()
850 : {
851 9033112448 : profile_count c;
852 9033112448 : c.m_val = uninitialized_count;
853 9033112448 : c.m_quality = GUESSED_LOCAL;
854 9002368987 : return c;
855 : }
856 :
857 : /* Conversion to gcov_type is lossy. */
858 42490924 : gcov_type to_gcov_type () const
859 : {
860 0 : gcc_checking_assert (initialized_p ());
861 42490924 : return m_val;
862 : }
863 :
864 : /* Return true if value has been initialized. */
865 31938808193 : bool initialized_p () const
866 : {
867 10722096724 : return m_val != uninitialized_count;
868 : }
869 :
870 : /* Return true if value can be trusted. */
871 26685949 : bool reliable_p () const
872 : {
873 26685679 : return m_quality >= ADJUSTED;
874 : }
875 :
876 : /* Return true if value can be operated inter-procedurally. */
877 8808246467 : bool ipa_p () const
878 : {
879 4904802418 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
880 : }
881 :
882 : /* Return true if quality of profile is precise. */
883 52755590 : bool precise_p () const
884 : {
885 52755590 : return m_quality == PRECISE;
886 : }
887 :
888 : /* Get the value of the count. */
889 0 : uint64_t value () const { return m_val; }
890 :
891 : /* Get the quality of the count. */
892 95208078 : enum profile_quality quality () const { return m_quality; }
893 :
894 : /* When merging basic blocks, the two different profile counts are unified.
895 : Return true if this can be done without losing info about profile.
896 : The only case we care about here is when first BB contains something
897 : that makes it terminate in a way not visible in CFG. */
898 3224124 : bool ok_for_merging (profile_count other) const
899 : {
900 3224124 : if (m_quality < ADJUSTED
901 15410 : || other.m_quality < ADJUSTED)
902 : return true;
903 14455 : return !(other < *this);
904 : }
905 :
906 : /* When merging two BBs with different counts, pick common count that looks
907 : most representative. */
908 16692282 : profile_count merge (profile_count other) const
909 : {
910 825682 : if (*this == other || !other.initialized_p ()
911 825391 : || m_quality > other.m_quality)
912 15868084 : return *this;
913 824198 : if (other.m_quality > m_quality
914 824198 : || other > *this)
915 0 : return other;
916 824198 : return *this;
917 : }
918 :
919 : /* Basic operations. */
920 19519291149 : bool operator== (const profile_count &other) const
921 : {
922 14840605889 : return m_val == other.m_val && m_quality == other.m_quality;
923 : }
924 :
925 3769928 : profile_count operator+ (const profile_count &other) const
926 : {
927 3769928 : if (other == zero ())
928 2407 : return *this;
929 3767521 : if (*this == zero ())
930 212089 : return other;
931 3555432 : if (!initialized_p () || !other.initialized_p ())
932 6024 : return uninitialized ();
933 :
934 3549408 : profile_count ret;
935 3549408 : gcc_checking_assert (compatible_p (other));
936 3549408 : uint64_t ret_val = m_val + other.m_val;
937 3549408 : ret.m_val = MIN (ret_val, max_count);
938 3549408 : ret.m_quality = MIN (m_quality, other.m_quality);
939 3549408 : return ret;
940 : }
941 :
942 73190813 : profile_count &operator+= (const profile_count &other)
943 : {
944 73190813 : if (other == zero ())
945 5487119 : return *this;
946 67703694 : if (*this == zero ())
947 : {
948 41927960 : *this = other;
949 41927960 : return *this;
950 : }
951 25775734 : if (!initialized_p () || !other.initialized_p ())
952 347743 : return *this = uninitialized ();
953 : else
954 : {
955 25427991 : gcc_checking_assert (compatible_p (other));
956 25427991 : uint64_t ret_val = m_val + other.m_val;
957 25427991 : m_val = MIN (ret_val, max_count);
958 25427991 : m_quality = MIN (m_quality, other.m_quality);
959 : }
960 25427991 : return *this;
961 : }
962 :
963 20418934 : profile_count operator- (const profile_count &other) const
964 : {
965 20437445 : if (*this == zero () || other == zero ())
966 1330511 : return *this;
967 19088423 : if (!initialized_p () || !other.initialized_p ())
968 8044354 : return uninitialized ();
969 11044069 : gcc_checking_assert (compatible_p (other));
970 11044069 : profile_count ret;
971 11044069 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
972 11044069 : ret.m_quality = MIN (m_quality, other.m_quality);
973 11044069 : return ret;
974 : }
975 :
976 8811492 : profile_count &operator-= (const profile_count &other)
977 : {
978 9246621 : if (*this == zero () || other == zero ())
979 280322 : return *this;
980 8531170 : if (!initialized_p () || !other.initialized_p ())
981 353699 : return *this = uninitialized ();
982 : else
983 : {
984 8177471 : gcc_checking_assert (compatible_p (other));
985 8177471 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
986 8177471 : m_quality = MIN (m_quality, other.m_quality);
987 : }
988 8177471 : return *this;
989 : }
990 :
991 : /* Return false if profile_count is bogus. */
992 3258087177 : bool verify () const
993 : {
994 3258087177 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
995 3258087177 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
996 : }
997 :
998 : /* Comparisons are three-state and conservative. False is returned if
999 : the inequality cannot be decided. */
1000 1076426530 : bool operator< (const profile_count &other) const
1001 : {
1002 1076426530 : if (!initialized_p () || !other.initialized_p ())
1003 : return false;
1004 1071787448 : if (*this == zero ())
1005 4037008 : return !(other == zero ());
1006 1069768944 : if (other == zero ())
1007 876 : return false;
1008 1069768068 : gcc_checking_assert (compatible_p (other));
1009 1069768068 : return m_val < other.m_val;
1010 : }
1011 :
1012 37442999 : bool operator> (const profile_count &other) const
1013 : {
1014 37442999 : if (!initialized_p () || !other.initialized_p ())
1015 : return false;
1016 32684734 : if (*this == zero ())
1017 221641 : return false;
1018 32463093 : if (other == zero ())
1019 1509970 : return !(*this == zero ());
1020 31708108 : gcc_checking_assert (compatible_p (other));
1021 31708108 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1022 : }
1023 :
1024 : bool operator< (const gcov_type other) const
1025 : {
1026 : gcc_checking_assert (ipa_p ());
1027 : gcc_checking_assert (other >= 0);
1028 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1029 : }
1030 :
1031 7208 : bool operator> (const gcov_type other) const
1032 : {
1033 7208 : gcc_checking_assert (ipa_p ());
1034 7208 : gcc_checking_assert (other >= 0);
1035 7208 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1036 : }
1037 :
1038 1108131 : bool operator<= (const profile_count &other) const
1039 : {
1040 1108131 : if (!initialized_p () || !other.initialized_p ())
1041 : return false;
1042 1108131 : if (*this == zero ())
1043 3953 : return true;
1044 1104178 : if (other == zero ())
1045 0 : return (*this == zero ());
1046 1104178 : gcc_checking_assert (compatible_p (other));
1047 1104178 : return m_val <= other.m_val;
1048 : }
1049 :
1050 1394463 : bool operator>= (const profile_count &other) const
1051 : {
1052 1394463 : if (!initialized_p () || !other.initialized_p ())
1053 : return false;
1054 1393336 : if (other == zero ())
1055 2704 : return true;
1056 1390632 : if (*this == zero ())
1057 22452 : return (other == zero ());
1058 1379406 : gcc_checking_assert (compatible_p (other));
1059 1379406 : return m_val >= other.m_val;
1060 : }
1061 :
1062 101519 : bool operator<= (const gcov_type other) const
1063 : {
1064 101519 : gcc_checking_assert (ipa_p ());
1065 101519 : gcc_checking_assert (other >= 0);
1066 101519 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1067 : }
1068 :
1069 64740 : bool operator>= (const gcov_type other) const
1070 : {
1071 64740 : gcc_checking_assert (ipa_p ());
1072 64740 : gcc_checking_assert (other >= 0);
1073 64740 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1074 : }
1075 :
1076 896666505 : profile_count operator* (int64_t num) const
1077 : {
1078 896593688 : return apply_scale (num, 1);
1079 : }
1080 :
1081 : profile_count operator*= (int64_t num)
1082 : {
1083 : *this = apply_scale (num, 1);
1084 : return *this;
1085 : }
1086 :
1087 : profile_count operator* (const sreal &num) const;
1088 : profile_count operator*= (const sreal &num);
1089 :
1090 2338197 : profile_count operator/ (int64_t den) const
1091 : {
1092 2338197 : return apply_scale (1, den);
1093 : }
1094 :
1095 8 : profile_count operator/= (int64_t den)
1096 : {
1097 0 : *this = apply_scale (1, den);
1098 8 : return *this;
1099 : }
1100 :
1101 : /* Return true when value is not zero and can be used for scaling.
1102 : This is different from *this > 0 because that requires counter to
1103 : be IPA. */
1104 7977052780 : bool nonzero_p () const
1105 : {
1106 7977052136 : return initialized_p () && m_val != 0;
1107 : }
1108 :
1109 : /* Make counter forcibly nonzero. */
1110 : profile_count force_nonzero () const;
1111 :
1112 :
1113 : /* Return maximum of A and B. If one of values is uninitialized return the
1114 : other. */
1115 :
1116 : static profile_count
1117 77595230 : max_prefer_initialized (const profile_count a, const profile_count b)
1118 : {
1119 77595230 : if (!a.initialized_p ())
1120 19453723 : return b;
1121 58141507 : if (!b.initialized_p ())
1122 1801924 : return a;
1123 56339583 : profile_count ret;
1124 56339583 : gcc_checking_assert (a.compatible_p (b));
1125 56339583 : ret.m_val = MAX (a.m_val, b.m_val);
1126 56339583 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1127 56339583 : return ret;
1128 : }
1129 :
1130 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1131 : accordingly. */
1132 : profile_count apply_probability (int prob) const
1133 : {
1134 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1135 : if (m_val == 0)
1136 : return *this;
1137 : if (!initialized_p ())
1138 : return uninitialized ();
1139 : profile_count ret;
1140 : uint64_t tmp;
1141 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1142 : ret.m_val = tmp;
1143 : ret.m_quality = MIN (m_quality, ADJUSTED);
1144 : return ret;
1145 : }
1146 :
1147 : /* Scale counter according to PROB. */
1148 381907814 : profile_count apply_probability (profile_probability prob) const
1149 : {
1150 390092655 : if (*this == zero () || prob == profile_probability::always ())
1151 124253547 : return *this;
1152 257654267 : if (prob == profile_probability::never ())
1153 19910538 : return zero ();
1154 237743729 : if (!initialized_p () || !prob.initialized_p ())
1155 29996744 : return uninitialized ();
1156 207746985 : profile_count ret;
1157 207746985 : uint64_t tmp;
1158 207746985 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1159 : &tmp);
1160 207746985 : ret.m_val = tmp;
1161 207769393 : ret.m_quality = MIN (m_quality, prob.quality ());
1162 207746985 : return ret;
1163 : }
1164 :
1165 : /* Return *THIS * NUM / DEN. */
1166 1021389507 : profile_count apply_scale (int64_t num, int64_t den) const
1167 : {
1168 1021389507 : if (m_val == 0)
1169 23738897 : return *this;
1170 997650610 : if (!initialized_p ())
1171 3861867 : return uninitialized ();
1172 993788743 : profile_count ret;
1173 993788743 : uint64_t tmp;
1174 :
1175 993788743 : gcc_checking_assert (num >= 0 && den > 0);
1176 993788743 : safe_scale_64bit (m_val, num, den, &tmp);
1177 993788743 : ret.m_val = MIN (tmp, max_count);
1178 993788743 : ret.m_quality = MIN (m_quality, ADJUSTED);
1179 993788743 : return ret;
1180 : }
1181 :
1182 27881262 : profile_count apply_scale (profile_count num, profile_count den) const
1183 : {
1184 27881262 : if (*this == zero ())
1185 2369706 : return *this;
1186 25511556 : if (num == zero ())
1187 83262 : return num;
1188 25428294 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1189 7017891 : return uninitialized ();
1190 18410403 : if (num == den)
1191 5443935 : return *this;
1192 12966468 : gcc_checking_assert (den.m_val);
1193 :
1194 12966468 : profile_count ret;
1195 12966468 : uint64_t val;
1196 12966468 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1197 12966468 : ret.m_val = MIN (val, max_count);
1198 12966468 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1199 : num.m_quality), den.m_quality);
1200 : /* Be sure that ret is not local if num is global.
1201 : Also ensure that ret is not global0 when num is global. */
1202 12966468 : if (num.ipa_p ())
1203 : {
1204 : /* This is common case of AFDO scaling when we upgrade
1205 : GLOBAL0_AFDO function to AFDO. Be sure that result
1206 : is AFDO and not GUESSED (which is unnecessarily low). */
1207 1818 : if (num.m_quality == AFDO
1208 0 : && (ret.m_quality != GUESSED
1209 0 : && ret.m_quality != GUESSED_LOCAL))
1210 0 : ret.m_quality = AFDO;
1211 : else
1212 3663 : ret.m_quality = MAX (ret.m_quality,
1213 : num == num.ipa () ? GUESSED : num.m_quality);
1214 : }
1215 12966468 : return ret;
1216 : }
1217 :
1218 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1219 17481494 : profile_count guessed_local () const
1220 : {
1221 17481494 : profile_count ret = *this;
1222 17481494 : if (!initialized_p ())
1223 0 : return *this;
1224 17481494 : ret.m_quality = GUESSED_LOCAL;
1225 17481494 : return ret;
1226 : }
1227 :
1228 : /* We know that profile is globally 0 but keep local profile if present. */
1229 794 : profile_count global0 () const
1230 : {
1231 794 : profile_count ret = *this;
1232 794 : if (!initialized_p ())
1233 0 : return *this;
1234 794 : ret.m_quality = GUESSED_GLOBAL0;
1235 794 : return ret;
1236 : }
1237 :
1238 : /* We know that profile is globally afdo 0 but keep local profile
1239 : if present. */
1240 0 : profile_count global0afdo () const
1241 : {
1242 0 : profile_count ret = *this;
1243 0 : if (!initialized_p ())
1244 0 : return *this;
1245 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1246 0 : return ret;
1247 : }
1248 :
1249 : /* We know that profile is globally adjusted 0 but keep local profile
1250 : if present. */
1251 48 : profile_count global0adjusted () const
1252 : {
1253 48 : profile_count ret = *this;
1254 48 : if (!initialized_p ())
1255 0 : return *this;
1256 48 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1257 48 : return ret;
1258 : }
1259 :
1260 : /* Return THIS with quality dropped to GUESSED. */
1261 394 : profile_count guessed () const
1262 : {
1263 394 : profile_count ret = *this;
1264 394 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1265 394 : return ret;
1266 : }
1267 :
1268 : /* Return THIS with quality GUESSED. */
1269 0 : profile_count force_guessed () const
1270 : {
1271 0 : profile_count ret = *this;
1272 0 : gcc_checking_assert (initialized_p ());
1273 0 : ret.m_quality = GUESSED;
1274 0 : return ret;
1275 : }
1276 :
1277 : /* Return variant of profile count which is always safe to compare
1278 : across functions. */
1279 8890492519 : profile_count ipa () const
1280 : {
1281 8890492519 : if (m_quality > GUESSED_GLOBAL0)
1282 13981544 : return *this;
1283 8876510975 : if (m_quality == GUESSED_GLOBAL0)
1284 211826 : return zero ();
1285 8876299149 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1286 10521 : return adjusted_zero ();
1287 8876288628 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1288 0 : return afdo_zero ();
1289 8876288628 : return uninitialized ();
1290 : }
1291 :
1292 : /* Return THIS with quality dropped to AFDO. */
1293 0 : profile_count afdo () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : ret.m_quality = AFDO;
1297 0 : return ret;
1298 : }
1299 :
1300 : /* Return probability of event with counter THIS within event with counter
1301 : OVERALL. */
1302 1067813690 : profile_probability probability_in (const profile_count overall) const
1303 : {
1304 1067831115 : if (*this == zero ()
1305 17425 : && !(overall == zero ()))
1306 2091 : return profile_probability::never ();
1307 1067811038 : if (!initialized_p () || !overall.initialized_p ()
1308 2135622637 : || !overall.m_val)
1309 60729 : return profile_probability::uninitialized ();
1310 1277545105 : if (*this == overall && m_quality == PRECISE)
1311 30733 : return profile_probability::always ();
1312 1067720137 : profile_probability ret;
1313 1067720137 : gcc_checking_assert (compatible_p (overall));
1314 :
1315 1067720137 : if (overall.m_val < m_val)
1316 : {
1317 166551 : ret.m_val = profile_probability::max_probability;
1318 166551 : ret.set_quality (GUESSED);
1319 166551 : return ret;
1320 : }
1321 : else
1322 : {
1323 1067553586 : gcc_checking_assert (overall.m_val);
1324 1067553586 : uint64_t tmp;
1325 1067553586 : safe_scale_64bit (m_val, profile_probability::max_probability,
1326 : overall.m_val, &tmp);
1327 1067553586 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1328 1067553586 : ret.m_val = tmp;
1329 : }
1330 1067553586 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1331 : GUESSED), ADJUSTED));
1332 1067553586 : return ret;
1333 : }
1334 :
1335 : /* Return true if profile count is very large, so we risk overflows
1336 : with loop transformations. */
1337 : bool
1338 1035019 : very_large_p ()
1339 : {
1340 1035019 : if (!initialized_p ())
1341 : return false;
1342 1035019 : return m_val > max_count / 65536;
1343 : }
1344 :
1345 : int to_frequency (struct function *fun) const;
1346 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1347 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1348 :
1349 : /* Output THIS to F. */
1350 : void dump (FILE *f, struct function *fun = NULL) const;
1351 :
1352 : /* Print THIS to stderr. */
1353 : void debug () const;
1354 :
1355 : /* Return true if THIS is known to differ significantly from OTHER. */
1356 : bool differs_from_p (profile_count other) const;
1357 :
1358 : /* We want to scale profile across function boundary from NUM to DEN.
1359 : Take care of the side case when NUM and DEN are zeros of incompatible
1360 : kinds. */
1361 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1362 :
1363 : /* THIS is a count of bb which is known to be executed IPA times.
1364 : Combine this information into bb counter. This means returning IPA
1365 : if it is nonzero, not changing anything if IPA is uninitialized
1366 : and if IPA is zero, turning THIS into corresponding local profile with
1367 : global0. */
1368 : profile_count combine_with_ipa_count (profile_count ipa);
1369 :
1370 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1371 : profile_count combine_with_ipa_count_within
1372 : (profile_count ipa, profile_count ipa2);
1373 :
1374 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1375 : Conversions back and forth are used to read the coverage and get it
1376 : into internal representation. */
1377 : static profile_count from_gcov_type (gcov_type v,
1378 : profile_quality quality = PRECISE);
1379 :
1380 : /* LTO streaming support. */
1381 : static profile_count stream_in (class lto_input_block *);
1382 : void stream_out (struct output_block *);
1383 : void stream_out (struct lto_output_stream *);
1384 : };
1385 : #endif
|
