Line data Source code
1 : /* Profile counter container type.
2 : Copyright (C) 2017-2026 Free Software Foundation, Inc.
3 : Contributed by Jan Hubicka
4 :
5 : This file is part of GCC.
6 :
7 : GCC is free software; you can redistribute it and/or modify it under
8 : the terms of the GNU General Public License as published by the Free
9 : Software Foundation; either version 3, or (at your option) any later
10 : version.
11 :
12 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : for more details.
16 :
17 : You should have received a copy of the GNU General Public License
18 : along with GCC; see the file COPYING3. If not see
19 : <http://www.gnu.org/licenses/>. */
20 :
21 : #ifndef GCC_PROFILE_COUNT_H
22 : #define GCC_PROFILE_COUNT_H
23 :
24 : struct function;
25 : struct profile_count;
26 : class sreal;
27 :
28 : /* Quality of the profile count. Because gengtype does not support enums
29 : inside of classes, this is in global namespace. */
30 : enum profile_quality {
31 : /* Uninitialized value. */
32 : UNINITIALIZED_PROFILE,
33 :
34 : /* Profile is based on static branch prediction heuristics and may
35 : or may not match reality. It is local to function and cannot be compared
36 : inter-procedurally. Never used by probabilities (they are always local).
37 : */
38 : GUESSED_LOCAL,
39 :
40 : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : GUESSED_GLOBAL0_AFDO,
42 :
43 : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : GUESSED_GLOBAL0_ADJUSTED,
45 :
46 : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : better. This is the case of functions not run in profile feedback.
48 : Never used by probabilities. */
49 : GUESSED_GLOBAL0,
50 :
51 : /* Profile is based on static branch prediction heuristics. It may or may
52 : not reflect the reality but it can be compared interprocedurally
53 : (for example, we inlined function w/o profile feedback into function
54 : with feedback and propagated from that). */
55 : GUESSED,
56 :
57 : /* Profile was determined by autofdo. */
58 : AFDO,
59 :
60 : /* Profile was originally based on feedback but it was adjusted
61 : by code duplicating optimization. It may not precisely reflect the
62 : particular code path. */
63 : ADJUSTED,
64 :
65 : /* Profile was read from profile feedback or determined by accurate static
66 : method. */
67 : PRECISE
68 : };
69 :
70 : extern const char *profile_quality_as_string (enum profile_quality);
71 : extern bool parse_profile_quality (const char *value,
72 : profile_quality *quality);
73 :
74 : /* The base value for branch probability notes and edge probabilities. */
75 : #define REG_BR_PROB_BASE 10000
76 :
77 : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 :
79 : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 :
81 : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 :
83 : inline bool
84 2375643427 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : {
86 : #if (GCC_VERSION >= 5000)
87 2375643427 : uint64_t tmp;
88 2375643427 : if (!__builtin_mul_overflow (a, b, &tmp)
89 2375643427 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : {
91 2373324802 : *res = tmp / c;
92 2373324802 : return true;
93 : }
94 2318625 : if (c == 1)
95 : {
96 41128 : *res = (uint64_t) -1;
97 41128 : return false;
98 : }
99 : #else
100 : if (a < ((uint64_t)1 << 31)
101 : && b < ((uint64_t)1 << 31)
102 : && c < ((uint64_t)1 << 31))
103 : {
104 : *res = (a * b + (c / 2)) / c;
105 : return true;
106 : }
107 : #endif
108 2277497 : return slow_safe_scale_64bit (a, b, c, res);
109 : }
110 :
111 : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : with capping so probability is always in range [0,1] and scaling requiring
113 : values greater than 1 needs to be represented otherwise.
114 :
115 : In addition to actual value the quality of profile is tracked and propagated
116 : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : that has not been determined yet (for example because of
118 : -fno-guess-branch-probability)
119 :
120 : Typically probabilities are derived from profile feedback (via
121 : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : thorough the compilation.
123 :
124 : Named probabilities are available:
125 : - never (0 probability)
126 : - guessed_never
127 : - very_unlikely (1/2000 probability)
128 : - unlikely (1/5 probability)
129 : - even (1/2 probability)
130 : - likely (4/5 probability)
131 : - very_likely (1999/2000 probability)
132 : - guessed_always
133 : - always
134 :
135 : Named probabilities except for never/always are assumed to be statically
136 : guessed and thus not necessarily accurate. The difference between never
137 : and guessed_never is that the first one should be used only in case that
138 : well behaving program will very likely not execute the "never" path.
139 : For example if the path is going to abort () call or it exception handling.
140 :
141 : Always and guessed_always probabilities are symmetric.
142 :
143 : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : integer arithmetics. Once the code is converted to branch probabilities,
145 : these conversions will probably go away because they are lossy.
146 : */
147 :
148 : class GTY((user)) profile_probability
149 : {
150 : static const int n_bits = 29;
151 : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : will lead to harder multiplication sequences. */
153 : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : static const uint32_t uninitialized_probability
155 : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : /* For probibilityes quality is either UNINITIALIZED (0)
157 : or greater then GUESSED. To save bits we store it in
158 : adjusted form which skips the invalid values. */
159 : static const int min_quality = GUESSED;
160 :
161 : uint32_t m_val : 29;
162 : unsigned m_adjusted_quality : 3;
163 :
164 : friend struct profile_count;
165 :
166 : /* Set the quality of the probability. */
167 : void
168 5526369713 : set_quality (profile_quality quality)
169 : {
170 1156177889 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : || (quality >= min_quality && quality <= PRECISE));
172 1156177889 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 1154437904 : }
174 :
175 : public:
176 4371984556 : profile_probability (): m_val (uninitialized_probability)
177 2397347248 : { set_quality (GUESSED); }
178 :
179 419 : profile_probability (uint32_t val, profile_quality quality):
180 419 : m_val (val)
181 419 : { set_quality (quality); }
182 :
183 : /* Named probabilities. */
184 768387569 : static profile_probability never ()
185 : {
186 768387569 : profile_probability ret;
187 768387569 : ret.m_val = 0;
188 768387569 : ret.set_quality (PRECISE);
189 460913692 : return ret;
190 : }
191 :
192 5358586 : static profile_probability guessed_never ()
193 : {
194 5358586 : profile_probability ret;
195 5358586 : ret.m_val = 0;
196 5358586 : ret.set_quality (GUESSED);
197 5358568 : return ret;
198 : }
199 :
200 5326182 : static profile_probability very_unlikely ()
201 : {
202 : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 10652364 : profile_probability r = guessed_always () / 2000;
204 5326182 : r.m_val--;
205 5326182 : return r;
206 : }
207 :
208 87675 : static profile_probability unlikely ()
209 : {
210 : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 175350 : profile_probability r = guessed_always () / 5;
212 87675 : r.m_val--;
213 87675 : return r;
214 : }
215 :
216 747237 : static profile_probability even ()
217 : {
218 1494474 : return guessed_always () / 2;
219 : }
220 :
221 347488 : static profile_probability very_likely ()
222 : {
223 347488 : return always () - very_unlikely ();
224 : }
225 :
226 33272 : static profile_probability likely ()
227 : {
228 33272 : return always () - unlikely ();
229 : }
230 : /* Return true when value is not zero and can be used for scaling. */
231 2386 : bool nonzero_p () const
232 : {
233 2386 : return initialized_p () && m_val != 0;
234 : }
235 :
236 6469387 : static profile_probability guessed_always ()
237 : {
238 6469387 : profile_probability ret;
239 6469387 : ret.m_val = max_probability;
240 6469387 : ret.set_quality (GUESSED);
241 6250518 : return ret;
242 : }
243 :
244 564196095 : static profile_probability always ()
245 : {
246 563440682 : profile_probability ret;
247 564196095 : ret.m_val = max_probability;
248 564196095 : ret.set_quality (PRECISE);
249 385176706 : return ret;
250 : }
251 :
252 : /* Probabilities which has not been initialized. Either because
253 : initialization did not happen yet or because profile is unknown. */
254 952385418 : static profile_probability uninitialized ()
255 : {
256 952385418 : profile_probability c;
257 952385418 : c.m_val = uninitialized_probability;
258 952385418 : c.set_quality (GUESSED);
259 938731422 : return c;
260 : }
261 :
262 : /* Return true if value has been initialized. */
263 5762374154 : bool initialized_p () const
264 : {
265 4017247273 : return m_val != uninitialized_probability;
266 : }
267 :
268 : /* Return true if value can be trusted. */
269 1577678 : bool reliable_p () const
270 : {
271 132276989 : return quality () >= ADJUSTED;
272 : }
273 :
274 : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : this is mostly to support legacy code and should go away. */
276 3177900 : static profile_probability from_reg_br_prob_base (int v)
277 : {
278 3177900 : profile_probability ret;
279 3177900 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 3177900 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 3177900 : ret.set_quality (GUESSED);
282 3177900 : return ret;
283 : }
284 :
285 : /* Return THIS with quality set to ADJUSTED. */
286 42 : profile_probability adjusted () const
287 : {
288 42 : profile_probability ret = *this;
289 42 : if (!initialized_p ())
290 0 : return *this;
291 42 : ret.set_quality (ADJUSTED);
292 42 : return ret;
293 : }
294 :
295 1136731918 : int to_reg_br_prob_base () const
296 : {
297 1136731918 : gcc_checking_assert (initialized_p ());
298 1136731918 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : }
300 :
301 : /* Conversion to and from RTL representation of profile probabilities. */
302 501489755 : static profile_probability from_reg_br_prob_note (int v)
303 : {
304 501489755 : profile_probability ret;
305 501489755 : ret.m_val = ((unsigned int)v) / 8;
306 501489755 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 16119725 : return ret;
308 : }
309 :
310 471912838 : int to_reg_br_prob_note () const
311 : {
312 471912838 : gcc_checking_assert (initialized_p ());
313 471912838 : int ret = m_val * 8 + m_adjusted_quality;
314 471912838 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 471912838 : return ret;
316 : }
317 :
318 : /* Return VAL1/VAL2. */
319 3209 : static profile_probability probability_in_gcov_type
320 : (gcov_type val1, gcov_type val2)
321 : {
322 3209 : profile_probability ret;
323 3209 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 3209 : if (val1 > val2)
325 : ret.m_val = max_probability;
326 : else
327 : {
328 3209 : uint64_t tmp;
329 3209 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 3209 : gcc_checking_assert (tmp <= max_probability);
331 3209 : ret.m_val = tmp;
332 : }
333 3209 : ret.set_quality (PRECISE);
334 3209 : return ret;
335 : }
336 :
337 : /* Basic operations. */
338 1670491501 : bool operator== (const profile_probability &other) const
339 : {
340 2355529140 : return m_val == other.m_val && quality () == other.quality ();
341 : }
342 :
343 6781263 : profile_probability operator+ (const profile_probability &other) const
344 : {
345 6781263 : if (other == never ())
346 34693 : return *this;
347 6746570 : if (*this == never ())
348 23861 : return other;
349 6722709 : if (!initialized_p () || !other.initialized_p ())
350 5510194 : return uninitialized ();
351 :
352 1212515 : profile_probability ret;
353 1212515 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 1212515 : ret.set_quality (MIN (quality (), other.quality ()));
355 1212515 : return ret;
356 : }
357 :
358 6558290 : profile_probability &operator+= (const profile_probability &other)
359 : {
360 6558290 : if (other == never ())
361 : return *this;
362 5244897 : if (*this == never ())
363 : {
364 903146 : *this = other;
365 903146 : return *this;
366 : }
367 4341751 : if (!initialized_p () || !other.initialized_p ())
368 2968212 : return *this = uninitialized ();
369 : else
370 : {
371 1373539 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 8232243 : set_quality (MIN (quality (), other.quality ()));
373 : }
374 1373539 : return *this;
375 : }
376 :
377 34907471 : profile_probability operator- (const profile_probability &other) const
378 : {
379 69814942 : if (*this == never ()
380 34907471 : || other == never ())
381 1898509 : return *this;
382 33008962 : if (!initialized_p () || !other.initialized_p ())
383 2194971 : return uninitialized ();
384 30813991 : profile_probability ret;
385 30813991 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 30813991 : ret.set_quality (MIN (quality (), other.quality ()));
387 30813991 : return ret;
388 : }
389 :
390 11752711 : profile_probability &operator-= (const profile_probability &other)
391 : {
392 23505422 : if (*this == never ()
393 11752711 : || other == never ())
394 : return *this;
395 9398571 : if (!initialized_p () || !other.initialized_p ())
396 2143873 : return *this = uninitialized ();
397 : else
398 : {
399 7254698 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 43528043 : set_quality (MIN (quality (), other.quality ()));
401 : }
402 7254698 : return *this;
403 : }
404 :
405 1254012 : profile_probability operator* (const profile_probability &other) const
406 : {
407 2508024 : if (*this == never ()
408 1254012 : || other == never ())
409 125081 : return never ();
410 1128931 : if (!initialized_p () || !other.initialized_p ())
411 58536 : return uninitialized ();
412 1070395 : profile_probability ret;
413 1070395 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 1070395 : ret.set_quality (MIN (quality (), other.quality ()));
415 1070395 : return ret;
416 : }
417 :
418 145307 : profile_probability &operator*= (const profile_probability &other)
419 : {
420 290614 : if (*this == never ()
421 145307 : || other == never ())
422 438 : return *this = never ();
423 144869 : if (!initialized_p () || !other.initialized_p ())
424 12 : return *this = uninitialized ();
425 : else
426 : {
427 144857 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 144857 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : }
430 144857 : return *this;
431 : }
432 :
433 1823592 : profile_probability operator/ (const profile_probability &other) const
434 : {
435 1823592 : if (*this == never ())
436 102526 : return never ();
437 1721066 : if (!initialized_p () || !other.initialized_p ())
438 228607 : return uninitialized ();
439 1492459 : profile_probability ret;
440 : /* If we get probability above 1, mark it as unreliable and return 1. */
441 1492459 : if (m_val >= other.m_val)
442 : {
443 50538 : ret.m_val = max_probability;
444 50538 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : GUESSED));
446 50538 : return ret;
447 : }
448 1441921 : else if (!m_val)
449 74312 : ret.m_val = 0;
450 : else
451 : {
452 1367609 : gcc_checking_assert (other.m_val);
453 1367609 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : }
455 2873650 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 1441921 : return ret;
457 : }
458 :
459 299986 : profile_probability &operator/= (const profile_probability &other)
460 : {
461 299986 : if (*this == never ())
462 1999 : return *this = never ();
463 297987 : if (!initialized_p () || !other.initialized_p ())
464 0 : return *this = uninitialized ();
465 : else
466 : {
467 : /* If we get probability above 1, mark it as unreliable
468 : and return 1. */
469 297987 : if (m_val > other.m_val)
470 : {
471 516 : m_val = max_probability;
472 516 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : GUESSED));
474 516 : return *this;
475 : }
476 297471 : else if (!m_val)
477 : ;
478 : else
479 : {
480 293354 : gcc_checking_assert (other.m_val);
481 293354 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : }
483 297471 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : }
485 297471 : return *this;
486 : }
487 :
488 : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : == ORIG. This is useful e.g. when splitting a conditional
492 : branch like:
493 : if (cond)
494 : goto lab; // ORIG probability
495 : into
496 : if (cond1)
497 : goto lab; // FIRST = ORIG * CPROB probability
498 : if (cond2)
499 : goto lab; // SECOND probability
500 : such that the overall probability of jumping to lab remains
501 : the same. CPROB gives the relative probability between the
502 : branches. */
503 301178 : profile_probability split (const profile_probability &cprob)
504 : {
505 301178 : profile_probability ret = *this * cprob;
506 : /* The following is equivalent to:
507 : *this = cprob.invert () * *this / ret.invert ();
508 : Avoid scaling when overall outcome is supposed to be always.
509 : Without knowing that one is inverse of other, the result would be
510 : conservative. */
511 301178 : if (!(*this == always ()))
512 291675 : *this = (*this - ret) / ret.invert ();
513 301178 : return ret;
514 : }
515 :
516 : /* Return VAL scaled by this probability. Round initialized probabilities
517 : to the nearest integer, with halfway values away from zero. Treat an
518 : uninitialized probability as one half and truncate toward zero. */
519 259994 : gcov_type apply (gcov_type val) const
520 : {
521 259994 : if (*this == uninitialized ())
522 69 : return val / 2;
523 :
524 : /* A unit probability leaves VAL unchanged. Return it directly because
525 : the magnitude of the minimum gcov_type value is one greater than the
526 : maximum gcov_type value. */
527 259925 : if (m_val == max_probability)
528 : return val;
529 :
530 : /* Convert to unsigned before negating so that the minimum gcov_type
531 : value has a representable magnitude. Scale the magnitude with
532 : overflow-safe arithmetic, then restore the sign. */
533 258166 : gcov_type_unsigned magnitude
534 258182 : = val < 0 ? -(gcov_type_unsigned) val : (gcov_type_unsigned) val;
535 258182 : uint64_t scaled;
536 258182 : bool scaled_p
537 258182 : = safe_scale_64bit (magnitude, m_val, max_probability, &scaled);
538 : /* The scaled result fits in uint64_t. With the unit case handled above,
539 : it also fits in the nonnegative range of gcov_type. */
540 258182 : gcc_checking_assert (scaled_p);
541 258182 : gcc_checking_assert
542 : (scaled <= (gcov_type_unsigned) INTTYPE_MAXIMUM (gcov_type));
543 :
544 258182 : return val < 0 ? -(gcov_type) scaled : (gcov_type) scaled;
545 : }
546 :
547 : /* Return 1-*THIS. */
548 21697305 : profile_probability invert () const
549 : {
550 31597706 : return always() - *this;
551 : }
552 :
553 : /* Return THIS with quality dropped to GUESSED. */
554 881948 : profile_probability guessed () const
555 : {
556 881948 : profile_probability ret = *this;
557 881948 : ret.set_quality (GUESSED);
558 881912 : return ret;
559 : }
560 :
561 : /* Return THIS with quality dropped to AFDO. */
562 : profile_probability afdo () const
563 : {
564 : profile_probability ret = *this;
565 : ret.set_quality (AFDO);
566 : return ret;
567 : }
568 :
569 : /* Return *THIS * NUM / DEN. */
570 14727022 : profile_probability apply_scale (int64_t num, int64_t den) const
571 : {
572 14727022 : if (*this == never ())
573 16153 : return *this;
574 14710869 : if (!initialized_p ())
575 5922006 : return uninitialized ();
576 8788863 : profile_probability ret;
577 8788863 : uint64_t tmp;
578 8788863 : safe_scale_64bit (m_val, num, den, &tmp);
579 8788863 : ret.m_val = MIN (tmp, max_probability);
580 8788863 : ret.set_quality (MIN (quality (), ADJUSTED));
581 8788863 : return ret;
582 : }
583 :
584 : /* Return *THIS * NUM / DEN. */
585 13992 : profile_probability apply_scale (profile_probability num,
586 : profile_probability den) const
587 : {
588 13992 : if (*this == never ())
589 306 : return *this;
590 13686 : if (num == never ())
591 0 : return num;
592 13686 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
593 0 : return uninitialized ();
594 13686 : if (num == den)
595 0 : return *this;
596 13686 : gcc_checking_assert (den.m_val);
597 :
598 13686 : profile_probability ret;
599 13686 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
600 : max_probability);
601 13686 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
602 : num.quality ()), den.quality ()));
603 13686 : return ret;
604 : }
605 :
606 : /* Return true when the probability of edge is reliable.
607 :
608 : The profile guessing code is good at predicting branch outcome (i.e.
609 : taken/not taken), that is predicted right slightly over 75% of time.
610 : It is however notoriously poor on predicting the probability itself.
611 : In general the profile appear a lot flatter (with probabilities closer
612 : to 50%) than the reality so it is bad idea to use it to drive optimization
613 : such as those disabling dynamic branch prediction for well predictable
614 : branches.
615 :
616 : There are two exceptions - edges leading to noreturn edges and edges
617 : predicted by number of iterations heuristics are predicted well. This macro
618 : should be able to distinguish those, but at the moment it simply check for
619 : noreturn heuristic that is only one giving probability over 99% or below
620 : 1%. In future we might want to propagate reliability information across the
621 : CFG if we find this information useful on multiple places. */
622 0 : bool probably_reliable_p () const
623 : {
624 0 : if (quality () >= ADJUSTED)
625 : return true;
626 0 : if (!initialized_p ())
627 : return false;
628 0 : return m_val < max_probability / 100
629 0 : || m_val > max_probability - max_probability / 100;
630 : }
631 :
632 : /* Return false if profile_probability is bogus. */
633 4426271228 : bool verify () const
634 : {
635 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
636 4426271228 : if (m_val == uninitialized_probability)
637 958527441 : return quality () == GUESSED;
638 3467743787 : else if (quality () < GUESSED)
639 : return false;
640 3467743787 : return m_val <= max_probability;
641 : }
642 :
643 : /* Comparisons are three-state and conservative. False is returned if
644 : the inequality cannot be decided. */
645 9202700 : bool operator< (const profile_probability &other) const
646 : {
647 9202700 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
648 : }
649 :
650 17997173 : bool operator> (const profile_probability &other) const
651 : {
652 17997173 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
653 : }
654 :
655 5482916 : bool operator<= (const profile_probability &other) const
656 : {
657 5482916 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
658 : }
659 :
660 970143 : bool operator>= (const profile_probability &other) const
661 : {
662 970143 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
663 : }
664 :
665 17 : profile_probability operator* (int64_t num) const
666 : {
667 17 : return apply_scale (num, 1);
668 : }
669 :
670 : profile_probability operator*= (int64_t num)
671 : {
672 : *this = apply_scale (num, 1);
673 : return *this;
674 : }
675 :
676 14053959 : profile_probability operator/ (int64_t den) const
677 : {
678 14053959 : return apply_scale (1, den);
679 : }
680 :
681 247059 : profile_probability operator/= (int64_t den)
682 : {
683 247059 : *this = apply_scale (1, den);
684 247059 : return *this;
685 : }
686 :
687 : /* Compute n-th power. */
688 : profile_probability pow (int) const;
689 :
690 : /* Compute sware root. */
691 : profile_probability sqrt () const;
692 :
693 : /* Get the value of the probability. */
694 0 : uint32_t value () const { return m_val; }
695 :
696 : /* Get the quality of the probability. */
697 6167201814 : enum profile_quality quality () const
698 : {
699 5949479888 : return (profile_quality) (m_adjusted_quality
700 5949503162 : ? m_adjusted_quality + min_quality - 1
701 51489453 : : UNINITIALIZED_PROFILE);
702 : }
703 :
704 : /* Output THIS to F. */
705 : void dump (FILE *f) const;
706 :
707 : /* Output THIS to BUFFER. */
708 : void dump (char *buffer) const;
709 :
710 : /* Print THIS to stderr. */
711 : void debug () const;
712 :
713 : /* Return true if THIS is known to differ significantly from OTHER. */
714 : bool differs_from_p (profile_probability other) const;
715 :
716 : /* Return if difference is greater than 50%. */
717 : bool differs_lot_from_p (profile_probability other) const;
718 :
719 : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
720 : happens with COUNT2 probability. Return probability that either *THIS or
721 : OTHER happens. */
722 : profile_probability combine_with_count (profile_count count1,
723 : profile_probability other,
724 : profile_count count2) const;
725 :
726 : /* Return probability as sreal. */
727 : sreal to_sreal () const;
728 : /* LTO streaming support. */
729 : static profile_probability stream_in (class lto_input_block *);
730 : void stream_out (struct output_block *);
731 : void stream_out (struct lto_output_stream *);
732 : };
733 :
734 : /* Main data type to hold profile counters in GCC. Profile counts originate
735 : either from profile feedback, static profile estimation or both. We do not
736 : perform whole program profile propagation and thus profile estimation
737 : counters are often local to function, while counters from profile feedback
738 : (or special cases of profile estimation) can be used inter-procedurally.
739 :
740 : There are 3 basic types
741 : 1) local counters which are result of intra-procedural static profile
742 : estimation.
743 : 2) ipa counters which are result of profile feedback or special case
744 : of static profile estimation (such as in function main).
745 : 3) counters which counts as 0 inter-procedurally (because given function
746 : was never run in train feedback) but they hold local static profile
747 : estimate.
748 :
749 : Counters of type 1 and 3 cannot be mixed with counters of different type
750 : within operation (because whole function should use one type of counter)
751 : with exception that global zero mix in most operations where outcome is
752 : well defined.
753 :
754 : To take local counter and use it inter-procedurally use ipa member function
755 : which strips information irrelevant at the inter-procedural level.
756 :
757 : Counters are 61bit integers representing number of executions during the
758 : train run or normalized frequency within the function.
759 :
760 : As the profile is maintained during the compilation, many adjustments are
761 : made. Not all transformations can be made precisely, most importantly
762 : when code is being duplicated. It also may happen that part of CFG has
763 : profile counts known while other do not - for example when LTO optimizing
764 : partly profiled program or when profile was lost due to COMDAT merging.
765 :
766 : For this reason profile_count tracks more information than
767 : just unsigned integer and it is also ready for profile mismatches.
768 : The API of this data type represent operations that are natural
769 : on profile counts - sum, difference and operation with scales and
770 : probabilities. All operations are safe by never getting negative counts
771 : and they do end up in uninitialized scale if any of the parameters is
772 : uninitialized.
773 :
774 : All comparisons that are three state and handling of probabilities. Thus
775 : a < b is not equal to !(a >= b).
776 :
777 : The following pre-defined counts are available:
778 :
779 : profile_count::zero () for code that is known to execute zero times at
780 : runtime (this can be detected statically i.e. for paths leading to
781 : abort ();
782 : profile_count::one () for code that is known to execute once (such as
783 : main () function
784 : profile_count::uninitialized () for unknown execution count.
785 :
786 : */
787 :
788 : struct GTY(()) profile_count
789 : {
790 : public:
791 : /* Use 60bit to hold basic block counters. Should be at least
792 : 64bit. Although a counter cannot be negative, we use a signed
793 : type to hold various extra stages. */
794 :
795 : static const int n_bits = 60;
796 : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
797 : private:
798 : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
799 :
800 : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
801 : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
802 : incorrectly detects the alignment of a structure where the only
803 : 64-bit aligned object is a bit-field. We force the alignment of
804 : the entire field to mitigate this. */
805 : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
806 : #else
807 : #define UINT64_BIT_FIELD_ALIGN
808 : #endif
809 : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
810 : #undef UINT64_BIT_FIELD_ALIGN
811 : enum profile_quality m_quality : 4;
812 : public:
813 :
814 : /* Return true if both values can meaningfully appear in single function
815 : body. We have either all counters in function local or global, otherwise
816 : operations between them are not really defined well. */
817 5019725936 : bool compatible_p (const profile_count other) const
818 : {
819 9221039644 : if (!initialized_p () || !other.initialized_p ())
820 : return true;
821 16634480717 : if (*this == zero ()
822 155091513 : || other == zero ())
823 : return true;
824 : /* Do not allow nonzero global profile together with local guesses
825 : that are globally0. */
826 4041429155 : if (ipa ().nonzero_p ()
827 251899 : && !(other.ipa () == other))
828 0 : return false;
829 4041429155 : if (other.ipa ().nonzero_p ()
830 261084 : && !(ipa () == *this))
831 0 : return false;
832 :
833 4041429155 : return ipa_p () == other.ipa_p ();
834 : }
835 :
836 : /* Used for counters which are expected to be never executed. */
837 18798975378 : static profile_count zero ()
838 : {
839 14696772811 : return from_gcov_type (0);
840 : }
841 :
842 : static profile_count adjusted_zero ()
843 : {
844 : profile_count c;
845 : c.m_val = 0;
846 : c.m_quality = ADJUSTED;
847 : return c;
848 : }
849 :
850 : static profile_count afdo_zero ()
851 : {
852 : profile_count c;
853 : c.m_val = 0;
854 : c.m_quality = AFDO;
855 : return c;
856 : }
857 :
858 : static profile_count guessed_zero ()
859 : {
860 : profile_count c;
861 : c.m_val = 0;
862 : c.m_quality = GUESSED;
863 : return c;
864 : }
865 :
866 53802 : static profile_count one ()
867 : {
868 53604 : return from_gcov_type (1);
869 : }
870 :
871 : /* Value of counters which has not been initialized. Either because
872 : initialization did not happen yet or because profile is unknown. */
873 113287917 : static profile_count uninitialized ()
874 : {
875 113287917 : profile_count c;
876 113287917 : c.m_val = uninitialized_count;
877 113287917 : c.m_quality = GUESSED_LOCAL;
878 113287917 : return c;
879 : }
880 :
881 : /* Conversion to gcov_type is lossy. */
882 42380858 : gcov_type to_gcov_type () const
883 : {
884 0 : gcc_checking_assert (initialized_p ());
885 42380858 : return m_val;
886 : }
887 :
888 : /* Return true if value has been initialized. */
889 33166941880 : bool initialized_p () const
890 : {
891 11124168483 : return m_val != uninitialized_count;
892 : }
893 :
894 : /* Return true if value can be trusted. */
895 27104947 : bool reliable_p () const
896 : {
897 27104669 : return m_quality >= ADJUSTED;
898 : }
899 :
900 : /* Return true if value can be operated inter-procedurally. */
901 9151970511 : bool ipa_p () const
902 : {
903 5096676210 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
904 : }
905 :
906 : /* Return true if quality of profile is precise. */
907 60136125 : bool precise_p () const
908 : {
909 60136125 : return m_quality == PRECISE;
910 : }
911 :
912 : /* Get the value of the count. */
913 0 : uint64_t value () const { return m_val; }
914 :
915 : /* Get the quality of the count. */
916 98821457 : enum profile_quality quality () const { return m_quality; }
917 :
918 : /* When merging basic blocks, the two different profile counts are unified.
919 : Return true if this can be done without losing info about profile.
920 : The only case we care about here is when first BB contains something
921 : that makes it terminate in a way not visible in CFG. */
922 3382353 : bool ok_for_merging (profile_count other) const
923 : {
924 3382353 : if (m_quality < ADJUSTED
925 17312 : || other.m_quality < ADJUSTED)
926 : return true;
927 16299 : return !(other < *this);
928 : }
929 :
930 : /* When merging two BBs with different counts, pick common count that looks
931 : most representative. */
932 17861240 : profile_count merge (profile_count other) const
933 : {
934 896017 : if (*this == other || !other.initialized_p ()
935 895723 : || m_quality > other.m_quality)
936 16966833 : return *this;
937 894407 : if (other.m_quality > m_quality
938 894407 : || other > *this)
939 0 : return other;
940 894407 : return *this;
941 : }
942 :
943 : /* Basic operations. */
944 20290885107 : bool operator== (const profile_count &other) const
945 : {
946 15437780802 : return m_val == other.m_val && m_quality == other.m_quality;
947 : }
948 :
949 3947308 : profile_count operator+ (const profile_count &other) const
950 : {
951 3947308 : if (other == zero ())
952 2463 : return *this;
953 3944845 : if (*this == zero ())
954 234151 : return other;
955 3710694 : if (!initialized_p () || !other.initialized_p ())
956 6348 : return uninitialized ();
957 :
958 3704346 : profile_count ret;
959 3704346 : gcc_checking_assert (compatible_p (other));
960 3704346 : uint64_t ret_val = m_val + other.m_val;
961 3704346 : ret.m_val = MIN (ret_val, max_count);
962 3704346 : ret.m_quality = MIN (m_quality, other.m_quality);
963 3704346 : return ret;
964 : }
965 :
966 74791026 : profile_count &operator+= (const profile_count &other)
967 : {
968 74791026 : if (other == zero ())
969 5491017 : return *this;
970 69300009 : if (*this == zero ())
971 : {
972 42553274 : *this = other;
973 42553274 : return *this;
974 : }
975 26746735 : if (!initialized_p () || !other.initialized_p ())
976 352016 : return *this = uninitialized ();
977 : else
978 : {
979 26394719 : gcc_checking_assert (compatible_p (other));
980 26394719 : uint64_t ret_val = m_val + other.m_val;
981 26394719 : m_val = MIN (ret_val, max_count);
982 26394719 : m_quality = MIN (m_quality, other.m_quality);
983 : }
984 26394719 : return *this;
985 : }
986 :
987 21092403 : profile_count operator- (const profile_count &other) const
988 : {
989 22625041 : if (*this == zero () || other == zero ())
990 1517361 : return *this;
991 19575042 : if (!initialized_p () || !other.initialized_p ())
992 8919597 : return uninitialized ();
993 10655445 : gcc_checking_assert (compatible_p (other));
994 10655445 : profile_count ret;
995 10655445 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
996 10655445 : ret.m_quality = MIN (m_quality, other.m_quality);
997 10655445 : return ret;
998 : }
999 :
1000 9193028 : profile_count &operator-= (const profile_count &other)
1001 : {
1002 9949021 : if (*this == zero () || other == zero ())
1003 : return *this;
1004 8902748 : if (!initialized_p () || !other.initialized_p ())
1005 353203 : return *this = uninitialized ();
1006 : else
1007 : {
1008 8549545 : gcc_checking_assert (compatible_p (other));
1009 8549545 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
1010 8549545 : m_quality = MIN (m_quality, other.m_quality);
1011 : }
1012 8549545 : return *this;
1013 : }
1014 :
1015 : /* Return false if profile_count is bogus. */
1016 3382722090 : bool verify () const
1017 : {
1018 3382722090 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1019 3382722090 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1020 : }
1021 :
1022 : /* Comparisons are three-state and conservative. False is returned if
1023 : the inequality cannot be decided. */
1024 1117649601 : bool operator< (const profile_count &other) const
1025 : {
1026 1117649601 : if (!initialized_p () || !other.initialized_p ())
1027 : return false;
1028 1112727819 : if (*this == zero ())
1029 4342190 : return !(other == zero ());
1030 1110556724 : if (other == zero ())
1031 720 : return false;
1032 1110556004 : gcc_checking_assert (compatible_p (other));
1033 1110556004 : return m_val < other.m_val;
1034 : }
1035 :
1036 39135515 : bool operator> (const profile_count &other) const
1037 : {
1038 39135515 : if (!initialized_p () || !other.initialized_p ())
1039 : return false;
1040 34026941 : if (*this == zero ())
1041 223986 : return false;
1042 33802955 : if (other == zero ())
1043 1543198 : return !(*this == zero ());
1044 33031356 : gcc_checking_assert (compatible_p (other));
1045 33031356 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1046 : }
1047 :
1048 : bool operator< (const gcov_type other) const
1049 : {
1050 : gcc_checking_assert (ipa_p ());
1051 : gcc_checking_assert (other >= 0);
1052 : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1053 : }
1054 :
1055 12002 : bool operator> (const gcov_type other) const
1056 : {
1057 12002 : gcc_checking_assert (ipa_p ());
1058 12002 : gcc_checking_assert (other >= 0);
1059 12002 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1060 : }
1061 :
1062 1244583 : bool operator<= (const profile_count &other) const
1063 : {
1064 1244583 : if (!initialized_p () || !other.initialized_p ())
1065 : return false;
1066 1244583 : if (*this == zero ())
1067 4261 : return true;
1068 1240322 : if (other == zero ())
1069 0 : return (*this == zero ());
1070 1240322 : gcc_checking_assert (compatible_p (other));
1071 1240322 : return m_val <= other.m_val;
1072 : }
1073 :
1074 1425211 : bool operator>= (const profile_count &other) const
1075 : {
1076 1425211 : if (!initialized_p () || !other.initialized_p ())
1077 : return false;
1078 1424089 : if (other == zero ())
1079 3040 : return true;
1080 1421049 : if (*this == zero ())
1081 23690 : return (other == zero ());
1082 1409204 : gcc_checking_assert (compatible_p (other));
1083 1409204 : return m_val >= other.m_val;
1084 : }
1085 :
1086 103144 : bool operator<= (const gcov_type other) const
1087 : {
1088 103144 : gcc_checking_assert (ipa_p ());
1089 103144 : gcc_checking_assert (other >= 0);
1090 103144 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1091 : }
1092 :
1093 65949 : bool operator>= (const gcov_type other) const
1094 : {
1095 65949 : gcc_checking_assert (ipa_p ());
1096 65949 : gcc_checking_assert (other >= 0);
1097 65949 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1098 : }
1099 :
1100 932238983 : profile_count operator* (int64_t num) const
1101 : {
1102 932162353 : return apply_scale (num, 1);
1103 : }
1104 :
1105 : profile_count operator*= (int64_t num)
1106 : {
1107 : *this = apply_scale (num, 1);
1108 : return *this;
1109 : }
1110 :
1111 : profile_count operator* (const sreal &num) const;
1112 : profile_count operator*= (const sreal &num);
1113 :
1114 2402926 : profile_count operator/ (int64_t den) const
1115 : {
1116 2402926 : return apply_scale (1, den);
1117 : }
1118 :
1119 8 : profile_count operator/= (int64_t den)
1120 : {
1121 0 : *this = apply_scale (1, den);
1122 8 : return *this;
1123 : }
1124 :
1125 : /* Return true when value is not zero and can be used for scaling.
1126 : This is different from *this > 0 because that requires counter to
1127 : be IPA. */
1128 8280198792 : bool nonzero_p () const
1129 : {
1130 8280198149 : return initialized_p () && m_val != 0;
1131 : }
1132 :
1133 : /* Make counter forcibly nonzero. */
1134 : profile_count force_nonzero () const;
1135 :
1136 :
1137 : /* Return maximum of A and B. If one of values is uninitialized return the
1138 : other. */
1139 :
1140 : static profile_count
1141 82544534 : max_prefer_initialized (const profile_count a, const profile_count b)
1142 : {
1143 82544534 : if (!a.initialized_p ())
1144 20687734 : return b;
1145 61856800 : if (!b.initialized_p ())
1146 2172507 : return a;
1147 59684293 : profile_count ret;
1148 59684293 : gcc_checking_assert (a.compatible_p (b));
1149 59684293 : ret.m_val = MAX (a.m_val, b.m_val);
1150 59684293 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1151 59684293 : return ret;
1152 : }
1153 :
1154 : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1155 : accordingly. */
1156 : profile_count apply_probability (int prob) const
1157 : {
1158 : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1159 : if (m_val == 0)
1160 : return *this;
1161 : if (!initialized_p ())
1162 : return uninitialized ();
1163 : profile_count ret;
1164 : uint64_t tmp;
1165 : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1166 : ret.m_val = tmp;
1167 : ret.m_quality = MIN (m_quality, ADJUSTED);
1168 : return ret;
1169 : }
1170 :
1171 : /* Scale counter according to PROB. */
1172 398905678 : profile_count apply_probability (profile_probability prob) const
1173 : {
1174 411203224 : if (*this == zero () || prob == profile_probability::always ())
1175 129549051 : return *this;
1176 269356627 : if (prob == profile_probability::never ())
1177 20564728 : return zero ();
1178 248791899 : if (!initialized_p () || !prob.initialized_p ())
1179 31108290 : return uninitialized ();
1180 217683609 : profile_count ret;
1181 217683609 : uint64_t tmp;
1182 217683609 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1183 : &tmp);
1184 217683609 : ret.m_val = tmp;
1185 217706883 : ret.m_quality = MIN (m_quality, prob.quality ());
1186 217683609 : return ret;
1187 : }
1188 :
1189 : /* Return *THIS * NUM / DEN. */
1190 1061461740 : profile_count apply_scale (int64_t num, int64_t den) const
1191 : {
1192 1061461740 : if (m_val == 0)
1193 25052959 : return *this;
1194 1036408781 : if (!initialized_p ())
1195 4198097 : return uninitialized ();
1196 1032210684 : profile_count ret;
1197 1032210684 : uint64_t tmp;
1198 :
1199 1032210684 : gcc_checking_assert (num >= 0 && den > 0);
1200 1032210684 : safe_scale_64bit (m_val, num, den, &tmp);
1201 1032210684 : ret.m_val = MIN (tmp, max_count);
1202 1032210684 : ret.m_quality = MIN (m_quality, ADJUSTED);
1203 1032210684 : return ret;
1204 : }
1205 :
1206 30634435 : profile_count apply_scale (profile_count num, profile_count den) const
1207 : {
1208 30634435 : if (*this == zero ())
1209 2709515 : return *this;
1210 27924920 : if (num == zero ())
1211 94198 : return num;
1212 27830722 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1213 7751020 : return uninitialized ();
1214 20079702 : if (num == den)
1215 5857127 : return *this;
1216 14222575 : gcc_checking_assert (den.m_val);
1217 :
1218 14222575 : profile_count ret;
1219 14222575 : uint64_t val;
1220 14222575 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1221 14222575 : ret.m_val = MIN (val, max_count);
1222 14222575 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1223 : num.m_quality), den.m_quality);
1224 : /* Be sure that ret is not local if num is global.
1225 : Also ensure that ret is not global0 when num is global. */
1226 14222575 : if (num.ipa_p ())
1227 : {
1228 : /* This is common case of AFDO scaling when we upgrade
1229 : GLOBAL0_AFDO function to AFDO. Be sure that result
1230 : is AFDO and not GUESSED (which is unnecessarily low). */
1231 1871 : if (num.m_quality == AFDO
1232 0 : && (ret.m_quality != GUESSED
1233 0 : && ret.m_quality != GUESSED_LOCAL))
1234 0 : ret.m_quality = AFDO;
1235 : else
1236 3439 : ret.m_quality = MAX (ret.m_quality,
1237 : num == num.ipa () ? GUESSED : num.m_quality);
1238 : }
1239 14222575 : return ret;
1240 : }
1241 :
1242 : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1243 18376120 : profile_count guessed_local () const
1244 : {
1245 18376120 : profile_count ret = *this;
1246 18376120 : if (!initialized_p ())
1247 0 : return *this;
1248 18376120 : ret.m_quality = GUESSED_LOCAL;
1249 18376120 : return ret;
1250 : }
1251 :
1252 : /* We know that profile is globally 0 but keep local profile if present. */
1253 794 : profile_count global0 () const
1254 : {
1255 794 : profile_count ret = *this;
1256 794 : if (!initialized_p ())
1257 0 : return *this;
1258 794 : ret.m_quality = GUESSED_GLOBAL0;
1259 794 : return ret;
1260 : }
1261 :
1262 : /* We know that profile is globally afdo 0 but keep local profile
1263 : if present. */
1264 0 : profile_count global0afdo () const
1265 : {
1266 0 : profile_count ret = *this;
1267 0 : if (!initialized_p ())
1268 0 : return *this;
1269 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1270 0 : return ret;
1271 : }
1272 :
1273 : /* We know that profile is globally adjusted 0 but keep local profile
1274 : if present. */
1275 64 : profile_count global0adjusted () const
1276 : {
1277 64 : profile_count ret = *this;
1278 64 : if (!initialized_p ())
1279 0 : return *this;
1280 64 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1281 64 : return ret;
1282 : }
1283 :
1284 : /* Return THIS with quality dropped to GUESSED. */
1285 397 : profile_count guessed () const
1286 : {
1287 397 : profile_count ret = *this;
1288 397 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1289 397 : return ret;
1290 : }
1291 :
1292 : /* Return THIS with quality GUESSED. */
1293 0 : profile_count force_guessed () const
1294 : {
1295 0 : profile_count ret = *this;
1296 0 : gcc_checking_assert (initialized_p ());
1297 0 : ret.m_quality = GUESSED;
1298 0 : return ret;
1299 : }
1300 :
1301 : /* Return variant of profile count which is always safe to compare
1302 : across functions. */
1303 9237015532 : profile_count ipa () const
1304 : {
1305 9237015532 : if (m_quality > GUESSED_GLOBAL0)
1306 14189792 : return *this;
1307 9222825740 : if (m_quality == GUESSED_GLOBAL0)
1308 211069 : return zero ();
1309 9222614671 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1310 17449 : return adjusted_zero ();
1311 9222597222 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1312 0 : return afdo_zero ();
1313 9222597222 : return uninitialized ();
1314 : }
1315 :
1316 : /* Return THIS with quality dropped to AFDO. */
1317 0 : profile_count afdo () const
1318 : {
1319 0 : profile_count ret = *this;
1320 0 : ret.m_quality = AFDO;
1321 0 : return ret;
1322 : }
1323 :
1324 : /* Return probability of event with counter THIS within event with counter
1325 : OVERALL. */
1326 1102225380 : profile_probability probability_in (const profile_count overall) const
1327 : {
1328 1102243276 : if (*this == zero ()
1329 17896 : && !(overall == zero ()))
1330 2219 : return profile_probability::never ();
1331 1102222632 : if (!initialized_p () || !overall.initialized_p ()
1332 2204445793 : || !overall.m_val)
1333 58761 : return profile_probability::uninitialized ();
1334 1315904237 : if (*this == overall && m_quality == PRECISE)
1335 30472 : return profile_probability::always ();
1336 1102133928 : profile_probability ret;
1337 1102133928 : gcc_checking_assert (compatible_p (overall));
1338 :
1339 1102133928 : if (overall.m_val < m_val)
1340 : {
1341 162060 : ret.m_val = profile_probability::max_probability;
1342 162060 : ret.set_quality (GUESSED);
1343 162060 : return ret;
1344 : }
1345 : else
1346 : {
1347 1101971868 : gcc_checking_assert (overall.m_val);
1348 1101971868 : uint64_t tmp;
1349 1101971868 : safe_scale_64bit (m_val, profile_probability::max_probability,
1350 : overall.m_val, &tmp);
1351 1101971868 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1352 1101971868 : ret.m_val = tmp;
1353 : }
1354 1101971868 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1355 : GUESSED), ADJUSTED));
1356 1101971868 : return ret;
1357 : }
1358 :
1359 : /* Return true if profile count is very large, so we risk overflows
1360 : with loop transformations. */
1361 : bool
1362 1057179 : very_large_p ()
1363 : {
1364 1057179 : if (!initialized_p ())
1365 : return false;
1366 1057179 : return m_val > max_count / 65536;
1367 : }
1368 :
1369 : int to_frequency (struct function *fun) const;
1370 : int to_cgraph_frequency (profile_count entry_bb_count) const;
1371 : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1372 :
1373 : /* Output THIS to F. */
1374 : void dump (FILE *f, struct function *fun = NULL) const;
1375 :
1376 : /* Print THIS to stderr. */
1377 : void debug () const;
1378 :
1379 : /* Return true if THIS is known to differ significantly from OTHER. */
1380 : bool differs_from_p (profile_count other) const;
1381 :
1382 : /* We want to scale profile across function boundary from NUM to DEN.
1383 : Take care of the side case when NUM and DEN are zeros of incompatible
1384 : kinds. */
1385 : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1386 :
1387 : /* THIS is a count of bb which is known to be executed IPA times.
1388 : Combine this information into bb counter. This means returning IPA
1389 : if it is nonzero, not changing anything if IPA is uninitialized
1390 : and if IPA is zero, turning THIS into corresponding local profile with
1391 : global0. */
1392 : profile_count combine_with_ipa_count (profile_count ipa);
1393 :
1394 : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1395 : profile_count combine_with_ipa_count_within
1396 : (profile_count ipa, profile_count ipa2);
1397 :
1398 : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1399 : Conversions back and forth are used to read the coverage and get it
1400 : into internal representation. */
1401 : static profile_count from_gcov_type (gcov_type v,
1402 : profile_quality quality = PRECISE);
1403 :
1404 : /* LTO streaming support. */
1405 : static profile_count stream_in (class lto_input_block *);
1406 : void stream_out (struct output_block *);
1407 : void stream_out (struct lto_output_stream *);
1408 : };
1409 : #endif
|
