Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2025 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : : GUESSED_GLOBAL0_AFDO,
42 : :
43 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : : GUESSED_GLOBAL0_ADJUSTED,
45 : :
46 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : : better. This is the case of functions not run in profile feedback.
48 : : Never used by probabilities. */
49 : : GUESSED_GLOBAL0,
50 : :
51 : : /* Profile is based on static branch prediction heuristics. It may or may
52 : : not reflect the reality but it can be compared interprocedurally
53 : : (for example, we inlined function w/o profile feedback into function
54 : : with feedback and propagated from that). */
55 : : GUESSED,
56 : :
57 : : /* Profile was determined by autofdo. */
58 : : AFDO,
59 : :
60 : : /* Profile was originally based on feedback but it was adjusted
61 : : by code duplicating optimization. It may not precisely reflect the
62 : : particular code path. */
63 : : ADJUSTED,
64 : :
65 : : /* Profile was read from profile feedback or determined by accurate static
66 : : method. */
67 : : PRECISE
68 : : };
69 : :
70 : : extern const char *profile_quality_as_string (enum profile_quality);
71 : : extern bool parse_profile_quality (const char *value,
72 : : profile_quality *quality);
73 : :
74 : : /* The base value for branch probability notes and edge probabilities. */
75 : : #define REG_BR_PROB_BASE 10000
76 : :
77 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 : :
79 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 : :
81 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 : :
83 : : inline bool
84 : 2351955718 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : : {
86 : : #if (GCC_VERSION >= 5000)
87 : 2351955718 : uint64_t tmp;
88 : 2351955718 : if (!__builtin_mul_overflow (a, b, &tmp)
89 : 2351955718 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : : {
91 : 2349694260 : *res = tmp / c;
92 : 2349694260 : return true;
93 : : }
94 : 2261458 : if (c == 1)
95 : : {
96 : 35757 : *res = (uint64_t) -1;
97 : 35757 : return false;
98 : : }
99 : : #else
100 : : if (a < ((uint64_t)1 << 31)
101 : : && b < ((uint64_t)1 << 31)
102 : : && c < ((uint64_t)1 << 31))
103 : : {
104 : : *res = (a * b + (c / 2)) / c;
105 : : return true;
106 : : }
107 : : #endif
108 : 2225701 : return slow_safe_scale_64bit (a, b, c, res);
109 : : }
110 : :
111 : : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : : with capping so probability is always in range [0,1] and scaling requiring
113 : : values greater than 1 needs to be represented otherwise.
114 : :
115 : : In addition to actual value the quality of profile is tracked and propagated
116 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : : that has not been determined yet (for example because of
118 : : -fno-guess-branch-probability)
119 : :
120 : : Typically probabilities are derived from profile feedback (via
121 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : : thorough the compilation.
123 : :
124 : : Named probabilities are available:
125 : : - never (0 probability)
126 : : - guessed_never
127 : : - very_unlikely (1/2000 probability)
128 : : - unlikely (1/5 probability)
129 : : - even (1/2 probability)
130 : : - likely (4/5 probability)
131 : : - very_likely (1999/2000 probability)
132 : : - guessed_always
133 : : - always
134 : :
135 : : Named probabilities except for never/always are assumed to be statically
136 : : guessed and thus not necessarily accurate. The difference between never
137 : : and guessed_never is that the first one should be used only in case that
138 : : well behaving program will very likely not execute the "never" path.
139 : : For example if the path is going to abort () call or it exception handling.
140 : :
141 : : Always and guessed_always probabilities are symmetric.
142 : :
143 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : : integer arithmetics. Once the code is converted to branch probabilities,
145 : : these conversions will probably go away because they are lossy.
146 : : */
147 : :
148 : : class GTY((user)) profile_probability
149 : : {
150 : : static const int n_bits = 29;
151 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : : will lead to harder multiplication sequences. */
153 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : : static const uint32_t uninitialized_probability
155 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : : /* For probibilityes quality is either UNINITIALIZED (0)
157 : : or greated then GUESSED. To save bits we store it in
158 : : adjusted form which skips the invalid values. */
159 : : static const int min_quality = GUESSED;
160 : :
161 : : uint32_t m_val : 29;
162 : : unsigned m_adjusted_quality : 3;
163 : :
164 : : friend struct profile_count;
165 : :
166 : : /* Set the quality of the probability. */
167 : : void
168 : 5398206329 : set_quality (profile_quality quality)
169 : : {
170 : 1140838843 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : : || (quality >= min_quality && quality <= PRECISE));
172 : 1140838843 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 : 1139092364 : }
174 : :
175 : : public:
176 : 4259113965 : profile_probability (): m_val (uninitialized_probability)
177 : 2340282540 : { set_quality (GUESSED); }
178 : :
179 : 372 : profile_probability (uint32_t val, profile_quality quality):
180 : 372 : m_val (val)
181 : 372 : { set_quality (quality); }
182 : :
183 : : /* Named probabilities. */
184 : 757831831 : static profile_probability never ()
185 : : {
186 : 757831831 : profile_probability ret;
187 : 757831831 : ret.m_val = 0;
188 : 757831831 : ret.set_quality (PRECISE);
189 : 448614072 : return ret;
190 : : }
191 : :
192 : 5287518 : static profile_probability guessed_never ()
193 : : {
194 : 5287518 : profile_probability ret;
195 : 5287518 : ret.m_val = 0;
196 : 5287518 : ret.set_quality (GUESSED);
197 : 5287518 : return ret;
198 : : }
199 : :
200 : 5499364 : static profile_probability very_unlikely ()
201 : : {
202 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 : 10998728 : profile_probability r = guessed_always () / 2000;
204 : 5499364 : r.m_val--;
205 : 5499364 : return r;
206 : : }
207 : :
208 : 81154 : static profile_probability unlikely ()
209 : : {
210 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 : 162308 : profile_probability r = guessed_always () / 5;
212 : 81154 : r.m_val--;
213 : 81154 : return r;
214 : : }
215 : :
216 : 802881 : static profile_probability even ()
217 : : {
218 : 1605762 : return guessed_always () / 2;
219 : : }
220 : :
221 : 338270 : static profile_probability very_likely ()
222 : : {
223 : 338270 : return always () - very_unlikely ();
224 : : }
225 : :
226 : 32178 : static profile_probability likely ()
227 : : {
228 : 32178 : return always () - unlikely ();
229 : : }
230 : : /* Return true when value is not zero and can be used for scaling. */
231 : 1433 : bool nonzero_p () const
232 : : {
233 : 1433 : return initialized_p () && m_val != 0;
234 : : }
235 : :
236 : 6691220 : static profile_probability guessed_always ()
237 : : {
238 : 6691220 : profile_probability ret;
239 : 6691220 : ret.m_val = max_probability;
240 : 6691220 : ret.set_quality (GUESSED);
241 : 6489137 : return ret;
242 : : }
243 : :
244 : 555218936 : static profile_probability always ()
245 : : {
246 : 555218936 : profile_probability ret;
247 : 555218936 : ret.m_val = max_probability;
248 : 555218936 : ret.set_quality (PRECISE);
249 : 376896725 : return ret;
250 : : }
251 : :
252 : : /* Probabilities which has not been initialized. Either because
253 : : initialization did not happen yet or because profile is unknown. */
254 : 903804106 : static profile_probability uninitialized ()
255 : : {
256 : 903804106 : profile_probability c;
257 : 903804106 : c.m_val = uninitialized_probability;
258 : 903804106 : c.set_quality (GUESSED);
259 : 890301101 : return c;
260 : : }
261 : :
262 : : /* Return true if value has been initialized. */
263 : 5708080332 : bool initialized_p () const
264 : : {
265 : 3985195471 : return m_val != uninitialized_probability;
266 : : }
267 : :
268 : : /* Return true if value can be trusted. */
269 : 1646389 : bool reliable_p () const
270 : : {
271 : 125766805 : return quality () >= ADJUSTED;
272 : : }
273 : :
274 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : : this is mostly to support legacy code and should go away. */
276 : 3223393 : static profile_probability from_reg_br_prob_base (int v)
277 : : {
278 : 3223393 : profile_probability ret;
279 : 3223393 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 : 3223393 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 : 3223393 : ret.set_quality (GUESSED);
282 : 3223393 : return ret;
283 : : }
284 : :
285 : : /* Return THIS with quality set to ADJUSTED. */
286 : 42 : profile_probability adjusted () const
287 : : {
288 : 42 : profile_probability ret = *this;
289 : 42 : if (!initialized_p ())
290 : 0 : return *this;
291 : 42 : ret.set_quality (ADJUSTED);
292 : 42 : return ret;
293 : : }
294 : :
295 : 1121102151 : int to_reg_br_prob_base () const
296 : : {
297 : 1121102151 : gcc_checking_assert (initialized_p ());
298 : 1121102151 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : : }
300 : :
301 : : /* Conversion to and from RTL representation of profile probabilities. */
302 : 493754441 : static profile_probability from_reg_br_prob_note (int v)
303 : : {
304 : 493754441 : profile_probability ret;
305 : 493754441 : ret.m_val = ((unsigned int)v) / 8;
306 : 493754441 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 : 15866558 : return ret;
308 : : }
309 : :
310 : 464591920 : int to_reg_br_prob_note () const
311 : : {
312 : 464591920 : gcc_checking_assert (initialized_p ());
313 : 464591920 : int ret = m_val * 8 + m_adjusted_quality;
314 : 464591920 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 : 464591920 : return ret;
316 : : }
317 : :
318 : : /* Return VAL1/VAL2. */
319 : 3094 : static profile_probability probability_in_gcov_type
320 : : (gcov_type val1, gcov_type val2)
321 : : {
322 : 3094 : profile_probability ret;
323 : 3094 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 : 3094 : if (val1 > val2)
325 : : ret.m_val = max_probability;
326 : : else
327 : : {
328 : 3094 : uint64_t tmp;
329 : 3094 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 : 3094 : gcc_checking_assert (tmp <= max_probability);
331 : 3094 : ret.m_val = tmp;
332 : : }
333 : 3094 : ret.set_quality (PRECISE);
334 : 3094 : return ret;
335 : : }
336 : :
337 : : /* Basic operations. */
338 : 1644806335 : bool operator== (const profile_probability &other) const
339 : : {
340 : 2306915243 : return m_val == other.m_val && quality () == other.quality ();
341 : : }
342 : :
343 : 6724252 : profile_probability operator+ (const profile_probability &other) const
344 : : {
345 : 6724252 : if (other == never ())
346 : 11802 : return *this;
347 : 6712450 : if (*this == never ())
348 : 23975 : return other;
349 : 6688475 : if (!initialized_p () || !other.initialized_p ())
350 : 5437543 : return uninitialized ();
351 : :
352 : 1250932 : profile_probability ret;
353 : 1250932 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 : 1250932 : ret.set_quality (MIN (quality (), other.quality ()));
355 : 1250932 : return ret;
356 : : }
357 : :
358 : 6242477 : profile_probability &operator+= (const profile_probability &other)
359 : : {
360 : 6242477 : if (other == never ())
361 : : return *this;
362 : 5333265 : if (*this == never ())
363 : : {
364 : 868167 : *this = other;
365 : 868167 : return *this;
366 : : }
367 : 4465098 : if (!initialized_p () || !other.initialized_p ())
368 : 2914163 : return *this = uninitialized ();
369 : : else
370 : : {
371 : 1550935 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 : 1550935 : set_quality (MIN (quality (), other.quality ()));
373 : : }
374 : 1550935 : return *this;
375 : : }
376 : :
377 : 34338328 : profile_probability operator- (const profile_probability &other) const
378 : : {
379 : 68676656 : if (*this == never ()
380 : 34338328 : || other == never ())
381 : 1758749 : return *this;
382 : 32579579 : if (!initialized_p () || !other.initialized_p ())
383 : 2156070 : return uninitialized ();
384 : 30423509 : profile_probability ret;
385 : 30423509 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 : 30423509 : ret.set_quality (MIN (quality (), other.quality ()));
387 : 30423509 : return ret;
388 : : }
389 : :
390 : 11313695 : profile_probability &operator-= (const profile_probability &other)
391 : : {
392 : 22627390 : if (*this == never ()
393 : 11313695 : || other == never ())
394 : 2141808 : return *this;
395 : 9171887 : if (!initialized_p () || !other.initialized_p ())
396 : 2132975 : return *this = uninitialized ();
397 : : else
398 : : {
399 : 7038912 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 : 7038912 : set_quality (MIN (quality (), other.quality ()));
401 : : }
402 : 7038912 : return *this;
403 : : }
404 : :
405 : 1177079 : profile_probability operator* (const profile_probability &other) const
406 : : {
407 : 2354158 : if (*this == never ()
408 : 1177079 : || other == never ())
409 : 109721 : return never ();
410 : 1067358 : if (!initialized_p () || !other.initialized_p ())
411 : 58010 : return uninitialized ();
412 : 1009348 : profile_probability ret;
413 : 1009348 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 : 1009348 : ret.set_quality (MIN (quality (), other.quality ()));
415 : 1009348 : return ret;
416 : : }
417 : :
418 : 161728 : profile_probability &operator*= (const profile_probability &other)
419 : : {
420 : 323456 : if (*this == never ()
421 : 161728 : || other == never ())
422 : 375 : return *this = never ();
423 : 161353 : if (!initialized_p () || !other.initialized_p ())
424 : 9 : return *this = uninitialized ();
425 : : else
426 : : {
427 : 161344 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 : 316231 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : : }
430 : 161344 : return *this;
431 : : }
432 : :
433 : 1801053 : profile_probability operator/ (const profile_probability &other) const
434 : : {
435 : 1801053 : if (*this == never ())
436 : 107040 : return never ();
437 : 1694013 : if (!initialized_p () || !other.initialized_p ())
438 : 224994 : return uninitialized ();
439 : 1469019 : profile_probability ret;
440 : : /* If we get probability above 1, mark it as unreliable and return 1. */
441 : 1469019 : if (m_val >= other.m_val)
442 : : {
443 : 20423 : ret.m_val = max_probability;
444 : 20423 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : : GUESSED));
446 : 20423 : return ret;
447 : : }
448 : 1448596 : else if (!m_val)
449 : 59010 : ret.m_val = 0;
450 : : else
451 : : {
452 : 1389586 : gcc_checking_assert (other.m_val);
453 : 1389586 : ret.m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
454 : : }
455 : 2888870 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
456 : 1448596 : return ret;
457 : : }
458 : :
459 : 378899 : profile_probability &operator/= (const profile_probability &other)
460 : : {
461 : 378899 : if (*this == never ())
462 : 1316 : return *this = never ();
463 : 377583 : if (!initialized_p () || !other.initialized_p ())
464 : 0 : return *this = uninitialized ();
465 : : else
466 : : {
467 : : /* If we get probability above 1, mark it as unreliable
468 : : and return 1. */
469 : 377583 : if (m_val > other.m_val)
470 : : {
471 : 643 : m_val = max_probability;
472 : 643 : set_quality (MIN (MIN (quality (), other.quality ()),
473 : : GUESSED));
474 : 643 : return *this;
475 : : }
476 : 376940 : else if (!m_val)
477 : : ;
478 : : else
479 : : {
480 : 374826 : gcc_checking_assert (other.m_val);
481 : 374826 : m_val = RDIV ((uint64_t)m_val * max_probability, other.m_val);
482 : : }
483 : 687657 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
484 : : }
485 : 376940 : return *this;
486 : : }
487 : :
488 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
489 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
490 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
491 : : == ORIG. This is useful e.g. when splitting a conditional
492 : : branch like:
493 : : if (cond)
494 : : goto lab; // ORIG probability
495 : : into
496 : : if (cond1)
497 : : goto lab; // FIRST = ORIG * CPROB probability
498 : : if (cond2)
499 : : goto lab; // SECOND probability
500 : : such that the overall probability of jumping to lab remains
501 : : the same. CPROB gives the relative probability between the
502 : : branches. */
503 : 289921 : profile_probability split (const profile_probability &cprob)
504 : : {
505 : 289921 : profile_probability ret = *this * cprob;
506 : : /* The following is equivalent to:
507 : : *this = cprob.invert () * *this / ret.invert ();
508 : : Avoid scaling when overall outcome is supposed to be always.
509 : : Without knowing that one is inverse of other, the result would be
510 : : conservative. */
511 : 289921 : if (!(*this == always ()))
512 : 281211 : *this = (*this - ret) / ret.invert ();
513 : 289921 : return ret;
514 : : }
515 : :
516 : 278561 : gcov_type apply (gcov_type val) const
517 : : {
518 : 278561 : if (*this == uninitialized ())
519 : 61 : return val / 2;
520 : 278500 : return RDIV (val * m_val, max_probability);
521 : : }
522 : :
523 : : /* Return 1-*THIS. */
524 : 21584104 : profile_probability invert () const
525 : : {
526 : 31019994 : return always() - *this;
527 : : }
528 : :
529 : : /* Return THIS with quality dropped to GUESSED. */
530 : 885307 : profile_probability guessed () const
531 : : {
532 : 885307 : profile_probability ret = *this;
533 : 885307 : ret.set_quality (GUESSED);
534 : 885307 : return ret;
535 : : }
536 : :
537 : : /* Return THIS with quality dropped to AFDO. */
538 : : profile_probability afdo () const
539 : : {
540 : : profile_probability ret = *this;
541 : : ret.set_quality (AFDO);
542 : : return ret;
543 : : }
544 : :
545 : : /* Return *THIS * NUM / DEN. */
546 : 14906657 : profile_probability apply_scale (int64_t num, int64_t den) const
547 : : {
548 : 14906657 : if (*this == never ())
549 : 14637 : return *this;
550 : 14892020 : if (!initialized_p ())
551 : 5841040 : return uninitialized ();
552 : 9050980 : profile_probability ret;
553 : 9050980 : uint64_t tmp;
554 : 9050980 : safe_scale_64bit (m_val, num, den, &tmp);
555 : 9050980 : ret.m_val = MIN (tmp, max_probability);
556 : 9050980 : ret.set_quality (MIN (quality (), ADJUSTED));
557 : 9050980 : return ret;
558 : : }
559 : :
560 : : /* Return *THIS * NUM / DEN. */
561 : 7177 : profile_probability apply_scale (profile_probability num,
562 : : profile_probability den) const
563 : : {
564 : 7177 : if (*this == never ())
565 : 42 : return *this;
566 : 7135 : if (num == never ())
567 : 0 : return num;
568 : 7135 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
569 : 0 : return uninitialized ();
570 : 7135 : if (num == den)
571 : 0 : return *this;
572 : 7135 : gcc_checking_assert (den.m_val);
573 : :
574 : 7135 : profile_probability ret;
575 : 7135 : ret.m_val = MIN (RDIV ((uint64_t)m_val * num.m_val, den.m_val),
576 : : max_probability);
577 : 7135 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
578 : : num.quality ()), den.quality ()));
579 : 7135 : return ret;
580 : : }
581 : :
582 : : /* Return true when the probability of edge is reliable.
583 : :
584 : : The profile guessing code is good at predicting branch outcome (i.e.
585 : : taken/not taken), that is predicted right slightly over 75% of time.
586 : : It is however notoriously poor on predicting the probability itself.
587 : : In general the profile appear a lot flatter (with probabilities closer
588 : : to 50%) than the reality so it is bad idea to use it to drive optimization
589 : : such as those disabling dynamic branch prediction for well predictable
590 : : branches.
591 : :
592 : : There are two exceptions - edges leading to noreturn edges and edges
593 : : predicted by number of iterations heuristics are predicted well. This macro
594 : : should be able to distinguish those, but at the moment it simply check for
595 : : noreturn heuristic that is only one giving probability over 99% or below
596 : : 1%. In future we might want to propagate reliability information across the
597 : : CFG if we find this information useful on multiple places. */
598 : 0 : bool probably_reliable_p () const
599 : : {
600 : 0 : if (quality () >= ADJUSTED)
601 : : return true;
602 : 0 : if (!initialized_p ())
603 : : return false;
604 : 0 : return m_val < max_probability / 100
605 : 0 : || m_val > max_probability - max_probability / 100;
606 : : }
607 : :
608 : : /* Return false if profile_probability is bogus. */
609 : 4380576990 : bool verify () const
610 : : {
611 : 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
612 : 4380576990 : if (m_val == uninitialized_probability)
613 : 937036814 : return quality () == GUESSED;
614 : 3443540176 : else if (quality () < GUESSED)
615 : : return false;
616 : 3443540176 : return m_val <= max_probability;
617 : : }
618 : :
619 : : /* Comparisons are three-state and conservative. False is returned if
620 : : the inequality cannot be decided. */
621 : 8546339 : bool operator< (const profile_probability &other) const
622 : : {
623 : 8546339 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
624 : : }
625 : :
626 : 17950095 : bool operator> (const profile_probability &other) const
627 : : {
628 : 17950095 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
629 : : }
630 : :
631 : 5615280 : bool operator<= (const profile_probability &other) const
632 : : {
633 : 5615280 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
634 : : }
635 : :
636 : 994441 : bool operator>= (const profile_probability &other) const
637 : : {
638 : 994441 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
639 : : }
640 : :
641 : 17 : profile_probability operator* (int64_t num) const
642 : : {
643 : 17 : return apply_scale (num, 1);
644 : : }
645 : :
646 : : profile_probability operator*= (int64_t num)
647 : : {
648 : : *this = apply_scale (num, 1);
649 : : return *this;
650 : : }
651 : :
652 : 14233331 : profile_probability operator/ (int64_t den) const
653 : : {
654 : 14233331 : return apply_scale (1, den);
655 : : }
656 : :
657 : 242511 : profile_probability operator/= (int64_t den)
658 : : {
659 : 242511 : *this = apply_scale (1, den);
660 : 242511 : return *this;
661 : : }
662 : :
663 : : /* Compute n-th power. */
664 : : profile_probability pow (int) const;
665 : :
666 : : /* Compute sware root. */
667 : : profile_probability sqrt () const;
668 : :
669 : : /* Get the value of the probability. */
670 : 0 : uint32_t value () const { return m_val; }
671 : :
672 : : /* Get the quality of the probability. */
673 : 6079862678 : enum profile_quality quality () const
674 : : {
675 : 5857589750 : return (profile_quality) (m_adjusted_quality
676 : 5857613507 : ? m_adjusted_quality + min_quality - 1
677 : : : UNINITIALIZED_PROFILE);
678 : : }
679 : :
680 : : /* Output THIS to F. */
681 : : void dump (FILE *f) const;
682 : :
683 : : /* Output THIS to BUFFER. */
684 : : void dump (char *buffer) const;
685 : :
686 : : /* Print THIS to stderr. */
687 : : void debug () const;
688 : :
689 : : /* Return true if THIS is known to differ significantly from OTHER. */
690 : : bool differs_from_p (profile_probability other) const;
691 : :
692 : : /* Return if difference is greater than 50%. */
693 : : bool differs_lot_from_p (profile_probability other) const;
694 : :
695 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
696 : : happens with COUNT2 probability. Return probability that either *THIS or
697 : : OTHER happens. */
698 : : profile_probability combine_with_count (profile_count count1,
699 : : profile_probability other,
700 : : profile_count count2) const;
701 : :
702 : : /* Return probability as sreal. */
703 : : sreal to_sreal () const;
704 : : /* LTO streaming support. */
705 : : static profile_probability stream_in (class lto_input_block *);
706 : : void stream_out (struct output_block *);
707 : : void stream_out (struct lto_output_stream *);
708 : : };
709 : :
710 : : /* Main data type to hold profile counters in GCC. Profile counts originate
711 : : either from profile feedback, static profile estimation or both. We do not
712 : : perform whole program profile propagation and thus profile estimation
713 : : counters are often local to function, while counters from profile feedback
714 : : (or special cases of profile estimation) can be used inter-procedurally.
715 : :
716 : : There are 3 basic types
717 : : 1) local counters which are result of intra-procedural static profile
718 : : estimation.
719 : : 2) ipa counters which are result of profile feedback or special case
720 : : of static profile estimation (such as in function main).
721 : : 3) counters which counts as 0 inter-procedurally (because given function
722 : : was never run in train feedback) but they hold local static profile
723 : : estimate.
724 : :
725 : : Counters of type 1 and 3 cannot be mixed with counters of different type
726 : : within operation (because whole function should use one type of counter)
727 : : with exception that global zero mix in most operations where outcome is
728 : : well defined.
729 : :
730 : : To take local counter and use it inter-procedurally use ipa member function
731 : : which strips information irrelevant at the inter-procedural level.
732 : :
733 : : Counters are 61bit integers representing number of executions during the
734 : : train run or normalized frequency within the function.
735 : :
736 : : As the profile is maintained during the compilation, many adjustments are
737 : : made. Not all transformations can be made precisely, most importantly
738 : : when code is being duplicated. It also may happen that part of CFG has
739 : : profile counts known while other do not - for example when LTO optimizing
740 : : partly profiled program or when profile was lost due to COMDAT merging.
741 : :
742 : : For this reason profile_count tracks more information than
743 : : just unsigned integer and it is also ready for profile mismatches.
744 : : The API of this data type represent operations that are natural
745 : : on profile counts - sum, difference and operation with scales and
746 : : probabilities. All operations are safe by never getting negative counts
747 : : and they do end up in uninitialized scale if any of the parameters is
748 : : uninitialized.
749 : :
750 : : All comparisons that are three state and handling of probabilities. Thus
751 : : a < b is not equal to !(a >= b).
752 : :
753 : : The following pre-defined counts are available:
754 : :
755 : : profile_count::zero () for code that is known to execute zero times at
756 : : runtime (this can be detected statically i.e. for paths leading to
757 : : abort ();
758 : : profile_count::one () for code that is known to execute once (such as
759 : : main () function
760 : : profile_count::uninitialized () for unknown execution count.
761 : :
762 : : */
763 : :
764 : : struct GTY(()) profile_count
765 : : {
766 : : public:
767 : : /* Use 60bit to hold basic block counters. Should be at least
768 : : 64bit. Although a counter cannot be negative, we use a signed
769 : : type to hold various extra stages. */
770 : :
771 : : static const int n_bits = 60;
772 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
773 : : private:
774 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
775 : :
776 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
777 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
778 : : incorrectly detects the alignment of a structure where the only
779 : : 64-bit aligned object is a bit-field. We force the alignment of
780 : : the entire field to mitigate this. */
781 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
782 : : #else
783 : : #define UINT64_BIT_FIELD_ALIGN
784 : : #endif
785 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
786 : : #undef UINT64_BIT_FIELD_ALIGN
787 : : enum profile_quality m_quality : 4;
788 : : public:
789 : :
790 : : /* Return true if both values can meaningfully appear in single function
791 : : body. We have either all counters in function local or global, otherwise
792 : : operations between them are not really defined well. */
793 : 4941286800 : bool compatible_p (const profile_count other) const
794 : : {
795 : 9084861547 : if (!initialized_p () || !other.initialized_p ())
796 : : return true;
797 : 16444999408 : if (*this == zero ()
798 : 8159142398 : || other == zero ())
799 : 128940865 : return true;
800 : : /* Do not allow nonzero global profile together with local guesses
801 : : that are globally0. */
802 : 4013987640 : if (ipa ().nonzero_p ()
803 : 241477 : && !(other.ipa () == other))
804 : 0 : return false;
805 : 4013987640 : if (other.ipa ().nonzero_p ()
806 : 250693 : && !(ipa () == *this))
807 : 0 : return false;
808 : :
809 : 4013987640 : return ipa_p () == other.ipa_p ();
810 : : }
811 : :
812 : : /* Used for counters which are expected to be never executed. */
813 : 18526893910 : static profile_count zero ()
814 : : {
815 : 14499472989 : return from_gcov_type (0);
816 : : }
817 : :
818 : 10537 : static profile_count adjusted_zero ()
819 : : {
820 : 10537 : profile_count c;
821 : 10537 : c.m_val = 0;
822 : 10537 : c.m_quality = ADJUSTED;
823 : 10537 : return c;
824 : : }
825 : :
826 : 0 : static profile_count afdo_zero ()
827 : : {
828 : 0 : profile_count c;
829 : 0 : c.m_val = 0;
830 : 0 : c.m_quality = AFDO;
831 : 0 : return c;
832 : : }
833 : :
834 : : static profile_count guessed_zero ()
835 : : {
836 : : profile_count c;
837 : : c.m_val = 0;
838 : : c.m_quality = GUESSED;
839 : : return c;
840 : : }
841 : :
842 : 49640 : static profile_count one ()
843 : : {
844 : 49433 : return from_gcov_type (1);
845 : : }
846 : :
847 : : /* Value of counters which has not been initialized. Either because
848 : : initialization did not happen yet or because profile is unknown. */
849 : 9294879231 : static profile_count uninitialized ()
850 : : {
851 : 9294879231 : profile_count c;
852 : 9294879231 : c.m_val = uninitialized_count;
853 : 9294879231 : c.m_quality = GUESSED_LOCAL;
854 : 9263251877 : return c;
855 : : }
856 : :
857 : : /* Conversion to gcov_type is lossy. */
858 : 44635231 : gcov_type to_gcov_type () const
859 : : {
860 : 0 : gcc_checking_assert (initialized_p ());
861 : 44635231 : return m_val;
862 : : }
863 : :
864 : : /* Return true if value has been initialized. */
865 : 32764697754 : bool initialized_p () const
866 : : {
867 : 10955148445 : return m_val != uninitialized_count;
868 : : }
869 : :
870 : : /* Return true if value can be trusted. */
871 : 26232600 : bool reliable_p () const
872 : : {
873 : 26232329 : return m_quality >= ADJUSTED;
874 : : }
875 : :
876 : : /* Return true if value can be operated inter-procedurally. */
877 : 9085913788 : bool ipa_p () const
878 : : {
879 : 5058462838 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
880 : : }
881 : :
882 : : /* Return true if quality of profile is precise. */
883 : 54626263 : bool precise_p () const
884 : : {
885 : 54626263 : return m_quality == PRECISE;
886 : : }
887 : :
888 : : /* Get the value of the count. */
889 : 0 : uint64_t value () const { return m_val; }
890 : :
891 : : /* Get the quality of the count. */
892 : 95382057 : enum profile_quality quality () const { return m_quality; }
893 : :
894 : : /* When merging basic blocks, the two different profile counts are unified.
895 : : Return true if this can be done without losing info about profile.
896 : : The only case we care about here is when first BB contains something
897 : : that makes it terminate in a way not visible in CFG. */
898 : 2960431 : bool ok_for_merging (profile_count other) const
899 : : {
900 : 2960431 : if (m_quality < ADJUSTED
901 : 10649 : || other.m_quality < ADJUSTED)
902 : : return true;
903 : 9800 : return !(other < *this);
904 : : }
905 : :
906 : : /* When merging two BBs with different counts, pick common count that looks
907 : : most representative. */
908 : 15648351 : profile_count merge (profile_count other) const
909 : : {
910 : 845330 : if (*this == other || !other.initialized_p ()
911 : 845085 : || m_quality > other.m_quality)
912 : 14804252 : return *this;
913 : 844099 : if (other.m_quality > m_quality
914 : 844099 : || other > *this)
915 : 0 : return other;
916 : 844099 : return *this;
917 : : }
918 : :
919 : : /* Basic operations. */
920 : 19977508136 : bool operator== (const profile_count &other) const
921 : : {
922 : 15182360068 : return m_val == other.m_val && m_quality == other.m_quality;
923 : : }
924 : :
925 : 3799648 : profile_count operator+ (const profile_count &other) const
926 : : {
927 : 3799648 : if (other == zero ())
928 : 3094 : return *this;
929 : 3796554 : if (*this == zero ())
930 : 197677 : return other;
931 : 3598877 : if (!initialized_p () || !other.initialized_p ())
932 : 6377 : return uninitialized ();
933 : :
934 : 3592500 : profile_count ret;
935 : 3592500 : gcc_checking_assert (compatible_p (other));
936 : 3592500 : uint64_t ret_val = m_val + other.m_val;
937 : 3592500 : ret.m_val = MIN (ret_val, max_count);
938 : 3592500 : ret.m_quality = MIN (m_quality, other.m_quality);
939 : 3592500 : return ret;
940 : : }
941 : :
942 : 74562593 : profile_count &operator+= (const profile_count &other)
943 : : {
944 : 74562593 : if (other == zero ())
945 : 4622225 : return *this;
946 : 69940368 : if (*this == zero ())
947 : : {
948 : 42922832 : *this = other;
949 : 42922832 : return *this;
950 : : }
951 : 27017536 : if (!initialized_p () || !other.initialized_p ())
952 : 341173 : return *this = uninitialized ();
953 : : else
954 : : {
955 : 26676363 : gcc_checking_assert (compatible_p (other));
956 : 26676363 : uint64_t ret_val = m_val + other.m_val;
957 : 26676363 : m_val = MIN (ret_val, max_count);
958 : 26676363 : m_quality = MIN (m_quality, other.m_quality);
959 : : }
960 : 26676363 : return *this;
961 : : }
962 : :
963 : 19711897 : profile_count operator- (const profile_count &other) const
964 : : {
965 : 19727271 : if (*this == zero () || other == zero ())
966 : 612174 : return *this;
967 : 19099723 : if (!initialized_p () || !other.initialized_p ())
968 : 8620428 : return uninitialized ();
969 : 10479295 : gcc_checking_assert (compatible_p (other));
970 : 10479295 : profile_count ret;
971 : 10479295 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
972 : 10479295 : ret.m_quality = MIN (m_quality, other.m_quality);
973 : 10479295 : return ret;
974 : : }
975 : :
976 : 9452131 : profile_count &operator-= (const profile_count &other)
977 : : {
978 : 9893567 : if (*this == zero () || other == zero ())
979 : 317689 : return *this;
980 : 9134442 : if (!initialized_p () || !other.initialized_p ())
981 : 348043 : return *this = uninitialized ();
982 : : else
983 : : {
984 : 8786399 : gcc_checking_assert (compatible_p (other));
985 : 8786399 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
986 : 8786399 : m_quality = MIN (m_quality, other.m_quality);
987 : : }
988 : 8786399 : return *this;
989 : : }
990 : :
991 : : /* Return false if profile_count is bogus. */
992 : 3340198817 : bool verify () const
993 : : {
994 : 3340198817 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
995 : 3340198817 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
996 : : }
997 : :
998 : : /* Comparisons are three-state and conservative. False is returned if
999 : : the inequality cannot be decided. */
1000 : 1103648048 : bool operator< (const profile_count &other) const
1001 : : {
1002 : 1103648048 : if (!initialized_p () || !other.initialized_p ())
1003 : : return false;
1004 : 1099007740 : if (*this == zero ())
1005 : 3902526 : return !(other == zero ());
1006 : 1097056477 : if (other == zero ())
1007 : 1045 : return false;
1008 : 1097055432 : gcc_checking_assert (compatible_p (other));
1009 : 1097055432 : return m_val < other.m_val;
1010 : : }
1011 : :
1012 : 39519809 : bool operator> (const profile_count &other) const
1013 : : {
1014 : 39519809 : if (!initialized_p () || !other.initialized_p ())
1015 : : return false;
1016 : 34542169 : if (*this == zero ())
1017 : 228725 : return false;
1018 : 34313444 : if (other == zero ())
1019 : 1622666 : return !(*this == zero ());
1020 : 33502111 : gcc_checking_assert (compatible_p (other));
1021 : 33502111 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1022 : : }
1023 : :
1024 : : bool operator< (const gcov_type other) const
1025 : : {
1026 : : gcc_checking_assert (ipa_p ());
1027 : : gcc_checking_assert (other >= 0);
1028 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1029 : : }
1030 : :
1031 : 59292 : bool operator> (const gcov_type other) const
1032 : : {
1033 : 59292 : gcc_checking_assert (ipa_p ());
1034 : 59292 : gcc_checking_assert (other >= 0);
1035 : 59292 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1036 : : }
1037 : :
1038 : 1124797 : bool operator<= (const profile_count &other) const
1039 : : {
1040 : 1124797 : if (!initialized_p () || !other.initialized_p ())
1041 : : return false;
1042 : 1124797 : if (*this == zero ())
1043 : 4386 : return true;
1044 : 1120411 : if (other == zero ())
1045 : 0 : return (*this == zero ());
1046 : 1120411 : gcc_checking_assert (compatible_p (other));
1047 : 1120411 : return m_val <= other.m_val;
1048 : : }
1049 : :
1050 : 1411490 : bool operator>= (const profile_count &other) const
1051 : : {
1052 : 1411490 : if (!initialized_p () || !other.initialized_p ())
1053 : : return false;
1054 : 1410407 : if (other == zero ())
1055 : 339 : return true;
1056 : 1410068 : if (*this == zero ())
1057 : 24554 : return (other == zero ());
1058 : 1397791 : gcc_checking_assert (compatible_p (other));
1059 : 1397791 : return m_val >= other.m_val;
1060 : : }
1061 : :
1062 : 95418 : bool operator<= (const gcov_type other) const
1063 : : {
1064 : 95418 : gcc_checking_assert (ipa_p ());
1065 : 95418 : gcc_checking_assert (other >= 0);
1066 : 95418 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1067 : : }
1068 : :
1069 : 59468 : bool operator>= (const gcov_type other) const
1070 : : {
1071 : 59468 : gcc_checking_assert (ipa_p ());
1072 : 59468 : gcc_checking_assert (other >= 0);
1073 : 59468 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1074 : : }
1075 : :
1076 : 925510289 : profile_count operator* (int64_t num) const
1077 : : {
1078 : 925440455 : return apply_scale (num, 1);
1079 : : }
1080 : :
1081 : : profile_count operator*= (int64_t num)
1082 : : {
1083 : : *this = apply_scale (num, 1);
1084 : : return *this;
1085 : : }
1086 : :
1087 : : profile_count operator* (const sreal &num) const;
1088 : : profile_count operator*= (const sreal &num);
1089 : :
1090 : 2330523 : profile_count operator/ (int64_t den) const
1091 : : {
1092 : 2330523 : return apply_scale (1, den);
1093 : : }
1094 : :
1095 : 75 : profile_count operator/= (int64_t den)
1096 : : {
1097 : 67 : *this = apply_scale (1, den);
1098 : 75 : return *this;
1099 : : }
1100 : :
1101 : : /* Return true when value is not zero and can be used for scaling.
1102 : : This is different from *this > 0 because that requires counter to
1103 : : be IPA. */
1104 : 8216866055 : bool nonzero_p () const
1105 : : {
1106 : 8216863193 : return initialized_p () && m_val != 0;
1107 : : }
1108 : :
1109 : : /* Make counter forcibly nonzero. */
1110 : : profile_count force_nonzero () const;
1111 : :
1112 : :
1113 : : /* Return maximum of A and B. If one of values is uninitialized return the
1114 : : other. */
1115 : :
1116 : : static profile_count
1117 : 78260236 : max_prefer_initialized (const profile_count a, const profile_count b)
1118 : : {
1119 : 78260236 : if (!a.initialized_p ())
1120 : 21730973 : return b;
1121 : 56529263 : if (!b.initialized_p ())
1122 : 315783 : return a;
1123 : 56213480 : profile_count ret;
1124 : 56213480 : gcc_checking_assert (a.compatible_p (b));
1125 : 56213480 : ret.m_val = MAX (a.m_val, b.m_val);
1126 : 56213480 : ret.m_quality = MIN (a.m_quality, b.m_quality);
1127 : 56213480 : return ret;
1128 : : }
1129 : :
1130 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1131 : : accordingly. */
1132 : : profile_count apply_probability (int prob) const
1133 : : {
1134 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1135 : : if (m_val == 0)
1136 : : return *this;
1137 : : if (!initialized_p ())
1138 : : return uninitialized ();
1139 : : profile_count ret;
1140 : : uint64_t tmp;
1141 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1142 : : ret.m_val = tmp;
1143 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1144 : : return ret;
1145 : : }
1146 : :
1147 : : /* Scale counter according to PROB. */
1148 : 392674086 : profile_count apply_probability (profile_probability prob) const
1149 : : {
1150 : 401521806 : if (*this == zero () || prob == profile_probability::always ())
1151 : 120601139 : return *this;
1152 : 272072947 : if (prob == profile_probability::never ())
1153 : 18688207 : return zero ();
1154 : 253384740 : if (!initialized_p () || !prob.initialized_p ())
1155 : 31135569 : return uninitialized ();
1156 : 222249171 : profile_count ret;
1157 : 222249171 : uint64_t tmp;
1158 : 222249171 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1159 : : &tmp);
1160 : 222249171 : ret.m_val = tmp;
1161 : 222272928 : ret.m_quality = MIN (m_quality, prob.quality ());
1162 : 222249171 : return ret;
1163 : : }
1164 : :
1165 : : /* Return *THIS * NUM / DEN. */
1166 : 1048628908 : profile_count apply_scale (int64_t num, int64_t den) const
1167 : : {
1168 : 1048628908 : if (m_val == 0)
1169 : 25103138 : return *this;
1170 : 1023525770 : if (!initialized_p ())
1171 : 4024427 : return uninitialized ();
1172 : 1019501343 : profile_count ret;
1173 : 1019501343 : uint64_t tmp;
1174 : :
1175 : 1019501343 : gcc_checking_assert (num >= 0 && den > 0);
1176 : 1019501343 : safe_scale_64bit (m_val, num, den, &tmp);
1177 : 1019501343 : ret.m_val = MIN (tmp, max_count);
1178 : 1019501343 : ret.m_quality = MIN (m_quality, ADJUSTED);
1179 : 1019501343 : return ret;
1180 : : }
1181 : :
1182 : 27569043 : profile_count apply_scale (profile_count num, profile_count den) const
1183 : : {
1184 : 27569043 : if (*this == zero ())
1185 : 1062356 : return *this;
1186 : 26506687 : if (num == zero ())
1187 : 83259 : return num;
1188 : 26423428 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1189 : 7350167 : return uninitialized ();
1190 : 19073261 : if (num == den)
1191 : 5312663 : return *this;
1192 : 13760598 : gcc_checking_assert (den.m_val);
1193 : :
1194 : 13760598 : profile_count ret;
1195 : 13760598 : uint64_t val;
1196 : 13760598 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1197 : 13760598 : ret.m_val = MIN (val, max_count);
1198 : 13760598 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1199 : : num.m_quality), den.m_quality);
1200 : : /* Be sure that ret is not local if num is global.
1201 : : Also ensure that ret is not global0 when num is global. */
1202 : 13760598 : if (num.ipa_p ())
1203 : : {
1204 : : /* This is common case of AFDO scaling when we upgrade
1205 : : GLOBAL0_AFDO function to AFDO. Be sure that result
1206 : : is AFDO and not GUESSED (which is unnecesarily low). */
1207 : 1798 : if (num.m_quality == AFDO
1208 : 0 : && (ret.m_quality != GUESSED
1209 : 0 : && ret.m_quality != GUESSED_LOCAL))
1210 : 0 : ret.m_quality = AFDO;
1211 : : else
1212 : 3600 : ret.m_quality = MAX (ret.m_quality,
1213 : : num == num.ipa () ? GUESSED : num.m_quality);
1214 : : }
1215 : 13760598 : return ret;
1216 : : }
1217 : :
1218 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1219 : 18145658 : profile_count guessed_local () const
1220 : : {
1221 : 18145658 : profile_count ret = *this;
1222 : 18145658 : if (!initialized_p ())
1223 : 0 : return *this;
1224 : 18145658 : ret.m_quality = GUESSED_LOCAL;
1225 : 18145658 : return ret;
1226 : : }
1227 : :
1228 : : /* We know that profile is globally 0 but keep local profile if present. */
1229 : 790 : profile_count global0 () const
1230 : : {
1231 : 790 : profile_count ret = *this;
1232 : 790 : if (!initialized_p ())
1233 : 0 : return *this;
1234 : 790 : ret.m_quality = GUESSED_GLOBAL0;
1235 : 790 : return ret;
1236 : : }
1237 : :
1238 : : /* We know that profile is globally afdo 0 but keep local profile
1239 : : if present. */
1240 : 0 : profile_count global0afdo () const
1241 : : {
1242 : 0 : profile_count ret = *this;
1243 : 0 : if (!initialized_p ())
1244 : 0 : return *this;
1245 : 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1246 : 0 : return ret;
1247 : : }
1248 : :
1249 : : /* We know that profile is globally adjusted 0 but keep local profile
1250 : : if present. */
1251 : 36 : profile_count global0adjusted () const
1252 : : {
1253 : 36 : profile_count ret = *this;
1254 : 36 : if (!initialized_p ())
1255 : 0 : return *this;
1256 : 36 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1257 : 36 : return ret;
1258 : : }
1259 : :
1260 : : /* Return THIS with quality dropped to GUESSED. */
1261 : 366 : profile_count guessed () const
1262 : : {
1263 : 366 : profile_count ret = *this;
1264 : 366 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1265 : 366 : return ret;
1266 : : }
1267 : :
1268 : : /* Return THIS with quality GUESSED. */
1269 : 0 : profile_count force_guessed () const
1270 : : {
1271 : 0 : profile_count ret = *this;
1272 : 0 : gcc_checking_assert (initialized_p ());
1273 : 0 : ret.m_quality = GUESSED;
1274 : 0 : return ret;
1275 : : }
1276 : :
1277 : : /* Return variant of profile count which is always safe to compare
1278 : : across functions. */
1279 : 9156794253 : profile_count ipa () const
1280 : : {
1281 : 9156794253 : if (m_quality > GUESSED_GLOBAL0)
1282 : 10778982 : return *this;
1283 : 9146015271 : if (m_quality == GUESSED_GLOBAL0)
1284 : 212938 : return zero ();
1285 : 9145802333 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1286 : 10537 : return adjusted_zero ();
1287 : 9145791796 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1288 : 0 : return afdo_zero ();
1289 : 9145791796 : return uninitialized ();
1290 : : }
1291 : :
1292 : : /* Return THIS with quality dropped to AFDO. */
1293 : 0 : profile_count afdo () const
1294 : : {
1295 : 0 : profile_count ret = *this;
1296 : 0 : ret.m_quality = AFDO;
1297 : 0 : return ret;
1298 : : }
1299 : :
1300 : : /* Return probability of event with counter THIS within event with counter
1301 : : OVERALL. */
1302 : 1087036071 : profile_probability probability_in (const profile_count overall) const
1303 : : {
1304 : 1087050887 : if (*this == zero ()
1305 : 14816 : && !(overall == zero ()))
1306 : 1157 : return profile_probability::never ();
1307 : 1087034347 : if (!initialized_p () || !overall.initialized_p ()
1308 : 2174069261 : || !overall.m_val)
1309 : 60441 : return profile_probability::uninitialized ();
1310 : 1292663129 : if (*this == overall && m_quality == PRECISE)
1311 : 26918 : return profile_probability::always ();
1312 : 1086947555 : profile_probability ret;
1313 : 1086947555 : gcc_checking_assert (compatible_p (overall));
1314 : :
1315 : 1086947555 : if (overall.m_val < m_val)
1316 : : {
1317 : 197798 : ret.m_val = profile_probability::max_probability;
1318 : 197798 : ret.set_quality (GUESSED);
1319 : 197798 : return ret;
1320 : : }
1321 : : else
1322 : : {
1323 : 1086749757 : gcc_checking_assert (overall.m_val);
1324 : 1086749757 : uint64_t tmp;
1325 : 1086749757 : safe_scale_64bit (m_val, profile_probability::max_probability,
1326 : : overall.m_val, &tmp);
1327 : 1086749757 : gcc_checking_assert (tmp <= profile_probability::max_probability);
1328 : 1086749757 : ret.m_val = tmp;
1329 : : }
1330 : 1086749757 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1331 : : GUESSED), ADJUSTED));
1332 : 1086749757 : return ret;
1333 : : }
1334 : :
1335 : : /* Return true if profile count is very large, so we risk overflows
1336 : : with loop transformations. */
1337 : : bool
1338 : 1029816 : very_large_p ()
1339 : : {
1340 : 1029816 : if (!initialized_p ())
1341 : : return false;
1342 : 1029816 : return m_val > max_count / 65536;
1343 : : }
1344 : :
1345 : : int to_frequency (struct function *fun) const;
1346 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1347 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1348 : :
1349 : : /* Output THIS to F. */
1350 : : void dump (FILE *f, struct function *fun = NULL) const;
1351 : :
1352 : : /* Print THIS to stderr. */
1353 : : void debug () const;
1354 : :
1355 : : /* Return true if THIS is known to differ significantly from OTHER. */
1356 : : bool differs_from_p (profile_count other) const;
1357 : :
1358 : : /* We want to scale profile across function boundary from NUM to DEN.
1359 : : Take care of the side case when NUM and DEN are zeros of incompatible
1360 : : kinds. */
1361 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1362 : :
1363 : : /* THIS is a count of bb which is known to be executed IPA times.
1364 : : Combine this information into bb counter. This means returning IPA
1365 : : if it is nonzero, not changing anything if IPA is uninitialized
1366 : : and if IPA is zero, turning THIS into corresponding local profile with
1367 : : global0. */
1368 : : profile_count combine_with_ipa_count (profile_count ipa);
1369 : :
1370 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1371 : : profile_count combine_with_ipa_count_within
1372 : : (profile_count ipa, profile_count ipa2);
1373 : :
1374 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1375 : : Conversions back and forth are used to read the coverage and get it
1376 : : into internal representation. */
1377 : : static profile_count from_gcov_type (gcov_type v,
1378 : : profile_quality quality = PRECISE);
1379 : :
1380 : : /* LTO streaming support. */
1381 : : static profile_count stream_in (class lto_input_block *);
1382 : : void stream_out (struct output_block *);
1383 : : void stream_out (struct lto_output_stream *);
1384 : : };
1385 : : #endif
|
