Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2025 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
41 : : better. This is the case of functions not run in profile feedback.
42 : : Never used by probabilities. */
43 : : GUESSED_GLOBAL0,
44 : :
45 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
46 : : GUESSED_GLOBAL0_ADJUSTED,
47 : :
48 : : /* Profile is based on static branch prediction heuristics. It may or may
49 : : not reflect the reality but it can be compared interprocedurally
50 : : (for example, we inlined function w/o profile feedback into function
51 : : with feedback and propagated from that).
52 : : Never used by probabilities. */
53 : : GUESSED,
54 : :
55 : : /* Profile was determined by autofdo. */
56 : : AFDO,
57 : :
58 : : /* Profile was originally based on feedback but it was adjusted
59 : : by code duplicating optimization. It may not precisely reflect the
60 : : particular code path. */
61 : : ADJUSTED,
62 : :
63 : : /* Profile was read from profile feedback or determined by accurate static
64 : : method. */
65 : : PRECISE
66 : : };
67 : :
68 : : extern const char *profile_quality_as_string (enum profile_quality);
69 : : extern bool parse_profile_quality (const char *value,
70 : : profile_quality *quality);
71 : :
72 : : /* The base value for branch probability notes and edge probabilities. */
73 : : #define REG_BR_PROB_BASE 10000
74 : :
75 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
76 : :
77 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
78 : :
79 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
80 : :
81 : : inline bool
82 : 1112086619 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
83 : : {
84 : : #if (GCC_VERSION >= 5000)
85 : 1112086619 : uint64_t tmp;
86 : 1112086619 : if (!__builtin_mul_overflow (a, b, &tmp)
87 : 1112086619 : && !__builtin_add_overflow (tmp, c/2, &tmp))
88 : : {
89 : 1111515740 : *res = tmp / c;
90 : 1111515740 : return true;
91 : : }
92 : 570879 : if (c == 1)
93 : : {
94 : 27611 : *res = (uint64_t) -1;
95 : 27611 : return false;
96 : : }
97 : : #else
98 : : if (a < ((uint64_t)1 << 31)
99 : : && b < ((uint64_t)1 << 31)
100 : : && c < ((uint64_t)1 << 31))
101 : : {
102 : : *res = (a * b + (c / 2)) / c;
103 : : return true;
104 : : }
105 : : #endif
106 : 543268 : return slow_safe_scale_64bit (a, b, c, res);
107 : : }
108 : :
109 : : /* Data type to hold probabilities. It implements fixed point arithmetics
110 : : with capping so probability is always in range [0,1] and scaling requiring
111 : : values greater than 1 needs to be represented otherwise.
112 : :
113 : : In addition to actual value the quality of profile is tracked and propagated
114 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
115 : : that has not been determined yet (for example because of
116 : : -fno-guess-branch-probability)
117 : :
118 : : Typically probabilities are derived from profile feedback (via
119 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
120 : : thorough the compilation.
121 : :
122 : : Named probabilities are available:
123 : : - never (0 probability)
124 : : - guessed_never
125 : : - very_unlikely (1/2000 probability)
126 : : - unlikely (1/5 probability)
127 : : - even (1/2 probability)
128 : : - likely (4/5 probability)
129 : : - very_likely (1999/2000 probability)
130 : : - guessed_always
131 : : - always
132 : :
133 : : Named probabilities except for never/always are assumed to be statically
134 : : guessed and thus not necessarily accurate. The difference between never
135 : : and guessed_never is that the first one should be used only in case that
136 : : well behaving program will very likely not execute the "never" path.
137 : : For example if the path is going to abort () call or it exception handling.
138 : :
139 : : Always and guessed_always probabilities are symmetric.
140 : :
141 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
142 : : integer arithmetics. Once the code is converted to branch probabilities,
143 : : these conversions will probably go away because they are lossy.
144 : : */
145 : :
146 : : class GTY((user)) profile_probability
147 : : {
148 : : static const int n_bits = 29;
149 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
150 : : will lead to harder multiplication sequences. */
151 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
152 : : static const uint32_t uninitialized_probability
153 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
154 : :
155 : : uint32_t m_val : 29;
156 : : enum profile_quality m_quality : 3;
157 : :
158 : : friend struct profile_count;
159 : : public:
160 : 3818550135 : profile_probability (): m_val (uninitialized_probability),
161 : 2046030350 : m_quality (GUESSED)
162 : : {}
163 : :
164 : 112 : profile_probability (uint32_t val, profile_quality quality):
165 : 112 : m_val (val), m_quality (quality)
166 : : {}
167 : :
168 : : /* Named probabilities. */
169 : 627199602 : static profile_probability never ()
170 : : {
171 : 627199602 : profile_probability ret;
172 : 627199602 : ret.m_val = 0;
173 : 627199602 : ret.m_quality = PRECISE;
174 : 164024906 : return ret;
175 : : }
176 : :
177 : 4925368 : static profile_probability guessed_never ()
178 : : {
179 : 4925368 : profile_probability ret;
180 : 4925368 : ret.m_val = 0;
181 : 4925368 : ret.m_quality = GUESSED;
182 : 4925368 : return ret;
183 : : }
184 : :
185 : 4771394 : static profile_probability very_unlikely ()
186 : : {
187 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
188 : 9542788 : profile_probability r = guessed_always () / 2000;
189 : 4771394 : r.m_val--;
190 : 4771394 : return r;
191 : : }
192 : :
193 : 80527 : static profile_probability unlikely ()
194 : : {
195 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
196 : 161054 : profile_probability r = guessed_always () / 5;
197 : 80527 : r.m_val--;
198 : 80527 : return r;
199 : : }
200 : :
201 : 746302 : static profile_probability even ()
202 : : {
203 : 1492604 : return guessed_always () / 2;
204 : : }
205 : :
206 : 337486 : static profile_probability very_likely ()
207 : : {
208 : 337486 : return always () - very_unlikely ();
209 : : }
210 : :
211 : 31682 : static profile_probability likely ()
212 : : {
213 : 31682 : return always () - unlikely ();
214 : : }
215 : : /* Return true when value is not zero and can be used for scaling. */
216 : 1300 : bool nonzero_p () const
217 : : {
218 : 1300 : return initialized_p () && m_val != 0;
219 : : }
220 : :
221 : 5900072 : static profile_probability guessed_always ()
222 : : {
223 : 5900072 : profile_probability ret;
224 : 5900072 : ret.m_val = max_probability;
225 : 5900072 : ret.m_quality = GUESSED;
226 : 5743732 : return ret;
227 : : }
228 : :
229 : 501168996 : static profile_probability always ()
230 : : {
231 : 501168996 : profile_probability ret;
232 : 501168996 : ret.m_val = max_probability;
233 : 501168996 : ret.m_quality = PRECISE;
234 : 330022365 : return ret;
235 : : }
236 : :
237 : : /* Probabilities which has not been initialized. Either because
238 : : initialization did not happen yet or because profile is unknown. */
239 : 838572655 : static profile_probability uninitialized ()
240 : : {
241 : 838572655 : profile_probability c;
242 : 838572655 : c.m_val = uninitialized_probability;
243 : 838572655 : c.m_quality = GUESSED;
244 : 820346846 : return c;
245 : : }
246 : :
247 : : /* Return true if value has been initialized. */
248 : 5318890134 : bool initialized_p () const
249 : : {
250 : 3660509673 : return m_val != uninitialized_probability;
251 : : }
252 : :
253 : : /* Return true if value can be trusted. */
254 : 400 : bool reliable_p () const
255 : : {
256 : 400 : return m_quality >= ADJUSTED;
257 : : }
258 : :
259 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
260 : : this is mostly to support legacy code and should go away. */
261 : 2976688 : static profile_probability from_reg_br_prob_base (int v)
262 : : {
263 : 2976688 : profile_probability ret;
264 : 2976688 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
265 : 2976688 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
266 : 2976688 : ret.m_quality = GUESSED;
267 : 2976688 : return ret;
268 : : }
269 : :
270 : : /* Return THIS with quality set to ADJUSTED. */
271 : 42 : profile_probability adjusted () const
272 : : {
273 : 42 : profile_probability ret = *this;
274 : 42 : if (!initialized_p ())
275 : 0 : return *this;
276 : 42 : ret.m_quality = ADJUSTED;
277 : 42 : return ret;
278 : : }
279 : :
280 : 1057869165 : int to_reg_br_prob_base () const
281 : : {
282 : 1057869165 : gcc_checking_assert (initialized_p ());
283 : 1057869165 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
284 : : }
285 : :
286 : : /* Conversion to and from RTL representation of profile probabilities. */
287 : 445086725 : static profile_probability from_reg_br_prob_note (int v)
288 : : {
289 : 445086725 : profile_probability ret;
290 : 445086725 : ret.m_val = ((unsigned int)v) / 8;
291 : 445086725 : ret.m_quality = (enum profile_quality)(v & 7);
292 : 14782100 : return ret;
293 : : }
294 : :
295 : 417841584 : int to_reg_br_prob_note () const
296 : : {
297 : 417841584 : gcc_checking_assert (initialized_p ());
298 : 417841584 : int ret = m_val * 8 + m_quality;
299 : 835683168 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
300 : 417841584 : return ret;
301 : : }
302 : :
303 : : /* Return VAL1/VAL2. */
304 : 3050 : static profile_probability probability_in_gcov_type
305 : : (gcov_type val1, gcov_type val2)
306 : : {
307 : 3050 : profile_probability ret;
308 : 3050 : gcc_checking_assert (val1 >= 0 && val2 > 0);
309 : 3050 : if (val1 > val2)
310 : : ret.m_val = max_probability;
311 : : else
312 : : {
313 : 3050 : uint64_t tmp;
314 : 3050 : safe_scale_64bit (val1, max_probability, val2, &tmp);
315 : 3050 : gcc_checking_assert (tmp <= max_probability);
316 : 3050 : ret.m_val = tmp;
317 : : }
318 : 3050 : ret.m_quality = PRECISE;
319 : 3050 : return ret;
320 : : }
321 : :
322 : : /* Basic operations. */
323 : 1419855073 : bool operator== (const profile_probability &other) const
324 : : {
325 : 845767209 : return m_val == other.m_val && m_quality == other.m_quality;
326 : : }
327 : :
328 : 6207314 : profile_probability operator+ (const profile_probability &other) const
329 : : {
330 : 6207314 : if (other == never ())
331 : 33857 : return *this;
332 : 6173457 : if (*this == never ())
333 : 22933 : return other;
334 : 6150524 : if (!initialized_p () || !other.initialized_p ())
335 : 5073195 : return uninitialized ();
336 : :
337 : 1077329 : profile_probability ret;
338 : 1077329 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
339 : 1077329 : ret.m_quality = MIN (m_quality, other.m_quality);
340 : 1077329 : return ret;
341 : : }
342 : :
343 : 5808880 : profile_probability &operator+= (const profile_probability &other)
344 : : {
345 : 5808880 : if (other == never ())
346 : 1057417 : return *this;
347 : 4751463 : if (*this == never ())
348 : : {
349 : 805854 : *this = other;
350 : 805854 : return *this;
351 : : }
352 : 3945609 : if (!initialized_p () || !other.initialized_p ())
353 : 2853547 : return *this = uninitialized ();
354 : : else
355 : : {
356 : 1092062 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
357 : 1092062 : m_quality = MIN (m_quality, other.m_quality);
358 : : }
359 : 1092062 : return *this;
360 : : }
361 : :
362 : 32009944 : profile_probability operator- (const profile_probability &other) const
363 : : {
364 : 32009944 : if (*this == never ()
365 : 31910499 : || other == never ())
366 : 1754099 : return *this;
367 : 30255845 : if (!initialized_p () || !other.initialized_p ())
368 : 2135399 : return uninitialized ();
369 : 28120446 : profile_probability ret;
370 : 28120446 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
371 : 28120446 : ret.m_quality = MIN (m_quality, other.m_quality);
372 : 28120446 : return ret;
373 : : }
374 : :
375 : 11084001 : profile_probability &operator-= (const profile_probability &other)
376 : : {
377 : 11084001 : if (*this == never ()
378 : 10958877 : || other == never ())
379 : 2165094 : return *this;
380 : 8918907 : if (!initialized_p () || !other.initialized_p ())
381 : 2106367 : return *this = uninitialized ();
382 : : else
383 : : {
384 : 6812540 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
385 : 6812540 : m_quality = MIN (m_quality, other.m_quality);
386 : : }
387 : 6812540 : return *this;
388 : : }
389 : :
390 : 1132246 : profile_probability operator* (const profile_probability &other) const
391 : : {
392 : 1132246 : if (*this == never ()
393 : 1029395 : || other == never ())
394 : 111491 : return never ();
395 : 1020755 : if (!initialized_p () || !other.initialized_p ())
396 : 57852 : return uninitialized ();
397 : 962903 : profile_probability ret;
398 : 962903 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
399 : 962903 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
400 : 962903 : return ret;
401 : : }
402 : :
403 : 131159 : profile_probability &operator*= (const profile_probability &other)
404 : : {
405 : 131159 : if (*this == never ()
406 : 131042 : || other == never ())
407 : 330 : return *this = never ();
408 : 130829 : if (!initialized_p () || !other.initialized_p ())
409 : 9 : return *this = uninitialized ();
410 : : else
411 : : {
412 : 130820 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
413 : 130820 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
414 : : }
415 : 130820 : return *this;
416 : : }
417 : :
418 : 1596835 : profile_probability operator/ (const profile_probability &other) const
419 : : {
420 : 1596835 : if (*this == never ())
421 : 106400 : return never ();
422 : 1490435 : if (!initialized_p () || !other.initialized_p ())
423 : 222194 : return uninitialized ();
424 : 1268241 : profile_probability ret;
425 : : /* If we get probability above 1, mark it as unreliable and return 1. */
426 : 1268241 : if (m_val >= other.m_val)
427 : : {
428 : 31556 : ret.m_val = max_probability;
429 : 31556 : ret.m_quality = MIN (MIN (m_quality, other.m_quality),
430 : : GUESSED);
431 : 31556 : return ret;
432 : : }
433 : 1236685 : else if (!m_val)
434 : : ret.m_val = 0;
435 : : else
436 : : {
437 : 1178635 : gcc_checking_assert (other.m_val);
438 : 1178635 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
439 : : other.m_val),
440 : : max_probability);
441 : : }
442 : 1236685 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
443 : 1236685 : return ret;
444 : : }
445 : :
446 : 305600 : profile_probability &operator/= (const profile_probability &other)
447 : : {
448 : 305600 : if (*this == never ())
449 : 1918 : return *this = never ();
450 : 303682 : if (!initialized_p () || !other.initialized_p ())
451 : 0 : return *this = uninitialized ();
452 : : else
453 : : {
454 : : /* If we get probability above 1, mark it as unreliable
455 : : and return 1. */
456 : 303682 : if (m_val > other.m_val)
457 : : {
458 : 557 : m_val = max_probability;
459 : 557 : m_quality = MIN (MIN (m_quality, other.m_quality),
460 : : GUESSED);
461 : 557 : return *this;
462 : : }
463 : 303125 : else if (!m_val)
464 : : ;
465 : : else
466 : : {
467 : 301516 : gcc_checking_assert (other.m_val);
468 : 301516 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
469 : : other.m_val),
470 : : max_probability);
471 : : }
472 : 303125 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
473 : : }
474 : 303125 : return *this;
475 : : }
476 : :
477 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
478 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
479 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
480 : : == ORIG. This is useful e.g. when splitting a conditional
481 : : branch like:
482 : : if (cond)
483 : : goto lab; // ORIG probability
484 : : into
485 : : if (cond1)
486 : : goto lab; // FIRST = ORIG * CPROB probability
487 : : if (cond2)
488 : : goto lab; // SECOND probability
489 : : such that the overall probability of jumping to lab remains
490 : : the same. CPROB gives the relative probability between the
491 : : branches. */
492 : 286278 : profile_probability split (const profile_probability &cprob)
493 : : {
494 : 286278 : profile_probability ret = *this * cprob;
495 : : /* The following is equivalent to:
496 : : *this = cprob.invert () * *this / ret.invert ();
497 : : Avoid scaling when overall outcome is supposed to be always.
498 : : Without knowing that one is inverse of other, the result would be
499 : : conservative. */
500 : 286278 : if (!(*this == always ()))
501 : 277444 : *this = (*this - ret) / ret.invert ();
502 : 286278 : return ret;
503 : : }
504 : :
505 : 246364 : gcov_type apply (gcov_type val) const
506 : : {
507 : 246364 : if (*this == uninitialized ())
508 : 62 : return val / 2;
509 : 246302 : return RDIV (val * m_val, max_probability);
510 : : }
511 : :
512 : : /* Return 1-*THIS. */
513 : 29055383 : profile_probability invert () const
514 : : {
515 : 29046822 : return always() - *this;
516 : : }
517 : :
518 : : /* Return THIS with quality dropped to GUESSED. */
519 : 202700 : profile_probability guessed () const
520 : : {
521 : 202700 : profile_probability ret = *this;
522 : 202700 : ret.m_quality = GUESSED;
523 : 202700 : return ret;
524 : : }
525 : :
526 : : /* Return THIS with quality dropped to AFDO. */
527 : : profile_probability afdo () const
528 : : {
529 : : profile_probability ret = *this;
530 : : ret.m_quality = AFDO;
531 : : return ret;
532 : : }
533 : :
534 : : /* Return *THIS * NUM / DEN. */
535 : 13506276 : profile_probability apply_scale (int64_t num, int64_t den) const
536 : : {
537 : 13506276 : if (*this == never ())
538 : 26630 : return *this;
539 : 13479646 : if (!initialized_p ())
540 : 5467553 : return uninitialized ();
541 : 8012093 : profile_probability ret;
542 : 8012093 : uint64_t tmp;
543 : 8012093 : safe_scale_64bit (m_val, num, den, &tmp);
544 : 8012093 : ret.m_val = MIN (tmp, max_probability);
545 : 8012093 : ret.m_quality = MIN (m_quality, ADJUSTED);
546 : 8012093 : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 6488 : profile_probability apply_scale (profile_probability num,
551 : : profile_probability den) const
552 : : {
553 : 6488 : if (*this == never ())
554 : 40 : return *this;
555 : 6448 : if (num == never ())
556 : 0 : return num;
557 : 6448 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
558 : 0 : return uninitialized ();
559 : 6448 : if (num == den)
560 : 0 : return *this;
561 : 6448 : gcc_checking_assert (den.m_val);
562 : :
563 : 6448 : profile_probability ret;
564 : 6448 : uint64_t val;
565 : 6448 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
566 : 6448 : ret.m_val = MIN (val, max_probability);
567 : 6448 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
568 : : num.m_quality), den.m_quality);
569 : 6448 : return ret;
570 : : }
571 : :
572 : : /* Return true when the probability of edge is reliable.
573 : :
574 : : The profile guessing code is good at predicting branch outcome (i.e.
575 : : taken/not taken), that is predicted right slightly over 75% of time.
576 : : It is however notoriously poor on predicting the probability itself.
577 : : In general the profile appear a lot flatter (with probabilities closer
578 : : to 50%) than the reality so it is bad idea to use it to drive optimization
579 : : such as those disabling dynamic branch prediction for well predictable
580 : : branches.
581 : :
582 : : There are two exceptions - edges leading to noreturn edges and edges
583 : : predicted by number of iterations heuristics are predicted well. This macro
584 : : should be able to distinguish those, but at the moment it simply check for
585 : : noreturn heuristic that is only one giving probability over 99% or bellow
586 : : 1%. In future we might want to propagate reliability information across the
587 : : CFG if we find this information useful on multiple places. */
588 : 0 : bool probably_reliable_p () const
589 : : {
590 : 0 : if (m_quality >= ADJUSTED)
591 : : return true;
592 : 0 : if (!initialized_p ())
593 : : return false;
594 : 0 : return m_val < max_probability / 100
595 : 0 : || m_val > max_probability - max_probability / 100;
596 : : }
597 : :
598 : : /* Return false if profile_probability is bogus. */
599 : 4102773290 : bool verify () const
600 : : {
601 : 4102773290 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
602 : 4102773290 : if (m_val == uninitialized_probability)
603 : 916705657 : return m_quality == GUESSED;
604 : 3186067633 : else if (m_quality < GUESSED)
605 : : return false;
606 : 3186067633 : return m_val <= max_probability;
607 : : }
608 : :
609 : : /* Comparisons are three-state and conservative. False is returned if
610 : : the inequality cannot be decided. */
611 : 5799135 : bool operator< (const profile_probability &other) const
612 : : {
613 : 5799135 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
614 : : }
615 : :
616 : 16520772 : bool operator> (const profile_probability &other) const
617 : : {
618 : 16520772 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
619 : : }
620 : :
621 : 4862459 : bool operator<= (const profile_probability &other) const
622 : : {
623 : 4862459 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
624 : : }
625 : :
626 : 918944 : bool operator>= (const profile_probability &other) const
627 : : {
628 : 918944 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
629 : : }
630 : :
631 : 17 : profile_probability operator* (int64_t num) const
632 : : {
633 : 17 : return apply_scale (num, 1);
634 : : }
635 : :
636 : : profile_probability operator*= (int64_t num)
637 : : {
638 : : *this = apply_scale (num, 1);
639 : : return *this;
640 : : }
641 : :
642 : 12892167 : profile_probability operator/ (int64_t den) const
643 : : {
644 : 12892167 : return apply_scale (1, den);
645 : : }
646 : :
647 : 243610 : profile_probability operator/= (int64_t den)
648 : : {
649 : 243610 : *this = apply_scale (1, den);
650 : 243610 : return *this;
651 : : }
652 : :
653 : : /* Compute n-th power. */
654 : : profile_probability pow (int) const;
655 : :
656 : : /* Compute sware root. */
657 : : profile_probability sqrt () const;
658 : :
659 : : /* Get the value of the count. */
660 : 0 : uint32_t value () const { return m_val; }
661 : :
662 : : /* Get the quality of the count. */
663 : 0 : enum profile_quality quality () const { return m_quality; }
664 : :
665 : : /* Output THIS to F. */
666 : : void dump (FILE *f) const;
667 : :
668 : : /* Output THIS to BUFFER. */
669 : : void dump (char *buffer) const;
670 : :
671 : : /* Print THIS to stderr. */
672 : : void debug () const;
673 : :
674 : : /* Return true if THIS is known to differ significantly from OTHER. */
675 : : bool differs_from_p (profile_probability other) const;
676 : :
677 : : /* Return if difference is greater than 50%. */
678 : : bool differs_lot_from_p (profile_probability other) const;
679 : :
680 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
681 : : happens with COUNT2 probability. Return probability that either *THIS or
682 : : OTHER happens. */
683 : : profile_probability combine_with_count (profile_count count1,
684 : : profile_probability other,
685 : : profile_count count2) const;
686 : :
687 : : /* Return probability as sreal. */
688 : : sreal to_sreal () const;
689 : : /* LTO streaming support. */
690 : : static profile_probability stream_in (class lto_input_block *);
691 : : void stream_out (struct output_block *);
692 : : void stream_out (struct lto_output_stream *);
693 : : };
694 : :
695 : : /* Main data type to hold profile counters in GCC. Profile counts originate
696 : : either from profile feedback, static profile estimation or both. We do not
697 : : perform whole program profile propagation and thus profile estimation
698 : : counters are often local to function, while counters from profile feedback
699 : : (or special cases of profile estimation) can be used inter-procedurally.
700 : :
701 : : There are 3 basic types
702 : : 1) local counters which are result of intra-procedural static profile
703 : : estimation.
704 : : 2) ipa counters which are result of profile feedback or special case
705 : : of static profile estimation (such as in function main).
706 : : 3) counters which counts as 0 inter-procedurally (because given function
707 : : was never run in train feedback) but they hold local static profile
708 : : estimate.
709 : :
710 : : Counters of type 1 and 3 cannot be mixed with counters of different type
711 : : within operation (because whole function should use one type of counter)
712 : : with exception that global zero mix in most operations where outcome is
713 : : well defined.
714 : :
715 : : To take local counter and use it inter-procedurally use ipa member function
716 : : which strips information irrelevant at the inter-procedural level.
717 : :
718 : : Counters are 61bit integers representing number of executions during the
719 : : train run or normalized frequency within the function.
720 : :
721 : : As the profile is maintained during the compilation, many adjustments are
722 : : made. Not all transformations can be made precisely, most importantly
723 : : when code is being duplicated. It also may happen that part of CFG has
724 : : profile counts known while other do not - for example when LTO optimizing
725 : : partly profiled program or when profile was lost due to COMDAT merging.
726 : :
727 : : For this reason profile_count tracks more information than
728 : : just unsigned integer and it is also ready for profile mismatches.
729 : : The API of this data type represent operations that are natural
730 : : on profile counts - sum, difference and operation with scales and
731 : : probabilities. All operations are safe by never getting negative counts
732 : : and they do end up in uninitialized scale if any of the parameters is
733 : : uninitialized.
734 : :
735 : : All comparisons that are three state and handling of probabilities. Thus
736 : : a < b is not equal to !(a >= b).
737 : :
738 : : The following pre-defined counts are available:
739 : :
740 : : profile_count::zero () for code that is known to execute zero times at
741 : : runtime (this can be detected statically i.e. for paths leading to
742 : : abort ();
743 : : profile_count::one () for code that is known to execute once (such as
744 : : main () function
745 : : profile_count::uninitialized () for unknown execution count.
746 : :
747 : : */
748 : :
749 : : struct GTY(()) profile_count
750 : : {
751 : : public:
752 : : /* Use 62bit to hold basic block counters. Should be at least
753 : : 64bit. Although a counter cannot be negative, we use a signed
754 : : type to hold various extra stages. */
755 : :
756 : : static const int n_bits = 61;
757 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
758 : : private:
759 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
760 : :
761 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
762 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
763 : : incorrectly detects the alignment of a structure where the only
764 : : 64-bit aligned object is a bit-field. We force the alignment of
765 : : the entire field to mitigate this. */
766 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
767 : : #else
768 : : #define UINT64_BIT_FIELD_ALIGN
769 : : #endif
770 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
771 : : #undef UINT64_BIT_FIELD_ALIGN
772 : : enum profile_quality m_quality : 3;
773 : : public:
774 : :
775 : : /* Return true if both values can meaningfully appear in single function
776 : : body. We have either all counters in function local or global, otherwise
777 : : operations between them are not really defined well. */
778 : 2468264571 : bool compatible_p (const profile_count other) const
779 : : {
780 : 4890960081 : if (!initialized_p () || !other.initialized_p ())
781 : : return true;
782 : 9678923698 : if (*this == zero ()
783 : 4834578406 : || other == zero ())
784 : 9767757 : return true;
785 : : /* Do not allow nonzero global profile together with local guesses
786 : : that are globally0. */
787 : 2412404889 : if (ipa ().nonzero_p ()
788 : 87865 : && !(other.ipa () == other))
789 : 0 : return false;
790 : 2412404889 : if (other.ipa ().nonzero_p ()
791 : 91531 : && !(ipa () == *this))
792 : 0 : return false;
793 : :
794 : 2412404889 : return ipa_p () == other.ipa_p ();
795 : : }
796 : :
797 : : /* Used for counters which are expected to be never executed. */
798 : 14281917673 : static profile_count zero ()
799 : : {
800 : 10513282872 : return from_gcov_type (0);
801 : : }
802 : :
803 : 4520 : static profile_count adjusted_zero ()
804 : : {
805 : 4520 : profile_count c;
806 : 4520 : c.m_val = 0;
807 : 4520 : c.m_quality = ADJUSTED;
808 : 4520 : return c;
809 : : }
810 : :
811 : : static profile_count guessed_zero ()
812 : : {
813 : : profile_count c;
814 : 0 : c.m_val = 0;
815 : 0 : c.m_quality = GUESSED;
816 : : return c;
817 : : }
818 : :
819 : 406 : static profile_count one ()
820 : : {
821 : 203 : return from_gcov_type (1);
822 : : }
823 : :
824 : : /* Value of counters which has not been initialized. Either because
825 : : initialization did not happen yet or because profile is unknown. */
826 : 6088624772 : static profile_count uninitialized ()
827 : : {
828 : 6088624772 : profile_count c;
829 : 6201270871 : c.m_val = uninitialized_count;
830 : 3861394018 : c.m_quality = GUESSED_LOCAL;
831 : 6088624772 : return c;
832 : : }
833 : :
834 : : /* Conversion to gcov_type is lossy. */
835 : 295259 : gcov_type to_gcov_type () const
836 : : {
837 : 0 : gcc_checking_assert (initialized_p ());
838 : 295259 : return m_val;
839 : : }
840 : :
841 : : /* Return true if value has been initialized. */
842 : 22925004974 : bool initialized_p () const
843 : : {
844 : 7962572370 : return m_val != uninitialized_count;
845 : : }
846 : :
847 : : /* Return true if value can be trusted. */
848 : 8025640 : bool reliable_p () const
849 : : {
850 : 8025383 : return m_quality >= ADJUSTED;
851 : : }
852 : :
853 : : /* Return true if value can be operated inter-procedurally. */
854 : 5760595857 : bool ipa_p () const
855 : : {
856 : 3337103259 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
857 : : }
858 : :
859 : : /* Return true if quality of profile is precise. */
860 : 48078225 : bool precise_p () const
861 : : {
862 : 48078225 : return m_quality == PRECISE;
863 : : }
864 : :
865 : : /* Get the value of the count. */
866 : 0 : uint64_t value () const { return m_val; }
867 : :
868 : : /* Get the quality of the count. */
869 : 50417 : enum profile_quality quality () const { return m_quality; }
870 : :
871 : : /* When merging basic blocks, the two different profile counts are unified.
872 : : Return true if this can be done without losing info about profile.
873 : : The only case we care about here is when first BB contains something
874 : : that makes it terminate in a way not visible in CFG. */
875 : 2767914 : bool ok_for_merging (profile_count other) const
876 : : {
877 : 2767914 : if (m_quality < ADJUSTED
878 : 9984 : || other.m_quality < ADJUSTED)
879 : : return true;
880 : 9237 : return !(other < *this);
881 : : }
882 : :
883 : : /* When merging two BBs with different counts, pick common count that looks
884 : : most representative. */
885 : 14048862 : profile_count merge (profile_count other) const
886 : : {
887 : 993235 : if (*this == other || !other.initialized_p ()
888 : 993006 : || m_quality > other.m_quality)
889 : 13056689 : return *this;
890 : 992173 : if (other.m_quality > m_quality
891 : 992173 : || other > *this)
892 : 403902 : return other;
893 : 588271 : return *this;
894 : : }
895 : :
896 : : /* Basic operations. */
897 : 15653636200 : bool operator== (const profile_count &other) const
898 : : {
899 : 11154183633 : return m_val == other.m_val && m_quality == other.m_quality;
900 : : }
901 : :
902 : 3443163 : profile_count operator+ (const profile_count &other) const
903 : : {
904 : 3443163 : if (other == zero ())
905 : 2169 : return *this;
906 : 3440994 : if (*this == zero ())
907 : 175545 : return other;
908 : 3265449 : if (!initialized_p () || !other.initialized_p ())
909 : 5318 : return uninitialized ();
910 : :
911 : 3260131 : profile_count ret;
912 : 3260131 : gcc_checking_assert (compatible_p (other));
913 : 3260131 : uint64_t ret_val = m_val + other.m_val;
914 : 3260131 : ret.m_val = MIN (ret_val, max_count);
915 : 3260131 : ret.m_quality = MIN (m_quality, other.m_quality);
916 : 3260131 : return ret;
917 : : }
918 : :
919 : 68533653 : profile_count &operator+= (const profile_count &other)
920 : : {
921 : 68533653 : if (other == zero ())
922 : 4822337 : return *this;
923 : 63711316 : if (*this == zero ())
924 : : {
925 : 38975674 : *this = other;
926 : 38975674 : return *this;
927 : : }
928 : 24735642 : if (!initialized_p () || !other.initialized_p ())
929 : 328854 : return *this = uninitialized ();
930 : : else
931 : : {
932 : 24406788 : gcc_checking_assert (compatible_p (other));
933 : 24406788 : uint64_t ret_val = m_val + other.m_val;
934 : 24406788 : m_val = MIN (ret_val, max_count);
935 : 24406788 : m_quality = MIN (m_quality, other.m_quality);
936 : : }
937 : 24406788 : return *this;
938 : : }
939 : :
940 : 17730859 : profile_count operator- (const profile_count &other) const
941 : : {
942 : 17745268 : if (*this == zero () || other == zero ())
943 : 1045002 : return *this;
944 : 16685857 : if (!initialized_p () || !other.initialized_p ())
945 : 7081338 : return uninitialized ();
946 : 9604519 : gcc_checking_assert (compatible_p (other));
947 : 9604519 : profile_count ret;
948 : 9604519 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
949 : 9604519 : ret.m_quality = MIN (m_quality, other.m_quality);
950 : 9604519 : return ret;
951 : : }
952 : :
953 : 8700921 : profile_count &operator-= (const profile_count &other)
954 : : {
955 : 9131633 : if (*this == zero () || other == zero ())
956 : 316811 : return *this;
957 : 8384110 : if (!initialized_p () || !other.initialized_p ())
958 : 347651 : return *this = uninitialized ();
959 : : else
960 : : {
961 : 8036459 : gcc_checking_assert (compatible_p (other));
962 : 8036459 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
963 : 8036459 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 8036459 : return *this;
966 : : }
967 : :
968 : : /* Return false if profile_count is bogus. */
969 : 3131220883 : bool verify () const
970 : : {
971 : 3131220883 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
972 : 3131220883 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
973 : : }
974 : :
975 : : /* Comparisons are three-state and conservative. False is returned if
976 : : the inequality cannot be decided. */
977 : 978933723 : bool operator< (const profile_count &other) const
978 : : {
979 : 978933723 : if (!initialized_p () || !other.initialized_p ())
980 : : return false;
981 : 974721762 : if (*this == zero ())
982 : 3813292 : return !(other == zero ());
983 : 972815116 : if (other == zero ())
984 : 1057 : return false;
985 : 972814059 : gcc_checking_assert (compatible_p (other));
986 : 972814059 : return m_val < other.m_val;
987 : : }
988 : :
989 : 32875669 : bool operator> (const profile_count &other) const
990 : : {
991 : 32875669 : if (!initialized_p () || !other.initialized_p ())
992 : : return false;
993 : 31948074 : if (*this == zero ())
994 : 265387 : return false;
995 : 31682687 : if (other == zero ())
996 : 1567204 : return !(*this == zero ());
997 : 30899085 : gcc_checking_assert (compatible_p (other));
998 : 30899085 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
999 : : }
1000 : :
1001 : : bool operator< (const gcov_type other) const
1002 : : {
1003 : : gcc_checking_assert (ipa_p ());
1004 : : gcc_checking_assert (other >= 0);
1005 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1006 : : }
1007 : :
1008 : 266042 : bool operator> (const gcov_type other) const
1009 : : {
1010 : 266042 : gcc_checking_assert (ipa_p ());
1011 : 266042 : gcc_checking_assert (other >= 0);
1012 : 266042 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1013 : : }
1014 : :
1015 : 982828 : bool operator<= (const profile_count &other) const
1016 : : {
1017 : 982828 : if (!initialized_p () || !other.initialized_p ())
1018 : : return false;
1019 : 982828 : if (*this == zero ())
1020 : 2676 : return true;
1021 : 980152 : if (other == zero ())
1022 : 0 : return (*this == zero ());
1023 : 980152 : gcc_checking_assert (compatible_p (other));
1024 : 980152 : return m_val <= other.m_val;
1025 : : }
1026 : :
1027 : 1306756 : bool operator>= (const profile_count &other) const
1028 : : {
1029 : 1306756 : if (!initialized_p () || !other.initialized_p ())
1030 : : return false;
1031 : 1305900 : if (other == zero ())
1032 : 271 : return true;
1033 : 1305629 : if (*this == zero ())
1034 : 23268 : return (other == zero ());
1035 : 1293995 : gcc_checking_assert (compatible_p (other));
1036 : 1293995 : return m_val >= other.m_val;
1037 : : }
1038 : :
1039 : 93114 : bool operator<= (const gcov_type other) const
1040 : : {
1041 : 93114 : gcc_checking_assert (ipa_p ());
1042 : 93114 : gcc_checking_assert (other >= 0);
1043 : 93114 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1044 : : }
1045 : :
1046 : 57269 : bool operator>= (const gcov_type other) const
1047 : : {
1048 : 57269 : gcc_checking_assert (ipa_p ());
1049 : 57269 : gcc_checking_assert (other >= 0);
1050 : 57269 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1051 : : }
1052 : :
1053 : 814912445 : profile_count operator* (int64_t num) const
1054 : : {
1055 : 814836372 : return apply_scale (num, 1);
1056 : : }
1057 : :
1058 : : profile_count operator*= (int64_t num)
1059 : : {
1060 : : *this = apply_scale (num, 1);
1061 : : return *this;
1062 : : }
1063 : :
1064 : 2245063 : profile_count operator/ (int64_t den) const
1065 : : {
1066 : 2245063 : return apply_scale (1, den);
1067 : : }
1068 : :
1069 : 65 : profile_count operator/= (int64_t den)
1070 : : {
1071 : 61 : *this = apply_scale (1, den);
1072 : 65 : return *this;
1073 : : }
1074 : :
1075 : : /* Return true when value is not zero and can be used for scaling.
1076 : : This is different from *this > 0 because that requires counter to
1077 : : be IPA. */
1078 : 5150051067 : bool nonzero_p () const
1079 : : {
1080 : 4992384757 : return initialized_p () && m_val != 0;
1081 : : }
1082 : :
1083 : : /* Make counter forcibly nonzero. */
1084 : 18099315 : profile_count force_nonzero () const
1085 : : {
1086 : 18099315 : if (!initialized_p ())
1087 : 26486 : return *this;
1088 : 18072829 : profile_count ret = *this;
1089 : 18072829 : if (ret.m_val == 0)
1090 : : {
1091 : 52840 : ret.m_val = 1;
1092 : 52840 : ret.m_quality = MIN (m_quality, ADJUSTED);
1093 : : }
1094 : 18072829 : return ret;
1095 : : }
1096 : :
1097 : 83526836 : profile_count max (profile_count other) const
1098 : : {
1099 : 83526836 : profile_count val = *this;
1100 : :
1101 : : /* Always prefer nonzero IPA counts over local counts. */
1102 : 92899796 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1103 : : {
1104 : 9306 : val = ipa ();
1105 : 9306 : other = other.ipa ();
1106 : : }
1107 : 83526836 : if (!initialized_p ())
1108 : 19929389 : return other;
1109 : 63597447 : if (!other.initialized_p ())
1110 : 1882785 : return *this;
1111 : 61714662 : if (*this == zero ())
1112 : 2124869 : return other;
1113 : 59589793 : if (other == zero ())
1114 : 4372851 : return *this;
1115 : 55216942 : gcc_checking_assert (compatible_p (other));
1116 : 55216942 : if (val.m_val < other.m_val || (m_val == other.m_val
1117 : 12965260 : && val.m_quality < other.m_quality))
1118 : 7845356 : return other;
1119 : 47371586 : return *this;
1120 : : }
1121 : :
1122 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1123 : : accordingly. */
1124 : : profile_count apply_probability (int prob) const
1125 : : {
1126 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1127 : : if (m_val == 0)
1128 : : return *this;
1129 : : if (!initialized_p ())
1130 : : return uninitialized ();
1131 : : profile_count ret;
1132 : : uint64_t tmp;
1133 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1134 : : ret.m_val = tmp;
1135 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1136 : : return ret;
1137 : : }
1138 : :
1139 : : /* Scale counter according to PROB. */
1140 : 354421275 : profile_count apply_probability (profile_probability prob) const
1141 : : {
1142 : 361833649 : if (*this == zero () || prob == profile_probability::always ())
1143 : 114619596 : return *this;
1144 : 239801679 : if (prob == profile_probability::never ())
1145 : 18878551 : return zero ();
1146 : 220923128 : if (!initialized_p () || !prob.initialized_p ())
1147 : 29091949 : return uninitialized ();
1148 : 191831179 : profile_count ret;
1149 : 191831179 : uint64_t tmp;
1150 : 191831179 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1151 : : &tmp);
1152 : 191831179 : ret.m_val = tmp;
1153 : 191831179 : ret.m_quality = MIN (m_quality, prob.m_quality);
1154 : 191831179 : return ret;
1155 : : }
1156 : :
1157 : : /* Return *THIS * NUM / DEN. */
1158 : 923789127 : profile_count apply_scale (int64_t num, int64_t den) const
1159 : : {
1160 : 923789127 : if (m_val == 0)
1161 : 22938969 : return *this;
1162 : 900850158 : if (!initialized_p ())
1163 : 6956 : return uninitialized ();
1164 : 900843202 : profile_count ret;
1165 : 900843202 : uint64_t tmp;
1166 : :
1167 : 900843202 : gcc_checking_assert (num >= 0 && den > 0);
1168 : 900843202 : safe_scale_64bit (m_val, num, den, &tmp);
1169 : 900843202 : ret.m_val = MIN (tmp, max_count);
1170 : 900843202 : ret.m_quality = MIN (m_quality, ADJUSTED);
1171 : 900843202 : return ret;
1172 : : }
1173 : :
1174 : 24717432 : profile_count apply_scale (profile_count num, profile_count den) const
1175 : : {
1176 : 24717432 : if (*this == zero ())
1177 : 1927477 : return *this;
1178 : 22789955 : if (num == zero ())
1179 : 71548 : return num;
1180 : 22718407 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1181 : 6296477 : return uninitialized ();
1182 : 16421930 : if (num == den)
1183 : 5484309 : return *this;
1184 : 10937621 : gcc_checking_assert (den.m_val);
1185 : :
1186 : 10937621 : profile_count ret;
1187 : 10937621 : uint64_t val;
1188 : 10937621 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1189 : 10937621 : ret.m_val = MIN (val, max_count);
1190 : 10937621 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1191 : : num.m_quality), den.m_quality);
1192 : : /* Be sure that ret is not local if num is global.
1193 : : Also ensure that ret is not global0 when num is global. */
1194 : 10937621 : if (num.ipa_p ())
1195 : 2992 : ret.m_quality = MAX (ret.m_quality,
1196 : : num == num.ipa () ? GUESSED : num.m_quality);
1197 : 10937621 : return ret;
1198 : : }
1199 : :
1200 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1201 : 16574332 : profile_count guessed_local () const
1202 : : {
1203 : 16574332 : profile_count ret = *this;
1204 : 16574332 : if (!initialized_p ())
1205 : 0 : return *this;
1206 : 16574332 : ret.m_quality = GUESSED_LOCAL;
1207 : 16574332 : return ret;
1208 : : }
1209 : :
1210 : : /* We know that profile is globally 0 but keep local profile if present. */
1211 : 766 : profile_count global0 () const
1212 : : {
1213 : 766 : profile_count ret = *this;
1214 : 766 : if (!initialized_p ())
1215 : 0 : return *this;
1216 : 766 : ret.m_quality = GUESSED_GLOBAL0;
1217 : 766 : return ret;
1218 : : }
1219 : :
1220 : : /* We know that profile is globally adjusted 0 but keep local profile
1221 : : if present. */
1222 : 37 : profile_count global0adjusted () const
1223 : : {
1224 : 37 : profile_count ret = *this;
1225 : 37 : if (!initialized_p ())
1226 : 0 : return *this;
1227 : 37 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1228 : 37 : return ret;
1229 : : }
1230 : :
1231 : : /* Return THIS with quality dropped to GUESSED. */
1232 : 255 : profile_count guessed () const
1233 : : {
1234 : 255 : profile_count ret = *this;
1235 : 255 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1236 : 255 : return ret;
1237 : : }
1238 : :
1239 : : /* Return variant of profile count which is always safe to compare
1240 : : across functions. */
1241 : 5970551056 : profile_count ipa () const
1242 : : {
1243 : 5970551056 : if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
1244 : 20389147 : return *this;
1245 : 5950161909 : if (m_quality == GUESSED_GLOBAL0)
1246 : 43936 : return zero ();
1247 : 5950117973 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1248 : 4520 : return adjusted_zero ();
1249 : 5950113453 : return uninitialized ();
1250 : : }
1251 : :
1252 : : /* Return THIS with quality dropped to AFDO. */
1253 : 0 : profile_count afdo () const
1254 : : {
1255 : 0 : profile_count ret = *this;
1256 : 0 : ret.m_quality = AFDO;
1257 : 0 : return ret;
1258 : : }
1259 : :
1260 : : /* Return probability of event with counter THIS within event with counter
1261 : : OVERALL. */
1262 : 1025996247 : profile_probability probability_in (const profile_count overall) const
1263 : : {
1264 : 1026011268 : if (*this == zero ()
1265 : 15021 : && !(overall == zero ()))
1266 : 1537 : return profile_probability::never ();
1267 : 1025994161 : if (!initialized_p () || !overall.initialized_p ()
1268 : 2051988871 : || !overall.m_val)
1269 : 59217 : return profile_probability::uninitialized ();
1270 : 1232874303 : if (*this == overall && m_quality == PRECISE)
1271 : 29731 : return profile_probability::always ();
1272 : 1025905762 : profile_probability ret;
1273 : 1025905762 : gcc_checking_assert (compatible_p (overall));
1274 : :
1275 : 1025905762 : if (overall.m_val < m_val)
1276 : : {
1277 : 89140 : ret.m_val = profile_probability::max_probability;
1278 : 89140 : ret.m_quality = GUESSED;
1279 : 89140 : return ret;
1280 : : }
1281 : : else
1282 : 1025816622 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1283 : : overall.m_val);
1284 : 1025816622 : ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
1285 : : GUESSED), ADJUSTED);
1286 : 1025816622 : return ret;
1287 : : }
1288 : :
1289 : : /* Return true if profile count is very large, so we risk overflows
1290 : : with loop transformations. */
1291 : : bool
1292 : 1002018 : very_large_p ()
1293 : : {
1294 : 1002018 : if (!initialized_p ())
1295 : : return false;
1296 : 1002018 : return m_val > max_count / 65536;
1297 : : }
1298 : :
1299 : : int to_frequency (struct function *fun) const;
1300 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1301 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1302 : :
1303 : : /* Output THIS to F. */
1304 : : void dump (FILE *f, struct function *fun = NULL) const;
1305 : :
1306 : : /* Print THIS to stderr. */
1307 : : void debug () const;
1308 : :
1309 : : /* Return true if THIS is known to differ significantly from OTHER. */
1310 : : bool differs_from_p (profile_count other) const;
1311 : :
1312 : : /* We want to scale profile across function boundary from NUM to DEN.
1313 : : Take care of the side case when NUM and DEN are zeros of incompatible
1314 : : kinds. */
1315 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1316 : :
1317 : : /* THIS is a count of bb which is known to be executed IPA times.
1318 : : Combine this information into bb counter. This means returning IPA
1319 : : if it is nonzero, not changing anything if IPA is uninitialized
1320 : : and if IPA is zero, turning THIS into corresponding local profile with
1321 : : global0. */
1322 : : profile_count combine_with_ipa_count (profile_count ipa);
1323 : :
1324 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1325 : : profile_count combine_with_ipa_count_within
1326 : : (profile_count ipa, profile_count ipa2);
1327 : :
1328 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1329 : : Conversions back and forth are used to read the coverage and get it
1330 : : into internal representation. */
1331 : : static profile_count from_gcov_type (gcov_type v,
1332 : : profile_quality quality = PRECISE);
1333 : :
1334 : : /* LTO streaming support. */
1335 : : static profile_count stream_in (class lto_input_block *);
1336 : : void stream_out (struct output_block *);
1337 : : void stream_out (struct lto_output_stream *);
1338 : : };
1339 : : #endif
|
