Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2025 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : : GUESSED_GLOBAL0_AFDO,
42 : :
43 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : : GUESSED_GLOBAL0_ADJUSTED,
45 : :
46 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : : better. This is the case of functions not run in profile feedback.
48 : : Never used by probabilities. */
49 : : GUESSED_GLOBAL0,
50 : :
51 : : /* Profile is based on static branch prediction heuristics. It may or may
52 : : not reflect the reality but it can be compared interprocedurally
53 : : (for example, we inlined function w/o profile feedback into function
54 : : with feedback and propagated from that). */
55 : : GUESSED,
56 : :
57 : : /* Profile was determined by autofdo. */
58 : : AFDO,
59 : :
60 : : /* Profile was originally based on feedback but it was adjusted
61 : : by code duplicating optimization. It may not precisely reflect the
62 : : particular code path. */
63 : : ADJUSTED,
64 : :
65 : : /* Profile was read from profile feedback or determined by accurate static
66 : : method. */
67 : : PRECISE
68 : : };
69 : :
70 : : extern const char *profile_quality_as_string (enum profile_quality);
71 : : extern bool parse_profile_quality (const char *value,
72 : : profile_quality *quality);
73 : :
74 : : /* The base value for branch probability notes and edge probabilities. */
75 : : #define REG_BR_PROB_BASE 10000
76 : :
77 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 : :
79 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 : :
81 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 : :
83 : : inline bool
84 : 1261149934 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : : {
86 : : #if (GCC_VERSION >= 5000)
87 : 1261149934 : uint64_t tmp;
88 : 1261149934 : if (!__builtin_mul_overflow (a, b, &tmp)
89 : 1261149934 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : : {
91 : 1260561252 : *res = tmp / c;
92 : 1260561252 : return true;
93 : : }
94 : 588682 : if (c == 1)
95 : : {
96 : 28199 : *res = (uint64_t) -1;
97 : 28199 : return false;
98 : : }
99 : : #else
100 : : if (a < ((uint64_t)1 << 31)
101 : : && b < ((uint64_t)1 << 31)
102 : : && c < ((uint64_t)1 << 31))
103 : : {
104 : : *res = (a * b + (c / 2)) / c;
105 : : return true;
106 : : }
107 : : #endif
108 : 560483 : return slow_safe_scale_64bit (a, b, c, res);
109 : : }
110 : :
111 : : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : : with capping so probability is always in range [0,1] and scaling requiring
113 : : values greater than 1 needs to be represented otherwise.
114 : :
115 : : In addition to actual value the quality of profile is tracked and propagated
116 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : : that has not been determined yet (for example because of
118 : : -fno-guess-branch-probability)
119 : :
120 : : Typically probabilities are derived from profile feedback (via
121 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : : thorough the compilation.
123 : :
124 : : Named probabilities are available:
125 : : - never (0 probability)
126 : : - guessed_never
127 : : - very_unlikely (1/2000 probability)
128 : : - unlikely (1/5 probability)
129 : : - even (1/2 probability)
130 : : - likely (4/5 probability)
131 : : - very_likely (1999/2000 probability)
132 : : - guessed_always
133 : : - always
134 : :
135 : : Named probabilities except for never/always are assumed to be statically
136 : : guessed and thus not necessarily accurate. The difference between never
137 : : and guessed_never is that the first one should be used only in case that
138 : : well behaving program will very likely not execute the "never" path.
139 : : For example if the path is going to abort () call or it exception handling.
140 : :
141 : : Always and guessed_always probabilities are symmetric.
142 : :
143 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : : integer arithmetics. Once the code is converted to branch probabilities,
145 : : these conversions will probably go away because they are lossy.
146 : : */
147 : :
148 : : class GTY((user)) profile_probability
149 : : {
150 : : static const int n_bits = 29;
151 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : : will lead to harder multiplication sequences. */
153 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : : static const uint32_t uninitialized_probability
155 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : : /* For probibilityes quality is either UNINITIALIZED (0)
157 : : or greated then GUESSED. To save bits we store it in
158 : : adjusted form which skips the invalid values. */
159 : : static const int min_quality = GUESSED;
160 : :
161 : : uint32_t m_val : 29;
162 : : unsigned m_adjusted_quality : 3;
163 : :
164 : : friend struct profile_count;
165 : :
166 : : /* Set the quality of the probability. */
167 : : void
168 : 5334872237 : set_quality (profile_quality quality)
169 : : {
170 : 1144307394 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : : || (quality >= min_quality && quality <= PRECISE));
172 : 1144307478 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 : 1142541924 : }
174 : :
175 : : public:
176 : 4192330313 : profile_probability (): m_val (uninitialized_probability)
177 : 2283454809 : { set_quality (GUESSED); }
178 : :
179 : 132 : profile_probability (uint32_t val, profile_quality quality):
180 : 132 : m_val (val)
181 : 132 : { set_quality (quality); }
182 : :
183 : : /* Named probabilities. */
184 : 685885269 : static profile_probability never ()
185 : : {
186 : 685885269 : profile_probability ret;
187 : 685885269 : ret.m_val = 0;
188 : 685885269 : ret.set_quality (PRECISE);
189 : 385193706 : return ret;
190 : : }
191 : :
192 : 5309652 : static profile_probability guessed_never ()
193 : : {
194 : 5309652 : profile_probability ret;
195 : 5309652 : ret.m_val = 0;
196 : 5309652 : ret.set_quality (GUESSED);
197 : 5309652 : return ret;
198 : : }
199 : :
200 : 5593432 : static profile_probability very_unlikely ()
201 : : {
202 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 : 11186864 : profile_probability r = guessed_always () / 2000;
204 : 5593432 : r.m_val--;
205 : 5593432 : return r;
206 : : }
207 : :
208 : 81127 : static profile_probability unlikely ()
209 : : {
210 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 : 162254 : profile_probability r = guessed_always () / 5;
212 : 81127 : r.m_val--;
213 : 81127 : return r;
214 : : }
215 : :
216 : 801332 : static profile_probability even ()
217 : : {
218 : 1602664 : return guessed_always () / 2;
219 : : }
220 : :
221 : 338282 : static profile_probability very_likely ()
222 : : {
223 : 338282 : return always () - very_unlikely ();
224 : : }
225 : :
226 : 32178 : static profile_probability likely ()
227 : : {
228 : 32178 : return always () - unlikely ();
229 : : }
230 : : /* Return true when value is not zero and can be used for scaling. */
231 : 1442 : bool nonzero_p () const
232 : : {
233 : 1442 : return initialized_p () && m_val != 0;
234 : : }
235 : :
236 : 6782323 : static profile_probability guessed_always ()
237 : : {
238 : 6782323 : profile_probability ret;
239 : 6782323 : ret.m_val = max_probability;
240 : 6782323 : ret.set_quality (GUESSED);
241 : 6579832 : return ret;
242 : : }
243 : :
244 : 536171910 : static profile_probability always ()
245 : : {
246 : 536171910 : profile_probability ret;
247 : 536171910 : ret.m_val = max_probability;
248 : 536171910 : ret.set_quality (PRECISE);
249 : 362504927 : return ret;
250 : : }
251 : :
252 : : /* Probabilities which has not been initialized. Either because
253 : : initialization did not happen yet or because profile is unknown. */
254 : 917309783 : static profile_probability uninitialized ()
255 : : {
256 : 917309783 : profile_probability c;
257 : 917309783 : c.m_val = uninitialized_probability;
258 : 917309783 : c.set_quality (GUESSED);
259 : 903752658 : return c;
260 : : }
261 : :
262 : : /* Return true if value has been initialized. */
263 : 5747078817 : bool initialized_p () const
264 : : {
265 : 4022124831 : return m_val != uninitialized_probability;
266 : : }
267 : :
268 : : /* Return true if value can be trusted. */
269 : 1672856 : bool reliable_p () const
270 : : {
271 : 114870324 : return quality () >= ADJUSTED;
272 : : }
273 : :
274 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : : this is mostly to support legacy code and should go away. */
276 : 3232288 : static profile_probability from_reg_br_prob_base (int v)
277 : : {
278 : 3232288 : profile_probability ret;
279 : 3232288 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 : 3232288 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 : 3232288 : ret.set_quality (GUESSED);
282 : 3232288 : return ret;
283 : : }
284 : :
285 : : /* Return THIS with quality set to ADJUSTED. */
286 : 42 : profile_probability adjusted () const
287 : : {
288 : 42 : profile_probability ret = *this;
289 : 42 : if (!initialized_p ())
290 : 0 : return *this;
291 : 42 : ret.set_quality (ADJUSTED);
292 : 42 : return ret;
293 : : }
294 : :
295 : 1124401768 : int to_reg_br_prob_base () const
296 : : {
297 : 1124401768 : gcc_checking_assert (initialized_p ());
298 : 1124401768 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : : }
300 : :
301 : : /* Conversion to and from RTL representation of profile probabilities. */
302 : 495027577 : static profile_probability from_reg_br_prob_note (int v)
303 : : {
304 : 495027577 : profile_probability ret;
305 : 495027577 : ret.m_val = ((unsigned int)v) / 8;
306 : 495027577 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 : 15904113 : return ret;
308 : : }
309 : :
310 : 465745861 : int to_reg_br_prob_note () const
311 : : {
312 : 465745861 : gcc_checking_assert (initialized_p ());
313 : 465745861 : int ret = m_val * 8 + m_adjusted_quality;
314 : 465745861 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 : 465745861 : return ret;
316 : : }
317 : :
318 : : /* Return VAL1/VAL2. */
319 : 3093 : static profile_probability probability_in_gcov_type
320 : : (gcov_type val1, gcov_type val2)
321 : : {
322 : 3093 : profile_probability ret;
323 : 3093 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 : 3093 : if (val1 > val2)
325 : : ret.m_val = max_probability;
326 : : else
327 : : {
328 : 3093 : uint64_t tmp;
329 : 3093 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 : 3093 : gcc_checking_assert (tmp <= max_probability);
331 : 3093 : ret.m_val = tmp;
332 : : }
333 : 3093 : ret.set_quality (PRECISE);
334 : 3093 : return ret;
335 : : }
336 : :
337 : : /* Basic operations. */
338 : 1554036240 : bool operator== (const profile_probability &other) const
339 : : {
340 : 2201017620 : return m_val == other.m_val && quality () == other.quality ();
341 : : }
342 : :
343 : 6748805 : profile_probability operator+ (const profile_probability &other) const
344 : : {
345 : 6748805 : if (other == never ())
346 : 11619 : return *this;
347 : 6737186 : if (*this == never ())
348 : 23632 : return other;
349 : 6713554 : if (!initialized_p () || !other.initialized_p ())
350 : 5459831 : return uninitialized ();
351 : :
352 : 1253723 : profile_probability ret;
353 : 1253723 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 : 1253723 : ret.set_quality (MIN (quality (), other.quality ()));
355 : 1253723 : return ret;
356 : : }
357 : :
358 : 6278163 : profile_probability &operator+= (const profile_probability &other)
359 : : {
360 : 6278163 : if (other == never ())
361 : : return *this;
362 : 5369421 : if (*this == never ())
363 : : {
364 : 868890 : *this = other;
365 : 868890 : return *this;
366 : : }
367 : 4500531 : if (!initialized_p () || !other.initialized_p ())
368 : 2917953 : return *this = uninitialized ();
369 : : else
370 : : {
371 : 1582578 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 : 1582578 : set_quality (MIN (quality (), other.quality ()));
373 : : }
374 : 1582578 : return *this;
375 : : }
376 : :
377 : 34373270 : profile_probability operator- (const profile_probability &other) const
378 : : {
379 : 68746540 : if (*this == never ()
380 : 34373270 : || other == never ())
381 : 1750997 : return *this;
382 : 32622273 : if (!initialized_p () || !other.initialized_p ())
383 : 2156480 : return uninitialized ();
384 : 30465793 : profile_probability ret;
385 : 30465793 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 : 30465793 : ret.set_quality (MIN (quality (), other.quality ()));
387 : 30465793 : return ret;
388 : : }
389 : :
390 : 11421853 : profile_probability &operator-= (const profile_probability &other)
391 : : {
392 : 22843706 : if (*this == never ()
393 : 11421853 : || other == never ())
394 : 2171707 : return *this;
395 : 9250146 : if (!initialized_p () || !other.initialized_p ())
396 : 2134241 : return *this = uninitialized ();
397 : : else
398 : : {
399 : 7115905 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 : 7115905 : set_quality (MIN (quality (), other.quality ()));
401 : : }
402 : 7115905 : return *this;
403 : : }
404 : :
405 : 1180227 : profile_probability operator* (const profile_probability &other) const
406 : : {
407 : 2360454 : if (*this == never ()
408 : 1180227 : || other == never ())
409 : 109245 : return never ();
410 : 1070982 : if (!initialized_p () || !other.initialized_p ())
411 : 58007 : return uninitialized ();
412 : 1012975 : profile_probability ret;
413 : 1012975 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 : 1012975 : ret.set_quality (MIN (quality (), other.quality ()));
415 : 1012975 : return ret;
416 : : }
417 : :
418 : 161539 : profile_probability &operator*= (const profile_probability &other)
419 : : {
420 : 323078 : if (*this == never ()
421 : 161539 : || other == never ())
422 : 336 : return *this = never ();
423 : 161203 : if (!initialized_p () || !other.initialized_p ())
424 : 9 : return *this = uninitialized ();
425 : : else
426 : : {
427 : 161194 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 : 315983 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : : }
430 : 161194 : return *this;
431 : : }
432 : :
433 : 1797028 : profile_probability operator/ (const profile_probability &other) const
434 : : {
435 : 1797028 : if (*this == never ())
436 : 106882 : return never ();
437 : 1690146 : if (!initialized_p () || !other.initialized_p ())
438 : 225144 : return uninitialized ();
439 : 1465002 : profile_probability ret;
440 : : /* If we get probability above 1, mark it as unreliable and return 1. */
441 : 1465002 : if (m_val >= other.m_val)
442 : : {
443 : 22055 : ret.m_val = max_probability;
444 : 22055 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : : GUESSED));
446 : 22055 : return ret;
447 : : }
448 : 1442947 : else if (!m_val)
449 : 60231 : ret.m_val = 0;
450 : : else
451 : : {
452 : 1382716 : gcc_checking_assert (other.m_val);
453 : 1382716 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
454 : : other.m_val),
455 : : max_probability);
456 : : }
457 : 2877623 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
458 : 1442947 : return ret;
459 : : }
460 : :
461 : 376049 : profile_probability &operator/= (const profile_probability &other)
462 : : {
463 : 376049 : if (*this == never ())
464 : 1315 : return *this = never ();
465 : 374734 : if (!initialized_p () || !other.initialized_p ())
466 : 0 : return *this = uninitialized ();
467 : : else
468 : : {
469 : : /* If we get probability above 1, mark it as unreliable
470 : : and return 1. */
471 : 374734 : if (m_val > other.m_val)
472 : : {
473 : 644 : m_val = max_probability;
474 : 644 : set_quality (MIN (MIN (quality (), other.quality ()),
475 : : GUESSED));
476 : 644 : return *this;
477 : : }
478 : 374090 : else if (!m_val)
479 : : ;
480 : : else
481 : : {
482 : 372241 : gcc_checking_assert (other.m_val);
483 : 372241 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
484 : : other.m_val),
485 : : max_probability);
486 : : }
487 : 682713 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
488 : : }
489 : 374090 : return *this;
490 : : }
491 : :
492 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
493 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
494 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
495 : : == ORIG. This is useful e.g. when splitting a conditional
496 : : branch like:
497 : : if (cond)
498 : : goto lab; // ORIG probability
499 : : into
500 : : if (cond1)
501 : : goto lab; // FIRST = ORIG * CPROB probability
502 : : if (cond2)
503 : : goto lab; // SECOND probability
504 : : such that the overall probability of jumping to lab remains
505 : : the same. CPROB gives the relative probability between the
506 : : branches. */
507 : 290370 : profile_probability split (const profile_probability &cprob)
508 : : {
509 : 290370 : profile_probability ret = *this * cprob;
510 : : /* The following is equivalent to:
511 : : *this = cprob.invert () * *this / ret.invert ();
512 : : Avoid scaling when overall outcome is supposed to be always.
513 : : Without knowing that one is inverse of other, the result would be
514 : : conservative. */
515 : 290370 : if (!(*this == always ()))
516 : 281609 : *this = (*this - ret) / ret.invert ();
517 : 290370 : return ret;
518 : : }
519 : :
520 : 282482 : gcov_type apply (gcov_type val) const
521 : : {
522 : 282482 : if (*this == uninitialized ())
523 : 61 : return val / 2;
524 : 282421 : return RDIV (val * m_val, max_probability);
525 : : }
526 : :
527 : : /* Return 1-*THIS. */
528 : 21643257 : profile_probability invert () const
529 : : {
530 : 31061113 : return always() - *this;
531 : : }
532 : :
533 : : /* Return THIS with quality dropped to GUESSED. */
534 : 891653 : profile_probability guessed () const
535 : : {
536 : 891653 : profile_probability ret = *this;
537 : 891653 : ret.set_quality (GUESSED);
538 : 891653 : return ret;
539 : : }
540 : :
541 : : /* Return THIS with quality dropped to AFDO. */
542 : : profile_probability afdo () const
543 : : {
544 : : profile_probability ret = *this;
545 : : ret.set_quality (AFDO);
546 : : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 15025313 : profile_probability apply_scale (int64_t num, int64_t den) const
551 : : {
552 : 15025313 : if (*this == never ())
553 : 14333 : return *this;
554 : 15010980 : if (!initialized_p ())
555 : 5862960 : return uninitialized ();
556 : 9148020 : profile_probability ret;
557 : 9148020 : uint64_t tmp;
558 : 9148020 : safe_scale_64bit (m_val, num, den, &tmp);
559 : 9148020 : ret.m_val = MIN (tmp, max_probability);
560 : 9148020 : ret.set_quality (MIN (quality (), ADJUSTED));
561 : 9148020 : return ret;
562 : : }
563 : :
564 : : /* Return *THIS * NUM / DEN. */
565 : 7239 : profile_probability apply_scale (profile_probability num,
566 : : profile_probability den) const
567 : : {
568 : 7239 : if (*this == never ())
569 : 42 : return *this;
570 : 7197 : if (num == never ())
571 : 0 : return num;
572 : 7197 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
573 : 0 : return uninitialized ();
574 : 7197 : if (num == den)
575 : 0 : return *this;
576 : 7197 : gcc_checking_assert (den.m_val);
577 : :
578 : 7197 : profile_probability ret;
579 : 7197 : uint64_t val;
580 : 7197 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
581 : 7197 : ret.m_val = MIN (val, max_probability);
582 : 7197 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
583 : : num.quality ()), den.quality ()));
584 : 7197 : return ret;
585 : : }
586 : :
587 : : /* Return true when the probability of edge is reliable.
588 : :
589 : : The profile guessing code is good at predicting branch outcome (i.e.
590 : : taken/not taken), that is predicted right slightly over 75% of time.
591 : : It is however notoriously poor on predicting the probability itself.
592 : : In general the profile appear a lot flatter (with probabilities closer
593 : : to 50%) than the reality so it is bad idea to use it to drive optimization
594 : : such as those disabling dynamic branch prediction for well predictable
595 : : branches.
596 : :
597 : : There are two exceptions - edges leading to noreturn edges and edges
598 : : predicted by number of iterations heuristics are predicted well. This macro
599 : : should be able to distinguish those, but at the moment it simply check for
600 : : noreturn heuristic that is only one giving probability over 99% or below
601 : : 1%. In future we might want to propagate reliability information across the
602 : : CFG if we find this information useful on multiple places. */
603 : 0 : bool probably_reliable_p () const
604 : : {
605 : 0 : if (quality () >= ADJUSTED)
606 : : return true;
607 : 0 : if (!initialized_p ())
608 : : return false;
609 : 0 : return m_val < max_probability / 100
610 : 0 : || m_val > max_probability - max_probability / 100;
611 : : }
612 : :
613 : : /* Return false if profile_probability is bogus. */
614 : 4432187542 : bool verify () const
615 : : {
616 : 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
617 : 4432187542 : if (m_val == uninitialized_probability)
618 : 944508604 : return quality () == GUESSED;
619 : 3487678938 : else if (quality () < GUESSED)
620 : : return false;
621 : 3487678938 : return m_val <= max_probability;
622 : : }
623 : :
624 : : /* Comparisons are three-state and conservative. False is returned if
625 : : the inequality cannot be decided. */
626 : 8464577 : bool operator< (const profile_probability &other) const
627 : : {
628 : 8464577 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
629 : : }
630 : :
631 : 17974499 : bool operator> (const profile_probability &other) const
632 : : {
633 : 17974499 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
634 : : }
635 : :
636 : 5709567 : bool operator<= (const profile_probability &other) const
637 : : {
638 : 5709567 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
639 : : }
640 : :
641 : 1005156 : bool operator>= (const profile_probability &other) const
642 : : {
643 : 1005156 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
644 : : }
645 : :
646 : 17 : profile_probability operator* (int64_t num) const
647 : : {
648 : 17 : return apply_scale (num, 1);
649 : : }
650 : :
651 : : profile_probability operator*= (int64_t num)
652 : : {
653 : : *this = apply_scale (num, 1);
654 : : return *this;
655 : : }
656 : :
657 : 14354962 : profile_probability operator/ (int64_t den) const
658 : : {
659 : 14354962 : return apply_scale (1, den);
660 : : }
661 : :
662 : 242596 : profile_probability operator/= (int64_t den)
663 : : {
664 : 242596 : *this = apply_scale (1, den);
665 : 242596 : return *this;
666 : : }
667 : :
668 : : /* Compute n-th power. */
669 : : profile_probability pow (int) const;
670 : :
671 : : /* Compute sware root. */
672 : : profile_probability sqrt () const;
673 : :
674 : : /* Get the value of the probability. */
675 : 0 : uint32_t value () const { return m_val; }
676 : :
677 : : /* Get the quality of the probability. */
678 : 6094347170 : enum profile_quality quality () const
679 : : {
680 : 5879596320 : return (profile_quality) (m_adjusted_quality
681 : 5879619440 : ? m_adjusted_quality + min_quality - 1
682 : : : UNINITIALIZED_PROFILE);
683 : : }
684 : :
685 : : /* Output THIS to F. */
686 : : void dump (FILE *f) const;
687 : :
688 : : /* Output THIS to BUFFER. */
689 : : void dump (char *buffer) const;
690 : :
691 : : /* Print THIS to stderr. */
692 : : void debug () const;
693 : :
694 : : /* Return true if THIS is known to differ significantly from OTHER. */
695 : : bool differs_from_p (profile_probability other) const;
696 : :
697 : : /* Return if difference is greater than 50%. */
698 : : bool differs_lot_from_p (profile_probability other) const;
699 : :
700 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
701 : : happens with COUNT2 probability. Return probability that either *THIS or
702 : : OTHER happens. */
703 : : profile_probability combine_with_count (profile_count count1,
704 : : profile_probability other,
705 : : profile_count count2) const;
706 : :
707 : : /* Return probability as sreal. */
708 : : sreal to_sreal () const;
709 : : /* LTO streaming support. */
710 : : static profile_probability stream_in (class lto_input_block *);
711 : : void stream_out (struct output_block *);
712 : : void stream_out (struct lto_output_stream *);
713 : : };
714 : :
715 : : /* Main data type to hold profile counters in GCC. Profile counts originate
716 : : either from profile feedback, static profile estimation or both. We do not
717 : : perform whole program profile propagation and thus profile estimation
718 : : counters are often local to function, while counters from profile feedback
719 : : (or special cases of profile estimation) can be used inter-procedurally.
720 : :
721 : : There are 3 basic types
722 : : 1) local counters which are result of intra-procedural static profile
723 : : estimation.
724 : : 2) ipa counters which are result of profile feedback or special case
725 : : of static profile estimation (such as in function main).
726 : : 3) counters which counts as 0 inter-procedurally (because given function
727 : : was never run in train feedback) but they hold local static profile
728 : : estimate.
729 : :
730 : : Counters of type 1 and 3 cannot be mixed with counters of different type
731 : : within operation (because whole function should use one type of counter)
732 : : with exception that global zero mix in most operations where outcome is
733 : : well defined.
734 : :
735 : : To take local counter and use it inter-procedurally use ipa member function
736 : : which strips information irrelevant at the inter-procedural level.
737 : :
738 : : Counters are 61bit integers representing number of executions during the
739 : : train run or normalized frequency within the function.
740 : :
741 : : As the profile is maintained during the compilation, many adjustments are
742 : : made. Not all transformations can be made precisely, most importantly
743 : : when code is being duplicated. It also may happen that part of CFG has
744 : : profile counts known while other do not - for example when LTO optimizing
745 : : partly profiled program or when profile was lost due to COMDAT merging.
746 : :
747 : : For this reason profile_count tracks more information than
748 : : just unsigned integer and it is also ready for profile mismatches.
749 : : The API of this data type represent operations that are natural
750 : : on profile counts - sum, difference and operation with scales and
751 : : probabilities. All operations are safe by never getting negative counts
752 : : and they do end up in uninitialized scale if any of the parameters is
753 : : uninitialized.
754 : :
755 : : All comparisons that are three state and handling of probabilities. Thus
756 : : a < b is not equal to !(a >= b).
757 : :
758 : : The following pre-defined counts are available:
759 : :
760 : : profile_count::zero () for code that is known to execute zero times at
761 : : runtime (this can be detected statically i.e. for paths leading to
762 : : abort ();
763 : : profile_count::one () for code that is known to execute once (such as
764 : : main () function
765 : : profile_count::uninitialized () for unknown execution count.
766 : :
767 : : */
768 : :
769 : : struct GTY(()) profile_count
770 : : {
771 : : public:
772 : : /* Use 60bit to hold basic block counters. Should be at least
773 : : 64bit. Although a counter cannot be negative, we use a signed
774 : : type to hold various extra stages. */
775 : :
776 : : static const int n_bits = 60;
777 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
778 : : private:
779 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
780 : :
781 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
782 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
783 : : incorrectly detects the alignment of a structure where the only
784 : : 64-bit aligned object is a bit-field. We force the alignment of
785 : : the entire field to mitigate this. */
786 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
787 : : #else
788 : : #define UINT64_BIT_FIELD_ALIGN
789 : : #endif
790 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
791 : : #undef UINT64_BIT_FIELD_ALIGN
792 : : enum profile_quality m_quality : 4;
793 : : public:
794 : :
795 : : /* Return true if both values can meaningfully appear in single function
796 : : body. We have either all counters in function local or global, otherwise
797 : : operations between them are not really defined well. */
798 : 5007097382 : bool compatible_p (const profile_count other) const
799 : : {
800 : 9208320834 : if (!initialized_p () || !other.initialized_p ())
801 : : return true;
802 : 16670925187 : if (*this == zero ()
803 : 8269774773 : || other == zero ())
804 : 131376860 : return true;
805 : : /* Do not allow nonzero global profile together with local guesses
806 : : that are globally0. */
807 : 4069198347 : if (ipa ().nonzero_p ()
808 : 245284 : && !(other.ipa () == other))
809 : 0 : return false;
810 : 4069198347 : if (other.ipa ().nonzero_p ()
811 : 254995 : && !(ipa () == *this))
812 : 0 : return false;
813 : :
814 : 4069198347 : return ipa_p () == other.ipa_p ();
815 : : }
816 : :
817 : : /* Used for counters which are expected to be never executed. */
818 : 18712328117 : static profile_count zero ()
819 : : {
820 : 14549609301 : return from_gcov_type (0);
821 : : }
822 : :
823 : 10698 : static profile_count adjusted_zero ()
824 : : {
825 : 10698 : profile_count c;
826 : 10698 : c.m_val = 0;
827 : 10698 : c.m_quality = ADJUSTED;
828 : 10698 : return c;
829 : : }
830 : :
831 : 0 : static profile_count afdo_zero ()
832 : : {
833 : 0 : profile_count c;
834 : 0 : c.m_val = 0;
835 : 0 : c.m_quality = AFDO;
836 : 0 : return c;
837 : : }
838 : :
839 : : static profile_count guessed_zero ()
840 : : {
841 : : profile_count c;
842 : : c.m_val = 0;
843 : : c.m_quality = GUESSED;
844 : : return c;
845 : : }
846 : :
847 : 50606 : static profile_count one ()
848 : : {
849 : 50412 : return from_gcov_type (1);
850 : : }
851 : :
852 : : /* Value of counters which has not been initialized. Either because
853 : : initialization did not happen yet or because profile is unknown. */
854 : 9573280908 : static profile_count uninitialized ()
855 : : {
856 : 9573280908 : profile_count c;
857 : 9573280908 : c.m_val = uninitialized_count;
858 : 9573280908 : c.m_quality = GUESSED_LOCAL;
859 : 9540925779 : return c;
860 : : }
861 : :
862 : : /* Conversion to gcov_type is lossy. */
863 : 296634 : gcov_type to_gcov_type () const
864 : : {
865 : 0 : gcc_checking_assert (initialized_p ());
866 : 296634 : return m_val;
867 : : }
868 : :
869 : : /* Return true if value has been initialized. */
870 : 33246135901 : bool initialized_p () const
871 : : {
872 : 10970280341 : return m_val != uninitialized_count;
873 : : }
874 : :
875 : : /* Return true if value can be trusted. */
876 : 23192268 : bool reliable_p () const
877 : : {
878 : 23191998 : return m_quality >= ADJUSTED;
879 : : }
880 : :
881 : : /* Return true if value can be operated inter-procedurally. */
882 : 9202632761 : bool ipa_p () const
883 : : {
884 : 5119436099 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
885 : : }
886 : :
887 : : /* Return true if quality of profile is precise. */
888 : 55013651 : bool precise_p () const
889 : : {
890 : 55013651 : return m_quality == PRECISE;
891 : : }
892 : :
893 : : /* Get the value of the count. */
894 : 0 : uint64_t value () const { return m_val; }
895 : :
896 : : /* Get the quality of the count. */
897 : 73767215 : enum profile_quality quality () const { return m_quality; }
898 : :
899 : : /* When merging basic blocks, the two different profile counts are unified.
900 : : Return true if this can be done without losing info about profile.
901 : : The only case we care about here is when first BB contains something
902 : : that makes it terminate in a way not visible in CFG. */
903 : 3015484 : bool ok_for_merging (profile_count other) const
904 : : {
905 : 3015484 : if (m_quality < ADJUSTED
906 : 11065 : || other.m_quality < ADJUSTED)
907 : : return true;
908 : 10226 : return !(other < *this);
909 : : }
910 : :
911 : : /* When merging two BBs with different counts, pick common count that looks
912 : : most representative. */
913 : 15854459 : profile_count merge (profile_count other) const
914 : : {
915 : 1457294 : if (*this == other || !other.initialized_p ()
916 : 1457053 : || m_quality > other.m_quality)
917 : 14398386 : return *this;
918 : 1456073 : if (other.m_quality > m_quality
919 : 1456073 : || other > *this)
920 : 599992 : return other;
921 : 856081 : return *this;
922 : : }
923 : :
924 : : /* Basic operations. */
925 : 20225309205 : bool operator== (const profile_count &other) const
926 : : {
927 : 15285487095 : return m_val == other.m_val && m_quality == other.m_quality;
928 : : }
929 : :
930 : 3744430 : profile_count operator+ (const profile_count &other) const
931 : : {
932 : 3744430 : if (other == zero ())
933 : 2339 : return *this;
934 : 3742091 : if (*this == zero ())
935 : 185573 : return other;
936 : 3556518 : if (!initialized_p () || !other.initialized_p ())
937 : 6043 : return uninitialized ();
938 : :
939 : 3550475 : profile_count ret;
940 : 3550475 : gcc_checking_assert (compatible_p (other));
941 : 3550475 : uint64_t ret_val = m_val + other.m_val;
942 : 3550475 : ret.m_val = MIN (ret_val, max_count);
943 : 3550475 : ret.m_quality = MIN (m_quality, other.m_quality);
944 : 3550475 : return ret;
945 : : }
946 : :
947 : 74536449 : profile_count &operator+= (const profile_count &other)
948 : : {
949 : 74536449 : if (other == zero ())
950 : 4717424 : return *this;
951 : 69819025 : if (*this == zero ())
952 : : {
953 : 42810118 : *this = other;
954 : 42810118 : return *this;
955 : : }
956 : 27008907 : if (!initialized_p () || !other.initialized_p ())
957 : 333524 : return *this = uninitialized ();
958 : : else
959 : : {
960 : 26675383 : gcc_checking_assert (compatible_p (other));
961 : 26675383 : uint64_t ret_val = m_val + other.m_val;
962 : 26675383 : m_val = MIN (ret_val, max_count);
963 : 26675383 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 26675383 : return *this;
966 : : }
967 : :
968 : 20040832 : profile_count operator- (const profile_count &other) const
969 : : {
970 : 20056213 : if (*this == zero () || other == zero ())
971 : 640419 : return *this;
972 : 19400413 : if (!initialized_p () || !other.initialized_p ())
973 : 8912904 : return uninitialized ();
974 : 10487509 : gcc_checking_assert (compatible_p (other));
975 : 10487509 : profile_count ret;
976 : 10487509 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
977 : 10487509 : ret.m_quality = MIN (m_quality, other.m_quality);
978 : 10487509 : return ret;
979 : : }
980 : :
981 : 9428608 : profile_count &operator-= (const profile_count &other)
982 : : {
983 : 9874168 : if (*this == zero () || other == zero ())
984 : 318566 : return *this;
985 : 9110042 : if (!initialized_p () || !other.initialized_p ())
986 : 349535 : return *this = uninitialized ();
987 : : else
988 : : {
989 : 8760507 : gcc_checking_assert (compatible_p (other));
990 : 8760507 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
991 : 8760507 : m_quality = MIN (m_quality, other.m_quality);
992 : : }
993 : 8760507 : return *this;
994 : : }
995 : :
996 : : /* Return false if profile_count is bogus. */
997 : 3380862399 : bool verify () const
998 : : {
999 : 3380862399 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1000 : 3380862399 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1001 : : }
1002 : :
1003 : : /* Comparisons are three-state and conservative. False is returned if
1004 : : the inequality cannot be decided. */
1005 : 1107003740 : bool operator< (const profile_count &other) const
1006 : : {
1007 : 1107003740 : if (!initialized_p () || !other.initialized_p ())
1008 : : return false;
1009 : 1102362644 : if (*this == zero ())
1010 : 4014456 : return !(other == zero ());
1011 : 1100355416 : if (other == zero ())
1012 : 1052 : return false;
1013 : 1100354364 : gcc_checking_assert (compatible_p (other));
1014 : 1100354364 : return m_val < other.m_val;
1015 : : }
1016 : :
1017 : 35994007 : bool operator> (const profile_count &other) const
1018 : : {
1019 : 35994007 : if (!initialized_p () || !other.initialized_p ())
1020 : : return false;
1021 : 35035708 : if (*this == zero ())
1022 : 220385 : return false;
1023 : 34815323 : if (other == zero ())
1024 : 1627020 : return !(*this == zero ());
1025 : 34001813 : gcc_checking_assert (compatible_p (other));
1026 : 34001813 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1027 : : }
1028 : :
1029 : : bool operator< (const gcov_type other) const
1030 : : {
1031 : : gcc_checking_assert (ipa_p ());
1032 : : gcc_checking_assert (other >= 0);
1033 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1034 : : }
1035 : :
1036 : 276948 : bool operator> (const gcov_type other) const
1037 : : {
1038 : 276948 : gcc_checking_assert (ipa_p ());
1039 : 276948 : gcc_checking_assert (other >= 0);
1040 : 276948 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1041 : : }
1042 : :
1043 : 1120338 : bool operator<= (const profile_count &other) const
1044 : : {
1045 : 1120338 : if (!initialized_p () || !other.initialized_p ())
1046 : : return false;
1047 : 1120338 : if (*this == zero ())
1048 : 3760 : return true;
1049 : 1116578 : if (other == zero ())
1050 : 0 : return (*this == zero ());
1051 : 1116578 : gcc_checking_assert (compatible_p (other));
1052 : 1116578 : return m_val <= other.m_val;
1053 : : }
1054 : :
1055 : 1402696 : bool operator>= (const profile_count &other) const
1056 : : {
1057 : 1402696 : if (!initialized_p () || !other.initialized_p ())
1058 : : return false;
1059 : 1401612 : if (other == zero ())
1060 : 328 : return true;
1061 : 1401284 : if (*this == zero ())
1062 : 24764 : return (other == zero ());
1063 : 1388902 : gcc_checking_assert (compatible_p (other));
1064 : 1388902 : return m_val >= other.m_val;
1065 : : }
1066 : :
1067 : 96832 : bool operator<= (const gcov_type other) const
1068 : : {
1069 : 96832 : gcc_checking_assert (ipa_p ());
1070 : 96832 : gcc_checking_assert (other >= 0);
1071 : 96832 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1072 : : }
1073 : :
1074 : 59882 : bool operator>= (const gcov_type other) const
1075 : : {
1076 : 59882 : gcc_checking_assert (ipa_p ());
1077 : 59882 : gcc_checking_assert (other >= 0);
1078 : 59882 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1079 : : }
1080 : :
1081 : 928528270 : profile_count operator* (int64_t num) const
1082 : : {
1083 : 928458806 : return apply_scale (num, 1);
1084 : : }
1085 : :
1086 : : profile_count operator*= (int64_t num)
1087 : : {
1088 : : *this = apply_scale (num, 1);
1089 : : return *this;
1090 : : }
1091 : :
1092 : : profile_count operator* (const sreal &num) const;
1093 : : profile_count operator*= (const sreal &num);
1094 : :
1095 : 2344876 : profile_count operator/ (int64_t den) const
1096 : : {
1097 : 2344876 : return apply_scale (1, den);
1098 : : }
1099 : :
1100 : 72 : profile_count operator/= (int64_t den)
1101 : : {
1102 : 64 : *this = apply_scale (1, den);
1103 : 72 : return *this;
1104 : : }
1105 : :
1106 : : /* Return true when value is not zero and can be used for scaling.
1107 : : This is different from *this > 0 because that requires counter to
1108 : : be IPA. */
1109 : 8520456139 : bool nonzero_p () const
1110 : : {
1111 : 8338330462 : return initialized_p () && m_val != 0;
1112 : : }
1113 : :
1114 : : /* Make counter forcibly nonzero. */
1115 : : profile_count force_nonzero () const;
1116 : :
1117 : 94036587 : profile_count max (profile_count other) const
1118 : : {
1119 : 94036587 : profile_count val = *this;
1120 : :
1121 : : /* Always prefer nonzero IPA counts over local counts. */
1122 : 99968952 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1123 : : {
1124 : 9707 : val = ipa ();
1125 : 9707 : other = other.ipa ();
1126 : : }
1127 : 94036587 : if (!initialized_p ())
1128 : 23973228 : return other;
1129 : 70063359 : if (!other.initialized_p ())
1130 : 826246 : return *this;
1131 : 69237113 : if (*this == zero ())
1132 : 1269532 : return other;
1133 : 67967581 : if (other == zero ())
1134 : 3177682 : return *this;
1135 : 64789899 : gcc_checking_assert (compatible_p (other));
1136 : 64789899 : if (val.m_val < other.m_val || (m_val == other.m_val
1137 : 13059668 : && val.m_quality < other.m_quality))
1138 : 10024844 : return other;
1139 : 54765055 : return *this;
1140 : : }
1141 : :
1142 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1143 : : accordingly. */
1144 : : profile_count apply_probability (int prob) const
1145 : : {
1146 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1147 : : if (m_val == 0)
1148 : : return *this;
1149 : : if (!initialized_p ())
1150 : : return uninitialized ();
1151 : : profile_count ret;
1152 : : uint64_t tmp;
1153 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1154 : : ret.m_val = tmp;
1155 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1156 : : return ret;
1157 : : }
1158 : :
1159 : : /* Scale counter according to PROB. */
1160 : 378587365 : profile_count apply_probability (profile_probability prob) const
1161 : : {
1162 : 386300499 : if (*this == zero () || prob == profile_probability::always ())
1163 : 115269279 : return *this;
1164 : 263318086 : if (prob == profile_probability::never ())
1165 : 17306145 : return zero ();
1166 : 246011941 : if (!initialized_p () || !prob.initialized_p ())
1167 : 31284211 : return uninitialized ();
1168 : 214727730 : profile_count ret;
1169 : 214727730 : uint64_t tmp;
1170 : 214727730 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1171 : : &tmp);
1172 : 214727730 : ret.m_val = tmp;
1173 : 214750850 : ret.m_quality = MIN (m_quality, prob.quality ());
1174 : 214727730 : return ret;
1175 : : }
1176 : :
1177 : : /* Return *THIS * NUM / DEN. */
1178 : 1047598565 : profile_count apply_scale (int64_t num, int64_t den) const
1179 : : {
1180 : 1047598565 : if (m_val == 0)
1181 : 25075363 : return *this;
1182 : 1022523202 : if (!initialized_p ())
1183 : 7773 : return uninitialized ();
1184 : 1022515429 : profile_count ret;
1185 : 1022515429 : uint64_t tmp;
1186 : :
1187 : 1022515429 : gcc_checking_assert (num >= 0 && den > 0);
1188 : 1022515429 : safe_scale_64bit (m_val, num, den, &tmp);
1189 : 1022515429 : ret.m_val = MIN (tmp, max_count);
1190 : 1022515429 : ret.m_quality = MIN (m_quality, ADJUSTED);
1191 : 1022515429 : return ret;
1192 : : }
1193 : :
1194 : 28175275 : profile_count apply_scale (profile_count num, profile_count den) const
1195 : : {
1196 : 28175275 : if (*this == zero ())
1197 : 1113320 : return *this;
1198 : 27061955 : if (num == zero ())
1199 : 90534 : return num;
1200 : 26971421 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1201 : 7541236 : return uninitialized ();
1202 : 19430185 : if (num == den)
1203 : 5339084 : return *this;
1204 : 14091101 : gcc_checking_assert (den.m_val);
1205 : :
1206 : 14091101 : profile_count ret;
1207 : 14091101 : uint64_t val;
1208 : 14091101 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1209 : 14091101 : ret.m_val = MIN (val, max_count);
1210 : 14091101 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1211 : : num.m_quality), den.m_quality);
1212 : : /* Be sure that ret is not local if num is global.
1213 : : Also ensure that ret is not global0 when num is global. */
1214 : 14091101 : if (num.ipa_p ())
1215 : : {
1216 : : /* This is common case of AFDO scaling when we upgrade
1217 : : GLOBAL0_AFDO function to AFDO. Be sure that result
1218 : : is AFDO and not GUESSED (which is unnecesarily low). */
1219 : 1800 : if (num.m_quality == AFDO
1220 : 0 : && (ret.m_quality != GUESSED
1221 : 0 : && ret.m_quality != GUESSED_LOCAL))
1222 : 0 : ret.m_quality = AFDO;
1223 : : else
1224 : 3604 : ret.m_quality = MAX (ret.m_quality,
1225 : : num == num.ipa () ? GUESSED : num.m_quality);
1226 : : }
1227 : 14091101 : return ret;
1228 : : }
1229 : :
1230 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1231 : 18399460 : profile_count guessed_local () const
1232 : : {
1233 : 18399460 : profile_count ret = *this;
1234 : 18399460 : if (!initialized_p ())
1235 : 0 : return *this;
1236 : 18399460 : ret.m_quality = GUESSED_LOCAL;
1237 : 18399460 : return ret;
1238 : : }
1239 : :
1240 : : /* We know that profile is globally 0 but keep local profile if present. */
1241 : 795 : profile_count global0 () const
1242 : : {
1243 : 795 : profile_count ret = *this;
1244 : 795 : if (!initialized_p ())
1245 : 0 : return *this;
1246 : 795 : ret.m_quality = GUESSED_GLOBAL0;
1247 : 795 : return ret;
1248 : : }
1249 : :
1250 : : /* We know that profile is globally afdo 0 but keep local profile
1251 : : if present. */
1252 : 0 : profile_count global0afdo () const
1253 : : {
1254 : 0 : profile_count ret = *this;
1255 : 0 : if (!initialized_p ())
1256 : 0 : return *this;
1257 : 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1258 : 0 : return ret;
1259 : : }
1260 : :
1261 : : /* We know that profile is globally adjusted 0 but keep local profile
1262 : : if present. */
1263 : 37 : profile_count global0adjusted () const
1264 : : {
1265 : 37 : profile_count ret = *this;
1266 : 37 : if (!initialized_p ())
1267 : 0 : return *this;
1268 : 37 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1269 : 37 : return ret;
1270 : : }
1271 : :
1272 : : /* Return THIS with quality dropped to GUESSED. */
1273 : 331 : profile_count guessed () const
1274 : : {
1275 : 331 : profile_count ret = *this;
1276 : 331 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1277 : 331 : return ret;
1278 : : }
1279 : :
1280 : : /* Return THIS with quality GUESSED. */
1281 : 0 : profile_count force_guessed () const
1282 : : {
1283 : 0 : profile_count ret = *this;
1284 : 0 : gcc_checking_assert (initialized_p ());
1285 : 0 : ret.m_quality = GUESSED;
1286 : 0 : return ret;
1287 : : }
1288 : :
1289 : : /* Return variant of profile count which is always safe to compare
1290 : : across functions. */
1291 : 9444037259 : profile_count ipa () const
1292 : : {
1293 : 9444037259 : if (m_quality > GUESSED_GLOBAL0)
1294 : 17029882 : return *this;
1295 : 9427007377 : if (m_quality == GUESSED_GLOBAL0)
1296 : 221927 : return zero ();
1297 : 9426785450 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1298 : 10698 : return adjusted_zero ();
1299 : 9426774752 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1300 : 0 : return afdo_zero ();
1301 : 9426774752 : return uninitialized ();
1302 : : }
1303 : :
1304 : : /* Return THIS with quality dropped to AFDO. */
1305 : 0 : profile_count afdo () const
1306 : : {
1307 : 0 : profile_count ret = *this;
1308 : 0 : ret.m_quality = AFDO;
1309 : 0 : return ret;
1310 : : }
1311 : :
1312 : : /* Return probability of event with counter THIS within event with counter
1313 : : OVERALL. */
1314 : 1090138053 : profile_probability probability_in (const profile_count overall) const
1315 : : {
1316 : 1090152093 : if (*this == zero ()
1317 : 14040 : && !(overall == zero ()))
1318 : 1110 : return profile_probability::never ();
1319 : 1090136376 : if (!initialized_p () || !overall.initialized_p ()
1320 : 2180273319 : || !overall.m_val)
1321 : 59928 : return profile_probability::uninitialized ();
1322 : 1295549109 : if (*this == overall && m_quality == PRECISE)
1323 : 30056 : return profile_probability::always ();
1324 : 1090046959 : profile_probability ret;
1325 : 1090046959 : gcc_checking_assert (compatible_p (overall));
1326 : :
1327 : 1090046959 : if (overall.m_val < m_val)
1328 : : {
1329 : 94820 : ret.m_val = profile_probability::max_probability;
1330 : 94820 : ret.set_quality (GUESSED);
1331 : 94820 : return ret;
1332 : : }
1333 : : else
1334 : 1089952139 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1335 : : overall.m_val);
1336 : 1089952139 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1337 : : GUESSED), ADJUSTED));
1338 : 1089952139 : return ret;
1339 : : }
1340 : :
1341 : : /* Return true if profile count is very large, so we risk overflows
1342 : : with loop transformations. */
1343 : : bool
1344 : 1037399 : very_large_p ()
1345 : : {
1346 : 1037399 : if (!initialized_p ())
1347 : : return false;
1348 : 1037399 : return m_val > max_count / 65536;
1349 : : }
1350 : :
1351 : : int to_frequency (struct function *fun) const;
1352 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1353 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1354 : :
1355 : : /* Output THIS to F. */
1356 : : void dump (FILE *f, struct function *fun = NULL) const;
1357 : :
1358 : : /* Print THIS to stderr. */
1359 : : void debug () const;
1360 : :
1361 : : /* Return true if THIS is known to differ significantly from OTHER. */
1362 : : bool differs_from_p (profile_count other) const;
1363 : :
1364 : : /* We want to scale profile across function boundary from NUM to DEN.
1365 : : Take care of the side case when NUM and DEN are zeros of incompatible
1366 : : kinds. */
1367 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1368 : :
1369 : : /* THIS is a count of bb which is known to be executed IPA times.
1370 : : Combine this information into bb counter. This means returning IPA
1371 : : if it is nonzero, not changing anything if IPA is uninitialized
1372 : : and if IPA is zero, turning THIS into corresponding local profile with
1373 : : global0. */
1374 : : profile_count combine_with_ipa_count (profile_count ipa);
1375 : :
1376 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1377 : : profile_count combine_with_ipa_count_within
1378 : : (profile_count ipa, profile_count ipa2);
1379 : :
1380 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1381 : : Conversions back and forth are used to read the coverage and get it
1382 : : into internal representation. */
1383 : : static profile_count from_gcov_type (gcov_type v,
1384 : : profile_quality quality = PRECISE);
1385 : :
1386 : : /* LTO streaming support. */
1387 : : static profile_count stream_in (class lto_input_block *);
1388 : : void stream_out (struct output_block *);
1389 : : void stream_out (struct lto_output_stream *);
1390 : : };
1391 : : #endif
|
