Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2024 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
41 : : better. This is the case of functions not run in profile feedback.
42 : : Never used by probabilities. */
43 : : GUESSED_GLOBAL0,
44 : :
45 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
46 : : GUESSED_GLOBAL0_ADJUSTED,
47 : :
48 : : /* Profile is based on static branch prediction heuristics. It may or may
49 : : not reflect the reality but it can be compared interprocedurally
50 : : (for example, we inlined function w/o profile feedback into function
51 : : with feedback and propagated from that).
52 : : Never used by probabilities. */
53 : : GUESSED,
54 : :
55 : : /* Profile was determined by autofdo. */
56 : : AFDO,
57 : :
58 : : /* Profile was originally based on feedback but it was adjusted
59 : : by code duplicating optimization. It may not precisely reflect the
60 : : particular code path. */
61 : : ADJUSTED,
62 : :
63 : : /* Profile was read from profile feedback or determined by accurate static
64 : : method. */
65 : : PRECISE
66 : : };
67 : :
68 : : extern const char *profile_quality_as_string (enum profile_quality);
69 : : extern bool parse_profile_quality (const char *value,
70 : : profile_quality *quality);
71 : :
72 : : /* The base value for branch probability notes and edge probabilities. */
73 : : #define REG_BR_PROB_BASE 10000
74 : :
75 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
76 : :
77 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
78 : :
79 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
80 : :
81 : : inline bool
82 : 1095104702 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
83 : : {
84 : : #if (GCC_VERSION >= 5000)
85 : 1095104702 : uint64_t tmp;
86 : 1095104702 : if (!__builtin_mul_overflow (a, b, &tmp)
87 : 1095104702 : && !__builtin_add_overflow (tmp, c/2, &tmp))
88 : : {
89 : 1094538711 : *res = tmp / c;
90 : 1094538711 : return true;
91 : : }
92 : 565991 : if (c == 1)
93 : : {
94 : 29607 : *res = (uint64_t) -1;
95 : 29607 : return false;
96 : : }
97 : : #else
98 : : if (a < ((uint64_t)1 << 31)
99 : : && b < ((uint64_t)1 << 31)
100 : : && c < ((uint64_t)1 << 31))
101 : : {
102 : : *res = (a * b + (c / 2)) / c;
103 : : return true;
104 : : }
105 : : #endif
106 : 536384 : return slow_safe_scale_64bit (a, b, c, res);
107 : : }
108 : :
109 : : /* Data type to hold probabilities. It implements fixed point arithmetics
110 : : with capping so probability is always in range [0,1] and scaling requiring
111 : : values greater than 1 needs to be represented otherwise.
112 : :
113 : : In addition to actual value the quality of profile is tracked and propagated
114 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
115 : : that has not been determined yet (for example because of
116 : : -fno-guess-branch-probability)
117 : :
118 : : Typically probabilities are derived from profile feedback (via
119 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
120 : : thorough the compilation.
121 : :
122 : : Named probabilities are available:
123 : : - never (0 probability)
124 : : - guessed_never
125 : : - very_unlikely (1/2000 probability)
126 : : - unlikely (1/5 probability)
127 : : - even (1/2 probability)
128 : : - likely (4/5 probability)
129 : : - very_likely (1999/2000 probability)
130 : : - guessed_always
131 : : - always
132 : :
133 : : Named probabilities except for never/always are assumed to be statically
134 : : guessed and thus not necessarily accurate. The difference between never
135 : : and guessed_never is that the first one should be used only in case that
136 : : well behaving program will very likely not execute the "never" path.
137 : : For example if the path is going to abort () call or it exception handling.
138 : :
139 : : Always and guessed_always probabilities are symmetric.
140 : :
141 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
142 : : integer arithmetics. Once the code is converted to branch probabilities,
143 : : these conversions will probably go away because they are lossy.
144 : : */
145 : :
146 : : class GTY((user)) profile_probability
147 : : {
148 : : static const int n_bits = 29;
149 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
150 : : will lead to harder multiplication sequences. */
151 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
152 : : static const uint32_t uninitialized_probability
153 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
154 : :
155 : : uint32_t m_val : 29;
156 : : enum profile_quality m_quality : 3;
157 : :
158 : : friend struct profile_count;
159 : : public:
160 : 3768941204 : profile_probability (): m_val (uninitialized_probability),
161 : 2017817561 : m_quality (GUESSED)
162 : : {}
163 : :
164 : 112 : profile_probability (uint32_t val, profile_quality quality):
165 : 112 : m_val (val), m_quality (quality)
166 : : {}
167 : :
168 : : /* Named probabilities. */
169 : 614436882 : static profile_probability never ()
170 : : {
171 : 614436882 : profile_probability ret;
172 : 614436882 : ret.m_val = 0;
173 : 614436882 : ret.m_quality = PRECISE;
174 : 160629829 : return ret;
175 : : }
176 : :
177 : 4843856 : static profile_probability guessed_never ()
178 : : {
179 : 4843856 : profile_probability ret;
180 : 4843856 : ret.m_val = 0;
181 : 4843856 : ret.m_quality = GUESSED;
182 : 4843856 : return ret;
183 : : }
184 : :
185 : 4680143 : static profile_probability very_unlikely ()
186 : : {
187 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
188 : 9360286 : profile_probability r = guessed_always () / 2000;
189 : 4680143 : r.m_val--;
190 : 4680143 : return r;
191 : : }
192 : :
193 : 76934 : static profile_probability unlikely ()
194 : : {
195 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
196 : 153868 : profile_probability r = guessed_always () / 5;
197 : 76934 : r.m_val--;
198 : 76934 : return r;
199 : : }
200 : :
201 : 729345 : static profile_probability even ()
202 : : {
203 : 1458690 : return guessed_always () / 2;
204 : : }
205 : :
206 : 337354 : static profile_probability very_likely ()
207 : : {
208 : 337354 : return always () - very_unlikely ();
209 : : }
210 : :
211 : 29806 : static profile_probability likely ()
212 : : {
213 : 29806 : return always () - unlikely ();
214 : : }
215 : : /* Return true when value is not zero and can be used for scaling. */
216 : 1279 : bool nonzero_p () const
217 : : {
218 : 1279 : return initialized_p () && m_val != 0;
219 : : }
220 : :
221 : 5785733 : static profile_probability guessed_always ()
222 : : {
223 : 5785733 : profile_probability ret;
224 : 5785733 : ret.m_val = max_probability;
225 : 5785733 : ret.m_quality = GUESSED;
226 : 5632393 : return ret;
227 : : }
228 : :
229 : 491190640 : static profile_probability always ()
230 : : {
231 : 491190640 : profile_probability ret;
232 : 491190640 : ret.m_val = max_probability;
233 : 491190640 : ret.m_quality = PRECISE;
234 : 321402743 : return ret;
235 : : }
236 : :
237 : : /* Probabilities which has not been initialized. Either because
238 : : initialization did not happen yet or because profile is unknown. */
239 : 816905982 : static profile_probability uninitialized ()
240 : : {
241 : 816905982 : profile_probability c;
242 : 816905982 : c.m_val = uninitialized_probability;
243 : 816905982 : c.m_quality = GUESSED;
244 : 798881431 : return c;
245 : : }
246 : :
247 : : /* Return true if value has been initialized. */
248 : 5281367419 : bool initialized_p () const
249 : : {
250 : 3619988734 : return m_val != uninitialized_probability;
251 : : }
252 : :
253 : : /* Return true if value can be trusted. */
254 : 400 : bool reliable_p () const
255 : : {
256 : 400 : return m_quality >= ADJUSTED;
257 : : }
258 : :
259 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
260 : : this is mostly to support legacy code and should go away. */
261 : 2932738 : static profile_probability from_reg_br_prob_base (int v)
262 : : {
263 : 2932738 : profile_probability ret;
264 : 2932738 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
265 : 2932738 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
266 : 2932738 : ret.m_quality = GUESSED;
267 : 2932738 : return ret;
268 : : }
269 : :
270 : : /* Return THIS with quality set to ADJUSTED. */
271 : 35 : profile_probability adjusted () const
272 : : {
273 : 35 : profile_probability ret = *this;
274 : 35 : if (!initialized_p ())
275 : 0 : return *this;
276 : 35 : ret.m_quality = ADJUSTED;
277 : 35 : return ret;
278 : : }
279 : :
280 : 1070640021 : int to_reg_br_prob_base () const
281 : : {
282 : 1070640021 : gcc_checking_assert (initialized_p ());
283 : 1070640021 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
284 : : }
285 : :
286 : : /* Conversion to and from RTL representation of profile probabilities. */
287 : 436729760 : static profile_probability from_reg_br_prob_note (int v)
288 : : {
289 : 436729760 : profile_probability ret;
290 : 436729760 : ret.m_val = ((unsigned int)v) / 8;
291 : 436729760 : ret.m_quality = (enum profile_quality)(v & 7);
292 : 14516855 : return ret;
293 : : }
294 : :
295 : 409981063 : int to_reg_br_prob_note () const
296 : : {
297 : 409981063 : gcc_checking_assert (initialized_p ());
298 : 409981063 : int ret = m_val * 8 + m_quality;
299 : 819962126 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
300 : 409981063 : return ret;
301 : : }
302 : :
303 : : /* Return VAL1/VAL2. */
304 : 3050 : static profile_probability probability_in_gcov_type
305 : : (gcov_type val1, gcov_type val2)
306 : : {
307 : 3050 : profile_probability ret;
308 : 3050 : gcc_checking_assert (val1 >= 0 && val2 > 0);
309 : 3050 : if (val1 > val2)
310 : : ret.m_val = max_probability;
311 : : else
312 : : {
313 : 3050 : uint64_t tmp;
314 : 3050 : safe_scale_64bit (val1, max_probability, val2, &tmp);
315 : 3050 : gcc_checking_assert (tmp <= max_probability);
316 : 3050 : ret.m_val = tmp;
317 : : }
318 : 3050 : ret.m_quality = PRECISE;
319 : 3050 : return ret;
320 : : }
321 : :
322 : : /* Basic operations. */
323 : 1391983783 : bool operator== (const profile_probability &other) const
324 : : {
325 : 829089918 : return m_val == other.m_val && m_quality == other.m_quality;
326 : : }
327 : :
328 : 6105968 : profile_probability operator+ (const profile_probability &other) const
329 : : {
330 : 6105968 : if (other == never ())
331 : 30490 : return *this;
332 : 6075478 : if (*this == never ())
333 : 22679 : return other;
334 : 6052799 : if (!initialized_p () || !other.initialized_p ())
335 : 4991043 : return uninitialized ();
336 : :
337 : 1061756 : profile_probability ret;
338 : 1061756 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
339 : 1061756 : ret.m_quality = MIN (m_quality, other.m_quality);
340 : 1061756 : return ret;
341 : : }
342 : :
343 : 5722173 : profile_probability &operator+= (const profile_probability &other)
344 : : {
345 : 5722173 : if (other == never ())
346 : 1026309 : return *this;
347 : 4695864 : if (*this == never ())
348 : : {
349 : 783750 : *this = other;
350 : 783750 : return *this;
351 : : }
352 : 3912114 : if (!initialized_p () || !other.initialized_p ())
353 : 2814538 : return *this = uninitialized ();
354 : : else
355 : : {
356 : 1097576 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
357 : 1097576 : m_quality = MIN (m_quality, other.m_quality);
358 : : }
359 : 1097576 : return *this;
360 : : }
361 : :
362 : 31504538 : profile_probability operator- (const profile_probability &other) const
363 : : {
364 : 31504538 : if (*this == never ()
365 : 31405270 : || other == never ())
366 : 1713890 : return *this;
367 : 29790648 : if (!initialized_p () || !other.initialized_p ())
368 : 2109961 : return uninitialized ();
369 : 27680687 : profile_probability ret;
370 : 27680687 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
371 : 27680687 : ret.m_quality = MIN (m_quality, other.m_quality);
372 : 27680687 : return ret;
373 : : }
374 : :
375 : 10898163 : profile_probability &operator-= (const profile_probability &other)
376 : : {
377 : 10898163 : if (*this == never ()
378 : 10781152 : || other == never ())
379 : 2107475 : return *this;
380 : 8790688 : if (!initialized_p () || !other.initialized_p ())
381 : 2103980 : return *this = uninitialized ();
382 : : else
383 : : {
384 : 6686708 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
385 : 6686708 : m_quality = MIN (m_quality, other.m_quality);
386 : : }
387 : 6686708 : return *this;
388 : : }
389 : :
390 : 1111385 : profile_probability operator* (const profile_probability &other) const
391 : : {
392 : 1111385 : if (*this == never ()
393 : 1008480 : || other == never ())
394 : 110713 : return never ();
395 : 1000672 : if (!initialized_p () || !other.initialized_p ())
396 : 57741 : return uninitialized ();
397 : 942931 : profile_probability ret;
398 : 942931 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
399 : 942931 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
400 : 942931 : return ret;
401 : : }
402 : :
403 : 129088 : profile_probability &operator*= (const profile_probability &other)
404 : : {
405 : 129088 : if (*this == never ()
406 : 129072 : || other == never ())
407 : 82 : return *this = never ();
408 : 129006 : if (!initialized_p () || !other.initialized_p ())
409 : 9 : return *this = uninitialized ();
410 : : else
411 : : {
412 : 128997 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
413 : 128997 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
414 : : }
415 : 128997 : return *this;
416 : : }
417 : :
418 : 1580725 : profile_probability operator/ (const profile_probability &other) const
419 : : {
420 : 1580725 : if (*this == never ())
421 : 105968 : return never ();
422 : 1474757 : if (!initialized_p () || !other.initialized_p ())
423 : 221397 : return uninitialized ();
424 : 1253360 : profile_probability ret;
425 : : /* If we get probability above 1, mark it as unreliable and return 1. */
426 : 1253360 : if (m_val >= other.m_val)
427 : : {
428 : 30332 : ret.m_val = max_probability;
429 : 30332 : ret.m_quality = MIN (MIN (m_quality, other.m_quality),
430 : : GUESSED);
431 : 30332 : return ret;
432 : : }
433 : 1223028 : else if (!m_val)
434 : : ret.m_val = 0;
435 : : else
436 : : {
437 : 1164991 : gcc_checking_assert (other.m_val);
438 : 1164991 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
439 : : other.m_val),
440 : : max_probability);
441 : : }
442 : 1223028 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
443 : 1223028 : return ret;
444 : : }
445 : :
446 : 297760 : profile_probability &operator/= (const profile_probability &other)
447 : : {
448 : 297760 : if (*this == never ())
449 : 1878 : return *this = never ();
450 : 295882 : if (!initialized_p () || !other.initialized_p ())
451 : 0 : return *this = uninitialized ();
452 : : else
453 : : {
454 : : /* If we get probability above 1, mark it as unreliable
455 : : and return 1. */
456 : 295882 : if (m_val > other.m_val)
457 : : {
458 : 570 : m_val = max_probability;
459 : 570 : m_quality = MIN (MIN (m_quality, other.m_quality),
460 : : GUESSED);
461 : 570 : return *this;
462 : : }
463 : 295312 : else if (!m_val)
464 : : ;
465 : : else
466 : : {
467 : 293673 : gcc_checking_assert (other.m_val);
468 : 293673 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
469 : : other.m_val),
470 : : max_probability);
471 : : }
472 : 295312 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
473 : : }
474 : 295312 : return *this;
475 : : }
476 : :
477 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
478 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
479 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
480 : : == ORIG. This is useful e.g. when splitting a conditional
481 : : branch like:
482 : : if (cond)
483 : : goto lab; // ORIG probability
484 : : into
485 : : if (cond1)
486 : : goto lab; // FIRST = ORIG * CPROB probability
487 : : if (cond2)
488 : : goto lab; // SECOND probability
489 : : such that the overall probability of jumping to lab remains
490 : : the same. CPROB gives the relative probability between the
491 : : branches. */
492 : 284300 : profile_probability split (const profile_probability &cprob)
493 : : {
494 : 284300 : profile_probability ret = *this * cprob;
495 : : /* The following is equivalent to:
496 : : *this = cprob.invert () * *this / ret.invert ();
497 : : Avoid scaling when overall outcome is supposed to be always.
498 : : Without knowing that one is inverse of other, the result would be
499 : : conservative. */
500 : 284300 : if (!(*this == always ()))
501 : 275701 : *this = (*this - ret) / ret.invert ();
502 : 284300 : return ret;
503 : : }
504 : :
505 : 243491 : gcov_type apply (gcov_type val) const
506 : : {
507 : 243491 : if (*this == uninitialized ())
508 : 62 : return val / 2;
509 : 243429 : return RDIV (val * m_val, max_probability);
510 : : }
511 : :
512 : : /* Return 1-*THIS. */
513 : 28589999 : profile_probability invert () const
514 : : {
515 : 28582372 : return always() - *this;
516 : : }
517 : :
518 : : /* Return THIS with quality dropped to GUESSED. */
519 : 196298 : profile_probability guessed () const
520 : : {
521 : 196298 : profile_probability ret = *this;
522 : 196298 : ret.m_quality = GUESSED;
523 : 196298 : return ret;
524 : : }
525 : :
526 : : /* Return THIS with quality dropped to AFDO. */
527 : : profile_probability afdo () const
528 : : {
529 : : profile_probability ret = *this;
530 : : ret.m_quality = AFDO;
531 : : return ret;
532 : : }
533 : :
534 : : /* Return *THIS * NUM / DEN. */
535 : 13269364 : profile_probability apply_scale (int64_t num, int64_t den) const
536 : : {
537 : 13269364 : if (*this == never ())
538 : 25742 : return *this;
539 : 13243622 : if (!initialized_p ())
540 : 5378725 : return uninitialized ();
541 : 7864897 : profile_probability ret;
542 : 7864897 : uint64_t tmp;
543 : 7864897 : safe_scale_64bit (m_val, num, den, &tmp);
544 : 7864897 : ret.m_val = MIN (tmp, max_probability);
545 : 7864897 : ret.m_quality = MIN (m_quality, ADJUSTED);
546 : 7864897 : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 6572 : profile_probability apply_scale (profile_probability num,
551 : : profile_probability den) const
552 : : {
553 : 6572 : if (*this == never ())
554 : 31 : return *this;
555 : 6541 : if (num == never ())
556 : 0 : return num;
557 : 6541 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
558 : 0 : return uninitialized ();
559 : 6541 : if (num == den)
560 : 0 : return *this;
561 : 6541 : gcc_checking_assert (den.m_val);
562 : :
563 : 6541 : profile_probability ret;
564 : 6541 : uint64_t val;
565 : 6541 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
566 : 6541 : ret.m_val = MIN (val, max_probability);
567 : 6541 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
568 : : num.m_quality), den.m_quality);
569 : 6541 : return ret;
570 : : }
571 : :
572 : : /* Return true when the probability of edge is reliable.
573 : :
574 : : The profile guessing code is good at predicting branch outcome (i.e.
575 : : taken/not taken), that is predicted right slightly over 75% of time.
576 : : It is however notoriously poor on predicting the probability itself.
577 : : In general the profile appear a lot flatter (with probabilities closer
578 : : to 50%) than the reality so it is bad idea to use it to drive optimization
579 : : such as those disabling dynamic branch prediction for well predictable
580 : : branches.
581 : :
582 : : There are two exceptions - edges leading to noreturn edges and edges
583 : : predicted by number of iterations heuristics are predicted well. This macro
584 : : should be able to distinguish those, but at the moment it simply check for
585 : : noreturn heuristic that is only one giving probability over 99% or bellow
586 : : 1%. In future we might want to propagate reliability information across the
587 : : CFG if we find this information useful on multiple places. */
588 : 0 : bool probably_reliable_p () const
589 : : {
590 : 0 : if (m_quality >= ADJUSTED)
591 : : return true;
592 : 0 : if (!initialized_p ())
593 : : return false;
594 : 0 : return m_val < max_probability / 100
595 : 0 : || m_val > max_probability - max_probability / 100;
596 : : }
597 : :
598 : : /* Return false if profile_probability is bogus. */
599 : 4022430653 : bool verify () const
600 : : {
601 : 4022430653 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
602 : 4022430653 : if (m_val == uninitialized_probability)
603 : 898528843 : return m_quality == GUESSED;
604 : 3123901810 : else if (m_quality < GUESSED)
605 : : return false;
606 : 3123901810 : return m_val <= max_probability;
607 : : }
608 : :
609 : : /* Comparisons are three-state and conservative. False is returned if
610 : : the inequality cannot be decided. */
611 : 5706772 : bool operator< (const profile_probability &other) const
612 : : {
613 : 5706772 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
614 : : }
615 : :
616 : 16248847 : bool operator> (const profile_probability &other) const
617 : : {
618 : 16248847 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
619 : : }
620 : :
621 : 4748633 : bool operator<= (const profile_probability &other) const
622 : : {
623 : 4748633 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
624 : : }
625 : :
626 : 908809 : bool operator>= (const profile_probability &other) const
627 : : {
628 : 908809 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
629 : : }
630 : :
631 : 17 : profile_probability operator* (int64_t num) const
632 : : {
633 : 17 : return apply_scale (num, 1);
634 : : }
635 : :
636 : : profile_probability operator*= (int64_t num)
637 : : {
638 : : *this = apply_scale (num, 1);
639 : : return *this;
640 : : }
641 : :
642 : 12663228 : profile_probability operator/ (int64_t den) const
643 : : {
644 : 12663228 : return apply_scale (1, den);
645 : : }
646 : :
647 : 243761 : profile_probability operator/= (int64_t den)
648 : : {
649 : 243761 : *this = apply_scale (1, den);
650 : 243761 : return *this;
651 : : }
652 : :
653 : : /* Compute n-th power. */
654 : : profile_probability pow (int) const;
655 : :
656 : : /* Compute sware root. */
657 : : profile_probability sqrt () const;
658 : :
659 : : /* Get the value of the count. */
660 : 0 : uint32_t value () const { return m_val; }
661 : :
662 : : /* Get the quality of the count. */
663 : 0 : enum profile_quality quality () const { return m_quality; }
664 : :
665 : : /* Output THIS to F. */
666 : : void dump (FILE *f) const;
667 : :
668 : : /* Output THIS to BUFFER. */
669 : : void dump (char *buffer) const;
670 : :
671 : : /* Print THIS to stderr. */
672 : : void debug () const;
673 : :
674 : : /* Return true if THIS is known to differ significantly from OTHER. */
675 : : bool differs_from_p (profile_probability other) const;
676 : :
677 : : /* Return if difference is greater than 50%. */
678 : : bool differs_lot_from_p (profile_probability other) const;
679 : :
680 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
681 : : happens with COUNT2 probability. Return probability that either *THIS or
682 : : OTHER happens. */
683 : : profile_probability combine_with_count (profile_count count1,
684 : : profile_probability other,
685 : : profile_count count2) const;
686 : :
687 : : /* Return probability as sreal. */
688 : : sreal to_sreal () const;
689 : : /* LTO streaming support. */
690 : : static profile_probability stream_in (class lto_input_block *);
691 : : void stream_out (struct output_block *);
692 : : void stream_out (struct lto_output_stream *);
693 : : };
694 : :
695 : : /* Main data type to hold profile counters in GCC. Profile counts originate
696 : : either from profile feedback, static profile estimation or both. We do not
697 : : perform whole program profile propagation and thus profile estimation
698 : : counters are often local to function, while counters from profile feedback
699 : : (or special cases of profile estimation) can be used inter-procedurally.
700 : :
701 : : There are 3 basic types
702 : : 1) local counters which are result of intra-procedural static profile
703 : : estimation.
704 : : 2) ipa counters which are result of profile feedback or special case
705 : : of static profile estimation (such as in function main).
706 : : 3) counters which counts as 0 inter-procedurally (because given function
707 : : was never run in train feedback) but they hold local static profile
708 : : estimate.
709 : :
710 : : Counters of type 1 and 3 cannot be mixed with counters of different type
711 : : within operation (because whole function should use one type of counter)
712 : : with exception that global zero mix in most operations where outcome is
713 : : well defined.
714 : :
715 : : To take local counter and use it inter-procedurally use ipa member function
716 : : which strips information irrelevant at the inter-procedural level.
717 : :
718 : : Counters are 61bit integers representing number of executions during the
719 : : train run or normalized frequency within the function.
720 : :
721 : : As the profile is maintained during the compilation, many adjustments are
722 : : made. Not all transformations can be made precisely, most importantly
723 : : when code is being duplicated. It also may happen that part of CFG has
724 : : profile counts known while other do not - for example when LTO optimizing
725 : : partly profiled program or when profile was lost due to COMDAT merging.
726 : :
727 : : For this reason profile_count tracks more information than
728 : : just unsigned integer and it is also ready for profile mismatches.
729 : : The API of this data type represent operations that are natural
730 : : on profile counts - sum, difference and operation with scales and
731 : : probabilities. All operations are safe by never getting negative counts
732 : : and they do end up in uninitialized scale if any of the parameters is
733 : : uninitialized.
734 : :
735 : : All comparisons that are three state and handling of probabilities. Thus
736 : : a < b is not equal to !(a >= b).
737 : :
738 : : The following pre-defined counts are available:
739 : :
740 : : profile_count::zero () for code that is known to execute zero times at
741 : : runtime (this can be detected statically i.e. for paths leading to
742 : : abort ();
743 : : profile_count::one () for code that is known to execute once (such as
744 : : main () function
745 : : profile_count::uninitialized () for unknown execution count.
746 : :
747 : : */
748 : :
749 : : struct GTY(()) profile_count
750 : : {
751 : : public:
752 : : /* Use 62bit to hold basic block counters. Should be at least
753 : : 64bit. Although a counter cannot be negative, we use a signed
754 : : type to hold various extra stages. */
755 : :
756 : : static const int n_bits = 61;
757 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
758 : : private:
759 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
760 : :
761 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
762 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
763 : : incorrectly detects the alignment of a structure where the only
764 : : 64-bit aligned object is a bit-field. We force the alignment of
765 : : the entire field to mitigate this. */
766 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
767 : : #else
768 : : #define UINT64_BIT_FIELD_ALIGN
769 : : #endif
770 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
771 : : #undef UINT64_BIT_FIELD_ALIGN
772 : : enum profile_quality m_quality : 3;
773 : : public:
774 : :
775 : : /* Return true if both values can meaningfully appear in single function
776 : : body. We have either all counters in function local or global, otherwise
777 : : operations between them are not really defined well. */
778 : 2457460461 : bool compatible_p (const profile_count other) const
779 : : {
780 : 4870806700 : if (!initialized_p () || !other.initialized_p ())
781 : : return true;
782 : 9641894703 : if (*this == zero ()
783 : 4816223883 : || other == zero ())
784 : 9447865 : return true;
785 : : /* Do not allow nonzero global profile together with local guesses
786 : : that are globally0. */
787 : 2403387545 : if (ipa ().nonzero_p ()
788 : 90612 : && !(other.ipa () == other))
789 : 0 : return false;
790 : 2403387545 : if (other.ipa ().nonzero_p ()
791 : 94094 : && !(ipa () == *this))
792 : 0 : return false;
793 : :
794 : 2403387545 : return ipa_p () == other.ipa_p ();
795 : : }
796 : :
797 : : /* Used for counters which are expected to be never executed. */
798 : 14112645082 : static profile_count zero ()
799 : : {
800 : 10374579216 : return from_gcov_type (0);
801 : : }
802 : :
803 : 3710 : static profile_count adjusted_zero ()
804 : : {
805 : 3710 : profile_count c;
806 : 3710 : c.m_val = 0;
807 : 3710 : c.m_quality = ADJUSTED;
808 : 3710 : return c;
809 : : }
810 : :
811 : : static profile_count guessed_zero ()
812 : : {
813 : : profile_count c;
814 : 0 : c.m_val = 0;
815 : 0 : c.m_quality = GUESSED;
816 : : return c;
817 : : }
818 : :
819 : 548 : static profile_count one ()
820 : : {
821 : 274 : return from_gcov_type (1);
822 : : }
823 : :
824 : : /* Value of counters which has not been initialized. Either because
825 : : initialization did not happen yet or because profile is unknown. */
826 : 6049840061 : static profile_count uninitialized ()
827 : : {
828 : 6049840061 : profile_count c;
829 : 6159453635 : c.m_val = uninitialized_count;
830 : 3807423415 : c.m_quality = GUESSED_LOCAL;
831 : 6049840061 : return c;
832 : : }
833 : :
834 : : /* Conversion to gcov_type is lossy. */
835 : 292274 : gcov_type to_gcov_type () const
836 : : {
837 : 0 : gcc_checking_assert (initialized_p ());
838 : 292274 : return m_val;
839 : : }
840 : :
841 : : /* Return true if value has been initialized. */
842 : 22821479074 : bool initialized_p () const
843 : : {
844 : 7927685728 : return m_val != uninitialized_count;
845 : : }
846 : :
847 : : /* Return true if value can be trusted. */
848 : 7977169 : bool reliable_p () const
849 : : {
850 : 7976912 : return m_quality >= ADJUSTED;
851 : : }
852 : :
853 : : /* Return true if value can be operated inter-procedurally. */
854 : 5729654009 : bool ipa_p () const
855 : : {
856 : 3315605853 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
857 : : }
858 : :
859 : : /* Return true if quality of profile is precise. */
860 : 47174817 : bool precise_p () const
861 : : {
862 : 47174817 : return m_quality == PRECISE;
863 : : }
864 : :
865 : : /* Get the value of the count. */
866 : 0 : uint64_t value () const { return m_val; }
867 : :
868 : : /* Get the quality of the count. */
869 : 49829 : enum profile_quality quality () const { return m_quality; }
870 : :
871 : : /* When merging basic blocks, the two different profile counts are unified.
872 : : Return true if this can be done without losing info about profile.
873 : : The only case we care about here is when first BB contains something
874 : : that makes it terminate in a way not visible in CFG. */
875 : 2699079 : bool ok_for_merging (profile_count other) const
876 : : {
877 : 2699079 : if (m_quality < ADJUSTED
878 : 9735 : || other.m_quality < ADJUSTED)
879 : : return true;
880 : 9000 : return !(other < *this);
881 : : }
882 : :
883 : : /* When merging two BBs with different counts, pick common count that looks
884 : : most representative. */
885 : 13526102 : profile_count merge (profile_count other) const
886 : : {
887 : 967456 : if (*this == other || !other.initialized_p ()
888 : 967227 : || m_quality > other.m_quality)
889 : 12560298 : return *this;
890 : 965804 : if (other.m_quality > m_quality
891 : 965804 : || other > *this)
892 : 393643 : return other;
893 : 572161 : return *this;
894 : : }
895 : :
896 : : /* Basic operations. */
897 : 15488704268 : bool operator== (const profile_count &other) const
898 : : {
899 : 11039149219 : return m_val == other.m_val && m_quality == other.m_quality;
900 : : }
901 : :
902 : 3370414 : profile_count operator+ (const profile_count &other) const
903 : : {
904 : 3370414 : if (other == zero ())
905 : 2100 : return *this;
906 : 3368314 : if (*this == zero ())
907 : 173227 : return other;
908 : 3195087 : if (!initialized_p () || !other.initialized_p ())
909 : 5814 : return uninitialized ();
910 : :
911 : 3189273 : profile_count ret;
912 : 3189273 : gcc_checking_assert (compatible_p (other));
913 : 3189273 : uint64_t ret_val = m_val + other.m_val;
914 : 3189273 : ret.m_val = MIN (ret_val, max_count);
915 : 3189273 : ret.m_quality = MIN (m_quality, other.m_quality);
916 : 3189273 : return ret;
917 : : }
918 : :
919 : 66901117 : profile_count &operator+= (const profile_count &other)
920 : : {
921 : 66901117 : if (other == zero ())
922 : 4637646 : return *this;
923 : 62263471 : if (*this == zero ())
924 : : {
925 : 38358309 : *this = other;
926 : 38358309 : return *this;
927 : : }
928 : 23905162 : if (!initialized_p () || !other.initialized_p ())
929 : 326461 : return *this = uninitialized ();
930 : : else
931 : : {
932 : 23578701 : gcc_checking_assert (compatible_p (other));
933 : 23578701 : uint64_t ret_val = m_val + other.m_val;
934 : 23578701 : m_val = MIN (ret_val, max_count);
935 : 23578701 : m_quality = MIN (m_quality, other.m_quality);
936 : : }
937 : 23578701 : return *this;
938 : : }
939 : :
940 : 17341483 : profile_count operator- (const profile_count &other) const
941 : : {
942 : 17354971 : if (*this == zero () || other == zero ())
943 : 1016436 : return *this;
944 : 16325047 : if (!initialized_p () || !other.initialized_p ())
945 : 6805842 : return uninitialized ();
946 : 9519205 : gcc_checking_assert (compatible_p (other));
947 : 9519205 : profile_count ret;
948 : 9519205 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
949 : 9519205 : ret.m_quality = MIN (m_quality, other.m_quality);
950 : 9519205 : return ret;
951 : : }
952 : :
953 : 8542354 : profile_count &operator-= (const profile_count &other)
954 : : {
955 : 8970487 : if (*this == zero () || other == zero ())
956 : 308498 : return *this;
957 : 8233856 : if (!initialized_p () || !other.initialized_p ())
958 : 343855 : return *this = uninitialized ();
959 : : else
960 : : {
961 : 7890001 : gcc_checking_assert (compatible_p (other));
962 : 7890001 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
963 : 7890001 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 7890001 : return *this;
966 : : }
967 : :
968 : : /* Return false if profile_count is bogus. */
969 : 3067696717 : bool verify () const
970 : : {
971 : 3067696717 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
972 : 3067696717 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
973 : : }
974 : :
975 : : /* Comparisons are three-state and conservative. False is returned if
976 : : the inequality cannot be decided. */
977 : 965804585 : bool operator< (const profile_count &other) const
978 : : {
979 : 965804585 : if (!initialized_p () || !other.initialized_p ())
980 : : return false;
981 : 961689157 : if (*this == zero ())
982 : 3679866 : return !(other == zero ());
983 : 959849224 : if (other == zero ())
984 : 4457 : return false;
985 : 959844767 : gcc_checking_assert (compatible_p (other));
986 : 959844767 : return m_val < other.m_val;
987 : : }
988 : :
989 : 31799110 : bool operator> (const profile_count &other) const
990 : : {
991 : 31799110 : if (!initialized_p () || !other.initialized_p ())
992 : : return false;
993 : 30890213 : if (*this == zero ())
994 : 263644 : return false;
995 : 30626569 : if (other == zero ())
996 : 1544420 : return !(*this == zero ());
997 : 29854359 : gcc_checking_assert (compatible_p (other));
998 : 29854359 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
999 : : }
1000 : :
1001 : : bool operator< (const gcov_type other) const
1002 : : {
1003 : : gcc_checking_assert (ipa_p ());
1004 : : gcc_checking_assert (other >= 0);
1005 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1006 : : }
1007 : :
1008 : 262161 : bool operator> (const gcov_type other) const
1009 : : {
1010 : 262161 : gcc_checking_assert (ipa_p ());
1011 : 262161 : gcc_checking_assert (other >= 0);
1012 : 262161 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1013 : : }
1014 : :
1015 : 968208 : bool operator<= (const profile_count &other) const
1016 : : {
1017 : 968208 : if (!initialized_p () || !other.initialized_p ())
1018 : : return false;
1019 : 968208 : if (*this == zero ())
1020 : 2585 : return true;
1021 : 965623 : if (other == zero ())
1022 : 0 : return (*this == zero ());
1023 : 965623 : gcc_checking_assert (compatible_p (other));
1024 : 965623 : return m_val <= other.m_val;
1025 : : }
1026 : :
1027 : 1284248 : bool operator>= (const profile_count &other) const
1028 : : {
1029 : 1284248 : if (!initialized_p () || !other.initialized_p ())
1030 : : return false;
1031 : 1283373 : if (other == zero ())
1032 : 277 : return true;
1033 : 1283096 : if (*this == zero ())
1034 : 22374 : return (other == zero ());
1035 : 1271909 : gcc_checking_assert (compatible_p (other));
1036 : 1271909 : return m_val >= other.m_val;
1037 : : }
1038 : :
1039 : 92651 : bool operator<= (const gcov_type other) const
1040 : : {
1041 : 92651 : gcc_checking_assert (ipa_p ());
1042 : 92651 : gcc_checking_assert (other >= 0);
1043 : 92651 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1044 : : }
1045 : :
1046 : 57294 : bool operator>= (const gcov_type other) const
1047 : : {
1048 : 57294 : gcc_checking_assert (ipa_p ());
1049 : 57294 : gcc_checking_assert (other >= 0);
1050 : 57294 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1051 : : }
1052 : :
1053 : 804847477 : profile_count operator* (int64_t num) const
1054 : : {
1055 : 804771929 : return apply_scale (num, 1);
1056 : : }
1057 : :
1058 : : profile_count operator*= (int64_t num)
1059 : : {
1060 : : *this = apply_scale (num, 1);
1061 : : return *this;
1062 : : }
1063 : :
1064 : 2219683 : profile_count operator/ (int64_t den) const
1065 : : {
1066 : 2219683 : return apply_scale (1, den);
1067 : : }
1068 : :
1069 : 65 : profile_count operator/= (int64_t den)
1070 : : {
1071 : 61 : *this = apply_scale (1, den);
1072 : 65 : return *this;
1073 : : }
1074 : :
1075 : : /* Return true when value is not zero and can be used for scaling.
1076 : : This is different from *this > 0 because that requires counter to
1077 : : be IPA. */
1078 : 5123626354 : bool nonzero_p () const
1079 : : {
1080 : 4970098983 : return initialized_p () && m_val != 0;
1081 : : }
1082 : :
1083 : : /* Make counter forcibly nonzero. */
1084 : 17310009 : profile_count force_nonzero () const
1085 : : {
1086 : 17310009 : if (!initialized_p ())
1087 : 24851 : return *this;
1088 : 17285158 : profile_count ret = *this;
1089 : 17285158 : if (ret.m_val == 0)
1090 : : {
1091 : 49572 : ret.m_val = 1;
1092 : 49572 : ret.m_quality = MIN (m_quality, ADJUSTED);
1093 : : }
1094 : 17285158 : return ret;
1095 : : }
1096 : :
1097 : 81365024 : profile_count max (profile_count other) const
1098 : : {
1099 : 81365024 : profile_count val = *this;
1100 : :
1101 : : /* Always prefer nonzero IPA counts over local counts. */
1102 : 90553430 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1103 : : {
1104 : 9236 : val = ipa ();
1105 : 9236 : other = other.ipa ();
1106 : : }
1107 : 81365024 : if (!initialized_p ())
1108 : 19118363 : return other;
1109 : 62246661 : if (!other.initialized_p ())
1110 : 1827584 : return *this;
1111 : 60419077 : if (*this == zero ())
1112 : 2106486 : return other;
1113 : 58312591 : if (other == zero ())
1114 : 4278826 : return *this;
1115 : 54033765 : gcc_checking_assert (compatible_p (other));
1116 : 54033765 : if (val.m_val < other.m_val || (m_val == other.m_val
1117 : 12728009 : && val.m_quality < other.m_quality))
1118 : 7675754 : return other;
1119 : 46358011 : return *this;
1120 : : }
1121 : :
1122 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1123 : : accordingly. */
1124 : : profile_count apply_probability (int prob) const
1125 : : {
1126 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1127 : : if (m_val == 0)
1128 : : return *this;
1129 : : if (!initialized_p ())
1130 : : return uninitialized ();
1131 : : profile_count ret;
1132 : : uint64_t tmp;
1133 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1134 : : ret.m_val = tmp;
1135 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1136 : : return ret;
1137 : : }
1138 : :
1139 : : /* Scale counter according to PROB. */
1140 : 347597166 : profile_count apply_probability (profile_probability prob) const
1141 : : {
1142 : 354995395 : if (*this == zero () || prob == profile_probability::always ())
1143 : 112759691 : return *this;
1144 : 234837475 : if (prob == profile_probability::never ())
1145 : 18561020 : return zero ();
1146 : 216276455 : if (!initialized_p () || !prob.initialized_p ())
1147 : 28575830 : return uninitialized ();
1148 : 187700625 : profile_count ret;
1149 : 187700625 : uint64_t tmp;
1150 : 187700625 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1151 : : &tmp);
1152 : 187700625 : ret.m_val = tmp;
1153 : 187700625 : ret.m_quality = MIN (m_quality, prob.m_quality);
1154 : 187700625 : return ret;
1155 : : }
1156 : :
1157 : : /* Return *THIS * NUM / DEN. */
1158 : 911498568 : profile_count apply_scale (int64_t num, int64_t den) const
1159 : : {
1160 : 911498568 : if (m_val == 0)
1161 : 22924224 : return *this;
1162 : 888574344 : if (!initialized_p ())
1163 : 6844 : return uninitialized ();
1164 : 888567500 : profile_count ret;
1165 : 888567500 : uint64_t tmp;
1166 : :
1167 : 888567500 : gcc_checking_assert (num >= 0 && den > 0);
1168 : 888567500 : safe_scale_64bit (m_val, num, den, &tmp);
1169 : 888567500 : ret.m_val = MIN (tmp, max_count);
1170 : 888567500 : ret.m_quality = MIN (m_quality, ADJUSTED);
1171 : 888567500 : return ret;
1172 : : }
1173 : :
1174 : 23857966 : profile_count apply_scale (profile_count num, profile_count den) const
1175 : : {
1176 : 23857966 : if (*this == zero ())
1177 : 1871538 : return *this;
1178 : 21986428 : if (num == zero ())
1179 : 69749 : return num;
1180 : 21916679 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1181 : 6020390 : return uninitialized ();
1182 : 15896289 : if (num == den)
1183 : 5379169 : return *this;
1184 : 10517120 : gcc_checking_assert (den.m_val);
1185 : :
1186 : 10517120 : profile_count ret;
1187 : 10517120 : uint64_t val;
1188 : 10517120 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1189 : 10517120 : ret.m_val = MIN (val, max_count);
1190 : 10517120 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1191 : : num.m_quality), den.m_quality);
1192 : : /* Be sure that ret is not local if num is global.
1193 : : Also ensure that ret is not global0 when num is global. */
1194 : 10517120 : if (num.ipa_p ())
1195 : 2958 : ret.m_quality = MAX (ret.m_quality,
1196 : : num == num.ipa () ? GUESSED : num.m_quality);
1197 : 10517120 : return ret;
1198 : : }
1199 : :
1200 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1201 : 16166272 : profile_count guessed_local () const
1202 : : {
1203 : 16166272 : profile_count ret = *this;
1204 : 16166272 : if (!initialized_p ())
1205 : 0 : return *this;
1206 : 16166272 : ret.m_quality = GUESSED_LOCAL;
1207 : 16166272 : return ret;
1208 : : }
1209 : :
1210 : : /* We know that profile is globally 0 but keep local profile if present. */
1211 : 766 : profile_count global0 () const
1212 : : {
1213 : 766 : profile_count ret = *this;
1214 : 766 : if (!initialized_p ())
1215 : 0 : return *this;
1216 : 766 : ret.m_quality = GUESSED_GLOBAL0;
1217 : 766 : return ret;
1218 : : }
1219 : :
1220 : : /* We know that profile is globally adjusted 0 but keep local profile
1221 : : if present. */
1222 : 34 : profile_count global0adjusted () const
1223 : : {
1224 : 34 : profile_count ret = *this;
1225 : 34 : if (!initialized_p ())
1226 : 0 : return *this;
1227 : 34 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1228 : 34 : return ret;
1229 : : }
1230 : :
1231 : : /* Return THIS with quality dropped to GUESSED. */
1232 : 261 : profile_count guessed () const
1233 : : {
1234 : 261 : profile_count ret = *this;
1235 : 261 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1236 : 261 : return ret;
1237 : : }
1238 : :
1239 : : /* Return variant of profile count which is always safe to compare
1240 : : across functions. */
1241 : 5933760958 : profile_count ipa () const
1242 : : {
1243 : 5933760958 : if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
1244 : 19781868 : return *this;
1245 : 5913979090 : if (m_quality == GUESSED_GLOBAL0)
1246 : 44915 : return zero ();
1247 : 5913934175 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1248 : 3710 : return adjusted_zero ();
1249 : 5913930465 : return uninitialized ();
1250 : : }
1251 : :
1252 : : /* Return THIS with quality dropped to AFDO. */
1253 : 0 : profile_count afdo () const
1254 : : {
1255 : 0 : profile_count ret = *this;
1256 : 0 : ret.m_quality = AFDO;
1257 : 0 : return ret;
1258 : : }
1259 : :
1260 : : /* Return probability of event with counter THIS within event with counter
1261 : : OVERALL. */
1262 : 1039221986 : profile_probability probability_in (const profile_count overall) const
1263 : : {
1264 : 1039237171 : if (*this == zero ()
1265 : 15185 : && !(overall == zero ()))
1266 : 1620 : return profile_probability::never ();
1267 : 1039219385 : if (!initialized_p () || !overall.initialized_p ()
1268 : 2078439751 : || !overall.m_val)
1269 : 58190 : return profile_probability::uninitialized ();
1270 : 1278920128 : if (*this == overall && m_quality == PRECISE)
1271 : 36055 : return profile_probability::always ();
1272 : 1039126121 : profile_probability ret;
1273 : 1039126121 : gcc_checking_assert (compatible_p (overall));
1274 : :
1275 : 1039126121 : if (overall.m_val < m_val)
1276 : : {
1277 : 88163 : ret.m_val = profile_probability::max_probability;
1278 : 88163 : ret.m_quality = GUESSED;
1279 : 88163 : return ret;
1280 : : }
1281 : : else
1282 : 1039037958 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1283 : : overall.m_val);
1284 : 1039037958 : ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
1285 : : GUESSED), ADJUSTED);
1286 : 1039037958 : return ret;
1287 : : }
1288 : :
1289 : : /* Return true if profile count is very large, so we risk overflows
1290 : : with loop transformations. */
1291 : : bool
1292 : 992167 : very_large_p ()
1293 : : {
1294 : 992167 : if (!initialized_p ())
1295 : : return false;
1296 : 992167 : return m_val > max_count / 65536;
1297 : : }
1298 : :
1299 : : int to_frequency (struct function *fun) const;
1300 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1301 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1302 : :
1303 : : /* Output THIS to F. */
1304 : : void dump (FILE *f, struct function *fun = NULL) const;
1305 : :
1306 : : /* Print THIS to stderr. */
1307 : : void debug () const;
1308 : :
1309 : : /* Return true if THIS is known to differ significantly from OTHER. */
1310 : : bool differs_from_p (profile_count other) const;
1311 : :
1312 : : /* We want to scale profile across function boundary from NUM to DEN.
1313 : : Take care of the side case when NUM and DEN are zeros of incompatible
1314 : : kinds. */
1315 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1316 : :
1317 : : /* THIS is a count of bb which is known to be executed IPA times.
1318 : : Combine this information into bb counter. This means returning IPA
1319 : : if it is nonzero, not changing anything if IPA is uninitialized
1320 : : and if IPA is zero, turning THIS into corresponding local profile with
1321 : : global0. */
1322 : : profile_count combine_with_ipa_count (profile_count ipa);
1323 : :
1324 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1325 : : profile_count combine_with_ipa_count_within
1326 : : (profile_count ipa, profile_count ipa2);
1327 : :
1328 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1329 : : Conversions back and forth are used to read the coverage and get it
1330 : : into internal representation. */
1331 : : static profile_count from_gcov_type (gcov_type v,
1332 : : profile_quality quality = PRECISE);
1333 : :
1334 : : /* LTO streaming support. */
1335 : : static profile_count stream_in (class lto_input_block *);
1336 : : void stream_out (struct output_block *);
1337 : : void stream_out (struct lto_output_stream *);
1338 : : };
1339 : : #endif
|
