Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2025 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Same as GUESSED_GLOBAL0 but global count is afdo 0. */
41 : : GUESSED_GLOBAL0_AFDO,
42 : :
43 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
44 : : GUESSED_GLOBAL0_ADJUSTED,
45 : :
46 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
47 : : better. This is the case of functions not run in profile feedback.
48 : : Never used by probabilities. */
49 : : GUESSED_GLOBAL0,
50 : :
51 : : /* Profile is based on static branch prediction heuristics. It may or may
52 : : not reflect the reality but it can be compared interprocedurally
53 : : (for example, we inlined function w/o profile feedback into function
54 : : with feedback and propagated from that). */
55 : : GUESSED,
56 : :
57 : : /* Profile was determined by autofdo. */
58 : : AFDO,
59 : :
60 : : /* Profile was originally based on feedback but it was adjusted
61 : : by code duplicating optimization. It may not precisely reflect the
62 : : particular code path. */
63 : : ADJUSTED,
64 : :
65 : : /* Profile was read from profile feedback or determined by accurate static
66 : : method. */
67 : : PRECISE
68 : : };
69 : :
70 : : extern const char *profile_quality_as_string (enum profile_quality);
71 : : extern bool parse_profile_quality (const char *value,
72 : : profile_quality *quality);
73 : :
74 : : /* The base value for branch probability notes and edge probabilities. */
75 : : #define REG_BR_PROB_BASE 10000
76 : :
77 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
78 : :
79 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
80 : :
81 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
82 : :
83 : : inline bool
84 : 1189523661 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
85 : : {
86 : : #if (GCC_VERSION >= 5000)
87 : 1189523661 : uint64_t tmp;
88 : 1189523661 : if (!__builtin_mul_overflow (a, b, &tmp)
89 : 1189523661 : && !__builtin_add_overflow (tmp, c/2, &tmp))
90 : : {
91 : 1188908801 : *res = tmp / c;
92 : 1188908801 : return true;
93 : : }
94 : 614860 : if (c == 1)
95 : : {
96 : 27748 : *res = (uint64_t) -1;
97 : 27748 : return false;
98 : : }
99 : : #else
100 : : if (a < ((uint64_t)1 << 31)
101 : : && b < ((uint64_t)1 << 31)
102 : : && c < ((uint64_t)1 << 31))
103 : : {
104 : : *res = (a * b + (c / 2)) / c;
105 : : return true;
106 : : }
107 : : #endif
108 : 587112 : return slow_safe_scale_64bit (a, b, c, res);
109 : : }
110 : :
111 : : /* Data type to hold probabilities. It implements fixed point arithmetics
112 : : with capping so probability is always in range [0,1] and scaling requiring
113 : : values greater than 1 needs to be represented otherwise.
114 : :
115 : : In addition to actual value the quality of profile is tracked and propagated
116 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
117 : : that has not been determined yet (for example because of
118 : : -fno-guess-branch-probability)
119 : :
120 : : Typically probabilities are derived from profile feedback (via
121 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
122 : : thorough the compilation.
123 : :
124 : : Named probabilities are available:
125 : : - never (0 probability)
126 : : - guessed_never
127 : : - very_unlikely (1/2000 probability)
128 : : - unlikely (1/5 probability)
129 : : - even (1/2 probability)
130 : : - likely (4/5 probability)
131 : : - very_likely (1999/2000 probability)
132 : : - guessed_always
133 : : - always
134 : :
135 : : Named probabilities except for never/always are assumed to be statically
136 : : guessed and thus not necessarily accurate. The difference between never
137 : : and guessed_never is that the first one should be used only in case that
138 : : well behaving program will very likely not execute the "never" path.
139 : : For example if the path is going to abort () call or it exception handling.
140 : :
141 : : Always and guessed_always probabilities are symmetric.
142 : :
143 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
144 : : integer arithmetics. Once the code is converted to branch probabilities,
145 : : these conversions will probably go away because they are lossy.
146 : : */
147 : :
148 : : class GTY((user)) profile_probability
149 : : {
150 : : static const int n_bits = 29;
151 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
152 : : will lead to harder multiplication sequences. */
153 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
154 : : static const uint32_t uninitialized_probability
155 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
156 : : /* For probibilityes quality is either UNINITIALIZED (0)
157 : : or greated then GUESSED. To save bits we store it in
158 : : adjusted form which skips the invalid values. */
159 : : static const int min_quality = GUESSED;
160 : :
161 : : uint32_t m_val : 29;
162 : : unsigned m_adjusted_quality : 3;
163 : :
164 : : friend struct profile_count;
165 : :
166 : : /* Set the quality of the probability. */
167 : : void
168 : 5192805524 : set_quality (profile_quality quality)
169 : : {
170 : 1124017987 : gcc_checking_assert (quality == UNINITIALIZED_PROFILE
171 : : || (quality >= min_quality && quality <= PRECISE));
172 : 1124018071 : m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
173 : 1122348602 : }
174 : :
175 : : public:
176 : 4070456922 : profile_probability (): m_val (uninitialized_probability)
177 : 2206244740 : { set_quality (GUESSED); }
178 : :
179 : 112 : profile_probability (uint32_t val, profile_quality quality):
180 : 112 : m_val (val)
181 : 112 : { set_quality (quality); }
182 : :
183 : : /* Named probabilities. */
184 : 663874773 : static profile_probability never ()
185 : : {
186 : 663874773 : profile_probability ret;
187 : 663874773 : ret.m_val = 0;
188 : 663874773 : ret.set_quality (PRECISE);
189 : 372350342 : return ret;
190 : : }
191 : :
192 : 5190843 : static profile_probability guessed_never ()
193 : : {
194 : 5190843 : profile_probability ret;
195 : 5190843 : ret.m_val = 0;
196 : 5190843 : ret.set_quality (GUESSED);
197 : 5190843 : return ret;
198 : : }
199 : :
200 : 5155306 : static profile_probability very_unlikely ()
201 : : {
202 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
203 : 10310612 : profile_probability r = guessed_always () / 2000;
204 : 5155306 : r.m_val--;
205 : 5155306 : return r;
206 : : }
207 : :
208 : 80788 : static profile_probability unlikely ()
209 : : {
210 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
211 : 161576 : profile_probability r = guessed_always () / 5;
212 : 80788 : r.m_val--;
213 : 80788 : return r;
214 : : }
215 : :
216 : 782653 : static profile_probability even ()
217 : : {
218 : 1565306 : return guessed_always () / 2;
219 : : }
220 : :
221 : 336899 : static profile_probability very_likely ()
222 : : {
223 : 336899 : return always () - very_unlikely ();
224 : : }
225 : :
226 : 31894 : static profile_probability likely ()
227 : : {
228 : 31894 : return always () - unlikely ();
229 : : }
230 : : /* Return true when value is not zero and can be used for scaling. */
231 : 1549 : bool nonzero_p () const
232 : : {
233 : 1549 : return initialized_p () && m_val != 0;
234 : : }
235 : :
236 : 6323783 : static profile_probability guessed_always ()
237 : : {
238 : 6323783 : profile_probability ret;
239 : 6323783 : ret.m_val = max_probability;
240 : 6323783 : ret.set_quality (GUESSED);
241 : 6123561 : return ret;
242 : : }
243 : :
244 : 526708135 : static profile_probability always ()
245 : : {
246 : 526708135 : profile_probability ret;
247 : 526708135 : ret.m_val = max_probability;
248 : 526708135 : ret.set_quality (PRECISE);
249 : 352951056 : return ret;
250 : : }
251 : :
252 : : /* Probabilities which has not been initialized. Either because
253 : : initialization did not happen yet or because profile is unknown. */
254 : 885430479 : static profile_probability uninitialized ()
255 : : {
256 : 885430479 : profile_probability c;
257 : 885430479 : c.m_val = uninitialized_probability;
258 : 885430479 : c.set_quality (GUESSED);
259 : 872035144 : return c;
260 : : }
261 : :
262 : : /* Return true if value has been initialized. */
263 : 5579897643 : bool initialized_p () const
264 : : {
265 : 3903884308 : return m_val != uninitialized_probability;
266 : : }
267 : :
268 : : /* Return true if value can be trusted. */
269 : 1211 : bool reliable_p () const
270 : : {
271 : 1197 : return quality () >= ADJUSTED;
272 : : }
273 : :
274 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
275 : : this is mostly to support legacy code and should go away. */
276 : 3069916 : static profile_probability from_reg_br_prob_base (int v)
277 : : {
278 : 3069916 : profile_probability ret;
279 : 3069916 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
280 : 3069916 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
281 : 3069916 : ret.set_quality (GUESSED);
282 : 3069916 : return ret;
283 : : }
284 : :
285 : : /* Return THIS with quality set to ADJUSTED. */
286 : 42 : profile_probability adjusted () const
287 : : {
288 : 42 : profile_probability ret = *this;
289 : 42 : if (!initialized_p ())
290 : 0 : return *this;
291 : 42 : ret.set_quality (ADJUSTED);
292 : 42 : return ret;
293 : : }
294 : :
295 : 1105256411 : int to_reg_br_prob_base () const
296 : : {
297 : 1105256411 : gcc_checking_assert (initialized_p ());
298 : 1105256411 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
299 : : }
300 : :
301 : : /* Conversion to and from RTL representation of profile probabilities. */
302 : 471660200 : static profile_probability from_reg_br_prob_note (int v)
303 : : {
304 : 471660200 : profile_probability ret;
305 : 471660200 : ret.m_val = ((unsigned int)v) / 8;
306 : 471660200 : ret.m_adjusted_quality = ((unsigned int)v) & 7;
307 : 15435993 : return ret;
308 : : }
309 : :
310 : 443206016 : int to_reg_br_prob_note () const
311 : : {
312 : 443206016 : gcc_checking_assert (initialized_p ());
313 : 443206016 : int ret = m_val * 8 + m_adjusted_quality;
314 : 443206016 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
315 : 443206016 : return ret;
316 : : }
317 : :
318 : : /* Return VAL1/VAL2. */
319 : 3072 : static profile_probability probability_in_gcov_type
320 : : (gcov_type val1, gcov_type val2)
321 : : {
322 : 3072 : profile_probability ret;
323 : 3072 : gcc_checking_assert (val1 >= 0 && val2 > 0);
324 : 3072 : if (val1 > val2)
325 : : ret.m_val = max_probability;
326 : : else
327 : : {
328 : 3072 : uint64_t tmp;
329 : 3072 : safe_scale_64bit (val1, max_probability, val2, &tmp);
330 : 3072 : gcc_checking_assert (tmp <= max_probability);
331 : 3072 : ret.m_val = tmp;
332 : : }
333 : 3072 : ret.set_quality (PRECISE);
334 : 3072 : return ret;
335 : : }
336 : :
337 : : /* Basic operations. */
338 : 1500977722 : bool operator== (const profile_probability &other) const
339 : : {
340 : 2131696798 : return m_val == other.m_val && quality () == other.quality ();
341 : : }
342 : :
343 : 6564131 : profile_probability operator+ (const profile_probability &other) const
344 : : {
345 : 6564131 : if (other == never ())
346 : 30850 : return *this;
347 : 6533281 : if (*this == never ())
348 : 23763 : return other;
349 : 6509518 : if (!initialized_p () || !other.initialized_p ())
350 : 5340643 : return uninitialized ();
351 : :
352 : 1168875 : profile_probability ret;
353 : 1168875 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
354 : 1168875 : ret.set_quality (MIN (quality (), other.quality ()));
355 : 1168875 : return ret;
356 : : }
357 : :
358 : 6160240 : profile_probability &operator+= (const profile_probability &other)
359 : : {
360 : 6160240 : if (other == never ())
361 : : return *this;
362 : 4989960 : if (*this == never ())
363 : : {
364 : 857309 : *this = other;
365 : 857309 : return *this;
366 : : }
367 : 4132651 : if (!initialized_p () || !other.initialized_p ())
368 : 2898294 : return *this = uninitialized ();
369 : : else
370 : : {
371 : 1234357 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
372 : 1234357 : set_quality (MIN (quality (), other.quality ()));
373 : : }
374 : 1234357 : return *this;
375 : : }
376 : :
377 : 33481090 : profile_probability operator- (const profile_probability &other) const
378 : : {
379 : 66962180 : if (*this == never ()
380 : 33481090 : || other == never ())
381 : 1807620 : return *this;
382 : 31673470 : if (!initialized_p () || !other.initialized_p ())
383 : 2147803 : return uninitialized ();
384 : 29525667 : profile_probability ret;
385 : 29525667 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
386 : 29525667 : ret.set_quality (MIN (quality (), other.quality ()));
387 : 29525667 : return ret;
388 : : }
389 : :
390 : 11487722 : profile_probability &operator-= (const profile_probability &other)
391 : : {
392 : 22975444 : if (*this == never ()
393 : 11487722 : || other == never ())
394 : 2264758 : return *this;
395 : 9222964 : if (!initialized_p () || !other.initialized_p ())
396 : 2131284 : return *this = uninitialized ();
397 : : else
398 : : {
399 : 7091680 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
400 : 7091680 : set_quality (MIN (quality (), other.quality ()));
401 : : }
402 : 7091680 : return *this;
403 : : }
404 : :
405 : 1176414 : profile_probability operator* (const profile_probability &other) const
406 : : {
407 : 2352828 : if (*this == never ()
408 : 1176414 : || other == never ())
409 : 120435 : return never ();
410 : 1055979 : if (!initialized_p () || !other.initialized_p ())
411 : 57800 : return uninitialized ();
412 : 998179 : profile_probability ret;
413 : 998179 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
414 : 998179 : ret.set_quality (MIN (quality (), other.quality ()));
415 : 998179 : return ret;
416 : : }
417 : :
418 : 156073 : profile_probability &operator*= (const profile_probability &other)
419 : : {
420 : 312146 : if (*this == never ()
421 : 156073 : || other == never ())
422 : 339 : return *this = never ();
423 : 155734 : if (!initialized_p () || !other.initialized_p ())
424 : 9 : return *this = uninitialized ();
425 : : else
426 : : {
427 : 155725 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
428 : 294211 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
429 : : }
430 : 155725 : return *this;
431 : : }
432 : :
433 : 1729389 : profile_probability operator/ (const profile_probability &other) const
434 : : {
435 : 1729389 : if (*this == never ())
436 : 107017 : return never ();
437 : 1622372 : if (!initialized_p () || !other.initialized_p ())
438 : 224559 : return uninitialized ();
439 : 1397813 : profile_probability ret;
440 : : /* If we get probability above 1, mark it as unreliable and return 1. */
441 : 1397813 : if (m_val >= other.m_val)
442 : : {
443 : 33047 : ret.m_val = max_probability;
444 : 33047 : ret.set_quality (MIN (MIN (quality (), other.quality ()),
445 : : GUESSED));
446 : 33047 : return ret;
447 : : }
448 : 1364766 : else if (!m_val)
449 : 60841 : ret.m_val = 0;
450 : : else
451 : : {
452 : 1303925 : gcc_checking_assert (other.m_val);
453 : 1303925 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
454 : : other.m_val),
455 : : max_probability);
456 : : }
457 : 2721563 : ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
458 : 1364766 : return ret;
459 : : }
460 : :
461 : 338777 : profile_probability &operator/= (const profile_probability &other)
462 : : {
463 : 338777 : if (*this == never ())
464 : 1368 : return *this = never ();
465 : 337409 : if (!initialized_p () || !other.initialized_p ())
466 : 0 : return *this = uninitialized ();
467 : : else
468 : : {
469 : : /* If we get probability above 1, mark it as unreliable
470 : : and return 1. */
471 : 337409 : if (m_val > other.m_val)
472 : : {
473 : 633 : m_val = max_probability;
474 : 633 : set_quality (MIN (MIN (quality (), other.quality ()),
475 : : GUESSED));
476 : 633 : return *this;
477 : : }
478 : 336776 : else if (!m_val)
479 : : ;
480 : : else
481 : : {
482 : 334983 : gcc_checking_assert (other.m_val);
483 : 334983 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
484 : : other.m_val),
485 : : max_probability);
486 : : }
487 : 610299 : set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
488 : : }
489 : 336776 : return *this;
490 : : }
491 : :
492 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
493 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
494 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
495 : : == ORIG. This is useful e.g. when splitting a conditional
496 : : branch like:
497 : : if (cond)
498 : : goto lab; // ORIG probability
499 : : into
500 : : if (cond1)
501 : : goto lab; // FIRST = ORIG * CPROB probability
502 : : if (cond2)
503 : : goto lab; // SECOND probability
504 : : such that the overall probability of jumping to lab remains
505 : : the same. CPROB gives the relative probability between the
506 : : branches. */
507 : 289198 : profile_probability split (const profile_probability &cprob)
508 : : {
509 : 289198 : profile_probability ret = *this * cprob;
510 : : /* The following is equivalent to:
511 : : *this = cprob.invert () * *this / ret.invert ();
512 : : Avoid scaling when overall outcome is supposed to be always.
513 : : Without knowing that one is inverse of other, the result would be
514 : : conservative. */
515 : 289198 : if (!(*this == always ()))
516 : 280519 : *this = (*this - ret) / ret.invert ();
517 : 289198 : return ret;
518 : : }
519 : :
520 : 254942 : gcov_type apply (gcov_type val) const
521 : : {
522 : 254942 : if (*this == uninitialized ())
523 : 63 : return val / 2;
524 : 254879 : return RDIV (val * m_val, max_probability);
525 : : }
526 : :
527 : : /* Return 1-*THIS. */
528 : 20981584 : profile_probability invert () const
529 : : {
530 : 30308357 : return always() - *this;
531 : : }
532 : :
533 : : /* Return THIS with quality dropped to GUESSED. */
534 : 865152 : profile_probability guessed () const
535 : : {
536 : 865152 : profile_probability ret = *this;
537 : 865152 : ret.set_quality (GUESSED);
538 : 865152 : return ret;
539 : : }
540 : :
541 : : /* Return THIS with quality dropped to AFDO. */
542 : : profile_probability afdo () const
543 : : {
544 : : profile_probability ret = *this;
545 : : ret.set_quality (AFDO);
546 : : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 14342851 : profile_probability apply_scale (int64_t num, int64_t den) const
551 : : {
552 : 14342851 : if (*this == never ())
553 : 16131 : return *this;
554 : 14326720 : if (!initialized_p ())
555 : 5741511 : return uninitialized ();
556 : 8585209 : profile_probability ret;
557 : 8585209 : uint64_t tmp;
558 : 8585209 : safe_scale_64bit (m_val, num, den, &tmp);
559 : 8585209 : ret.m_val = MIN (tmp, max_probability);
560 : 8585209 : ret.set_quality (MIN (quality (), ADJUSTED));
561 : 8585209 : return ret;
562 : : }
563 : :
564 : : /* Return *THIS * NUM / DEN. */
565 : 10367 : profile_probability apply_scale (profile_probability num,
566 : : profile_probability den) const
567 : : {
568 : 10367 : if (*this == never ())
569 : 56 : return *this;
570 : 10311 : if (num == never ())
571 : 0 : return num;
572 : 10311 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
573 : 0 : return uninitialized ();
574 : 10311 : if (num == den)
575 : 0 : return *this;
576 : 10311 : gcc_checking_assert (den.m_val);
577 : :
578 : 10311 : profile_probability ret;
579 : 10311 : uint64_t val;
580 : 10311 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
581 : 10311 : ret.m_val = MIN (val, max_probability);
582 : 10311 : ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
583 : : num.quality ()), den.quality ()));
584 : 10311 : return ret;
585 : : }
586 : :
587 : : /* Return true when the probability of edge is reliable.
588 : :
589 : : The profile guessing code is good at predicting branch outcome (i.e.
590 : : taken/not taken), that is predicted right slightly over 75% of time.
591 : : It is however notoriously poor on predicting the probability itself.
592 : : In general the profile appear a lot flatter (with probabilities closer
593 : : to 50%) than the reality so it is bad idea to use it to drive optimization
594 : : such as those disabling dynamic branch prediction for well predictable
595 : : branches.
596 : :
597 : : There are two exceptions - edges leading to noreturn edges and edges
598 : : predicted by number of iterations heuristics are predicted well. This macro
599 : : should be able to distinguish those, but at the moment it simply check for
600 : : noreturn heuristic that is only one giving probability over 99% or bellow
601 : : 1%. In future we might want to propagate reliability information across the
602 : : CFG if we find this information useful on multiple places. */
603 : 0 : bool probably_reliable_p () const
604 : : {
605 : 0 : if (quality () >= ADJUSTED)
606 : : return true;
607 : 0 : if (!initialized_p ())
608 : : return false;
609 : 0 : return m_val < max_probability / 100
610 : 0 : || m_val > max_probability - max_probability / 100;
611 : : }
612 : :
613 : : /* Return false if profile_probability is bogus. */
614 : 4285475652 : bool verify () const
615 : : {
616 : 0 : gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
617 : 4285475652 : if (m_val == uninitialized_probability)
618 : 932941597 : return quality () == GUESSED;
619 : 3352534055 : else if (quality () < GUESSED)
620 : : return false;
621 : 3352534055 : return m_val <= max_probability;
622 : : }
623 : :
624 : : /* Comparisons are three-state and conservative. False is returned if
625 : : the inequality cannot be decided. */
626 : 6143418 : bool operator< (const profile_probability &other) const
627 : : {
628 : 6143418 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
629 : : }
630 : :
631 : 17525919 : bool operator> (const profile_probability &other) const
632 : : {
633 : 17525919 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
634 : : }
635 : :
636 : 5280300 : bool operator<= (const profile_probability &other) const
637 : : {
638 : 5280300 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
639 : : }
640 : :
641 : 936519 : bool operator>= (const profile_probability &other) const
642 : : {
643 : 936519 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
644 : : }
645 : :
646 : 17 : profile_probability operator* (int64_t num) const
647 : : {
648 : 17 : return apply_scale (num, 1);
649 : : }
650 : :
651 : : profile_probability operator*= (int64_t num)
652 : : {
653 : : *this = apply_scale (num, 1);
654 : : return *this;
655 : : }
656 : :
657 : 13691282 : profile_probability operator/ (int64_t den) const
658 : : {
659 : 13691282 : return apply_scale (1, den);
660 : : }
661 : :
662 : 242777 : profile_probability operator/= (int64_t den)
663 : : {
664 : 242777 : *this = apply_scale (1, den);
665 : 242777 : return *this;
666 : : }
667 : :
668 : : /* Compute n-th power. */
669 : : profile_probability pow (int) const;
670 : :
671 : : /* Compute sware root. */
672 : : profile_probability sqrt () const;
673 : :
674 : : /* Get the value of the probability. */
675 : 0 : uint32_t value () const { return m_val; }
676 : :
677 : : /* Get the quality of the probability. */
678 : 5897278348 : enum profile_quality quality () const
679 : : {
680 : 5692739356 : return (profile_quality) (m_adjusted_quality
681 : 5692762390 : ? m_adjusted_quality + min_quality - 1
682 : : : UNINITIALIZED_PROFILE);
683 : : }
684 : :
685 : : /* Output THIS to F. */
686 : : void dump (FILE *f) const;
687 : :
688 : : /* Output THIS to BUFFER. */
689 : : void dump (char *buffer) const;
690 : :
691 : : /* Print THIS to stderr. */
692 : : void debug () const;
693 : :
694 : : /* Return true if THIS is known to differ significantly from OTHER. */
695 : : bool differs_from_p (profile_probability other) const;
696 : :
697 : : /* Return if difference is greater than 50%. */
698 : : bool differs_lot_from_p (profile_probability other) const;
699 : :
700 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
701 : : happens with COUNT2 probability. Return probability that either *THIS or
702 : : OTHER happens. */
703 : : profile_probability combine_with_count (profile_count count1,
704 : : profile_probability other,
705 : : profile_count count2) const;
706 : :
707 : : /* Return probability as sreal. */
708 : : sreal to_sreal () const;
709 : : /* LTO streaming support. */
710 : : static profile_probability stream_in (class lto_input_block *);
711 : : void stream_out (struct output_block *);
712 : : void stream_out (struct lto_output_stream *);
713 : : };
714 : :
715 : : /* Main data type to hold profile counters in GCC. Profile counts originate
716 : : either from profile feedback, static profile estimation or both. We do not
717 : : perform whole program profile propagation and thus profile estimation
718 : : counters are often local to function, while counters from profile feedback
719 : : (or special cases of profile estimation) can be used inter-procedurally.
720 : :
721 : : There are 3 basic types
722 : : 1) local counters which are result of intra-procedural static profile
723 : : estimation.
724 : : 2) ipa counters which are result of profile feedback or special case
725 : : of static profile estimation (such as in function main).
726 : : 3) counters which counts as 0 inter-procedurally (because given function
727 : : was never run in train feedback) but they hold local static profile
728 : : estimate.
729 : :
730 : : Counters of type 1 and 3 cannot be mixed with counters of different type
731 : : within operation (because whole function should use one type of counter)
732 : : with exception that global zero mix in most operations where outcome is
733 : : well defined.
734 : :
735 : : To take local counter and use it inter-procedurally use ipa member function
736 : : which strips information irrelevant at the inter-procedural level.
737 : :
738 : : Counters are 61bit integers representing number of executions during the
739 : : train run or normalized frequency within the function.
740 : :
741 : : As the profile is maintained during the compilation, many adjustments are
742 : : made. Not all transformations can be made precisely, most importantly
743 : : when code is being duplicated. It also may happen that part of CFG has
744 : : profile counts known while other do not - for example when LTO optimizing
745 : : partly profiled program or when profile was lost due to COMDAT merging.
746 : :
747 : : For this reason profile_count tracks more information than
748 : : just unsigned integer and it is also ready for profile mismatches.
749 : : The API of this data type represent operations that are natural
750 : : on profile counts - sum, difference and operation with scales and
751 : : probabilities. All operations are safe by never getting negative counts
752 : : and they do end up in uninitialized scale if any of the parameters is
753 : : uninitialized.
754 : :
755 : : All comparisons that are three state and handling of probabilities. Thus
756 : : a < b is not equal to !(a >= b).
757 : :
758 : : The following pre-defined counts are available:
759 : :
760 : : profile_count::zero () for code that is known to execute zero times at
761 : : runtime (this can be detected statically i.e. for paths leading to
762 : : abort ();
763 : : profile_count::one () for code that is known to execute once (such as
764 : : main () function
765 : : profile_count::uninitialized () for unknown execution count.
766 : :
767 : : */
768 : :
769 : : struct GTY(()) profile_count
770 : : {
771 : : public:
772 : : /* Use 60bit to hold basic block counters. Should be at least
773 : : 64bit. Although a counter cannot be negative, we use a signed
774 : : type to hold various extra stages. */
775 : :
776 : : static const int n_bits = 60;
777 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
778 : : private:
779 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
780 : :
781 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
782 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
783 : : incorrectly detects the alignment of a structure where the only
784 : : 64-bit aligned object is a bit-field. We force the alignment of
785 : : the entire field to mitigate this. */
786 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
787 : : #else
788 : : #define UINT64_BIT_FIELD_ALIGN
789 : : #endif
790 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
791 : : #undef UINT64_BIT_FIELD_ALIGN
792 : : enum profile_quality m_quality : 4;
793 : : public:
794 : :
795 : : /* Return true if both values can meaningfully appear in single function
796 : : body. We have either all counters in function local or global, otherwise
797 : : operations between them are not really defined well. */
798 : 4832293058 : bool compatible_p (const profile_count other) const
799 : : {
800 : 8876504742 : if (!initialized_p () || !other.initialized_p ())
801 : : return true;
802 : 16014997365 : if (*this == zero ()
803 : 7933854279 : || other == zero ())
804 : 147289899 : return true;
805 : : /* Do not allow nonzero global profile together with local guesses
806 : : that are globally0. */
807 : 3893281644 : if (ipa ().nonzero_p ()
808 : 244815 : && !(other.ipa () == other))
809 : 0 : return false;
810 : 3893281644 : if (other.ipa ().nonzero_p ()
811 : 254121 : && !(ipa () == *this))
812 : 0 : return false;
813 : :
814 : 3893281644 : return ipa_p () == other.ipa_p ();
815 : : }
816 : :
817 : : /* Used for counters which are expected to be never executed. */
818 : 17942344136 : static profile_count zero ()
819 : : {
820 : 13950122866 : return from_gcov_type (0);
821 : : }
822 : :
823 : 10527 : static profile_count adjusted_zero ()
824 : : {
825 : 10527 : profile_count c;
826 : 10527 : c.m_val = 0;
827 : 10527 : c.m_quality = ADJUSTED;
828 : 10527 : return c;
829 : : }
830 : :
831 : 0 : static profile_count afdo_zero ()
832 : : {
833 : 0 : profile_count c;
834 : 0 : c.m_val = 0;
835 : 0 : c.m_quality = AFDO;
836 : 0 : return c;
837 : : }
838 : :
839 : : static profile_count guessed_zero ()
840 : : {
841 : : profile_count c;
842 : : c.m_val = 0;
843 : : c.m_quality = GUESSED;
844 : : return c;
845 : : }
846 : :
847 : 48562 : static profile_count one ()
848 : : {
849 : 48365 : return from_gcov_type (1);
850 : : }
851 : :
852 : : /* Value of counters which has not been initialized. Either because
853 : : initialization did not happen yet or because profile is unknown. */
854 : 9134244056 : static profile_count uninitialized ()
855 : : {
856 : 9134244056 : profile_count c;
857 : 9134244056 : c.m_val = uninitialized_count;
858 : 9134244056 : c.m_quality = GUESSED_LOCAL;
859 : 9104102364 : return c;
860 : : }
861 : :
862 : : /* Conversion to gcov_type is lossy. */
863 : 299015 : gcov_type to_gcov_type () const
864 : : {
865 : 0 : gcc_checking_assert (initialized_p ());
866 : 299015 : return m_val;
867 : : }
868 : :
869 : : /* Return true if value has been initialized. */
870 : 31959782761 : bool initialized_p () const
871 : : {
872 : 10586008168 : return m_val != uninitialized_count;
873 : : }
874 : :
875 : : /* Return true if value can be trusted. */
876 : 13868678 : bool reliable_p () const
877 : : {
878 : 13868414 : return m_quality >= ADJUSTED;
879 : : }
880 : :
881 : : /* Return true if value can be operated inter-procedurally. */
882 : 8786884349 : bool ipa_p () const
883 : : {
884 : 4881332204 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
885 : : }
886 : :
887 : : /* Return true if quality of profile is precise. */
888 : 53063291 : bool precise_p () const
889 : : {
890 : 53063291 : return m_quality == PRECISE;
891 : : }
892 : :
893 : : /* Get the value of the count. */
894 : 0 : uint64_t value () const { return m_val; }
895 : :
896 : : /* Get the quality of the count. */
897 : 67276363 : enum profile_quality quality () const { return m_quality; }
898 : :
899 : : /* When merging basic blocks, the two different profile counts are unified.
900 : : Return true if this can be done without losing info about profile.
901 : : The only case we care about here is when first BB contains something
902 : : that makes it terminate in a way not visible in CFG. */
903 : 2938119 : bool ok_for_merging (profile_count other) const
904 : : {
905 : 2938119 : if (m_quality < ADJUSTED
906 : 10252 : || other.m_quality < ADJUSTED)
907 : : return true;
908 : 9467 : return !(other < *this);
909 : : }
910 : :
911 : : /* When merging two BBs with different counts, pick common count that looks
912 : : most representative. */
913 : 15197040 : profile_count merge (profile_count other) const
914 : : {
915 : 1112063 : if (*this == other || !other.initialized_p ()
916 : 1111822 : || m_quality > other.m_quality)
917 : 14086163 : return *this;
918 : 1110877 : if (other.m_quality > m_quality
919 : 1110877 : || other > *this)
920 : 438515 : return other;
921 : 672362 : return *this;
922 : : }
923 : :
924 : : /* Basic operations. */
925 : 19401583081 : bool operator== (const profile_count &other) const
926 : : {
927 : 14637816112 : return m_val == other.m_val && m_quality == other.m_quality;
928 : : }
929 : :
930 : 3643493 : profile_count operator+ (const profile_count &other) const
931 : : {
932 : 3643493 : if (other == zero ())
933 : 2235 : return *this;
934 : 3641258 : if (*this == zero ())
935 : 178100 : return other;
936 : 3463158 : if (!initialized_p () || !other.initialized_p ())
937 : 6050 : return uninitialized ();
938 : :
939 : 3457108 : profile_count ret;
940 : 3457108 : gcc_checking_assert (compatible_p (other));
941 : 3457108 : uint64_t ret_val = m_val + other.m_val;
942 : 3457108 : ret.m_val = MIN (ret_val, max_count);
943 : 3457108 : ret.m_quality = MIN (m_quality, other.m_quality);
944 : 3457108 : return ret;
945 : : }
946 : :
947 : 72293657 : profile_count &operator+= (const profile_count &other)
948 : : {
949 : 72293657 : if (other == zero ())
950 : 5099415 : return *this;
951 : 67194242 : if (*this == zero ())
952 : : {
953 : 41228795 : *this = other;
954 : 41228795 : return *this;
955 : : }
956 : 25965447 : if (!initialized_p () || !other.initialized_p ())
957 : 331705 : return *this = uninitialized ();
958 : : else
959 : : {
960 : 25633742 : gcc_checking_assert (compatible_p (other));
961 : 25633742 : uint64_t ret_val = m_val + other.m_val;
962 : 25633742 : m_val = MIN (ret_val, max_count);
963 : 25633742 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 25633742 : return *this;
966 : : }
967 : :
968 : 19174748 : profile_count operator- (const profile_count &other) const
969 : : {
970 : 19190930 : if (*this == zero () || other == zero ())
971 : 1173559 : return *this;
972 : 18001189 : if (!initialized_p () || !other.initialized_p ())
973 : 7728226 : return uninitialized ();
974 : 10272963 : gcc_checking_assert (compatible_p (other));
975 : 10272963 : profile_count ret;
976 : 10272963 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
977 : 10272963 : ret.m_quality = MIN (m_quality, other.m_quality);
978 : 10272963 : return ret;
979 : : }
980 : :
981 : 9172026 : profile_count &operator-= (const profile_count &other)
982 : : {
983 : 9613548 : if (*this == zero () || other == zero ())
984 : 312725 : return *this;
985 : 8859301 : if (!initialized_p () || !other.initialized_p ())
986 : 353850 : return *this = uninitialized ();
987 : : else
988 : : {
989 : 8505451 : gcc_checking_assert (compatible_p (other));
990 : 8505451 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
991 : 8505451 : m_quality = MIN (m_quality, other.m_quality);
992 : : }
993 : 8505451 : return *this;
994 : : }
995 : :
996 : : /* Return false if profile_count is bogus. */
997 : 3270008820 : bool verify () const
998 : : {
999 : 3270008820 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
1000 : 3270008820 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
1001 : : }
1002 : :
1003 : : /* Comparisons are three-state and conservative. False is returned if
1004 : : the inequality cannot be decided. */
1005 : 1045234334 : bool operator< (const profile_count &other) const
1006 : : {
1007 : 1045234334 : if (!initialized_p () || !other.initialized_p ())
1008 : : return false;
1009 : 1040669101 : if (*this == zero ())
1010 : 4021840 : return !(other == zero ());
1011 : 1038658181 : if (other == zero ())
1012 : 1051 : return false;
1013 : 1038657130 : gcc_checking_assert (compatible_p (other));
1014 : 1038657130 : return m_val < other.m_val;
1015 : : }
1016 : :
1017 : 34670791 : bool operator> (const profile_count &other) const
1018 : : {
1019 : 34670791 : if (!initialized_p () || !other.initialized_p ())
1020 : : return false;
1021 : 33719053 : if (*this == zero ())
1022 : 218145 : return false;
1023 : 33500908 : if (other == zero ())
1024 : 1609630 : return !(*this == zero ());
1025 : 32696093 : gcc_checking_assert (compatible_p (other));
1026 : 32696093 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
1027 : : }
1028 : :
1029 : : bool operator< (const gcov_type other) const
1030 : : {
1031 : : gcc_checking_assert (ipa_p ());
1032 : : gcc_checking_assert (other >= 0);
1033 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1034 : : }
1035 : :
1036 : 274560 : bool operator> (const gcov_type other) const
1037 : : {
1038 : 274560 : gcc_checking_assert (ipa_p ());
1039 : 274560 : gcc_checking_assert (other >= 0);
1040 : 274560 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1041 : : }
1042 : :
1043 : 1097643 : bool operator<= (const profile_count &other) const
1044 : : {
1045 : 1097643 : if (!initialized_p () || !other.initialized_p ())
1046 : : return false;
1047 : 1097643 : if (*this == zero ())
1048 : 3669 : return true;
1049 : 1093974 : if (other == zero ())
1050 : 0 : return (*this == zero ());
1051 : 1093974 : gcc_checking_assert (compatible_p (other));
1052 : 1093974 : return m_val <= other.m_val;
1053 : : }
1054 : :
1055 : 1372747 : bool operator>= (const profile_count &other) const
1056 : : {
1057 : 1372747 : if (!initialized_p () || !other.initialized_p ())
1058 : : return false;
1059 : 1371833 : if (other == zero ())
1060 : 292 : return true;
1061 : 1371541 : if (*this == zero ())
1062 : 25370 : return (other == zero ());
1063 : 1358856 : gcc_checking_assert (compatible_p (other));
1064 : 1358856 : return m_val >= other.m_val;
1065 : : }
1066 : :
1067 : 92794 : bool operator<= (const gcov_type other) const
1068 : : {
1069 : 92794 : gcc_checking_assert (ipa_p ());
1070 : 92794 : gcc_checking_assert (other >= 0);
1071 : 92794 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1072 : : }
1073 : :
1074 : 57884 : bool operator>= (const gcov_type other) const
1075 : : {
1076 : 57884 : gcc_checking_assert (ipa_p ());
1077 : 57884 : gcc_checking_assert (other >= 0);
1078 : 57884 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1079 : : }
1080 : :
1081 : 872457967 : profile_count operator* (int64_t num) const
1082 : : {
1083 : 872389941 : return apply_scale (num, 1);
1084 : : }
1085 : :
1086 : : profile_count operator*= (int64_t num)
1087 : : {
1088 : : *this = apply_scale (num, 1);
1089 : : return *this;
1090 : : }
1091 : :
1092 : : profile_count operator* (const sreal &num) const;
1093 : : profile_count operator*= (const sreal &num);
1094 : :
1095 : 2292739 : profile_count operator/ (int64_t den) const
1096 : : {
1097 : 2292739 : return apply_scale (1, den);
1098 : : }
1099 : :
1100 : 70 : profile_count operator/= (int64_t den)
1101 : : {
1102 : 62 : *this = apply_scale (1, den);
1103 : 70 : return *this;
1104 : : }
1105 : :
1106 : : /* Return true when value is not zero and can be used for scaling.
1107 : : This is different from *this > 0 because that requires counter to
1108 : : be IPA. */
1109 : 8151465335 : bool nonzero_p () const
1110 : : {
1111 : 7982997793 : return initialized_p () && m_val != 0;
1112 : : }
1113 : :
1114 : : /* Make counter forcibly nonzero. */
1115 : 19898193 : profile_count force_nonzero () const
1116 : : {
1117 : 19898193 : if (!initialized_p ())
1118 : 10977 : return *this;
1119 : 19887216 : profile_count ret = *this;
1120 : 19887216 : if (ret.m_val == 0)
1121 : : {
1122 : 21904 : ret.m_val = 1;
1123 : 21904 : ret.m_quality = MIN (m_quality, ADJUSTED);
1124 : : }
1125 : 19887216 : return ret;
1126 : : }
1127 : :
1128 : 89190838 : profile_count max (profile_count other) const
1129 : : {
1130 : 89190838 : profile_count val = *this;
1131 : :
1132 : : /* Always prefer nonzero IPA counts over local counts. */
1133 : 99089920 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1134 : : {
1135 : 9693 : val = ipa ();
1136 : 9693 : other = other.ipa ();
1137 : : }
1138 : 89190838 : if (!initialized_p ())
1139 : 21474926 : return other;
1140 : 67715912 : if (!other.initialized_p ())
1141 : 2168367 : return *this;
1142 : 65547545 : if (*this == zero ())
1143 : 2131956 : return other;
1144 : 63415589 : if (other == zero ())
1145 : 4602111 : return *this;
1146 : 58813478 : gcc_checking_assert (compatible_p (other));
1147 : 58813478 : if (val.m_val < other.m_val || (m_val == other.m_val
1148 : 13568744 : && val.m_quality < other.m_quality))
1149 : 8233301 : return other;
1150 : 50580177 : return *this;
1151 : : }
1152 : :
1153 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1154 : : accordingly. */
1155 : : profile_count apply_probability (int prob) const
1156 : : {
1157 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1158 : : if (m_val == 0)
1159 : : return *this;
1160 : : if (!initialized_p ())
1161 : : return uninitialized ();
1162 : : profile_count ret;
1163 : : uint64_t tmp;
1164 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1165 : : ret.m_val = tmp;
1166 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1167 : : return ret;
1168 : : }
1169 : :
1170 : : /* Scale counter according to PROB. */
1171 : 372772670 : profile_count apply_probability (profile_probability prob) const
1172 : : {
1173 : 380511765 : if (*this == zero () || prob == profile_probability::always ())
1174 : 118489323 : return *this;
1175 : 254283347 : if (prob == profile_probability::never ())
1176 : 19448637 : return zero ();
1177 : 234834710 : if (!initialized_p () || !prob.initialized_p ())
1178 : 30318752 : return uninitialized ();
1179 : 204515958 : profile_count ret;
1180 : 204515958 : uint64_t tmp;
1181 : 204515958 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1182 : : &tmp);
1183 : 204515958 : ret.m_val = tmp;
1184 : 204538992 : ret.m_quality = MIN (m_quality, prob.quality ());
1185 : 204515958 : return ret;
1186 : : }
1187 : :
1188 : : /* Return *THIS * NUM / DEN. */
1189 : 987381808 : profile_count apply_scale (int64_t num, int64_t den) const
1190 : : {
1191 : 987381808 : if (m_val == 0)
1192 : 23816715 : return *this;
1193 : 963565093 : if (!initialized_p ())
1194 : 7944 : return uninitialized ();
1195 : 963557149 : profile_count ret;
1196 : 963557149 : uint64_t tmp;
1197 : :
1198 : 963557149 : gcc_checking_assert (num >= 0 && den > 0);
1199 : 963557149 : safe_scale_64bit (m_val, num, den, &tmp);
1200 : 963557149 : ret.m_val = MIN (tmp, max_count);
1201 : 963557149 : ret.m_quality = MIN (m_quality, ADJUSTED);
1202 : 963557149 : return ret;
1203 : : }
1204 : :
1205 : 26632064 : profile_count apply_scale (profile_count num, profile_count den) const
1206 : : {
1207 : 26632064 : if (*this == zero ())
1208 : 2057217 : return *this;
1209 : 24574847 : if (num == zero ())
1210 : 76581 : return num;
1211 : 24498266 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1212 : 6824600 : return uninitialized ();
1213 : 17673666 : if (num == den)
1214 : 5309606 : return *this;
1215 : 12364060 : gcc_checking_assert (den.m_val);
1216 : :
1217 : 12364060 : profile_count ret;
1218 : 12364060 : uint64_t val;
1219 : 12364060 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1220 : 12364060 : ret.m_val = MIN (val, max_count);
1221 : 12364060 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1222 : : num.m_quality), den.m_quality);
1223 : : /* Be sure that ret is not local if num is global.
1224 : : Also ensure that ret is not global0 when num is global. */
1225 : 12364060 : if (num.ipa_p ())
1226 : 3600 : ret.m_quality = MAX (ret.m_quality,
1227 : : num == num.ipa () ? GUESSED : num.m_quality);
1228 : 12364060 : return ret;
1229 : : }
1230 : :
1231 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1232 : 17530733 : profile_count guessed_local () const
1233 : : {
1234 : 17530733 : profile_count ret = *this;
1235 : 17530733 : if (!initialized_p ())
1236 : 0 : return *this;
1237 : 17530733 : ret.m_quality = GUESSED_LOCAL;
1238 : 17530733 : return ret;
1239 : : }
1240 : :
1241 : : /* We know that profile is globally 0 but keep local profile if present. */
1242 : 788 : profile_count global0 () const
1243 : : {
1244 : 788 : profile_count ret = *this;
1245 : 788 : if (!initialized_p ())
1246 : 0 : return *this;
1247 : 788 : ret.m_quality = GUESSED_GLOBAL0;
1248 : 788 : return ret;
1249 : : }
1250 : :
1251 : : /* We know that profile is globally afdo 0 but keep local profile
1252 : : if present. */
1253 : 0 : profile_count global0afdo () const
1254 : : {
1255 : 0 : profile_count ret = *this;
1256 : 0 : if (!initialized_p ())
1257 : 0 : return *this;
1258 : 0 : ret.m_quality = GUESSED_GLOBAL0_AFDO;
1259 : 0 : return ret;
1260 : : }
1261 : :
1262 : : /* We know that profile is globally adjusted 0 but keep local profile
1263 : : if present. */
1264 : 36 : profile_count global0adjusted () const
1265 : : {
1266 : 36 : profile_count ret = *this;
1267 : 36 : if (!initialized_p ())
1268 : 0 : return *this;
1269 : 36 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1270 : 36 : return ret;
1271 : : }
1272 : :
1273 : : /* Return THIS with quality dropped to GUESSED. */
1274 : 293 : profile_count guessed () const
1275 : : {
1276 : 293 : profile_count ret = *this;
1277 : 293 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1278 : 293 : return ret;
1279 : : }
1280 : :
1281 : : /* Return THIS with quality GUESSED. */
1282 : 0 : profile_count force_guessed () const
1283 : : {
1284 : 0 : profile_count ret = *this;
1285 : 0 : gcc_checking_assert (initialized_p ());
1286 : 0 : ret.m_quality = GUESSED;
1287 : 0 : return ret;
1288 : : }
1289 : :
1290 : : /* Return variant of profile count which is always safe to compare
1291 : : across functions. */
1292 : 9015089908 : profile_count ipa () const
1293 : : {
1294 : 9015089908 : if (m_quality > GUESSED_GLOBAL0)
1295 : 22147107 : return *this;
1296 : 8992942801 : if (m_quality == GUESSED_GLOBAL0)
1297 : 220626 : return zero ();
1298 : 8992722175 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1299 : 10527 : return adjusted_zero ();
1300 : 8992711648 : if (m_quality == GUESSED_GLOBAL0_AFDO)
1301 : 0 : return afdo_zero ();
1302 : 8992711648 : return uninitialized ();
1303 : : }
1304 : :
1305 : : /* Return THIS with quality dropped to AFDO. */
1306 : 0 : profile_count afdo () const
1307 : : {
1308 : 0 : profile_count ret = *this;
1309 : 0 : ret.m_quality = AFDO;
1310 : 0 : return ret;
1311 : : }
1312 : :
1313 : : /* Return probability of event with counter THIS within event with counter
1314 : : OVERALL. */
1315 : 1072028407 : profile_probability probability_in (const profile_count overall) const
1316 : : {
1317 : 1072043114 : if (*this == zero ()
1318 : 14707 : && !(overall == zero ()))
1319 : 1594 : return profile_probability::never ();
1320 : 1072026255 : if (!initialized_p () || !overall.initialized_p ()
1321 : 2144053068 : || !overall.m_val)
1322 : 62439 : return profile_probability::uninitialized ();
1323 : 1281652876 : if (*this == overall && m_quality == PRECISE)
1324 : 29732 : return profile_probability::always ();
1325 : 1071934642 : profile_probability ret;
1326 : 1071934642 : gcc_checking_assert (compatible_p (overall));
1327 : :
1328 : 1071934642 : if (overall.m_val < m_val)
1329 : : {
1330 : 93850 : ret.m_val = profile_probability::max_probability;
1331 : 93850 : ret.set_quality (GUESSED);
1332 : 93850 : return ret;
1333 : : }
1334 : : else
1335 : 1071840792 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1336 : : overall.m_val);
1337 : 1071840792 : ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
1338 : : GUESSED), ADJUSTED));
1339 : 1071840792 : return ret;
1340 : : }
1341 : :
1342 : : /* Return true if profile count is very large, so we risk overflows
1343 : : with loop transformations. */
1344 : : bool
1345 : 1015839 : very_large_p ()
1346 : : {
1347 : 1015839 : if (!initialized_p ())
1348 : : return false;
1349 : 1015839 : return m_val > max_count / 65536;
1350 : : }
1351 : :
1352 : : int to_frequency (struct function *fun) const;
1353 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1354 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1355 : :
1356 : : /* Output THIS to F. */
1357 : : void dump (FILE *f, struct function *fun = NULL) const;
1358 : :
1359 : : /* Print THIS to stderr. */
1360 : : void debug () const;
1361 : :
1362 : : /* Return true if THIS is known to differ significantly from OTHER. */
1363 : : bool differs_from_p (profile_count other) const;
1364 : :
1365 : : /* We want to scale profile across function boundary from NUM to DEN.
1366 : : Take care of the side case when NUM and DEN are zeros of incompatible
1367 : : kinds. */
1368 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1369 : :
1370 : : /* THIS is a count of bb which is known to be executed IPA times.
1371 : : Combine this information into bb counter. This means returning IPA
1372 : : if it is nonzero, not changing anything if IPA is uninitialized
1373 : : and if IPA is zero, turning THIS into corresponding local profile with
1374 : : global0. */
1375 : : profile_count combine_with_ipa_count (profile_count ipa);
1376 : :
1377 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1378 : : profile_count combine_with_ipa_count_within
1379 : : (profile_count ipa, profile_count ipa2);
1380 : :
1381 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1382 : : Conversions back and forth are used to read the coverage and get it
1383 : : into internal representation. */
1384 : : static profile_count from_gcov_type (gcov_type v,
1385 : : profile_quality quality = PRECISE);
1386 : :
1387 : : /* LTO streaming support. */
1388 : : static profile_count stream_in (class lto_input_block *);
1389 : : void stream_out (struct output_block *);
1390 : : void stream_out (struct lto_output_stream *);
1391 : : };
1392 : : #endif
|