Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2025 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
41 : : better. This is the case of functions not run in profile feedback.
42 : : Never used by probabilities. */
43 : : GUESSED_GLOBAL0,
44 : :
45 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
46 : : GUESSED_GLOBAL0_ADJUSTED,
47 : :
48 : : /* Profile is based on static branch prediction heuristics. It may or may
49 : : not reflect the reality but it can be compared interprocedurally
50 : : (for example, we inlined function w/o profile feedback into function
51 : : with feedback and propagated from that).
52 : : Never used by probabilities. */
53 : : GUESSED,
54 : :
55 : : /* Profile was determined by autofdo. */
56 : : AFDO,
57 : :
58 : : /* Profile was originally based on feedback but it was adjusted
59 : : by code duplicating optimization. It may not precisely reflect the
60 : : particular code path. */
61 : : ADJUSTED,
62 : :
63 : : /* Profile was read from profile feedback or determined by accurate static
64 : : method. */
65 : : PRECISE
66 : : };
67 : :
68 : : extern const char *profile_quality_as_string (enum profile_quality);
69 : : extern bool parse_profile_quality (const char *value,
70 : : profile_quality *quality);
71 : :
72 : : /* The base value for branch probability notes and edge probabilities. */
73 : : #define REG_BR_PROB_BASE 10000
74 : :
75 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
76 : :
77 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
78 : :
79 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
80 : :
81 : : inline bool
82 : 1200989825 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
83 : : {
84 : : #if (GCC_VERSION >= 5000)
85 : 1200989825 : uint64_t tmp;
86 : 1200989825 : if (!__builtin_mul_overflow (a, b, &tmp)
87 : 1200989825 : && !__builtin_add_overflow (tmp, c/2, &tmp))
88 : : {
89 : 1200378607 : *res = tmp / c;
90 : 1200378607 : return true;
91 : : }
92 : 611218 : if (c == 1)
93 : : {
94 : 28115 : *res = (uint64_t) -1;
95 : 28115 : return false;
96 : : }
97 : : #else
98 : : if (a < ((uint64_t)1 << 31)
99 : : && b < ((uint64_t)1 << 31)
100 : : && c < ((uint64_t)1 << 31))
101 : : {
102 : : *res = (a * b + (c / 2)) / c;
103 : : return true;
104 : : }
105 : : #endif
106 : 583103 : return slow_safe_scale_64bit (a, b, c, res);
107 : : }
108 : :
109 : : /* Data type to hold probabilities. It implements fixed point arithmetics
110 : : with capping so probability is always in range [0,1] and scaling requiring
111 : : values greater than 1 needs to be represented otherwise.
112 : :
113 : : In addition to actual value the quality of profile is tracked and propagated
114 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
115 : : that has not been determined yet (for example because of
116 : : -fno-guess-branch-probability)
117 : :
118 : : Typically probabilities are derived from profile feedback (via
119 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
120 : : thorough the compilation.
121 : :
122 : : Named probabilities are available:
123 : : - never (0 probability)
124 : : - guessed_never
125 : : - very_unlikely (1/2000 probability)
126 : : - unlikely (1/5 probability)
127 : : - even (1/2 probability)
128 : : - likely (4/5 probability)
129 : : - very_likely (1999/2000 probability)
130 : : - guessed_always
131 : : - always
132 : :
133 : : Named probabilities except for never/always are assumed to be statically
134 : : guessed and thus not necessarily accurate. The difference between never
135 : : and guessed_never is that the first one should be used only in case that
136 : : well behaving program will very likely not execute the "never" path.
137 : : For example if the path is going to abort () call or it exception handling.
138 : :
139 : : Always and guessed_always probabilities are symmetric.
140 : :
141 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
142 : : integer arithmetics. Once the code is converted to branch probabilities,
143 : : these conversions will probably go away because they are lossy.
144 : : */
145 : :
146 : : class GTY((user)) profile_probability
147 : : {
148 : : static const int n_bits = 29;
149 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
150 : : will lead to harder multiplication sequences. */
151 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
152 : : static const uint32_t uninitialized_probability
153 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
154 : :
155 : : uint32_t m_val : 29;
156 : : enum profile_quality m_quality : 3;
157 : :
158 : : friend struct profile_count;
159 : : public:
160 : 2868195002 : profile_probability (): m_val (uninitialized_probability),
161 : 1291767922 : m_quality (GUESSED)
162 : : {}
163 : :
164 : 112 : profile_probability (uint32_t val, profile_quality quality):
165 : 112 : m_val (val), m_quality (quality)
166 : : {}
167 : :
168 : : /* Named probabilities. */
169 : 668975917 : static profile_probability never ()
170 : : {
171 : 668975917 : profile_probability ret;
172 : 668975917 : ret.m_val = 0;
173 : 668975917 : ret.m_quality = PRECISE;
174 : 176457576 : return ret;
175 : : }
176 : :
177 : 5216638 : static profile_probability guessed_never ()
178 : : {
179 : 5216638 : profile_probability ret;
180 : 5216638 : ret.m_val = 0;
181 : 5216638 : ret.m_quality = GUESSED;
182 : 5216638 : return ret;
183 : : }
184 : :
185 : 5148888 : static profile_probability very_unlikely ()
186 : : {
187 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
188 : 10297776 : profile_probability r = guessed_always () / 2000;
189 : 5148888 : r.m_val--;
190 : 5148888 : return r;
191 : : }
192 : :
193 : 80855 : static profile_probability unlikely ()
194 : : {
195 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
196 : 161710 : profile_probability r = guessed_always () / 5;
197 : 80855 : r.m_val--;
198 : 80855 : return r;
199 : : }
200 : :
201 : 789206 : static profile_probability even ()
202 : : {
203 : 1578412 : return guessed_always () / 2;
204 : : }
205 : :
206 : 335530 : static profile_probability very_likely ()
207 : : {
208 : 335530 : return always () - very_unlikely ();
209 : : }
210 : :
211 : 31821 : static profile_probability likely ()
212 : : {
213 : 31821 : return always () - unlikely ();
214 : : }
215 : : /* Return true when value is not zero and can be used for scaling. */
216 : 1441 : bool nonzero_p () const
217 : : {
218 : 1441 : return initialized_p () && m_val != 0;
219 : : }
220 : :
221 : 6323693 : static profile_probability guessed_always ()
222 : : {
223 : 6323693 : profile_probability ret;
224 : 6323693 : ret.m_val = max_probability;
225 : 6323693 : ret.m_quality = GUESSED;
226 : 6157334 : return ret;
227 : : }
228 : :
229 : 530941104 : static profile_probability always ()
230 : : {
231 : 530941104 : profile_probability ret;
232 : 530941104 : ret.m_val = max_probability;
233 : 530941104 : ret.m_quality = PRECISE;
234 : 354213473 : return ret;
235 : : }
236 : :
237 : : /* Probabilities which has not been initialized. Either because
238 : : initialization did not happen yet or because profile is unknown. */
239 : 895791114 : static profile_probability uninitialized ()
240 : : {
241 : 895791114 : profile_probability c;
242 : 895791114 : c.m_val = uninitialized_probability;
243 : 895791114 : c.m_quality = GUESSED;
244 : 876950774 : return c;
245 : : }
246 : :
247 : : /* Return true if value has been initialized. */
248 : 5635128313 : bool initialized_p () const
249 : : {
250 : 3878453358 : return m_val != uninitialized_probability;
251 : : }
252 : :
253 : : /* Return true if value can be trusted. */
254 : 1210 : bool reliable_p () const
255 : : {
256 : 1210 : return m_quality >= ADJUSTED;
257 : : }
258 : :
259 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
260 : : this is mostly to support legacy code and should go away. */
261 : 3102026 : static profile_probability from_reg_br_prob_base (int v)
262 : : {
263 : 3102026 : profile_probability ret;
264 : 3102026 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
265 : 3102026 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
266 : 3102026 : ret.m_quality = GUESSED;
267 : 3102026 : return ret;
268 : : }
269 : :
270 : : /* Return THIS with quality set to ADJUSTED. */
271 : 42 : profile_probability adjusted () const
272 : : {
273 : 42 : profile_probability ret = *this;
274 : 42 : if (!initialized_p ())
275 : 0 : return *this;
276 : 42 : ret.m_quality = ADJUSTED;
277 : 42 : return ret;
278 : : }
279 : :
280 : 1118039529 : int to_reg_br_prob_base () const
281 : : {
282 : 1118039529 : gcc_checking_assert (initialized_p ());
283 : 1118039529 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
284 : : }
285 : :
286 : : /* Conversion to and from RTL representation of profile probabilities. */
287 : 475873373 : static profile_probability from_reg_br_prob_note (int v)
288 : : {
289 : 475873373 : profile_probability ret;
290 : 475873373 : ret.m_val = ((unsigned int)v) / 8;
291 : 475873373 : ret.m_quality = (enum profile_quality)(v & 7);
292 : 15560575 : return ret;
293 : : }
294 : :
295 : 447182489 : int to_reg_br_prob_note () const
296 : : {
297 : 447182489 : gcc_checking_assert (initialized_p ());
298 : 447182489 : int ret = m_val * 8 + m_quality;
299 : 894364978 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
300 : 447182489 : return ret;
301 : : }
302 : :
303 : : /* Return VAL1/VAL2. */
304 : 3068 : static profile_probability probability_in_gcov_type
305 : : (gcov_type val1, gcov_type val2)
306 : : {
307 : 3068 : profile_probability ret;
308 : 3068 : gcc_checking_assert (val1 >= 0 && val2 > 0);
309 : 3068 : if (val1 > val2)
310 : : ret.m_val = max_probability;
311 : : else
312 : : {
313 : 3068 : uint64_t tmp;
314 : 3068 : safe_scale_64bit (val1, max_probability, val2, &tmp);
315 : 3068 : gcc_checking_assert (tmp <= max_probability);
316 : 3068 : ret.m_val = tmp;
317 : : }
318 : 3068 : ret.m_quality = PRECISE;
319 : 3068 : return ret;
320 : : }
321 : :
322 : : /* Basic operations. */
323 : 1513999699 : bool operator== (const profile_probability &other) const
324 : : {
325 : 901363832 : return m_val == other.m_val && m_quality == other.m_quality;
326 : : }
327 : :
328 : 6599186 : profile_probability operator+ (const profile_probability &other) const
329 : : {
330 : 6599186 : if (other == never ())
331 : 32155 : return *this;
332 : 6567031 : if (*this == never ())
333 : 24661 : return other;
334 : 6542370 : if (!initialized_p () || !other.initialized_p ())
335 : 5366246 : return uninitialized ();
336 : :
337 : 1176124 : profile_probability ret;
338 : 1176124 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
339 : 1176124 : ret.m_quality = MIN (m_quality, other.m_quality);
340 : 1176124 : return ret;
341 : : }
342 : :
343 : 6212578 : profile_probability &operator+= (const profile_probability &other)
344 : : {
345 : 6212578 : if (other == never ())
346 : : return *this;
347 : 5023004 : if (*this == never ())
348 : : {
349 : 863931 : *this = other;
350 : 863931 : return *this;
351 : : }
352 : 4159073 : if (!initialized_p () || !other.initialized_p ())
353 : 2897395 : return *this = uninitialized ();
354 : : else
355 : : {
356 : 1261678 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
357 : 1261678 : m_quality = MIN (m_quality, other.m_quality);
358 : : }
359 : 1261678 : return *this;
360 : : }
361 : :
362 : 33752994 : profile_probability operator- (const profile_probability &other) const
363 : : {
364 : 33752994 : if (*this == never ()
365 : 33653432 : || other == never ())
366 : 1814082 : return *this;
367 : 31938912 : if (!initialized_p () || !other.initialized_p ())
368 : 2143744 : return uninitialized ();
369 : 29795168 : profile_probability ret;
370 : 29795168 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
371 : 29795168 : ret.m_quality = MIN (m_quality, other.m_quality);
372 : 29795168 : return ret;
373 : : }
374 : :
375 : 11578417 : profile_probability &operator-= (const profile_probability &other)
376 : : {
377 : 11578417 : if (*this == never ()
378 : 11442848 : || other == never ())
379 : : return *this;
380 : 9270183 : if (!initialized_p () || !other.initialized_p ())
381 : 2130340 : return *this = uninitialized ();
382 : : else
383 : : {
384 : 7139843 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
385 : 7139843 : m_quality = MIN (m_quality, other.m_quality);
386 : : }
387 : 7139843 : return *this;
388 : : }
389 : :
390 : 1174599 : profile_probability operator* (const profile_probability &other) const
391 : : {
392 : 1174599 : if (*this == never ()
393 : 1070806 : || other == never ())
394 : 120294 : return never ();
395 : 1054305 : if (!initialized_p () || !other.initialized_p ())
396 : 57800 : return uninitialized ();
397 : 996505 : profile_probability ret;
398 : 996505 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
399 : 996505 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
400 : 996505 : return ret;
401 : : }
402 : :
403 : 155681 : profile_probability &operator*= (const profile_probability &other)
404 : : {
405 : 155681 : if (*this == never ()
406 : 155557 : || other == never ())
407 : 339 : return *this = never ();
408 : 155342 : if (!initialized_p () || !other.initialized_p ())
409 : 9 : return *this = uninitialized ();
410 : : else
411 : : {
412 : 155333 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
413 : 155333 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
414 : : }
415 : 155333 : return *this;
416 : : }
417 : :
418 : 1745537 : profile_probability operator/ (const profile_probability &other) const
419 : : {
420 : 1745537 : if (*this == never ())
421 : 109536 : return never ();
422 : 1636001 : if (!initialized_p () || !other.initialized_p ())
423 : 224366 : return uninitialized ();
424 : 1411635 : profile_probability ret;
425 : : /* If we get probability above 1, mark it as unreliable and return 1. */
426 : 1411635 : if (m_val >= other.m_val)
427 : : {
428 : 33257 : ret.m_val = max_probability;
429 : 33257 : ret.m_quality = MIN (MIN (m_quality, other.m_quality),
430 : : GUESSED);
431 : 33257 : return ret;
432 : : }
433 : 1378378 : else if (!m_val)
434 : : ret.m_val = 0;
435 : : else
436 : : {
437 : 1317944 : gcc_checking_assert (other.m_val);
438 : 1317944 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
439 : : other.m_val),
440 : : max_probability);
441 : : }
442 : 1378378 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
443 : 1378378 : return ret;
444 : : }
445 : :
446 : 348941 : profile_probability &operator/= (const profile_probability &other)
447 : : {
448 : 348941 : if (*this == never ())
449 : 1418 : return *this = never ();
450 : 347523 : if (!initialized_p () || !other.initialized_p ())
451 : 0 : return *this = uninitialized ();
452 : : else
453 : : {
454 : : /* If we get probability above 1, mark it as unreliable
455 : : and return 1. */
456 : 347523 : if (m_val > other.m_val)
457 : : {
458 : 629 : m_val = max_probability;
459 : 629 : m_quality = MIN (MIN (m_quality, other.m_quality),
460 : : GUESSED);
461 : 629 : return *this;
462 : : }
463 : 346894 : else if (!m_val)
464 : : ;
465 : : else
466 : : {
467 : 345084 : gcc_checking_assert (other.m_val);
468 : 345084 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
469 : : other.m_val),
470 : : max_probability);
471 : : }
472 : 346894 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
473 : : }
474 : 346894 : return *this;
475 : : }
476 : :
477 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
478 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
479 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
480 : : == ORIG. This is useful e.g. when splitting a conditional
481 : : branch like:
482 : : if (cond)
483 : : goto lab; // ORIG probability
484 : : into
485 : : if (cond1)
486 : : goto lab; // FIRST = ORIG * CPROB probability
487 : : if (cond2)
488 : : goto lab; // SECOND probability
489 : : such that the overall probability of jumping to lab remains
490 : : the same. CPROB gives the relative probability between the
491 : : branches. */
492 : 289752 : profile_probability split (const profile_probability &cprob)
493 : : {
494 : 289752 : profile_probability ret = *this * cprob;
495 : : /* The following is equivalent to:
496 : : *this = cprob.invert () * *this / ret.invert ();
497 : : Avoid scaling when overall outcome is supposed to be always.
498 : : Without knowing that one is inverse of other, the result would be
499 : : conservative. */
500 : 289752 : if (!(*this == always ()))
501 : 281067 : *this = (*this - ret) / ret.invert ();
502 : 289752 : return ret;
503 : : }
504 : :
505 : 259100 : gcov_type apply (gcov_type val) const
506 : : {
507 : 259100 : if (*this == uninitialized ())
508 : 63 : return val / 2;
509 : 259037 : return RDIV (val * m_val, max_probability);
510 : : }
511 : :
512 : : /* Return 1-*THIS. */
513 : 30569073 : profile_probability invert () const
514 : : {
515 : 30561011 : return always() - *this;
516 : : }
517 : :
518 : : /* Return THIS with quality dropped to GUESSED. */
519 : 880642 : profile_probability guessed () const
520 : : {
521 : 880642 : profile_probability ret = *this;
522 : 880642 : ret.m_quality = GUESSED;
523 : 880642 : return ret;
524 : : }
525 : :
526 : : /* Return THIS with quality dropped to AFDO. */
527 : : profile_probability afdo () const
528 : : {
529 : : profile_probability ret = *this;
530 : : ret.m_quality = AFDO;
531 : : return ret;
532 : : }
533 : :
534 : : /* Return *THIS * NUM / DEN. */
535 : 14419072 : profile_probability apply_scale (int64_t num, int64_t den) const
536 : : {
537 : 14419072 : if (*this == never ())
538 : 24194 : return *this;
539 : 14394878 : if (!initialized_p ())
540 : 5766671 : return uninitialized ();
541 : 8628207 : profile_probability ret;
542 : 8628207 : uint64_t tmp;
543 : 8628207 : safe_scale_64bit (m_val, num, den, &tmp);
544 : 8628207 : ret.m_val = MIN (tmp, max_probability);
545 : 8628207 : ret.m_quality = MIN (m_quality, ADJUSTED);
546 : 8628207 : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 7235 : profile_probability apply_scale (profile_probability num,
551 : : profile_probability den) const
552 : : {
553 : 7235 : if (*this == never ())
554 : 56 : return *this;
555 : 7179 : if (num == never ())
556 : 0 : return num;
557 : 7179 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
558 : 0 : return uninitialized ();
559 : 7179 : if (num == den)
560 : 0 : return *this;
561 : 7179 : gcc_checking_assert (den.m_val);
562 : :
563 : 7179 : profile_probability ret;
564 : 7179 : uint64_t val;
565 : 7179 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
566 : 7179 : ret.m_val = MIN (val, max_probability);
567 : 7179 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
568 : : num.m_quality), den.m_quality);
569 : 7179 : return ret;
570 : : }
571 : :
572 : : /* Return true when the probability of edge is reliable.
573 : :
574 : : The profile guessing code is good at predicting branch outcome (i.e.
575 : : taken/not taken), that is predicted right slightly over 75% of time.
576 : : It is however notoriously poor on predicting the probability itself.
577 : : In general the profile appear a lot flatter (with probabilities closer
578 : : to 50%) than the reality so it is bad idea to use it to drive optimization
579 : : such as those disabling dynamic branch prediction for well predictable
580 : : branches.
581 : :
582 : : There are two exceptions - edges leading to noreturn edges and edges
583 : : predicted by number of iterations heuristics are predicted well. This macro
584 : : should be able to distinguish those, but at the moment it simply check for
585 : : noreturn heuristic that is only one giving probability over 99% or bellow
586 : : 1%. In future we might want to propagate reliability information across the
587 : : CFG if we find this information useful on multiple places. */
588 : 0 : bool probably_reliable_p () const
589 : : {
590 : 0 : if (m_quality >= ADJUSTED)
591 : : return true;
592 : 0 : if (!initialized_p ())
593 : : return false;
594 : 0 : return m_val < max_probability / 100
595 : 0 : || m_val > max_probability - max_probability / 100;
596 : : }
597 : :
598 : : /* Return false if profile_probability is bogus. */
599 : 4323088526 : bool verify () const
600 : : {
601 : 4323088526 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
602 : 4323088526 : if (m_val == uninitialized_probability)
603 : 938885633 : return m_quality == GUESSED;
604 : 3384202893 : else if (m_quality < GUESSED)
605 : : return false;
606 : 3384202893 : return m_val <= max_probability;
607 : : }
608 : :
609 : : /* Comparisons are three-state and conservative. False is returned if
610 : : the inequality cannot be decided. */
611 : 6217415 : bool operator< (const profile_probability &other) const
612 : : {
613 : 6217415 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
614 : : }
615 : :
616 : 17630799 : bool operator> (const profile_probability &other) const
617 : : {
618 : 17630799 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
619 : : }
620 : :
621 : 5274250 : bool operator<= (const profile_probability &other) const
622 : : {
623 : 5274250 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
624 : : }
625 : :
626 : 949339 : bool operator>= (const profile_probability &other) const
627 : : {
628 : 949339 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
629 : : }
630 : :
631 : 17 : profile_probability operator* (int64_t num) const
632 : : {
633 : 17 : return apply_scale (num, 1);
634 : : }
635 : :
636 : : profile_probability operator*= (int64_t num)
637 : : {
638 : : *this = apply_scale (num, 1);
639 : : return *this;
640 : : }
641 : :
642 : 13752686 : profile_probability operator/ (int64_t den) const
643 : : {
644 : 13752686 : return apply_scale (1, den);
645 : : }
646 : :
647 : 245440 : profile_probability operator/= (int64_t den)
648 : : {
649 : 245440 : *this = apply_scale (1, den);
650 : 245440 : return *this;
651 : : }
652 : :
653 : : /* Compute n-th power. */
654 : : profile_probability pow (int) const;
655 : :
656 : : /* Compute sware root. */
657 : : profile_probability sqrt () const;
658 : :
659 : : /* Get the value of the count. */
660 : 0 : uint32_t value () const { return m_val; }
661 : :
662 : : /* Get the quality of the count. */
663 : 0 : enum profile_quality quality () const { return m_quality; }
664 : :
665 : : /* Output THIS to F. */
666 : : void dump (FILE *f) const;
667 : :
668 : : /* Output THIS to BUFFER. */
669 : : void dump (char *buffer) const;
670 : :
671 : : /* Print THIS to stderr. */
672 : : void debug () const;
673 : :
674 : : /* Return true if THIS is known to differ significantly from OTHER. */
675 : : bool differs_from_p (profile_probability other) const;
676 : :
677 : : /* Return if difference is greater than 50%. */
678 : : bool differs_lot_from_p (profile_probability other) const;
679 : :
680 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
681 : : happens with COUNT2 probability. Return probability that either *THIS or
682 : : OTHER happens. */
683 : : profile_probability combine_with_count (profile_count count1,
684 : : profile_probability other,
685 : : profile_count count2) const;
686 : :
687 : : /* Return probability as sreal. */
688 : : sreal to_sreal () const;
689 : : /* LTO streaming support. */
690 : : static profile_probability stream_in (class lto_input_block *);
691 : : void stream_out (struct output_block *);
692 : : void stream_out (struct lto_output_stream *);
693 : : };
694 : :
695 : : /* Main data type to hold profile counters in GCC. Profile counts originate
696 : : either from profile feedback, static profile estimation or both. We do not
697 : : perform whole program profile propagation and thus profile estimation
698 : : counters are often local to function, while counters from profile feedback
699 : : (or special cases of profile estimation) can be used inter-procedurally.
700 : :
701 : : There are 3 basic types
702 : : 1) local counters which are result of intra-procedural static profile
703 : : estimation.
704 : : 2) ipa counters which are result of profile feedback or special case
705 : : of static profile estimation (such as in function main).
706 : : 3) counters which counts as 0 inter-procedurally (because given function
707 : : was never run in train feedback) but they hold local static profile
708 : : estimate.
709 : :
710 : : Counters of type 1 and 3 cannot be mixed with counters of different type
711 : : within operation (because whole function should use one type of counter)
712 : : with exception that global zero mix in most operations where outcome is
713 : : well defined.
714 : :
715 : : To take local counter and use it inter-procedurally use ipa member function
716 : : which strips information irrelevant at the inter-procedural level.
717 : :
718 : : Counters are 61bit integers representing number of executions during the
719 : : train run or normalized frequency within the function.
720 : :
721 : : As the profile is maintained during the compilation, many adjustments are
722 : : made. Not all transformations can be made precisely, most importantly
723 : : when code is being duplicated. It also may happen that part of CFG has
724 : : profile counts known while other do not - for example when LTO optimizing
725 : : partly profiled program or when profile was lost due to COMDAT merging.
726 : :
727 : : For this reason profile_count tracks more information than
728 : : just unsigned integer and it is also ready for profile mismatches.
729 : : The API of this data type represent operations that are natural
730 : : on profile counts - sum, difference and operation with scales and
731 : : probabilities. All operations are safe by never getting negative counts
732 : : and they do end up in uninitialized scale if any of the parameters is
733 : : uninitialized.
734 : :
735 : : All comparisons that are three state and handling of probabilities. Thus
736 : : a < b is not equal to !(a >= b).
737 : :
738 : : The following pre-defined counts are available:
739 : :
740 : : profile_count::zero () for code that is known to execute zero times at
741 : : runtime (this can be detected statically i.e. for paths leading to
742 : : abort ();
743 : : profile_count::one () for code that is known to execute once (such as
744 : : main () function
745 : : profile_count::uninitialized () for unknown execution count.
746 : :
747 : : */
748 : :
749 : : struct GTY(()) profile_count
750 : : {
751 : : public:
752 : : /* Use 62bit to hold basic block counters. Should be at least
753 : : 64bit. Although a counter cannot be negative, we use a signed
754 : : type to hold various extra stages. */
755 : :
756 : : static const int n_bits = 61;
757 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
758 : : private:
759 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
760 : :
761 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
762 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
763 : : incorrectly detects the alignment of a structure where the only
764 : : 64-bit aligned object is a bit-field. We force the alignment of
765 : : the entire field to mitigate this. */
766 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
767 : : #else
768 : : #define UINT64_BIT_FIELD_ALIGN
769 : : #endif
770 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
771 : : #undef UINT64_BIT_FIELD_ALIGN
772 : : enum profile_quality m_quality : 3;
773 : : public:
774 : :
775 : : /* Return true if both values can meaningfully appear in single function
776 : : body. We have either all counters in function local or global, otherwise
777 : : operations between them are not really defined well. */
778 : 4878489215 : bool compatible_p (const profile_count other) const
779 : : {
780 : 8963247839 : if (!initialized_p () || !other.initialized_p ())
781 : : return true;
782 : 16172689096 : if (*this == zero ()
783 : 8011425210 : || other == zero ())
784 : 149839750 : return true;
785 : : /* Do not allow nonzero global profile together with local guesses
786 : : that are globally0. */
787 : 3930792193 : if (ipa ().nonzero_p ()
788 : 244591 : && !(other.ipa () == other))
789 : 0 : return false;
790 : 3930792193 : if (other.ipa ().nonzero_p ()
791 : 254404 : && !(ipa () == *this))
792 : 0 : return false;
793 : :
794 : 3930792193 : return ipa_p () == other.ipa_p ();
795 : : }
796 : :
797 : : /* Used for counters which are expected to be never executed. */
798 : 18114973308 : static profile_count zero ()
799 : : {
800 : 14082036283 : return from_gcov_type (0);
801 : : }
802 : :
803 : 10535 : static profile_count adjusted_zero ()
804 : : {
805 : 10535 : profile_count c;
806 : 10535 : c.m_val = 0;
807 : 10535 : c.m_quality = ADJUSTED;
808 : 10535 : return c;
809 : : }
810 : :
811 : : static profile_count guessed_zero ()
812 : : {
813 : : profile_count c;
814 : : c.m_val = 0;
815 : : c.m_quality = GUESSED;
816 : : return c;
817 : : }
818 : :
819 : 560 : static profile_count one ()
820 : : {
821 : 363 : return from_gcov_type (1);
822 : : }
823 : :
824 : : /* Value of counters which has not been initialized. Either because
825 : : initialization did not happen yet or because profile is unknown. */
826 : 9222139102 : static profile_count uninitialized ()
827 : : {
828 : 9222139102 : profile_count c;
829 : 9222139102 : c.m_val = uninitialized_count;
830 : 9222139102 : c.m_quality = GUESSED_LOCAL;
831 : 9191710168 : return c;
832 : : }
833 : :
834 : : /* Conversion to gcov_type is lossy. */
835 : 299200 : gcov_type to_gcov_type () const
836 : : {
837 : 0 : gcc_checking_assert (initialized_p ());
838 : 299200 : return m_val;
839 : : }
840 : :
841 : : /* Return true if value has been initialized. */
842 : 32277991291 : bool initialized_p () const
843 : : {
844 : 10690661186 : return m_val != uninitialized_count;
845 : : }
846 : :
847 : : /* Return true if value can be trusted. */
848 : 13989661 : bool reliable_p () const
849 : : {
850 : 13989405 : return m_quality >= ADJUSTED;
851 : : }
852 : :
853 : : /* Return true if value can be operated inter-procedurally. */
854 : 8872786960 : bool ipa_p () const
855 : : {
856 : 4929498067 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
857 : : }
858 : :
859 : : /* Return true if quality of profile is precise. */
860 : 53590249 : bool precise_p () const
861 : : {
862 : 53590249 : return m_quality == PRECISE;
863 : : }
864 : :
865 : : /* Get the value of the count. */
866 : 0 : uint64_t value () const { return m_val; }
867 : :
868 : : /* Get the quality of the count. */
869 : 67482427 : enum profile_quality quality () const { return m_quality; }
870 : :
871 : : /* When merging basic blocks, the two different profile counts are unified.
872 : : Return true if this can be done without losing info about profile.
873 : : The only case we care about here is when first BB contains something
874 : : that makes it terminate in a way not visible in CFG. */
875 : 2955972 : bool ok_for_merging (profile_count other) const
876 : : {
877 : 2955972 : if (m_quality < ADJUSTED
878 : 10333 : || other.m_quality < ADJUSTED)
879 : : return true;
880 : 9548 : return !(other < *this);
881 : : }
882 : :
883 : : /* When merging two BBs with different counts, pick common count that looks
884 : : most representative. */
885 : 15365907 : profile_count merge (profile_count other) const
886 : : {
887 : 1135704 : if (*this == other || !other.initialized_p ()
888 : 1135463 : || m_quality > other.m_quality)
889 : 14231388 : return *this;
890 : 1134519 : if (other.m_quality > m_quality
891 : 1134519 : || other > *this)
892 : 457032 : return other;
893 : 677487 : return *this;
894 : : }
895 : :
896 : : /* Basic operations. */
897 : 19592135066 : bool operator== (const profile_count &other) const
898 : : {
899 : 14775772636 : return m_val == other.m_val && m_quality == other.m_quality;
900 : : }
901 : :
902 : 3661378 : profile_count operator+ (const profile_count &other) const
903 : : {
904 : 3661378 : if (other == zero ())
905 : 2243 : return *this;
906 : 3659135 : if (*this == zero ())
907 : 175206 : return other;
908 : 3483929 : if (!initialized_p () || !other.initialized_p ())
909 : 5239 : return uninitialized ();
910 : :
911 : 3478690 : profile_count ret;
912 : 3478690 : gcc_checking_assert (compatible_p (other));
913 : 3478690 : uint64_t ret_val = m_val + other.m_val;
914 : 3478690 : ret.m_val = MIN (ret_val, max_count);
915 : 3478690 : ret.m_quality = MIN (m_quality, other.m_quality);
916 : 3478690 : return ret;
917 : : }
918 : :
919 : 72926510 : profile_count &operator+= (const profile_count &other)
920 : : {
921 : 72926510 : if (other == zero ())
922 : 5171860 : return *this;
923 : 67754650 : if (*this == zero ())
924 : : {
925 : 41450445 : *this = other;
926 : 41450445 : return *this;
927 : : }
928 : 26304205 : if (!initialized_p () || !other.initialized_p ())
929 : 337651 : return *this = uninitialized ();
930 : : else
931 : : {
932 : 25966554 : gcc_checking_assert (compatible_p (other));
933 : 25966554 : uint64_t ret_val = m_val + other.m_val;
934 : 25966554 : m_val = MIN (ret_val, max_count);
935 : 25966554 : m_quality = MIN (m_quality, other.m_quality);
936 : : }
937 : 25966554 : return *this;
938 : : }
939 : :
940 : 19326832 : profile_count operator- (const profile_count &other) const
941 : : {
942 : 19343022 : if (*this == zero () || other == zero ())
943 : 1220558 : return *this;
944 : 18106274 : if (!initialized_p () || !other.initialized_p ())
945 : 7848918 : return uninitialized ();
946 : 10257356 : gcc_checking_assert (compatible_p (other));
947 : 10257356 : profile_count ret;
948 : 10257356 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
949 : 10257356 : ret.m_quality = MIN (m_quality, other.m_quality);
950 : 10257356 : return ret;
951 : : }
952 : :
953 : 9300075 : profile_count &operator-= (const profile_count &other)
954 : : {
955 : 9745944 : if (*this == zero () || other == zero ())
956 : 325945 : return *this;
957 : 8974130 : if (!initialized_p () || !other.initialized_p ())
958 : 352145 : return *this = uninitialized ();
959 : : else
960 : : {
961 : 8621985 : gcc_checking_assert (compatible_p (other));
962 : 8621985 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
963 : 8621985 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 8621985 : return *this;
966 : : }
967 : :
968 : : /* Return false if profile_count is bogus. */
969 : 3298782954 : bool verify () const
970 : : {
971 : 3298782954 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
972 : 3298782954 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
973 : : }
974 : :
975 : : /* Comparisons are three-state and conservative. False is returned if
976 : : the inequality cannot be decided. */
977 : 1055240620 : bool operator< (const profile_count &other) const
978 : : {
979 : 1055240620 : if (!initialized_p () || !other.initialized_p ())
980 : : return false;
981 : 1050666648 : if (*this == zero ())
982 : 4118848 : return !(other == zero ());
983 : 1048607224 : if (other == zero ())
984 : 1052 : return false;
985 : 1048606172 : gcc_checking_assert (compatible_p (other));
986 : 1048606172 : return m_val < other.m_val;
987 : : }
988 : :
989 : 35070687 : bool operator> (const profile_count &other) const
990 : : {
991 : 35070687 : if (!initialized_p () || !other.initialized_p ())
992 : : return false;
993 : 34118901 : if (*this == zero ())
994 : 219428 : return false;
995 : 33899473 : if (other == zero ())
996 : 1612734 : return !(*this == zero ());
997 : 33093106 : gcc_checking_assert (compatible_p (other));
998 : 33093106 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
999 : : }
1000 : :
1001 : : bool operator< (const gcov_type other) const
1002 : : {
1003 : : gcc_checking_assert (ipa_p ());
1004 : : gcc_checking_assert (other >= 0);
1005 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1006 : : }
1007 : :
1008 : 273028 : bool operator> (const gcov_type other) const
1009 : : {
1010 : 273028 : gcc_checking_assert (ipa_p ());
1011 : 273028 : gcc_checking_assert (other >= 0);
1012 : 273028 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1013 : : }
1014 : :
1015 : 1106887 : bool operator<= (const profile_count &other) const
1016 : : {
1017 : 1106887 : if (!initialized_p () || !other.initialized_p ())
1018 : : return false;
1019 : 1106887 : if (*this == zero ())
1020 : 4803 : return true;
1021 : 1102084 : if (other == zero ())
1022 : 0 : return (*this == zero ());
1023 : 1102084 : gcc_checking_assert (compatible_p (other));
1024 : 1102084 : return m_val <= other.m_val;
1025 : : }
1026 : :
1027 : 1386614 : bool operator>= (const profile_count &other) const
1028 : : {
1029 : 1386614 : if (!initialized_p () || !other.initialized_p ())
1030 : : return false;
1031 : 1385700 : if (other == zero ())
1032 : 286 : return true;
1033 : 1385414 : if (*this == zero ())
1034 : 27826 : return (other == zero ());
1035 : 1371501 : gcc_checking_assert (compatible_p (other));
1036 : 1371501 : return m_val >= other.m_val;
1037 : : }
1038 : :
1039 : 93833 : bool operator<= (const gcov_type other) const
1040 : : {
1041 : 93833 : gcc_checking_assert (ipa_p ());
1042 : 93833 : gcc_checking_assert (other >= 0);
1043 : 93833 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1044 : : }
1045 : :
1046 : 58434 : bool operator>= (const gcov_type other) const
1047 : : {
1048 : 58434 : gcc_checking_assert (ipa_p ());
1049 : 58434 : gcc_checking_assert (other >= 0);
1050 : 58434 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1051 : : }
1052 : :
1053 : 882081191 : profile_count operator* (int64_t num) const
1054 : : {
1055 : 882013271 : return apply_scale (num, 1);
1056 : : }
1057 : :
1058 : : profile_count operator*= (int64_t num)
1059 : : {
1060 : : *this = apply_scale (num, 1);
1061 : : return *this;
1062 : : }
1063 : :
1064 : : profile_count operator* (const sreal &num) const;
1065 : : profile_count operator*= (const sreal &num);
1066 : :
1067 : 2303451 : profile_count operator/ (int64_t den) const
1068 : : {
1069 : 2303451 : return apply_scale (1, den);
1070 : : }
1071 : :
1072 : 70 : profile_count operator/= (int64_t den)
1073 : : {
1074 : 62 : *this = apply_scale (1, den);
1075 : 70 : return *this;
1076 : : }
1077 : :
1078 : : /* Return true when value is not zero and can be used for scaling.
1079 : : This is different from *this > 0 because that requires counter to
1080 : : be IPA. */
1081 : 8230195442 : bool nonzero_p () const
1082 : : {
1083 : 8060388726 : return initialized_p () && m_val != 0;
1084 : : }
1085 : :
1086 : : /* Make counter forcibly nonzero. */
1087 : 20323381 : profile_count force_nonzero () const
1088 : : {
1089 : 20323381 : if (!initialized_p ())
1090 : 41929 : return *this;
1091 : 20281452 : profile_count ret = *this;
1092 : 20281452 : if (ret.m_val == 0)
1093 : : {
1094 : 83808 : ret.m_val = 1;
1095 : 83808 : ret.m_quality = MIN (m_quality, ADJUSTED);
1096 : : }
1097 : 20281452 : return ret;
1098 : : }
1099 : :
1100 : 90006764 : profile_count max (profile_count other) const
1101 : : {
1102 : 90006764 : profile_count val = *this;
1103 : :
1104 : : /* Always prefer nonzero IPA counts over local counts. */
1105 : 100198595 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1106 : : {
1107 : 9653 : val = ipa ();
1108 : 9653 : other = other.ipa ();
1109 : : }
1110 : 90006764 : if (!initialized_p ())
1111 : 21639532 : return other;
1112 : 68367232 : if (!other.initialized_p ())
1113 : 2316162 : return *this;
1114 : 66051070 : if (*this == zero ())
1115 : 2165978 : return other;
1116 : 63885092 : if (other == zero ())
1117 : 4659243 : return *this;
1118 : 59225849 : gcc_checking_assert (compatible_p (other));
1119 : 59225849 : if (val.m_val < other.m_val || (m_val == other.m_val
1120 : 13589756 : && val.m_quality < other.m_quality))
1121 : 8257852 : return other;
1122 : 50967997 : return *this;
1123 : : }
1124 : :
1125 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1126 : : accordingly. */
1127 : : profile_count apply_probability (int prob) const
1128 : : {
1129 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1130 : : if (m_val == 0)
1131 : : return *this;
1132 : : if (!initialized_p ())
1133 : : return uninitialized ();
1134 : : profile_count ret;
1135 : : uint64_t tmp;
1136 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1137 : : ret.m_val = tmp;
1138 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1139 : : return ret;
1140 : : }
1141 : :
1142 : : /* Scale counter according to PROB. */
1143 : 376643296 : profile_count apply_probability (profile_probability prob) const
1144 : : {
1145 : 384464064 : if (*this == zero () || prob == profile_probability::always ())
1146 : 120404393 : return *this;
1147 : 256238903 : if (prob == profile_probability::never ())
1148 : 19573170 : return zero ();
1149 : 236665733 : if (!initialized_p () || !prob.initialized_p ())
1150 : 30450834 : return uninitialized ();
1151 : 206214899 : profile_count ret;
1152 : 206214899 : uint64_t tmp;
1153 : 206214899 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1154 : : &tmp);
1155 : 206214899 : ret.m_val = tmp;
1156 : 206214899 : ret.m_quality = MIN (m_quality, prob.m_quality);
1157 : 206214899 : return ret;
1158 : : }
1159 : :
1160 : : /* Return *THIS * NUM / DEN. */
1161 : 996907393 : profile_count apply_scale (int64_t num, int64_t den) const
1162 : : {
1163 : 996907393 : if (m_val == 0)
1164 : 23843239 : return *this;
1165 : 973064154 : if (!initialized_p ())
1166 : 7077 : return uninitialized ();
1167 : 973057077 : profile_count ret;
1168 : 973057077 : uint64_t tmp;
1169 : :
1170 : 973057077 : gcc_checking_assert (num >= 0 && den > 0);
1171 : 973057077 : safe_scale_64bit (m_val, num, den, &tmp);
1172 : 973057077 : ret.m_val = MIN (tmp, max_count);
1173 : 973057077 : ret.m_quality = MIN (m_quality, ADJUSTED);
1174 : 973057077 : return ret;
1175 : : }
1176 : :
1177 : 27059747 : profile_count apply_scale (profile_count num, profile_count den) const
1178 : : {
1179 : 27059747 : if (*this == zero ())
1180 : 2158023 : return *this;
1181 : 24901724 : if (num == zero ())
1182 : 78016 : return num;
1183 : 24823708 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1184 : 6876764 : return uninitialized ();
1185 : 17946944 : if (num == den)
1186 : 5356140 : return *this;
1187 : 12590804 : gcc_checking_assert (den.m_val);
1188 : :
1189 : 12590804 : profile_count ret;
1190 : 12590804 : uint64_t val;
1191 : 12590804 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1192 : 12590804 : ret.m_val = MIN (val, max_count);
1193 : 12590804 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1194 : : num.m_quality), den.m_quality);
1195 : : /* Be sure that ret is not local if num is global.
1196 : : Also ensure that ret is not global0 when num is global. */
1197 : 12590804 : if (num.ipa_p ())
1198 : 3570 : ret.m_quality = MAX (ret.m_quality,
1199 : : num == num.ipa () ? GUESSED : num.m_quality);
1200 : 12590804 : return ret;
1201 : : }
1202 : :
1203 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1204 : 17611341 : profile_count guessed_local () const
1205 : : {
1206 : 17611341 : profile_count ret = *this;
1207 : 17611341 : if (!initialized_p ())
1208 : 0 : return *this;
1209 : 17611341 : ret.m_quality = GUESSED_LOCAL;
1210 : 17611341 : return ret;
1211 : : }
1212 : :
1213 : : /* We know that profile is globally 0 but keep local profile if present. */
1214 : 790 : profile_count global0 () const
1215 : : {
1216 : 790 : profile_count ret = *this;
1217 : 790 : if (!initialized_p ())
1218 : 0 : return *this;
1219 : 790 : ret.m_quality = GUESSED_GLOBAL0;
1220 : 790 : return ret;
1221 : : }
1222 : :
1223 : : /* We know that profile is globally adjusted 0 but keep local profile
1224 : : if present. */
1225 : 39 : profile_count global0adjusted () const
1226 : : {
1227 : 39 : profile_count ret = *this;
1228 : 39 : if (!initialized_p ())
1229 : 0 : return *this;
1230 : 39 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1231 : 39 : return ret;
1232 : : }
1233 : :
1234 : : /* Return THIS with quality dropped to GUESSED. */
1235 : 293 : profile_count guessed () const
1236 : : {
1237 : 293 : profile_count ret = *this;
1238 : 293 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1239 : 293 : return ret;
1240 : : }
1241 : :
1242 : : /* Return THIS with quality GUESSED. */
1243 : 0 : profile_count force_guessed () const
1244 : : {
1245 : 0 : profile_count ret = *this;
1246 : 0 : gcc_checking_assert (initialized_p ());
1247 : 0 : ret.m_quality = GUESSED;
1248 : 0 : return ret;
1249 : : }
1250 : :
1251 : : /* Return variant of profile count which is always safe to compare
1252 : : across functions. */
1253 : 9103468707 : profile_count ipa () const
1254 : : {
1255 : 9103468707 : if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
1256 : 22792278 : return *this;
1257 : 9080676429 : if (m_quality == GUESSED_GLOBAL0)
1258 : 217428 : return zero ();
1259 : 9080459001 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1260 : 10535 : return adjusted_zero ();
1261 : 9080448466 : return uninitialized ();
1262 : : }
1263 : :
1264 : : /* Return THIS with quality dropped to AFDO. */
1265 : 0 : profile_count afdo () const
1266 : : {
1267 : 0 : profile_count ret = *this;
1268 : 0 : ret.m_quality = AFDO;
1269 : 0 : return ret;
1270 : : }
1271 : :
1272 : : /* Return probability of event with counter THIS within event with counter
1273 : : OVERALL. */
1274 : 1084475244 : profile_probability probability_in (const profile_count overall) const
1275 : : {
1276 : 1084490515 : if (*this == zero ()
1277 : 15271 : && !(overall == zero ()))
1278 : 1570 : return profile_probability::never ();
1279 : 1084473116 : if (!initialized_p () || !overall.initialized_p ()
1280 : 2168946790 : || !overall.m_val)
1281 : 64201 : return profile_probability::uninitialized ();
1282 : 1294391776 : if (*this == overall && m_quality == PRECISE)
1283 : 29775 : return profile_probability::always ();
1284 : 1084379698 : profile_probability ret;
1285 : 1084379698 : gcc_checking_assert (compatible_p (overall));
1286 : :
1287 : 1084379698 : if (overall.m_val < m_val)
1288 : : {
1289 : 93634 : ret.m_val = profile_probability::max_probability;
1290 : 93634 : ret.m_quality = GUESSED;
1291 : 93634 : return ret;
1292 : : }
1293 : : else
1294 : 1084286064 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1295 : : overall.m_val);
1296 : 1084286064 : ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
1297 : : GUESSED), ADJUSTED);
1298 : 1084286064 : return ret;
1299 : : }
1300 : :
1301 : : /* Return true if profile count is very large, so we risk overflows
1302 : : with loop transformations. */
1303 : : bool
1304 : 1016329 : very_large_p ()
1305 : : {
1306 : 1016329 : if (!initialized_p ())
1307 : : return false;
1308 : 1016329 : return m_val > max_count / 65536;
1309 : : }
1310 : :
1311 : : int to_frequency (struct function *fun) const;
1312 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1313 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1314 : :
1315 : : /* Output THIS to F. */
1316 : : void dump (FILE *f, struct function *fun = NULL) const;
1317 : :
1318 : : /* Print THIS to stderr. */
1319 : : void debug () const;
1320 : :
1321 : : /* Return true if THIS is known to differ significantly from OTHER. */
1322 : : bool differs_from_p (profile_count other) const;
1323 : :
1324 : : /* We want to scale profile across function boundary from NUM to DEN.
1325 : : Take care of the side case when NUM and DEN are zeros of incompatible
1326 : : kinds. */
1327 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1328 : :
1329 : : /* THIS is a count of bb which is known to be executed IPA times.
1330 : : Combine this information into bb counter. This means returning IPA
1331 : : if it is nonzero, not changing anything if IPA is uninitialized
1332 : : and if IPA is zero, turning THIS into corresponding local profile with
1333 : : global0. */
1334 : : profile_count combine_with_ipa_count (profile_count ipa);
1335 : :
1336 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1337 : : profile_count combine_with_ipa_count_within
1338 : : (profile_count ipa, profile_count ipa2);
1339 : :
1340 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1341 : : Conversions back and forth are used to read the coverage and get it
1342 : : into internal representation. */
1343 : : static profile_count from_gcov_type (gcov_type v,
1344 : : profile_quality quality = PRECISE);
1345 : :
1346 : : /* LTO streaming support. */
1347 : : static profile_count stream_in (class lto_input_block *);
1348 : : void stream_out (struct output_block *);
1349 : : void stream_out (struct lto_output_stream *);
1350 : : };
1351 : : #endif
|
