Branch data Line data Source code
1 : : /* Profile counter container type.
2 : : Copyright (C) 2017-2024 Free Software Foundation, Inc.
3 : : Contributed by Jan Hubicka
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #ifndef GCC_PROFILE_COUNT_H
22 : : #define GCC_PROFILE_COUNT_H
23 : :
24 : : struct function;
25 : : struct profile_count;
26 : : class sreal;
27 : :
28 : : /* Quality of the profile count. Because gengtype does not support enums
29 : : inside of classes, this is in global namespace. */
30 : : enum profile_quality {
31 : : /* Uninitialized value. */
32 : : UNINITIALIZED_PROFILE,
33 : :
34 : : /* Profile is based on static branch prediction heuristics and may
35 : : or may not match reality. It is local to function and cannot be compared
36 : : inter-procedurally. Never used by probabilities (they are always local).
37 : : */
38 : : GUESSED_LOCAL,
39 : :
40 : : /* Profile was read by feedback and was 0, we used local heuristics to guess
41 : : better. This is the case of functions not run in profile feedback.
42 : : Never used by probabilities. */
43 : : GUESSED_GLOBAL0,
44 : :
45 : : /* Same as GUESSED_GLOBAL0 but global count is adjusted 0. */
46 : : GUESSED_GLOBAL0_ADJUSTED,
47 : :
48 : : /* Profile is based on static branch prediction heuristics. It may or may
49 : : not reflect the reality but it can be compared interprocedurally
50 : : (for example, we inlined function w/o profile feedback into function
51 : : with feedback and propagated from that).
52 : : Never used by probabilities. */
53 : : GUESSED,
54 : :
55 : : /* Profile was determined by autofdo. */
56 : : AFDO,
57 : :
58 : : /* Profile was originally based on feedback but it was adjusted
59 : : by code duplicating optimization. It may not precisely reflect the
60 : : particular code path. */
61 : : ADJUSTED,
62 : :
63 : : /* Profile was read from profile feedback or determined by accurate static
64 : : method. */
65 : : PRECISE
66 : : };
67 : :
68 : : extern const char *profile_quality_as_string (enum profile_quality);
69 : : extern bool parse_profile_quality (const char *value,
70 : : profile_quality *quality);
71 : :
72 : : /* The base value for branch probability notes and edge probabilities. */
73 : : #define REG_BR_PROB_BASE 10000
74 : :
75 : : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
76 : :
77 : : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
78 : :
79 : : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened. */
80 : :
81 : : inline bool
82 : 980235269 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
83 : : {
84 : : #if (GCC_VERSION >= 5000)
85 : 980235269 : uint64_t tmp;
86 : 980235269 : if (!__builtin_mul_overflow (a, b, &tmp)
87 : 980235269 : && !__builtin_add_overflow (tmp, c/2, &tmp))
88 : : {
89 : 979678064 : *res = tmp / c;
90 : 979678064 : return true;
91 : : }
92 : 557205 : if (c == 1)
93 : : {
94 : 30728 : *res = (uint64_t) -1;
95 : 30728 : return false;
96 : : }
97 : : #else
98 : : if (a < ((uint64_t)1 << 31)
99 : : && b < ((uint64_t)1 << 31)
100 : : && c < ((uint64_t)1 << 31))
101 : : {
102 : : *res = (a * b + (c / 2)) / c;
103 : : return true;
104 : : }
105 : : #endif
106 : 526477 : return slow_safe_scale_64bit (a, b, c, res);
107 : : }
108 : :
109 : : /* Data type to hold probabilities. It implements fixed point arithmetics
110 : : with capping so probability is always in range [0,1] and scaling requiring
111 : : values greater than 1 needs to be represented otherwise.
112 : :
113 : : In addition to actual value the quality of profile is tracked and propagated
114 : : through all operations. Special value UNINITIALIZED_PROFILE is used for probabilities
115 : : that has not been determined yet (for example because of
116 : : -fno-guess-branch-probability)
117 : :
118 : : Typically probabilities are derived from profile feedback (via
119 : : probability_in_gcov_type), autoFDO or guessed statically and then propagated
120 : : thorough the compilation.
121 : :
122 : : Named probabilities are available:
123 : : - never (0 probability)
124 : : - guessed_never
125 : : - very_unlikely (1/2000 probability)
126 : : - unlikely (1/5 probability)
127 : : - even (1/2 probability)
128 : : - likely (4/5 probability)
129 : : - very_likely (1999/2000 probability)
130 : : - guessed_always
131 : : - always
132 : :
133 : : Named probabilities except for never/always are assumed to be statically
134 : : guessed and thus not necessarily accurate. The difference between never
135 : : and guessed_never is that the first one should be used only in case that
136 : : well behaving program will very likely not execute the "never" path.
137 : : For example if the path is going to abort () call or it exception handling.
138 : :
139 : : Always and guessed_always probabilities are symmetric.
140 : :
141 : : For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
142 : : integer arithmetics. Once the code is converted to branch probabilities,
143 : : these conversions will probably go away because they are lossy.
144 : : */
145 : :
146 : : class GTY((user)) profile_probability
147 : : {
148 : : static const int n_bits = 29;
149 : : /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
150 : : will lead to harder multiplication sequences. */
151 : : static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
152 : : static const uint32_t uninitialized_probability
153 : : = ((uint32_t) 1 << (n_bits - 1)) - 1;
154 : :
155 : : uint32_t m_val : 29;
156 : : enum profile_quality m_quality : 3;
157 : :
158 : : friend struct profile_count;
159 : : public:
160 : 3475103794 : profile_probability (): m_val (uninitialized_probability),
161 : 1857649623 : m_quality (GUESSED)
162 : : {}
163 : :
164 : 112 : profile_probability (uint32_t val, profile_quality quality):
165 : 112 : m_val (val), m_quality (quality)
166 : : {}
167 : :
168 : : /* Named probabilities. */
169 : 566720130 : static profile_probability never ()
170 : : {
171 : 566720130 : profile_probability ret;
172 : 566720130 : ret.m_val = 0;
173 : 566720130 : ret.m_quality = PRECISE;
174 : 152229421 : return ret;
175 : : }
176 : :
177 : 4633014 : static profile_probability guessed_never ()
178 : : {
179 : 4633014 : profile_probability ret;
180 : 4633014 : ret.m_val = 0;
181 : 4633014 : ret.m_quality = GUESSED;
182 : 4633014 : return ret;
183 : : }
184 : :
185 : 4277915 : static profile_probability very_unlikely ()
186 : : {
187 : : /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
188 : 8555830 : profile_probability r = guessed_always () / 2000;
189 : 4277915 : r.m_val--;
190 : 4277915 : return r;
191 : : }
192 : :
193 : 78770 : static profile_probability unlikely ()
194 : : {
195 : : /* Be consistent with PROB_VERY_LIKELY in predict.h. */
196 : 157540 : profile_probability r = guessed_always () / 5;
197 : 78770 : r.m_val--;
198 : 78770 : return r;
199 : : }
200 : :
201 : 686037 : static profile_probability even ()
202 : : {
203 : 1372074 : return guessed_always () / 2;
204 : : }
205 : :
206 : 336154 : static profile_probability very_likely ()
207 : : {
208 : 336154 : return always () - very_unlikely ();
209 : : }
210 : :
211 : 30834 : static profile_probability likely ()
212 : : {
213 : 30834 : return always () - unlikely ();
214 : : }
215 : : /* Return true when value is not zero and can be used for scaling. */
216 : 1255 : bool nonzero_p () const
217 : : {
218 : 1255 : return initialized_p () && m_val != 0;
219 : : }
220 : :
221 : 5324932 : static profile_probability guessed_always ()
222 : : {
223 : 5324932 : profile_probability ret;
224 : 5324932 : ret.m_val = max_probability;
225 : 5324932 : ret.m_quality = GUESSED;
226 : 5126725 : return ret;
227 : : }
228 : :
229 : 455916565 : static profile_probability always ()
230 : : {
231 : 455916565 : profile_probability ret;
232 : 455916565 : ret.m_val = max_probability;
233 : 455916565 : ret.m_quality = PRECISE;
234 : 294052471 : return ret;
235 : : }
236 : :
237 : : /* Probabilities which has not been initialized. Either because
238 : : initialization did not happen yet or because profile is unknown. */
239 : 769465047 : static profile_probability uninitialized ()
240 : : {
241 : 769465047 : profile_probability c;
242 : 769465047 : c.m_val = uninitialized_probability;
243 : 769465047 : c.m_quality = GUESSED;
244 : 756733429 : return c;
245 : : }
246 : :
247 : : /* Return true if value has been initialized. */
248 : 4802311405 : bool initialized_p () const
249 : : {
250 : 3314124569 : return m_val != uninitialized_probability;
251 : : }
252 : :
253 : : /* Return true if value can be trusted. */
254 : 396 : bool reliable_p () const
255 : : {
256 : 396 : return m_quality >= ADJUSTED;
257 : : }
258 : :
259 : : /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
260 : : this is mostly to support legacy code and should go away. */
261 : 2834178 : static profile_probability from_reg_br_prob_base (int v)
262 : : {
263 : 2834178 : profile_probability ret;
264 : 2834178 : gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
265 : 2834178 : ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
266 : 2834178 : ret.m_quality = GUESSED;
267 : 2834178 : return ret;
268 : : }
269 : :
270 : : /* Return THIS with quality set to ADJUSTED. */
271 : 35 : profile_probability adjusted () const
272 : : {
273 : 35 : profile_probability ret = *this;
274 : 35 : if (!initialized_p ())
275 : 0 : return *this;
276 : 35 : ret.m_quality = ADJUSTED;
277 : 35 : return ret;
278 : : }
279 : :
280 : 941800465 : int to_reg_br_prob_base () const
281 : : {
282 : 941800465 : gcc_checking_assert (initialized_p ());
283 : 941800465 : return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
284 : : }
285 : :
286 : : /* Conversion to and from RTL representation of profile probabilities. */
287 : 406735063 : static profile_probability from_reg_br_prob_note (int v)
288 : : {
289 : 406735063 : profile_probability ret;
290 : 406735063 : ret.m_val = ((unsigned int)v) / 8;
291 : 406735063 : ret.m_quality = (enum profile_quality)(v & 7);
292 : 13957802 : return ret;
293 : : }
294 : :
295 : 381003481 : int to_reg_br_prob_note () const
296 : : {
297 : 381003481 : gcc_checking_assert (initialized_p ());
298 : 381003481 : int ret = m_val * 8 + m_quality;
299 : 762006962 : gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
300 : 381003481 : return ret;
301 : : }
302 : :
303 : : /* Return VAL1/VAL2. */
304 : 3046 : static profile_probability probability_in_gcov_type
305 : : (gcov_type val1, gcov_type val2)
306 : : {
307 : 3046 : profile_probability ret;
308 : 3046 : gcc_checking_assert (val1 >= 0 && val2 > 0);
309 : 3046 : if (val1 > val2)
310 : : ret.m_val = max_probability;
311 : : else
312 : : {
313 : 3046 : uint64_t tmp;
314 : 3046 : safe_scale_64bit (val1, max_probability, val2, &tmp);
315 : 3046 : gcc_checking_assert (tmp <= max_probability);
316 : 3046 : ret.m_val = tmp;
317 : : }
318 : 3046 : ret.m_quality = PRECISE;
319 : 3046 : return ret;
320 : : }
321 : :
322 : : /* Basic operations. */
323 : 1285612430 : bool operator== (const profile_probability &other) const
324 : : {
325 : 771815670 : return m_val == other.m_val && m_quality == other.m_quality;
326 : : }
327 : :
328 : 5651449 : profile_probability operator+ (const profile_probability &other) const
329 : : {
330 : 5651449 : if (other == never ())
331 : 24562 : return *this;
332 : 5626887 : if (*this == never ())
333 : 3409 : return other;
334 : 5623478 : if (!initialized_p () || !other.initialized_p ())
335 : 4631626 : return uninitialized ();
336 : :
337 : 991852 : profile_probability ret;
338 : 991852 : ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
339 : 991852 : ret.m_quality = MIN (m_quality, other.m_quality);
340 : 991852 : return ret;
341 : : }
342 : :
343 : 3378076 : profile_probability &operator+= (const profile_probability &other)
344 : : {
345 : 3378076 : if (other == never ())
346 : : return *this;
347 : 2566992 : if (*this == never ())
348 : : {
349 : 691792 : *this = other;
350 : 691792 : return *this;
351 : : }
352 : 1875200 : if (!initialized_p () || !other.initialized_p ())
353 : 805628 : return *this = uninitialized ();
354 : : else
355 : : {
356 : 1069572 : m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
357 : 1069572 : m_quality = MIN (m_quality, other.m_quality);
358 : : }
359 : 1069572 : return *this;
360 : : }
361 : :
362 : 29920249 : profile_probability operator- (const profile_probability &other) const
363 : : {
364 : 29920249 : if (*this == never ()
365 : 29821118 : || other == never ())
366 : 1651195 : return *this;
367 : 28269054 : if (!initialized_p () || !other.initialized_p ())
368 : 1804053 : return uninitialized ();
369 : 26465001 : profile_probability ret;
370 : 26465001 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
371 : 26465001 : ret.m_quality = MIN (m_quality, other.m_quality);
372 : 26465001 : return ret;
373 : : }
374 : :
375 : 8290444 : profile_probability &operator-= (const profile_probability &other)
376 : : {
377 : 8290444 : if (*this == never ()
378 : 8180875 : || other == never ())
379 : 1878672 : return *this;
380 : 6411772 : if (!initialized_p () || !other.initialized_p ())
381 : 19521 : return *this = uninitialized ();
382 : : else
383 : : {
384 : 6392251 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
385 : 6392251 : m_quality = MIN (m_quality, other.m_quality);
386 : : }
387 : 6392251 : return *this;
388 : : }
389 : :
390 : 1000947 : profile_probability operator* (const profile_probability &other) const
391 : : {
392 : 1000947 : if (*this == never ()
393 : 899453 : || other == never ())
394 : 104301 : return never ();
395 : 896646 : if (!initialized_p () || !other.initialized_p ())
396 : 57744 : return uninitialized ();
397 : 838902 : profile_probability ret;
398 : 838902 : ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
399 : 838902 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
400 : 838902 : return ret;
401 : : }
402 : :
403 : 38952 : profile_probability &operator*= (const profile_probability &other)
404 : : {
405 : 38952 : if (*this == never ()
406 : 38947 : || other == never ())
407 : 5 : return *this = never ();
408 : 38947 : if (!initialized_p () || !other.initialized_p ())
409 : 0 : return *this = uninitialized ();
410 : : else
411 : : {
412 : 38947 : m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
413 : 38947 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
414 : : }
415 : 38947 : return *this;
416 : : }
417 : :
418 : 1346456 : profile_probability operator/ (const profile_probability &other) const
419 : : {
420 : 1346456 : if (*this == never ())
421 : 104946 : return never ();
422 : 1241510 : if (!initialized_p () || !other.initialized_p ())
423 : 73719 : return uninitialized ();
424 : 1167791 : profile_probability ret;
425 : : /* If we get probability above 1, mark it as unreliable and return 1. */
426 : 1167791 : if (m_val >= other.m_val)
427 : : {
428 : 25564 : ret.m_val = max_probability;
429 : 25564 : ret.m_quality = MIN (MIN (m_quality, other.m_quality),
430 : : GUESSED);
431 : 25564 : return ret;
432 : : }
433 : 1142227 : else if (!m_val)
434 : : ret.m_val = 0;
435 : : else
436 : : {
437 : 1090261 : gcc_checking_assert (other.m_val);
438 : 1090261 : ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
439 : : other.m_val),
440 : : max_probability);
441 : : }
442 : 1142227 : ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
443 : 1142227 : return ret;
444 : : }
445 : :
446 : 338515 : profile_probability &operator/= (const profile_probability &other)
447 : : {
448 : 338515 : if (*this == never ())
449 : 2029 : return *this = never ();
450 : 336486 : if (!initialized_p () || !other.initialized_p ())
451 : 0 : return *this = uninitialized ();
452 : : else
453 : : {
454 : : /* If we get probability above 1, mark it as unreliable
455 : : and return 1. */
456 : 336486 : if (m_val > other.m_val)
457 : : {
458 : 565 : m_val = max_probability;
459 : 565 : m_quality = MIN (MIN (m_quality, other.m_quality),
460 : : GUESSED);
461 : 565 : return *this;
462 : : }
463 : 335921 : else if (!m_val)
464 : : ;
465 : : else
466 : : {
467 : 334340 : gcc_checking_assert (other.m_val);
468 : 334340 : m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
469 : : other.m_val),
470 : : max_probability);
471 : : }
472 : 335921 : m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
473 : : }
474 : 335921 : return *this;
475 : : }
476 : :
477 : : /* Split *THIS (ORIG) probability into 2 probabilities, such that
478 : : the returned one (FIRST) is *THIS * CPROB and *THIS is
479 : : adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
480 : : == ORIG. This is useful e.g. when splitting a conditional
481 : : branch like:
482 : : if (cond)
483 : : goto lab; // ORIG probability
484 : : into
485 : : if (cond1)
486 : : goto lab; // FIRST = ORIG * CPROB probability
487 : : if (cond2)
488 : : goto lab; // SECOND probability
489 : : such that the overall probability of jumping to lab remains
490 : : the same. CPROB gives the relative probability between the
491 : : branches. */
492 : 270921 : profile_probability split (const profile_probability &cprob)
493 : : {
494 : 270921 : profile_probability ret = *this * cprob;
495 : : /* The following is equivalent to:
496 : : *this = cprob.invert () * *this / ret.invert ();
497 : : Avoid scaling when overall outcome is supposed to be always.
498 : : Without knowing that one is inverse of other, the result would be
499 : : conservative. */
500 : 270921 : if (!(*this == always ()))
501 : 262569 : *this = (*this - ret) / ret.invert ();
502 : 270921 : return ret;
503 : : }
504 : :
505 : 267842 : gcov_type apply (gcov_type val) const
506 : : {
507 : 267842 : if (*this == uninitialized ())
508 : 63 : return val / 2;
509 : 267779 : return RDIV (val * m_val, max_probability);
510 : : }
511 : :
512 : : /* Return 1-*THIS. */
513 : 27340718 : profile_probability invert () const
514 : : {
515 : 27333178 : return always() - *this;
516 : : }
517 : :
518 : : /* Return THIS with quality dropped to GUESSED. */
519 : 183021 : profile_probability guessed () const
520 : : {
521 : 183021 : profile_probability ret = *this;
522 : 183021 : ret.m_quality = GUESSED;
523 : 183021 : return ret;
524 : : }
525 : :
526 : : /* Return THIS with quality dropped to AFDO. */
527 : : profile_probability afdo () const
528 : : {
529 : : profile_probability ret = *this;
530 : : ret.m_quality = AFDO;
531 : : return ret;
532 : : }
533 : :
534 : : /* Return *THIS * NUM / DEN. */
535 : 12407473 : profile_probability apply_scale (int64_t num, int64_t den) const
536 : : {
537 : 12407473 : if (*this == never ())
538 : 23723 : return *this;
539 : 12383750 : if (!initialized_p ())
540 : 4969495 : return uninitialized ();
541 : 7414255 : profile_probability ret;
542 : 7414255 : uint64_t tmp;
543 : 7414255 : safe_scale_64bit (m_val, num, den, &tmp);
544 : 7414255 : ret.m_val = MIN (tmp, max_probability);
545 : 7414255 : ret.m_quality = MIN (m_quality, ADJUSTED);
546 : 7414255 : return ret;
547 : : }
548 : :
549 : : /* Return *THIS * NUM / DEN. */
550 : 6552 : profile_probability apply_scale (profile_probability num,
551 : : profile_probability den) const
552 : : {
553 : 6552 : if (*this == never ())
554 : 27 : return *this;
555 : 6525 : if (num == never ())
556 : 0 : return num;
557 : 6525 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
558 : 0 : return uninitialized ();
559 : 6525 : if (num == den)
560 : 0 : return *this;
561 : 6525 : gcc_checking_assert (den.m_val);
562 : :
563 : 6525 : profile_probability ret;
564 : 6525 : uint64_t val;
565 : 6525 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
566 : 6525 : ret.m_val = MIN (val, max_probability);
567 : 6525 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
568 : : num.m_quality), den.m_quality);
569 : 6525 : return ret;
570 : : }
571 : :
572 : : /* Return true when the probability of edge is reliable.
573 : :
574 : : The profile guessing code is good at predicting branch outcome (i.e.
575 : : taken/not taken), that is predicted right slightly over 75% of time.
576 : : It is however notoriously poor on predicting the probability itself.
577 : : In general the profile appear a lot flatter (with probabilities closer
578 : : to 50%) than the reality so it is bad idea to use it to drive optimization
579 : : such as those disabling dynamic branch prediction for well predictable
580 : : branches.
581 : :
582 : : There are two exceptions - edges leading to noreturn edges and edges
583 : : predicted by number of iterations heuristics are predicted well. This macro
584 : : should be able to distinguish those, but at the moment it simply check for
585 : : noreturn heuristic that is only one giving probability over 99% or bellow
586 : : 1%. In future we might want to propagate reliability information across the
587 : : CFG if we find this information useful on multiple places. */
588 : 0 : bool probably_reliable_p () const
589 : : {
590 : 0 : if (m_quality >= ADJUSTED)
591 : : return true;
592 : 0 : if (!initialized_p ())
593 : : return false;
594 : 0 : return m_val < max_probability / 100
595 : 0 : || m_val > max_probability - max_probability / 100;
596 : : }
597 : :
598 : : /* Return false if profile_probability is bogus. */
599 : 3772827718 : bool verify () const
600 : : {
601 : 3772827718 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
602 : 3772827718 : if (m_val == uninitialized_probability)
603 : 864203050 : return m_quality == GUESSED;
604 : 2908624668 : else if (m_quality < GUESSED)
605 : : return false;
606 : 2908624668 : return m_val <= max_probability;
607 : : }
608 : :
609 : : /* Comparisons are three-state and conservative. False is returned if
610 : : the inequality cannot be decided. */
611 : 3593739 : bool operator< (const profile_probability &other) const
612 : : {
613 : 3593739 : return initialized_p () && other.initialized_p () && m_val < other.m_val;
614 : : }
615 : :
616 : 15336770 : bool operator> (const profile_probability &other) const
617 : : {
618 : 15336770 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
619 : : }
620 : :
621 : 4303046 : bool operator<= (const profile_probability &other) const
622 : : {
623 : 4303046 : return initialized_p () && other.initialized_p () && m_val <= other.m_val;
624 : : }
625 : :
626 : 0 : bool operator>= (const profile_probability &other) const
627 : : {
628 : 0 : return initialized_p () && other.initialized_p () && m_val >= other.m_val;
629 : : }
630 : :
631 : 20 : profile_probability operator* (int64_t num) const
632 : : {
633 : 20 : return apply_scale (num, 1);
634 : : }
635 : :
636 : : profile_probability operator*= (int64_t num)
637 : : {
638 : : *this = apply_scale (num, 1);
639 : : return *this;
640 : : }
641 : :
642 : 11798338 : profile_probability operator/ (int64_t den) const
643 : : {
644 : 11798338 : return apply_scale (1, den);
645 : : }
646 : :
647 : 186405 : profile_probability operator/= (int64_t den)
648 : : {
649 : 186405 : *this = apply_scale (1, den);
650 : 186405 : return *this;
651 : : }
652 : :
653 : : /* Compute n-th power. */
654 : : profile_probability pow (int) const;
655 : :
656 : : /* Compute sware root. */
657 : : profile_probability sqrt () const;
658 : :
659 : : /* Get the value of the count. */
660 : 0 : uint32_t value () const { return m_val; }
661 : :
662 : : /* Get the quality of the count. */
663 : 0 : enum profile_quality quality () const { return m_quality; }
664 : :
665 : : /* Output THIS to F. */
666 : : void dump (FILE *f) const;
667 : :
668 : : /* Output THIS to BUFFER. */
669 : : void dump (char *buffer) const;
670 : :
671 : : /* Print THIS to stderr. */
672 : : void debug () const;
673 : :
674 : : /* Return true if THIS is known to differ significantly from OTHER. */
675 : : bool differs_from_p (profile_probability other) const;
676 : :
677 : : /* Return if difference is greater than 50%. */
678 : : bool differs_lot_from_p (profile_probability other) const;
679 : :
680 : : /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
681 : : happens with COUNT2 probability. Return probability that either *THIS or
682 : : OTHER happens. */
683 : : profile_probability combine_with_count (profile_count count1,
684 : : profile_probability other,
685 : : profile_count count2) const;
686 : :
687 : : /* Return probability as sreal. */
688 : : sreal to_sreal () const;
689 : : /* LTO streaming support. */
690 : : static profile_probability stream_in (class lto_input_block *);
691 : : void stream_out (struct output_block *);
692 : : void stream_out (struct lto_output_stream *);
693 : : };
694 : :
695 : : /* Main data type to hold profile counters in GCC. Profile counts originate
696 : : either from profile feedback, static profile estimation or both. We do not
697 : : perform whole program profile propagation and thus profile estimation
698 : : counters are often local to function, while counters from profile feedback
699 : : (or special cases of profile estimation) can be used inter-procedurally.
700 : :
701 : : There are 3 basic types
702 : : 1) local counters which are result of intra-procedural static profile
703 : : estimation.
704 : : 2) ipa counters which are result of profile feedback or special case
705 : : of static profile estimation (such as in function main).
706 : : 3) counters which counts as 0 inter-procedurally (because given function
707 : : was never run in train feedback) but they hold local static profile
708 : : estimate.
709 : :
710 : : Counters of type 1 and 3 cannot be mixed with counters of different type
711 : : within operation (because whole function should use one type of counter)
712 : : with exception that global zero mix in most operations where outcome is
713 : : well defined.
714 : :
715 : : To take local counter and use it inter-procedurally use ipa member function
716 : : which strips information irrelevant at the inter-procedural level.
717 : :
718 : : Counters are 61bit integers representing number of executions during the
719 : : train run or normalized frequency within the function.
720 : :
721 : : As the profile is maintained during the compilation, many adjustments are
722 : : made. Not all transformations can be made precisely, most importantly
723 : : when code is being duplicated. It also may happen that part of CFG has
724 : : profile counts known while other do not - for example when LTO optimizing
725 : : partly profiled program or when profile was lost due to COMDAT merging.
726 : :
727 : : For this reason profile_count tracks more information than
728 : : just unsigned integer and it is also ready for profile mismatches.
729 : : The API of this data type represent operations that are natural
730 : : on profile counts - sum, difference and operation with scales and
731 : : probabilities. All operations are safe by never getting negative counts
732 : : and they do end up in uninitialized scale if any of the parameters is
733 : : uninitialized.
734 : :
735 : : All comparisons that are three state and handling of probabilities. Thus
736 : : a < b is not equal to !(a >= b).
737 : :
738 : : The following pre-defined counts are available:
739 : :
740 : : profile_count::zero () for code that is known to execute zero times at
741 : : runtime (this can be detected statically i.e. for paths leading to
742 : : abort ();
743 : : profile_count::one () for code that is known to execute once (such as
744 : : main () function
745 : : profile_count::uninitialized () for unknown execution count.
746 : :
747 : : */
748 : :
749 : : struct GTY(()) profile_count
750 : : {
751 : : public:
752 : : /* Use 62bit to hold basic block counters. Should be at least
753 : : 64bit. Although a counter cannot be negative, we use a signed
754 : : type to hold various extra stages. */
755 : :
756 : : static const int n_bits = 61;
757 : : static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
758 : : private:
759 : : static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
760 : :
761 : : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
762 : : /* Work-around for PR88469. A bug in the gcc-6/7/8 PCS layout code
763 : : incorrectly detects the alignment of a structure where the only
764 : : 64-bit aligned object is a bit-field. We force the alignment of
765 : : the entire field to mitigate this. */
766 : : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
767 : : #else
768 : : #define UINT64_BIT_FIELD_ALIGN
769 : : #endif
770 : : uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
771 : : #undef UINT64_BIT_FIELD_ALIGN
772 : : enum profile_quality m_quality : 3;
773 : : public:
774 : :
775 : : /* Return true if both values can meaningfully appear in single function
776 : : body. We have either all counters in function local or global, otherwise
777 : : operations between them are not really defined well. */
778 : 2184123496 : bool compatible_p (const profile_count other) const
779 : : {
780 : 4325755910 : if (!initialized_p () || !other.initialized_p ())
781 : : return true;
782 : 8558091236 : if (*this == zero ()
783 : 4275222266 : || other == zero ())
784 : 7647587 : return true;
785 : : /* Do not allow nonzero global profile together with local guesses
786 : : that are globally0. */
787 : 2133786898 : if (ipa ().nonzero_p ()
788 : 85950 : && !(other.ipa () == other))
789 : 0 : return false;
790 : 2133786898 : if (other.ipa ().nonzero_p ()
791 : 89300 : && !(ipa () == *this))
792 : 0 : return false;
793 : :
794 : 2133786898 : return ipa_p () == other.ipa_p ();
795 : : }
796 : :
797 : : /* Used for counters which are expected to be never executed. */
798 : 12550646316 : static profile_count zero ()
799 : : {
800 : 9235579725 : return from_gcov_type (0);
801 : : }
802 : :
803 : 3237 : static profile_count adjusted_zero ()
804 : : {
805 : 3237 : profile_count c;
806 : 3237 : c.m_val = 0;
807 : 3237 : c.m_quality = ADJUSTED;
808 : 3237 : return c;
809 : : }
810 : :
811 : : static profile_count guessed_zero ()
812 : : {
813 : : profile_count c;
814 : 0 : c.m_val = 0;
815 : 0 : c.m_quality = GUESSED;
816 : : return c;
817 : : }
818 : :
819 : 714 : static profile_count one ()
820 : : {
821 : 357 : return from_gcov_type (1);
822 : : }
823 : :
824 : : /* Value of counters which has not been initialized. Either because
825 : : initialization did not happen yet or because profile is unknown. */
826 : 5382269522 : static profile_count uninitialized ()
827 : : {
828 : 5382269522 : profile_count c;
829 : 5486019993 : c.m_val = uninitialized_count;
830 : 3393830423 : c.m_quality = GUESSED_LOCAL;
831 : 5382269522 : return c;
832 : : }
833 : :
834 : : /* Conversion to gcov_type is lossy. */
835 : 274217 : gcov_type to_gcov_type () const
836 : : {
837 : 0 : gcc_checking_assert (initialized_p ());
838 : 274217 : return m_val;
839 : : }
840 : :
841 : : /* Return true if value has been initialized. */
842 : 20326881564 : bool initialized_p () const
843 : : {
844 : 7094040390 : return m_val != uninitialized_count;
845 : : }
846 : :
847 : : /* Return true if value can be trusted. */
848 : 7187859 : bool reliable_p () const
849 : : {
850 : 7187606 : return m_quality >= ADJUSTED;
851 : : }
852 : :
853 : : /* Return true if value can be operated inter-procedurally. */
854 : 5084333174 : bool ipa_p () const
855 : : {
856 : 2940890216 : return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
857 : : }
858 : :
859 : : /* Return true if quality of profile is precise. */
860 : 44262598 : bool precise_p () const
861 : : {
862 : 44262598 : return m_quality == PRECISE;
863 : : }
864 : :
865 : : /* Get the value of the count. */
866 : 0 : uint64_t value () const { return m_val; }
867 : :
868 : : /* Get the quality of the count. */
869 : 36129 : enum profile_quality quality () const { return m_quality; }
870 : :
871 : : /* When merging basic blocks, the two different profile counts are unified.
872 : : Return true if this can be done without losing info about profile.
873 : : The only case we care about here is when first BB contains something
874 : : that makes it terminate in a way not visible in CFG. */
875 : 2620542 : bool ok_for_merging (profile_count other) const
876 : : {
877 : 2620542 : if (m_quality < ADJUSTED
878 : 10256 : || other.m_quality < ADJUSTED)
879 : : return true;
880 : 9635 : return !(other < *this);
881 : : }
882 : :
883 : : /* When merging two BBs with different counts, pick common count that looks
884 : : most representative. */
885 : 12664954 : profile_count merge (profile_count other) const
886 : : {
887 : 877711 : if (*this == other || !other.initialized_p ()
888 : 877480 : || m_quality > other.m_quality)
889 : 11788850 : return *this;
890 : 876104 : if (other.m_quality > m_quality
891 : 876104 : || other > *this)
892 : 360532 : return other;
893 : 515572 : return *this;
894 : : }
895 : :
896 : : /* Basic operations. */
897 : 13773968204 : bool operator== (const profile_count &other) const
898 : : {
899 : 9810950652 : return m_val == other.m_val && m_quality == other.m_quality;
900 : : }
901 : :
902 : 2449371 : profile_count operator+ (const profile_count &other) const
903 : : {
904 : 2449371 : if (other == zero ())
905 : 1849 : return *this;
906 : 2447522 : if (*this == zero ())
907 : 158679 : return other;
908 : 2288843 : if (!initialized_p () || !other.initialized_p ())
909 : 5281 : return uninitialized ();
910 : :
911 : 2283562 : profile_count ret;
912 : 2283562 : gcc_checking_assert (compatible_p (other));
913 : 2283562 : uint64_t ret_val = m_val + other.m_val;
914 : 2283562 : ret.m_val = MIN (ret_val, max_count);
915 : 2283562 : ret.m_quality = MIN (m_quality, other.m_quality);
916 : 2283562 : return ret;
917 : : }
918 : :
919 : 55092241 : profile_count &operator+= (const profile_count &other)
920 : : {
921 : 55092241 : if (other == zero ())
922 : 2868059 : return *this;
923 : 52224182 : if (*this == zero ())
924 : : {
925 : 30041138 : *this = other;
926 : 30041138 : return *this;
927 : : }
928 : 22183044 : if (!initialized_p () || !other.initialized_p ())
929 : 291339 : return *this = uninitialized ();
930 : : else
931 : : {
932 : 21891705 : gcc_checking_assert (compatible_p (other));
933 : 21891705 : uint64_t ret_val = m_val + other.m_val;
934 : 21891705 : m_val = MIN (ret_val, max_count);
935 : 21891705 : m_quality = MIN (m_quality, other.m_quality);
936 : : }
937 : 21891705 : return *this;
938 : : }
939 : :
940 : 15524046 : profile_count operator- (const profile_count &other) const
941 : : {
942 : 15535132 : if (*this == zero () || other == zero ())
943 : 643223 : return *this;
944 : 14880823 : if (!initialized_p () || !other.initialized_p ())
945 : 6311815 : return uninitialized ();
946 : 8569008 : gcc_checking_assert (compatible_p (other));
947 : 8569008 : profile_count ret;
948 : 8569008 : ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
949 : 8569008 : ret.m_quality = MIN (m_quality, other.m_quality);
950 : 8569008 : return ret;
951 : : }
952 : :
953 : 8014096 : profile_count &operator-= (const profile_count &other)
954 : : {
955 : 8467278 : if (*this == zero () || other == zero ())
956 : 290358 : return *this;
957 : 7723738 : if (!initialized_p () || !other.initialized_p ())
958 : 334486 : return *this = uninitialized ();
959 : : else
960 : : {
961 : 7389252 : gcc_checking_assert (compatible_p (other));
962 : 7389252 : m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
963 : 7389252 : m_quality = MIN (m_quality, other.m_quality);
964 : : }
965 : 7389252 : return *this;
966 : : }
967 : :
968 : : /* Return false if profile_count is bogus. */
969 : 2880543080 : bool verify () const
970 : : {
971 : 2880543080 : gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
972 : 2880543080 : return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
973 : : }
974 : :
975 : : /* Comparisons are three-state and conservative. False is returned if
976 : : the inequality cannot be decided. */
977 : 848264246 : bool operator< (const profile_count &other) const
978 : : {
979 : 848264246 : if (!initialized_p () || !other.initialized_p ())
980 : : return false;
981 : 844608756 : if (*this == zero ())
982 : 3280182 : return !(other == zero ());
983 : 842968665 : if (other == zero ())
984 : 4458 : return false;
985 : 842964207 : gcc_checking_assert (compatible_p (other));
986 : 842964207 : return m_val < other.m_val;
987 : : }
988 : :
989 : 28580561 : bool operator> (const profile_count &other) const
990 : : {
991 : 28580561 : if (!initialized_p () || !other.initialized_p ())
992 : : return false;
993 : 27741694 : if (*this == zero ())
994 : 216852 : return false;
995 : 27524842 : if (other == zero ())
996 : 1516590 : return !(*this == zero ());
997 : 26766547 : gcc_checking_assert (compatible_p (other));
998 : 26766547 : return initialized_p () && other.initialized_p () && m_val > other.m_val;
999 : : }
1000 : :
1001 : : bool operator< (const gcov_type other) const
1002 : : {
1003 : : gcc_checking_assert (ipa_p ());
1004 : : gcc_checking_assert (other >= 0);
1005 : : return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
1006 : : }
1007 : :
1008 : 260065 : bool operator> (const gcov_type other) const
1009 : : {
1010 : 260065 : gcc_checking_assert (ipa_p ());
1011 : 260065 : gcc_checking_assert (other >= 0);
1012 : 260065 : return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
1013 : : }
1014 : :
1015 : 888262 : bool operator<= (const profile_count &other) const
1016 : : {
1017 : 888262 : if (!initialized_p () || !other.initialized_p ())
1018 : : return false;
1019 : 888262 : if (*this == zero ())
1020 : 1191 : return true;
1021 : 887071 : if (other == zero ())
1022 : 0 : return (*this == zero ());
1023 : 887071 : gcc_checking_assert (compatible_p (other));
1024 : 887071 : return m_val <= other.m_val;
1025 : : }
1026 : :
1027 : 9771851 : bool operator>= (const profile_count &other) const
1028 : : {
1029 : 9771851 : if (!initialized_p () || !other.initialized_p ())
1030 : : return false;
1031 : 9769675 : if (other == zero ())
1032 : 78973 : return true;
1033 : 9690702 : if (*this == zero ())
1034 : 23458 : return (other == zero ());
1035 : 9678973 : gcc_checking_assert (compatible_p (other));
1036 : 9678973 : return m_val >= other.m_val;
1037 : : }
1038 : :
1039 : 85839 : bool operator<= (const gcov_type other) const
1040 : : {
1041 : 85839 : gcc_checking_assert (ipa_p ());
1042 : 85839 : gcc_checking_assert (other >= 0);
1043 : 85839 : return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
1044 : : }
1045 : :
1046 : 53378 : bool operator>= (const gcov_type other) const
1047 : : {
1048 : 53378 : gcc_checking_assert (ipa_p ());
1049 : 53378 : gcc_checking_assert (other >= 0);
1050 : 53378 : return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
1051 : : }
1052 : :
1053 : 722885438 : profile_count operator* (int64_t num) const
1054 : : {
1055 : 714286415 : return apply_scale (num, 1);
1056 : : }
1057 : :
1058 : : profile_count operator*= (int64_t num)
1059 : : {
1060 : : *this = apply_scale (num, 1);
1061 : : return *this;
1062 : : }
1063 : :
1064 : 1231249 : profile_count operator/ (int64_t den) const
1065 : : {
1066 : 1231249 : return apply_scale (1, den);
1067 : : }
1068 : :
1069 : 67 : profile_count operator/= (int64_t den)
1070 : : {
1071 : 63 : *this = apply_scale (1, den);
1072 : 67 : return *this;
1073 : : }
1074 : :
1075 : : /* Return true when value is not zero and can be used for scaling.
1076 : : This is different from *this > 0 because that requires counter to
1077 : : be IPA. */
1078 : 4566154687 : bool nonzero_p () const
1079 : : {
1080 : 4421381424 : return initialized_p () && m_val != 0;
1081 : : }
1082 : :
1083 : : /* Make counter forcibly nonzero. */
1084 : 15534413 : profile_count force_nonzero () const
1085 : : {
1086 : 15534413 : if (!initialized_p ())
1087 : 19857 : return *this;
1088 : 15514556 : profile_count ret = *this;
1089 : 15514556 : if (ret.m_val == 0)
1090 : : {
1091 : 39534 : ret.m_val = 1;
1092 : 39534 : ret.m_quality = MIN (m_quality, ADJUSTED);
1093 : : }
1094 : 15514556 : return ret;
1095 : : }
1096 : :
1097 : 76173004 : profile_count max (profile_count other) const
1098 : : {
1099 : 76173004 : profile_count val = *this;
1100 : :
1101 : : /* Always prefer nonzero IPA counts over local counts. */
1102 : 83731123 : if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
1103 : : {
1104 : 9218 : val = ipa ();
1105 : 9218 : other = other.ipa ();
1106 : : }
1107 : 76173004 : if (!initialized_p ())
1108 : 18570583 : return other;
1109 : 57602421 : if (!other.initialized_p ())
1110 : 910852 : return *this;
1111 : 56691569 : if (*this == zero ())
1112 : 2044451 : return other;
1113 : 54647118 : if (other == zero ())
1114 : 3721915 : return *this;
1115 : 50925203 : gcc_checking_assert (compatible_p (other));
1116 : 50925203 : if (val.m_val < other.m_val || (m_val == other.m_val
1117 : 12170450 : && val.m_quality < other.m_quality))
1118 : 7146772 : return other;
1119 : 43778431 : return *this;
1120 : : }
1121 : :
1122 : : /* PROB is a probability in scale 0...REG_BR_PROB_BASE. Scale counter
1123 : : accordingly. */
1124 : : profile_count apply_probability (int prob) const
1125 : : {
1126 : : gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
1127 : : if (m_val == 0)
1128 : : return *this;
1129 : : if (!initialized_p ())
1130 : : return uninitialized ();
1131 : : profile_count ret;
1132 : : uint64_t tmp;
1133 : : safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
1134 : : ret.m_val = tmp;
1135 : : ret.m_quality = MIN (m_quality, ADJUSTED);
1136 : : return ret;
1137 : : }
1138 : :
1139 : : /* Scale counter according to PROB. */
1140 : 320302253 : profile_count apply_probability (profile_probability prob) const
1141 : : {
1142 : 327793321 : if (*this == zero () || prob == profile_probability::always ())
1143 : 104134000 : return *this;
1144 : 216168253 : if (prob == profile_probability::never ())
1145 : 16545222 : return zero ();
1146 : 199623031 : if (!initialized_p () || !prob.initialized_p ())
1147 : 27379640 : return uninitialized ();
1148 : 172243391 : profile_count ret;
1149 : 172243391 : uint64_t tmp;
1150 : 172243391 : safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
1151 : : &tmp);
1152 : 172243391 : ret.m_val = tmp;
1153 : 172243391 : ret.m_quality = MIN (m_quality, prob.m_quality);
1154 : 172243391 : return ret;
1155 : : }
1156 : :
1157 : : /* Return *THIS * NUM / DEN. */
1158 : 812692862 : profile_count apply_scale (int64_t num, int64_t den) const
1159 : : {
1160 : 812692862 : if (m_val == 0)
1161 : 22054385 : return *this;
1162 : 790638477 : if (!initialized_p ())
1163 : 8666 : return uninitialized ();
1164 : 790629811 : profile_count ret;
1165 : 790629811 : uint64_t tmp;
1166 : :
1167 : 790629811 : gcc_checking_assert (num >= 0 && den > 0);
1168 : 790629811 : safe_scale_64bit (m_val, num, den, &tmp);
1169 : 790629811 : ret.m_val = MIN (tmp, max_count);
1170 : 790629811 : ret.m_quality = MIN (m_quality, ADJUSTED);
1171 : 790629811 : return ret;
1172 : : }
1173 : :
1174 : 21569673 : profile_count apply_scale (profile_count num, profile_count den) const
1175 : : {
1176 : 21569673 : if (*this == zero ())
1177 : 1192240 : return *this;
1178 : 20377433 : if (num == zero ())
1179 : 55469 : return num;
1180 : 20321964 : if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
1181 : 5634052 : return uninitialized ();
1182 : 14687912 : if (num == den)
1183 : 5178396 : return *this;
1184 : 9509516 : gcc_checking_assert (den.m_val);
1185 : :
1186 : 9509516 : profile_count ret;
1187 : 9509516 : uint64_t val;
1188 : 9509516 : safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
1189 : 9509516 : ret.m_val = MIN (val, max_count);
1190 : 9509516 : ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
1191 : : num.m_quality), den.m_quality);
1192 : : /* Be sure that ret is not local if num is global.
1193 : : Also ensure that ret is not global0 when num is global. */
1194 : 9509516 : if (num.ipa_p ())
1195 : 2878 : ret.m_quality = MAX (ret.m_quality,
1196 : : num == num.ipa () ? GUESSED : num.m_quality);
1197 : 9509516 : return ret;
1198 : : }
1199 : :
1200 : : /* Return THIS with quality dropped to GUESSED_LOCAL. */
1201 : 15456221 : profile_count guessed_local () const
1202 : : {
1203 : 15456221 : profile_count ret = *this;
1204 : 15456221 : if (!initialized_p ())
1205 : 0 : return *this;
1206 : 15456221 : ret.m_quality = GUESSED_LOCAL;
1207 : 15456221 : return ret;
1208 : : }
1209 : :
1210 : : /* We know that profile is globally 0 but keep local profile if present. */
1211 : 768 : profile_count global0 () const
1212 : : {
1213 : 768 : profile_count ret = *this;
1214 : 768 : if (!initialized_p ())
1215 : 0 : return *this;
1216 : 768 : ret.m_quality = GUESSED_GLOBAL0;
1217 : 768 : return ret;
1218 : : }
1219 : :
1220 : : /* We know that profile is globally adjusted 0 but keep local profile
1221 : : if present. */
1222 : 35 : profile_count global0adjusted () const
1223 : : {
1224 : 35 : profile_count ret = *this;
1225 : 35 : if (!initialized_p ())
1226 : 0 : return *this;
1227 : 35 : ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
1228 : 35 : return ret;
1229 : : }
1230 : :
1231 : : /* Return THIS with quality dropped to GUESSED. */
1232 : 210 : profile_count guessed () const
1233 : : {
1234 : 210 : profile_count ret = *this;
1235 : 210 : ret.m_quality = MIN (ret.m_quality, GUESSED);
1236 : 210 : return ret;
1237 : : }
1238 : :
1239 : : /* Return variant of profile count which is always safe to compare
1240 : : across functions. */
1241 : 5272025445 : profile_count ipa () const
1242 : : {
1243 : 5272025445 : if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
1244 : 16653695 : return *this;
1245 : 5255371750 : if (m_quality == GUESSED_GLOBAL0)
1246 : 44573 : return zero ();
1247 : 5255327177 : if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
1248 : 3237 : return adjusted_zero ();
1249 : 5255323940 : return uninitialized ();
1250 : : }
1251 : :
1252 : : /* Return THIS with quality dropped to AFDO. */
1253 : 0 : profile_count afdo () const
1254 : : {
1255 : 0 : profile_count ret = *this;
1256 : 0 : ret.m_quality = AFDO;
1257 : 0 : return ret;
1258 : : }
1259 : :
1260 : : /* Return probability of event with counter THIS within event with counter
1261 : : OVERALL. */
1262 : 911532404 : profile_probability probability_in (const profile_count overall) const
1263 : : {
1264 : 911543428 : if (*this == zero ()
1265 : 11024 : && !(overall == zero ()))
1266 : 951 : return profile_probability::never ();
1267 : 911530284 : if (!initialized_p () || !overall.initialized_p ()
1268 : 1823061737 : || !overall.m_val)
1269 : 46760 : return profile_probability::uninitialized ();
1270 : 1102663086 : if (*this == overall && m_quality == PRECISE)
1271 : 27588 : return profile_probability::always ();
1272 : 911457105 : profile_probability ret;
1273 : 911457105 : gcc_checking_assert (compatible_p (overall));
1274 : :
1275 : 911457105 : if (overall.m_val < m_val)
1276 : : {
1277 : 89345 : ret.m_val = profile_probability::max_probability;
1278 : 89345 : ret.m_quality = GUESSED;
1279 : 89345 : return ret;
1280 : : }
1281 : : else
1282 : 911367760 : ret.m_val = RDIV (m_val * profile_probability::max_probability,
1283 : : overall.m_val);
1284 : 911367760 : ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
1285 : : GUESSED), ADJUSTED);
1286 : 911367760 : return ret;
1287 : : }
1288 : :
1289 : : /* Return true if profile count is very large, so we risk overflows
1290 : : with loop transformations. */
1291 : : bool
1292 : 966718 : very_large_p ()
1293 : : {
1294 : 966718 : if (!initialized_p ())
1295 : : return false;
1296 : 966718 : return m_val > max_count / 65536;
1297 : : }
1298 : :
1299 : : int to_frequency (struct function *fun) const;
1300 : : int to_cgraph_frequency (profile_count entry_bb_count) const;
1301 : : sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
1302 : :
1303 : : /* Output THIS to F. */
1304 : : void dump (FILE *f, struct function *fun = NULL) const;
1305 : :
1306 : : /* Print THIS to stderr. */
1307 : : void debug () const;
1308 : :
1309 : : /* Return true if THIS is known to differ significantly from OTHER. */
1310 : : bool differs_from_p (profile_count other) const;
1311 : :
1312 : : /* We want to scale profile across function boundary from NUM to DEN.
1313 : : Take care of the side case when NUM and DEN are zeros of incompatible
1314 : : kinds. */
1315 : : static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
1316 : :
1317 : : /* THIS is a count of bb which is known to be executed IPA times.
1318 : : Combine this information into bb counter. This means returning IPA
1319 : : if it is nonzero, not changing anything if IPA is uninitialized
1320 : : and if IPA is zero, turning THIS into corresponding local profile with
1321 : : global0. */
1322 : : profile_count combine_with_ipa_count (profile_count ipa);
1323 : :
1324 : : /* Same as combine_with_ipa_count but inside function with count IPA2. */
1325 : : profile_count combine_with_ipa_count_within
1326 : : (profile_count ipa, profile_count ipa2);
1327 : :
1328 : : /* The profiling runtime uses gcov_type, which is usually 64bit integer.
1329 : : Conversions back and forth are used to read the coverage and get it
1330 : : into internal representation. */
1331 : : static profile_count from_gcov_type (gcov_type v,
1332 : : profile_quality quality = PRECISE);
1333 : :
1334 : : /* LTO streaming support. */
1335 : : static profile_count stream_in (class lto_input_block *);
1336 : : void stream_out (struct output_block *);
1337 : : void stream_out (struct lto_output_stream *);
1338 : : };
1339 : : #endif
|
