LCOV - code coverage report
Current view: top level - gcc - profile-count.h (source / functions) Coverage Total Hit
Test: gcc.info Lines: 92.7 % 558 517
Test Date: 2025-07-05 13:26:22 Functions: 94.5 % 55 52
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: - 0 0

             Branch data     Line data    Source code
       1                 :             : /* Profile counter container type.
       2                 :             :    Copyright (C) 2017-2025 Free Software Foundation, Inc.
       3                 :             :    Contributed by Jan Hubicka
       4                 :             : 
       5                 :             : This file is part of GCC.
       6                 :             : 
       7                 :             : GCC is free software; you can redistribute it and/or modify it under
       8                 :             : the terms of the GNU General Public License as published by the Free
       9                 :             : Software Foundation; either version 3, or (at your option) any later
      10                 :             : version.
      11                 :             : 
      12                 :             : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13                 :             : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14                 :             : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15                 :             : for more details.
      16                 :             : 
      17                 :             : You should have received a copy of the GNU General Public License
      18                 :             : along with GCC; see the file COPYING3.  If not see
      19                 :             : <http://www.gnu.org/licenses/>.  */
      20                 :             : 
      21                 :             : #ifndef GCC_PROFILE_COUNT_H
      22                 :             : #define GCC_PROFILE_COUNT_H
      23                 :             : 
      24                 :             : struct function;
      25                 :             : struct profile_count;
      26                 :             : class sreal;
      27                 :             : 
      28                 :             : /* Quality of the profile count.  Because gengtype does not support enums
      29                 :             :    inside of classes, this is in global namespace.  */
      30                 :             : enum profile_quality {
      31                 :             :   /* Uninitialized value.  */
      32                 :             :   UNINITIALIZED_PROFILE,
      33                 :             : 
      34                 :             :   /* Profile is based on static branch prediction heuristics and may
      35                 :             :      or may not match reality.  It is local to function and cannot be compared
      36                 :             :      inter-procedurally.  Never used by probabilities (they are always local).
      37                 :             :    */
      38                 :             :   GUESSED_LOCAL,
      39                 :             : 
      40                 :             :   /* Same as GUESSED_GLOBAL0 but global count is afdo 0.  */
      41                 :             :   GUESSED_GLOBAL0_AFDO,
      42                 :             : 
      43                 :             :   /* Same as GUESSED_GLOBAL0 but global count is adjusted 0.  */
      44                 :             :   GUESSED_GLOBAL0_ADJUSTED,
      45                 :             : 
      46                 :             :   /* Profile was read by feedback and was 0, we used local heuristics to guess
      47                 :             :      better.  This is the case of functions not run in profile feedback.
      48                 :             :      Never used by probabilities.  */
      49                 :             :   GUESSED_GLOBAL0,
      50                 :             : 
      51                 :             :   /* Profile is based on static branch prediction heuristics.  It may or may
      52                 :             :      not reflect the reality but it can be compared interprocedurally
      53                 :             :      (for example, we inlined function w/o profile feedback into function
      54                 :             :       with feedback and propagated from that).  */
      55                 :             :   GUESSED,
      56                 :             : 
      57                 :             :   /* Profile was determined by autofdo.  */
      58                 :             :   AFDO,
      59                 :             : 
      60                 :             :   /* Profile was originally based on feedback but it was adjusted
      61                 :             :      by code duplicating optimization.  It may not precisely reflect the
      62                 :             :      particular code path.  */
      63                 :             :   ADJUSTED,
      64                 :             : 
      65                 :             :   /* Profile was read from profile feedback or determined by accurate static
      66                 :             :      method.  */
      67                 :             :   PRECISE
      68                 :             : };
      69                 :             : 
      70                 :             : extern const char *profile_quality_as_string (enum profile_quality);
      71                 :             : extern bool parse_profile_quality (const char *value,
      72                 :             :                                    profile_quality *quality);
      73                 :             : 
      74                 :             : /* The base value for branch probability notes and edge probabilities.  */
      75                 :             : #define REG_BR_PROB_BASE  10000
      76                 :             : 
      77                 :             : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
      78                 :             : 
      79                 :             : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
      80                 :             : 
      81                 :             : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened.  */
      82                 :             : 
      83                 :             : inline bool
      84                 :  1189523661 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
      85                 :             : {
      86                 :             : #if (GCC_VERSION >= 5000)
      87                 :  1189523661 :   uint64_t tmp;
      88                 :  1189523661 :   if (!__builtin_mul_overflow (a, b, &tmp)
      89                 :  1189523661 :       && !__builtin_add_overflow (tmp, c/2, &tmp))
      90                 :             :     {
      91                 :  1188908801 :       *res = tmp / c;
      92                 :  1188908801 :       return true;
      93                 :             :     }
      94                 :      614860 :   if (c == 1)
      95                 :             :     {
      96                 :       27748 :       *res = (uint64_t) -1;
      97                 :       27748 :       return false;
      98                 :             :     }
      99                 :             : #else
     100                 :             :   if (a < ((uint64_t)1 << 31)
     101                 :             :       && b < ((uint64_t)1 << 31)
     102                 :             :       && c < ((uint64_t)1 << 31))
     103                 :             :     {
     104                 :             :       *res = (a * b + (c / 2)) / c;
     105                 :             :       return true;
     106                 :             :     }
     107                 :             : #endif
     108                 :      587112 :   return slow_safe_scale_64bit (a, b, c, res);
     109                 :             : }
     110                 :             : 
     111                 :             : /* Data type to hold probabilities.  It implements fixed point arithmetics
     112                 :             :    with capping so probability is always in range [0,1] and scaling requiring
     113                 :             :    values greater than 1 needs to be represented otherwise.
     114                 :             : 
     115                 :             :    In addition to actual value the quality of profile is tracked and propagated
     116                 :             :    through all operations.  Special value UNINITIALIZED_PROFILE is used for probabilities
     117                 :             :    that has not been determined yet (for example because of
     118                 :             :    -fno-guess-branch-probability)
     119                 :             : 
     120                 :             :    Typically probabilities are derived from profile feedback (via
     121                 :             :    probability_in_gcov_type), autoFDO or guessed statically and then propagated
     122                 :             :    thorough the compilation.
     123                 :             : 
     124                 :             :    Named probabilities are available:
     125                 :             :      - never           (0 probability)
     126                 :             :      - guessed_never
     127                 :             :      - very_unlikely   (1/2000 probability)
     128                 :             :      - unlikely        (1/5 probability)
     129                 :             :      - even            (1/2 probability)
     130                 :             :      - likely          (4/5 probability)
     131                 :             :      - very_likely     (1999/2000 probability)
     132                 :             :      - guessed_always
     133                 :             :      - always
     134                 :             : 
     135                 :             :    Named probabilities except for never/always are assumed to be statically
     136                 :             :    guessed and thus not necessarily accurate.  The difference between never
     137                 :             :    and guessed_never is that the first one should be used only in case that
     138                 :             :    well behaving program will very likely not execute the "never" path.
     139                 :             :    For example if the path is going to abort () call or it exception handling.
     140                 :             : 
     141                 :             :    Always and guessed_always probabilities are symmetric.
     142                 :             : 
     143                 :             :    For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
     144                 :             :    integer arithmetics. Once the code is converted to branch probabilities,
     145                 :             :    these conversions will probably go away because they are lossy.
     146                 :             : */
     147                 :             : 
     148                 :             : class GTY((user)) profile_probability
     149                 :             : {
     150                 :             :   static const int n_bits = 29;
     151                 :             :   /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
     152                 :             :      will lead to harder multiplication sequences.  */
     153                 :             :   static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
     154                 :             :   static const uint32_t uninitialized_probability
     155                 :             :                  = ((uint32_t) 1 << (n_bits - 1)) - 1;
     156                 :             :   /* For probibilityes quality is either UNINITIALIZED (0)
     157                 :             :      or greated then GUESSED.  To save bits we store it in
     158                 :             :      adjusted form which skips the invalid values.  */
     159                 :             :   static const int min_quality = GUESSED;
     160                 :             : 
     161                 :             :   uint32_t m_val : 29;
     162                 :             :   unsigned m_adjusted_quality : 3;
     163                 :             : 
     164                 :             :   friend struct profile_count;
     165                 :             : 
     166                 :             :   /* Set the quality of the probability.  */
     167                 :             :   void
     168                 :  5192805524 :   set_quality (profile_quality quality)
     169                 :             :   {
     170                 :  1124017987 :     gcc_checking_assert (quality == UNINITIALIZED_PROFILE
     171                 :             :                          || (quality >= min_quality && quality <= PRECISE));
     172                 :  1124018071 :     m_adjusted_quality = quality ? quality - min_quality + 1 : 0;
     173                 :  1122348602 :   }
     174                 :             : 
     175                 :             : public:
     176                 :  4070456922 :   profile_probability (): m_val (uninitialized_probability)
     177                 :  2206244740 :   { set_quality (GUESSED); }
     178                 :             : 
     179                 :         112 :   profile_probability (uint32_t val, profile_quality quality):
     180                 :         112 :     m_val (val)
     181                 :         112 :   { set_quality (quality); }
     182                 :             : 
     183                 :             :   /* Named probabilities.  */
     184                 :   663874773 :   static profile_probability never ()
     185                 :             :     {
     186                 :   663874773 :       profile_probability ret;
     187                 :   663874773 :       ret.m_val = 0;
     188                 :   663874773 :       ret.set_quality (PRECISE);
     189                 :   372350342 :       return ret;
     190                 :             :     }
     191                 :             : 
     192                 :     5190843 :   static profile_probability guessed_never ()
     193                 :             :     {
     194                 :     5190843 :       profile_probability ret;
     195                 :     5190843 :       ret.m_val = 0;
     196                 :     5190843 :       ret.set_quality (GUESSED);
     197                 :     5190843 :       return ret;
     198                 :             :     }
     199                 :             : 
     200                 :     5155306 :   static profile_probability very_unlikely ()
     201                 :             :     {
     202                 :             :       /* Be consistent with PROB_VERY_UNLIKELY in predict.h.  */
     203                 :    10310612 :       profile_probability r = guessed_always () / 2000;
     204                 :     5155306 :       r.m_val--;
     205                 :     5155306 :       return r;
     206                 :             :     }
     207                 :             : 
     208                 :       80788 :   static profile_probability unlikely ()
     209                 :             :     {
     210                 :             :       /* Be consistent with PROB_VERY_LIKELY in predict.h.  */
     211                 :      161576 :       profile_probability r = guessed_always () / 5;
     212                 :       80788 :       r.m_val--;
     213                 :       80788 :       return r;
     214                 :             :     }
     215                 :             : 
     216                 :      782653 :   static profile_probability even ()
     217                 :             :     {
     218                 :     1565306 :       return guessed_always () / 2;
     219                 :             :     }
     220                 :             : 
     221                 :      336899 :   static profile_probability very_likely ()
     222                 :             :     {
     223                 :      336899 :       return always () - very_unlikely ();
     224                 :             :     }
     225                 :             : 
     226                 :       31894 :   static profile_probability likely ()
     227                 :             :     {
     228                 :       31894 :       return always () - unlikely ();
     229                 :             :     }
     230                 :             :   /* Return true when value is not zero and can be used for scaling.   */
     231                 :        1549 :   bool nonzero_p () const
     232                 :             :     {
     233                 :        1549 :       return initialized_p () && m_val != 0;
     234                 :             :     }
     235                 :             : 
     236                 :     6323783 :   static profile_probability guessed_always ()
     237                 :             :     {
     238                 :     6323783 :       profile_probability ret;
     239                 :     6323783 :       ret.m_val = max_probability;
     240                 :     6323783 :       ret.set_quality (GUESSED);
     241                 :     6123561 :       return ret;
     242                 :             :     }
     243                 :             : 
     244                 :   526708135 :   static profile_probability always ()
     245                 :             :     {
     246                 :   526708135 :       profile_probability ret;
     247                 :   526708135 :       ret.m_val = max_probability;
     248                 :   526708135 :       ret.set_quality (PRECISE);
     249                 :   352951056 :       return ret;
     250                 :             :     }
     251                 :             : 
     252                 :             :   /* Probabilities which has not been initialized. Either because
     253                 :             :      initialization did not happen yet or because profile is unknown.  */
     254                 :   885430479 :   static profile_probability uninitialized ()
     255                 :             :     {
     256                 :   885430479 :       profile_probability c;
     257                 :   885430479 :       c.m_val = uninitialized_probability;
     258                 :   885430479 :       c.set_quality (GUESSED);
     259                 :   872035144 :       return c;
     260                 :             :     }
     261                 :             : 
     262                 :             :   /* Return true if value has been initialized.  */
     263                 :  5579897643 :   bool initialized_p () const
     264                 :             :     {
     265                 :  3903884308 :       return m_val != uninitialized_probability;
     266                 :             :     }
     267                 :             : 
     268                 :             :   /* Return true if value can be trusted.  */
     269                 :        1211 :   bool reliable_p () const
     270                 :             :     {
     271                 :        1197 :       return quality () >= ADJUSTED;
     272                 :             :     }
     273                 :             : 
     274                 :             :   /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
     275                 :             :      this is mostly to support legacy code and should go away.  */
     276                 :     3069916 :   static profile_probability from_reg_br_prob_base (int v)
     277                 :             :     {
     278                 :     3069916 :       profile_probability ret;
     279                 :     3069916 :       gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
     280                 :     3069916 :       ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
     281                 :     3069916 :       ret.set_quality (GUESSED);
     282                 :     3069916 :       return ret;
     283                 :             :     }
     284                 :             : 
     285                 :             :   /* Return THIS with quality set to ADJUSTED.  */
     286                 :          42 :   profile_probability adjusted () const
     287                 :             :     {
     288                 :          42 :       profile_probability ret = *this;
     289                 :          42 :       if (!initialized_p ())
     290                 :           0 :         return *this;
     291                 :          42 :       ret.set_quality (ADJUSTED);
     292                 :          42 :       return ret;
     293                 :             :     }
     294                 :             : 
     295                 :  1105256411 :   int to_reg_br_prob_base () const
     296                 :             :     {
     297                 :  1105256411 :       gcc_checking_assert (initialized_p ());
     298                 :  1105256411 :       return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
     299                 :             :     }
     300                 :             : 
     301                 :             :   /* Conversion to and from RTL representation of profile probabilities.  */
     302                 :   471660200 :   static profile_probability from_reg_br_prob_note (int v)
     303                 :             :     {
     304                 :   471660200 :       profile_probability ret;
     305                 :   471660200 :       ret.m_val = ((unsigned int)v) / 8;
     306                 :   471660200 :       ret.m_adjusted_quality = ((unsigned int)v) & 7;
     307                 :    15435993 :       return ret;
     308                 :             :     }
     309                 :             : 
     310                 :   443206016 :   int to_reg_br_prob_note () const
     311                 :             :     {
     312                 :   443206016 :       gcc_checking_assert (initialized_p ());
     313                 :   443206016 :       int ret = m_val * 8 + m_adjusted_quality;
     314                 :   443206016 :       gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
     315                 :   443206016 :       return ret;
     316                 :             :     }
     317                 :             : 
     318                 :             :   /* Return VAL1/VAL2.  */
     319                 :        3072 :   static profile_probability probability_in_gcov_type
     320                 :             :                                  (gcov_type val1, gcov_type val2)
     321                 :             :     {
     322                 :        3072 :       profile_probability ret;
     323                 :        3072 :       gcc_checking_assert (val1 >= 0 && val2 > 0);
     324                 :        3072 :       if (val1 > val2)
     325                 :             :         ret.m_val = max_probability;
     326                 :             :       else
     327                 :             :         {
     328                 :        3072 :           uint64_t tmp;
     329                 :        3072 :           safe_scale_64bit (val1, max_probability, val2, &tmp);
     330                 :        3072 :           gcc_checking_assert (tmp <= max_probability);
     331                 :        3072 :           ret.m_val = tmp;
     332                 :             :         }
     333                 :        3072 :       ret.set_quality (PRECISE);
     334                 :        3072 :       return ret;
     335                 :             :     }
     336                 :             : 
     337                 :             :   /* Basic operations.  */
     338                 :  1500977722 :   bool operator== (const profile_probability &other) const
     339                 :             :     {
     340                 :  2131696798 :       return m_val == other.m_val && quality () == other.quality ();
     341                 :             :     }
     342                 :             : 
     343                 :     6564131 :   profile_probability operator+ (const profile_probability &other) const
     344                 :             :     {
     345                 :     6564131 :       if (other == never ())
     346                 :       30850 :         return *this;
     347                 :     6533281 :       if (*this == never ())
     348                 :       23763 :         return other;
     349                 :     6509518 :       if (!initialized_p () || !other.initialized_p ())
     350                 :     5340643 :         return uninitialized ();
     351                 :             : 
     352                 :     1168875 :       profile_probability ret;
     353                 :     1168875 :       ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
     354                 :     1168875 :       ret.set_quality (MIN (quality (), other.quality ()));
     355                 :     1168875 :       return ret;
     356                 :             :     }
     357                 :             : 
     358                 :     6160240 :   profile_probability &operator+= (const profile_probability &other)
     359                 :             :     {
     360                 :     6160240 :       if (other == never ())
     361                 :             :         return *this;
     362                 :     4989960 :       if (*this == never ())
     363                 :             :         {
     364                 :      857309 :           *this = other;
     365                 :      857309 :           return *this;
     366                 :             :         }
     367                 :     4132651 :       if (!initialized_p () || !other.initialized_p ())
     368                 :     2898294 :         return *this = uninitialized ();
     369                 :             :       else
     370                 :             :         {
     371                 :     1234357 :           m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
     372                 :     1234357 :           set_quality (MIN (quality (), other.quality ()));
     373                 :             :         }
     374                 :     1234357 :       return *this;
     375                 :             :     }
     376                 :             : 
     377                 :    33481090 :   profile_probability operator- (const profile_probability &other) const
     378                 :             :     {
     379                 :    66962180 :       if (*this == never ()
     380                 :    33481090 :           || other == never ())
     381                 :     1807620 :         return *this;
     382                 :    31673470 :       if (!initialized_p () || !other.initialized_p ())
     383                 :     2147803 :         return uninitialized ();
     384                 :    29525667 :       profile_probability ret;
     385                 :    29525667 :       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     386                 :    29525667 :       ret.set_quality (MIN (quality (), other.quality ()));
     387                 :    29525667 :       return ret;
     388                 :             :     }
     389                 :             : 
     390                 :    11487722 :   profile_probability &operator-= (const profile_probability &other)
     391                 :             :     {
     392                 :    22975444 :       if (*this == never ()
     393                 :    11487722 :           || other == never ())
     394                 :     2264758 :         return *this;
     395                 :     9222964 :       if (!initialized_p () || !other.initialized_p ())
     396                 :     2131284 :         return *this = uninitialized ();
     397                 :             :       else
     398                 :             :         {
     399                 :     7091680 :           m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     400                 :     7091680 :           set_quality (MIN (quality (), other.quality ()));
     401                 :             :         }
     402                 :     7091680 :       return *this;
     403                 :             :     }
     404                 :             : 
     405                 :     1176414 :   profile_probability operator* (const profile_probability &other) const
     406                 :             :     {
     407                 :     2352828 :       if (*this == never ()
     408                 :     1176414 :           || other == never ())
     409                 :      120435 :         return never ();
     410                 :     1055979 :       if (!initialized_p () || !other.initialized_p ())
     411                 :       57800 :         return uninitialized ();
     412                 :      998179 :       profile_probability ret;
     413                 :      998179 :       ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
     414                 :      998179 :       ret.set_quality (MIN (quality (), other.quality ()));
     415                 :      998179 :       return ret;
     416                 :             :     }
     417                 :             : 
     418                 :      156073 :   profile_probability &operator*= (const profile_probability &other)
     419                 :             :     {
     420                 :      312146 :       if (*this == never ()
     421                 :      156073 :           || other == never ())
     422                 :         339 :         return *this = never ();
     423                 :      155734 :       if (!initialized_p () || !other.initialized_p ())
     424                 :           9 :         return *this = uninitialized ();
     425                 :             :       else
     426                 :             :         {
     427                 :      155725 :           m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
     428                 :      294211 :           set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
     429                 :             :         }
     430                 :      155725 :       return *this;
     431                 :             :     }
     432                 :             : 
     433                 :     1729389 :   profile_probability operator/ (const profile_probability &other) const
     434                 :             :     {
     435                 :     1729389 :       if (*this == never ())
     436                 :      107017 :         return never ();
     437                 :     1622372 :       if (!initialized_p () || !other.initialized_p ())
     438                 :      224559 :         return uninitialized ();
     439                 :     1397813 :       profile_probability ret;
     440                 :             :       /* If we get probability above 1, mark it as unreliable and return 1. */
     441                 :     1397813 :       if (m_val >= other.m_val)
     442                 :             :         {
     443                 :       33047 :           ret.m_val = max_probability;
     444                 :       33047 :           ret.set_quality (MIN (MIN (quality (), other.quality ()),
     445                 :             :                                 GUESSED));
     446                 :       33047 :           return ret;
     447                 :             :         }
     448                 :     1364766 :       else if (!m_val)
     449                 :       60841 :         ret.m_val = 0;
     450                 :             :       else
     451                 :             :         {
     452                 :     1303925 :           gcc_checking_assert (other.m_val);
     453                 :     1303925 :           ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
     454                 :             :                                  other.m_val),
     455                 :             :                            max_probability);
     456                 :             :         }
     457                 :     2721563 :       ret.set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
     458                 :     1364766 :       return ret;
     459                 :             :     }
     460                 :             : 
     461                 :      338777 :   profile_probability &operator/= (const profile_probability &other)
     462                 :             :     {
     463                 :      338777 :       if (*this == never ())
     464                 :        1368 :         return *this = never ();
     465                 :      337409 :       if (!initialized_p () || !other.initialized_p ())
     466                 :           0 :         return *this = uninitialized ();
     467                 :             :       else
     468                 :             :         {
     469                 :             :           /* If we get probability above 1, mark it as unreliable
     470                 :             :              and return 1. */
     471                 :      337409 :           if (m_val > other.m_val)
     472                 :             :             {
     473                 :         633 :               m_val = max_probability;
     474                 :         633 :               set_quality (MIN (MIN (quality (), other.quality ()),
     475                 :             :                                 GUESSED));
     476                 :         633 :               return *this;
     477                 :             :             }
     478                 :      336776 :           else if (!m_val)
     479                 :             :             ;
     480                 :             :           else
     481                 :             :             {
     482                 :      334983 :               gcc_checking_assert (other.m_val);
     483                 :      334983 :               m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
     484                 :             :                                  other.m_val),
     485                 :             :                            max_probability);
     486                 :             :             }
     487                 :      610299 :           set_quality (MIN (MIN (quality (), other.quality ()), ADJUSTED));
     488                 :             :         }
     489                 :      336776 :       return *this;
     490                 :             :     }
     491                 :             : 
     492                 :             :   /* Split *THIS (ORIG) probability into 2 probabilities, such that
     493                 :             :      the returned one (FIRST) is *THIS * CPROB and *THIS is
     494                 :             :      adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
     495                 :             :      == ORIG.  This is useful e.g. when splitting a conditional
     496                 :             :      branch like:
     497                 :             :      if (cond)
     498                 :             :        goto lab; // ORIG probability
     499                 :             :      into
     500                 :             :      if (cond1)
     501                 :             :        goto lab; // FIRST = ORIG * CPROB probability
     502                 :             :      if (cond2)
     503                 :             :        goto lab; // SECOND probability
     504                 :             :      such that the overall probability of jumping to lab remains
     505                 :             :      the same.  CPROB gives the relative probability between the
     506                 :             :      branches.  */
     507                 :      289198 :   profile_probability split (const profile_probability &cprob)
     508                 :             :     {
     509                 :      289198 :       profile_probability ret = *this * cprob;
     510                 :             :       /* The following is equivalent to:
     511                 :             :          *this = cprob.invert () * *this / ret.invert ();
     512                 :             :          Avoid scaling when overall outcome is supposed to be always.
     513                 :             :          Without knowing that one is inverse of other, the result would be
     514                 :             :          conservative.  */
     515                 :      289198 :       if (!(*this == always ()))
     516                 :      280519 :         *this = (*this - ret) / ret.invert ();
     517                 :      289198 :       return ret;
     518                 :             :     }
     519                 :             : 
     520                 :      254942 :   gcov_type apply (gcov_type val) const
     521                 :             :     {
     522                 :      254942 :       if (*this == uninitialized ())
     523                 :          63 :         return val / 2;
     524                 :      254879 :       return RDIV (val * m_val, max_probability);
     525                 :             :     }
     526                 :             : 
     527                 :             :   /* Return 1-*THIS.  */
     528                 :    20981584 :   profile_probability invert () const
     529                 :             :     {
     530                 :    30308357 :       return always() - *this;
     531                 :             :     }
     532                 :             : 
     533                 :             :   /* Return THIS with quality dropped to GUESSED.  */
     534                 :      865152 :   profile_probability guessed () const
     535                 :             :     {
     536                 :      865152 :       profile_probability ret = *this;
     537                 :      865152 :       ret.set_quality (GUESSED);
     538                 :      865152 :       return ret;
     539                 :             :     }
     540                 :             : 
     541                 :             :   /* Return THIS with quality dropped to AFDO.  */
     542                 :             :   profile_probability afdo () const
     543                 :             :     {
     544                 :             :       profile_probability ret = *this;
     545                 :             :       ret.set_quality (AFDO);
     546                 :             :       return ret;
     547                 :             :     }
     548                 :             : 
     549                 :             :   /* Return *THIS * NUM / DEN.  */
     550                 :    14342851 :   profile_probability apply_scale (int64_t num, int64_t den) const
     551                 :             :     {
     552                 :    14342851 :       if (*this == never ())
     553                 :       16131 :         return *this;
     554                 :    14326720 :       if (!initialized_p ())
     555                 :     5741511 :         return uninitialized ();
     556                 :     8585209 :       profile_probability ret;
     557                 :     8585209 :       uint64_t tmp;
     558                 :     8585209 :       safe_scale_64bit (m_val, num, den, &tmp);
     559                 :     8585209 :       ret.m_val = MIN (tmp, max_probability);
     560                 :     8585209 :       ret.set_quality (MIN (quality (), ADJUSTED));
     561                 :     8585209 :       return ret;
     562                 :             :     }
     563                 :             : 
     564                 :             :   /* Return *THIS * NUM / DEN.  */
     565                 :       10367 :   profile_probability apply_scale (profile_probability num,
     566                 :             :                                    profile_probability den) const
     567                 :             :     {
     568                 :       10367 :       if (*this == never ())
     569                 :          56 :         return *this;
     570                 :       10311 :       if (num == never ())
     571                 :           0 :         return num;
     572                 :       10311 :       if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
     573                 :           0 :         return uninitialized ();
     574                 :       10311 :       if (num == den)
     575                 :           0 :         return *this;
     576                 :       10311 :       gcc_checking_assert (den.m_val);
     577                 :             : 
     578                 :       10311 :       profile_probability ret;
     579                 :       10311 :       uint64_t val;
     580                 :       10311 :       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
     581                 :       10311 :       ret.m_val = MIN (val, max_probability);
     582                 :       10311 :       ret.set_quality (MIN (MIN (MIN (quality (), ADJUSTED),
     583                 :             :                             num.quality ()), den.quality ()));
     584                 :       10311 :       return ret;
     585                 :             :     }
     586                 :             : 
     587                 :             :   /* Return true when the probability of edge is reliable.
     588                 :             : 
     589                 :             :      The profile guessing code is good at predicting branch outcome (i.e.
     590                 :             :      taken/not taken), that is predicted right slightly over 75% of time.
     591                 :             :      It is however notoriously poor on predicting the probability itself.
     592                 :             :      In general the profile appear a lot flatter (with probabilities closer
     593                 :             :      to 50%) than the reality so it is bad idea to use it to drive optimization
     594                 :             :      such as those disabling dynamic branch prediction for well predictable
     595                 :             :      branches.
     596                 :             : 
     597                 :             :      There are two exceptions - edges leading to noreturn edges and edges
     598                 :             :      predicted by number of iterations heuristics are predicted well.  This macro
     599                 :             :      should be able to distinguish those, but at the moment it simply check for
     600                 :             :      noreturn heuristic that is only one giving probability over 99% or bellow
     601                 :             :      1%.  In future we might want to propagate reliability information across the
     602                 :             :      CFG if we find this information useful on multiple places.   */
     603                 :           0 :   bool probably_reliable_p () const
     604                 :             :     {
     605                 :           0 :       if (quality () >= ADJUSTED)
     606                 :             :         return true;
     607                 :           0 :       if (!initialized_p ())
     608                 :             :         return false;
     609                 :           0 :       return m_val < max_probability / 100
     610                 :           0 :              || m_val > max_probability - max_probability / 100;
     611                 :             :     }
     612                 :             : 
     613                 :             :   /* Return false if profile_probability is bogus.  */
     614                 :  4285475652 :   bool verify () const
     615                 :             :     {
     616                 :           0 :       gcc_checking_assert (quality () != UNINITIALIZED_PROFILE);
     617                 :  4285475652 :       if (m_val == uninitialized_probability)
     618                 :   932941597 :         return quality () == GUESSED;
     619                 :  3352534055 :       else if (quality () < GUESSED)
     620                 :             :         return false;
     621                 :  3352534055 :       return m_val <= max_probability;
     622                 :             :     }
     623                 :             : 
     624                 :             :   /* Comparisons are three-state and conservative.  False is returned if
     625                 :             :      the inequality cannot be decided.  */
     626                 :     6143418 :   bool operator< (const profile_probability &other) const
     627                 :             :     {
     628                 :     6143418 :       return initialized_p () && other.initialized_p () && m_val < other.m_val;
     629                 :             :     }
     630                 :             : 
     631                 :    17525919 :   bool operator> (const profile_probability &other) const
     632                 :             :     {
     633                 :    17525919 :       return initialized_p () && other.initialized_p () && m_val > other.m_val;
     634                 :             :     }
     635                 :             : 
     636                 :     5280300 :   bool operator<= (const profile_probability &other) const
     637                 :             :     {
     638                 :     5280300 :       return initialized_p () && other.initialized_p () && m_val <= other.m_val;
     639                 :             :     }
     640                 :             : 
     641                 :      936519 :   bool operator>= (const profile_probability &other) const
     642                 :             :     {
     643                 :      936519 :       return initialized_p () && other.initialized_p () && m_val >= other.m_val;
     644                 :             :     }
     645                 :             : 
     646                 :          17 :   profile_probability operator* (int64_t num) const
     647                 :             :     {
     648                 :          17 :       return apply_scale (num, 1);
     649                 :             :     }
     650                 :             : 
     651                 :             :   profile_probability operator*= (int64_t num)
     652                 :             :     {
     653                 :             :       *this = apply_scale (num, 1);
     654                 :             :       return *this;
     655                 :             :     }
     656                 :             : 
     657                 :    13691282 :   profile_probability operator/ (int64_t den) const
     658                 :             :     {
     659                 :    13691282 :       return apply_scale (1, den);
     660                 :             :     }
     661                 :             : 
     662                 :      242777 :   profile_probability operator/= (int64_t den)
     663                 :             :     {
     664                 :      242777 :       *this = apply_scale (1, den);
     665                 :      242777 :       return *this;
     666                 :             :     }
     667                 :             : 
     668                 :             :   /* Compute n-th power.  */
     669                 :             :   profile_probability pow (int) const;
     670                 :             : 
     671                 :             :   /* Compute sware root.  */
     672                 :             :   profile_probability sqrt () const;
     673                 :             : 
     674                 :             :   /* Get the value of the probability.  */
     675                 :           0 :   uint32_t value () const { return m_val; }
     676                 :             : 
     677                 :             :   /* Get the quality of the probability.  */
     678                 :  5897278348 :   enum profile_quality quality () const
     679                 :             :   {
     680                 :  5692739356 :     return (profile_quality) (m_adjusted_quality
     681                 :  5692762390 :                               ? m_adjusted_quality + min_quality - 1
     682                 :             :                               : UNINITIALIZED_PROFILE);
     683                 :             :   }
     684                 :             : 
     685                 :             :   /* Output THIS to F.  */
     686                 :             :   void dump (FILE *f) const;
     687                 :             : 
     688                 :             :   /* Output THIS to BUFFER.  */
     689                 :             :   void dump (char *buffer) const;
     690                 :             : 
     691                 :             :   /* Print THIS to stderr.  */
     692                 :             :   void debug () const;
     693                 :             : 
     694                 :             :   /* Return true if THIS is known to differ significantly from OTHER.  */
     695                 :             :   bool differs_from_p (profile_probability other) const;
     696                 :             : 
     697                 :             :   /* Return if difference is greater than 50%.  */
     698                 :             :   bool differs_lot_from_p (profile_probability other) const;
     699                 :             : 
     700                 :             :   /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
     701                 :             :      happens with COUNT2 probability. Return probability that either *THIS or
     702                 :             :      OTHER happens.  */
     703                 :             :   profile_probability combine_with_count (profile_count count1,
     704                 :             :                                           profile_probability other,
     705                 :             :                                           profile_count count2) const;
     706                 :             : 
     707                 :             :   /* Return probability as sreal.  */
     708                 :             :   sreal to_sreal () const;
     709                 :             :   /* LTO streaming support.  */
     710                 :             :   static profile_probability stream_in (class lto_input_block *);
     711                 :             :   void stream_out (struct output_block *);
     712                 :             :   void stream_out (struct lto_output_stream *);
     713                 :             : };
     714                 :             : 
     715                 :             : /* Main data type to hold profile counters in GCC. Profile counts originate
     716                 :             :    either from profile feedback, static profile estimation or both.  We do not
     717                 :             :    perform whole program profile propagation and thus profile estimation
     718                 :             :    counters are often local to function, while counters from profile feedback
     719                 :             :    (or special cases of profile estimation) can be used inter-procedurally.
     720                 :             : 
     721                 :             :    There are 3 basic types
     722                 :             :      1) local counters which are result of intra-procedural static profile
     723                 :             :         estimation.
     724                 :             :      2) ipa counters which are result of profile feedback or special case
     725                 :             :         of static profile estimation (such as in function main).
     726                 :             :      3) counters which counts as 0 inter-procedurally (because given function
     727                 :             :         was never run in train feedback) but they hold local static profile
     728                 :             :         estimate.
     729                 :             : 
     730                 :             :    Counters of type 1 and 3 cannot be mixed with counters of different type
     731                 :             :    within operation (because whole function should use one type of counter)
     732                 :             :    with exception that global zero mix in most operations where outcome is
     733                 :             :    well defined.
     734                 :             : 
     735                 :             :    To take local counter and use it inter-procedurally use ipa member function
     736                 :             :    which strips information irrelevant at the inter-procedural level.
     737                 :             : 
     738                 :             :    Counters are 61bit integers representing number of executions during the
     739                 :             :    train run or normalized frequency within the function.
     740                 :             : 
     741                 :             :    As the profile is maintained during the compilation, many adjustments are
     742                 :             :    made.  Not all transformations can be made precisely, most importantly
     743                 :             :    when code is being duplicated.  It also may happen that part of CFG has
     744                 :             :    profile counts known while other do not - for example when LTO optimizing
     745                 :             :    partly profiled program or when profile was lost due to COMDAT merging.
     746                 :             : 
     747                 :             :    For this reason profile_count tracks more information than
     748                 :             :    just unsigned integer and it is also ready for profile mismatches.
     749                 :             :    The API of this data type represent operations that are natural
     750                 :             :    on profile counts - sum, difference and operation with scales and
     751                 :             :    probabilities.  All operations are safe by never getting negative counts
     752                 :             :    and they do end up in uninitialized scale if any of the parameters is
     753                 :             :    uninitialized.
     754                 :             : 
     755                 :             :    All comparisons that are three state and handling of probabilities.  Thus
     756                 :             :    a < b is not equal to !(a >= b).
     757                 :             : 
     758                 :             :    The following pre-defined counts are available:
     759                 :             : 
     760                 :             :    profile_count::zero ()  for code that is known to execute zero times at
     761                 :             :       runtime (this can be detected statically i.e. for paths leading to
     762                 :             :       abort ();
     763                 :             :    profile_count::one () for code that is known to execute once (such as
     764                 :             :       main () function
     765                 :             :    profile_count::uninitialized ()  for unknown execution count.
     766                 :             : 
     767                 :             :  */
     768                 :             : 
     769                 :             : struct GTY(()) profile_count
     770                 :             : {
     771                 :             : public:
     772                 :             :   /* Use 60bit to hold basic block counters.  Should be at least
     773                 :             :      64bit.  Although a counter cannot be negative, we use a signed
     774                 :             :      type to hold various extra stages.  */
     775                 :             : 
     776                 :             :   static const int n_bits = 60;
     777                 :             :   static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
     778                 :             : private:
     779                 :             :   static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
     780                 :             : 
     781                 :             : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
     782                 :             :   /* Work-around for PR88469.  A bug in the gcc-6/7/8 PCS layout code
     783                 :             :      incorrectly detects the alignment of a structure where the only
     784                 :             :      64-bit aligned object is a bit-field.  We force the alignment of
     785                 :             :      the entire field to mitigate this.  */
     786                 :             : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
     787                 :             : #else
     788                 :             : #define UINT64_BIT_FIELD_ALIGN
     789                 :             : #endif
     790                 :             :   uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
     791                 :             : #undef UINT64_BIT_FIELD_ALIGN
     792                 :             :   enum profile_quality m_quality : 4;
     793                 :             : public:
     794                 :             : 
     795                 :             :   /* Return true if both values can meaningfully appear in single function
     796                 :             :      body.  We have either all counters in function local or global, otherwise
     797                 :             :      operations between them are not really defined well.  */
     798                 :  4832293058 :   bool compatible_p (const profile_count other) const
     799                 :             :     {
     800                 :  8876504742 :       if (!initialized_p () || !other.initialized_p ())
     801                 :             :         return true;
     802                 : 16014997365 :       if (*this == zero ()
     803                 :  7933854279 :           || other == zero ())
     804                 :   147289899 :         return true;
     805                 :             :       /* Do not allow nonzero global profile together with local guesses
     806                 :             :          that are globally0.  */
     807                 :  3893281644 :       if (ipa ().nonzero_p ()
     808                 :      244815 :           && !(other.ipa () == other))
     809                 :           0 :         return false;
     810                 :  3893281644 :       if (other.ipa ().nonzero_p ()
     811                 :      254121 :           && !(ipa () == *this))
     812                 :           0 :         return false;
     813                 :             : 
     814                 :  3893281644 :       return ipa_p () == other.ipa_p ();
     815                 :             :     }
     816                 :             : 
     817                 :             :   /* Used for counters which are expected to be never executed.  */
     818                 : 17942344136 :   static profile_count zero ()
     819                 :             :     {
     820                 : 13950122866 :       return from_gcov_type (0);
     821                 :             :     }
     822                 :             : 
     823                 :       10527 :   static profile_count adjusted_zero ()
     824                 :             :     {
     825                 :       10527 :       profile_count c;
     826                 :       10527 :       c.m_val = 0;
     827                 :       10527 :       c.m_quality = ADJUSTED;
     828                 :       10527 :       return c;
     829                 :             :     }
     830                 :             : 
     831                 :           0 :   static profile_count afdo_zero ()
     832                 :             :     {
     833                 :           0 :       profile_count c;
     834                 :           0 :       c.m_val = 0;
     835                 :           0 :       c.m_quality = AFDO;
     836                 :           0 :       return c;
     837                 :             :     }
     838                 :             : 
     839                 :             :   static profile_count guessed_zero ()
     840                 :             :     {
     841                 :             :       profile_count c;
     842                 :             :       c.m_val = 0;
     843                 :             :       c.m_quality = GUESSED;
     844                 :             :       return c;
     845                 :             :     }
     846                 :             : 
     847                 :       48562 :   static profile_count one ()
     848                 :             :     {
     849                 :       48365 :       return from_gcov_type (1);
     850                 :             :     }
     851                 :             : 
     852                 :             :   /* Value of counters which has not been initialized. Either because
     853                 :             :      initialization did not happen yet or because profile is unknown.  */
     854                 :  9134244056 :   static profile_count uninitialized ()
     855                 :             :     {
     856                 :  9134244056 :       profile_count c;
     857                 :  9134244056 :       c.m_val = uninitialized_count;
     858                 :  9134244056 :       c.m_quality = GUESSED_LOCAL;
     859                 :  9104102364 :       return c;
     860                 :             :     }
     861                 :             : 
     862                 :             :   /* Conversion to gcov_type is lossy.  */
     863                 :      299015 :   gcov_type to_gcov_type () const
     864                 :             :     {
     865                 :           0 :       gcc_checking_assert (initialized_p ());
     866                 :      299015 :       return m_val;
     867                 :             :     }
     868                 :             : 
     869                 :             :   /* Return true if value has been initialized.  */
     870                 : 31959782761 :   bool initialized_p () const
     871                 :             :     {
     872                 : 10586008168 :       return m_val != uninitialized_count;
     873                 :             :     }
     874                 :             : 
     875                 :             :   /* Return true if value can be trusted.  */
     876                 :    13868678 :   bool reliable_p () const
     877                 :             :     {
     878                 :    13868414 :       return m_quality >= ADJUSTED;
     879                 :             :     }
     880                 :             : 
     881                 :             :   /* Return true if value can be operated inter-procedurally.  */
     882                 :  8786884349 :   bool ipa_p () const
     883                 :             :     {
     884                 :  4881332204 :       return !initialized_p () || m_quality >= GUESSED_GLOBAL0_AFDO;
     885                 :             :     }
     886                 :             : 
     887                 :             :   /* Return true if quality of profile is precise.  */
     888                 :    53063291 :   bool precise_p () const
     889                 :             :     {
     890                 :    53063291 :       return m_quality == PRECISE;
     891                 :             :     }
     892                 :             : 
     893                 :             :   /* Get the value of the count.  */
     894                 :           0 :   uint64_t value () const { return m_val; }
     895                 :             : 
     896                 :             :   /* Get the quality of the count.  */
     897                 :    67276363 :   enum profile_quality quality () const { return m_quality; }
     898                 :             : 
     899                 :             :   /* When merging basic blocks, the two different profile counts are unified.
     900                 :             :      Return true if this can be done without losing info about profile.
     901                 :             :      The only case we care about here is when first BB contains something
     902                 :             :      that makes it terminate in a way not visible in CFG.  */
     903                 :     2938119 :   bool ok_for_merging (profile_count other) const
     904                 :             :     {
     905                 :     2938119 :       if (m_quality < ADJUSTED
     906                 :       10252 :           || other.m_quality < ADJUSTED)
     907                 :             :         return true;
     908                 :        9467 :       return !(other < *this);
     909                 :             :     }
     910                 :             : 
     911                 :             :   /* When merging two BBs with different counts, pick common count that looks
     912                 :             :      most representative.  */
     913                 :    15197040 :   profile_count merge (profile_count other) const
     914                 :             :     {
     915                 :     1112063 :       if (*this == other || !other.initialized_p ()
     916                 :     1111822 :           || m_quality > other.m_quality)
     917                 :    14086163 :         return *this;
     918                 :     1110877 :       if (other.m_quality > m_quality
     919                 :     1110877 :           || other > *this)
     920                 :      438515 :         return other;
     921                 :      672362 :       return *this;
     922                 :             :     }
     923                 :             : 
     924                 :             :   /* Basic operations.  */
     925                 : 19401583081 :   bool operator== (const profile_count &other) const
     926                 :             :     {
     927                 : 14637816112 :       return m_val == other.m_val && m_quality == other.m_quality;
     928                 :             :     }
     929                 :             : 
     930                 :     3643493 :   profile_count operator+ (const profile_count &other) const
     931                 :             :     {
     932                 :     3643493 :       if (other == zero ())
     933                 :        2235 :         return *this;
     934                 :     3641258 :       if (*this == zero ())
     935                 :      178100 :         return other;
     936                 :     3463158 :       if (!initialized_p () || !other.initialized_p ())
     937                 :        6050 :         return uninitialized ();
     938                 :             : 
     939                 :     3457108 :       profile_count ret;
     940                 :     3457108 :       gcc_checking_assert (compatible_p (other));
     941                 :     3457108 :       uint64_t ret_val = m_val + other.m_val;
     942                 :     3457108 :       ret.m_val = MIN (ret_val, max_count);
     943                 :     3457108 :       ret.m_quality = MIN (m_quality, other.m_quality);
     944                 :     3457108 :       return ret;
     945                 :             :     }
     946                 :             : 
     947                 :    72293657 :   profile_count &operator+= (const profile_count &other)
     948                 :             :     {
     949                 :    72293657 :       if (other == zero ())
     950                 :     5099415 :         return *this;
     951                 :    67194242 :       if (*this == zero ())
     952                 :             :         {
     953                 :    41228795 :           *this = other;
     954                 :    41228795 :           return *this;
     955                 :             :         }
     956                 :    25965447 :       if (!initialized_p () || !other.initialized_p ())
     957                 :      331705 :         return *this = uninitialized ();
     958                 :             :       else
     959                 :             :         {
     960                 :    25633742 :           gcc_checking_assert (compatible_p (other));
     961                 :    25633742 :           uint64_t ret_val = m_val + other.m_val;
     962                 :    25633742 :           m_val = MIN (ret_val, max_count);
     963                 :    25633742 :           m_quality = MIN (m_quality, other.m_quality);
     964                 :             :         }
     965                 :    25633742 :       return *this;
     966                 :             :     }
     967                 :             : 
     968                 :    19174748 :   profile_count operator- (const profile_count &other) const
     969                 :             :     {
     970                 :    19190930 :       if (*this == zero () || other == zero ())
     971                 :     1173559 :         return *this;
     972                 :    18001189 :       if (!initialized_p () || !other.initialized_p ())
     973                 :     7728226 :         return uninitialized ();
     974                 :    10272963 :       gcc_checking_assert (compatible_p (other));
     975                 :    10272963 :       profile_count ret;
     976                 :    10272963 :       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     977                 :    10272963 :       ret.m_quality = MIN (m_quality, other.m_quality);
     978                 :    10272963 :       return ret;
     979                 :             :     }
     980                 :             : 
     981                 :     9172026 :   profile_count &operator-= (const profile_count &other)
     982                 :             :     {
     983                 :     9613548 :       if (*this == zero () || other == zero ())
     984                 :      312725 :         return *this;
     985                 :     8859301 :       if (!initialized_p () || !other.initialized_p ())
     986                 :      353850 :         return *this = uninitialized ();
     987                 :             :       else
     988                 :             :         {
     989                 :     8505451 :           gcc_checking_assert (compatible_p (other));
     990                 :     8505451 :           m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     991                 :     8505451 :           m_quality = MIN (m_quality, other.m_quality);
     992                 :             :         }
     993                 :     8505451 :       return *this;
     994                 :             :     }
     995                 :             : 
     996                 :             :   /* Return false if profile_count is bogus.  */
     997                 :  3270008820 :   bool verify () const
     998                 :             :     {
     999                 :  3270008820 :       gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
    1000                 :  3270008820 :       return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
    1001                 :             :     }
    1002                 :             : 
    1003                 :             :   /* Comparisons are three-state and conservative.  False is returned if
    1004                 :             :      the inequality cannot be decided.  */
    1005                 :  1045234334 :   bool operator< (const profile_count &other) const
    1006                 :             :     {
    1007                 :  1045234334 :       if (!initialized_p () || !other.initialized_p ())
    1008                 :             :         return false;
    1009                 :  1040669101 :       if (*this == zero ())
    1010                 :     4021840 :         return !(other == zero ());
    1011                 :  1038658181 :       if (other == zero ())
    1012                 :        1051 :         return false;
    1013                 :  1038657130 :       gcc_checking_assert (compatible_p (other));
    1014                 :  1038657130 :       return m_val < other.m_val;
    1015                 :             :     }
    1016                 :             : 
    1017                 :    34670791 :   bool operator> (const profile_count &other) const
    1018                 :             :     {
    1019                 :    34670791 :       if (!initialized_p () || !other.initialized_p ())
    1020                 :             :         return false;
    1021                 :    33719053 :       if (*this  == zero ())
    1022                 :      218145 :         return false;
    1023                 :    33500908 :       if (other == zero ())
    1024                 :     1609630 :         return !(*this == zero ());
    1025                 :    32696093 :       gcc_checking_assert (compatible_p (other));
    1026                 :    32696093 :       return initialized_p () && other.initialized_p () && m_val > other.m_val;
    1027                 :             :     }
    1028                 :             : 
    1029                 :             :   bool operator< (const gcov_type other) const
    1030                 :             :     {
    1031                 :             :       gcc_checking_assert (ipa_p ());
    1032                 :             :       gcc_checking_assert (other >= 0);
    1033                 :             :       return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
    1034                 :             :     }
    1035                 :             : 
    1036                 :      274560 :   bool operator> (const gcov_type other) const
    1037                 :             :     {
    1038                 :      274560 :       gcc_checking_assert (ipa_p ());
    1039                 :      274560 :       gcc_checking_assert (other >= 0);
    1040                 :      274560 :       return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
    1041                 :             :     }
    1042                 :             : 
    1043                 :     1097643 :   bool operator<= (const profile_count &other) const
    1044                 :             :     {
    1045                 :     1097643 :       if (!initialized_p () || !other.initialized_p ())
    1046                 :             :         return false;
    1047                 :     1097643 :       if (*this == zero ())
    1048                 :        3669 :         return true;
    1049                 :     1093974 :       if (other == zero ())
    1050                 :           0 :         return (*this == zero ());
    1051                 :     1093974 :       gcc_checking_assert (compatible_p (other));
    1052                 :     1093974 :       return m_val <= other.m_val;
    1053                 :             :     }
    1054                 :             : 
    1055                 :     1372747 :   bool operator>= (const profile_count &other) const
    1056                 :             :     {
    1057                 :     1372747 :       if (!initialized_p () || !other.initialized_p ())
    1058                 :             :         return false;
    1059                 :     1371833 :       if (other == zero ())
    1060                 :         292 :         return true;
    1061                 :     1371541 :       if (*this == zero ())
    1062                 :       25370 :         return (other == zero ());
    1063                 :     1358856 :       gcc_checking_assert (compatible_p (other));
    1064                 :     1358856 :       return m_val >= other.m_val;
    1065                 :             :     }
    1066                 :             : 
    1067                 :       92794 :   bool operator<= (const gcov_type other) const
    1068                 :             :     {
    1069                 :       92794 :       gcc_checking_assert (ipa_p ());
    1070                 :       92794 :       gcc_checking_assert (other >= 0);
    1071                 :       92794 :       return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
    1072                 :             :     }
    1073                 :             : 
    1074                 :       57884 :   bool operator>= (const gcov_type other) const
    1075                 :             :     {
    1076                 :       57884 :       gcc_checking_assert (ipa_p ());
    1077                 :       57884 :       gcc_checking_assert (other >= 0);
    1078                 :       57884 :       return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
    1079                 :             :     }
    1080                 :             : 
    1081                 :   872457967 :   profile_count operator* (int64_t num) const
    1082                 :             :     {
    1083                 :   872389941 :       return apply_scale (num, 1);
    1084                 :             :     }
    1085                 :             : 
    1086                 :             :   profile_count operator*= (int64_t num)
    1087                 :             :     {
    1088                 :             :       *this = apply_scale (num, 1);
    1089                 :             :       return *this;
    1090                 :             :     }
    1091                 :             : 
    1092                 :             :   profile_count operator* (const sreal &num) const;
    1093                 :             :   profile_count operator*= (const sreal &num);
    1094                 :             : 
    1095                 :     2292739 :   profile_count operator/ (int64_t den) const
    1096                 :             :     {
    1097                 :     2292739 :       return apply_scale (1, den);
    1098                 :             :     }
    1099                 :             : 
    1100                 :          70 :   profile_count operator/= (int64_t den)
    1101                 :             :     {
    1102                 :          62 :       *this = apply_scale (1, den);
    1103                 :          70 :       return *this;
    1104                 :             :     }
    1105                 :             : 
    1106                 :             :   /* Return true when value is not zero and can be used for scaling.
    1107                 :             :      This is different from *this > 0 because that requires counter to
    1108                 :             :      be IPA.  */
    1109                 :  8151465335 :   bool nonzero_p () const
    1110                 :             :     {
    1111                 :  7982997793 :       return initialized_p () && m_val != 0;
    1112                 :             :     }
    1113                 :             : 
    1114                 :             :   /* Make counter forcibly nonzero.  */
    1115                 :    19898193 :   profile_count force_nonzero () const
    1116                 :             :     {
    1117                 :    19898193 :       if (!initialized_p ())
    1118                 :       10977 :         return *this;
    1119                 :    19887216 :       profile_count ret = *this;
    1120                 :    19887216 :       if (ret.m_val == 0)
    1121                 :             :         {
    1122                 :       21904 :           ret.m_val = 1;
    1123                 :       21904 :           ret.m_quality = MIN (m_quality, ADJUSTED);
    1124                 :             :         }
    1125                 :    19887216 :       return ret;
    1126                 :             :     }
    1127                 :             : 
    1128                 :    89190838 :   profile_count max (profile_count other) const
    1129                 :             :     {
    1130                 :    89190838 :       profile_count val = *this;
    1131                 :             : 
    1132                 :             :       /* Always prefer nonzero IPA counts over local counts.  */
    1133                 :    99089920 :       if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
    1134                 :             :         {
    1135                 :        9693 :           val = ipa ();
    1136                 :        9693 :           other = other.ipa ();
    1137                 :             :         }
    1138                 :    89190838 :       if (!initialized_p ())
    1139                 :    21474926 :         return other;
    1140                 :    67715912 :       if (!other.initialized_p ())
    1141                 :     2168367 :         return *this;
    1142                 :    65547545 :       if (*this == zero ())
    1143                 :     2131956 :         return other;
    1144                 :    63415589 :       if (other == zero ())
    1145                 :     4602111 :         return *this;
    1146                 :    58813478 :       gcc_checking_assert (compatible_p (other));
    1147                 :    58813478 :       if (val.m_val < other.m_val || (m_val == other.m_val
    1148                 :    13568744 :                                       && val.m_quality < other.m_quality))
    1149                 :     8233301 :         return other;
    1150                 :    50580177 :       return *this;
    1151                 :             :     }
    1152                 :             : 
    1153                 :             :   /* PROB is a probability in scale 0...REG_BR_PROB_BASE.  Scale counter
    1154                 :             :      accordingly.  */
    1155                 :             :   profile_count apply_probability (int prob) const
    1156                 :             :     {
    1157                 :             :       gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
    1158                 :             :       if (m_val == 0)
    1159                 :             :         return *this;
    1160                 :             :       if (!initialized_p ())
    1161                 :             :         return uninitialized ();
    1162                 :             :       profile_count ret;
    1163                 :             :       uint64_t tmp;
    1164                 :             :       safe_scale_64bit (m_val, prob, REG_BR_PROB_BASE, &tmp);
    1165                 :             :       ret.m_val = tmp;
    1166                 :             :       ret.m_quality = MIN (m_quality, ADJUSTED);
    1167                 :             :       return ret;
    1168                 :             :     }
    1169                 :             : 
    1170                 :             :   /* Scale counter according to PROB.  */
    1171                 :   372772670 :   profile_count apply_probability (profile_probability prob) const
    1172                 :             :     {
    1173                 :   380511765 :       if (*this == zero () || prob == profile_probability::always ())
    1174                 :   118489323 :         return *this;
    1175                 :   254283347 :       if (prob == profile_probability::never ())
    1176                 :    19448637 :         return zero ();
    1177                 :   234834710 :       if (!initialized_p () || !prob.initialized_p ())
    1178                 :    30318752 :         return uninitialized ();
    1179                 :   204515958 :       profile_count ret;
    1180                 :   204515958 :       uint64_t tmp;
    1181                 :   204515958 :       safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
    1182                 :             :                         &tmp);
    1183                 :   204515958 :       ret.m_val = tmp;
    1184                 :   204538992 :       ret.m_quality = MIN (m_quality, prob.quality ());
    1185                 :   204515958 :       return ret;
    1186                 :             :     }
    1187                 :             : 
    1188                 :             :   /* Return *THIS * NUM / DEN.  */
    1189                 :   987381808 :   profile_count apply_scale (int64_t num, int64_t den) const
    1190                 :             :     {
    1191                 :   987381808 :       if (m_val == 0)
    1192                 :    23816715 :         return *this;
    1193                 :   963565093 :       if (!initialized_p ())
    1194                 :        7944 :         return uninitialized ();
    1195                 :   963557149 :       profile_count ret;
    1196                 :   963557149 :       uint64_t tmp;
    1197                 :             : 
    1198                 :   963557149 :       gcc_checking_assert (num >= 0 && den > 0);
    1199                 :   963557149 :       safe_scale_64bit (m_val, num, den, &tmp);
    1200                 :   963557149 :       ret.m_val = MIN (tmp, max_count);
    1201                 :   963557149 :       ret.m_quality = MIN (m_quality, ADJUSTED);
    1202                 :   963557149 :       return ret;
    1203                 :             :     }
    1204                 :             : 
    1205                 :    26632064 :   profile_count apply_scale (profile_count num, profile_count den) const
    1206                 :             :     {
    1207                 :    26632064 :       if (*this == zero ())
    1208                 :     2057217 :         return *this;
    1209                 :    24574847 :       if (num == zero ())
    1210                 :       76581 :         return num;
    1211                 :    24498266 :       if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
    1212                 :     6824600 :         return uninitialized ();
    1213                 :    17673666 :       if (num == den)
    1214                 :     5309606 :         return *this;
    1215                 :    12364060 :       gcc_checking_assert (den.m_val);
    1216                 :             : 
    1217                 :    12364060 :       profile_count ret;
    1218                 :    12364060 :       uint64_t val;
    1219                 :    12364060 :       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
    1220                 :    12364060 :       ret.m_val = MIN (val, max_count);
    1221                 :    12364060 :       ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
    1222                 :             :                            num.m_quality), den.m_quality);
    1223                 :             :       /* Be sure that ret is not local if num is global.
    1224                 :             :          Also ensure that ret is not global0 when num is global.  */
    1225                 :    12364060 :       if (num.ipa_p ())
    1226                 :        3600 :         ret.m_quality = MAX (ret.m_quality,
    1227                 :             :                              num == num.ipa () ? GUESSED : num.m_quality);
    1228                 :    12364060 :       return ret;
    1229                 :             :     }
    1230                 :             : 
    1231                 :             :   /* Return THIS with quality dropped to GUESSED_LOCAL.  */
    1232                 :    17530733 :   profile_count guessed_local () const
    1233                 :             :     {
    1234                 :    17530733 :       profile_count ret = *this;
    1235                 :    17530733 :       if (!initialized_p ())
    1236                 :           0 :         return *this;
    1237                 :    17530733 :       ret.m_quality = GUESSED_LOCAL;
    1238                 :    17530733 :       return ret;
    1239                 :             :     }
    1240                 :             : 
    1241                 :             :   /* We know that profile is globally 0 but keep local profile if present.  */
    1242                 :         788 :   profile_count global0 () const
    1243                 :             :     {
    1244                 :         788 :       profile_count ret = *this;
    1245                 :         788 :       if (!initialized_p ())
    1246                 :           0 :         return *this;
    1247                 :         788 :       ret.m_quality = GUESSED_GLOBAL0;
    1248                 :         788 :       return ret;
    1249                 :             :     }
    1250                 :             : 
    1251                 :             :   /* We know that profile is globally afdo 0 but keep local profile
    1252                 :             :      if present.  */
    1253                 :           0 :   profile_count global0afdo () const
    1254                 :             :     {
    1255                 :           0 :       profile_count ret = *this;
    1256                 :           0 :       if (!initialized_p ())
    1257                 :           0 :         return *this;
    1258                 :           0 :       ret.m_quality = GUESSED_GLOBAL0_AFDO;
    1259                 :           0 :       return ret;
    1260                 :             :     }
    1261                 :             : 
    1262                 :             :   /* We know that profile is globally adjusted 0 but keep local profile
    1263                 :             :      if present.  */
    1264                 :          36 :   profile_count global0adjusted () const
    1265                 :             :     {
    1266                 :          36 :       profile_count ret = *this;
    1267                 :          36 :       if (!initialized_p ())
    1268                 :           0 :         return *this;
    1269                 :          36 :       ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
    1270                 :          36 :       return ret;
    1271                 :             :     }
    1272                 :             : 
    1273                 :             :   /* Return THIS with quality dropped to GUESSED.  */
    1274                 :         293 :   profile_count guessed () const
    1275                 :             :     {
    1276                 :         293 :       profile_count ret = *this;
    1277                 :         293 :       ret.m_quality = MIN (ret.m_quality, GUESSED);
    1278                 :         293 :       return ret;
    1279                 :             :     }
    1280                 :             : 
    1281                 :             :   /* Return THIS with quality GUESSED.  */
    1282                 :           0 :   profile_count force_guessed () const
    1283                 :             :     {
    1284                 :           0 :       profile_count ret = *this;
    1285                 :           0 :       gcc_checking_assert (initialized_p ());
    1286                 :           0 :       ret.m_quality = GUESSED;
    1287                 :           0 :       return ret;
    1288                 :             :     }
    1289                 :             : 
    1290                 :             :   /* Return variant of profile count which is always safe to compare
    1291                 :             :      across functions.  */
    1292                 :  9015089908 :   profile_count ipa () const
    1293                 :             :     {
    1294                 :  9015089908 :       if (m_quality > GUESSED_GLOBAL0)
    1295                 :    22147107 :         return *this;
    1296                 :  8992942801 :       if (m_quality == GUESSED_GLOBAL0)
    1297                 :      220626 :         return zero ();
    1298                 :  8992722175 :       if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
    1299                 :       10527 :         return adjusted_zero ();
    1300                 :  8992711648 :       if (m_quality == GUESSED_GLOBAL0_AFDO)
    1301                 :           0 :         return afdo_zero ();
    1302                 :  8992711648 :       return uninitialized ();
    1303                 :             :     }
    1304                 :             : 
    1305                 :             :   /* Return THIS with quality dropped to AFDO.  */
    1306                 :           0 :   profile_count afdo () const
    1307                 :             :     {
    1308                 :           0 :       profile_count ret = *this;
    1309                 :           0 :       ret.m_quality = AFDO;
    1310                 :           0 :       return ret;
    1311                 :             :     }
    1312                 :             : 
    1313                 :             :   /* Return probability of event with counter THIS within event with counter
    1314                 :             :      OVERALL.  */
    1315                 :  1072028407 :   profile_probability probability_in (const profile_count overall) const
    1316                 :             :     {
    1317                 :  1072043114 :       if (*this == zero ()
    1318                 :       14707 :           && !(overall == zero ()))
    1319                 :        1594 :         return profile_probability::never ();
    1320                 :  1072026255 :       if (!initialized_p () || !overall.initialized_p ()
    1321                 :  2144053068 :           || !overall.m_val)
    1322                 :       62439 :         return profile_probability::uninitialized ();
    1323                 :  1281652876 :       if (*this == overall && m_quality == PRECISE)
    1324                 :       29732 :         return profile_probability::always ();
    1325                 :  1071934642 :       profile_probability ret;
    1326                 :  1071934642 :       gcc_checking_assert (compatible_p (overall));
    1327                 :             : 
    1328                 :  1071934642 :       if (overall.m_val < m_val)
    1329                 :             :         {
    1330                 :       93850 :           ret.m_val = profile_probability::max_probability;
    1331                 :       93850 :           ret.set_quality (GUESSED);
    1332                 :       93850 :           return ret;
    1333                 :             :         }
    1334                 :             :       else
    1335                 :  1071840792 :         ret.m_val = RDIV (m_val * profile_probability::max_probability,
    1336                 :             :                           overall.m_val);
    1337                 :  1071840792 :       ret.set_quality (MIN (MAX (MIN (m_quality, overall.m_quality),
    1338                 :             :                                  GUESSED), ADJUSTED));
    1339                 :  1071840792 :       return ret;
    1340                 :             :     }
    1341                 :             : 
    1342                 :             :   /* Return true if profile count is very large, so we risk overflows
    1343                 :             :      with loop transformations.  */
    1344                 :             :   bool
    1345                 :     1015839 :   very_large_p ()
    1346                 :             :   {
    1347                 :     1015839 :     if (!initialized_p ())
    1348                 :             :       return false;
    1349                 :     1015839 :     return m_val > max_count / 65536;
    1350                 :             :   }
    1351                 :             : 
    1352                 :             :   int to_frequency (struct function *fun) const;
    1353                 :             :   int to_cgraph_frequency (profile_count entry_bb_count) const;
    1354                 :             :   sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
    1355                 :             : 
    1356                 :             :   /* Output THIS to F.  */
    1357                 :             :   void dump (FILE *f, struct function *fun = NULL) const;
    1358                 :             : 
    1359                 :             :   /* Print THIS to stderr.  */
    1360                 :             :   void debug () const;
    1361                 :             : 
    1362                 :             :   /* Return true if THIS is known to differ significantly from OTHER.  */
    1363                 :             :   bool differs_from_p (profile_count other) const;
    1364                 :             : 
    1365                 :             :   /* We want to scale profile across function boundary from NUM to DEN.
    1366                 :             :      Take care of the side case when NUM and DEN are zeros of incompatible
    1367                 :             :      kinds.  */
    1368                 :             :   static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
    1369                 :             : 
    1370                 :             :   /* THIS is a count of bb which is known to be executed IPA times.
    1371                 :             :      Combine this information into bb counter.  This means returning IPA
    1372                 :             :      if it is nonzero, not changing anything if IPA is uninitialized
    1373                 :             :      and if IPA is zero, turning THIS into corresponding local profile with
    1374                 :             :      global0.  */
    1375                 :             :   profile_count combine_with_ipa_count (profile_count ipa);
    1376                 :             : 
    1377                 :             :   /* Same as combine_with_ipa_count but inside function with count IPA2.  */
    1378                 :             :   profile_count combine_with_ipa_count_within
    1379                 :             :                  (profile_count ipa, profile_count ipa2);
    1380                 :             : 
    1381                 :             :   /* The profiling runtime uses gcov_type, which is usually 64bit integer.
    1382                 :             :      Conversions back and forth are used to read the coverage and get it
    1383                 :             :      into internal representation.  */
    1384                 :             :   static profile_count from_gcov_type (gcov_type v,
    1385                 :             :                                        profile_quality quality = PRECISE);
    1386                 :             : 
    1387                 :             :   /* LTO streaming support.  */
    1388                 :             :   static profile_count stream_in (class lto_input_block *);
    1389                 :             :   void stream_out (struct output_block *);
    1390                 :             :   void stream_out (struct lto_output_stream *);
    1391                 :             : };
    1392                 :             : #endif
        

Generated by: LCOV version 2.1-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.