LCOV - code coverage report
Current view: top level - gcc - ipa-fnsummary.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 96.6 % 2681 2589
Test Date: 2026-11-14 16:39:50 Functions: 98.9 % 94 93
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Function summary pass.
       2              :    Copyright (C) 2003-2026 Free Software Foundation, Inc.
       3              :    Contributed by Jan Hubicka
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it under
       8              : the terms of the GNU General Public License as published by the Free
       9              : Software Foundation; either version 3, or (at your option) any later
      10              : version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : /* Analysis of function bodies used by inter-procedural passes
      22              : 
      23              :    We estimate for each function
      24              :      - function body size and size after specializing into given context
      25              :      - average function execution time in a given context
      26              :      - function frame size
      27              :    For each call
      28              :      - call statement size, time and how often the parameters change
      29              : 
      30              :    ipa_fn_summary data structures store above information locally (i.e.
      31              :    parameters of the function itself) and globally (i.e. parameters of
      32              :    the function created by applying all the inline decisions already
      33              :    present in the callgraph).
      34              : 
      35              :    We provide access to the ipa_fn_summary data structure and
      36              :    basic logic updating the parameters when inlining is performed.
      37              : 
      38              :    The summaries are context sensitive.  Context means
      39              :      1) partial assignment of known constant values of operands
      40              :      2) whether function is inlined into the call or not.
      41              :    It is easy to add more variants.  To represent function size and time
      42              :    that depends on context (i.e. it is known to be optimized away when
      43              :    context is known either by inlining or from IP-CP and cloning),
      44              :    we use predicates.
      45              : 
      46              :    estimate_edge_size_and_time can be used to query
      47              :    function size/time in the given context.  ipa_merge_fn_summary_after_inlining merges
      48              :    properties of caller and callee after inlining.
      49              : 
      50              :    Finally pass_inline_parameters is exported.  This is used to drive
      51              :    computation of function parameters used by the early inliner. IPA
      52              :    inlined performs analysis via its analyze_function method. */
      53              : 
      54              : #include "config.h"
      55              : #define INCLUDE_VECTOR
      56              : #include "system.h"
      57              : #include "coretypes.h"
      58              : #include "backend.h"
      59              : #include "target.h"
      60              : #include "tree.h"
      61              : #include "gimple.h"
      62              : #include "alloc-pool.h"
      63              : #include "tree-pass.h"
      64              : #include "ssa.h"
      65              : #include "tree-streamer.h"
      66              : #include "cgraph.h"
      67              : #include "diagnostic.h"
      68              : #include "fold-const.h"
      69              : #include "print-tree.h"
      70              : #include "tree-inline.h"
      71              : #include "gimple-pretty-print.h"
      72              : #include "cfganal.h"
      73              : #include "gimple-iterator.h"
      74              : #include "tree-cfg.h"
      75              : #include "tree-ssa-loop-niter.h"
      76              : #include "tree-ssa-loop.h"
      77              : #include "symbol-summary.h"
      78              : #include "sreal.h"
      79              : #include "ipa-cp.h"
      80              : #include "ipa-prop.h"
      81              : #include "ipa-fnsummary.h"
      82              : #include "cfgloop.h"
      83              : #include "tree-scalar-evolution.h"
      84              : #include "ipa-utils.h"
      85              : #include "cfgexpand.h"
      86              : #include "gimplify.h"
      87              : #include "stringpool.h"
      88              : #include "attribs.h"
      89              : #include "tree-into-ssa.h"
      90              : #include "symtab-clones.h"
      91              : #include "gimple-range.h"
      92              : #include "tree-dfa.h"
      93              : #include "attr-callback.h"
      94              : #include "callback-info.h"
      95              : 
      96              : /* Summaries.  */
      97              : fast_function_summary <ipa_fn_summary *, va_gc> *ipa_fn_summaries;
      98              : fast_function_summary <ipa_size_summary *, va_heap> *ipa_size_summaries;
      99              : fast_call_summary <ipa_call_summary *, va_heap> *ipa_call_summaries;
     100              : 
     101              : /* Edge predicates goes here.  */
     102              : static object_allocator<ipa_predicate> edge_predicate_pool ("edge predicates");
     103              : 
     104              : 
     105              : /* Dump IPA hints.  */
     106              : void
     107          185 : ipa_dump_hints (FILE *f, ipa_hints hints)
     108              : {
     109          185 :   if (!hints)
     110              :     return;
     111          148 :   fprintf (f, "IPA hints:");
     112          148 :   if (hints & INLINE_HINT_indirect_call)
     113              :     {
     114           23 :       hints &= ~INLINE_HINT_indirect_call;
     115           23 :       fprintf (f, " indirect_call");
     116              :     }
     117          148 :   if (hints & INLINE_HINT_loop_iterations)
     118              :     {
     119            3 :       hints &= ~INLINE_HINT_loop_iterations;
     120            3 :       fprintf (f, " loop_iterations");
     121              :     }
     122          148 :   if (hints & INLINE_HINT_loop_stride)
     123              :     {
     124            3 :       hints &= ~INLINE_HINT_loop_stride;
     125            3 :       fprintf (f, " loop_stride");
     126              :     }
     127          148 :   if (hints & INLINE_HINT_same_scc)
     128              :     {
     129            4 :       hints &= ~INLINE_HINT_same_scc;
     130            4 :       fprintf (f, " same_scc");
     131              :     }
     132          148 :   if (hints & INLINE_HINT_in_scc)
     133              :     {
     134           11 :       hints &= ~INLINE_HINT_in_scc;
     135           11 :       fprintf (f, " in_scc");
     136              :     }
     137          148 :   if (hints & INLINE_HINT_cross_module)
     138              :     {
     139            2 :       hints &= ~INLINE_HINT_cross_module;
     140            2 :       fprintf (f, " cross_module");
     141              :     }
     142          148 :   if (hints & INLINE_HINT_declared_inline)
     143              :     {
     144          122 :       hints &= ~INLINE_HINT_declared_inline;
     145          122 :       fprintf (f, " declared_inline");
     146              :     }
     147          148 :   if (hints & INLINE_HINT_known_hot)
     148              :     {
     149            1 :       hints &= ~INLINE_HINT_known_hot;
     150            1 :       fprintf (f, " known_hot");
     151              :     }
     152          148 :   if (hints & INLINE_HINT_builtin_constant_p)
     153              :     {
     154            4 :       hints &= ~INLINE_HINT_builtin_constant_p;
     155            4 :       fprintf (f, " builtin_constant_p");
     156              :     }
     157          148 :   gcc_assert (!hints);
     158              : }
     159              : 
     160              : 
     161              : /* Record SIZE and TIME to SUMMARY.
     162              :    The accounted code will be executed when EXEC_PRED is true.
     163              :    When NONCONST_PRED is false the code will evaluate to constant and
     164              :    will get optimized out in specialized clones of the function.
     165              :    If CALL is true account to call_size_time_table rather than
     166              :    size_time_table.   */
     167              : 
     168              : void
     169    139417274 : ipa_fn_summary::account_size_time (int size, sreal time,
     170              :                                    const ipa_predicate &exec_pred,
     171              :                                    const ipa_predicate &nonconst_pred_in,
     172              :                                    bool call)
     173              : {
     174    139417274 :   size_time_entry *e;
     175    139417274 :   bool found = false;
     176    139417274 :   int i;
     177    139417274 :   ipa_predicate nonconst_pred;
     178    139417274 :   vec<size_time_entry> *table = call ? &call_size_time_table : &size_time_table;
     179              : 
     180    139417274 :   if (exec_pred == false)
     181      5032789 :     return;
     182              : 
     183    139126637 :   nonconst_pred = nonconst_pred_in & exec_pred;
     184              : 
     185    139126637 :   if (nonconst_pred == false)
     186              :     return;
     187              : 
     188              :   /* We need to create initial empty unconditional clause, but otherwise
     189              :      we don't need to account empty times and sizes.  */
     190    163023156 :   if (!size && time == 0 && table->length ())
     191      3974553 :     return;
     192              : 
     193              :   /* Only for calls we are unaccounting what we previously recorded.  */
     194    134384485 :   gcc_checking_assert (time >= 0 || call);
     195              : 
     196    486892267 :   for (i = 0; table->iterate (i, &e); i++)
     197    456684811 :     if (e->exec_predicate == exec_pred
     198    456684811 :         && e->nonconst_predicate == nonconst_pred)
     199              :       {
     200              :         found = true;
     201              :         break;
     202              :       }
     203    134384485 :   if (i == max_size_time_table_size)
     204              :     {
     205         4896 :       i = 0;
     206         4896 :       found = true;
     207         4896 :       e = &(*table)[0];
     208         4896 :       if (dump_file && (dump_flags & TDF_DETAILS))
     209            0 :         fprintf (dump_file,
     210              :                  "\t\tReached limit on number of entries, "
     211              :                  "ignoring the predicate.");
     212              :     }
     213    134388846 :   if (dump_file && (dump_flags & TDF_DETAILS) && (time != 0 || size))
     214              :     {
     215         4037 :       fprintf (dump_file,
     216              :                "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
     217         3777 :                ((double) size) / ipa_fn_summary::size_scale,
     218              :                (time.to_double ()), found ? "" : "new ");
     219         3777 :       exec_pred.dump (dump_file, conds, 0);
     220         3777 :       if (exec_pred != nonconst_pred)
     221              :         {
     222           96 :           fprintf (dump_file, " nonconst:");
     223           96 :           nonconst_pred.dump (dump_file, conds);
     224              :         }
     225              :       else
     226         3681 :         fprintf (dump_file, "\n");
     227              :     }
     228    134384485 :   if (!found)
     229              :     {
     230     30202560 :       class size_time_entry new_entry;
     231     30202560 :       new_entry.size = size;
     232     30202560 :       new_entry.time = time;
     233     30202560 :       new_entry.exec_predicate = exec_pred;
     234     30202560 :       new_entry.nonconst_predicate = nonconst_pred;
     235     30202560 :       if (call)
     236      1364926 :         call_size_time_table.safe_push (new_entry);
     237              :       else
     238     28837634 :         size_time_table.safe_push (new_entry);
     239              :     }
     240              :   else
     241              :     {
     242    104181925 :       e->size += size;
     243    104181925 :       e->time += time;
     244              :       /* FIXME: PR bootstrap/92653 gcc_checking_assert (e->time >= -1); */
     245              :       /* Tolerate small roundoff issues.  */
     246    104181925 :       if (e->time < 0)
     247          231 :         e->time = 0;
     248              :     }
     249              : }
     250              : 
     251              : /* We proved E to be unreachable, redirect it to __builtin_unreachable.  */
     252              : 
     253              : static struct cgraph_edge *
     254       344800 : redirect_to_unreachable (struct cgraph_edge *e)
     255              : {
     256       344800 :   struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
     257       344800 :   struct cgraph_node *target
     258       344800 :     = cgraph_node::get_create (builtin_decl_unreachable ());
     259              : 
     260       344800 :   gcc_checking_assert (lookup_attribute ("cold",
     261              :                                          DECL_ATTRIBUTES (target->decl)));
     262              : 
     263       344800 :   if (e->speculative)
     264          311 :     e = cgraph_edge::resolve_speculation (e, target->decl);
     265       344489 :   else if (!e->callee)
     266          789 :     e = cgraph_edge::make_direct (e, target);
     267              :   else
     268       343700 :     e->redirect_callee (target);
     269       344800 :   class ipa_call_summary *es = ipa_call_summaries->get (e);
     270       344800 :   e->inline_failed = CIF_UNREACHABLE;
     271       344800 :   e->count = profile_count::zero ();
     272       344800 :   es->call_stmt_size = 0;
     273       344800 :   es->call_stmt_time = 0;
     274       344800 :   if (callee)
     275            0 :     callee->remove_symbol_and_inline_clones ();
     276       344800 :   if (e->has_callback)
     277            6 :     for (cgraph_edge *cbe = e->first_callback_edge (); cbe;
     278            2 :          cbe = cbe->next_callback_edge ())
     279              :       /* If the carrying edge is unreachable, so are the callback calls.  */
     280            2 :       redirect_to_unreachable (cbe);
     281       344800 :   return e;
     282              : }
     283              : 
     284              : /* Set predicate for edge E.  */
     285              : 
     286              : static void
     287     32127114 : edge_set_predicate (struct cgraph_edge *e, ipa_predicate *predicate)
     288              : {
     289              :   /* If the edge is determined to be never executed, redirect it
     290              :      to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
     291              :      be optimized out.  */
     292     29120555 :   if (predicate && *predicate == false
     293              :       /* When handling speculative edges, we need to do the redirection
     294              :          just once.  Do it always on the direct edge, so we do not
     295              :          attempt to resolve speculation while duplicating the edge.  */
     296     32471989 :       && (!e->speculative || e->callee))
     297       344798 :     e = redirect_to_unreachable (e);
     298              : 
     299     32127114 :   class ipa_call_summary *es = ipa_call_summaries->get (e);
     300     61247669 :   if (predicate && *predicate != true)
     301              :     {
     302      4677916 :       if (!es->predicate)
     303      4167570 :         es->predicate = edge_predicate_pool.allocate ();
     304      4677916 :       *es->predicate = *predicate;
     305              :     }
     306              :   else
     307              :     {
     308     27449198 :       if (es->predicate)
     309       828903 :         edge_predicate_pool.remove (es->predicate);
     310     27449198 :       es->predicate = NULL;
     311              :     }
     312     32127114 : }
     313              : 
     314              : /* Set predicate for hint *P.  */
     315              : 
     316              : static void
     317       155428 : set_hint_predicate (ipa_predicate **p, ipa_predicate new_predicate)
     318              : {
     319       155428 :   if (new_predicate == false || new_predicate == true)
     320              :     {
     321         2986 :       if (*p)
     322            0 :         edge_predicate_pool.remove (*p);
     323         2986 :       *p = NULL;
     324              :     }
     325              :   else
     326              :     {
     327       152442 :       if (!*p)
     328       152442 :         *p = edge_predicate_pool.allocate ();
     329       152442 :       **p = new_predicate;
     330              :     }
     331       155428 : }
     332              : 
     333              : /* Find if NEW_PREDICATE is already in V and if so, increment its freq.
     334              :    Otherwise add a new item to the vector with this predicate and frerq equal
     335              :    to add_freq, unless the number of predicates would exceed MAX_NUM_PREDICATES
     336              :    in which case the function does nothing.  */
     337              : 
     338              : static void
     339      1145232 : add_freqcounting_predicate (vec<ipa_freqcounting_predicate, va_gc> **v,
     340              :                             const ipa_predicate &new_predicate, sreal add_freq,
     341              :                             unsigned max_num_predicates)
     342              : {
     343      1145232 :   if (new_predicate == false || new_predicate == true)
     344              :     return;
     345              :   ipa_freqcounting_predicate *f;
     346       145384 :   for (int i = 0; vec_safe_iterate (*v, i, &f); i++)
     347        72577 :     if (new_predicate == f->predicate)
     348              :       {
     349            0 :         f->freq += add_freq;
     350            0 :         return;
     351              :       }
     352        99396 :   if (vec_safe_length (*v) >= max_num_predicates)
     353              :     /* Too many different predicates to account for.  */
     354              :     return;
     355              : 
     356        72535 :   ipa_freqcounting_predicate fcp;
     357        72535 :   fcp.predicate = NULL;
     358        72535 :   set_hint_predicate (&fcp.predicate, new_predicate);
     359        72535 :   fcp.freq = add_freq;
     360        72535 :   vec_safe_push (*v, fcp);
     361        72535 :   return;
     362              : }
     363              : 
     364              : /* Compute what conditions may or may not hold given information about
     365              :    parameters.  RET_CLAUSE returns truths that may hold in a specialized copy,
     366              :    while RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
     367              :    copy when called in a given context.  It is a bitmask of conditions. Bit
     368              :    0 means that condition is known to be false, while bit 1 means that condition
     369              :    may or may not be true.  These differs - for example NOT_INLINED condition
     370              :    is always false in the second and also builtin_constant_p tests cannot use
     371              :    the fact that parameter is indeed a constant.
     372              : 
     373              :    When INLINE_P is true, assume that we are inlining.  AVAL contains known
     374              :    information about argument values.  The function does not modify its content
     375              :    and so AVALs could also be of type ipa_call_arg_values but so far all
     376              :    callers work with the auto version and so we avoid the conversion for
     377              :    convenience.
     378              : 
     379              :    ERROR_MARK value of an argument means compile time invariant.  */
     380              : 
     381              : static void
     382     23001217 : evaluate_conditions_for_known_args (struct cgraph_node *node,
     383              :                                     bool inline_p,
     384              :                                     ipa_auto_call_arg_values *avals,
     385              :                                     clause_t *ret_clause,
     386              :                                     clause_t *ret_nonspec_clause,
     387              :                                     ipa_call_summary *es)
     388              : {
     389     23001217 :   clause_t clause = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
     390     23001217 :   clause_t nonspec_clause = 1 << ipa_predicate::not_inlined_condition;
     391     23001217 :   class ipa_fn_summary *info = ipa_fn_summaries->get (node);
     392     23001217 :   int i;
     393     23001217 :   struct condition *c;
     394              : 
     395     94622881 :   for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
     396              :     {
     397     71621664 :       tree val = NULL;
     398     71621664 :       tree res;
     399     71621664 :       int j;
     400     71621664 :       struct expr_eval_op *op;
     401              : 
     402     71621664 :       if (c->code == ipa_predicate::not_sra_candidate)
     403              :         {
     404     16768842 :           if (!inline_p
     405     16768842 :               || !es
     406     16446671 :               || (int)es->param.length () <= c->operand_num
     407     24992150 :               || !es->param[c->operand_num].points_to_possible_sra_candidate)
     408     12552918 :             clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     409     16768842 :           nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     410     71621664 :           continue;
     411              :         }
     412              : 
     413     54852822 :       if (c->agg_contents)
     414              :         {
     415     30259725 :           if (c->code == ipa_predicate::changed
     416     19453027 :               && !c->by_ref
     417     33147260 :               && (avals->safe_sval_at(c->operand_num) == error_mark_node))
     418         3462 :             continue;
     419              : 
     420     30252801 :           if (tree sval = avals->safe_sval_at (c->operand_num))
     421     13982161 :             val = ipa_find_agg_cst_from_init (sval, c->offset, c->by_ref);
     422              :           /* ipa_argagg_value_list is indexed by byte offsets, so a condition
     423              :              which does not start at a byte boundary (a bit-field) cannot be
     424              :              looked up in it; the containing byte would be reinterpreted as
     425              :              the whole field below.  */
     426     30252801 :           if (!val && (c->offset % BITS_PER_UNIT) == 0)
     427              :             {
     428     30215402 :               ipa_argagg_value_list avs (avals);
     429     30215402 :               val = avs.get_value (c->operand_num, c->offset / BITS_PER_UNIT,
     430     30215402 :                                    c->by_ref);
     431              :             }
     432              :         }
     433              :       else
     434              :         {
     435     24596559 :           val = avals->safe_sval_at (c->operand_num);
     436     24596559 :           if (val && val == error_mark_node
     437      2192626 :               && c->code != ipa_predicate::changed)
     438              :             val = NULL_TREE;
     439              :         }
     440              : 
     441     41507525 :       if (!val
     442     41426822 :           && (c->code == ipa_predicate::changed
     443              :               || c->code == ipa_predicate::is_not_constant))
     444              :         {
     445     26294782 :           clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     446     26294782 :           nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     447     26294782 :           continue;
     448              :         }
     449     28554578 :       if (c->code == ipa_predicate::changed)
     450              :         {
     451      7675844 :           nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     452      7675844 :           continue;
     453              :         }
     454              : 
     455     20878734 :       if (c->code == ipa_predicate::is_not_constant)
     456              :         {
     457         8779 :           nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     458         8779 :           continue;
     459              :         }
     460              : 
     461     20869955 :       if (val
     462     20869955 :           && (c->type == TREE_TYPE (val)
     463      1074614 :               || (TYPE_SIZE (c->type) == TYPE_SIZE (TREE_TYPE (val))
     464              :                   /* Avoid precision mismatch like with bit-fields where the
     465              :                      VIEW_CONVERT_EXPR does not truncate excess bits
     466              :                      appropriately.  */
     467      1011996 :                   && ((!INTEGRAL_TYPE_P (c->type)
     468       240648 :                        || type_has_mode_precision_p (c->type))
     469      1010273 :                       && (!INTEGRAL_TYPE_P (TREE_TYPE (val))
     470       239105 :                           || type_has_mode_precision_p (TREE_TYPE (val)))))))
     471              :         {
     472      5673560 :           if (c->type != TREE_TYPE (val))
     473      1010259 :             val = fold_unary (VIEW_CONVERT_EXPR, c->type, val);
     474      6197521 :           for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
     475              :             {
     476       524416 :               if (!val)
     477              :                 break;
     478       523961 :               if (!op->val[0])
     479       226150 :                 val = fold_unary (op->code, op->type, val);
     480       297811 :               else if (!op->val[1])
     481       595622 :                 val = fold_binary (op->code, op->type,
     482              :                                    op->index ? op->val[0] : val,
     483              :                                    op->index ? val : op->val[0]);
     484            0 :               else if (op->index == 0)
     485            0 :                 val = fold_ternary (op->code, op->type,
     486              :                                     val, op->val[0], op->val[1]);
     487            0 :               else if (op->index == 1)
     488            0 :                 val = fold_ternary (op->code, op->type,
     489              :                                     op->val[0], val, op->val[1]);
     490            0 :               else if (op->index == 2)
     491            0 :                 val = fold_ternary (op->code, op->type,
     492              :                                     op->val[0], op->val[1], val);
     493              :               else
     494              :                 val = NULL_TREE;
     495              :             }
     496              : 
     497      5673560 :           res = val
     498      5673560 :             ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
     499              :             : NULL;
     500              : 
     501      5673085 :           if (res && integer_zerop (res))
     502      2882058 :             continue;
     503      2791502 :           if (res && integer_onep (res))
     504              :             {
     505      2768840 :               clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     506      2768840 :               nonspec_clause
     507      2768840 :                 |= 1 << (i + ipa_predicate::first_dynamic_condition);
     508      2768840 :               continue;
     509              :             }
     510              :         }
     511     15219057 :       if (c->operand_num < (int) avals->m_known_value_ranges.length ()
     512      8825161 :           && !c->agg_contents
     513     17426315 :           && (!val || TREE_CODE (val) != INTEGER_CST))
     514              :         {
     515      2206523 :           value_range vr (avals->m_known_value_ranges[c->operand_num]);
     516      2206523 :           if (!vr.undefined_p ()
     517      1550254 :               && !vr.varying_p ()
     518      3756777 :               && (TYPE_SIZE (c->type) == TYPE_SIZE (vr.type ())))
     519              :             {
     520      1549717 :               if (!useless_type_conversion_p (c->type, vr.type ()))
     521          536 :                 range_cast (vr, c->type);
     522              : 
     523      1974235 :               for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
     524              :                 {
     525       526581 :                   if (vr.varying_p () || vr.undefined_p ())
     526              :                     break;
     527              : 
     528       424518 :                   value_range res (op->type);
     529       424518 :                   if (!op->val[0])
     530              :                     {
     531       150506 :                       value_range varying (op->type);
     532       150506 :                       varying.set_varying (op->type);
     533       150506 :                       range_op_handler handler (op->code);
     534       150506 :                       if (!handler
     535       150506 :                           || !res.supports_type_p (op->type)
     536       301012 :                           || !handler.fold_range (res, op->type, vr, varying))
     537            0 :                         res.set_varying (op->type);
     538       150506 :                     }
     539       274012 :                   else if (!op->val[1])
     540              :                     {
     541       273840 :                       value_range op0 (TREE_TYPE (op->val[0]));
     542       273840 :                       range_op_handler handler (op->code);
     543              : 
     544       273840 :                       ipa_get_range_from_ip_invariant (op0, op->val[0], node);
     545              : 
     546       273840 :                       if (!handler
     547       273840 :                           || !res.supports_type_p (op->type)
     548       547680 :                           || !handler.fold_range (res, op->type,
     549       273840 :                                                   op->index ? op0 : vr,
     550       545912 :                                                   op->index ? vr : op0))
     551            0 :                         res.set_varying (op->type);
     552       273840 :                     }
     553              :                   else
     554          172 :                     res.set_varying (op->type);
     555       424518 :                   vr = res;
     556       424518 :                 }
     557      1549717 :               if (!vr.varying_p () && !vr.undefined_p ())
     558              :                 {
     559      1438479 :                   int_range<2> res;
     560      1438479 :                   value_range val_vr (TREE_TYPE (c->val));
     561      1438479 :                   range_op_handler handler (c->code);
     562              : 
     563      1438479 :                   ipa_get_range_from_ip_invariant (val_vr, c->val, node);
     564              : 
     565      1438479 :                   if (!handler
     566      1438479 :                       || !val_vr.supports_type_p (TREE_TYPE (c->val))
     567      2876958 :                       || !handler.fold_range (res, boolean_type_node, vr, val_vr))
     568            0 :                     res.set_varying (boolean_type_node);
     569              : 
     570      1438479 :                   if (res.zero_p ())
     571       243469 :                     continue;
     572      1438479 :                 }
     573              :             }
     574      2206523 :         }
     575              : 
     576     14975588 :       clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     577     14975588 :       nonspec_clause |= 1 << (i + ipa_predicate::first_dynamic_condition);
     578              :     }
     579     23001217 :   *ret_clause = clause;
     580     23001217 :   if (ret_nonspec_clause)
     581     19920174 :     *ret_nonspec_clause = nonspec_clause;
     582     23001217 : }
     583              : 
     584              : /* Return true if VRP will be executed on the function.
     585              :    We do not want to anticipate optimizations that will not happen.
     586              : 
     587              :    FIXME: This can be confused with -fdisable and debug counters and thus
     588              :    it should not be used for correctness (only to make heuristics work).
     589              :    This means that inliner should do its own optimizations of expressions
     590              :    that it predicts to be constant so wrong code can not be triggered by
     591              :    builtin_constant_p.  */
     592              : 
     593              : static bool
     594     12520813 : vrp_will_run_p (struct cgraph_node *node)
     595              : {
     596     12520813 :   return (opt_for_fn (node->decl, optimize)
     597     12520813 :           && !opt_for_fn (node->decl, optimize_debug)
     598     25040858 :           && opt_for_fn (node->decl, flag_tree_vrp));
     599              : }
     600              : 
     601              : /* Similarly about FRE.  */
     602              : 
     603              : static bool
     604     14746263 : fre_will_run_p (struct cgraph_node *node)
     605              : {
     606     14746263 :   return (opt_for_fn (node->decl, optimize)
     607     14746263 :           && !opt_for_fn (node->decl, optimize_debug)
     608     29490878 :           && opt_for_fn (node->decl, flag_tree_fre));
     609              : }
     610              : 
     611              : /* Work out what conditions might be true at invocation of E.
     612              :    Compute costs for inlined edge if INLINE_P is true.
     613              : 
     614              :    Return in CLAUSE_PTR the evaluated conditions and in NONSPEC_CLAUSE_PTR
     615              :    (if non-NULL) conditions evaluated for nonspecialized clone called
     616              :    in a given context.
     617              : 
     618              :    Vectors in AVALS will be populated with useful known information about
     619              :    argument values - information not known to have any uses will be omitted -
     620              :    except for m_known_contexts which will only be calculated if
     621              :    COMPUTE_CONTEXTS is true.  */
     622              : 
     623              : void
     624     22751247 : evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
     625              :                               clause_t *clause_ptr,
     626              :                               clause_t *nonspec_clause_ptr,
     627              :                               ipa_auto_call_arg_values *avals,
     628              :                               bool compute_contexts)
     629              : {
     630     22751247 :   struct cgraph_node *callee = e->callee->ultimate_alias_target ();
     631     22751247 :   class ipa_fn_summary *info = ipa_fn_summaries->get (callee);
     632     22751247 :   class ipa_edge_args *args;
     633     22751247 :   class ipa_call_summary *es = NULL;
     634              : 
     635     22751247 :   if (clause_ptr)
     636     22751247 :     *clause_ptr = inline_p ? 0 : 1 << ipa_predicate::not_inlined_condition;
     637              : 
     638     22751247 :   if (ipa_node_params_sum
     639     10294637 :       && !e->call_stmt_cannot_inline_p
     640     10294637 :       && (info->conds || compute_contexts)
     641     33045884 :       && (args = ipa_edge_args_sum->get (e)) != NULL)
     642              :     {
     643     10239368 :       struct cgraph_node *caller;
     644     10239368 :       class ipa_node_params *caller_parms_info, *callee_pi = NULL;
     645     10239368 :       int i, count = ipa_get_cs_argument_count (args);
     646     10239368 :       es = ipa_call_summaries->get (e);
     647              : 
     648     10239368 :       if (count)
     649              :         {
     650      9501470 :           if (e->caller->inlined_to)
     651              :             caller = e->caller->inlined_to;
     652              :           else
     653      7557704 :             caller = e->caller;
     654      9501470 :           caller_parms_info = ipa_node_params_sum->get (caller);
     655      9501470 :           callee_pi = ipa_node_params_sum->get (callee);
     656              : 
     657              :           /* Watch for thunks.  */
     658      9501470 :           if (callee_pi)
     659              :             /* Watch for variadic functions.  */
     660      9501096 :             count = MIN (count, ipa_get_param_count (callee_pi));
     661              :         }
     662              : 
     663      9501096 :       if (callee_pi)
     664     31328562 :         for (i = 0; i < count; i++)
     665              :           {
     666     21827466 :             struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
     667              : 
     668     21827466 :             if (ipa_is_param_used_by_indirect_call (callee_pi, i)
     669     21827466 :                 || ipa_is_param_used_by_ipa_predicates (callee_pi, i))
     670              :               {
     671              :                 /* Determine if we know constant value of the parameter.  */
     672     14746263 :                 tree type = ipa_get_type (callee_pi, i);
     673     14746263 :                 tree cst = ipa_value_from_jfunc (caller_parms_info, jf, type);
     674              : 
     675     11833776 :                 if (!cst && e->call_stmt
     676     26526317 :                     && i < (int)gimple_call_num_args (e->call_stmt))
     677              :                   {
     678     11780054 :                     cst = gimple_call_arg (e->call_stmt, i);
     679     11780054 :                     if (!is_gimple_min_invariant (cst))
     680              :                       cst = NULL;
     681              :                   }
     682      3160836 :                 if (cst)
     683              :                   {
     684      3107114 :                     gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
     685      3107114 :                     if (!avals->m_known_vals.length ())
     686      1787425 :                       avals->m_known_vals.safe_grow_cleared (count, true);
     687      3107114 :                     avals->m_known_vals[i] = cst;
     688              :                   }
     689     11639149 :                 else if (inline_p && !es->param[i].change_prob)
     690              :                   {
     691      4251013 :                     if (!avals->m_known_vals.length ())
     692      3449580 :                       avals->m_known_vals.safe_grow_cleared (count, true);
     693      4251013 :                     avals->m_known_vals[i] = error_mark_node;
     694              :                   }
     695              : 
     696              :                 /* If we failed to get simple constant, try value range.  */
     697      3107114 :                 if ((!cst || TREE_CODE (cst) != INTEGER_CST)
     698     12520813 :                     && vrp_will_run_p (caller)
     699     27099154 :                     && ipa_is_param_used_by_ipa_predicates (callee_pi, i))
     700              :                   {
     701     12311984 :                     value_range vr (type);
     702              : 
     703     12311984 :                     ipa_value_range_from_jfunc (vr, caller_parms_info, e, jf, type);
     704     12311984 :                     if (!vr.undefined_p () && !vr.varying_p ())
     705              :                       {
     706      8746581 :                         if (!avals->m_known_value_ranges.length ())
     707              :                           {
     708      6357043 :                             avals->m_known_value_ranges.safe_grow_cleared (count,
     709              :                                                                            true);
     710     26671039 :                             for (int i = 0; i < count; ++i)
     711     13956953 :                               avals->m_known_value_ranges[i].set_range_class
     712     13956953 :                                 (void_type_node);
     713              :                           }
     714      8746581 :                         avals->m_known_value_ranges[i] = vr;
     715              :                       }
     716     12311984 :                   }
     717              : 
     718              :                 /* Determine known aggregate values.  */
     719     14746263 :                 if (fre_will_run_p (caller))
     720     14744211 :                   ipa_push_agg_values_from_jfunc (caller_parms_info,
     721              :                                                   caller, &jf->agg, i,
     722              :                                                   &avals->m_known_aggs);
     723              :               }
     724              : 
     725              :             /* For calls used in polymorphic calls we further determine
     726              :                polymorphic call context.  */
     727     21827466 :             if (compute_contexts
     728     21827466 :                 && ipa_is_param_used_by_polymorphic_call (callee_pi, i))
     729              :               {
     730       185376 :                 ipa_polymorphic_call_context
     731       185376 :                    ctx = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
     732       370752 :                 if (!ctx.useless_p ())
     733              :                   {
     734       183410 :                     if (!avals->m_known_contexts.length ())
     735       183236 :                       avals->m_known_contexts.safe_grow_cleared (count, true);
     736       183410 :                     avals->m_known_contexts[i]
     737       366820 :                       = ipa_context_from_jfunc (caller_parms_info, e, i, jf);
     738              :                   }
     739              :                }
     740              :           }
     741              :         else
     742       738272 :           gcc_assert (!count || callee->thunk);
     743              :     }
     744     12511879 :   else if (e->call_stmt && !e->call_stmt_cannot_inline_p && info->conds)
     745              :     {
     746      9840205 :       int i, count = (int)gimple_call_num_args (e->call_stmt);
     747              : 
     748     27997048 :       for (i = 0; i < count; i++)
     749              :         {
     750     18156843 :           tree cst = gimple_call_arg (e->call_stmt, i);
     751     18156843 :           if (!is_gimple_min_invariant (cst))
     752              :             cst = NULL;
     753      6509928 :           if (cst)
     754              :             {
     755      6509928 :               if (!avals->m_known_vals.length ())
     756      4514956 :                 avals->m_known_vals.safe_grow_cleared (count, true);
     757      6509928 :               avals->m_known_vals[i] = cst;
     758              :             }
     759              :         }
     760              :     }
     761              : 
     762     22751247 :   evaluate_conditions_for_known_args (callee, inline_p, avals, clause_ptr,
     763              :                                       nonspec_clause_ptr, es);
     764     22751247 : }
     765              : 
     766              : 
     767              : /* Allocate the function summary. */
     768              : 
     769              : static void
     770       472283 : ipa_fn_summary_alloc (void)
     771              : {
     772       472283 :   gcc_checking_assert (!ipa_fn_summaries);
     773       472283 :   ipa_size_summaries = new ipa_size_summary_t (symtab);
     774       472283 :   ipa_fn_summaries = ipa_fn_summary_t::create_ggc (symtab);
     775       472283 :   ipa_call_summaries = new ipa_call_summary_t (symtab);
     776       472283 : }
     777              : 
     778     28143343 : ipa_call_summary::~ipa_call_summary ()
     779              : {
     780     28143343 :   if (predicate)
     781      3338667 :     edge_predicate_pool.remove (predicate);
     782              : 
     783     28143343 :   param.release ();
     784     28143343 : }
     785              : 
     786     10369843 : ipa_fn_summary::~ipa_fn_summary ()
     787              : {
     788     10369843 :   unsigned len = vec_safe_length (loop_iterations);
     789     10486068 :   for (unsigned i = 0; i < len; i++)
     790       116225 :     edge_predicate_pool.remove ((*loop_iterations)[i].predicate);
     791     10369843 :   len = vec_safe_length (loop_strides);
     792     10406060 :   for (unsigned i = 0; i < len; i++)
     793        36217 :     edge_predicate_pool.remove ((*loop_strides)[i].predicate);
     794     10369843 :   vec_free (conds);
     795     10369843 :   call_size_time_table.release ();
     796     10369843 :   vec_free (loop_iterations);
     797     10369843 :   vec_free (loop_strides);
     798     10369843 :   builtin_constant_p_parms.release ();
     799     10369843 : }
     800              : 
     801              : void
     802      7452066 : ipa_fn_summary_t::remove_callees (cgraph_node *node)
     803              : {
     804      7452066 :   cgraph_edge *e;
     805     30589425 :   for (e = node->callees; e; e = e->next_callee)
     806     23137359 :     ipa_call_summaries->remove (e);
     807      7938393 :   for (e = node->indirect_calls; e; e = e->next_callee)
     808       486327 :     ipa_call_summaries->remove (e);
     809      7452066 : }
     810              : 
     811              : /* Duplicate predicates in loop hint vector, allocating memory for them and
     812              :    remove and deallocate any uninteresting (true or false) ones.  Return the
     813              :    result.  */
     814              : 
     815              : static vec<ipa_freqcounting_predicate, va_gc> *
     816        29076 : remap_freqcounting_preds_after_dup (vec<ipa_freqcounting_predicate, va_gc> *v,
     817              :                                     clause_t possible_truths)
     818              : {
     819        29076 :   if (vec_safe_length (v) == 0)
     820              :     return NULL;
     821              : 
     822         4771 :   vec<ipa_freqcounting_predicate, va_gc> *res = v->copy ();
     823         4771 :   int len = res->length();
     824        11095 :   for (int i = len - 1; i >= 0; i--)
     825              :     {
     826         6324 :       ipa_predicate new_predicate
     827         6324 :         = (*res)[i].predicate->remap_after_duplication (possible_truths);
     828              :       /* We do not want to free previous predicate; it is used by node
     829              :          origin.  */
     830         6324 :       (*res)[i].predicate = NULL;
     831         6324 :       set_hint_predicate (&(*res)[i].predicate, new_predicate);
     832              : 
     833         6324 :       if (!(*res)[i].predicate)
     834         2986 :         res->unordered_remove (i);
     835              :     }
     836              : 
     837              :   return res;
     838              : }
     839              : 
     840              : 
     841              : /* Hook that is called by cgraph.cc when a node is duplicated.  */
     842              : void
     843      2828353 : ipa_fn_summary_t::duplicate (cgraph_node *src,
     844              :                              cgraph_node *dst,
     845              :                              ipa_fn_summary *src_info,
     846              :                              ipa_fn_summary *info)
     847              : {
     848      2828353 :   new (info) ipa_fn_summary (*src_info);
     849              :   /* TODO: as an optimization, we may avoid copying conditions
     850              :      that are known to be false or true.  */
     851      2828353 :   info->conds = vec_safe_copy (info->conds);
     852              : 
     853      2828353 :   clone_info *cinfo = clone_info::get (dst);
     854              :   /* When there are any replacements in the function body, see if we can figure
     855              :      out that something was optimized out.  */
     856      2828353 :   if (ipa_node_params_sum && cinfo && cinfo->tree_map)
     857              :     {
     858              :       /* Use SRC parm info since it may not be copied yet.  */
     859        14538 :       ipa_node_params *parms_info = ipa_node_params_sum->get (src);
     860        14538 :       ipa_auto_call_arg_values avals;
     861        14538 :       int count = ipa_get_param_count (parms_info);
     862        14538 :       int i, j;
     863        14538 :       clause_t possible_truths;
     864        14538 :       ipa_predicate true_pred = true;
     865        14538 :       size_time_entry *e;
     866        14538 :       int optimized_out_size = 0;
     867        14538 :       bool inlined_to_p = false;
     868        14538 :       struct cgraph_edge *edge, *next;
     869              : 
     870        14538 :       info->size_time_table.release ();
     871        14538 :       avals.m_known_vals.safe_grow_cleared (count, true);
     872        72865 :       for (i = 0; i < count; i++)
     873              :         {
     874              :           struct ipa_replace_map *r;
     875              : 
     876       142706 :           for (j = 0; vec_safe_iterate (cinfo->tree_map, j, &r); j++)
     877              :             {
     878        80100 :               if (r->parm_num == i)
     879              :                 {
     880        24972 :                   avals.m_known_vals[i] = r->new_tree;
     881        24972 :                   break;
     882              :                 }
     883              :             }
     884              :         }
     885        14538 :       evaluate_conditions_for_known_args (dst, false,
     886              :                                           &avals,
     887              :                                           &possible_truths,
     888              :                                           /* We are going to specialize,
     889              :                                              so ignore nonspec truths.  */
     890              :                                           NULL,
     891              :                                           NULL);
     892              : 
     893        14538 :       info->account_size_time (0, 0, true_pred, true_pred);
     894              : 
     895              :       /* Remap size_time vectors.
     896              :          Simplify the predicate by pruning out alternatives that are known
     897              :          to be false.
     898              :          TODO: as on optimization, we can also eliminate conditions known
     899              :          to be true.  */
     900       100913 :       for (i = 0; src_info->size_time_table.iterate (i, &e); i++)
     901              :         {
     902        86375 :           ipa_predicate new_exec_pred;
     903        86375 :           ipa_predicate new_nonconst_pred;
     904        86375 :           new_exec_pred = e->exec_predicate.remap_after_duplication
     905        86375 :                                  (possible_truths);
     906        86375 :           new_nonconst_pred = e->nonconst_predicate.remap_after_duplication
     907        86375 :                                  (possible_truths);
     908        86375 :           if (new_exec_pred == false || new_nonconst_pred == false)
     909        11913 :             optimized_out_size += e->size;
     910              :           else
     911        74462 :             info->account_size_time (e->size, e->time, new_exec_pred,
     912              :                                      new_nonconst_pred);
     913              :         }
     914              : 
     915              :       /* Remap edge predicates with the same simplification as above.
     916              :          Also copy constantness arrays.   */
     917       184580 :       for (edge = dst->callees; edge; edge = next)
     918              :         {
     919       170042 :           ipa_predicate new_predicate;
     920       170042 :           class ipa_call_summary *es = ipa_call_summaries->get (edge);
     921       170042 :           next = edge->next_callee;
     922              : 
     923       170042 :           if (!edge->inline_failed)
     924            0 :             inlined_to_p = true;
     925       170042 :           if (!es->predicate)
     926       157551 :             continue;
     927        12491 :           new_predicate = es->predicate->remap_after_duplication
     928        12491 :             (possible_truths);
     929        12491 :           if (new_predicate == false && *es->predicate != false)
     930         3247 :             optimized_out_size += es->call_stmt_size * ipa_fn_summary::size_scale;
     931        12491 :           edge_set_predicate (edge, &new_predicate);
     932              :         }
     933              : 
     934              :       /* Remap indirect edge predicates with the same simplification as above.
     935              :          Also copy constantness arrays.   */
     936        15457 :       for (edge = dst->indirect_calls; edge; edge = next)
     937              :         {
     938          919 :           ipa_predicate new_predicate;
     939          919 :           class ipa_call_summary *es = ipa_call_summaries->get (edge);
     940          919 :           next = edge->next_callee;
     941              : 
     942          919 :           gcc_checking_assert (edge->inline_failed);
     943          919 :           if (!es->predicate)
     944          815 :             continue;
     945          104 :           new_predicate = es->predicate->remap_after_duplication
     946          104 :                                  (possible_truths);
     947          104 :           if (new_predicate == false && *es->predicate != false)
     948           21 :             optimized_out_size
     949           21 :                  += es->call_stmt_size * ipa_fn_summary::size_scale;
     950          104 :           edge_set_predicate (edge, &new_predicate);
     951              :         }
     952        14538 :       info->loop_iterations
     953        14538 :         = remap_freqcounting_preds_after_dup (info->loop_iterations,
     954              :                                               possible_truths);
     955        14538 :       info->loop_strides
     956        14538 :         = remap_freqcounting_preds_after_dup (info->loop_strides,
     957              :                                               possible_truths);
     958        14538 :       if (info->builtin_constant_p_parms.length())
     959              :         {
     960           23 :           vec <int, va_heap, vl_ptr> parms = info->builtin_constant_p_parms;
     961           23 :           int ip;
     962           23 :           info->builtin_constant_p_parms = vNULL;
     963           46 :           for (i = 0; parms.iterate (i, &ip); i++)
     964           23 :             if (!avals.m_known_vals[ip])
     965            8 :               info->builtin_constant_p_parms.safe_push (ip);
     966              :         }
     967              : 
     968              :       /* If inliner or someone after inliner will ever start producing
     969              :          non-trivial clones, we will get trouble with lack of information
     970              :          about updating self sizes, because size vectors already contains
     971              :          sizes of the callees.  */
     972        14538 :       gcc_assert (!inlined_to_p || !optimized_out_size);
     973        14538 :     }
     974              :   else
     975              :     {
     976      2813815 :       info->size_time_table = src_info->size_time_table.copy ();
     977      2813815 :       info->loop_iterations = vec_safe_copy (src_info->loop_iterations);
     978      2813815 :       info->loop_strides = vec_safe_copy (info->loop_strides);
     979              : 
     980      2813815 :       info->builtin_constant_p_parms
     981      2813815 :              = info->builtin_constant_p_parms.copy ();
     982              : 
     983      2813815 :       ipa_freqcounting_predicate *f;
     984      5677915 :       for (int i = 0; vec_safe_iterate (info->loop_iterations, i, &f); i++)
     985              :         {
     986        50285 :           ipa_predicate p = *f->predicate;
     987        50285 :           f->predicate = NULL;
     988        50285 :           set_hint_predicate (&f->predicate, p);
     989              :         }
     990      2838288 :       for (int i = 0; vec_safe_iterate (info->loop_strides, i, &f); i++)
     991              :         {
     992        24473 :           ipa_predicate p = *f->predicate;
     993        24473 :           f->predicate = NULL;
     994        24473 :           set_hint_predicate (&f->predicate, p);
     995              :         }
     996              :     }
     997      2828353 :   if (!dst->inlined_to)
     998       145390 :     ipa_update_overall_fn_summary (dst);
     999      2828353 : }
    1000              : 
    1001              : 
    1002              : /* Hook that is called by cgraph.cc when a node is duplicated.  */
    1003              : 
    1004              : void
    1005      4168582 : ipa_call_summary_t::duplicate (struct cgraph_edge *src,
    1006              :                                struct cgraph_edge *dst,
    1007              :                                class ipa_call_summary *srcinfo,
    1008              :                                class ipa_call_summary *info)
    1009              : {
    1010      4168582 :   new (info) ipa_call_summary (*srcinfo);
    1011      4168582 :   info->predicate = NULL;
    1012      4168582 :   edge_set_predicate (dst, srcinfo->predicate);
    1013      4168582 :   info->param = srcinfo->param.copy ();
    1014      4168582 :   if (!dst->indirect_unknown_callee && src->indirect_unknown_callee
    1015              :       /* Don't subtract the size when dealing with callback pairs, since the
    1016              :          edge has no real size.  */
    1017        36734 :       && !src->has_callback && !dst->callback)
    1018              :     {
    1019        36734 :       info->call_stmt_size -= (eni_size_weights.indirect_call_cost
    1020        36734 :                                - eni_size_weights.call_cost);
    1021        36734 :       info->call_stmt_time -= (eni_time_weights.indirect_call_cost
    1022        36734 :                                - eni_time_weights.call_cost);
    1023              :     }
    1024      4168582 : }
    1025              : 
    1026              : /* Dump edge summaries associated to NODE and recursively to all clones.
    1027              :    Indent by INDENT.  */
    1028              : 
    1029              : static void
    1030         1969 : dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node,
    1031              :                        class ipa_fn_summary *info)
    1032              : {
    1033         1969 :   struct cgraph_edge *edge;
    1034         9787 :   for (edge = node->callees; edge; edge = edge->next_callee)
    1035              :     {
    1036         7818 :       class ipa_call_summary *es = ipa_call_summaries->get (edge);
    1037         7818 :       struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
    1038         7818 :       int i;
    1039              : 
    1040        15636 :       fprintf (f,
    1041              :                "%*s%s %s\n%*s  freq:%4.2f",
    1042              :                indent, "", callee->dump_name (),
    1043         7818 :                !edge->inline_failed
    1044         7265 :                ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
    1045         7818 :                indent, "", edge->sreal_frequency ().to_double ());
    1046              : 
    1047         7818 :       if (cross_module_call_p (edge))
    1048            4 :         fprintf (f, " cross module");
    1049              : 
    1050         7818 :       if (es)
    1051         7265 :         fprintf (f, " loop depth:%2i size:%2i time: %2i",
    1052              :                  es->loop_depth, es->call_stmt_size, es->call_stmt_time);
    1053              : 
    1054         7818 :       ipa_fn_summary *s = ipa_fn_summaries->get (callee);
    1055         7818 :       ipa_size_summary *ss = ipa_size_summaries->get (callee);
    1056         7818 :       if (s != NULL)
    1057          620 :         fprintf (f, " callee size:%2i stack:%2i",
    1058          620 :                  (int) (ss->size / ipa_fn_summary::size_scale),
    1059          620 :                  (int) s->estimated_stack_size);
    1060              : 
    1061         7818 :       if (es && es->predicate)
    1062              :         {
    1063         4627 :           fprintf (f, " predicate: ");
    1064         4627 :           es->predicate->dump (f, info->conds);
    1065              :         }
    1066              :       else
    1067         3191 :         fprintf (f, "\n");
    1068         7818 :       if (es && es->param.exists ())
    1069         5348 :         for (i = 0; i < (int) es->param.length (); i++)
    1070              :           {
    1071         1582 :             int prob = es->param[i].change_prob;
    1072              : 
    1073         1582 :             if (!prob)
    1074          620 :               fprintf (f, "%*s op%i is compile time invariant\n",
    1075              :                        indent + 2, "", i);
    1076          962 :             else if (prob != REG_BR_PROB_BASE)
    1077           35 :               fprintf (f, "%*s op%i change %f%% of time\n", indent + 2, "", i,
    1078           35 :                        prob * 100.0 / REG_BR_PROB_BASE);
    1079         1582 :             if (es->param[i].points_to_local_or_readonly_memory)
    1080          351 :               fprintf (f, "%*s op%i points to local or readonly memory\n",
    1081              :                        indent + 2, "", i);
    1082         1582 :             if (es->param[i].points_to_possible_sra_candidate)
    1083          227 :               fprintf (f, "%*s op%i points to possible sra candidate\n",
    1084              :                        indent + 2, "", i);
    1085              :           }
    1086         7818 :       if (!edge->inline_failed)
    1087              :         {
    1088          553 :           ipa_size_summary *ss = ipa_size_summaries->get (callee);
    1089          553 :           fprintf (f, "%*sStack frame offset %i, callee self size %i\n",
    1090              :                    indent + 2, "",
    1091          553 :                    (int) ipa_get_stack_frame_offset (callee),
    1092          553 :                    (int) ss->estimated_self_stack_size);
    1093          553 :           dump_ipa_call_summary (f, indent + 2, callee, info);
    1094              :         }
    1095              :     }
    1096         2267 :   for (edge = node->indirect_calls; edge; edge = edge->next_callee)
    1097              :     {
    1098          298 :       class ipa_call_summary *es = ipa_call_summaries->get (edge);
    1099          298 :       fprintf (f, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
    1100              :                " time: %2i",
    1101              :                indent, "",
    1102              :                es->loop_depth,
    1103          298 :                edge->sreal_frequency ().to_double (), es->call_stmt_size,
    1104              :                es->call_stmt_time);
    1105          298 :       if (es->predicate)
    1106              :         {
    1107            6 :           fprintf (f, "predicate: ");
    1108            6 :           es->predicate->dump (f, info->conds);
    1109              :         }
    1110              :       else
    1111          292 :         fprintf (f, "\n");
    1112              :     }
    1113         1969 : }
    1114              : 
    1115              : 
    1116              : void
    1117         1628 : ipa_dump_fn_summary (FILE *f, struct cgraph_node *node)
    1118              : {
    1119         1628 :   if (node->definition)
    1120              :     {
    1121         1628 :       class ipa_fn_summary *s = ipa_fn_summaries->get (node);
    1122         1628 :       class ipa_size_summary *ss = ipa_size_summaries->get (node);
    1123         1628 :       if (s != NULL)
    1124              :         {
    1125         1416 :           size_time_entry *e;
    1126         1416 :           int i;
    1127         1416 :           fprintf (f, "IPA function summary for %s", node->dump_name ());
    1128         1416 :           if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
    1129            0 :             fprintf (f, " always_inline");
    1130         1416 :           if (s->inlinable)
    1131         1235 :             fprintf (f, " inlinable");
    1132         1416 :           if (s->fp_expressions)
    1133           44 :             fprintf (f, " fp_expression");
    1134         1416 :           if (s->builtin_constant_p_parms.length ())
    1135              :             {
    1136            2 :               fprintf (f, " builtin_constant_p_parms");
    1137            4 :               for (unsigned int i = 0;
    1138            4 :                    i < s->builtin_constant_p_parms.length (); i++)
    1139            2 :                 fprintf (f, " %i", s->builtin_constant_p_parms[i]);
    1140              :             }
    1141         1416 :           fprintf (f, "\n  global time:     %f\n", s->time.to_double ());
    1142         1416 :           fprintf (f, "  self size:       %i\n", ss->self_size);
    1143         1416 :           fprintf (f, "  global size:     %i\n", ss->size);
    1144         1416 :           fprintf (f, "  min size:       %i\n", s->min_size);
    1145         1416 :           fprintf (f, "  self stack:      %i\n",
    1146         1416 :                    (int) ss->estimated_self_stack_size);
    1147         1416 :           fprintf (f, "  global stack:    %i\n", (int) s->estimated_stack_size);
    1148         1416 :           if (s->growth)
    1149          168 :             fprintf (f, "  estimated growth:%i\n", (int) s->growth);
    1150         1416 :           if (s->scc_no)
    1151            5 :             fprintf (f, "  In SCC:          %i\n", (int) s->scc_no);
    1152         5908 :           for (i = 0; s->size_time_table.iterate (i, &e); i++)
    1153              :             {
    1154         4492 :               fprintf (f, "    size:%f, time:%f",
    1155         4492 :                        (double) e->size / ipa_fn_summary::size_scale,
    1156              :                        e->time.to_double ());
    1157         4492 :               if (e->exec_predicate != true)
    1158              :                 {
    1159         2623 :                   fprintf (f, ",  executed if:");
    1160         2623 :                   e->exec_predicate.dump (f, s->conds, 0);
    1161              :                 }
    1162         4492 :               if (e->exec_predicate != e->nonconst_predicate)
    1163              :                 {
    1164         1167 :                   fprintf (f, ",  nonconst if:");
    1165         1167 :                   e->nonconst_predicate.dump (f, s->conds, 0);
    1166              :                 }
    1167         4492 :               fprintf (f, "\n");
    1168              :             }
    1169              :           ipa_freqcounting_predicate *fcp;
    1170              :           bool first_fcp = true;
    1171         1522 :           for (int i = 0; vec_safe_iterate (s->loop_iterations, i, &fcp); i++)
    1172              :             {
    1173          106 :               if (first_fcp)
    1174              :                 {
    1175           78 :                   fprintf (f, "  loop iterations:");
    1176           78 :                   first_fcp = false;
    1177              :                 }
    1178          106 :               fprintf (f, "  %3.2f for ", fcp->freq.to_double ());
    1179          106 :               fcp->predicate->dump (f, s->conds);
    1180              :             }
    1181              :           first_fcp = true;
    1182         1434 :           for (int i = 0; vec_safe_iterate (s->loop_strides, i, &fcp); i++)
    1183              :             {
    1184           18 :               if (first_fcp)
    1185              :                 {
    1186           10 :                   fprintf (f, "  loop strides:");
    1187           10 :                   first_fcp = false;
    1188              :                 }
    1189           18 :               fprintf (f, "  %3.2f for :", fcp->freq.to_double ());
    1190           18 :               fcp->predicate->dump (f, s->conds);
    1191              :             }
    1192         1416 :           fprintf (f, "  calls:\n");
    1193         1416 :           dump_ipa_call_summary (f, 4, node, s);
    1194         1416 :           fprintf (f, "\n");
    1195         1416 :           if (s->target_info)
    1196            0 :             fprintf (f, "  target_info: %x\n", s->target_info);
    1197              :         }
    1198              :       else
    1199          212 :         fprintf (f, "IPA summary for %s is missing.\n", node->dump_name ());
    1200              :     }
    1201         1628 : }
    1202              : 
    1203              : DEBUG_FUNCTION void
    1204            0 : ipa_debug_fn_summary (struct cgraph_node *node)
    1205              : {
    1206            0 :   ipa_dump_fn_summary (stderr, node);
    1207            0 : }
    1208              : 
    1209              : void
    1210          356 : ipa_dump_fn_summaries (FILE *f)
    1211              : {
    1212          356 :   struct cgraph_node *node;
    1213              : 
    1214         2288 :   FOR_EACH_DEFINED_FUNCTION (node)
    1215         1932 :     if (!node->inlined_to)
    1216         1379 :       ipa_dump_fn_summary (f, node);
    1217          356 : }
    1218              : 
    1219              : /* Callback of walk_aliased_vdefs.  Flags that it has been invoked to the
    1220              :    boolean variable pointed to by DATA.  */
    1221              : 
    1222              : static bool
    1223       848988 : mark_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef ATTRIBUTE_UNUSED,
    1224              :                void *data)
    1225              : {
    1226       848988 :   bool *b = (bool *) data;
    1227       848988 :   *b = true;
    1228       848988 :   return true;
    1229              : }
    1230              : 
    1231              : /* If OP refers to value of function parameter, return the corresponding
    1232              :    parameter.  If non-NULL, the size of the memory load (or the SSA_NAME of the
    1233              :    PARM_DECL) will be stored to *SIZE_P in that case too.  */
    1234              : 
    1235              : static tree
    1236    190535823 : unmodified_parm_1 (ipa_func_body_info *fbi, gimple *stmt, tree op,
    1237              :                    poly_int64 *size_p)
    1238              : {
    1239              :   /* SSA_NAME referring to parm default def?  */
    1240    190535823 :   if (TREE_CODE (op) == SSA_NAME
    1241    111640166 :       && SSA_NAME_IS_DEFAULT_DEF (op)
    1242    216791108 :       && TREE_CODE (SSA_NAME_VAR (op)) == PARM_DECL)
    1243              :     {
    1244     26044255 :       if (size_p)
    1245            0 :         *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
    1246     26044255 :       return SSA_NAME_VAR (op);
    1247              :     }
    1248              :   /* Non-SSA parm reference?  */
    1249    164491568 :   if (TREE_CODE (op) == PARM_DECL
    1250      1431397 :       && fbi->aa_walk_budget > 0)
    1251              :     {
    1252      1428851 :       bool modified = false;
    1253              : 
    1254      1428851 :       ao_ref refd;
    1255      1428851 :       ao_ref_init (&refd, op);
    1256      2857702 :       int walked = walk_aliased_vdefs (&refd, gimple_vuse (stmt),
    1257              :                                        mark_modified, &modified, NULL, NULL,
    1258              :                                        fbi->aa_walk_budget);
    1259      1428851 :       if (walked < 0)
    1260              :         {
    1261            8 :           fbi->aa_walk_budget = 0;
    1262       916996 :           return NULL_TREE;
    1263              :         }
    1264      1428843 :       fbi->aa_walk_budget -= walked;
    1265      1428843 :       if (!modified)
    1266              :         {
    1267       916988 :           if (size_p)
    1268            0 :             *size_p = tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op)));
    1269              :           return op;
    1270              :         }
    1271              :     }
    1272              :   return NULL_TREE;
    1273              : }
    1274              : 
    1275              : /* If OP refers to value of function parameter, return the corresponding
    1276              :    parameter.  Also traverse chains of SSA register assignments.  If non-NULL,
    1277              :    the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
    1278              :    stored to *SIZE_P in that case too.  */
    1279              : 
    1280              : static tree
    1281    124640169 : unmodified_parm (ipa_func_body_info *fbi, gimple *stmt, tree op,
    1282              :                  poly_int64 *size_p)
    1283              : {
    1284    154907438 :   tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
    1285    154907438 :   if (res)
    1286              :     return res;
    1287              : 
    1288    128795591 :   if (TREE_CODE (op) == SSA_NAME
    1289     73175822 :       && !SSA_NAME_IS_DEFAULT_DEF (op)
    1290    201765144 :       && gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
    1291     30267269 :     return unmodified_parm (fbi, SSA_NAME_DEF_STMT (op),
    1292     30267269 :                             gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op)),
    1293     30267269 :                             size_p);
    1294              :   return NULL_TREE;
    1295              : }
    1296              : 
    1297              : /* If OP refers to a value of a function parameter or value loaded from an
    1298              :    aggregate passed to a parameter (either by value or reference), return TRUE
    1299              :    and store the number of the parameter to *INDEX_P, the access size into
    1300              :    *SIZE_P, and information whether and how it has been loaded from an
    1301              :    aggregate into *AGGPOS.  INFO describes the function parameters, STMT is the
    1302              :    statement in which OP is used or loaded.  */
    1303              : 
    1304              : static bool
    1305     34162100 : unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
    1306              :                                   gimple *stmt, tree op, int *index_p,
    1307              :                                   poly_int64 *size_p,
    1308              :                                   struct agg_position_info *aggpos)
    1309              : {
    1310     35628385 :   tree res = unmodified_parm_1 (fbi, stmt, op, size_p);
    1311              : 
    1312     35628385 :   gcc_checking_assert (aggpos);
    1313     35628385 :   if (res)
    1314              :     {
    1315       849396 :       *index_p = ipa_get_param_decl_index (fbi->info, res);
    1316       849396 :       if (*index_p < 0)
    1317              :         return false;
    1318       849266 :       aggpos->agg_contents = false;
    1319       849266 :       aggpos->by_ref = false;
    1320       849266 :       return true;
    1321              :     }
    1322              : 
    1323     34778989 :   if (TREE_CODE (op) == SSA_NAME)
    1324              :     {
    1325     12420089 :       if (SSA_NAME_IS_DEFAULT_DEF (op)
    1326     12420089 :           || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op)))
    1327              :         return false;
    1328      4966412 :       stmt = SSA_NAME_DEF_STMT (op);
    1329      4966412 :       op = gimple_assign_rhs1 (stmt);
    1330      4966412 :       if (!REFERENCE_CLASS_P (op))
    1331              :         return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p, size_p,
    1332              :                                                  aggpos);
    1333              :     }
    1334              : 
    1335     25859027 :   aggpos->agg_contents = true;
    1336     25859027 :   return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
    1337              :                                  stmt, op, index_p, &aggpos->offset,
    1338     25859027 :                                  size_p, &aggpos->by_ref);
    1339              : }
    1340              : 
    1341              : /* If stmt is simple load or store of value pointed to by a function parameter,
    1342              :    return its index.  */
    1343              : 
    1344              : static int
    1345    114735321 : load_or_store_of_ptr_parameter (ipa_func_body_info *fbi, gimple *stmt)
    1346              : {
    1347    114735321 :   if (!optimize)
    1348              :     return -1;
    1349     96226725 :   gassign *assign = dyn_cast <gassign *> (stmt);
    1350     57315590 :   if (!assign)
    1351              :     return -1;
    1352     57315590 :   tree param;
    1353     57315590 :   if (gimple_assign_load_p (stmt))
    1354     20960047 :     param = gimple_assign_rhs1 (stmt);
    1355     36355543 :   else if (gimple_store_p (stmt))
    1356     19338585 :     param = gimple_assign_lhs (stmt);
    1357              :   else
    1358              :     return -1;
    1359     40298632 :   tree base = get_base_address (param);
    1360     40298632 :   if (TREE_CODE (base) != MEM_REF
    1361     13984879 :       || TREE_CODE (TREE_OPERAND (base, 0)) != SSA_NAME
    1362     54279381 :       || !SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0)))
    1363              :     return -1;
    1364      7545765 :   tree p = SSA_NAME_VAR (TREE_OPERAND (base, 0));
    1365      7545765 :   if (TREE_CODE (p) != PARM_DECL)
    1366              :     return -1;
    1367      7457750 :   return ipa_get_param_decl_index (fbi->info, p);
    1368              : }
    1369              : 
    1370              : /* See if statement might disappear after inlining.
    1371              :    0 - means not eliminated
    1372              :    1 - half of statements goes away
    1373              :    2 - for sure it is eliminated.
    1374              :    We are not terribly sophisticated, basically looking for simple abstraction
    1375              :    penalty wrappers.  */
    1376              : 
    1377              : static int
    1378    114735321 : eliminated_by_inlining_prob (ipa_func_body_info *fbi, gimple *stmt)
    1379              : {
    1380    114735321 :   enum gimple_code code = gimple_code (stmt);
    1381    114735321 :   enum tree_code rhs_code;
    1382              : 
    1383    114735321 :   if (!optimize)
    1384              :     return 0;
    1385              : 
    1386     96226725 :   switch (code)
    1387              :     {
    1388              :     case GIMPLE_RETURN:
    1389              :       return 2;
    1390     57315590 :     case GIMPLE_ASSIGN:
    1391     57315590 :       if (gimple_num_ops (stmt) != 2)
    1392              :         return 0;
    1393              : 
    1394     41135677 :       rhs_code = gimple_assign_rhs_code (stmt);
    1395              : 
    1396              :       /* Casts of parameters, loads from parameters passed by reference
    1397              :          and stores to return value or parameters are often free after
    1398              :          inlining due to SRA and further combining.
    1399              :          Assume that half of statements goes away.  */
    1400     41135677 :       if (CONVERT_EXPR_CODE_P (rhs_code)
    1401              :           || rhs_code == VIEW_CONVERT_EXPR
    1402              :           || rhs_code == ADDR_EXPR
    1403     38258075 :           || gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
    1404              :         {
    1405     40298632 :           tree rhs = gimple_assign_rhs1 (stmt);
    1406     40298632 :           tree lhs = gimple_assign_lhs (stmt);
    1407     40298632 :           tree inner_rhs = get_base_address (rhs);
    1408     40298632 :           tree inner_lhs = get_base_address (lhs);
    1409     40298632 :           bool rhs_free = false;
    1410     40298632 :           bool lhs_free = false;
    1411              : 
    1412     40298632 :           if (!inner_rhs)
    1413            0 :             inner_rhs = rhs;
    1414     40298632 :           if (!inner_lhs)
    1415            0 :             inner_lhs = lhs;
    1416              : 
    1417              :           /* Reads of parameter are expected to be free.  */
    1418     40298632 :           if (unmodified_parm (fbi, stmt, inner_rhs, NULL))
    1419              :             rhs_free = true;
    1420              :           /* Match expressions of form &this->field. Those will most likely
    1421              :              combine with something upstream after inlining.  */
    1422     38982221 :           else if (TREE_CODE (inner_rhs) == ADDR_EXPR)
    1423              :             {
    1424      2589068 :               tree op = get_base_address (TREE_OPERAND (inner_rhs, 0));
    1425      2589068 :               if (TREE_CODE (op) == PARM_DECL)
    1426              :                 rhs_free = true;
    1427      2544958 :               else if (TREE_CODE (op) == MEM_REF
    1428      2544958 :                        && unmodified_parm (fbi, stmt, TREE_OPERAND (op, 0),
    1429              :                                            NULL))
    1430              :                 rhs_free = true;
    1431              :             }
    1432              : 
    1433              :           /* When parameter is not SSA register because its address is taken
    1434              :              and it is just copied into one, the statement will be completely
    1435              :              free after inlining (we will copy propagate backward).   */
    1436      1360521 :           if (rhs_free && is_gimple_reg (lhs))
    1437              :             return 2;
    1438              : 
    1439              :           /* Reads of parameters passed by reference
    1440              :              expected to be free (i.e. optimized out after inlining).  */
    1441     39610657 :           if (TREE_CODE (inner_rhs) == MEM_REF
    1442     39610657 :               && unmodified_parm (fbi, stmt, TREE_OPERAND (inner_rhs, 0), NULL))
    1443              :             rhs_free = true;
    1444              : 
    1445              :           /* Copying parameter passed by reference into gimple register is
    1446              :              probably also going to copy propagate, but we can't be quite
    1447              :              sure.  */
    1448     39610657 :           if (rhs_free && is_gimple_reg (lhs))
    1449              :             lhs_free = true;
    1450              : 
    1451              :           /* Writes to parameters, parameters passed by value and return value
    1452              :              (either directly or passed via invisible reference) are free.
    1453              : 
    1454              :              TODO: We ought to handle testcase like
    1455              :              struct a {int a,b;};
    1456              :              struct a
    1457              :              returnstruct (void)
    1458              :              {
    1459              :              struct a a ={1,2};
    1460              :              return a;
    1461              :              }
    1462              : 
    1463              :              This translate into:
    1464              : 
    1465              :              returnstruct ()
    1466              :              {
    1467              :              int a$b;
    1468              :              int a$a;
    1469              :              struct a a;
    1470              :              struct a D.2739;
    1471              : 
    1472              :              <bb 2>:
    1473              :              D.2739.a = 1;
    1474              :              D.2739.b = 2;
    1475              :              return D.2739;
    1476              : 
    1477              :              }
    1478              :              For that we either need to copy ipa-split logic detecting writes
    1479              :              to return value.  */
    1480     39610657 :           if (TREE_CODE (inner_lhs) == PARM_DECL
    1481     39520998 :               || TREE_CODE (inner_lhs) == RESULT_DECL
    1482     78283105 :               || (TREE_CODE (inner_lhs) == MEM_REF
    1483      5209625 :                   && (unmodified_parm (fbi, stmt, TREE_OPERAND (inner_lhs, 0),
    1484              :                                        NULL)
    1485      2692445 :                       || (TREE_CODE (TREE_OPERAND (inner_lhs, 0)) == SSA_NAME
    1486      2691247 :                           && SSA_NAME_VAR (TREE_OPERAND (inner_lhs, 0))
    1487       378745 :                           && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
    1488              :                                                       (inner_lhs,
    1489              :                                                        0))) == RESULT_DECL))))
    1490              :             lhs_free = true;
    1491     36067362 :           if (lhs_free
    1492     39610657 :               && (is_gimple_reg (rhs) || is_gimple_min_invariant (rhs)))
    1493              :             rhs_free = true;
    1494     39610657 :           if (lhs_free && rhs_free)
    1495      8229825 :             return 1;
    1496              :         }
    1497              :       return 0;
    1498              :     default:
    1499              :       return 0;
    1500              :     }
    1501              : }
    1502              : 
    1503              : /* Analyze EXPR if it represents a series of simple operations performed on
    1504              :    a function parameter and return true if so.  FBI, STMT, EXPR, INDEX_P and
    1505              :    AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
    1506              :    Type of the parameter or load from an aggregate via the parameter is
    1507              :    stored in *TYPE_P.  Operations on the parameter are recorded to
    1508              :    PARAM_OPS_P if it is not NULL.  */
    1509              : 
    1510              : static bool
    1511     28022724 : decompose_param_expr (struct ipa_func_body_info *fbi,
    1512              :                       gimple *stmt, tree expr,
    1513              :                       int *index_p, tree *type_p,
    1514              :                       struct agg_position_info *aggpos,
    1515              :                       expr_eval_ops *param_ops_p = NULL)
    1516              : {
    1517     28022724 :   int op_limit = opt_for_fn (fbi->node->decl, param_ipa_max_param_expr_ops);
    1518     28022724 :   int op_count = 0;
    1519              : 
    1520     28022724 :   if (param_ops_p)
    1521      9607774 :     *param_ops_p = NULL;
    1522              : 
    1523     34162100 :   while (true)
    1524              :     {
    1525     34162100 :       expr_eval_op eval_op;
    1526     34162100 :       unsigned rhs_count;
    1527     34162100 :       unsigned cst_count = 0;
    1528              : 
    1529     34162100 :       if (unmodified_parm_or_parm_agg_item (fbi, stmt, expr, index_p, NULL,
    1530              :                                             aggpos))
    1531              :         {
    1532      4736180 :           tree type = TREE_TYPE (expr);
    1533              : 
    1534      4736180 :           if (aggpos->agg_contents)
    1535              :             {
    1536              :               /* Stop if containing bit-field.  */
    1537      3886914 :               if (TREE_CODE (expr) == BIT_FIELD_REF
    1538      3886914 :                   || contains_bitfld_component_ref_p (expr))
    1539              :                 break;
    1540              :             }
    1541              : 
    1542      4708687 :           *type_p = type;
    1543      4708687 :           return true;
    1544              :         }
    1545              : 
    1546     29425920 :       if (TREE_CODE (expr) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (expr))
    1547              :         break;
    1548     11070801 :       stmt = SSA_NAME_DEF_STMT (expr);
    1549              : 
    1550     11070801 :       if (gcall *call = dyn_cast <gcall *> (stmt))
    1551              :         {
    1552      2598465 :           int flags = gimple_call_return_flags (call);
    1553      2598465 :           if (!(flags & ERF_RETURNS_ARG))
    1554      3192517 :             goto fail;
    1555       231555 :           int arg = flags & ERF_RETURN_ARG_MASK;
    1556       231555 :           if (arg >= (int)gimple_call_num_args (call))
    1557            0 :             goto fail;
    1558       231555 :           expr = gimple_call_arg (stmt, arg);
    1559      4197963 :           continue;
    1560       231555 :         }
    1561              : 
    1562      8472336 :       if (!is_gimple_assign (stmt = SSA_NAME_DEF_STMT (expr)))
    1563              :         break;
    1564              : 
    1565      6733516 :       switch (gimple_assign_rhs_class (stmt))
    1566              :         {
    1567      3966408 :         case GIMPLE_SINGLE_RHS:
    1568      3966408 :           expr = gimple_assign_rhs1 (stmt);
    1569      3966408 :           continue;
    1570              : 
    1571              :         case GIMPLE_UNARY_RHS:
    1572              :           rhs_count = 1;
    1573              :           break;
    1574              : 
    1575      1679378 :         case GIMPLE_BINARY_RHS:
    1576      1679378 :           rhs_count = 2;
    1577      1679378 :           break;
    1578              : 
    1579          696 :         case GIMPLE_TERNARY_RHS:
    1580          696 :           rhs_count = 3;
    1581          696 :           break;
    1582              : 
    1583            0 :         default:
    1584            0 :           goto fail;
    1585              :         }
    1586              : 
    1587              :       /* Stop if expression is too complex.  */
    1588      2767108 :       if (op_count++ == op_limit)
    1589              :         break;
    1590              : 
    1591      2767020 :       if (param_ops_p)
    1592              :         {
    1593      2732723 :           eval_op.code = gimple_assign_rhs_code (stmt);
    1594      2732723 :           eval_op.type = TREE_TYPE (gimple_assign_lhs (stmt));
    1595      2732723 :           eval_op.val[0] = NULL_TREE;
    1596      2732723 :           eval_op.val[1] = NULL_TREE;
    1597              :         }
    1598              : 
    1599      2767020 :       expr = NULL_TREE;
    1600      6388512 :       for (unsigned i = 0; i < rhs_count; i++)
    1601              :         {
    1602      4447099 :           tree op = gimple_op (stmt, i + 1);
    1603              : 
    1604      4447099 :           gcc_assert (op && !TYPE_P (op));
    1605      4447099 :           if (is_gimple_ip_invariant (op))
    1606              :             {
    1607       857214 :               if (++cst_count == rhs_count)
    1608         1371 :                 goto fail;
    1609              : 
    1610       855843 :               eval_op.val[cst_count - 1] = op;
    1611              :             }
    1612      3589885 :           else if (!expr)
    1613              :             {
    1614              :               /* Found a non-constant operand, and record its index in rhs
    1615              :                  operands.  */
    1616      2765649 :               eval_op.index = i;
    1617      2765649 :               expr = op;
    1618              :             }
    1619              :           else
    1620              :             {
    1621              :               /* Found more than one non-constant operands.  */
    1622       824236 :               goto fail;
    1623              :             }
    1624              :         }
    1625              : 
    1626      1941413 :       if (param_ops_p)
    1627      1907736 :         vec_safe_insert (*param_ops_p, 0, eval_op);
    1628              :     }
    1629              : 
    1630              :   /* Failed to decompose, free resource and return.  */
    1631     23314037 : fail:
    1632     23314037 :   if (param_ops_p)
    1633      9433954 :     vec_free (*param_ops_p);
    1634              : 
    1635              :   return false;
    1636              : }
    1637              : 
    1638              : /* Record to SUMMARY that PARM is used by builtin_constant_p.  */
    1639              : 
    1640              : static void
    1641        12085 : add_builtin_constant_p_parm (class ipa_fn_summary *summary, int parm)
    1642              : {
    1643        12085 :   int ip;
    1644              : 
    1645              :   /* Avoid duplicates.  */
    1646        12143 :   for (unsigned int i = 0;
    1647        12143 :        summary->builtin_constant_p_parms.iterate (i, &ip); i++)
    1648         5987 :     if (ip == parm)
    1649        12085 :       return;
    1650         6156 :   summary->builtin_constant_p_parms.safe_push (parm);
    1651              : }
    1652              : 
    1653              : /* If BB ends by a conditional we can turn into predicates, attach corresponding
    1654              :    predicates to the CFG edges.   */
    1655              : 
    1656              : static void
    1657     36155811 : set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
    1658              :                                    class ipa_fn_summary *summary,
    1659              :                                    class ipa_node_params *params_summary,
    1660              :                                    basic_block bb)
    1661              : {
    1662     36155811 :   tree op, op2;
    1663     36155811 :   int index;
    1664     36155811 :   struct agg_position_info aggpos;
    1665     36155811 :   enum tree_code code, inverted_code;
    1666     36155811 :   edge e;
    1667     36155811 :   edge_iterator ei;
    1668     36155811 :   gimple *set_stmt;
    1669     36155811 :   tree param_type;
    1670     36155811 :   expr_eval_ops param_ops;
    1671              : 
    1672     72311622 :   gcond *last = safe_dyn_cast <gcond *> (*gsi_last_bb (bb));
    1673     11986371 :   if (!last)
    1674     36142729 :     return;
    1675     11986371 :   if (!is_gimple_ip_invariant (gimple_cond_rhs (last)))
    1676              :     return;
    1677      9524196 :   op = gimple_cond_lhs (last);
    1678              : 
    1679      9524196 :   if (decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
    1680              :                             &param_ops))
    1681              :     {
    1682      1351686 :       code = gimple_cond_code (last);
    1683      1351686 :       inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
    1684              : 
    1685      4055058 :       FOR_EACH_EDGE (e, ei, bb->succs)
    1686              :         {
    1687      1351686 :           enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
    1688      2703372 :                                       ? code : inverted_code);
    1689              :           /* invert_tree_comparison will return ERROR_MARK on FP
    1690              :              comparisons that are not EQ/NE instead of returning proper
    1691              :              unordered one.  Be sure it is not confused with NON_CONSTANT.
    1692              : 
    1693              :              And if the edge's target is the final block of diamond CFG graph
    1694              :              of this conditional statement, we do not need to compute
    1695              :              predicate for the edge because the final block's predicate must
    1696              :              be at least as that of the first block of the statement.  */
    1697      2703372 :           if (this_code != ERROR_MARK
    1698      2703372 :               && !dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
    1699              :             {
    1700      2300602 :               ipa_predicate p
    1701      2300602 :                 = add_condition (summary, params_summary, index,
    1702              :                                  param_type, &aggpos,
    1703              :                                  this_code, gimple_cond_rhs (last), param_ops);
    1704      2300602 :               e->aux = edge_predicate_pool.allocate ();
    1705      2300602 :               *(ipa_predicate *) e->aux = p;
    1706              :             }
    1707              :         }
    1708     36371085 :       vec_free (param_ops);
    1709              :       return;
    1710              :     }
    1711              : 
    1712      8172510 :   if (TREE_CODE (op) != SSA_NAME)
    1713              :     return;
    1714              :   /* Special case
    1715              :      if (builtin_constant_p (op))
    1716              :      constant_code
    1717              :      else
    1718              :      nonconstant_code.
    1719              :      Here we can predicate nonconstant_code.  We can't
    1720              :      really handle constant_code since we have no predicate
    1721              :      for this and also the constant code is not known to be
    1722              :      optimized away when inliner doesn't see operand is constant.
    1723              :      Other optimizers might think otherwise.  */
    1724      8171425 :   if (gimple_cond_code (last) != NE_EXPR
    1725      8171425 :       || !integer_zerop (gimple_cond_rhs (last)))
    1726              :     return;
    1727      3600173 :   set_stmt = SSA_NAME_DEF_STMT (op);
    1728      3600173 :   if (!gimple_call_builtin_p (set_stmt, BUILT_IN_CONSTANT_P)
    1729      3600173 :       || gimple_call_num_args (set_stmt) != 1)
    1730              :     return;
    1731        46462 :   op2 = gimple_call_arg (set_stmt, 0);
    1732        46462 :   if (!decompose_param_expr (fbi, set_stmt, op2, &index, &param_type, &aggpos))
    1733              :     return;
    1734        13082 :   if (!aggpos.by_ref)
    1735        12023 :     add_builtin_constant_p_parm (summary, index);
    1736        39246 :   FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
    1737              :     {
    1738        13082 :       ipa_predicate p = add_condition (summary, params_summary, index,
    1739              :                                    param_type, &aggpos,
    1740              :                                    ipa_predicate::is_not_constant, NULL_TREE);
    1741        13082 :       e->aux = edge_predicate_pool.allocate ();
    1742        13082 :       *(ipa_predicate *) e->aux = p;
    1743              :     }
    1744              : }
    1745              : 
    1746              : 
    1747              : /* If BB ends by a switch we can turn into predicates, attach corresponding
    1748              :    predicates to the CFG edges.   */
    1749              : 
    1750              : static void
    1751     36155811 : set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
    1752              :                                      class ipa_fn_summary *summary,
    1753              :                                      class ipa_node_params *params_summary,
    1754              :                                      basic_block bb)
    1755              : {
    1756     36155811 :   tree op;
    1757     36155811 :   int index;
    1758     36155811 :   struct agg_position_info aggpos;
    1759     36155811 :   edge e;
    1760     36155811 :   edge_iterator ei;
    1761     36155811 :   size_t n;
    1762     36155811 :   size_t case_idx;
    1763     36155811 :   tree param_type;
    1764     36155811 :   expr_eval_ops param_ops;
    1765              : 
    1766     72311622 :   gswitch *last = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb));
    1767        83578 :   if (!last)
    1768     36126071 :     return;
    1769        83578 :   op = gimple_switch_index (last);
    1770        83578 :   if (!decompose_param_expr (fbi, last, op, &index, &param_type, &aggpos,
    1771              :                              &param_ops))
    1772              :     return;
    1773              : 
    1774        44966 :   auto_vec<std::pair<tree, tree> > ranges;
    1775        44966 :   tree type = TREE_TYPE (op);
    1776        44966 :   int bound_limit = opt_for_fn (fbi->node->decl,
    1777              :                                 param_ipa_max_switch_predicate_bounds);
    1778        44966 :   int bound_count = 0;
    1779              :   // This can safely be an integer range, as switches can only hold
    1780              :   // integers.
    1781        44966 :   int_range<2> vr;
    1782              : 
    1783        89932 :   get_range_query (cfun)->range_of_expr (vr, op);
    1784        44966 :   if (vr.undefined_p ())
    1785            0 :     vr.set_varying (TREE_TYPE (op));
    1786        44966 :   tree vr_min, vr_max;
    1787              :   // TODO: This entire function could use a rewrite to use the irange
    1788              :   // API, instead of trying to recreate its intersection/union logic.
    1789              :   // Any use of get_legacy_range() is a serious code smell.
    1790        44966 :   value_range_kind vr_type = get_legacy_range (vr, vr_min, vr_max);
    1791        44966 :   wide_int vr_wmin = wi::to_wide (vr_min);
    1792        44966 :   wide_int vr_wmax = wi::to_wide (vr_max);
    1793              : 
    1794       311620 :   FOR_EACH_EDGE (e, ei, bb->succs)
    1795              :     {
    1796       266654 :       e->aux = edge_predicate_pool.allocate ();
    1797       266654 :       *(ipa_predicate *) e->aux = false;
    1798              :     }
    1799              : 
    1800        44966 :   e = gimple_switch_edge (cfun, last, 0);
    1801              :   /* Set BOUND_COUNT to maximum count to bypass computing predicate for
    1802              :      default case if its target basic block is in convergence point of all
    1803              :      switch cases, which can be determined by checking whether it
    1804              :      post-dominates the switch statement.  */
    1805        44966 :   if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
    1806        12658 :     bound_count = INT_MAX;
    1807              : 
    1808        44966 :   n = gimple_switch_num_labels (last);
    1809       297923 :   for (case_idx = 1; case_idx < n; ++case_idx)
    1810              :     {
    1811       252957 :       tree cl = gimple_switch_label (last, case_idx);
    1812       252957 :       tree min = CASE_LOW (cl);
    1813       252957 :       tree max = CASE_HIGH (cl);
    1814       252957 :       ipa_predicate p;
    1815              : 
    1816       252957 :       e = gimple_switch_edge (cfun, last, case_idx);
    1817              : 
    1818              :       /* The case value might not have same type as switch expression,
    1819              :          extend the value based on the expression type.  */
    1820       252957 :       if (TREE_TYPE (min) != type)
    1821        45173 :         min = wide_int_to_tree (type, wi::to_widest (min));
    1822              : 
    1823       252957 :       if (!max)
    1824              :         max = min;
    1825        12691 :       else if (TREE_TYPE (max) != type)
    1826         3522 :         max = wide_int_to_tree (type, wi::to_widest (max));
    1827              : 
    1828              :       /* The case's target basic block is in convergence point of all switch
    1829              :          cases, its predicate should be at least as that of the switch
    1830              :          statement.  */
    1831       252957 :       if (dominated_by_p (CDI_POST_DOMINATORS, bb, e->dest))
    1832         5229 :         p = true;
    1833       247728 :       else if (min == max)
    1834       236977 :         p = add_condition (summary, params_summary, index, param_type,
    1835              :                            &aggpos, EQ_EXPR, min, param_ops);
    1836              :       else
    1837              :         {
    1838        10751 :           ipa_predicate p1, p2;
    1839        10751 :           p1 = add_condition (summary, params_summary, index, param_type,
    1840              :                               &aggpos, GE_EXPR, min, param_ops);
    1841        10751 :           p2 = add_condition (summary,  params_summary,index, param_type,
    1842              :                               &aggpos, LE_EXPR, max, param_ops);
    1843        10751 :           p = p1 & p2;
    1844              :         }
    1845       252957 :       *(ipa_predicate *) e->aux
    1846       252957 :         = p.or_with (summary->conds, *(ipa_predicate *) e->aux);
    1847              : 
    1848              :       /* If there are too many disjoint case ranges, predicate for default
    1849              :          case might become too complicated.  So add a limit here.  */
    1850       252957 :       if (bound_count > bound_limit)
    1851        98136 :         continue;
    1852              : 
    1853       154821 :       bool new_range = true;
    1854              : 
    1855       154821 :       if (!ranges.is_empty ())
    1856              :         {
    1857       122513 :           wide_int curr_wmin = wi::to_wide (min);
    1858       122513 :           wide_int last_wmax = wi::to_wide (ranges.last ().second);
    1859              : 
    1860              :           /* Merge case ranges if they are continuous.  */
    1861       122513 :           if (curr_wmin == last_wmax + 1)
    1862              :             new_range = false;
    1863        42314 :           else if (vr_type == VR_ANTI_RANGE)
    1864              :             {
    1865              :               /* If two disjoint case ranges can be connected by anti-range
    1866              :                  of switch index, combine them to one range.  */
    1867          141 :               if (wi::lt_p (vr_wmax, curr_wmin - 1, TYPE_SIGN (type)))
    1868              :                 vr_type = VR_UNDEFINED;
    1869            0 :               else if (wi::le_p (vr_wmin, last_wmax + 1, TYPE_SIGN (type)))
    1870            0 :                 new_range = false;
    1871              :             }
    1872       122513 :         }
    1873              : 
    1874              :       /* Create/extend a case range.  And we count endpoints of range set,
    1875              :          this number nearly equals to number of conditions that we will create
    1876              :          for predicate of default case.  */
    1877       122513 :       if (new_range)
    1878              :         {
    1879        74622 :           bound_count += (min == max) ? 1 : 2;
    1880        74622 :           ranges.safe_push (std::make_pair (min, max));
    1881              :         }
    1882              :       else
    1883              :         {
    1884        80199 :           bound_count += (ranges.last ().first == ranges.last ().second);
    1885        80199 :           ranges.last ().second = max;
    1886              :         }
    1887              :     }
    1888              : 
    1889        44966 :   e = gimple_switch_edge (cfun, last, 0);
    1890        44966 :   if (bound_count > bound_limit)
    1891              :     {
    1892        15226 :       *(ipa_predicate *) e->aux = true;
    1893        15226 :       vec_free (param_ops);
    1894        15226 :       return;
    1895              :     }
    1896              : 
    1897        29740 :   ipa_predicate p_seg = true;
    1898        29740 :   ipa_predicate p_all = false;
    1899              : 
    1900        29740 :   if (vr_type != VR_RANGE)
    1901              :     {
    1902        29140 :       vr_wmin = wi::to_wide (TYPE_MIN_VALUE (type));
    1903        29140 :       vr_wmax = wi::to_wide (TYPE_MAX_VALUE (type));
    1904              :     }
    1905              : 
    1906              :   /* Construct predicate to represent default range set that is negation of
    1907              :      all case ranges.  Case range is classified as containing single/non-single
    1908              :      values.  Suppose a piece of case ranges in the following.
    1909              : 
    1910              :                 [D1...D2]  [S1] ... [Sn]  [D3...D4]
    1911              : 
    1912              :      To represent default case's range sets between two non-single value
    1913              :      case ranges (From D2 to D3), we construct predicate as:
    1914              : 
    1915              :               D2 < x < D3 && x != S1 && ... && x != Sn
    1916              :    */
    1917        91716 :   for (size_t i = 0; i < ranges.length (); i++)
    1918              :     {
    1919        62073 :       tree min = ranges[i].first;
    1920        62073 :       tree max = ranges[i].second;
    1921              : 
    1922        62073 :       if (min == max)
    1923        73902 :         p_seg &= add_condition (summary, params_summary, index,
    1924              :                                 param_type, &aggpos, NE_EXPR,
    1925        36951 :                                 min, param_ops);
    1926              :       else
    1927              :         {
    1928              :           /* Do not create sub-predicate for range that is beyond low bound
    1929              :              of switch index.  */
    1930        25122 :           if (wi::lt_p (vr_wmin, wi::to_wide (min), TYPE_SIGN (type)))
    1931              :             {
    1932        18012 :               p_seg &= add_condition (summary, params_summary, index,
    1933              :                                       param_type, &aggpos,
    1934        18012 :                                       LT_EXPR, min, param_ops);
    1935        18012 :               p_all = p_all.or_with (summary->conds, p_seg);
    1936              :             }
    1937              : 
    1938              :           /* Do not create sub-predicate for range that is beyond up bound
    1939              :              of switch index.  */
    1940        25122 :           if (wi::le_p (vr_wmax, wi::to_wide (max), TYPE_SIGN (type)))
    1941              :             {
    1942           97 :               p_seg = false;
    1943           97 :               break;
    1944              :             }
    1945              : 
    1946        25025 :           p_seg = add_condition (summary, params_summary, index,
    1947              :                                  param_type, &aggpos, GT_EXPR,
    1948              :                                  max, param_ops);
    1949              :         }
    1950              :     }
    1951              : 
    1952        29740 :   p_all = p_all.or_with (summary->conds, p_seg);
    1953        29740 :   *(ipa_predicate *) e->aux
    1954        29740 :     = p_all.or_with (summary->conds, *(ipa_predicate *) e->aux);
    1955              : 
    1956        36333 :   vec_free (param_ops);
    1957        44966 : }
    1958              : 
    1959              : 
    1960              : /* For each BB in NODE attach to its AUX pointer predicate under
    1961              :    which it is executable.  */
    1962              : 
    1963              : static void
    1964      6517766 : compute_bb_predicates (struct ipa_func_body_info *fbi,
    1965              :                        struct cgraph_node *node,
    1966              :                        class ipa_fn_summary *summary,
    1967              :                        class ipa_node_params *params_summary)
    1968              : {
    1969      6517766 :   struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
    1970      6517766 :   bool done = false;
    1971      6517766 :   basic_block bb;
    1972              : 
    1973     42673577 :   FOR_EACH_BB_FN (bb, my_function)
    1974              :     {
    1975     36155811 :       set_cond_stmt_execution_predicate (fbi, summary, params_summary, bb);
    1976     36155811 :       set_switch_stmt_execution_predicate (fbi, summary, params_summary, bb);
    1977              :     }
    1978              : 
    1979              :   /* Entry block is always executable.  */
    1980      6517766 :   ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
    1981      6517766 :     = edge_predicate_pool.allocate ();
    1982      6517766 :   *(ipa_predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux = true;
    1983              : 
    1984              :   /* A simple dataflow propagation of predicates forward in the CFG.
    1985              :      TODO: work in reverse postorder.  */
    1986     19678448 :   while (!done)
    1987              :     {
    1988     13160682 :       done = true;
    1989     90850511 :       FOR_EACH_BB_FN (bb, my_function)
    1990              :         {
    1991     77689829 :           ipa_predicate p = false;
    1992     77689829 :           edge e;
    1993     77689829 :           edge_iterator ei;
    1994     96214953 :           FOR_EACH_EDGE (e, ei, bb->preds)
    1995              :             {
    1996     82570356 :               if (e->src->aux)
    1997              :                 {
    1998     80663836 :                   ipa_predicate this_bb_predicate
    1999              :                     = *(ipa_predicate *) e->src->aux;
    2000     80663836 :                   if (e->aux)
    2001      5367678 :                     this_bb_predicate &= (*(ipa_predicate *) e->aux);
    2002     80663836 :                   p = p.or_with (summary->conds, this_bb_predicate);
    2003     80663836 :                   if (p == true)
    2004              :                     break;
    2005              :                 }
    2006              :             }
    2007     77689829 :           if (p != false)
    2008              :             {
    2009     76423338 :               basic_block pdom_bb;
    2010              : 
    2011     76423338 :               if (!bb->aux)
    2012              :                 {
    2013     28399642 :                   done = false;
    2014     28399642 :                   bb->aux = edge_predicate_pool.allocate ();
    2015     28399642 :                   *((ipa_predicate *) bb->aux) = p;
    2016              :                 }
    2017     48023696 :               else if (p != *(ipa_predicate *) bb->aux)
    2018              :                 {
    2019              :                   /* This OR operation is needed to ensure monotonous data flow
    2020              :                      in the case we hit the limit on number of clauses and the
    2021              :                      and/or operations above give approximate answers.  */
    2022       128262 :                   p = p.or_with (summary->conds, *(ipa_predicate *)bb->aux);
    2023       128262 :                   if (p != *(ipa_predicate *)bb->aux)
    2024              :                     {
    2025       103477 :                       done = false;
    2026       103477 :                       *((ipa_predicate *)bb->aux) = p;
    2027              :                     }
    2028              :                 }
    2029              : 
    2030              :               /* For switch/if statement, we can OR-combine predicates of all
    2031              :                  its cases/branches to get predicate for basic block in their
    2032              :                  convergence point, but sometimes this will generate very
    2033              :                  complicated predicate.  Actually, we can get simplified
    2034              :                  predicate in another way by using the fact that predicate
    2035              :                  for a basic block must also hold true for its post dominators.
    2036              :                  To be specific, basic block in convergence point of
    2037              :                  conditional statement should include predicate of the
    2038              :                  statement.  */
    2039     76423338 :               pdom_bb = get_immediate_dominator (CDI_POST_DOMINATORS, bb);
    2040     76423338 :               if (pdom_bb == EXIT_BLOCK_PTR_FOR_FN (my_function) || !pdom_bb)
    2041              :                 ;
    2042     39199082 :               else if (!pdom_bb->aux)
    2043              :                 {
    2044      7756153 :                   done = false;
    2045      7756153 :                   pdom_bb->aux = edge_predicate_pool.allocate ();
    2046      7756153 :                   *((ipa_predicate *)pdom_bb->aux) = p;
    2047              :                 }
    2048     31442929 :               else if (p != *(ipa_predicate *)pdom_bb->aux)
    2049              :                 {
    2050      3436261 :                   p = p.or_with (summary->conds,
    2051              :                                  *(ipa_predicate *)pdom_bb->aux);
    2052      3436261 :                   if (p != *(ipa_predicate *)pdom_bb->aux)
    2053              :                     {
    2054       140980 :                       done = false;
    2055       140980 :                       *((ipa_predicate *)pdom_bb->aux) = p;
    2056              :                     }
    2057              :                 }
    2058              :             }
    2059              :         }
    2060              :     }
    2061      6517766 : }
    2062              : 
    2063              : 
    2064              : /* Return predicate specifying when the STMT might have result that is not
    2065              :    a compile time constant.  */
    2066              : 
    2067              : static ipa_predicate
    2068      4302669 : will_be_nonconstant_expr_predicate (ipa_func_body_info *fbi,
    2069              :                                     class ipa_fn_summary *summary,
    2070              :                                     class ipa_node_params *params_summary,
    2071              :                                     tree expr,
    2072              :                                     vec<ipa_predicate> nonconstant_names)
    2073              : {
    2074      4302669 :   tree parm;
    2075      4302669 :   int index;
    2076              : 
    2077      4551444 :   while (UNARY_CLASS_P (expr))
    2078       248775 :     expr = TREE_OPERAND (expr, 0);
    2079              : 
    2080      4302669 :   parm = unmodified_parm (fbi, NULL, expr, NULL);
    2081      4302669 :   if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
    2082       222390 :     return add_condition (summary, params_summary, index, TREE_TYPE (parm), NULL,
    2083       222390 :                           ipa_predicate::changed, NULL_TREE);
    2084      4080279 :   if (is_gimple_min_invariant (expr))
    2085        64369 :     return false;
    2086      4015910 :   if (TREE_CODE (expr) == SSA_NAME)
    2087      3832269 :     return nonconstant_names[SSA_NAME_VERSION (expr)];
    2088       183641 :   if (BINARY_CLASS_P (expr) || COMPARISON_CLASS_P (expr))
    2089              :     {
    2090       183039 :       ipa_predicate p1
    2091       183039 :         = will_be_nonconstant_expr_predicate (fbi, summary,
    2092              :                                               params_summary,
    2093       183039 :                                               TREE_OPERAND (expr, 0),
    2094              :                                               nonconstant_names);
    2095       183039 :       if (p1 == true)
    2096       105073 :         return p1;
    2097              : 
    2098        77966 :       ipa_predicate p2
    2099        77966 :         = will_be_nonconstant_expr_predicate (fbi, summary,
    2100              :                                               params_summary,
    2101        77966 :                                               TREE_OPERAND (expr, 1),
    2102              :                                               nonconstant_names);
    2103        77966 :       return p1.or_with (summary->conds, p2);
    2104              :     }
    2105          602 :   else if (TREE_CODE (expr) == COND_EXPR)
    2106              :     {
    2107           89 :       ipa_predicate p1
    2108           89 :         = will_be_nonconstant_expr_predicate (fbi, summary,
    2109              :                                               params_summary,
    2110           89 :                                               TREE_OPERAND (expr, 0),
    2111              :                                               nonconstant_names);
    2112           89 :       if (p1 == true)
    2113           27 :         return p1;
    2114              : 
    2115           62 :       ipa_predicate p2
    2116           62 :         = will_be_nonconstant_expr_predicate (fbi, summary,
    2117              :                                               params_summary,
    2118           62 :                                               TREE_OPERAND (expr, 1),
    2119              :                                               nonconstant_names);
    2120           62 :       if (p2 == true)
    2121           62 :         return p2;
    2122            0 :       p1 = p1.or_with (summary->conds, p2);
    2123            0 :       p2 = will_be_nonconstant_expr_predicate (fbi, summary,
    2124              :                                                params_summary,
    2125            0 :                                                TREE_OPERAND (expr, 2),
    2126              :                                                nonconstant_names);
    2127            0 :       return p2.or_with (summary->conds, p1);
    2128              :     }
    2129          513 :   else if (TREE_CODE (expr) == CALL_EXPR)
    2130          513 :     return true;
    2131              :   else
    2132              :     {
    2133            0 :       debug_tree (expr);
    2134            0 :       gcc_unreachable ();
    2135              :     }
    2136              : }
    2137              : 
    2138              : 
    2139              : /* Return predicate specifying when the STMT might have result that is not
    2140              :    a compile time constant.  */
    2141              : 
    2142              : static ipa_predicate
    2143    117712834 : will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
    2144              :                                class ipa_fn_summary *summary,
    2145              :                                class ipa_node_params *params_summary,
    2146              :                                gimple *stmt,
    2147              :                                vec<ipa_predicate> nonconstant_names)
    2148              : {
    2149    117712834 :   ipa_predicate p = true;
    2150    117712834 :   ssa_op_iter iter;
    2151    117712834 :   tree use;
    2152    117712834 :   tree param_type = NULL_TREE;
    2153    117712834 :   ipa_predicate op_non_const;
    2154    117712834 :   bool is_load;
    2155    117712834 :   int base_index;
    2156    117712834 :   struct agg_position_info aggpos;
    2157              : 
    2158              :   /* What statements might be optimized away
    2159              :      when their arguments are constant.  */
    2160    117712834 :   if (gimple_code (stmt) != GIMPLE_ASSIGN
    2161              :       && gimple_code (stmt) != GIMPLE_COND
    2162              :       && gimple_code (stmt) != GIMPLE_SWITCH
    2163              :       && (gimple_code (stmt) != GIMPLE_CALL
    2164     21478764 :           || !(gimple_call_flags (stmt) & ECF_CONST)))
    2165     26872855 :     return p;
    2166              : 
    2167              :   /* Stores will stay anyway.  */
    2168     90839979 :   if (gimple_store_p (stmt))
    2169     26790466 :     return p;
    2170              : 
    2171     64049513 :   is_load = gimple_assign_load_p (stmt);
    2172              : 
    2173              :   /* Loads can be optimized when the value is known.  */
    2174     64049513 :   if (is_load)
    2175              :     {
    2176     18368488 :       tree op = gimple_assign_rhs1 (stmt);
    2177     18368488 :       if (!decompose_param_expr (fbi, stmt, op, &base_index, &param_type,
    2178              :                                  &aggpos))
    2179     15069535 :         return p;
    2180              :     }
    2181              :   else
    2182     45681025 :     base_index = -1;
    2183              : 
    2184              :   /* See if we understand all operands before we start
    2185              :      adding conditionals.  */
    2186     66726533 :   FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
    2187              :     {
    2188     48811163 :       tree parm = unmodified_parm (fbi, stmt, use, NULL);
    2189              :       /* For arguments we can build a condition.  */
    2190     48811163 :       if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
    2191      8664044 :         continue;
    2192     40147119 :       if (TREE_CODE (use) != SSA_NAME)
    2193            0 :         return p;
    2194              :       /* If we know when operand is constant,
    2195              :          we still can say something useful.  */
    2196     40147119 :       if (nonconstant_names[SSA_NAME_VERSION (use)] != true)
    2197      9082511 :         continue;
    2198     31064608 :       return p;
    2199              :     }
    2200              : 
    2201     17915370 :   if (is_load)
    2202      3298872 :     op_non_const =
    2203      3298872 :       add_condition (summary, params_summary,
    2204              :                      base_index, param_type, &aggpos,
    2205              :                      ipa_predicate::changed, NULL_TREE);
    2206              :   else
    2207     14616498 :     op_non_const = false;
    2208     34613195 :   FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
    2209              :     {
    2210     16697825 :       tree parm = unmodified_parm (fbi, stmt, use, NULL);
    2211     16697825 :       int index;
    2212              : 
    2213     16697825 :       if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
    2214              :         {
    2215      8165933 :           if (index != base_index)
    2216      5575651 :             p = add_condition (summary, params_summary, index,
    2217      5575651 :                                TREE_TYPE (parm), NULL,
    2218              :                                ipa_predicate::changed, NULL_TREE);
    2219              :           else
    2220      2590282 :             continue;
    2221              :         }
    2222              :       else
    2223      8531892 :         p = nonconstant_names[SSA_NAME_VERSION (use)];
    2224     14107543 :       op_non_const = p.or_with (summary->conds, op_non_const);
    2225              :     }
    2226     17915370 :   if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
    2227     15571717 :       && gimple_op (stmt, 0)
    2228     33190345 :       && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
    2229     15274975 :     nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
    2230     15274975 :       = op_non_const;
    2231     17915370 :   return op_non_const;
    2232              : }
    2233              : 
    2234              : struct record_modified_bb_info
    2235              : {
    2236              :   tree op;
    2237              :   bitmap bb_set;
    2238              :   gimple *stmt;
    2239              : };
    2240              : 
    2241              : /* Value is initialized in INIT_BB and used in USE_BB.  We want to compute
    2242              :    probability how often it changes between USE_BB.
    2243              :    INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
    2244              :    is in different loop nest, we can do better.
    2245              :    This is all just estimate.  In theory we look for minimal cut separating
    2246              :    INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
    2247              :    anyway.  */
    2248              : 
    2249              : static basic_block
    2250     10705206 : get_minimal_bb (basic_block init_bb, basic_block use_bb)
    2251              : {
    2252     10705206 :   class loop *l = find_common_loop (init_bb->loop_father, use_bb->loop_father);
    2253     10705206 :   if (l && l->header->count < init_bb->count)
    2254       386292 :     return l->header;
    2255              :   return init_bb;
    2256              : }
    2257              : 
    2258              : /* Callback of walk_aliased_vdefs.  Records basic blocks where the value may be
    2259              :    set except for info->stmt.  */
    2260              : 
    2261              : static bool
    2262      4954181 : record_modified (ao_ref *ao ATTRIBUTE_UNUSED, tree vdef, void *data)
    2263              : {
    2264      4954181 :   struct record_modified_bb_info *info =
    2265              :     (struct record_modified_bb_info *) data;
    2266      4954181 :   if (SSA_NAME_DEF_STMT (vdef) == info->stmt)
    2267              :     return false;
    2268      4926074 :   if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef)))
    2269              :     return false;
    2270      4876922 :   bitmap_set_bit (info->bb_set,
    2271      4876922 :                   SSA_NAME_IS_DEFAULT_DEF (vdef)
    2272            0 :                   ? ENTRY_BLOCK_PTR_FOR_FN (cfun)->index
    2273              :                   : get_minimal_bb
    2274      4876922 :                          (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
    2275      4876922 :                           gimple_bb (info->stmt))->index);
    2276      4876922 :   if (dump_file)
    2277              :     {
    2278            0 :       fprintf (dump_file, "     Param ");
    2279            0 :       print_generic_expr (dump_file, info->op, TDF_SLIM);
    2280            0 :       fprintf (dump_file, " changed at bb %i, minimal: %i stmt: ",
    2281            0 :                gimple_bb (SSA_NAME_DEF_STMT (vdef))->index,
    2282              :                get_minimal_bb
    2283            0 :                          (gimple_bb (SSA_NAME_DEF_STMT (vdef)),
    2284            0 :                           gimple_bb (info->stmt))->index);
    2285            0 :       print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (vdef), 0);
    2286              :     }
    2287              :   return false;
    2288              : }
    2289              : 
    2290              : /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
    2291              :    will change since last invocation of STMT.
    2292              : 
    2293              :    Value 0 is reserved for compile time invariants.
    2294              :    For common parameters it is REG_BR_PROB_BASE.  For loop invariants it
    2295              :    ought to be REG_BR_PROB_BASE / estimated_iters.  */
    2296              : 
    2297              : static int
    2298     39808244 : param_change_prob (ipa_func_body_info *fbi, gimple *stmt, int i)
    2299              : {
    2300     39808244 :   tree op = gimple_call_arg (stmt, i);
    2301     39808244 :   basic_block bb = gimple_bb (stmt);
    2302              : 
    2303     39808244 :   if (TREE_CODE (op) == WITH_SIZE_EXPR)
    2304          299 :     op = TREE_OPERAND (op, 0);
    2305              : 
    2306     39808244 :   tree base = get_base_address (op);
    2307              : 
    2308              :   /* Global invariants never change.  */
    2309     39808244 :   if (is_gimple_min_invariant (base))
    2310              :     return 0;
    2311              : 
    2312              :   /* We would have to do non-trivial analysis to really work out what
    2313              :      is the probability of value to change (i.e. when init statement
    2314              :      is in a sibling loop of the call).
    2315              : 
    2316              :      We do an conservative estimate: when call is executed N times more often
    2317              :      than the statement defining value, we take the frequency 1/N.  */
    2318     19924240 :   if (TREE_CODE (base) == SSA_NAME)
    2319              :     {
    2320     17511040 :       profile_count init_count;
    2321              : 
    2322     26870259 :       if (!bb->count.nonzero_p ())
    2323              :         return REG_BR_PROB_BASE;
    2324              : 
    2325      8932553 :       if (SSA_NAME_IS_DEFAULT_DEF (base))
    2326      3104269 :         init_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    2327              :       else
    2328      5828284 :         init_count = get_minimal_bb
    2329      5828284 :                       (gimple_bb (SSA_NAME_DEF_STMT (base)),
    2330              :                        gimple_bb (stmt))->count;
    2331              : 
    2332      8932553 :       if (init_count < bb->count)
    2333       466787 :         return MAX ((init_count.to_sreal_scale (bb->count)
    2334              :                      * REG_BR_PROB_BASE).to_int (), 1);
    2335              :       return REG_BR_PROB_BASE;
    2336              :     }
    2337              :   else
    2338              :     {
    2339      2413200 :       ao_ref refd;
    2340      2413200 :       profile_count max = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    2341      2413200 :       struct record_modified_bb_info info;
    2342      2413200 :       tree init = ctor_for_folding (base);
    2343              : 
    2344      2413200 :       if (init != error_mark_node)
    2345              :         return 0;
    2346      5386314 :       if (!bb->count.nonzero_p () || fbi->aa_walk_budget == 0)
    2347              :         return REG_BR_PROB_BASE;
    2348      1474298 :       if (dump_file)
    2349              :         {
    2350            0 :           fprintf (dump_file, "     Analyzing param change probability of ");
    2351            0 :           print_generic_expr (dump_file, op, TDF_SLIM);
    2352            0 :           fprintf (dump_file, "\n");
    2353              :         }
    2354      1474298 :       ao_ref_init (&refd, op);
    2355      1474298 :       info.op = op;
    2356      1474298 :       info.stmt = stmt;
    2357      1474298 :       info.bb_set = BITMAP_ALLOC (NULL);
    2358      1474298 :       int walked
    2359      2948596 :         = walk_aliased_vdefs (&refd, gimple_vuse (stmt), record_modified, &info,
    2360              :                               NULL, NULL, fbi->aa_walk_budget);
    2361      1474298 :       if (walked > 0)
    2362      1337686 :         fbi->aa_walk_budget -= walked;
    2363      1474298 :       if (walked < 0 || bitmap_bit_p (info.bb_set, bb->index))
    2364              :         {
    2365       894484 :           if (walked < 0)
    2366          212 :             fbi->aa_walk_budget = 0;
    2367       894484 :           if (dump_file)
    2368              :             {
    2369            0 :               if (walked < 0)
    2370            0 :                 fprintf (dump_file, "     Ran out of AA walking budget.\n");
    2371              :               else
    2372            0 :                 fprintf (dump_file, "     Set in same BB as used.\n");
    2373              :             }
    2374       894484 :           BITMAP_FREE (info.bb_set);
    2375       894484 :           return REG_BR_PROB_BASE;
    2376              :         }
    2377              : 
    2378       579814 :       bitmap_iterator bi;
    2379       579814 :       unsigned index;
    2380              :       /* Lookup the most frequent update of the value and believe that
    2381              :          it dominates all the other; precise analysis here is difficult.  */
    2382      1620360 :       EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
    2383      1040546 :         max = profile_count::max_prefer_initialized
    2384      1040546 :                 (max, BASIC_BLOCK_FOR_FN (cfun, index)->count);
    2385       579814 :       if (dump_file)
    2386              :         {
    2387            0 :           fprintf (dump_file, "     Set with count ");
    2388            0 :           max.dump (dump_file);
    2389            0 :           fprintf (dump_file, " and used with count ");
    2390            0 :           bb->count.dump (dump_file);
    2391            0 :           fprintf (dump_file, " freq %f\n",
    2392            0 :                    max.to_sreal_scale (bb->count).to_double ());
    2393              :         }
    2394              : 
    2395       579814 :       BITMAP_FREE (info.bb_set);
    2396       579814 :       if (max < bb->count)
    2397        97070 :         return MAX ((max.to_sreal_scale (bb->count)
    2398              :                      * REG_BR_PROB_BASE).to_int (), 1);
    2399              :       return REG_BR_PROB_BASE;
    2400              :     }
    2401              : }
    2402              : 
    2403              : /* Find whether a basic block BB is the final block of a (half) diamond CFG
    2404              :    sub-graph and if the predicate the condition depends on is known.  If so,
    2405              :    return true and store the pointer the predicate in *P.  */
    2406              : 
    2407              : static bool
    2408      6516640 : phi_result_unknown_predicate (ipa_func_body_info *fbi,
    2409              :                               ipa_fn_summary *summary,
    2410              :                               class ipa_node_params *params_summary,
    2411              :                               basic_block bb,
    2412              :                               ipa_predicate *p,
    2413              :                               vec<ipa_predicate> nonconstant_names)
    2414              : {
    2415      6516640 :   edge e;
    2416      6516640 :   edge_iterator ei;
    2417      6516640 :   basic_block first_bb = NULL;
    2418              : 
    2419      6516640 :   if (single_pred_p (bb))
    2420              :     {
    2421        82650 :       *p = false;
    2422        82650 :       return true;
    2423              :     }
    2424              : 
    2425     14791857 :   FOR_EACH_EDGE (e, ei, bb->preds)
    2426              :     {
    2427     12670138 :       if (single_succ_p (e->src))
    2428              :         {
    2429      8486617 :           if (!single_pred_p (e->src))
    2430              :             return false;
    2431      7453240 :           if (!first_bb)
    2432      3278918 :             first_bb = single_pred (e->src);
    2433      4174322 :           else if (single_pred (e->src) != first_bb)
    2434              :             return false;
    2435              :         }
    2436              :       else
    2437              :         {
    2438      4183521 :           if (!first_bb)
    2439              :             first_bb = e->src;
    2440      1464771 :           else if (e->src != first_bb)
    2441              :             return false;
    2442              :         }
    2443              :     }
    2444              : 
    2445      2121719 :   if (!first_bb)
    2446              :     return false;
    2447              : 
    2448      4243438 :   gcond *stmt = safe_dyn_cast <gcond *> (*gsi_last_bb (first_bb));
    2449      2076085 :   if (!stmt
    2450      2076085 :       || !is_gimple_ip_invariant (gimple_cond_rhs (stmt)))
    2451              :     return false;
    2452              : 
    2453      1819311 :   *p = will_be_nonconstant_expr_predicate (fbi, summary, params_summary,
    2454              :                                            gimple_cond_lhs (stmt),
    2455              :                                            nonconstant_names);
    2456      1819311 :   if (*p == true)
    2457              :     return false;
    2458              :   else
    2459       601470 :     return true;
    2460              : }
    2461              : 
    2462              : /* Given a PHI statement in a function described by inline properties SUMMARY
    2463              :    and *P being the predicate describing whether the selected PHI argument is
    2464              :    known, store a predicate for the result of the PHI statement into
    2465              :    NONCONSTANT_NAMES, if possible.  */
    2466              : 
    2467              : static void
    2468       780422 : predicate_for_phi_result (class ipa_fn_summary *summary, gphi *phi,
    2469              :                           ipa_predicate *p,
    2470              :                           vec<ipa_predicate> nonconstant_names)
    2471              : {
    2472       780422 :   unsigned i;
    2473              : 
    2474      1102837 :   for (i = 0; i < gimple_phi_num_args (phi); i++)
    2475              :     {
    2476       942619 :       tree arg = gimple_phi_arg (phi, i)->def;
    2477       942619 :       if (!is_gimple_min_invariant (arg))
    2478              :         {
    2479       790419 :           gcc_assert (TREE_CODE (arg) == SSA_NAME);
    2480      1580838 :           *p = p->or_with (summary->conds,
    2481       790419 :                            nonconstant_names[SSA_NAME_VERSION (arg)]);
    2482       790419 :           if (*p == true)
    2483              :             return;
    2484              :         }
    2485              :     }
    2486              : 
    2487       160218 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2488              :     {
    2489            3 :       fprintf (dump_file, "\t\tphi predicate: ");
    2490            3 :       p->dump (dump_file, summary->conds);
    2491              :     }
    2492       160218 :   nonconstant_names[SSA_NAME_VERSION (gimple_phi_result (phi))] = *p;
    2493              : }
    2494              : 
    2495              : /* For a typical usage of __builtin_expect (a<b, 1), we
    2496              :    may introduce an extra relation stmt:
    2497              :    With the builtin, we have
    2498              :      t1 = a <= b;
    2499              :      t2 = (long int) t1;
    2500              :      t3 = __builtin_expect (t2, 1);
    2501              :      if (t3 != 0)
    2502              :        goto ...
    2503              :    Without the builtin, we have
    2504              :      if (a<=b)
    2505              :        goto...
    2506              :    This affects the size/time estimation and may have
    2507              :    an impact on the earlier inlining.
    2508              :    Here find this pattern and fix it up later.  */
    2509              : 
    2510              : static gimple *
    2511     42202673 : find_foldable_builtin_expect (basic_block bb)
    2512              : {
    2513     42202673 :   gimple_stmt_iterator bsi;
    2514              : 
    2515    296391931 :   for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
    2516              :     {
    2517    212171316 :       gimple *stmt = gsi_stmt (bsi);
    2518    212171316 :       if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
    2519    211935181 :           || gimple_call_builtin_p (stmt, BUILT_IN_EXPECT_WITH_PROBABILITY)
    2520    424106420 :           || gimple_call_internal_p (stmt, IFN_BUILTIN_EXPECT))
    2521              :         {
    2522       372052 :           tree var = gimple_call_lhs (stmt);
    2523       372052 :           tree arg = gimple_call_arg (stmt, 0);
    2524       372052 :           use_operand_p use_p;
    2525       372052 :           gimple *use_stmt;
    2526       372052 :           bool match = false;
    2527       372052 :           bool done = false;
    2528              : 
    2529       372052 :           if (!var || !arg)
    2530            3 :             continue;
    2531       372049 :           gcc_assert (TREE_CODE (var) == SSA_NAME);
    2532              : 
    2533       742715 :           while (TREE_CODE (arg) == SSA_NAME)
    2534              :             {
    2535       742715 :               gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
    2536       742715 :               if (!is_gimple_assign (stmt_tmp))
    2537              :                 break;
    2538       735364 :               switch (gimple_assign_rhs_code (stmt_tmp))
    2539              :                 {
    2540              :                   case LT_EXPR:
    2541              :                   case LE_EXPR:
    2542              :                   case GT_EXPR:
    2543              :                   case GE_EXPR:
    2544              :                   case EQ_EXPR:
    2545              :                   case NE_EXPR:
    2546              :                     match = true;
    2547              :                     done = true;
    2548              :                     break;
    2549              :                   CASE_CONVERT:
    2550              :                     break;
    2551              :                   default:
    2552              :                     done = true;
    2553              :                     break;
    2554              :                 }
    2555       370666 :               if (done)
    2556              :                 break;
    2557       370666 :               arg = gimple_assign_rhs1 (stmt_tmp);
    2558              :             }
    2559              : 
    2560       332083 :           if (match && single_imm_use (var, &use_p, &use_stmt)
    2561       703814 :               && gimple_code (use_stmt) == GIMPLE_COND)
    2562       184731 :             return use_stmt;
    2563              :         }
    2564              :     }
    2565              :   return NULL;
    2566              : }
    2567              : 
    2568              : /* Return true when the basic blocks contains only clobbers followed by RESX.
    2569              :    Such BBs are kept around to make removal of dead stores possible with
    2570              :    presence of EH and will be optimized out by optimize_clobbers later in the
    2571              :    game.
    2572              : 
    2573              :    NEED_EH is used to recurse in case the clobber has non-EH predecessors
    2574              :    that can be clobber only, too.. When it is false, the RESX is not necessary
    2575              :    on the end of basic block.  */
    2576              : 
    2577              : static bool
    2578     42856097 : clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
    2579              : {
    2580     42856097 :   gimple_stmt_iterator gsi = gsi_last_bb (bb);
    2581     42856097 :   edge_iterator ei;
    2582     42856097 :   edge e;
    2583              : 
    2584     42856097 :   if (need_eh)
    2585              :     {
    2586     42781790 :       if (gsi_end_p (gsi))
    2587              :         return false;
    2588     41626932 :       if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
    2589              :         return false;
    2590      1167325 :       gsi_prev (&gsi);
    2591              :     }
    2592        74307 :   else if (!single_succ_p (bb))
    2593              :     return false;
    2594              : 
    2595      3081412 :   for (; !gsi_end_p (gsi); gsi_prev (&gsi))
    2596              :     {
    2597      3000622 :       gimple *stmt = gsi_stmt (gsi);
    2598      3000622 :       if (is_gimple_debug (stmt))
    2599      1014289 :         continue;
    2600      1986333 :       if (gimple_clobber_p (stmt))
    2601       940830 :         continue;
    2602      1045503 :       if (gimple_code (stmt) == GIMPLE_LABEL)
    2603              :         break;
    2604              :       return false;
    2605              :     }
    2606              : 
    2607              :   /* See if all predecessors are either throws or clobber only BBs.  */
    2608      3142110 :   FOR_EACH_EDGE (e, ei, bb->preds)
    2609      2561491 :     if (!(e->flags & EDGE_EH)
    2610      2561491 :         && !clobber_only_eh_bb_p (e->src, false))
    2611              :       return false;
    2612              : 
    2613              :   return true;
    2614              : }
    2615              : 
    2616              : /* Return true if STMT compute a floating point expression that may be affected
    2617              :    by -ffast-math and similar flags.  */
    2618              : 
    2619              : static bool
    2620     96869056 : fp_expression_p (gimple *stmt)
    2621              : {
    2622     96869056 :   ssa_op_iter i;
    2623     96869056 :   tree op;
    2624              : 
    2625    217271714 :   FOR_EACH_SSA_TREE_OPERAND (op, stmt, i, SSA_OP_DEF|SSA_OP_USE)
    2626    121026959 :     if (FLOAT_TYPE_P (TREE_TYPE (op)))
    2627              :       return true;
    2628              :   return false;
    2629              : }
    2630              : 
    2631              : /* Return true if T references memory location that is local
    2632              :    for the function (that means, dead after return) or read-only.  */
    2633              : 
    2634              : bool
    2635     61262979 : refs_local_or_readonly_memory_p (tree t)
    2636              : {
    2637              :   /* Non-escaping memory is fine.  */
    2638     61262979 :   t = get_base_address (t);
    2639     61262979 :   if ((TREE_CODE (t) == MEM_REF
    2640     61262979 :       || TREE_CODE (t) == TARGET_MEM_REF))
    2641     25870491 :     return points_to_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
    2642              : 
    2643              :   /* Automatic variables are fine.  */
    2644     35392488 :   if (DECL_P (t)
    2645     35392488 :       && auto_var_in_fn_p (t, current_function_decl))
    2646              :     return true;
    2647              : 
    2648              :   /* Read-only variables are fine.  */
    2649     11484497 :   if (DECL_P (t) && TREE_READONLY (t))
    2650      1684997 :     return true;
    2651              : 
    2652              :   return false;
    2653              : }
    2654              : 
    2655              : /* Return true if T is a pointer pointing to memory location that is local
    2656              :    for the function (that means, dead after return) or read-only.  */
    2657              : 
    2658              : bool
    2659     74515856 : points_to_local_or_readonly_memory_p (tree t)
    2660              : {
    2661              :   /* See if memory location is clearly invalid.  */
    2662     74515856 :   if (integer_zerop (t))
    2663      2490287 :     return flag_delete_null_pointer_checks;
    2664     72025569 :   if (TREE_CODE (t) == SSA_NAME)
    2665              :     {
    2666              :       /* For IPA passes we can consinder accesses to return slot local
    2667              :          even if it is not local in the sense that memory is dead by
    2668              :          the end of founction.
    2669              :          The outer function will see a store in the call assignment
    2670              :          and thus this will do right thing for all uses of this
    2671              :          function in the current IPA passes (modref, pure/const discovery
    2672              :          and inlining heuristics).  */
    2673     45571767 :       if (DECL_RESULT (current_function_decl)
    2674     45571767 :           && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl))
    2675     46693071 :           && t == ssa_default_def (cfun, DECL_RESULT (current_function_decl)))
    2676              :         return true;
    2677     45246374 :       return !ptr_deref_may_alias_global_p (t, false);
    2678              :     }
    2679     26453802 :   if (TREE_CODE (t) == ADDR_EXPR
    2680     26453802 :       && (TREE_CODE (TREE_OPERAND (t, 0)) != TARGET_MEM_REF
    2681         4447 :           || TREE_CODE (TREE_OPERAND (TREE_OPERAND (t, 0), 0)) != INTEGER_CST))
    2682     13704139 :     return refs_local_or_readonly_memory_p (TREE_OPERAND (t, 0));
    2683              :   return false;
    2684              : }
    2685              : 
    2686              : /* Return true if T is a pointer pointing to memory location that is possible
    2687              :    sra candidate if all functions it is passed to are inlined.  */
    2688              : 
    2689              : static bool
    2690     39808244 : points_to_possible_sra_candidate_p (tree t)
    2691              : {
    2692     39808244 :   if (TREE_CODE (t) != ADDR_EXPR)
    2693              :     return false;
    2694              : 
    2695     11940558 :   t = get_base_address (TREE_OPERAND (t, 0));
    2696              : 
    2697              :   /* Automatic variables are fine.  */
    2698     11940558 :   if (DECL_P (t)
    2699     11940558 :       && auto_var_in_fn_p (t, current_function_decl))
    2700              :     return true;
    2701              :   return false;
    2702              : }
    2703              : 
    2704              : /* Return true if BB only calls builtin_unreachable.
    2705              :    We skip empty basic blocks, debug statements, clobbers and predicts.
    2706              :    CACHE is used to memoize already analyzed blocks.  */
    2707              : 
    2708              : static bool
    2709     28032769 : builtin_unreachable_bb_p (basic_block bb, vec<unsigned char> &cache)
    2710              : {
    2711     28032769 :   if (cache[bb->index])
    2712      2770482 :     return cache[bb->index] - 1;
    2713     25262287 :   gimple_stmt_iterator si;
    2714     25262287 :   auto_vec <basic_block, 4> visited_bbs;
    2715     25262287 :   bool ret = false;
    2716     26083621 :   while (true)
    2717              :     {
    2718     26083621 :       bool empty_bb = true;
    2719     26083621 :       visited_bbs.safe_push (bb);
    2720     26083621 :       cache[bb->index] = 3;
    2721     26083621 :       for (si = gsi_start_nondebug_bb (bb);
    2722     27603224 :            !gsi_end_p (si) && empty_bb;
    2723      1519603 :            gsi_next_nondebug (&si))
    2724              :         {
    2725     26221378 :           if (gimple_code (gsi_stmt (si)) != GIMPLE_PREDICT
    2726     25852564 :               && !gimple_clobber_p (gsi_stmt (si))
    2727     50956660 :               && !gimple_nop_p (gsi_stmt (si)))
    2728              :             {
    2729              :               empty_bb = false;
    2730              :               break;
    2731              :             }
    2732              :         }
    2733     26083621 :       if (!empty_bb)
    2734              :         break;
    2735              :       else
    2736      1381846 :         bb = single_succ_edge (bb)->dest;
    2737      1381846 :       if (cache[bb->index])
    2738              :         {
    2739       560512 :           ret = cache[bb->index] == 3 ? false : cache[bb->index] - 1;
    2740       560512 :           goto done;
    2741              :         }
    2742              :     }
    2743     24701775 :   if (gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE)
    2744     24701775 :       || gimple_call_builtin_p (gsi_stmt (si), BUILT_IN_UNREACHABLE_TRAP))
    2745              :     ret = true;
    2746     25262287 : done:
    2747    101870482 :   for (basic_block vbb:visited_bbs)
    2748     26083621 :     cache[vbb->index] = (unsigned char)ret + 1;
    2749     25262287 :   return ret;
    2750     25262287 : }
    2751              : 
    2752              : static bool
    2753     14141245 : guards_builtin_unreachable (basic_block bb, vec<unsigned char> &cache)
    2754              : {
    2755     14141245 :   edge_iterator ei;
    2756     14141245 :   edge e;
    2757     41909929 :   FOR_EACH_EDGE (e, ei, bb->succs)
    2758     28032769 :     if (builtin_unreachable_bb_p (e->dest, cache))
    2759              :       {
    2760       264085 :         if (dump_file && (dump_flags & TDF_DETAILS))
    2761            1 :           fprintf (dump_file,
    2762              :                    "BB %i ends with conditional guarding __builtin_unreachable;"
    2763              :                    " conditinal is unnecessary\n", bb->index);
    2764              :         return true;
    2765              :       }
    2766              :   return false;
    2767              : }
    2768              : 
    2769              : #define STMT_NECESSARY GF_PLF_1
    2770              : 
    2771              : /* If STMT is not already marked necessary, mark it, and add it to the
    2772              :    worklist if ADD_TO_WORKLIST is true.  */
    2773              : 
    2774              : static inline void
    2775    176976858 : mark_stmt_necessary (gimple *stmt, auto_vec<gimple *> &worklist)
    2776              : {
    2777    176976858 :   gcc_assert (stmt);
    2778              : 
    2779    176976858 :   if (gimple_plf (stmt, STMT_NECESSARY))
    2780              :     return;
    2781              : 
    2782    146926089 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2783              :     {
    2784          211 :       fprintf (dump_file, "Marking useful stmt: ");
    2785          211 :       print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
    2786          211 :       fprintf (dump_file, "\n");
    2787              :     }
    2788              : 
    2789    146926089 :   gimple_set_plf (stmt, STMT_NECESSARY, true);
    2790    146926089 :   worklist.safe_push (stmt);
    2791              : }
    2792              : 
    2793              : /* Mark the statement defining operand OP as necessary.  */
    2794              : 
    2795              : static inline void
    2796    124422530 : mark_operand_necessary (tree op, auto_vec<gimple *> &worklist)
    2797              : {
    2798    124422530 :   gimple *stmt = SSA_NAME_DEF_STMT (op);
    2799    124422530 :   if (gimple_nop_p (stmt))
    2800              :     return;
    2801    100878314 :   mark_stmt_necessary (stmt, worklist);
    2802              : }
    2803              : 
    2804              : /* Mark all statements that will remain in the body after optimizing out
    2805              :    conditionals guarding __builtin_unreachable which we keep to preserve
    2806              :    value ranges.  */
    2807              : 
    2808              : static void
    2809      7450714 : find_necessary_statements (struct cgraph_node *node)
    2810              : {
    2811      7450714 :   struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
    2812      7450714 :   auto_vec<unsigned char, 10> cache;
    2813      7450714 :   basic_block bb;
    2814      7450714 :   auto_vec<gimple *> worklist;
    2815              : 
    2816      7450714 :   cache.safe_grow_cleared (last_basic_block_for_fn (cfun));
    2817              :   /* Mark all obviously necessary statements.  */
    2818     50232504 :   FOR_EACH_BB_FN (bb, my_function)
    2819              :     {
    2820     42781790 :       for (gimple_stmt_iterator gsi = gsi_start_phis (bb);
    2821     54675539 :            !gsi_end_p (gsi); gsi_next (&gsi))
    2822     11893749 :         gimple_set_plf (gsi_stmt (gsi), STMT_NECESSARY, false);
    2823              : 
    2824    239829341 :       for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
    2825    154265761 :            gsi_next_nondebug (&bsi))
    2826              :         {
    2827    154265761 :           gimple *stmt = gsi_stmt (bsi);
    2828              : 
    2829    154265761 :           gimple_set_plf (stmt, STMT_NECESSARY, false);
    2830    154265761 :           if (gimple_has_side_effects (stmt)
    2831    126142079 :               || (is_ctrl_stmt (stmt)
    2832     22693948 :                   && (gimple_code (stmt) != GIMPLE_COND
    2833     14141245 :                       || !guards_builtin_unreachable (bb, cache)))
    2834    103712216 :               || gimple_store_p (stmt)
    2835    232454679 :               || gimple_code (stmt) == GIMPLE_ASM)
    2836     76098544 :             mark_stmt_necessary (stmt, worklist);
    2837              :         }
    2838              :     }
    2839    154376803 :   while (worklist.length () > 0)
    2840              :     {
    2841    146926089 :       gimple *stmt = worklist.pop ();
    2842              : 
    2843    146926089 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2844              :         {
    2845          211 :           fprintf (dump_file, "processing: ");
    2846          211 :           print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
    2847          211 :           fprintf (dump_file, "\n");
    2848              :         }
    2849    146926089 :       if (gimple_code (stmt) == GIMPLE_PHI)
    2850     18818596 :         for (unsigned int k = 0; k < gimple_phi_num_args (stmt); k++)
    2851              :           {
    2852     13317578 :             tree arg = PHI_ARG_DEF (stmt, k);
    2853              : 
    2854     13317578 :             if (TREE_CODE (arg) == SSA_NAME)
    2855     10197430 :               mark_operand_necessary (arg, worklist);
    2856              :           }
    2857              :       else
    2858              :         {
    2859    141425071 :           ssa_op_iter iter;
    2860    141425071 :           tree use;
    2861              : 
    2862    255650171 :           FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
    2863    114225100 :             mark_operand_necessary (use, worklist);
    2864              :         }
    2865              :     }
    2866      7450714 : }
    2867              : 
    2868              : /* Analyze function body for NODE.
    2869              :    EARLY indicates run from early optimization pipeline.  */
    2870              : 
    2871              : static void
    2872      7450714 : analyze_function_body (struct cgraph_node *node, bool early)
    2873              : {
    2874      7450714 :   sreal time = opt_for_fn (node->decl, param_uninlined_function_time);
    2875              :   /* Estimate static overhead for function prologue/epilogue and alignment. */
    2876      7450714 :   int size = opt_for_fn (node->decl, param_uninlined_function_insns);
    2877              :   /* Benefits are scaled by probability of elimination that is in range
    2878              :      <0,2>.  */
    2879      7450714 :   basic_block bb;
    2880      7450714 :   struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
    2881      7450714 :   sreal freq;
    2882      7450714 :   class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
    2883      7450714 :   ipa_node_params *params_summary
    2884      7450714 :     = early ? NULL : ipa_node_params_sum->get (node);
    2885      7450714 :   ipa_predicate bb_predicate;
    2886      7450714 :   struct ipa_func_body_info fbi;
    2887      7450714 :   vec<ipa_predicate> nonconstant_names = vNULL;
    2888      7450714 :   int nblocks, n;
    2889      7450714 :   int *order;
    2890      7450714 :   gimple *fix_builtin_expect_stmt;
    2891              : 
    2892      7450714 :   gcc_assert (my_function && my_function->cfg);
    2893      7450714 :   gcc_assert (cfun == my_function);
    2894              : 
    2895      7450714 :   memset(&fbi, 0, sizeof(fbi));
    2896      7450714 :   vec_free (info->conds);
    2897      7450714 :   info->conds = NULL;
    2898      7450714 :   info->size_time_table.release ();
    2899      7450714 :   info->call_size_time_table.release ();
    2900              : 
    2901              :   /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
    2902              :      so we can produce proper inline hints.
    2903              : 
    2904              :      When optimizing and analyzing for early inliner, initialize node params
    2905              :      so we can produce correct BB predicates.  */
    2906              : 
    2907      7450714 :   if (opt_for_fn (node->decl, optimize))
    2908              :     {
    2909      6517766 :       calculate_dominance_info (CDI_DOMINATORS);
    2910      6517766 :       calculate_dominance_info (CDI_POST_DOMINATORS);
    2911      6517766 :       if (!early)
    2912      1415887 :         loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
    2913              :       else
    2914              :         {
    2915      5101879 :           ipa_check_create_node_params ();
    2916      5101879 :           ipa_initialize_node_params (node);
    2917              :         }
    2918              : 
    2919      6517766 :       if (ipa_node_params_sum)
    2920              :         {
    2921      6517766 :           fbi.node = node;
    2922      6517766 :           fbi.info = ipa_node_params_sum->get (node);
    2923      6517766 :           fbi.bb_infos = vNULL;
    2924      6517766 :           fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
    2925      6517766 :           fbi.param_count = count_formal_params (node->decl);
    2926      6517766 :           fbi.aa_walk_budget = opt_for_fn (node->decl, param_ipa_max_aa_steps);
    2927              : 
    2928      6517766 :           nonconstant_names.safe_grow_cleared
    2929      6517766 :             (SSANAMES (my_function)->length (), true);
    2930              :         }
    2931              :     }
    2932              : 
    2933      7450714 :   if (dump_file)
    2934          249 :     fprintf (dump_file, "\nAnalyzing function body size: %s\n",
    2935              :              node->dump_name ());
    2936              : 
    2937              :   /* When we run into maximal number of entries, we assign everything to the
    2938              :      constant truth case.  Be sure to have it in list. */
    2939      7450714 :   bb_predicate = true;
    2940      7450714 :   info->account_size_time (0, 0, bb_predicate, bb_predicate);
    2941              : 
    2942      7450714 :   bb_predicate = ipa_predicate::not_inlined ();
    2943      7450714 :   info->account_size_time (opt_for_fn (node->decl,
    2944              :                                 param_uninlined_function_insns)
    2945              :                            * ipa_fn_summary::size_scale,
    2946      7450714 :                            opt_for_fn (node->decl,
    2947              :                                 param_uninlined_function_time),
    2948              :                            bb_predicate,
    2949              :                            bb_predicate);
    2950              : 
    2951              :   /* Only look for target information for inlinable functions.  */
    2952      7450714 :   bool scan_for_target_info =
    2953      7450714 :     info->inlinable
    2954     13399567 :     && targetm.target_option.need_ipa_fn_target_info (node->decl,
    2955      5948853 :                                                       info->target_info);
    2956              : 
    2957      7450714 :   if (fbi.info)
    2958      6517766 :     compute_bb_predicates (&fbi, node, info, params_summary);
    2959      7450714 :   find_necessary_statements (node);
    2960      7450714 :   const profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    2961      7450714 :   order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
    2962      7450714 :   nblocks = pre_and_rev_post_order_compute (NULL, order, false);
    2963     57683218 :   for (n = 0; n < nblocks; n++)
    2964              :     {
    2965     42781790 :       bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
    2966     42781790 :       freq = bb->count.to_sreal_scale (entry_count);
    2967     42781790 :       if (clobber_only_eh_bb_p (bb))
    2968              :         {
    2969       579117 :           if (dump_file && (dump_flags & TDF_DETAILS))
    2970            0 :             fprintf (dump_file, "\n Ignoring BB %i;"
    2971              :                      " it will be optimized away by cleanup_clobbers\n",
    2972              :                      bb->index);
    2973       579117 :           continue;
    2974              :         }
    2975              : 
    2976              :       /* TODO: Obviously predicates can be propagated down across CFG.  */
    2977     42202673 :       if (fbi.info)
    2978              :         {
    2979     35642723 :           if (bb->aux)
    2980     35642707 :             bb_predicate = *(ipa_predicate *)bb->aux;
    2981              :           else
    2982           16 :             bb_predicate = false;
    2983              :         }
    2984              :       else
    2985      6559950 :         bb_predicate = true;
    2986              : 
    2987     42202673 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2988              :         {
    2989           69 :           fprintf (dump_file, "\n BB %i predicate:", bb->index);
    2990           69 :           bb_predicate.dump (dump_file, info->conds);
    2991              :         }
    2992              : 
    2993     42202673 :       if (fbi.info && nonconstant_names.exists ())
    2994              :         {
    2995     35642723 :           ipa_predicate phi_predicate;
    2996     35642723 :           bool first_phi = true;
    2997              : 
    2998     36423145 :           for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
    2999       780422 :                gsi_next (&bsi))
    3000              :             {
    3001      6612942 :               if (first_phi
    3002      6612942 :                   && !phi_result_unknown_predicate (&fbi, info,
    3003              :                                                     params_summary,
    3004              :                                                     bb,
    3005              :                                                     &phi_predicate,
    3006              :                                                     nonconstant_names))
    3007              :                 break;
    3008       780422 :               first_phi = false;
    3009       780422 :               if (dump_file && (dump_flags & TDF_DETAILS))
    3010              :                 {
    3011            3 :                   fprintf (dump_file, "  ");
    3012            3 :                   print_gimple_stmt (dump_file, gsi_stmt (bsi), 0);
    3013              :                 }
    3014       780422 :               predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
    3015              :                                         nonconstant_names);
    3016              :             }
    3017              :         }
    3018              : 
    3019     42202673 :       fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
    3020              : 
    3021     42202673 :       for (gimple_stmt_iterator bsi = gsi_start_nondebug_bb (bb);
    3022    188032986 :            !gsi_end_p (bsi); gsi_next_nondebug (&bsi))
    3023              :         {
    3024    145830313 :           gimple *stmt = gsi_stmt (bsi);
    3025    145830313 :           if (!gimple_plf (stmt, STMT_NECESSARY))
    3026              :             {
    3027      5772472 :               if (dump_file && (dump_flags & TDF_DETAILS))
    3028              :                 {
    3029           19 :                   fprintf (dump_file, "  skipping unnecessary stmt ");
    3030           19 :                   print_gimple_stmt (dump_file, stmt, 0);
    3031              :                 }
    3032              :               /* TODO: const calls used only to produce values for
    3033              :                  builtion_unreachable guards should not be accounted.  However
    3034              :                  we still want to inline them and this does does not work well
    3035              :                  with the cost model.  For now account them as usual.  */
    3036      5772472 :               if (!is_gimple_call (stmt)
    3037      5772472 :                   || gimple_call_internal_p (stmt))
    3038      5492092 :                 continue;
    3039              :             }
    3040    140338221 :           int this_size = estimate_num_insns (stmt, &eni_size_weights);
    3041    140338221 :           int this_time = estimate_num_insns (stmt, &eni_time_weights);
    3042    140338221 :           int prob;
    3043    140338221 :           ipa_predicate will_be_nonconstant;
    3044              : 
    3045              :           /* This relation stmt should be folded after we remove
    3046              :              __builtin_expect call. Adjust the cost here.  */
    3047    140338221 :           if (stmt == fix_builtin_expect_stmt)
    3048              :             {
    3049       184293 :               this_size--;
    3050       184293 :               this_time--;
    3051              :             }
    3052              : 
    3053    140338221 :           if (dump_file && (dump_flags & TDF_DETAILS))
    3054              :             {
    3055          205 :               fprintf (dump_file, "  ");
    3056          205 :               print_gimple_stmt (dump_file, stmt, 0);
    3057          205 :               fprintf (dump_file, "\t\tfreq:%3.2f size:%3i time:%3i\n",
    3058              :                        freq.to_double (), this_size,
    3059              :                        this_time);
    3060              :             }
    3061              : 
    3062    140338221 :           struct cgraph_edge *edge;
    3063    140338221 :           if (is_gimple_call (stmt)
    3064     24756881 :               && (!gimple_call_internal_p (stmt)
    3065      1149925 :                   || gimple_call_internal_fn (stmt) == IFN_ASSUME)
    3066    163945693 :               && (edge = node->get_edge (stmt)))
    3067              :             {
    3068     23607131 :               ipa_call_summary *es = ipa_call_summaries->get_create (edge);
    3069              : 
    3070              :               /* Special case: results of BUILT_IN_CONSTANT_P will be always
    3071              :                  resolved as constant.  We however don't want to optimize
    3072              :                  out the cgraph edges.  */
    3073     23607131 :               if (nonconstant_names.exists ()
    3074     20650988 :                   && gimple_call_builtin_p (stmt, BUILT_IN_CONSTANT_P)
    3075        48326 :                   && gimple_call_lhs (stmt)
    3076     23655457 :                   && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
    3077              :                 {
    3078        48326 :                   ipa_predicate false_p = false;
    3079        48326 :                   nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
    3080        48326 :                     = false_p;
    3081              :                 }
    3082     23607131 :               if (ipa_node_params_sum)
    3083              :                 {
    3084     20663821 :                   int count = gimple_call_num_args (stmt);
    3085     20663821 :                   int i;
    3086              : 
    3087     20663821 :                   if (count)
    3088     17757936 :                     es->param.safe_grow_cleared (count, true);
    3089     60472065 :                   for (i = 0; i < count; i++)
    3090              :                     {
    3091     39808244 :                       int prob = param_change_prob (&fbi, stmt, i);
    3092     39808244 :                       gcc_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
    3093     39808244 :                       es->param[i].change_prob = prob;
    3094     39808244 :                       es->param[i].points_to_local_or_readonly_memory
    3095     39808244 :                          = points_to_local_or_readonly_memory_p
    3096     39808244 :                              (gimple_call_arg (stmt, i));
    3097     39808244 :                       es->param[i].points_to_possible_sra_candidate
    3098     39808244 :                          = points_to_possible_sra_candidate_p
    3099     39808244 :                              (gimple_call_arg (stmt, i));
    3100              :                     }
    3101              :                 }
    3102              :               /* We cannot setup VLA parameters during inlining.  */
    3103     69395857 :               for (unsigned int i = 0; i < gimple_call_num_args (stmt); ++i)
    3104     45789122 :                 if (TREE_CODE (gimple_call_arg (stmt, i)) == WITH_SIZE_EXPR)
    3105              :                   {
    3106          396 :                     edge->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
    3107          396 :                     break;
    3108              :                   }
    3109     23607131 :               es->call_stmt_size = this_size;
    3110     23607131 :               es->call_stmt_time = this_time;
    3111     23607131 :               es->loop_depth = bb_loop_depth (bb);
    3112     23607131 :               edge_set_predicate (edge, &bb_predicate);
    3113     23607131 :               if (edge->speculative)
    3114              :                 {
    3115            0 :                   cgraph_edge *indirect
    3116            0 :                         = edge->speculative_call_indirect_edge ();
    3117            0 :                   ipa_call_summary *es2
    3118            0 :                          = ipa_call_summaries->get_create (indirect);
    3119            0 :                   ipa_call_summaries->duplicate (edge, indirect,
    3120              :                                                  es, es2);
    3121              : 
    3122              :                   /* Edge is the first direct call.
    3123              :                      create and duplicate call summaries for multiple
    3124              :                      speculative call targets.  */
    3125            0 :                   for (cgraph_edge *direct
    3126            0 :                          = edge->next_speculative_call_target ();
    3127            0 :                        direct;
    3128            0 :                        direct = direct->next_speculative_call_target ())
    3129              :                     {
    3130            0 :                       ipa_call_summary *es3
    3131            0 :                         = ipa_call_summaries->get_create (direct);
    3132            0 :                       ipa_call_summaries->duplicate (edge, direct,
    3133              :                                                      es, es3);
    3134              :                     }
    3135              :                 }
    3136              : 
    3137              :               /* If dealing with a carrying edge, copy its summary over to its
    3138              :                  attached edges as well.  */
    3139     23607131 :               if (edge->has_callback)
    3140              :                 {
    3141        15202 :                   cgraph_edge *cbe;
    3142        30405 :                   for (cbe = edge->first_callback_edge (); cbe;
    3143        15203 :                        cbe = cbe->next_callback_edge ())
    3144              :                     {
    3145        15203 :                       ipa_call_summary *es2 = ipa_call_summaries->get_create (cbe);
    3146        15203 :                       ipa_call_summaries->duplicate (edge, cbe, es, es2);
    3147              :                       /* Unlike speculative edges, callback edges have no real
    3148              :                          size or time; the call doesn't exist.  Reflect that in
    3149              :                          their summaries.  */
    3150        15203 :                       es2->call_stmt_size = 0;
    3151        15203 :                       es2->call_stmt_time = 0;
    3152              :                     }
    3153              :                 }
    3154              :               /* Internal calls are not in the callgraph, so edges pointing to
    3155              :                  .ASSUME bodies are not covered by the above branch, as the
    3156              :                  dispatching edge doesn't exist.  */
    3157     23591929 :               else if (edge->callback)
    3158              :                 {
    3159          175 :                   gcc_checking_assert (edge->callee->assume_function);
    3160          175 :                   es->call_stmt_size = 0;
    3161          175 :                   es->call_stmt_time = 0;
    3162              :                 }
    3163              :             }
    3164              : 
    3165              :           /* TODO: When conditional jump or switch is known to be constant, but
    3166              :              we did not translate it into the predicates, we really can account
    3167              :              just maximum of the possible paths.  */
    3168    140338221 :           if (fbi.info)
    3169    117712834 :             will_be_nonconstant
    3170    117712834 :               = will_be_nonconstant_predicate (&fbi, info, params_summary,
    3171              :                                                stmt, nonconstant_names);
    3172              :           else
    3173     22625387 :             will_be_nonconstant = true;
    3174    140338221 :           if (this_time || this_size)
    3175              :             {
    3176    114735321 :               sreal final_time = (sreal)this_time * freq;
    3177    114735321 :               prob = eliminated_by_inlining_prob (&fbi, stmt);
    3178    114735321 :               if (prob == 1 && dump_file && (dump_flags & TDF_DETAILS))
    3179           11 :                 fprintf (dump_file,
    3180              :                          "\t\t50%% will be eliminated by inlining\n");
    3181    114735321 :               if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
    3182           20 :                 fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
    3183              : 
    3184    114735321 :               ipa_predicate p = bb_predicate & will_be_nonconstant;
    3185    114735321 :               int parm = load_or_store_of_ptr_parameter (&fbi, stmt);
    3186    114735321 :               ipa_predicate sra_predicate = true;
    3187    114735321 :               if (parm != -1)
    3188     14598576 :                 sra_predicate &= add_condition (info, params_summary, parm,
    3189              :                                                 ptr_type_node, NULL,
    3190      7299288 :                                                 ipa_predicate::not_sra_candidate, NULL, 0);
    3191              : 
    3192              :               /* We can ignore statement when we proved it is never going
    3193              :                  to happen, but we cannot do that for call statements
    3194              :                  because edges are accounted specially.  */
    3195              : 
    3196    229470642 :               if (*(is_gimple_call (stmt) ? &bb_predicate : &p) != false)
    3197              :                 {
    3198    113967722 :                   time += final_time;
    3199    113967722 :                   size += this_size;
    3200              :                 }
    3201              : 
    3202              :               /* We account everything but the calls.  Calls have their own
    3203              :                  size/time info attached to cgraph edges.  This is necessary
    3204              :                  in order to make the cost disappear after inlining.  */
    3205    114735321 :               if (!is_gimple_call (stmt))
    3206              :                 {
    3207     91854922 :                   if (prob)
    3208              :                     {
    3209     15327071 :                       ipa_predicate ip
    3210     15327071 :                         = bb_predicate & ipa_predicate::not_inlined () & sra_predicate;
    3211     30654142 :                       info->account_size_time (this_size * prob,
    3212     15327071 :                                                (final_time * prob) / 2, ip,
    3213              :                                                p);
    3214              :                     }
    3215     15327071 :                   if (prob != 2)
    3216    169515352 :                     info->account_size_time (this_size * (2 - prob),
    3217     84757676 :                                              (final_time * (2 - prob) / 2),
    3218    169515352 :                                              bb_predicate & sra_predicate,
    3219              :                                              p);
    3220              :                 }
    3221              : 
    3222    114735321 :               if (!info->fp_expressions && fp_expression_p (stmt))
    3223              :                 {
    3224       624301 :                   info->fp_expressions = true;
    3225       624301 :                   if (dump_file)
    3226            9 :                     fprintf (dump_file, "   fp_expression set\n");
    3227              :                 }
    3228              :             }
    3229              : 
    3230              :           /* For target specific information, we want to scan all statements
    3231              :              rather than those statements with non-zero weights, to avoid
    3232              :              missing to scan something interesting for target information,
    3233              :              such as: internal function calls.  */
    3234    140338221 :           if (scan_for_target_info)
    3235            0 :             scan_for_target_info =
    3236            0 :               targetm.target_option.update_ipa_fn_target_info
    3237            0 :               (info->target_info, stmt);
    3238              : 
    3239              :           /* Account cost of address calculations in the statements.  */
    3240    542692720 :           for (unsigned int i = 0; i < gimple_num_ops (stmt); i++)
    3241              :             {
    3242    402354499 :               for (tree op = gimple_op (stmt, i);
    3243    774927880 :                    op && handled_component_p (op);
    3244     48818043 :                    op = TREE_OPERAND (op, 0))
    3245     48818043 :                 if ((TREE_CODE (op) == ARRAY_REF
    3246     48818043 :                      || TREE_CODE (op) == ARRAY_RANGE_REF)
    3247     48818043 :                     && TREE_CODE (TREE_OPERAND (op, 1)) == SSA_NAME)
    3248              :                   {
    3249      2440173 :                     ipa_predicate p = bb_predicate;
    3250      2440173 :                     if (fbi.info)
    3251      1911815 :                       p = p & will_be_nonconstant_expr_predicate
    3252      1911815 :                                  (&fbi, info, params_summary,
    3253      1911815 :                                   TREE_OPERAND (op, 1),
    3254      1911815 :                                   nonconstant_names);
    3255      2440173 :                     if (p != false)
    3256              :                       {
    3257      2434748 :                         time += freq;
    3258      2434748 :                         size += 1;
    3259      2434748 :                         if (dump_file)
    3260           24 :                           fprintf (dump_file,
    3261              :                                    "\t\tAccounting address calculation.\n");
    3262      2434748 :                         info->account_size_time (ipa_fn_summary::size_scale,
    3263              :                                                  freq,
    3264              :                                                  bb_predicate,
    3265              :                                                  p);
    3266              :                       }
    3267              :                   }
    3268              :             }
    3269              : 
    3270              :         }
    3271              :     }
    3272      7450714 :   free (order);
    3273              : 
    3274      7450714 :   if (nonconstant_names.exists () && !early)
    3275              :     {
    3276      1415887 :       ipa_fn_summary *s = ipa_fn_summaries->get (node);
    3277      1415887 :       unsigned max_loop_predicates = opt_for_fn (node->decl,
    3278              :                                                  param_ipa_max_loop_predicates);
    3279              : 
    3280      1415887 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3281           16 :         flow_loops_dump (dump_file, NULL, 0);
    3282      1415887 :       scev_initialize ();
    3283      4888219 :       for (auto loop : loops_list (cfun, 0))
    3284              :         {
    3285       640558 :           ipa_predicate loop_iterations = true;
    3286       640558 :           sreal header_freq;
    3287       640558 :           edge ex;
    3288       640558 :           unsigned int j;
    3289       640558 :           class tree_niter_desc niter_desc;
    3290       640558 :           if (!loop->header->aux)
    3291            0 :             continue;
    3292              : 
    3293       640558 :           profile_count hdr_count = loop->header->count;
    3294       640558 :           sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
    3295              : 
    3296       640558 :           bb_predicate = *(ipa_predicate *)loop->header->aux;
    3297       640558 :           auto_vec<edge> exits = get_loop_exit_edges (loop);
    3298      2320536 :           FOR_EACH_VEC_ELT (exits, j, ex)
    3299      1039420 :             if (number_of_iterations_exit (loop, ex, &niter_desc, false)
    3300      1039420 :                 && !is_gimple_min_invariant (niter_desc.niter))
    3301              :             {
    3302       157712 :               ipa_predicate will_be_nonconstant
    3303       157712 :                 = will_be_nonconstant_expr_predicate (&fbi, info,
    3304              :                                                       params_summary,
    3305              :                                                       niter_desc.niter,
    3306              :                                                       nonconstant_names);
    3307       157712 :               if (will_be_nonconstant != true)
    3308        63739 :                 will_be_nonconstant = bb_predicate & will_be_nonconstant;
    3309       157712 :               if (will_be_nonconstant != true
    3310       221451 :                   && will_be_nonconstant != false)
    3311        63185 :                 loop_iterations &= will_be_nonconstant;
    3312              :             }
    3313       640558 :           add_freqcounting_predicate (&s->loop_iterations, loop_iterations,
    3314              :                                       hdr_freq, max_loop_predicates);
    3315       640560 :         }
    3316              : 
    3317              :       /* To avoid quadratic behavior we analyze stride predicates only
    3318              :          with respect to the containing loop.  Thus we simply iterate
    3319              :          over all defs in the outermost loop body.  */
    3320      1415887 :       for (class loop *loop = loops_for_fn (cfun)->tree_root->inner;
    3321      1920561 :            loop != NULL; loop = loop->next)
    3322              :         {
    3323       504674 :           ipa_predicate loop_stride = true;
    3324       504674 :           basic_block *body = get_loop_body (loop);
    3325       504674 :           profile_count hdr_count = loop->header->count;
    3326       504674 :           sreal hdr_freq = hdr_count.to_sreal_scale (entry_count);
    3327      3472618 :           for (unsigned i = 0; i < loop->num_nodes; i++)
    3328              :             {
    3329      2463270 :               gimple_stmt_iterator gsi;
    3330      2463270 :               if (!body[i]->aux)
    3331            8 :                 continue;
    3332              : 
    3333      2463262 :               bb_predicate = *(ipa_predicate *)body[i]->aux;
    3334     16710533 :               for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
    3335     11784009 :                    gsi_next (&gsi))
    3336              :                 {
    3337     11784009 :                   gimple *stmt = gsi_stmt (gsi);
    3338              : 
    3339     11784009 :                   if (!is_gimple_assign (stmt))
    3340     11631334 :                     continue;
    3341              : 
    3342      5609597 :                   tree def = gimple_assign_lhs (stmt);
    3343      5609597 :                   if (TREE_CODE (def) != SSA_NAME)
    3344      1109027 :                     continue;
    3345              : 
    3346      4500570 :                   affine_iv iv;
    3347      9001140 :                   if (!simple_iv (loop_containing_stmt (stmt),
    3348              :                                   loop_containing_stmt (stmt),
    3349              :                                   def, &iv, true)
    3350      4500570 :                       || is_gimple_min_invariant (iv.step))
    3351      4347895 :                     continue;
    3352              : 
    3353       152675 :                   ipa_predicate will_be_nonconstant
    3354       152675 :                     = will_be_nonconstant_expr_predicate (&fbi, info,
    3355              :                                                           params_summary,
    3356              :                                                           iv.step,
    3357              :                                                           nonconstant_names);
    3358       152675 :                   if (will_be_nonconstant != true)
    3359        56512 :                     will_be_nonconstant = bb_predicate & will_be_nonconstant;
    3360       152675 :                   if (will_be_nonconstant != true
    3361       209187 :                       && will_be_nonconstant != false)
    3362        46914 :                     loop_stride = loop_stride & will_be_nonconstant;
    3363              :                 }
    3364              :             }
    3365       504674 :           add_freqcounting_predicate (&s->loop_strides, loop_stride,
    3366              :                                       hdr_freq, max_loop_predicates);
    3367       504674 :           free (body);
    3368              :         }
    3369      1415887 :       scev_finalize ();
    3370              :     }
    3371     65133932 :   FOR_ALL_BB_FN (bb, my_function)
    3372              :     {
    3373     57683218 :       edge e;
    3374     57683218 :       edge_iterator ei;
    3375              : 
    3376     57683218 :       if (bb->aux)
    3377     42673561 :         edge_predicate_pool.remove ((ipa_predicate *)bb->aux);
    3378     57683218 :       bb->aux = NULL;
    3379    122704966 :       FOR_EACH_EDGE (e, ei, bb->succs)
    3380              :         {
    3381     65021748 :           if (e->aux)
    3382      2580338 :             edge_predicate_pool.remove ((ipa_predicate *)e->aux);
    3383     65021748 :           e->aux = NULL;
    3384              :         }
    3385              :     }
    3386      7450714 :   ipa_fn_summary *s = ipa_fn_summaries->get (node);
    3387      7450714 :   ipa_size_summary *ss = ipa_size_summaries->get (node);
    3388      7450714 :   s->time = time;
    3389      7450714 :   ss->self_size = size;
    3390      7450714 :   nonconstant_names.release ();
    3391      7450714 :   ipa_release_body_info (&fbi);
    3392      7450714 :   if (opt_for_fn (node->decl, optimize))
    3393              :     {
    3394      6517766 :       if (!early)
    3395      1415887 :         loop_optimizer_finalize ();
    3396      5101879 :       else if (!ipa_edge_args_sum)
    3397      5101865 :         ipa_free_all_node_params ();
    3398      6517766 :       free_dominance_info (CDI_DOMINATORS);
    3399      6517766 :       free_dominance_info (CDI_POST_DOMINATORS);
    3400              :     }
    3401      7450714 :   if (dump_file)
    3402              :     {
    3403          249 :       fprintf (dump_file, "\n");
    3404          249 :       ipa_dump_fn_summary (dump_file, node);
    3405              :     }
    3406      7450714 : }
    3407              : 
    3408              : 
    3409              : /* Compute function summary.
    3410              :    EARLY is true when we compute parameters during early opts.  */
    3411              : 
    3412              : void
    3413      7452066 : compute_fn_summary (struct cgraph_node *node, bool early)
    3414              : {
    3415      7452066 :   HOST_WIDE_INT self_stack_size;
    3416      7452066 :   struct cgraph_edge *e;
    3417              : 
    3418      7452066 :   gcc_assert (!node->inlined_to);
    3419              : 
    3420      7452066 :   if (!ipa_fn_summaries)
    3421       220583 :     ipa_fn_summary_alloc ();
    3422              : 
    3423              :   /* Create a new ipa_fn_summary.  */
    3424      7452066 :   ((ipa_fn_summary_t *)ipa_fn_summaries)->remove_callees (node);
    3425      7452066 :   ipa_fn_summaries->remove (node);
    3426      7452066 :   class ipa_fn_summary *info = ipa_fn_summaries->get_create (node);
    3427      7452066 :   class ipa_size_summary *size_info = ipa_size_summaries->get_create (node);
    3428              : 
    3429              :   /* Estimate the stack size for the function if we're optimizing.  */
    3430     13036868 :   self_stack_size = optimize && !node->thunk
    3431     13969832 :                     ? estimated_stack_frame_size (node) : 0;
    3432      7452066 :   size_info->estimated_self_stack_size = self_stack_size;
    3433      7452066 :   info->estimated_stack_size = self_stack_size;
    3434              : 
    3435      7452066 :   if (node->thunk)
    3436              :     {
    3437         1352 :       ipa_call_summary *es = ipa_call_summaries->get_create (node->callees);
    3438         1352 :       ipa_predicate t = true;
    3439              : 
    3440         1352 :       node->can_change_signature = false;
    3441         1352 :       es->call_stmt_size = eni_size_weights.call_cost;
    3442         1352 :       es->call_stmt_time = eni_time_weights.call_cost;
    3443         4056 :       info->account_size_time (ipa_fn_summary::size_scale
    3444         1352 :                                * opt_for_fn (node->decl,
    3445              :                                  param_uninlined_function_thunk_insns),
    3446         1352 :                                opt_for_fn (node->decl,
    3447              :                                  param_uninlined_function_thunk_time), t, t);
    3448         1352 :       t = ipa_predicate::not_inlined ();
    3449         1352 :       info->account_size_time (2 * ipa_fn_summary::size_scale, 0, t, t);
    3450         1352 :       ipa_update_overall_fn_summary (node);
    3451         1352 :       size_info->self_size = size_info->size;
    3452         1352 :       if (stdarg_p (TREE_TYPE (node->decl)))
    3453              :         {
    3454            9 :           info->inlinable = false;
    3455            9 :           node->callees->inline_failed = CIF_VARIADIC_THUNK;
    3456              :         }
    3457              :       else
    3458         1343 :         info->inlinable = true;
    3459              :     }
    3460              :   else
    3461              :     {
    3462              :        /* Even is_gimple_min_invariant rely on current_function_decl.  */
    3463      7450714 :        push_cfun (DECL_STRUCT_FUNCTION (node->decl));
    3464              : 
    3465              :        /* During IPA profile merging we may be called w/o virtual SSA form
    3466              :           built.  */
    3467      7450714 :        update_ssa (TODO_update_ssa_only_virtuals);
    3468              : 
    3469              :        /* Can this function be inlined at all?  */
    3470      7450714 :        if (!opt_for_fn (node->decl, optimize)
    3471      8383662 :            && !lookup_attribute ("always_inline",
    3472       932948 :                                  DECL_ATTRIBUTES (node->decl)))
    3473       861830 :          info->inlinable = false;
    3474              :        else
    3475      6588884 :          info->inlinable = tree_inlinable_function_p (node->decl);
    3476              : 
    3477      7450714 :        bool no_signature = false;
    3478              : 
    3479              :        /* Don't allow signature changes for functions which have
    3480              :           [[gnu::musttail]] or [[clang::musttail]] calls.  Sometimes
    3481              :           (more often on targets which pass everything on the stack)
    3482              :           signature changes can result in tail calls being impossible
    3483              :           even when without the signature changes they would be ok.
    3484              :           See PR121023.  */
    3485      7450714 :        if (cfun->has_musttail)
    3486              :          {
    3487         1441 :            if (dump_file)
    3488            0 :             fprintf (dump_file, "No signature change:"
    3489              :                      " function has calls with musttail attribute.\n");
    3490              :            no_signature = true;
    3491              :          }
    3492              : 
    3493              :        /* Type attributes can use parameter indices to describe them.
    3494              :           Special case fn spec since we can safely preserve them in
    3495              :           modref summaries.  */
    3496      7450714 :        for (tree list = TYPE_ATTRIBUTES (TREE_TYPE (node->decl));
    3497      7855136 :             list && !no_signature; list = TREE_CHAIN (list))
    3498       404422 :         if (!ipa_param_adjustments::type_attribute_allowed_p
    3499       404422 :                         (get_attribute_name (list)))
    3500              :            {
    3501       159430 :              if (dump_file)
    3502              :                 {
    3503            0 :                   fprintf (dump_file, "No signature change:"
    3504              :                            " function type has unhandled attribute %s.\n",
    3505            0 :                            IDENTIFIER_POINTER (get_attribute_name (list)));
    3506              :                 }
    3507              :              no_signature = true;
    3508              :            }
    3509      7450714 :        for (tree parm = DECL_ARGUMENTS (node->decl);
    3510     22216768 :             parm && !no_signature; parm = DECL_CHAIN (parm))
    3511     14766054 :          if (variably_modified_type_p (TREE_TYPE (parm), node->decl))
    3512              :            {
    3513        16036 :              if (dump_file)
    3514              :                 {
    3515            0 :                   fprintf (dump_file, "No signature change:"
    3516              :                            " has parameter with variably modified type.\n");
    3517              :                 }
    3518              :              no_signature = true;
    3519              :            }
    3520              : 
    3521              :        /* Likewise for #pragma omp declare simd functions or functions
    3522              :           with simd attribute.  */
    3523      7450714 :        if (no_signature
    3524     14724521 :            || lookup_attribute ("omp declare simd",
    3525      7273807 :                                 DECL_ATTRIBUTES (node->decl)))
    3526       178622 :          node->can_change_signature = false;
    3527              :        else
    3528              :          {
    3529              :            /* Otherwise, inlinable functions always can change signature.  */
    3530      7272092 :            if (info->inlinable)
    3531      5868974 :              node->can_change_signature = true;
    3532              :            else
    3533              :              {
    3534              :                /* Functions calling builtin_apply cannot change signature.  */
    3535      5428658 :                for (e = node->callees; e; e = e->next_callee)
    3536              :                  {
    3537      4058325 :                    tree cdecl = e->callee->decl;
    3538      4058325 :                    if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS,
    3539              :                                                  BUILT_IN_VA_START))
    3540              :                      break;
    3541              :                  }
    3542      1403118 :                node->can_change_signature = !e;
    3543              :              }
    3544              :          }
    3545      7450714 :        analyze_function_body (node, early);
    3546      7450714 :        pop_cfun ();
    3547              :      }
    3548              : 
    3549              :   /* Inlining characteristics are maintained by the cgraph_mark_inline.  */
    3550      7452066 :   size_info->size = size_info->self_size;
    3551      7452066 :   info->estimated_stack_size = size_info->estimated_self_stack_size;
    3552              : 
    3553              :   /* Code above should compute exactly the same result as
    3554              :      ipa_update_overall_fn_summary except for case when speculative
    3555              :      edges are present since these are accounted to size but not
    3556              :      self_size. Do not compare time since different order the roundoff
    3557              :      errors result in slight changes.  */
    3558      7452066 :   ipa_update_overall_fn_summary (node);
    3559      7452066 :   if (flag_checking)
    3560              :     {
    3561      7938311 :       for (e = node->indirect_calls; e; e = e->next_callee)
    3562       486327 :        if (e->speculative)
    3563              :          break;
    3564      7451984 :       gcc_assert (e || size_info->size == size_info->self_size);
    3565              :     }
    3566      7452066 : }
    3567              : 
    3568              : 
    3569              : /* Compute parameters of functions used by inliner using
    3570              :    current_function_decl.  */
    3571              : 
    3572              : static unsigned int
    3573      5972550 : compute_fn_summary_for_current (void)
    3574              : {
    3575      5972550 :   compute_fn_summary (cgraph_node::get (current_function_decl), true);
    3576      5972550 :   return 0;
    3577              : }
    3578              : 
    3579              : /* Estimate benefit devirtualizing indirect edge IE and return true if it can
    3580              :    be devirtualized and inlined, provided m_known_vals, m_known_contexts and
    3581              :    m_known_aggs in AVALS.  Return false straight away if AVALS is NULL.  */
    3582              : 
    3583              : static bool
    3584      4937894 : estimate_edge_devirt_benefit (struct cgraph_edge *ie,
    3585              :                               int *size, int *time,
    3586              :                               ipa_call_arg_values *avals)
    3587              : {
    3588      4937894 :   tree target;
    3589      4937894 :   struct cgraph_node *callee;
    3590      4937894 :   class ipa_fn_summary *isummary;
    3591      4937894 :   enum availability avail;
    3592      4937894 :   bool speculative;
    3593              : 
    3594      4937894 :   if (!avals
    3595      4937894 :       || (!avals->m_known_vals.length() && !avals->m_known_contexts.length ()))
    3596              :     return false;
    3597      1506826 :   if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
    3598              :     return false;
    3599              : 
    3600      1500279 :   target = ipa_get_indirect_edge_target
    3601      1500279 :     (ie->callee ? ie->speculative_call_indirect_edge () : ie,
    3602              :      avals, &speculative);
    3603      1500279 :   if (!target || speculative)
    3604              :     return false;
    3605              : 
    3606              :   /* If this is speculative call, turn its cost into 0; we will account
    3607              :      the call when processing the indirect call.  */
    3608       219653 :   if (ie->callee)
    3609              :     {
    3610         4549 :       gcc_checking_assert (ie->speculative && *size > 0);
    3611         4549 :       *size = 0;
    3612         4549 :       *time = 0;
    3613              :     }
    3614              :   else
    3615              :     {
    3616              :       /* Account for difference in cost between indirect and direct calls.  */
    3617       215104 :       *size -= (eni_size_weights.indirect_call_cost - eni_size_weights.call_cost);
    3618       215104 :       *time -= (eni_time_weights.indirect_call_cost - eni_time_weights.call_cost);
    3619              :     }
    3620       219653 :   gcc_checking_assert (*time >= 0);
    3621       219653 :   gcc_checking_assert (*size >= 0);
    3622              : 
    3623       219653 :   callee = cgraph_node::get (target);
    3624       219653 :   if (!callee || !callee->definition)
    3625              :     return false;
    3626       198961 :   callee = callee->function_symbol (&avail);
    3627       198961 :   if (avail < AVAIL_AVAILABLE)
    3628              :     return false;
    3629       198911 :   isummary = ipa_fn_summaries->get (callee);
    3630       198911 :   if (isummary == NULL)
    3631              :     return false;
    3632              : 
    3633       198897 :   return isummary->inlinable;
    3634              : }
    3635              : 
    3636              : /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
    3637              :    handle edge E with probability PROB.  Set HINTS accordingly if edge may be
    3638              :    devirtualized.  AVALS, if non-NULL, describes the context of the call site
    3639              :    as far as values of parameters are concerned.  */
    3640              : 
    3641              : static inline void
    3642    208139933 : estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
    3643              :                              sreal *time, ipa_call_arg_values *avals,
    3644              :                              ipa_hints *hints)
    3645              : {
    3646    208139933 :   class ipa_call_summary *es = ipa_call_summaries->get (e);
    3647    208139933 :   int call_size = es->call_stmt_size;
    3648    208139933 :   int call_time = es->call_stmt_time;
    3649    208139933 :   int cur_size;
    3650              : 
    3651    203815627 :   if ((!e->callee || e->speculative)
    3652    208753521 :       && estimate_edge_devirt_benefit (e, &call_size, &call_time, avals))
    3653              :     {
    3654       198651 :       if (hints && e->maybe_hot_p ())
    3655       189281 :         *hints |= INLINE_HINT_indirect_call;
    3656              :     }
    3657    208139933 :   cur_size = call_size * ipa_fn_summary::size_scale;
    3658    208139933 :   *size += cur_size;
    3659    208139933 :   if (min_size)
    3660     29628319 :     *min_size += cur_size;
    3661    208139933 :   if (time)
    3662    197992578 :     *time += ((sreal)call_time) * e->sreal_frequency ();
    3663    208139933 : }
    3664              : 
    3665              : 
    3666              : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
    3667              :    calls in NODE.  POSSIBLE_TRUTHS and AVALS describe the context of the call
    3668              :    site.
    3669              : 
    3670              :    Helper for estimate_calls_size_and_time which does the same but
    3671              :    (in most cases) faster.  */
    3672              : 
    3673              : static void
    3674     73737375 : estimate_calls_size_and_time_1 (struct cgraph_node *node, int *size,
    3675              :                                 int *min_size, sreal *time,
    3676              :                                 ipa_hints *hints,
    3677              :                                 clause_t possible_truths,
    3678              :                                 ipa_call_arg_values *avals)
    3679              : {
    3680     73737375 :   struct cgraph_edge *e;
    3681    351585049 :   for (e = node->callees; e; e = e->next_callee)
    3682              :     {
    3683    277847674 :       if (!e->inline_failed)
    3684              :         {
    3685     52948228 :           gcc_checking_assert (!ipa_call_summaries->get (e));
    3686     52948228 :           estimate_calls_size_and_time_1 (e->callee, size, min_size, time,
    3687              :                                           hints, possible_truths, avals);
    3688              : 
    3689     52948228 :           continue;
    3690              :         }
    3691    224899446 :       class ipa_call_summary *es = ipa_call_summaries->get (e);
    3692              : 
    3693              :       /* Do not care about zero sized builtins.  */
    3694    224899446 :       if (!es->call_stmt_size)
    3695              :         {
    3696     30178771 :           gcc_checking_assert (!es->call_stmt_time);
    3697     30178771 :           continue;
    3698              :         }
    3699    194720675 :       if (!es->predicate
    3700    194720675 :           || es->predicate->evaluate (possible_truths))
    3701              :         {
    3702              :           /* Predicates of calls shall not use NOT_CHANGED codes,
    3703              :              so we do not need to compute probabilities.  */
    3704    192452679 :           estimate_edge_size_and_time (e, size,
    3705    192452679 :                                        es->predicate ? NULL : min_size,
    3706              :                                        time, avals, hints);
    3707              :         }
    3708              :     }
    3709     77771228 :   for (e = node->indirect_calls; e; e = e->next_callee)
    3710              :     {
    3711      4033853 :       class ipa_call_summary *es = ipa_call_summaries->get (e);
    3712      4033853 :       if (!es->predicate
    3713      4033853 :           || es->predicate->evaluate (possible_truths))
    3714      4006339 :         estimate_edge_size_and_time (e, size,
    3715      4006339 :                                      es->predicate ? NULL : min_size,
    3716              :                                      time, avals, hints);
    3717              :     }
    3718     73737375 : }
    3719              : 
    3720              : /* Populate sum->call_size_time_table for edges from NODE.  */
    3721              : 
    3722              : static void
    3723      3247786 : summarize_calls_size_and_time (struct cgraph_node *node,
    3724              :                                ipa_fn_summary *sum)
    3725              : {
    3726      3247786 :   struct cgraph_edge *e;
    3727     15180740 :   for (e = node->callees; e; e = e->next_callee)
    3728              :     {
    3729     11932954 :       if (!e->inline_failed)
    3730              :         {
    3731      1462769 :           gcc_checking_assert (!ipa_call_summaries->get (e));
    3732      1462769 :           summarize_calls_size_and_time (e->callee, sum);
    3733      1462769 :           continue;
    3734              :         }
    3735     10470185 :       int size = 0;
    3736     10470185 :       sreal time = 0;
    3737              : 
    3738     10470185 :       estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
    3739              : 
    3740     10470185 :       ipa_predicate pred = true;
    3741     10470185 :       class ipa_call_summary *es = ipa_call_summaries->get (e);
    3742              : 
    3743     10470185 :       if (es->predicate)
    3744      1887532 :         pred = *es->predicate;
    3745     10470185 :       sum->account_size_time (size, time, pred, pred, true);
    3746              :     }
    3747      3565753 :   for (e = node->indirect_calls; e; e = e->next_callee)
    3748              :     {
    3749       317967 :       int size = 0;
    3750       317967 :       sreal time = 0;
    3751              : 
    3752       317967 :       estimate_edge_size_and_time (e, &size, NULL, &time, NULL, NULL);
    3753       317967 :       ipa_predicate pred = true;
    3754       317967 :       class ipa_call_summary *es = ipa_call_summaries->get (e);
    3755              : 
    3756       317967 :       if (es->predicate)
    3757        81270 :         pred = *es->predicate;
    3758       317967 :       sum->account_size_time (size, time, pred, pred, true);
    3759              :     }
    3760      3247786 : }
    3761              : 
    3762              : /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
    3763              :    calls in NODE.  POSSIBLE_TRUTHS and AVALS (the latter if non-NULL) describe
    3764              :    context of the call site.  */
    3765              : 
    3766              : static void
    3767     20789168 : estimate_calls_size_and_time (struct cgraph_node *node, int *size,
    3768              :                               int *min_size, sreal *time,
    3769              :                               ipa_hints *hints,
    3770              :                               clause_t possible_truths,
    3771              :                               ipa_call_arg_values *avals)
    3772              : {
    3773     20789168 :   class ipa_fn_summary *sum = ipa_fn_summaries->get (node);
    3774     20789168 :   bool use_table = true;
    3775              : 
    3776     20789168 :   gcc_assert (node->callees || node->indirect_calls);
    3777              : 
    3778              :   /* During early inlining we do not calculate info for very
    3779              :      large functions and thus there is no need for producing
    3780              :      summaries.  */
    3781     20789168 :   if (!ipa_node_params_sum)
    3782              :     use_table = false;
    3783              :   /* Do not calculate summaries for simple wrappers; it is waste
    3784              :      of memory.  */
    3785     11218326 :   else if (node->callees && !node->indirect_calls
    3786     10442302 :            && node->callees->inline_failed && !node->callees->next_callee)
    3787              :     use_table = false;
    3788              :   /* If there is an indirect edge that may be optimized, we need
    3789              :      to go the slow way.  */
    3790      8995535 :   else if (avals
    3791      8995535 :            && (avals->m_known_vals.length ()
    3792      3456440 :                || avals->m_known_contexts.length ()
    3793      3302093 :                || avals->m_known_aggs.length ()))
    3794              :     {
    3795      4163717 :       ipa_node_params *params_summary = ipa_node_params_sum->get (node);
    3796      4163717 :       unsigned int nargs = params_summary
    3797      4163717 :                            ? ipa_get_param_count (params_summary) : 0;
    3798              : 
    3799     14004378 :       for (unsigned int i = 0; i < nargs && use_table; i++)
    3800              :         {
    3801      9840661 :           if (ipa_is_param_used_by_indirect_call (params_summary, i)
    3802      9840661 :               && (avals->safe_sval_at (i)
    3803       280202 :                   || (ipa_argagg_value_list (avals).value_for_index_p (i))))
    3804              :             use_table = false;
    3805      9743674 :           else if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
    3806      9743674 :                    && (avals->m_known_contexts.length () > i
    3807      9992304 :                        && !avals->m_known_contexts[i].useless_p ()))
    3808              :             use_table = false;
    3809              :         }
    3810              :     }
    3811              : 
    3812              :   /* Fast path is via the call size time table.  */
    3813      4163717 :   if (use_table)
    3814              :     {
    3815              :       /* Build summary if it is absent.  */
    3816      8723868 :       if (!sum->call_size_time_table.length ())
    3817              :         {
    3818       892254 :           ipa_predicate true_pred = true;
    3819       892254 :           sum->account_size_time (0, 0, true_pred, true_pred, true);
    3820       892254 :           summarize_calls_size_and_time (node, sum);
    3821              :         }
    3822              : 
    3823      8723868 :       int old_size = *size;
    3824      8723868 :       sreal old_time = time ? *time : 0;
    3825              : 
    3826      8723868 :       if (min_size)
    3827      8723868 :         *min_size += sum->call_size_time_table[0].size;
    3828              : 
    3829      8723868 :       unsigned int i;
    3830      8723868 :       size_time_entry *e;
    3831              : 
    3832              :       /* Walk the table and account sizes and times.  */
    3833     23007212 :       for (i = 0; sum->call_size_time_table.iterate (i, &e);
    3834              :            i++)
    3835     14283344 :         if (e->exec_predicate.evaluate (possible_truths))
    3836              :           {
    3837     13366556 :             *size += e->size;
    3838     13366556 :             if (time)
    3839     11180275 :               *time += e->time;
    3840              :           }
    3841              : 
    3842              :       /* Be careful and see if both methods agree.  */
    3843           21 :       if ((flag_checking || dump_file)
    3844              :           /* Do not try to sanity check when we know we lost some
    3845              :              precision.  */
    3846      8723868 :           && sum->call_size_time_table.length ()
    3847              :              < ipa_fn_summary::max_size_time_table_size)
    3848              :         {
    3849      8723847 :           estimate_calls_size_and_time_1 (node, &old_size, NULL, &old_time, NULL,
    3850              :                                           possible_truths, avals);
    3851      8723847 :           gcc_assert (*size == old_size);
    3852     16003122 :           if (time && (*time - old_time > 1 || *time - old_time < -1)
    3853      8723880 :               && dump_file)
    3854            0 :             fprintf (dump_file, "Time mismatch in call summary %f!=%f\n",
    3855              :                      old_time.to_double (),
    3856              :                      time->to_double ());
    3857              :         }
    3858              :     }
    3859              :   /* Slow path by walking all edges.  */
    3860              :   else
    3861     12065300 :     estimate_calls_size_and_time_1 (node, size, min_size, time, hints,
    3862              :                                     possible_truths, avals);
    3863     20789168 : }
    3864              : 
    3865              : /* Main constructor for ipa call context.  Memory allocation of ARG_VALUES
    3866              :    is owned by the caller.  INLINE_PARAM_SUMMARY is also owned by the
    3867              :    caller.  */
    3868              : 
    3869     19920174 : ipa_call_context::ipa_call_context (cgraph_node *node, clause_t possible_truths,
    3870              :                                     clause_t nonspec_possible_truths,
    3871              :                                     vec<inline_param_summary>
    3872              :                                       inline_param_summary,
    3873              :                                     ipa_auto_call_arg_values *arg_values)
    3874     19920174 : : m_node (node), m_possible_truths (possible_truths),
    3875     19920174 :   m_nonspec_possible_truths (nonspec_possible_truths),
    3876     19920174 :   m_inline_param_summary (inline_param_summary),
    3877     19920174 :   m_avals (arg_values)
    3878              : {
    3879     19920174 : }
    3880              : 
    3881              : /* Set THIS to be a duplicate of CTX.  Copy all relevant info.  */
    3882              : 
    3883              : void
    3884      1995701 : ipa_cached_call_context::duplicate_from (const ipa_call_context &ctx)
    3885              : {
    3886      1995701 :   m_node = ctx.m_node;
    3887      1995701 :   m_possible_truths = ctx.m_possible_truths;
    3888      1995701 :   m_nonspec_possible_truths = ctx.m_nonspec_possible_truths;
    3889      1995701 :   ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
    3890      1995701 :   unsigned int nargs = params_summary
    3891      1995701 :                        ? ipa_get_param_count (params_summary) : 0;
    3892              : 
    3893      1995701 :   m_inline_param_summary = vNULL;
    3894              :   /* Copy the info only if there is at least one useful entry.  */
    3895      1995701 :   if (ctx.m_inline_param_summary.exists ())
    3896              :     {
    3897      1772624 :       unsigned int n = MIN (ctx.m_inline_param_summary.length (), nargs);
    3898              : 
    3899      4058482 :       for (unsigned int i = 0; i < n; i++)
    3900      3311171 :         if (ipa_is_param_used_by_ipa_predicates (params_summary, i)
    3901      4712938 :             && !ctx.m_inline_param_summary[i].useless_p ())
    3902              :           {
    3903      1025313 :             m_inline_param_summary
    3904      1025313 :                     = ctx.m_inline_param_summary.copy ();
    3905      1025313 :             break;
    3906              :           }
    3907              :     }
    3908      1995701 :   m_avals.m_known_vals = vNULL;
    3909      1995701 :   if (ctx.m_avals.m_known_vals.exists ())
    3910              :     {
    3911      1995701 :       unsigned int n = MIN (ctx.m_avals.m_known_vals.length (), nargs);
    3912              : 
    3913      4615157 :       for (unsigned int i = 0; i < n; i++)
    3914      2646421 :         if (ipa_is_param_used_by_indirect_call (params_summary, i)
    3915      2646421 :             && ctx.m_avals.m_known_vals[i])
    3916              :           {
    3917        26965 :             m_avals.m_known_vals = ctx.m_avals.m_known_vals.copy ();
    3918        26965 :             break;
    3919              :           }
    3920              :     }
    3921              : 
    3922      1995701 :   m_avals.m_known_contexts = vNULL;
    3923      1995701 :   if (ctx.m_avals.m_known_contexts.exists ())
    3924              :     {
    3925      1995701 :       unsigned int n = MIN (ctx.m_avals.m_known_contexts.length (), nargs);
    3926              : 
    3927      1998569 :       for (unsigned int i = 0; i < n; i++)
    3928        28497 :         if (ipa_is_param_used_by_polymorphic_call (params_summary, i)
    3929        28497 :             && !ctx.m_avals.m_known_contexts[i].useless_p ())
    3930              :           {
    3931        25629 :             m_avals.m_known_contexts = ctx.m_avals.m_known_contexts.copy ();
    3932        25629 :             break;
    3933              :           }
    3934              :     }
    3935              : 
    3936      1995701 :   m_avals.m_known_aggs = vNULL;
    3937      1995701 :   if (ctx.m_avals.m_known_aggs.exists ())
    3938              :     {
    3939      1995701 :       const ipa_argagg_value_list avl (&ctx.m_avals);
    3940      6643054 :       for (unsigned int i = 0; i < nargs; i++)
    3941      4650304 :         if (ipa_is_param_used_by_indirect_call (params_summary, i)
    3942      4650304 :             && avl.value_for_index_p (i))
    3943              :           {
    3944         2951 :             m_avals.m_known_aggs = ctx.m_avals.m_known_aggs.copy ();
    3945         2951 :             break;
    3946              :           }
    3947              :     }
    3948              : 
    3949      1995701 :   m_avals.m_known_value_ranges = vNULL;
    3950      1995701 : }
    3951              : 
    3952              : /* Release memory used by known_vals/contexts/aggs vectors.  and
    3953              :    inline_param_summary.  */
    3954              : 
    3955              : void
    3956      3248610 : ipa_cached_call_context::release ()
    3957              : {
    3958              :   /* See if context is initialized at first place.  */
    3959      3248610 :   if (!m_node)
    3960              :     return;
    3961      1995701 :   m_avals.m_known_aggs.release ();
    3962      1995701 :   m_avals.m_known_vals.release ();
    3963      1995701 :   m_avals.m_known_contexts.release ();
    3964      1995701 :   m_inline_param_summary.release ();
    3965              : }
    3966              : 
    3967              : /* Return true if CTX describes the same call context as THIS.  */
    3968              : 
    3969              : bool
    3970      7004800 : ipa_call_context::equal_to (const ipa_call_context &ctx)
    3971              : {
    3972      7004800 :   if (m_node != ctx.m_node
    3973      5751891 :       || m_possible_truths != ctx.m_possible_truths
    3974      5150317 :       || m_nonspec_possible_truths != ctx.m_nonspec_possible_truths)
    3975              :     return false;
    3976              : 
    3977      5150317 :   ipa_node_params *params_summary = ipa_node_params_sum->get (m_node);
    3978      5150317 :   unsigned int nargs = params_summary
    3979      5150317 :                        ? ipa_get_param_count (params_summary) : 0;
    3980              : 
    3981      5150317 :   if (m_inline_param_summary.exists () || ctx.m_inline_param_summary.exists ())
    3982              :     {
    3983     15456113 :       for (unsigned int i = 0; i < nargs; i++)
    3984              :         {
    3985     10668448 :           if (!ipa_is_param_used_by_ipa_predicates (params_summary, i))
    3986      3469426 :             continue;
    3987      7199022 :           if (i >= m_inline_param_summary.length ()
    3988      4947324 :               || m_inline_param_summary[i].useless_p ())
    3989              :             {
    3990      3240434 :               if (i < ctx.m_inline_param_summary.length ()
    3991      3240434 :                   && !ctx.m_inline_param_summary[i].useless_p ())
    3992              :                 return false;
    3993      3190896 :               continue;
    3994              :             }
    3995      3958588 :           if (i >= ctx.m_inline_param_summary.length ()
    3996      3958585 :               || ctx.m_inline_param_summary[i].useless_p ())
    3997              :             {
    3998        47589 :               if (i < m_inline_param_summary.length ()
    3999        47589 :                   && !m_inline_param_summary[i].useless_p ())
    4000              :                 return false;
    4001            0 :               continue;
    4002              :             }
    4003      3910999 :           if (!m_inline_param_summary[i].equal_to
    4004      3910999 :                  (ctx.m_inline_param_summary[i]))
    4005              :             return false;
    4006              :         }
    4007              :     }
    4008      5022718 :   if (m_avals.m_known_vals.exists () || ctx.m_avals.m_known_vals.exists ())
    4009              :     {
    4010     15484459 :       for (unsigned int i = 0; i < nargs; i++)
    4011              :         {
    4012     10472468 :           if (!ipa_is_param_used_by_indirect_call (params_summary, i))
    4013     10254262 :             continue;
    4014       218206 :           if (i >= m_avals.m_known_vals.length () || !m_avals.m_known_vals[i])
    4015              :             {
    4016       160662 :               if (i < ctx.m_avals.m_known_vals.length ()
    4017       160662 :                   && ctx.m_avals.m_known_vals[i])
    4018              :                 return false;
    4019       160573 :               continue;
    4020              :             }
    4021        57544 :           if (i >= ctx.m_avals.m_known_vals.length ()
    4022        57544 :               || !ctx.m_avals.m_known_vals[i])
    4023              :             {
    4024              :               if (i < m_avals.m_known_vals.length () && m_avals.m_known_vals[i])
    4025              :                 return false;
    4026              :               continue;
    4027              :             }
    4028        57454 :           if (m_avals.m_known_vals[i] != ctx.m_avals.m_known_vals[i])
    4029              :             return false;
    4030              :         }
    4031              :     }
    4032      5011991 :   if (m_avals.m_known_contexts.exists ()
    4033      5011991 :       || ctx.m_avals.m_known_contexts.exists ())
    4034              :     {
    4035     15469078 :       for (unsigned int i = 0; i < nargs; i++)
    4036              :         {
    4037     10458565 :           if (!ipa_is_param_used_by_polymorphic_call (params_summary, i))
    4038     10314385 :             continue;
    4039       144180 :           if (i >= m_avals.m_known_contexts.length ()
    4040       143019 :               || m_avals.m_known_contexts[i].useless_p ())
    4041              :             {
    4042         1161 :               if (i < ctx.m_avals.m_known_contexts.length ()
    4043         1161 :                   && !ctx.m_avals.m_known_contexts[i].useless_p ())
    4044              :                 return false;
    4045         1149 :               continue;
    4046              :             }
    4047       143019 :           if (i >= ctx.m_avals.m_known_contexts.length ()
    4048       143019 :               || ctx.m_avals.m_known_contexts[i].useless_p ())
    4049              :             {
    4050            8 :               if (i < m_avals.m_known_contexts.length ()
    4051            8 :                   && !m_avals.m_known_contexts[i].useless_p ())
    4052              :                 return false;
    4053            0 :               continue;
    4054              :             }
    4055       143011 :           if (!m_avals.m_known_contexts[i].equal_to
    4056       143011 :                  (ctx.m_avals.m_known_contexts[i]))
    4057              :             return false;
    4058              :         }
    4059              :     }
    4060      5010513 :   if (m_avals.m_known_aggs.exists () || ctx.m_avals.m_known_aggs.exists ())
    4061              :     {
    4062              :       unsigned i = 0, j = 0;
    4063     11880060 :       while (i < m_avals.m_known_aggs.length ()
    4064      5939276 :              || j < ctx.m_avals.m_known_aggs.length ())
    4065              :         {
    4066       930177 :           if (i >= m_avals.m_known_aggs.length ())
    4067              :             {
    4068       925276 :               int idx2 = ctx.m_avals.m_known_aggs[j].index;
    4069       925276 :               if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
    4070              :                 return false;
    4071       924716 :               j++;
    4072       924716 :               continue;
    4073       924716 :             }
    4074         4901 :           if (j >= ctx.m_avals.m_known_aggs.length ())
    4075              :             {
    4076          603 :               int idx1 = m_avals.m_known_aggs[i].index;
    4077          603 :               if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
    4078              :                 return false;
    4079            8 :               i++;
    4080            8 :               continue;
    4081            8 :             }
    4082              : 
    4083         4298 :           int idx1 = m_avals.m_known_aggs[i].index;
    4084         4298 :           int idx2 = ctx.m_avals.m_known_aggs[j].index;
    4085         4298 :           if (idx1 < idx2)
    4086              :             {
    4087            0 :               if (ipa_is_param_used_by_indirect_call (params_summary, idx1))
    4088              :                 return false;
    4089            0 :               i++;
    4090            0 :               continue;
    4091              :             }
    4092         4298 :           if (idx1 > idx2)
    4093              :             {
    4094            0 :               if (ipa_is_param_used_by_indirect_call (params_summary, idx2))
    4095              :                 return false;
    4096            0 :               j++;
    4097            0 :               continue;
    4098              :             }
    4099         4298 :           if (!ipa_is_param_used_by_indirect_call (params_summary, idx1))
    4100              :             {
    4101          482 :               i++;
    4102          482 :               j++;
    4103          482 :               continue;
    4104              :             }
    4105              : 
    4106         3816 :           if ((m_avals.m_known_aggs[i].unit_offset
    4107         3816 :                != ctx.m_avals.m_known_aggs[j].unit_offset)
    4108         3808 :               || (m_avals.m_known_aggs[i].by_ref
    4109         3808 :                != ctx.m_avals.m_known_aggs[j].by_ref)
    4110         7624 :               || !operand_equal_p (m_avals.m_known_aggs[i].value,
    4111         3808 :                                    ctx.m_avals.m_known_aggs[j].value))
    4112              :             return false;
    4113         3557 :           i++;
    4114         3557 :           j++;
    4115              :         }
    4116              :     }
    4117              :   return true;
    4118              : }
    4119              : 
    4120              : /* Fill in the selected fields in ESTIMATES with value estimated for call in
    4121              :    this context.  Always compute size and min_size.  Only compute time and
    4122              :    nonspecialized_time if EST_TIMES is true.  Only compute hints if EST_HINTS
    4123              :    is true.  */
    4124              : 
    4125              : void
    4126     19887656 : ipa_call_context::estimate_size_and_time (ipa_call_estimates *estimates,
    4127              :                                           bool est_times, bool est_hints)
    4128              : {
    4129     19887656 :   class ipa_fn_summary *info = ipa_fn_summaries->get (m_node);
    4130     19887656 :   size_time_entry *e;
    4131     19887656 :   int size = 0;
    4132     19887656 :   sreal time = 0;
    4133     19887656 :   int min_size = 0;
    4134     19887656 :   ipa_hints hints = 0;
    4135     19887656 :   sreal loops_with_known_iterations = 0;
    4136     19887656 :   sreal loops_with_known_strides = 0;
    4137     19887656 :   int i;
    4138              : 
    4139     19887656 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4140              :     {
    4141         1333 :       bool found = false;
    4142         1333 :       fprintf (dump_file, "   Estimating body: %s\n"
    4143              :                "   Known to be false: ", m_node->dump_name ());
    4144              : 
    4145         4347 :       for (i = ipa_predicate::not_inlined_condition;
    4146         8694 :            i < (ipa_predicate::first_dynamic_condition
    4147         7144 :                 + (int) vec_safe_length (info->conds)); i++)
    4148         3014 :         if (!(m_possible_truths & (1 << i)))
    4149              :           {
    4150         1795 :             if (found)
    4151          519 :               fprintf (dump_file, ", ");
    4152         1795 :             found = true;
    4153         1795 :             dump_condition (dump_file, info->conds, i);
    4154              :           }
    4155              :     }
    4156              : 
    4157     19887656 :   if (m_node->callees || m_node->indirect_calls)
    4158     26432821 :     estimate_calls_size_and_time (m_node, &size, &min_size,
    4159              :                                   est_times ? &time : NULL,
    4160              :                                   est_hints ? &hints : NULL, m_possible_truths,
    4161              :                                   &m_avals);
    4162              : 
    4163     19887656 :   sreal nonspecialized_time = time;
    4164              : 
    4165     19887656 :   min_size += info->size_time_table[0].size;
    4166    135594765 :   for (i = 0; info->size_time_table.iterate (i, &e); i++)
    4167              :     {
    4168    115707109 :       bool exec = e->exec_predicate.evaluate (m_nonspec_possible_truths);
    4169              : 
    4170              :       /* Because predicates are conservative, it can happen that nonconst is 1
    4171              :          but exec is 0.  */
    4172    115707109 :       if (exec)
    4173              :         {
    4174    113269235 :           bool nonconst = e->nonconst_predicate.evaluate (m_possible_truths);
    4175              : 
    4176    113269235 :           gcc_checking_assert (e->time >= 0);
    4177    113269235 :           gcc_checking_assert (time >= 0);
    4178              : 
    4179              :           /* We compute specialized size only because size of nonspecialized
    4180              :              copy is context independent.
    4181              : 
    4182              :              The difference between nonspecialized execution and specialized is
    4183              :              that nonspecialized is not going to have optimized out computations
    4184              :              known to be constant in a specialized setting.  */
    4185    113269235 :           if (nonconst)
    4186     56992416 :             size += e->size;
    4187    113269235 :           if (!est_times)
    4188     59317853 :             continue;
    4189     53951382 :           nonspecialized_time += e->time;
    4190     53951382 :           if (!nonconst)
    4191              :             ;
    4192     25946873 :           else if (!m_inline_param_summary.exists ())
    4193              :             {
    4194      2464399 :               if (nonconst)
    4195      2464399 :                 time += e->time;
    4196              :             }
    4197              :           else
    4198              :             {
    4199     23482474 :               int prob = e->nonconst_predicate.probability
    4200     23482474 :                                                (info->conds, m_possible_truths,
    4201              :                                                 m_inline_param_summary);
    4202     23482474 :               gcc_checking_assert (prob >= 0);
    4203     23482474 :               gcc_checking_assert (prob <= REG_BR_PROB_BASE);
    4204     23482474 :               if (prob == REG_BR_PROB_BASE)
    4205     19623522 :                 time += e->time;
    4206              :               else
    4207      3858952 :                 time += e->time * prob / REG_BR_PROB_BASE;
    4208              :             }
    4209     53951382 :           gcc_checking_assert (time >= 0);
    4210              :         }
    4211              :      }
    4212     19887656 :   gcc_checking_assert (info->size_time_table[0].exec_predicate == true);
    4213     19887656 :   gcc_checking_assert (info->size_time_table[0].nonconst_predicate == true);
    4214     19887656 :   gcc_checking_assert (min_size >= 0);
    4215     19887656 :   gcc_checking_assert (size >= 0);
    4216     19887656 :   gcc_checking_assert (time >= 0);
    4217              :   /* nonspecialized_time should be always bigger than specialized time.
    4218              :      Roundoff issues however may get into the way.  */
    4219     19887656 :   gcc_checking_assert ((nonspecialized_time - time * 99 / 100) >= -1);
    4220              : 
    4221              :   /* Roundoff issues may make specialized time bigger than nonspecialized
    4222              :      time.  We do not really want that to happen because some heuristics
    4223              :      may get confused by seeing negative speedups.  */
    4224     19887656 :   if (time > nonspecialized_time)
    4225            0 :     time = nonspecialized_time;
    4226              : 
    4227     19887656 :   if (est_hints)
    4228              :     {
    4229      7211515 :       if (info->scc_no)
    4230       234170 :         hints |= INLINE_HINT_in_scc;
    4231      7211515 :       if (DECL_DECLARED_INLINE_P (m_node->decl))
    4232      4540092 :         hints |= INLINE_HINT_declared_inline;
    4233      7211515 :       if (info->builtin_constant_p_parms.length ()
    4234         9826 :           && DECL_DECLARED_INLINE_P (m_node->decl))
    4235         9637 :         hints |= INLINE_HINT_builtin_constant_p;
    4236              : 
    4237      7211515 :       ipa_freqcounting_predicate *fcp;
    4238      7771705 :       for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
    4239       560190 :         if (!fcp->predicate->evaluate (m_possible_truths))
    4240              :           {
    4241       372728 :             hints |= INLINE_HINT_loop_iterations;
    4242       372728 :             loops_with_known_iterations += fcp->freq;
    4243              :           }
    4244      7211515 :       estimates->loops_with_known_iterations = loops_with_known_iterations;
    4245              : 
    4246      7506112 :       for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
    4247       294597 :         if (!fcp->predicate->evaluate (m_possible_truths))
    4248              :           {
    4249       263751 :             hints |= INLINE_HINT_loop_stride;
    4250       263751 :             loops_with_known_strides += fcp->freq;
    4251              :           }
    4252      7211515 :       estimates->loops_with_known_strides = loops_with_known_strides;
    4253              :     }
    4254              : 
    4255     19887656 :   size = RDIV (size, ipa_fn_summary::size_scale);
    4256     19887656 :   min_size = RDIV (min_size, ipa_fn_summary::size_scale);
    4257              : 
    4258     19887656 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4259              :     {
    4260         1333 :       fprintf (dump_file, "\n   size:%i", (int) size);
    4261         1333 :       if (est_times)
    4262         1096 :         fprintf (dump_file, " time:%f nonspec time:%f",
    4263              :                  time.to_double (), nonspecialized_time.to_double ());
    4264         1333 :       if (est_hints)
    4265         1096 :         fprintf (dump_file, " loops with known iterations:%f "
    4266              :                  "known strides:%f", loops_with_known_iterations.to_double (),
    4267              :                  loops_with_known_strides.to_double ());
    4268         1333 :       fprintf (dump_file, "\n");
    4269              :     }
    4270     19887656 :   if (est_times)
    4271              :     {
    4272      7211515 :       estimates->time = time;
    4273      7211515 :       estimates->nonspecialized_time = nonspecialized_time;
    4274              :     }
    4275     19887656 :   estimates->size = size;
    4276     19887656 :   estimates->min_size = min_size;
    4277     19887656 :   if (est_hints)
    4278      7211515 :     estimates->hints = hints;
    4279     19887656 :   return;
    4280              : }
    4281              : 
    4282              : 
    4283              : /* Estimate size and time needed to execute callee of EDGE assuming that
    4284              :    parameters known to be constant at caller of EDGE are propagated.
    4285              :    KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
    4286              :    and types for parameters.  */
    4287              : 
    4288              : void
    4289       235432 : estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
    4290              :                                    ipa_auto_call_arg_values *avals,
    4291              :                                    ipa_call_estimates *estimates)
    4292              : {
    4293       235432 :   clause_t clause, nonspec_clause;
    4294              : 
    4295       235432 :   evaluate_conditions_for_known_args (node, false, avals, &clause,
    4296              :                                       &nonspec_clause, NULL);
    4297       235432 :   ipa_call_context ctx (node, clause, nonspec_clause, vNULL, avals);
    4298       235432 :   ctx.estimate_size_and_time (estimates);
    4299       235432 : }
    4300              : 
    4301              : /* Return stack frame offset where frame of NODE is supposed to start inside
    4302              :    of the function it is inlined to.
    4303              :    Return 0 for functions that are not inlined.  */
    4304              : 
    4305              : HOST_WIDE_INT
    4306      5265656 : ipa_get_stack_frame_offset (struct cgraph_node *node)
    4307              : {
    4308      5265656 :   HOST_WIDE_INT offset = 0;
    4309      5265656 :   if (!node->inlined_to)
    4310              :     return 0;
    4311      4185744 :   node = node->callers->caller;
    4312      5420028 :   while (true)
    4313              :     {
    4314      4802886 :       offset += ipa_size_summaries->get (node)->estimated_self_stack_size;
    4315      4802886 :       if (!node->inlined_to)
    4316              :         return offset;
    4317       617142 :       node = node->callers->caller;
    4318              :     }
    4319              : }
    4320              : 
    4321              : 
    4322              : /* Update summary information of inline clones after inlining.
    4323              :    Compute peak stack usage.  */
    4324              : 
    4325              : static void
    4326      4913614 : inline_update_callee_summaries (struct cgraph_node *node, int depth)
    4327              : {
    4328      4913614 :   struct cgraph_edge *e;
    4329              : 
    4330      4913614 :   ipa_propagate_frequency (node);
    4331      9513012 :   for (e = node->callees; e; e = e->next_callee)
    4332              :     {
    4333      4599398 :       if (!e->inline_failed)
    4334       728423 :         inline_update_callee_summaries (e->callee, depth);
    4335              :       else
    4336      3870975 :         ipa_call_summaries->get (e)->loop_depth += depth;
    4337              :     }
    4338      5030603 :   for (e = node->indirect_calls; e; e = e->next_callee)
    4339       116989 :     ipa_call_summaries->get (e)->loop_depth += depth;
    4340      4913614 : }
    4341              : 
    4342              : /* Update change_prob and points_to_local_or_readonly_memory of EDGE after
    4343              :    INLINED_EDGE has been inlined.
    4344              : 
    4345              :    When function A is inlined in B and A calls C with parameter that
    4346              :    changes with probability PROB1 and C is known to be passthrough
    4347              :    of argument if B that change with probability PROB2, the probability
    4348              :    of change is now PROB1*PROB2.  */
    4349              : 
    4350              : static void
    4351      3988274 : remap_edge_params (struct cgraph_edge *inlined_edge,
    4352              :                    struct cgraph_edge *edge)
    4353              : {
    4354      3988274 :   if (ipa_node_params_sum)
    4355              :     {
    4356      2797138 :       int i;
    4357      2797138 :       ipa_edge_args *args = ipa_edge_args_sum->get (edge);
    4358      2797138 :       if (!args)
    4359              :         return;
    4360      1598264 :       class ipa_call_summary *es = ipa_call_summaries->get (edge);
    4361      1598264 :       class ipa_call_summary *inlined_es
    4362      1598264 :         = ipa_call_summaries->get (inlined_edge);
    4363              : 
    4364      1598264 :       if (es->param.length () == 0)
    4365              :         return;
    4366              : 
    4367      9286697 :       for (i = 0; i < ipa_get_cs_argument_count (args); i++)
    4368              :         {
    4369      3129944 :           struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
    4370      3129944 :           if (jfunc->type == IPA_JF_PASS_THROUGH
    4371      2302034 :               || jfunc->type == IPA_JF_ANCESTOR)
    4372              :             {
    4373       970972 :               int id = jfunc->type == IPA_JF_PASS_THROUGH
    4374       970972 :                        ? ipa_get_jf_pass_through_formal_id (jfunc)
    4375       143062 :                        : ipa_get_jf_ancestor_formal_id (jfunc);
    4376      1941861 :               if (id < (int) inlined_es->param.length ())
    4377              :                 {
    4378       970881 :                   int prob1 = es->param[i].change_prob;
    4379       970881 :                   int prob2 = inlined_es->param[id].change_prob;
    4380       970881 :                   int prob = combine_probabilities (prob1, prob2);
    4381              : 
    4382       970881 :                   if (prob1 && prob2 && !prob)
    4383       970881 :                     prob = 1;
    4384              : 
    4385       970881 :                   es->param[i].change_prob = prob;
    4386              : 
    4387      1941762 :                   if (inlined_es
    4388       970881 :                         ->param[id].points_to_local_or_readonly_memory)
    4389       215325 :                     es->param[i].points_to_local_or_readonly_memory = true;
    4390      1941762 :                   if (inlined_es
    4391       970881 :                         ->param[id].points_to_possible_sra_candidate)
    4392       179288 :                     es->param[i].points_to_possible_sra_candidate = true;
    4393              :                 }
    4394       970972 :               if (!es->param[i].points_to_local_or_readonly_memory
    4395              :                   && jfunc->type == IPA_JF_CONST
    4396              :                   && points_to_local_or_readonly_memory_p
    4397              :                          (ipa_get_jf_constant (jfunc)))
    4398              :                 es->param[i].points_to_local_or_readonly_memory = true;
    4399              :             }
    4400              :         }
    4401              :     }
    4402              : }
    4403              : 
    4404              : /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
    4405              : 
    4406              :    Remap predicates of callees of NODE.  Rest of arguments match
    4407              :    remap_predicate.
    4408              : 
    4409              :    Also update change probabilities.  */
    4410              : 
    4411              : static void
    4412      4913614 : remap_edge_summaries (struct cgraph_edge *inlined_edge,
    4413              :                       struct cgraph_node *node,
    4414              :                       class ipa_fn_summary *info,
    4415              :                       class ipa_node_params *params_summary,
    4416              :                       class ipa_fn_summary *callee_info,
    4417              :                       const vec<int> &operand_map,
    4418              :                       const vec<HOST_WIDE_INT> &offset_map,
    4419              :                       clause_t possible_truths,
    4420              :                       ipa_predicate *toplev_predicate)
    4421              : {
    4422      4913614 :   struct cgraph_edge *e, *next;
    4423      9512555 :   for (e = node->callees; e; e = next)
    4424              :     {
    4425      4598941 :       ipa_predicate p;
    4426      4598941 :       next = e->next_callee;
    4427              : 
    4428      4598941 :       if (e->inline_failed)
    4429              :         {
    4430      3870518 :           class ipa_call_summary *es = ipa_call_summaries->get (e);
    4431      3870518 :           remap_edge_params (inlined_edge, e);
    4432              : 
    4433      3870518 :           if (es->predicate)
    4434              :             {
    4435      1301434 :               p = es->predicate->remap_after_inlining
    4436      1301434 :                                      (info, params_summary,
    4437              :                                       callee_info, operand_map,
    4438              :                                       offset_map, possible_truths,
    4439              :                                       *toplev_predicate);
    4440      1301434 :               edge_set_predicate (e, &p);
    4441              :             }
    4442              :           else
    4443      2569084 :             edge_set_predicate (e, toplev_predicate);
    4444              :         }
    4445              :       else
    4446       728423 :         remap_edge_summaries (inlined_edge, e->callee, info,
    4447              :                               params_summary, callee_info,
    4448              :                               operand_map, offset_map, possible_truths,
    4449              :                               toplev_predicate);
    4450              :     }
    4451      5031370 :   for (e = node->indirect_calls; e; e = next)
    4452              :     {
    4453       117756 :       class ipa_call_summary *es = ipa_call_summaries->get (e);
    4454       117756 :       ipa_predicate p;
    4455       117756 :       next = e->next_callee;
    4456              : 
    4457       117756 :       remap_edge_params (inlined_edge, e);
    4458       117756 :       if (es->predicate)
    4459              :         {
    4460        25220 :           p = es->predicate->remap_after_inlining
    4461        25220 :                                  (info, params_summary,
    4462              :                                   callee_info, operand_map, offset_map,
    4463              :                                   possible_truths, *toplev_predicate);
    4464        25220 :           edge_set_predicate (e, &p);
    4465              :         }
    4466              :       else
    4467        92536 :         edge_set_predicate (e, toplev_predicate);
    4468              :     }
    4469      4913614 : }
    4470              : 
    4471              : /* Run remap_after_inlining on each predicate in V.  */
    4472              : 
    4473              : static void
    4474      8370382 : remap_freqcounting_predicate (class ipa_fn_summary *info,
    4475              :                               class ipa_node_params *params_summary,
    4476              :                               class ipa_fn_summary *callee_info,
    4477              :                               vec<ipa_freqcounting_predicate, va_gc> *v,
    4478              :                               const vec<int> &operand_map,
    4479              :                               const vec<HOST_WIDE_INT> &offset_map,
    4480              :                               clause_t possible_truths,
    4481              :                               ipa_predicate *toplev_predicate)
    4482              : 
    4483              : {
    4484      8370382 :   ipa_freqcounting_predicate *fcp;
    4485      8398134 :   for (int i = 0; vec_safe_iterate (v, i, &fcp); i++)
    4486              :     {
    4487        27752 :       ipa_predicate p
    4488        27752 :         = fcp->predicate->remap_after_inlining (info, params_summary,
    4489              :                                                 callee_info, operand_map,
    4490              :                                                 offset_map, possible_truths,
    4491              :                                                 *toplev_predicate);
    4492        39015 :       if (p != false && p != true)
    4493         2679 :         *fcp->predicate &= p;
    4494              :     }
    4495      8370382 : }
    4496              : 
    4497              : /* We inlined EDGE.  Update summary of the function we inlined into.  */
    4498              : 
    4499              : void
    4500      4185191 : ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge)
    4501              : {
    4502      4185191 :   ipa_fn_summary *callee_info = ipa_fn_summaries->get (edge->callee);
    4503      3894428 :   struct cgraph_node *to = (edge->caller->inlined_to
    4504      4185191 :                             ? edge->caller->inlined_to : edge->caller);
    4505      4185191 :   class ipa_fn_summary *info = ipa_fn_summaries->get (to);
    4506      4185191 :   clause_t clause = 0;  /* not_inline is known to be false.  */
    4507      4185191 :   size_time_entry *e;
    4508      4185191 :   auto_vec<int, 8> operand_map;
    4509      4185191 :   auto_vec<HOST_WIDE_INT, 8> offset_map;
    4510      4185191 :   int i;
    4511      4185191 :   ipa_predicate toplev_predicate;
    4512      4185191 :   class ipa_call_summary *es = ipa_call_summaries->get (edge);
    4513      4185191 :   ipa_node_params *params_summary = (ipa_node_params_sum
    4514      4185191 :                                      ? ipa_node_params_sum->get (to) : NULL);
    4515              : 
    4516      4185191 :   if (es->predicate)
    4517       352046 :     toplev_predicate = *es->predicate;
    4518              :   else
    4519      3833145 :     toplev_predicate = true;
    4520              : 
    4521      4185191 :   info->fp_expressions |= callee_info->fp_expressions;
    4522      4185191 :   info->target_info |= callee_info->target_info;
    4523              : 
    4524      4185191 :   if (callee_info->conds)
    4525              :     {
    4526      3066505 :       ipa_auto_call_arg_values avals;
    4527      3066505 :       evaluate_properties_for_edge (edge, true, &clause, NULL, &avals, false);
    4528      3066505 :     }
    4529      4185191 :   if (ipa_node_params_sum && callee_info->conds)
    4530              :     {
    4531       829494 :       ipa_edge_args *args = ipa_edge_args_sum->get (edge);
    4532       829494 :       int count = args ? ipa_get_cs_argument_count (args) : 0;
    4533       829355 :       int i;
    4534              : 
    4535       829355 :       if (count)
    4536              :         {
    4537       829355 :           operand_map.safe_grow_cleared (count, true);
    4538       829355 :           offset_map.safe_grow_cleared (count, true);
    4539              :         }
    4540      2664641 :       for (i = 0; i < count; i++)
    4541              :         {
    4542      1835147 :           struct ipa_jump_func *jfunc = ipa_get_ith_jump_func (args, i);
    4543      1835147 :           int map = -1;
    4544              : 
    4545              :           /* TODO: handle non-NOPs when merging.  */
    4546      1835147 :           if (jfunc->type == IPA_JF_PASS_THROUGH)
    4547              :             {
    4548       298640 :               if (ipa_get_jf_pass_through_operation (jfunc) == NOP_EXPR)
    4549       295125 :                 map = ipa_get_jf_pass_through_formal_id (jfunc);
    4550       298640 :               if (!ipa_get_jf_pass_through_agg_preserved (jfunc))
    4551       200918 :                 offset_map[i] = -1;
    4552              :             }
    4553      1536507 :           else if (jfunc->type == IPA_JF_ANCESTOR)
    4554              :             {
    4555        95954 :               HOST_WIDE_INT offset = ipa_get_jf_ancestor_offset (jfunc);
    4556        95954 :               if (offset >= 0 && offset < INT_MAX)
    4557              :                 {
    4558        95954 :                   map = ipa_get_jf_ancestor_formal_id (jfunc);
    4559        95954 :                   if (!ipa_get_jf_ancestor_agg_preserved (jfunc))
    4560        57702 :                     offset = -1;
    4561        95954 :                   offset_map[i] = offset;
    4562              :                 }
    4563              :             }
    4564      1835147 :           operand_map[i] = map;
    4565      3312684 :           gcc_assert (map < ipa_get_param_count (params_summary));
    4566              :         }
    4567              : 
    4568              :       int ip;
    4569       832416 :       for (i = 0; callee_info->builtin_constant_p_parms.iterate (i, &ip); i++)
    4570         2922 :         if (ip < count && operand_map[ip] >= 0)
    4571           62 :           add_builtin_constant_p_parm (info, operand_map[ip]);
    4572              :     }
    4573      4185191 :   sreal freq = edge->sreal_frequency ();
    4574     27250755 :   for (i = 0; callee_info->size_time_table.iterate (i, &e); i++)
    4575              :     {
    4576     18880373 :       ipa_predicate p;
    4577     18880373 :       p = e->exec_predicate.remap_after_inlining
    4578     18880373 :                              (info, params_summary,
    4579              :                               callee_info, operand_map,
    4580              :                               offset_map, clause,
    4581              :                               toplev_predicate);
    4582     18880373 :       ipa_predicate nonconstp;
    4583     18880373 :       nonconstp = e->nonconst_predicate.remap_after_inlining
    4584     18880373 :                                      (info, params_summary,
    4585              :                                       callee_info, operand_map,
    4586              :                                       offset_map, clause,
    4587              :                                       toplev_predicate);
    4588     28543717 :       if (p != false && nonconstp != false)
    4589              :         {
    4590      9331478 :           sreal add_time = ((sreal)e->time * freq);
    4591      9331478 :           int prob = e->nonconst_predicate.probability (callee_info->conds,
    4592              :                                                         clause, es->param);
    4593      9331478 :           if (prob != REG_BR_PROB_BASE)
    4594      1137559 :             add_time = add_time * prob / REG_BR_PROB_BASE;
    4595      1137559 :           if (prob != REG_BR_PROB_BASE
    4596      1137559 :               && dump_file && (dump_flags & TDF_DETAILS))
    4597              :             {
    4598            3 :               fprintf (dump_file, "\t\tScaling time by probability:%f\n",
    4599            3 :                        (double) prob / REG_BR_PROB_BASE);
    4600              :             }
    4601      9331478 :           info->account_size_time (e->size, add_time, p, nonconstp);
    4602              :         }
    4603              :     }
    4604      4185191 :   remap_edge_summaries (edge, edge->callee, info, params_summary,
    4605              :                         callee_info, operand_map,
    4606              :                         offset_map, clause, &toplev_predicate);
    4607      4185191 :   remap_freqcounting_predicate (info, params_summary, callee_info,
    4608              :                                 info->loop_iterations, operand_map,
    4609              :                                 offset_map, clause, &toplev_predicate);
    4610      4185191 :   remap_freqcounting_predicate (info, params_summary, callee_info,
    4611              :                                 info->loop_strides, operand_map,
    4612              :                                 offset_map, clause, &toplev_predicate);
    4613              : 
    4614      4185191 :   HOST_WIDE_INT stack_frame_offset = ipa_get_stack_frame_offset (edge->callee);
    4615      4185191 :   HOST_WIDE_INT peak = stack_frame_offset + callee_info->estimated_stack_size;
    4616              : 
    4617      4185191 :   if (info->estimated_stack_size < peak)
    4618       126222 :     info->estimated_stack_size = peak;
    4619              : 
    4620      4185191 :   inline_update_callee_summaries (edge->callee, es->loop_depth);
    4621      4185191 :   if (info->call_size_time_table.length ())
    4622              :     {
    4623       892763 :       int edge_size = 0;
    4624       892763 :       sreal edge_time = 0;
    4625              : 
    4626       892763 :       estimate_edge_size_and_time (edge, &edge_size, NULL, &edge_time, NULL, 0);
    4627              :       /* Unaccount size and time of the optimized out call.  */
    4628       892763 :       info->account_size_time (-edge_size, -edge_time,
    4629       892763 :                                es->predicate ? *es->predicate : true,
    4630       892763 :                                es->predicate ? *es->predicate : true,
    4631              :                                true);
    4632              :       /* Account new calls.  */
    4633       892763 :       summarize_calls_size_and_time (edge->callee, info);
    4634              :     }
    4635              : 
    4636              :   /* Free summaries that are not maintained for inline clones/edges.  */
    4637      4185191 :   ipa_call_summaries->remove (edge);
    4638      4185191 :   ipa_fn_summaries->remove (edge->callee);
    4639      4185191 :   ipa_remove_from_growth_caches (edge);
    4640      4185191 : }
    4641              : 
    4642              : /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
    4643              :    overall size and time.  Recompute it.
    4644              :    If RESET is true also recompute call_time_size_table.  */
    4645              : 
    4646              : void
    4647      9776806 : ipa_update_overall_fn_summary (struct cgraph_node *node, bool reset)
    4648              : {
    4649      9776806 :   class ipa_fn_summary *info = ipa_fn_summaries->get (node);
    4650      9776806 :   class ipa_size_summary *size_info = ipa_size_summaries->get (node);
    4651      9776806 :   size_time_entry *e;
    4652      9776806 :   int i;
    4653              : 
    4654      9776806 :   size_info->size = 0;
    4655      9776806 :   info->time = 0;
    4656     49505031 :   for (i = 0; info->size_time_table.iterate (i, &e); i++)
    4657              :     {
    4658     39728225 :       size_info->size += e->size;
    4659     39728225 :       info->time += e->time;
    4660              :     }
    4661      9776806 :   info->min_size = info->size_time_table[0].size;
    4662      9776806 :   if (reset)
    4663      8840453 :     info->call_size_time_table.release ();
    4664      9776806 :   if (node->callees || node->indirect_calls)
    4665      7443231 :     estimate_calls_size_and_time (node, &size_info->size, &info->min_size,
    4666              :                                   &info->time, NULL,
    4667              :                                   ~(clause_t) (1 << ipa_predicate::false_condition),
    4668              :                                   NULL);
    4669      9776806 :   size_info->size = RDIV (size_info->size, ipa_fn_summary::size_scale);
    4670      9776806 :   info->min_size = RDIV (info->min_size, ipa_fn_summary::size_scale);
    4671      9776806 : }
    4672              : 
    4673              : 
    4674              : /* This function performs intraprocedural analysis in NODE that is required to
    4675              :    inline indirect calls.  */
    4676              : 
    4677              : static void
    4678      1415883 : inline_indirect_intraprocedural_analysis (struct cgraph_node *node)
    4679              : {
    4680      1415883 :   ipa_analyze_node (node);
    4681      1415883 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4682              :     {
    4683           16 :       ipa_print_node_params (dump_file, node);
    4684           16 :       ipa_print_node_jump_functions (dump_file, node);
    4685              :     }
    4686      1415883 : }
    4687              : 
    4688              : 
    4689              : /* Note function body size.  */
    4690              : 
    4691              : void
    4692      1426686 : inline_analyze_function (struct cgraph_node *node)
    4693              : {
    4694      1426686 :   push_cfun (DECL_STRUCT_FUNCTION (node->decl));
    4695              : 
    4696      1426686 :   if (dump_file)
    4697           96 :     fprintf (dump_file, "\nAnalyzing function: %s\n", node->dump_name ());
    4698      1426686 :   if (opt_for_fn (node->decl, optimize) && !node->thunk)
    4699      1415883 :     inline_indirect_intraprocedural_analysis (node);
    4700      1426686 :   compute_fn_summary (node, false);
    4701      1426686 :   if (!optimize)
    4702              :     {
    4703         9467 :       struct cgraph_edge *e;
    4704        22238 :       for (e = node->callees; e; e = e->next_callee)
    4705        12771 :         e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
    4706         9541 :       for (e = node->indirect_calls; e; e = e->next_callee)
    4707           74 :         e->inline_failed = CIF_FUNCTION_NOT_OPTIMIZED;
    4708              :     }
    4709              : 
    4710      1426686 :   pop_cfun ();
    4711      1426686 : }
    4712              : 
    4713              : 
    4714              : /* Called when new function is inserted to callgraph late.  */
    4715              : 
    4716              : void
    4717        20092 : ipa_fn_summary_t::insert (struct cgraph_node *node, ipa_fn_summary *)
    4718              : {
    4719        20092 :   inline_analyze_function (node);
    4720        20092 : }
    4721              : 
    4722              : /* Note function body size.  */
    4723              : 
    4724              : static void
    4725       238918 : ipa_fn_summary_generate (void)
    4726              : {
    4727       238918 :   struct cgraph_node *node;
    4728              : 
    4729      2160704 :   FOR_EACH_DEFINED_FUNCTION (node)
    4730      1921786 :     if (DECL_STRUCT_FUNCTION (node->decl))
    4731      1907972 :       node->versionable = tree_versionable_function_p (node->decl);
    4732              : 
    4733       238918 :   ipa_fn_summary_alloc ();
    4734              : 
    4735       238918 :   ipa_fn_summaries->enable_insertion_hook ();
    4736              : 
    4737       238918 :   ipa_register_cgraph_hooks ();
    4738              : 
    4739      2160704 :   FOR_EACH_DEFINED_FUNCTION (node)
    4740      1921786 :     if (!node->alias
    4741      1921786 :         && (flag_generate_lto || flag_generate_offload|| flag_wpa
    4742      1725870 :             || opt_for_fn (node->decl, optimize)))
    4743      1387698 :       inline_analyze_function (node);
    4744       238918 : }
    4745              : 
    4746              : 
    4747              : /* Write inline summary for edge E to OB.  */
    4748              : 
    4749              : static void
    4750       350581 : read_ipa_call_summary (class lto_input_block *ib, struct cgraph_edge *e,
    4751              :                        bool prevails)
    4752              : {
    4753       350581 :   class ipa_call_summary *es = prevails
    4754       350581 :                                 ? ipa_call_summaries->get_create (e) : NULL;
    4755       350581 :   ipa_predicate p;
    4756       350581 :   int length, i;
    4757              : 
    4758       350581 :   int size = streamer_read_uhwi (ib);
    4759       350581 :   int time = streamer_read_uhwi (ib);
    4760       350581 :   int depth = streamer_read_uhwi (ib);
    4761              : 
    4762       350581 :   if (es)
    4763              :     {
    4764       350532 :       es->call_stmt_size = size;
    4765       350532 :       es->call_stmt_time = time;
    4766       350532 :       es->loop_depth = depth;
    4767              :     }
    4768              : 
    4769       350581 :   bitpack_d bp = streamer_read_bitpack (ib);
    4770       350581 :   if (es)
    4771       350532 :     es->is_return_callee_uncaptured = bp_unpack_value (&bp, 1);
    4772              :   else
    4773           49 :     bp_unpack_value (&bp, 1);
    4774              : 
    4775       350581 :   p.stream_in (ib);
    4776       350581 :   if (es)
    4777       350532 :     edge_set_predicate (e, &p);
    4778       350581 :   length = streamer_read_uhwi (ib);
    4779       350581 :   if (length && es
    4780       350581 :       && (e->possibly_call_in_translation_unit_p ()
    4781              :           /* Also stream in jump functions to builtins in hope that they
    4782              :              will get fnspecs.  */
    4783       119553 :           || fndecl_built_in_p (e->callee->decl, BUILT_IN_NORMAL)))
    4784              :     {
    4785       234428 :       es->param.safe_grow_cleared (length, true);
    4786      1040909 :       for (i = 0; i < length; i++)
    4787              :         {
    4788       572053 :           es->param[i].change_prob = streamer_read_uhwi (ib);
    4789       572053 :           bitpack_d bp = streamer_read_bitpack (ib);
    4790      1716159 :           es->param[i].points_to_local_or_readonly_memory
    4791       572053 :             = bp_unpack_value (&bp, 1);
    4792      1716159 :           es->param[i].points_to_possible_sra_candidate
    4793       572053 :             = bp_unpack_value (&bp, 1);
    4794              :         }
    4795              :     }
    4796              :   else
    4797              :     {
    4798       139316 :       for (i = 0; i < length; i++)
    4799              :         {
    4800        23163 :           streamer_read_uhwi (ib);
    4801        23163 :           streamer_read_uhwi (ib);
    4802              :         }
    4803              :     }
    4804       350581 : }
    4805              : 
    4806              : 
    4807              : /* Stream in inline summaries from the section.  */
    4808              : 
    4809              : static void
    4810        13896 : inline_read_section (struct lto_file_decl_data *file_data, const char *data,
    4811              :                      size_t len)
    4812              : {
    4813        13896 :   const struct lto_function_header *header =
    4814              :     (const struct lto_function_header *) data;
    4815        13896 :   const int cfg_offset = sizeof (struct lto_function_header);
    4816        13896 :   const int main_offset = cfg_offset + header->cfg_size;
    4817        13896 :   const int string_offset = main_offset + header->main_size;
    4818        13896 :   class data_in *data_in;
    4819        13896 :   unsigned int i, count2, j;
    4820        13896 :   unsigned int f_count;
    4821              : 
    4822        13896 :   lto_input_block ib ((const char *) data + main_offset, header->main_size,
    4823        13896 :                       file_data);
    4824              : 
    4825        13896 :   data_in =
    4826        27792 :     lto_data_in_create (file_data, (const char *) data + string_offset,
    4827        13896 :                         header->string_size, vNULL);
    4828        13896 :   f_count = streamer_read_uhwi (&ib);
    4829       102191 :   for (i = 0; i < f_count; i++)
    4830              :     {
    4831        88295 :       unsigned int index;
    4832        88295 :       struct cgraph_node *node;
    4833        88295 :       class ipa_fn_summary *info;
    4834        88295 :       class ipa_node_params *params_summary;
    4835        88295 :       class ipa_size_summary *size_info;
    4836        88295 :       lto_symtab_encoder_t encoder;
    4837        88295 :       struct bitpack_d bp;
    4838        88295 :       struct cgraph_edge *e;
    4839        88295 :       ipa_predicate p;
    4840              : 
    4841        88295 :       index = streamer_read_uhwi (&ib);
    4842        88295 :       encoder = file_data->symtab_node_encoder;
    4843        88295 :       node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
    4844              :                                                                 index));
    4845        88295 :       info = node->prevailing_p () ? ipa_fn_summaries->get_create (node) : NULL;
    4846        88295 :       params_summary = node->prevailing_p ()
    4847        88295 :                        ? ipa_node_params_sum->get (node) : NULL;
    4848        88295 :       size_info = node->prevailing_p ()
    4849        88295 :                   ? ipa_size_summaries->get_create (node) : NULL;
    4850              : 
    4851        88295 :       int stack_size = streamer_read_uhwi (&ib);
    4852        88295 :       int size = streamer_read_uhwi (&ib);
    4853        88295 :       sreal time = sreal::stream_in (&ib);
    4854              : 
    4855        88295 :       if (info)
    4856              :         {
    4857        88237 :           info->estimated_stack_size
    4858        88237 :             = size_info->estimated_self_stack_size = stack_size;
    4859        88237 :           size_info->size = size_info->self_size = size;
    4860        88237 :           info->time = time;
    4861              :         }
    4862              : 
    4863        88295 :       bp = streamer_read_bitpack (&ib);
    4864        88295 :       if (info)
    4865              :         {
    4866        88237 :           info->inlinable = bp_unpack_value (&bp, 1);
    4867        88237 :           info->fp_expressions = bp_unpack_value (&bp, 1);
    4868        88237 :           if (!lto_stream_offload_p)
    4869        88237 :             info->target_info = streamer_read_uhwi (&ib);
    4870              :         }
    4871              :       else
    4872              :         {
    4873           58 :           bp_unpack_value (&bp, 1);
    4874           58 :           bp_unpack_value (&bp, 1);
    4875           58 :           if (!lto_stream_offload_p)
    4876           58 :             streamer_read_uhwi (&ib);
    4877              :         }
    4878              : 
    4879        88295 :       count2 = streamer_read_uhwi (&ib);
    4880        88295 :       gcc_assert (!info || !info->conds);
    4881        88237 :       if (info)
    4882        88237 :         vec_safe_reserve_exact (info->conds, count2);
    4883       167677 :       for (j = 0; j < count2; j++)
    4884              :         {
    4885        79382 :           struct condition c;
    4886        79382 :           unsigned int k, count3;
    4887        79382 :           c.operand_num = streamer_read_uhwi (&ib);
    4888        79382 :           c.code = (enum tree_code) streamer_read_uhwi (&ib);
    4889        79382 :           c.type = stream_read_tree (&ib, data_in);
    4890        79382 :           c.val = stream_read_tree (&ib, data_in);
    4891        79382 :           bp = streamer_read_bitpack (&ib);
    4892        79382 :           c.agg_contents = bp_unpack_value (&bp, 1);
    4893        79382 :           c.by_ref = bp_unpack_value (&bp, 1);
    4894        79382 :           if (c.agg_contents)
    4895        12683 :             c.offset = streamer_read_uhwi (&ib);
    4896        79382 :           count3 = streamer_read_uhwi (&ib);
    4897        79382 :           c.param_ops = NULL;
    4898        79382 :           if (info)
    4899        79382 :             vec_safe_reserve_exact (c.param_ops, count3);
    4900        79382 :           if (params_summary)
    4901        79382 :             ipa_set_param_used_by_ipa_predicates
    4902        79382 :                     (params_summary, c.operand_num, true);
    4903        83187 :           for (k = 0; k < count3; k++)
    4904              :             {
    4905         3805 :               struct expr_eval_op op;
    4906         3805 :               enum gimple_rhs_class rhs_class;
    4907         3805 :               op.code = (enum tree_code) streamer_read_uhwi (&ib);
    4908         3805 :               op.type = stream_read_tree (&ib, data_in);
    4909         3805 :               switch (rhs_class = get_gimple_rhs_class (op.code))
    4910              :                 {
    4911         1577 :                 case GIMPLE_UNARY_RHS:
    4912         1577 :                   op.index = 0;
    4913         1577 :                   op.val[0] = NULL_TREE;
    4914         1577 :                   op.val[1] = NULL_TREE;
    4915         1577 :                   break;
    4916              : 
    4917         2228 :                 case GIMPLE_BINARY_RHS:
    4918         2228 :                 case GIMPLE_TERNARY_RHS:
    4919         2228 :                   bp = streamer_read_bitpack (&ib);
    4920         2228 :                   op.index = bp_unpack_value (&bp, 2);
    4921         2228 :                   op.val[0] = stream_read_tree (&ib, data_in);
    4922         2228 :                   if (rhs_class == GIMPLE_BINARY_RHS)
    4923         2228 :                     op.val[1] = NULL_TREE;
    4924              :                   else
    4925            0 :                     op.val[1] = stream_read_tree (&ib, data_in);
    4926              :                   break;
    4927              : 
    4928            0 :                 default:
    4929            0 :                   fatal_error (UNKNOWN_LOCATION,
    4930              :                                "invalid fnsummary in LTO stream");
    4931              :                 }
    4932         3805 :               if (info)
    4933         3805 :                 c.param_ops->quick_push (op);
    4934              :             }
    4935        79382 :           if (info)
    4936        79382 :             info->conds->quick_push (c);
    4937              :         }
    4938        88295 :       count2 = streamer_read_uhwi (&ib);
    4939        88295 :       gcc_assert (!info || !info->size_time_table.length ());
    4940        88295 :       if (info && count2)
    4941        88237 :         info->size_time_table.reserve_exact (count2);
    4942       336110 :       for (j = 0; j < count2; j++)
    4943              :         {
    4944       247815 :           class size_time_entry e;
    4945              : 
    4946       247815 :           e.size = streamer_read_uhwi (&ib);
    4947       247815 :           e.time = sreal::stream_in (&ib);
    4948       247815 :           e.exec_predicate.stream_in (&ib);
    4949       247815 :           e.nonconst_predicate.stream_in (&ib);
    4950              : 
    4951       247815 :           if (info)
    4952       247699 :             info->size_time_table.quick_push (e);
    4953              :         }
    4954              : 
    4955        88295 :       count2 = streamer_read_uhwi (&ib);
    4956        89816 :       for (j = 0; j < count2; j++)
    4957              :         {
    4958         1521 :           p.stream_in (&ib);
    4959         1521 :           sreal fcp_freq = sreal::stream_in (&ib);
    4960         1521 :           if (info)
    4961              :             {
    4962         1521 :               ipa_freqcounting_predicate fcp;
    4963         1521 :               fcp.predicate = NULL;
    4964         1521 :               set_hint_predicate (&fcp.predicate, p);
    4965         1521 :               fcp.freq = fcp_freq;
    4966         1521 :               vec_safe_push (info->loop_iterations, fcp);
    4967              :             }
    4968              :         }
    4969        88295 :       count2 = streamer_read_uhwi (&ib);
    4970        88585 :       for (j = 0; j < count2; j++)
    4971              :         {
    4972          290 :           p.stream_in (&ib);
    4973          290 :           sreal fcp_freq = sreal::stream_in (&ib);
    4974          290 :           if (info)
    4975              :             {
    4976          290 :               ipa_freqcounting_predicate fcp;
    4977          290 :               fcp.predicate = NULL;
    4978          290 :               set_hint_predicate (&fcp.predicate, p);
    4979          290 :               fcp.freq = fcp_freq;
    4980          290 :               vec_safe_push (info->loop_strides, fcp);
    4981              :             }
    4982              :         }
    4983        88295 :       count2 = streamer_read_uhwi (&ib);
    4984        88295 :       if (info && count2)
    4985            8 :         info->builtin_constant_p_parms.reserve_exact (count2);
    4986        88303 :       for (j = 0; j < count2; j++)
    4987              :         {
    4988            8 :           int parm = streamer_read_uhwi (&ib);
    4989            8 :           if (info)
    4990            8 :             info->builtin_constant_p_parms.quick_push (parm);
    4991              :         }
    4992       437373 :       for (e = node->callees; e; e = e->next_callee)
    4993       349078 :         read_ipa_call_summary (&ib, e, info != NULL);
    4994        89798 :       for (e = node->indirect_calls; e; e = e->next_callee)
    4995         1503 :         read_ipa_call_summary (&ib, e, info != NULL);
    4996              :     }
    4997              : 
    4998        13896 :   lto_free_section_data (file_data, LTO_section_ipa_fn_summary, NULL, data,
    4999              :                          len);
    5000        13896 :   lto_data_in_delete (data_in);
    5001        13896 : }
    5002              : 
    5003              : 
    5004              : /* Read inline summary.  Jump functions are shared among ipa-cp
    5005              :    and inliner, so when ipa-cp is active, we don't need to write them
    5006              :    twice.  */
    5007              : 
    5008              : static void
    5009        12782 : ipa_fn_summary_read (void)
    5010              : {
    5011        12782 :   struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
    5012        12782 :   struct lto_file_decl_data *file_data;
    5013        12782 :   unsigned int j = 0;
    5014              : 
    5015        12782 :   ipa_prop_read_jump_functions ();
    5016        12782 :   ipa_fn_summary_alloc ();
    5017              : 
    5018        39460 :   while ((file_data = file_data_vec[j++]))
    5019              :     {
    5020        13896 :       size_t len;
    5021        13896 :       const char *data
    5022        13896 :         = lto_get_summary_section_data (file_data, LTO_section_ipa_fn_summary,
    5023              :                                         &len);
    5024        13896 :       if (data)
    5025        13896 :         inline_read_section (file_data, data, len);
    5026              :       else
    5027              :         /* Fatal error here.  We do not want to support compiling ltrans units
    5028              :            with different version of compiler or different flags than the WPA
    5029              :            unit, so this should never happen.  */
    5030            0 :         fatal_error (input_location,
    5031              :                      "ipa inline summary is missing in input file");
    5032              :     }
    5033        12782 :   ipa_register_cgraph_hooks ();
    5034              : 
    5035        12782 :   gcc_assert (ipa_fn_summaries);
    5036        12782 :   ipa_fn_summaries->enable_insertion_hook ();
    5037        12782 : }
    5038              : 
    5039              : 
    5040              : /* Write inline summary for edge E to OB.  */
    5041              : 
    5042              : static void
    5043       382410 : write_ipa_call_summary (struct output_block *ob, struct cgraph_edge *e)
    5044              : {
    5045       382410 :   class ipa_call_summary *es = ipa_call_summaries->get (e);
    5046       382410 :   int i;
    5047              : 
    5048       382410 :   streamer_write_uhwi (ob, es->call_stmt_size);
    5049       382410 :   streamer_write_uhwi (ob, es->call_stmt_time);
    5050       382410 :   streamer_write_uhwi (ob, es->loop_depth);
    5051              : 
    5052       382410 :   bitpack_d bp = bitpack_create (ob->main_stream);
    5053       382410 :   bp_pack_value (&bp, es->is_return_callee_uncaptured, 1);
    5054       382410 :   streamer_write_bitpack (&bp);
    5055              : 
    5056       382410 :   if (es->predicate)
    5057        10821 :     es->predicate->stream_out (ob);
    5058              :   else
    5059       371589 :     streamer_write_uhwi (ob, 0);
    5060       382410 :   streamer_write_uhwi (ob, es->param.length ());
    5061      2307327 :   for (i = 0; i < (int) es->param.length (); i++)
    5062              :     {
    5063       636043 :       streamer_write_uhwi (ob, es->param[i].change_prob);
    5064       636043 :       bp = bitpack_create (ob->main_stream);
    5065       636043 :       bp_pack_value (&bp, es->param[i].points_to_local_or_readonly_memory, 1);
    5066       636043 :       bp_pack_value (&bp, es->param[i].points_to_possible_sra_candidate, 1);
    5067       636043 :       streamer_write_bitpack (&bp);
    5068              :     }
    5069       382410 : }
    5070              : 
    5071              : 
    5072              : /* Write inline summary for node in SET.
    5073              :    Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
    5074              :    active, we don't need to write them twice.  */
    5075              : 
    5076              : static void
    5077        23943 : ipa_fn_summary_write (void)
    5078              : {
    5079        23943 :   struct output_block *ob = create_output_block (LTO_section_ipa_fn_summary);
    5080        23943 :   lto_symtab_encoder_iterator lsei;
    5081        23943 :   lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
    5082        23943 :   unsigned int count = 0;
    5083              : 
    5084       131446 :   for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
    5085       107503 :        lsei_next_function_in_partition (&lsei))
    5086              :     {
    5087       107503 :       cgraph_node *cnode = lsei_cgraph_node (lsei);
    5088       107503 :       if (cnode->definition && !cnode->alias)
    5089       105023 :         count++;
    5090              :     }
    5091        23943 :   streamer_write_uhwi (ob, count);
    5092              : 
    5093       131446 :   for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
    5094       107503 :        lsei_next_function_in_partition (&lsei))
    5095              :     {
    5096       107503 :       cgraph_node *cnode = lsei_cgraph_node (lsei);
    5097       107503 :       if (cnode->definition && !cnode->alias)
    5098              :         {
    5099       105023 :           class ipa_fn_summary *info = ipa_fn_summaries->get (cnode);
    5100       105023 :           class ipa_size_summary *size_info = ipa_size_summaries->get (cnode);
    5101       105023 :           struct bitpack_d bp;
    5102       105023 :           struct cgraph_edge *edge;
    5103       105023 :           int i;
    5104       105023 :           size_time_entry *e;
    5105       105023 :           struct condition *c;
    5106              : 
    5107       105023 :           streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode));
    5108       105023 :           streamer_write_hwi (ob, size_info->estimated_self_stack_size);
    5109       105023 :           streamer_write_hwi (ob, size_info->self_size);
    5110       105023 :           info->time.stream_out (ob);
    5111       105023 :           bp = bitpack_create (ob->main_stream);
    5112       105023 :           bp_pack_value (&bp, info->inlinable, 1);
    5113       105023 :           bp_pack_value (&bp, info->fp_expressions, 1);
    5114       105023 :           streamer_write_bitpack (&bp);
    5115       105023 :           if (!lto_stream_offload_p)
    5116       105023 :             streamer_write_uhwi (ob, info->target_info);
    5117       105023 :           streamer_write_uhwi (ob, vec_safe_length (info->conds));
    5118       305675 :           for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
    5119              :             {
    5120        95629 :               int j;
    5121        95629 :               struct expr_eval_op *op;
    5122              : 
    5123        95629 :               streamer_write_uhwi (ob, c->operand_num);
    5124        95629 :               streamer_write_uhwi (ob, c->code);
    5125        95629 :               stream_write_tree (ob, c->type, true);
    5126        95629 :               stream_write_tree (ob, c->val, true);
    5127        95629 :               bp = bitpack_create (ob->main_stream);
    5128        95629 :               bp_pack_value (&bp, c->agg_contents, 1);
    5129        95629 :               bp_pack_value (&bp, c->by_ref, 1);
    5130        95629 :               streamer_write_bitpack (&bp);
    5131        95629 :               if (c->agg_contents)
    5132        15956 :                 streamer_write_uhwi (ob, c->offset);
    5133        95629 :               streamer_write_uhwi (ob, vec_safe_length (c->param_ops));
    5134       198809 :               for (j = 0; vec_safe_iterate (c->param_ops, j, &op); j++)
    5135              :                 {
    5136         4467 :                   streamer_write_uhwi (ob, op->code);
    5137         4467 :                   stream_write_tree (ob, op->type, true);
    5138         4467 :                   if (op->val[0])
    5139              :                     {
    5140         2616 :                       bp = bitpack_create (ob->main_stream);
    5141         2616 :                       bp_pack_value (&bp, op->index, 2);
    5142         2616 :                       streamer_write_bitpack (&bp);
    5143         2616 :                       stream_write_tree (ob, op->val[0], true);
    5144         2616 :                       if (op->val[1])
    5145            4 :                         stream_write_tree (ob, op->val[1], true);
    5146              :                     }
    5147              :                 }
    5148              :             }
    5149       105023 :           streamer_write_uhwi (ob, info->size_time_table.length ());
    5150       508387 :           for (i = 0; info->size_time_table.iterate (i, &e); i++)
    5151              :             {
    5152       298341 :               streamer_write_uhwi (ob, e->size);
    5153       298341 :               e->time.stream_out (ob);
    5154       298341 :               e->exec_predicate.stream_out (ob);
    5155       298341 :               e->nonconst_predicate.stream_out (ob);
    5156              :             }
    5157       105023 :           ipa_freqcounting_predicate *fcp;
    5158       105023 :           streamer_write_uhwi (ob, vec_safe_length (info->loop_iterations));
    5159       212257 :           for (i = 0; vec_safe_iterate (info->loop_iterations, i, &fcp); i++)
    5160              :             {
    5161         2211 :               fcp->predicate->stream_out (ob);
    5162         2211 :               fcp->freq.stream_out (ob);
    5163              :             }
    5164       105023 :           streamer_write_uhwi (ob, vec_safe_length (info->loop_strides));
    5165       210392 :           for (i = 0; vec_safe_iterate (info->loop_strides, i, &fcp); i++)
    5166              :             {
    5167          346 :               fcp->predicate->stream_out (ob);
    5168          346 :               fcp->freq.stream_out (ob);
    5169              :             }
    5170       105023 :           streamer_write_uhwi (ob, info->builtin_constant_p_parms.length ());
    5171       105023 :           int ip;
    5172       210062 :           for (i = 0; info->builtin_constant_p_parms.iterate (i, &ip);
    5173              :                i++)
    5174           16 :             streamer_write_uhwi (ob, ip);
    5175       484722 :           for (edge = cnode->callees; edge; edge = edge->next_callee)
    5176       379699 :             write_ipa_call_summary (ob, edge);
    5177       107734 :           for (edge = cnode->indirect_calls; edge; edge = edge->next_callee)
    5178         2711 :             write_ipa_call_summary (ob, edge);
    5179              :         }
    5180              :     }
    5181        23943 :   streamer_write_char_stream (ob->main_stream, 0);
    5182        23943 :   produce_asm (ob);
    5183        23943 :   destroy_output_block (ob);
    5184              : 
    5185        23943 :   ipa_prop_write_jump_functions ();
    5186        23943 : }
    5187              : 
    5188              : 
    5189              : /* Release function summary.  */
    5190              : 
    5191              : void
    5192       746266 : ipa_free_fn_summary (void)
    5193              : {
    5194       746266 :   if (!ipa_call_summaries)
    5195              :     return;
    5196       472278 :   ggc_delete (ipa_fn_summaries);
    5197       472278 :   ipa_fn_summaries = NULL;
    5198       472278 :   delete ipa_call_summaries;
    5199       472278 :   ipa_call_summaries = NULL;
    5200       472278 :   edge_predicate_pool.release ();
    5201              :   /* During IPA this is one of largest datastructures to release.  */
    5202       472278 :   if (flag_wpa)
    5203         8147 :     ggc_trim ();
    5204              : }
    5205              : 
    5206              : /* Release function summary.  */
    5207              : 
    5208              : void
    5209       746266 : ipa_free_size_summary (void)
    5210              : {
    5211       746266 :   if (!ipa_size_summaries)
    5212              :     return;
    5213       472278 :   delete ipa_size_summaries;
    5214       472278 :   ipa_size_summaries = NULL;
    5215              : }
    5216              : 
    5217              : namespace {
    5218              : 
    5219              : const pass_data pass_data_local_fn_summary =
    5220              : {
    5221              :   GIMPLE_PASS, /* type */
    5222              :   "local-fnsummary", /* name */
    5223              :   OPTGROUP_INLINE, /* optinfo_flags */
    5224              :   TV_INLINE_PARAMETERS, /* tv_id */
    5225              :   0, /* properties_required */
    5226              :   0, /* properties_provided */
    5227              :   0, /* properties_destroyed */
    5228              :   0, /* todo_flags_start */
    5229              :   0, /* todo_flags_finish */
    5230              : };
    5231              : 
    5232              : class pass_local_fn_summary : public gimple_opt_pass
    5233              : {
    5234              : public:
    5235       594360 :   pass_local_fn_summary (gcc::context *ctxt)
    5236      1188720 :     : gimple_opt_pass (pass_data_local_fn_summary, ctxt)
    5237              :   {}
    5238              : 
    5239              :   /* opt_pass methods: */
    5240       297180 :   opt_pass * clone () final override
    5241              :   {
    5242       297180 :     return new pass_local_fn_summary (m_ctxt);
    5243              :   }
    5244      5972550 :   unsigned int execute (function *) final override
    5245              :     {
    5246      5972550 :       return compute_fn_summary_for_current ();
    5247              :     }
    5248              : 
    5249              : }; // class pass_local_fn_summary
    5250              : 
    5251              : } // anon namespace
    5252              : 
    5253              : gimple_opt_pass *
    5254       297180 : make_pass_local_fn_summary (gcc::context *ctxt)
    5255              : {
    5256       297180 :   return new pass_local_fn_summary (ctxt);
    5257              : }
    5258              : 
    5259              : 
    5260              : /* Free inline summary.  */
    5261              : 
    5262              : namespace {
    5263              : 
    5264              : const pass_data pass_data_ipa_free_fn_summary =
    5265              : {
    5266              :   SIMPLE_IPA_PASS, /* type */
    5267              :   "free-fnsummary", /* name */
    5268              :   OPTGROUP_NONE, /* optinfo_flags */
    5269              :   TV_IPA_FREE_INLINE_SUMMARY, /* tv_id */
    5270              :   0, /* properties_required */
    5271              :   0, /* properties_provided */
    5272              :   0, /* properties_destroyed */
    5273              :   0, /* todo_flags_start */
    5274              :   0, /* todo_flags_finish */
    5275              : };
    5276              : 
    5277              : class pass_ipa_free_fn_summary : public simple_ipa_opt_pass
    5278              : {
    5279              : public:
    5280       594360 :   pass_ipa_free_fn_summary (gcc::context *ctxt)
    5281              :     : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary, ctxt),
    5282      1188720 :       small_p (false)
    5283              :   {}
    5284              : 
    5285              :   /* opt_pass methods: */
    5286       297180 :   opt_pass *clone () final override
    5287              :   {
    5288       297180 :     return new pass_ipa_free_fn_summary (m_ctxt);
    5289              :   }
    5290       594360 :   void set_pass_param (unsigned int n, bool param) final override
    5291              :     {
    5292       594360 :       gcc_assert (n == 0);
    5293       594360 :       small_p = param;
    5294       594360 :     }
    5295       499756 :   bool gate (function *) final override { return true; }
    5296       478426 :   unsigned int execute (function *) final override
    5297              :     {
    5298       478426 :       ipa_free_fn_summary ();
    5299              :       /* Free ipa-prop structures if they are no longer needed.  */
    5300       478426 :       ipa_free_all_structures_after_iinln ();
    5301       478426 :       if (!flag_wpa)
    5302       470279 :         ipa_free_size_summary ();
    5303       478426 :       return 0;
    5304              :     }
    5305              : 
    5306              : private:
    5307              :   bool small_p;
    5308              : }; // class pass_ipa_free_fn_summary
    5309              : 
    5310              : } // anon namespace
    5311              : 
    5312              : simple_ipa_opt_pass *
    5313       297180 : make_pass_ipa_free_fn_summary (gcc::context *ctxt)
    5314              : {
    5315       297180 :   return new pass_ipa_free_fn_summary (ctxt);
    5316              : }
    5317              : 
    5318              : namespace {
    5319              : 
    5320              : const pass_data pass_data_ipa_fn_summary =
    5321              : {
    5322              :   IPA_PASS, /* type */
    5323              :   "fnsummary", /* name */
    5324              :   OPTGROUP_INLINE, /* optinfo_flags */
    5325              :   TV_IPA_FNSUMMARY, /* tv_id */
    5326              :   0, /* properties_required */
    5327              :   0, /* properties_provided */
    5328              :   0, /* properties_destroyed */
    5329              :   0, /* todo_flags_start */
    5330              :   ( TODO_dump_symtab ), /* todo_flags_finish */
    5331              : };
    5332              : 
    5333              : class pass_ipa_fn_summary : public ipa_opt_pass_d
    5334              : {
    5335              : public:
    5336       297180 :   pass_ipa_fn_summary (gcc::context *ctxt)
    5337              :     : ipa_opt_pass_d (pass_data_ipa_fn_summary, ctxt,
    5338              :                       ipa_fn_summary_generate, /* generate_summary */
    5339              :                       ipa_fn_summary_write, /* write_summary */
    5340              :                       ipa_fn_summary_read, /* read_summary */
    5341              :                       NULL, /* write_optimization_summary */
    5342              :                       NULL, /* read_optimization_summary */
    5343              :                       NULL, /* stmt_fixup */
    5344              :                       0, /* function_transform_todo_flags_start */
    5345              :                       NULL, /* function_transform */
    5346       297180 :                       NULL) /* variable_transform */
    5347       297180 :   {}
    5348              : 
    5349              :   /* opt_pass methods: */
    5350       239358 :   unsigned int execute (function *) final override { return 0; }
    5351              : 
    5352              : }; // class pass_ipa_fn_summary
    5353              : 
    5354              : } // anon namespace
    5355              : 
    5356              : ipa_opt_pass_d *
    5357       297180 : make_pass_ipa_fn_summary (gcc::context *ctxt)
    5358              : {
    5359       297180 :   return new pass_ipa_fn_summary (ctxt);
    5360              : }
    5361              : 
    5362              : /* Reset all state within ipa-fnsummary.cc so that we can rerun the compiler
    5363              :    within the same process.  For use by toplev::finalize.  */
    5364              : 
    5365              : void
    5366       266934 : ipa_fnsummary_cc_finalize (void)
    5367              : {
    5368       266934 :   ipa_free_fn_summary ();
    5369       266934 :   ipa_free_size_summary ();
    5370       266934 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.