LCOV - code coverage report
Current view: top level - gcc - ipa-sra.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 93.3 % 2243 2093
Test Date: 2026-11-14 16:39:50 Functions: 97.8 % 92 90
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Interprocedural scalar replacement of aggregates
       2              :    Copyright (C) 2019-2026 Free Software Foundation, Inc.
       3              :    Contributed by Martin Jambor <mjambor@suse.cz>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it under
       8              : the terms of the GNU General Public License as published by the Free
       9              : Software Foundation; either version 3, or (at your option) any later
      10              : version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : /* IPA-SRA is an interprocedural pass that removes unused function return
      22              :    values (turning functions returning a value which is never used into void
      23              :    functions) and removes unused function parameters.  It can also replace an
      24              :    aggregate parameter by a set of other parameters representing part of the
      25              :    original, turning those passed by reference into new ones which pass the
      26              :    value directly.
      27              : 
      28              :    The pass is a true IPA one, which means that it works in three stages in
      29              :    order to be able to take advantage of LTO.  First, summaries about functions
      30              :    and each calls are generated.  Function summaries (often called call graph
      31              :    node summaries) contain mainly information about which parameters are
      32              :    potential transformation candidates and which bits of candidates are
      33              :    accessed.  We differentiate between accesses done as a part of a call
      34              :    statement (which might be not necessary if the callee is also transformed)
      35              :    and others (which are mandatory).  Call summaries (often called call graph
      36              :    edge summaries) contain information about which function formal parameters
      37              :    feed into which actual call arguments so that if two parameters are only
      38              :    used in a sum which is then passed to another function which then however
      39              :    does not use this parameter, all three parameters of the two functions can
      40              :    be eliminated.  Edge summaries also have flags whether the return value is
      41              :    used or if it is only returned in the caller too.  In LTO mode these
      42              :    summaries are then streamed to the object file in the compilation phase and
      43              :    streamed back in in the WPA analysis stage.
      44              : 
      45              :    The interprocedural analysis phase traverses the graph in topological order
      46              :    in two sweeps, one in each direction.  First, from callees to callers for
      47              :    parameter removal and splitting.  Each strongly-connected component is
      48              :    processed iteratively until the situation in it stabilizes.  The pass from
      49              :    callers to callees is then carried out to remove unused return values in a
      50              :    very similar fashion.
      51              : 
      52              :    Because parameter manipulation has big implications for call redirection
      53              :    which is done only after all call graph nodes materialize, the
      54              :    transformation phase is not part of this patch but is carried out by the
      55              :    clone materialization and edge redirection itself, see comments in
      56              :    ipa-param-manipulation.h for more details.  */
      57              : 
      58              : 
      59              : #include "config.h"
      60              : #include "system.h"
      61              : #include "coretypes.h"
      62              : #include "backend.h"
      63              : #include "tree.h"
      64              : #include "gimple.h"
      65              : #include "predict.h"
      66              : #include "tree-pass.h"
      67              : #include "ssa.h"
      68              : #include "cgraph.h"
      69              : #include "gimple-pretty-print.h"
      70              : #include "alias.h"
      71              : #include "tree-eh.h"
      72              : #include "gimple-iterator.h"
      73              : #include "gimple-walk.h"
      74              : #include "tree-dfa.h"
      75              : #include "tree-sra.h"
      76              : #include "alloc-pool.h"
      77              : #include "symbol-summary.h"
      78              : #include "dbgcnt.h"
      79              : #include "tree-inline.h"
      80              : #include "ipa-utils.h"
      81              : #include "builtins.h"
      82              : #include "cfganal.h"
      83              : #include "tree-streamer.h"
      84              : #include "internal-fn.h"
      85              : #include "symtab-clones.h"
      86              : #include "attribs.h"
      87              : #include "sreal.h"
      88              : #include "ipa-cp.h"
      89              : #include "ipa-prop.h"
      90              : #include "attr-callback.h"
      91              : #include "callback-info.h"
      92              : 
      93              : static void ipa_sra_summarize_function (cgraph_node *);
      94              : 
      95              : /* Bits used to track size of an aggregate in bytes interprocedurally.  */
      96              : #define ISRA_ARG_SIZE_LIMIT_BITS 16
      97              : #define ISRA_ARG_SIZE_LIMIT (1 << ISRA_ARG_SIZE_LIMIT_BITS)
      98              : /* How many parameters can feed into a call actual argument and still be
      99              :    tracked.  */
     100              : #define IPA_SRA_MAX_PARAM_FLOW_LEN 7
     101              : 
     102              : /* Structure describing accesses to a specific portion of an aggregate
     103              :    parameter, as given by the offset and size.  Any smaller accesses that occur
     104              :    within a function that fall within another access form a tree.  The pass
     105              :    cannot analyze parameters with only partially overlapping accesses.  */
     106              : 
     107              : struct GTY(()) param_access
     108              : {
     109              :   /* Type that a potential replacement should have.  This field only has
     110              :      meaning in the summary building and transformation phases, when it is
     111              :      reconstructed from the body.  Must not be touched in IPA analysis
     112              :      stage.  */
     113              :   tree type;
     114              : 
     115              :   /* Alias reference type to be used in MEM_REFs when adjusting caller
     116              :      arguments.  */
     117              :   tree alias_ptr_type;
     118              : 
     119              :   /* Values returned by get_ref_base_and_extent but converted to bytes and
     120              :      stored as unsigned ints.  */
     121              :   unsigned unit_offset;
     122              :   unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
     123              : 
     124              :   /* Set once we are sure that the access will really end up in a potentially
     125              :      transformed function - initially not set for portions of formal parameters
     126              :      that are only used as actual function arguments passed to callees.  */
     127              :   unsigned certain : 1;
     128              :   /* Set if the access has reverse scalar storage order.  */
     129              :   unsigned reverse : 1;
     130              : };
     131              : 
     132              : /* This structure has the same purpose as the one above and additionally it
     133              :    contains some fields that are only necessary in the summary generation
     134              :    phase.  */
     135              : 
     136              : struct gensum_param_access
     137              : {
     138              :   /* Values returned by get_ref_base_and_extent.  */
     139              :   HOST_WIDE_INT offset;
     140              :   HOST_WIDE_INT size;
     141              : 
     142              :   /* if this access has any children (in terms of the definition above), this
     143              :      points to the first one.  */
     144              :   struct gensum_param_access *first_child;
     145              :   /* In intraprocedural SRA, pointer to the next sibling in the access tree as
     146              :      described above.  */
     147              :   struct gensum_param_access *next_sibling;
     148              : 
     149              :   /* Type that a potential replacement should have.  This field only has
     150              :      meaning in the summary building and transformation phases, when it is
     151              :      reconstructed from the body.  Must not be touched in IPA analysis
     152              :      stage.  */
     153              :   tree type;
     154              :   /* Alias reference type to be used in MEM_REFs when adjusting caller
     155              :      arguments.  */
     156              :   tree alias_ptr_type;
     157              : 
     158              :   /* Cumulative count of all loads. */
     159              :   profile_count load_count;
     160              :   /* Have there been writes to or reads from this exact location except for as
     161              :      arguments to a function call that can be tracked.  */
     162              :   bool nonarg;
     163              : 
     164              :   /* Set if the access has reverse scalar storage order.  */
     165              :   bool reverse;
     166              : };
     167              : 
     168              : /* Summary describing a parameter in the IPA stages.  */
     169              : 
     170              : struct GTY(()) isra_param_desc
     171              : {
     172              :   /* List of access representatives to the parameters, sorted according to
     173              :      their offset.  */
     174              :   vec <param_access *, va_gc> *accesses;
     175              : 
     176              :   /* Unit size limit of total size of all replacements.  */
     177              :   unsigned param_size_limit : ISRA_ARG_SIZE_LIMIT_BITS;
     178              :   /* Sum of unit sizes of all certain replacements.  */
     179              :   unsigned size_reached : ISRA_ARG_SIZE_LIMIT_BITS;
     180              :   /* Minimum offset that is known to be safe to dereference because of callers
     181              :      pass pointers to DECLs of at least this size or because of dereferences in
     182              :      callers.  */
     183              :   unsigned safe_size : ISRA_ARG_SIZE_LIMIT_BITS;
     184              : 
     185              :   /* A parameter that is used only in call arguments and can be removed if all
     186              :      concerned actual arguments are removed.  */
     187              :   unsigned locally_unused : 1;
     188              :   /* An aggregate that is a candidate for breaking up or complete removal.  */
     189              :   unsigned split_candidate : 1;
     190              :   /* Is this a parameter passing stuff by reference?  */
     191              :   unsigned by_ref : 1;
     192              :   /* If set, this parameter can only be a candidate for removal if the function
     193              :      is going to loose its return value.  */
     194              :   unsigned remove_only_when_retval_removed : 1;
     195              :   /* If set, this parameter can only be a candidate for splitting if the
     196              :      function is going to loose its return value.  Can only be meaningfully set
     197              :      for by_ref parameters.  */
     198              :   unsigned split_only_when_retval_removed : 1;
     199              :   /* Parameter hint set during IPA analysis when there is a caller which does
     200              :      not construct the argument just to pass it to calls.  Only meaningful for
     201              :      by_ref parameters.  */
     202              :   unsigned not_specially_constructed : 1;
     203              :   /* Only meaningful for by_ref parameters.  If set, this parameter can only be
     204              :      a split candidate if all callers pass pointers that are known to point to
     205              :      a chunk of memory large enough to contain all accesses.  */
     206              :   unsigned conditionally_dereferenceable : 1;
     207              :   /* Set when safe_size has been updated from at least one caller.  */
     208              :   unsigned safe_size_set : 1;
     209              : };
     210              : 
     211              : /* Structure used when generating summaries that describes a parameter.  */
     212              : 
     213              : struct gensum_param_desc
     214              : {
     215              :   /* Roots of param_accesses.  */
     216              :   gensum_param_access *accesses;
     217              :   /* Number of accesses in the access tree rooted in field accesses.  */
     218              :   unsigned access_count;
     219              : 
     220              :   /* If the below is non-zero, this is the number of uses as actual
     221              :      arguments.  */
     222              :   int call_uses;
     223              :   /* Number of times this parameter has been directly passed to.  */
     224              :   unsigned ptr_pt_count;
     225              : 
     226              :   /* Size limit of total size of all replacements.  */
     227              :   unsigned param_size_limit;
     228              :   /* Sum of sizes of nonarg accesses.  */
     229              :   unsigned nonarg_acc_size;
     230              : 
     231              :   /* A parameter that is used only in call arguments and can be removed if all
     232              :      concerned actual arguments are removed.  */
     233              :   bool locally_unused;
     234              :   /* An aggregate that is a candidate for breaking up or a pointer passing data
     235              :      by reference that is a candidate for being converted to a set of
     236              :      parameters passing those data by value.  */
     237              :   bool split_candidate;
     238              :   /* Is this a parameter passing stuff by reference (either a pointer or a
     239              :      source language reference type)?  */
     240              :   bool by_ref;
     241              :   /* If this parameter passes stuff by reference, can it be safely dereferenced
     242              :      without performing further checks (for example because it is a
     243              :      REFERENCE_TYPE)?  */
     244              :   bool safe_ref;
     245              :   /* If set, this parameter can only be a candidate for removal if the function
     246              :      is going to loose its return value.  */
     247              :   bool remove_only_when_retval_removed;
     248              :   /* If set, this parameter can only be a candidate for splitting if the
     249              :      function is going to loose its return value.  Can only be meaningfully set
     250              :      for by_ref parameters.  */
     251              :   bool split_only_when_retval_removed;
     252              :   /* Only meaningful for by_ref parameters.  If set, this parameter can only be
     253              :      a split candidate if all callers pass pointers that are known to point to
     254              :      a chunk of memory large enough to contain all accesses.  */
     255              :   bool conditionally_dereferenceable;
     256              : 
     257              :   /* The number of this parameter as they are ordered in function decl.  */
     258              :   int param_number;
     259              :   /* For parameters passing data by reference, this is parameter index to
     260              :      compute indices to bb_dereferences.  */
     261              :   int deref_index;
     262              : };
     263              : 
     264              : /* Properly deallocate accesses of DESC.  TODO: Since this data structure is
     265              :    allocated in GC memory, this is not necessary and we can consider removing
     266              :    the function.  */
     267              : 
     268              : static void
     269      2872725 : free_param_decl_accesses (isra_param_desc *desc)
     270              : {
     271      2872725 :   unsigned len = vec_safe_length (desc->accesses);
     272      3491073 :   for (unsigned i = 0; i < len; ++i)
     273       618348 :     ggc_free ((*desc->accesses)[i]);
     274      2872725 :   vec_free (desc->accesses);
     275      2872725 : }
     276              : 
     277              : /* Class used to convey information about functions from the
     278              :    intra-procedural analysis stage to inter-procedural one.  */
     279              : 
     280              : class GTY((for_user)) isra_func_summary
     281              : {
     282              : public:
     283              :   /* initialize the object.  */
     284              : 
     285      1244490 :   isra_func_summary ()
     286      1244490 :     : m_parameters (NULL), m_candidate (false), m_returns_value (false),
     287      1244490 :     m_return_ignored (false), m_queued (false)
     288              :   {}
     289              : 
     290              :   /* Destroy m_parameters.  */
     291              : 
     292              :   ~isra_func_summary ();
     293              : 
     294              :   /* Mark the function as not a candidate for any IPA-SRA transformation.
     295              :      Return true if it was a candidate until now.  */
     296              : 
     297              :   bool zap ();
     298              : 
     299              :   /* Vector of parameter descriptors corresponding to the function being
     300              :      analyzed.  */
     301              :   vec<isra_param_desc, va_gc> *m_parameters;
     302              : 
     303              :   /* Whether the node is even a candidate for any IPA-SRA transformation at
     304              :      all.  */
     305              :   unsigned m_candidate : 1;
     306              : 
     307              :   /* Whether the original function returns any value.  */
     308              :   unsigned m_returns_value : 1;
     309              : 
     310              :   /* Set to true if all call statements do not actually use the returned
     311              :      value.  */
     312              : 
     313              :   unsigned m_return_ignored : 1;
     314              : 
     315              :   /* Whether the node is already queued in IPA SRA stack during processing of
     316              :      call graphs SCCs.  */
     317              : 
     318              :   unsigned m_queued : 1;
     319              : };
     320              : 
     321              : /* Deallocate the memory pointed to by isra_func_summary.  TODO: Since this
     322              :    data structure is allocated in GC memory, this is not necessary and we can
     323              :    consider removing the destructor.  */
     324              : 
     325      1244489 : isra_func_summary::~isra_func_summary ()
     326              : {
     327      1244489 :   unsigned len = vec_safe_length (m_parameters);
     328      2588768 :   for (unsigned i = 0; i < len; ++i)
     329      1344279 :     free_param_decl_accesses (&(*m_parameters)[i]);
     330      1244489 :   vec_free (m_parameters);
     331      1244489 : }
     332              : 
     333              : /* Mark the function as not a candidate for any IPA-SRA transformation.  Return
     334              :    true if it was a candidate until now.  */
     335              : 
     336              : bool
     337       670426 : isra_func_summary::zap ()
     338              : {
     339       670426 :   bool ret = m_candidate;
     340       670426 :   m_candidate = false;
     341              : 
     342              :   /* TODO: see the destructor above.  */
     343       670426 :   unsigned len = vec_safe_length (m_parameters);
     344      2198872 :   for (unsigned i = 0; i < len; ++i)
     345      1528446 :     free_param_decl_accesses (&(*m_parameters)[i]);
     346       670426 :   vec_free (m_parameters);
     347              : 
     348       670426 :   return ret;
     349              : }
     350              : 
     351              : /* Structure to describe which formal parameters feed into a particular actual
     352              :    argument.  */
     353              : 
     354              : struct isra_param_flow
     355              : {
     356              :   /* Number of elements in array inputs that contain valid data.  */
     357              :   char length;
     358              :   /* Indices of formal parameters that feed into the described actual argument.
     359              :      If aggregate_pass_through or pointer_pass_through below are true, it must
     360              :      contain exactly one element which is passed through from a formal
     361              :      parameter if the given number.  Otherwise, the array contains indices of
     362              :      callee's formal parameters which are used to calculate value of this
     363              :      actual argument. */
     364              :   unsigned char inputs[IPA_SRA_MAX_PARAM_FLOW_LEN];
     365              : 
     366              :   /* Offset within the formal parameter.  */
     367              :   unsigned unit_offset;
     368              :   /* When aggregate_pass_through is set, this is the size of the portion of an
     369              :      aggregate formal parameter that is being passed.  Otherwise, this is size
     370              :      of pointed to memory that is known to be valid be dereferenced.  */
     371              :   unsigned unit_size : ISRA_ARG_SIZE_LIMIT_BITS;
     372              : 
     373              :   /* True when the value of this actual argument is a portion of a formal
     374              :      parameter.  */
     375              :   unsigned aggregate_pass_through : 1;
     376              :   /* True when the value of this actual copy is a verbatim pass through of an
     377              :      obtained pointer.  */
     378              :   unsigned pointer_pass_through : 1;
     379              :   /* True when it is safe to copy access candidates here from the callee, which
     380              :      would mean introducing dereferences into callers of the caller.  */
     381              :   unsigned safe_to_import_accesses : 1;
     382              :   /* True when the passed value is an address of a structure that has been
     383              :      constructed in the caller just to be passed by reference to functions
     384              :      (i.e. is never read).  */
     385              :   unsigned constructed_for_calls : 1;
     386              : };
     387              : 
     388              : /* Structure used to convey information about calls from the intra-procedural
     389              :    analysis stage to inter-procedural one.  */
     390              : 
     391      6317071 : class isra_call_summary
     392              : {
     393              : public:
     394      6317071 :   isra_call_summary ()
     395            0 :     : m_arg_flow (), m_return_ignored (false), m_return_returned (false),
     396            0 :       m_bit_aligned_arg (false), m_before_any_store (false)
     397              :   {}
     398              : 
     399              :   void init_inputs (unsigned arg_count);
     400              :   void dump (FILE *f);
     401              : 
     402              :   /* Information about what formal parameters of the caller are used to compute
     403              :      individual actual arguments of this call.  */
     404              :   auto_vec <isra_param_flow> m_arg_flow;
     405              : 
     406              :   /* Set to true if the call statement does not have a LHS.  */
     407              :   unsigned m_return_ignored : 1;
     408              : 
     409              :   /* Set to true if the LHS of call statement is only used to construct the
     410              :      return value of the caller.  */
     411              :   unsigned m_return_returned : 1;
     412              : 
     413              :   /* Set when any of the call arguments are not byte-aligned.  */
     414              :   unsigned m_bit_aligned_arg : 1;
     415              : 
     416              :   /* Set to true if the call happened before any (other) store to memory in the
     417              :      caller.  */
     418              :   unsigned m_before_any_store : 1;
     419              : };
     420              : 
     421              : /* Class to manage function summaries.  */
     422              : 
     423              : class GTY((user)) ipa_sra_function_summaries
     424              :   : public function_summary <isra_func_summary *>
     425              : {
     426              : public:
     427       140040 :   ipa_sra_function_summaries (symbol_table *table, bool ggc):
     428       280080 :     function_summary<isra_func_summary *> (table, ggc) { }
     429              : 
     430              :   void duplicate (cgraph_node *, cgraph_node *,
     431              :                   isra_func_summary *old_sum,
     432              :                   isra_func_summary *new_sum) final override;
     433              :   void insert (cgraph_node *, isra_func_summary *) final override;
     434              : };
     435              : 
     436              : /* Hook that is called by summary when a node is duplicated.  */
     437              : 
     438              : void
     439       142038 : ipa_sra_function_summaries::duplicate (cgraph_node *, cgraph_node *,
     440              :                                        isra_func_summary *old_sum,
     441              :                                        isra_func_summary *new_sum)
     442              : {
     443              :   /* TODO: Somehow stop copying when ISRA is doing the cloning, it is
     444              :      useless.  */
     445       142038 :   new_sum->m_candidate  = old_sum->m_candidate;
     446       142038 :   new_sum->m_returns_value = old_sum->m_returns_value;
     447       142038 :   new_sum->m_return_ignored = old_sum->m_return_ignored;
     448       142038 :   gcc_assert (!old_sum->m_queued);
     449       142038 :   new_sum->m_queued = false;
     450              : 
     451       142038 :   unsigned param_count = vec_safe_length (old_sum->m_parameters);
     452       140472 :   if (!param_count)
     453              :     return;
     454       140472 :   vec_safe_reserve_exact (new_sum->m_parameters, param_count);
     455       140472 :   new_sum->m_parameters->quick_grow_cleared (param_count);
     456       676997 :   for (unsigned i = 0; i < param_count; i++)
     457              :     {
     458       396053 :       isra_param_desc *s = &(*old_sum->m_parameters)[i];
     459       396053 :       isra_param_desc *d = &(*new_sum->m_parameters)[i];
     460              : 
     461       396053 :       d->param_size_limit = s->param_size_limit;
     462       396053 :       d->size_reached = s->size_reached;
     463       396053 :       d->safe_size = s->safe_size;
     464       396053 :       d->locally_unused = s->locally_unused;
     465       396053 :       d->split_candidate = s->split_candidate;
     466       396053 :       d->by_ref = s->by_ref;
     467       396053 :       d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
     468       396053 :       d->split_only_when_retval_removed = s->split_only_when_retval_removed;
     469       396053 :       d->not_specially_constructed = s->not_specially_constructed;
     470       396053 :       d->conditionally_dereferenceable = s->conditionally_dereferenceable;
     471       396053 :       d->safe_size_set = s->safe_size_set;
     472              : 
     473       396053 :       unsigned acc_count = vec_safe_length (s->accesses);
     474       396053 :       vec_safe_reserve_exact (d->accesses, acc_count);
     475       953730 :       for (unsigned j = 0; j < acc_count; j++)
     476              :         {
     477       161624 :           param_access *from = (*s->accesses)[j];
     478       161624 :           param_access *to = ggc_cleared_alloc<param_access> ();
     479       161624 :           to->type = from->type;
     480       161624 :           to->alias_ptr_type = from->alias_ptr_type;
     481       161624 :           to->unit_offset = from->unit_offset;
     482       161624 :           to->unit_size = from->unit_size;
     483       161624 :           to->certain = from->certain;
     484       161624 :           to->reverse = from->reverse;
     485       161624 :           d->accesses->quick_push (to);
     486              :         }
     487              :     }
     488              : }
     489              : 
     490              : /* Pointer to the pass function summary holder.  */
     491              : 
     492              : static GTY(()) ipa_sra_function_summaries *func_sums;
     493              : 
     494              : /* Hook that is called by summary when new node appears.  */
     495              : 
     496              : void
     497        18871 : ipa_sra_function_summaries::insert (cgraph_node *node, isra_func_summary *)
     498              : {
     499        18871 :   if (opt_for_fn (node->decl, flag_ipa_sra))
     500              :     {
     501        18871 :       push_cfun (DECL_STRUCT_FUNCTION (node->decl));
     502        18871 :       ipa_sra_summarize_function (node);
     503        18871 :       pop_cfun ();
     504              :     }
     505              :   else
     506            0 :     func_sums->remove (node);
     507        18871 : }
     508              : 
     509              : /* Class to manage call summaries.  */
     510              : 
     511              : class ipa_sra_call_summaries: public call_summary <isra_call_summary *>
     512              : {
     513              : public:
     514       140040 :   ipa_sra_call_summaries (symbol_table *table):
     515       280080 :     call_summary<isra_call_summary *> (table) { }
     516              : 
     517              :   /* Duplicate info when an edge is cloned.  */
     518              :   void duplicate (cgraph_edge *, cgraph_edge *,
     519              :                   isra_call_summary *old_sum,
     520              :                   isra_call_summary *new_sum) final override;
     521              : };
     522              : 
     523              : static ipa_sra_call_summaries *call_sums;
     524              : 
     525              : 
     526              : /* Initialize m_arg_flow of a particular instance of isra_call_summary.
     527              :    ARG_COUNT is the number of actual arguments passed.  */
     528              : 
     529              : void
     530      7323386 : isra_call_summary::init_inputs (unsigned arg_count)
     531              : {
     532      7323386 :   if (arg_count == 0)
     533              :     {
     534       370748 :       gcc_checking_assert (m_arg_flow.length () == 0);
     535              :       return;
     536              :     }
     537      6952638 :   if (m_arg_flow.length () == 0)
     538              :     {
     539      3438195 :       m_arg_flow.reserve_exact (arg_count);
     540      3438195 :       m_arg_flow.quick_grow_cleared (arg_count);
     541              :     }
     542              :   else
     543      3514443 :     gcc_checking_assert (arg_count == m_arg_flow.length ());
     544              : }
     545              : 
     546              : /* Dump all information in call summary to F.  */
     547              : 
     548              : void
     549          296 : isra_call_summary::dump (FILE *f)
     550              : {
     551          296 :   if (m_return_ignored)
     552          188 :     fprintf (f, "    return value ignored\n");
     553          296 :   if (m_return_returned)
     554           38 :     fprintf (f, "    return value used only to compute caller return value\n");
     555          296 :   if (m_before_any_store)
     556           57 :     fprintf (f, "    happens before any store to memory\n");
     557          627 :   for (unsigned i = 0; i < m_arg_flow.length (); i++)
     558              :     {
     559          331 :       fprintf (f, "    Parameter %u:\n", i);
     560          331 :       isra_param_flow *ipf = &m_arg_flow[i];
     561              : 
     562          331 :       if (ipf->length)
     563              :         {
     564          206 :           bool first = true;
     565          206 :           fprintf (f, "      Scalar param sources: ");
     566          621 :           for (int j = 0; j < ipf->length; j++)
     567              :             {
     568          209 :               if (!first)
     569            3 :                 fprintf (f, ", ");
     570              :               else
     571              :                 first = false;
     572          209 :               fprintf (f, "%i", (int) ipf->inputs[j]);
     573              :             }
     574          206 :           fprintf (f, "\n");
     575              :         }
     576          331 :       if (ipf->aggregate_pass_through)
     577           26 :         fprintf (f, "      Aggregate pass through from the param given above, "
     578              :                  "unit offset: %u , unit size: %u\n",
     579           26 :                  ipf->unit_offset, ipf->unit_size);
     580          305 :       else if (ipf->unit_size > 0)
     581           72 :         fprintf (f, "      Known dereferenceable size: %u\n", ipf->unit_size);
     582          331 :       if (ipf->pointer_pass_through)
     583           30 :         fprintf (f, "      Pointer pass through from the param given above, "
     584           30 :                  "safe_to_import_accesses: %u\n", ipf->safe_to_import_accesses);
     585          331 :       if (ipf->constructed_for_calls)
     586           38 :         fprintf (f, "      Variable constructed just to be passed to "
     587              :                  "calls.\n");
     588              :     }
     589          296 : }
     590              : 
     591              : /* Duplicate edge summary when an edge is cloned.  */
     592              : 
     593              : void
     594       722932 : ipa_sra_call_summaries::duplicate (cgraph_edge *, cgraph_edge *,
     595              :                                    isra_call_summary *old_sum,
     596              :                                    isra_call_summary *new_sum)
     597              : {
     598       722932 :   unsigned arg_count = old_sum->m_arg_flow.length ();
     599       722932 :   new_sum->init_inputs (arg_count);
     600      2800364 :   for (unsigned i = 0; i < arg_count; i++)
     601      1354500 :     new_sum->m_arg_flow[i] = old_sum->m_arg_flow[i];
     602              : 
     603       722932 :   new_sum->m_return_ignored = old_sum->m_return_ignored;
     604       722932 :   new_sum->m_return_returned = old_sum->m_return_returned;
     605       722932 :   new_sum->m_bit_aligned_arg = old_sum->m_bit_aligned_arg;
     606       722932 :   new_sum->m_before_any_store = old_sum->m_before_any_store;
     607       722932 : }
     608              : 
     609              : 
     610              : /* With all GTY stuff done, we can move to anonymous namespace.  */
     611              : namespace {
     612              : /* Quick mapping from a decl to its param descriptor.  */
     613              : 
     614              : hash_map<tree, gensum_param_desc *> *decl2desc;
     615              : 
     616              : /* All local DECLs ever loaded from of and of those that have their address
     617              :    assigned to a variable.  */
     618              : 
     619              : hash_set <tree> *loaded_decls;
     620              : 
     621              : /* Countdown of allowed Alias Analysis steps during summary building.  */
     622              : 
     623              : int aa_walking_limit;
     624              : 
     625              : /* This is a table in which for each basic block and parameter there is a
     626              :    distance (offset + size) in that parameter which is dereferenced and
     627              :    accessed in that BB.  */
     628              : HOST_WIDE_INT *bb_dereferences = NULL;
     629              : /* How many by-reference parameters there are in the current function.  */
     630              : int unsafe_by_ref_count;
     631              : 
     632              : /* Bitmap of BBs that can cause the function to "stop" progressing by
     633              :    returning, throwing externally, looping infinitely or calling a function
     634              :    which might abort etc.. */
     635              : bitmap final_bbs;
     636              : 
     637              : /* Obstack to allocate various small structures required only when generating
     638              :    summary for a function.  */
     639              : struct obstack gensum_obstack;
     640              : 
     641              : /* Return false the function is apparently unsuitable for IPA-SRA based on it's
     642              :    attributes, return true otherwise.  NODE is the cgraph node of the current
     643              :    function.  */
     644              : 
     645              : static bool
     646      1304084 : ipa_sra_preliminary_function_checks (cgraph_node *node)
     647              : {
     648      1304084 :   if (!node->can_change_signature)
     649              :     {
     650        35928 :       if (dump_file)
     651            0 :         fprintf (dump_file, "Function cannot change signature.\n");
     652              :       return false;
     653              :     }
     654              : 
     655      1268156 :   if (!tree_versionable_function_p (node->decl))
     656              :     {
     657       122233 :       if (dump_file)
     658            7 :         fprintf (dump_file, "Function is not versionable.\n");
     659              :       return false;
     660              :     }
     661              : 
     662      1145923 :   if (!opt_for_fn (node->decl, optimize)
     663      1145923 :       || !opt_for_fn (node->decl, flag_ipa_sra))
     664              :     {
     665          301 :       if (dump_file)
     666            0 :         fprintf (dump_file, "Not optimizing or IPA-SRA turned off for this "
     667              :                  "function.\n");
     668              :       return false;
     669              :     }
     670              : 
     671      1145622 :   if (DECL_VIRTUAL_P (node->decl))
     672              :     {
     673        31086 :       if (dump_file)
     674            0 :         fprintf (dump_file, "Function is a virtual method.\n");
     675              :       return false;
     676              :     }
     677              : 
     678      1114536 :   struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
     679      1114536 :   if (fun->stdarg)
     680              :     {
     681          124 :       if (dump_file)
     682            0 :         fprintf (dump_file, "Function uses stdarg. \n");
     683              :       return false;
     684              :     }
     685              : 
     686      1114412 :   if (DECL_DISREGARD_INLINE_LIMITS (node->decl))
     687              :     {
     688        51103 :       if (dump_file)
     689            0 :         fprintf (dump_file, "Always inline function will be inlined "
     690              :                  "anyway. \n");
     691              :       return false;
     692              :     }
     693              : 
     694              :   /* A special case.  If the callback edge pointing to the assume body is not
     695              :      created before ipa-sra analysis is run, the newly created edge won't be
     696              :      analyzed and will thus lack the appropriate summaries.  Be conservative and
     697              :      skip the function entirely.  We aren't missing anything, because we can't
     698              :      use an unsummarized call anyway.  */
     699      1063309 :   if (node->assume_function && !node->callers)
     700              :     {
     701            0 :       if (dump_file)
     702            0 :         fprintf (dump_file,
     703              :                  "Assume body without a callback edge, skipping. \n");
     704              :       return false;
     705              :     }
     706              : 
     707              :   return true;
     708              : }
     709              : 
     710              : /* Print access tree starting at ACCESS to F.  */
     711              : 
     712              : static void
     713          117 : dump_gensum_access (FILE *f, gensum_param_access *access, unsigned indent)
     714              : {
     715          117 :   fprintf (f, "  ");
     716          488 :   for (unsigned i = 0; i < indent; i++)
     717          254 :     fprintf (f, " ");
     718          117 :   fprintf (f, "    * Access to offset: " HOST_WIDE_INT_PRINT_DEC,
     719              :            access->offset);
     720          117 :   fprintf (f, ", size: " HOST_WIDE_INT_PRINT_DEC, access->size);
     721          117 :   fprintf (f, ", type: ");
     722          117 :   print_generic_expr (f, access->type);
     723          117 :   fprintf (f, ", alias_ptr_type: ");
     724          117 :   print_generic_expr (f, access->alias_ptr_type);
     725          117 :   fprintf (f, ", load_count: ");
     726          117 :   access->load_count.dump (f);
     727          117 :   fprintf (f, ", nonarg: %u, reverse: %u\n", access->nonarg, access->reverse);
     728          117 :   for (gensum_param_access *ch = access->first_child;
     729          127 :        ch;
     730           10 :        ch = ch->next_sibling)
     731           10 :     dump_gensum_access (f, ch, indent + 2);
     732          117 : }
     733              : 
     734              : 
     735              : /* Print access tree starting at ACCESS to F.  */
     736              : 
     737              : static void
     738          281 : dump_isra_access (FILE *f, param_access *access)
     739              : {
     740          281 :   fprintf (f, "    * Access to unit offset: %u", access->unit_offset);
     741          281 :   fprintf (f, ", unit size: %u", access->unit_size);
     742          281 :   fprintf (f, ", type: ");
     743          281 :   print_generic_expr (f, access->type);
     744          281 :   fprintf (f, ", alias_ptr_type: ");
     745          281 :   print_generic_expr (f, access->alias_ptr_type);
     746          281 :   if (access->certain)
     747          241 :     fprintf (f, ", certain");
     748              :   else
     749           40 :     fprintf (f, ", not certain");
     750          281 :   if (access->reverse)
     751            0 :     fprintf (f, ", reverse");
     752          281 :   fprintf (f, "\n");
     753          281 : }
     754              : 
     755              : /* Dump access tree starting at ACCESS to stderr.  */
     756              : 
     757              : DEBUG_FUNCTION void
     758            0 : debug_isra_access (param_access *access)
     759              : {
     760            0 :   dump_isra_access (stderr, access);
     761            0 : }
     762              : 
     763              : /* Dump DESC to F.  */
     764              : 
     765              : static void
     766          369 : dump_gensum_param_descriptor (FILE *f, gensum_param_desc *desc)
     767              : {
     768          369 :   if (desc->locally_unused)
     769          175 :     fprintf (f, "    unused with %i call_uses%s\n", desc->call_uses,
     770          175 :              desc->remove_only_when_retval_removed ?
     771              :              " remove_only_when_retval_removed" : "");
     772          369 :   if (!desc->split_candidate)
     773              :     {
     774          248 :       fprintf (f, "    not a candidate\n");
     775          248 :       return;
     776              :     }
     777          121 :   if (desc->by_ref)
     778           90 :     fprintf (f, "    %s%s%s by_ref with %u pass throughs\n",
     779           90 :              desc->safe_ref ? "safe" : "unsafe",
     780           90 :              desc->conditionally_dereferenceable
     781              :              ? " conditionally_dereferenceable" : "",
     782           90 :              desc->split_only_when_retval_removed
     783              :              ? " split_only_when_retval_removed" : "",
     784              :              desc->ptr_pt_count);
     785              : 
     786          228 :   for (gensum_param_access *acc = desc->accesses; acc; acc = acc->next_sibling)
     787          107 :     dump_gensum_access (f, acc, 2);
     788              : }
     789              : 
     790              : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to
     791              :    F.  */
     792              : 
     793              : static void
     794          172 : dump_gensum_param_descriptors (FILE *f, tree fndecl,
     795              :                                vec<gensum_param_desc> *param_descriptions)
     796              : {
     797          172 :   tree parm = DECL_ARGUMENTS (fndecl);
     798          172 :   for (unsigned i = 0;
     799          541 :        i < param_descriptions->length ();
     800          369 :        ++i, parm = DECL_CHAIN (parm))
     801              :     {
     802          369 :       fprintf (f, "  Descriptor for parameter %i ", i);
     803          369 :       print_generic_expr (f, parm, TDF_UID);
     804          369 :       fprintf (f, "\n");
     805          369 :       dump_gensum_param_descriptor (f, &(*param_descriptions)[i]);
     806              :     }
     807          172 : }
     808              : 
     809              : 
     810              : /* Dump DESC to F.  If HINTS is true, also dump IPA-analysis computed
     811              :    hints.  */
     812              : 
     813              : static void
     814          743 : dump_isra_param_descriptor (FILE *f, isra_param_desc *desc, bool hints)
     815              : {
     816          743 :   if (desc->locally_unused)
     817              :     {
     818          352 :       fprintf (f, "    (locally) unused\n");
     819              :     }
     820          743 :   if (!desc->split_candidate)
     821              :     {
     822          480 :       fprintf (f, "    not a candidate for splitting");
     823          480 :       if (hints && desc->by_ref && desc->safe_size_set)
     824            9 :         fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
     825          480 :       fprintf (f, "\n");
     826          480 :       return;
     827              :     }
     828          526 :   fprintf (f, "    param_size_limit: %u, size_reached: %u%s",
     829          263 :            desc->param_size_limit, desc->size_reached,
     830          263 :            desc->by_ref ? ", by_ref" : "");
     831          263 :   if (desc->remove_only_when_retval_removed)
     832           48 :     fprintf (f, ", remove_only_when_retval_removed");
     833          263 :   if (desc->split_only_when_retval_removed)
     834            4 :     fprintf (f, ", split_only_when_retval_removed");
     835          263 :   if (desc->by_ref && desc->conditionally_dereferenceable)
     836           54 :     fprintf (f, ", conditionally_dereferenceable");
     837          263 :   if (hints)
     838              :     {
     839           29 :       if (desc->by_ref && !desc->not_specially_constructed)
     840            8 :         fprintf (f, ", args_specially_constructed");
     841           29 :       if (desc->by_ref && desc->safe_size_set)
     842           18 :         fprintf (f, ", safe_size: %u", (unsigned) desc->safe_size);
     843              :     }
     844          263 :   fprintf (f, "\n");
     845              : 
     846          807 :   for (unsigned i = 0; i < vec_safe_length (desc->accesses); ++i)
     847              :     {
     848          281 :       param_access *access = (*desc->accesses)[i];
     849          281 :       dump_isra_access (f, access);
     850              :     }
     851              : }
     852              : 
     853              : /* Dump all parameter descriptors in IFS, assuming it describes FNDECL, to F.
     854              :    If HINTS is true, also dump IPA-analysis computed hints.  */
     855              : 
     856              : static void
     857          172 : dump_isra_param_descriptors (FILE *f, tree fndecl, isra_func_summary *ifs,
     858              :                              bool hints)
     859              : {
     860          172 :   tree parm = DECL_ARGUMENTS (fndecl);
     861          172 :   if (!ifs->m_parameters)
     862              :     {
     863            0 :       fprintf (f, "  parameter descriptors not available\n");
     864            0 :       return;
     865              :     }
     866              : 
     867              :   for (unsigned i = 0;
     868          541 :        i < ifs->m_parameters->length ();
     869          369 :        ++i, parm = DECL_CHAIN (parm))
     870              :     {
     871          369 :       fprintf (f, "  Descriptor for parameter %i ", i);
     872          369 :       print_generic_expr (f, parm, TDF_UID);
     873          369 :       fprintf (f, "\n");
     874          369 :       dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
     875              :     }
     876              : }
     877              : 
     878              : /* Add SRC to inputs of PARAM_FLOW, unless it would exceed storage.  If the
     879              :    function fails return false, otherwise return true.  SRC must fit into an
     880              :    unsigned char.  Used for purposes of transitive unused parameter
     881              :    removal.  */
     882              : 
     883              : static bool
     884      1366204 : add_src_to_param_flow (isra_param_flow *param_flow, int src)
     885              : {
     886      1366204 :   gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
     887      1366204 :   if (param_flow->length == IPA_SRA_MAX_PARAM_FLOW_LEN)
     888              :     return false;
     889              : 
     890      1366202 :   param_flow->inputs[(int) param_flow->length] = src;
     891      1366202 :   param_flow->length++;
     892      1366202 :   return true;
     893              : }
     894              : 
     895              : /* Add a SRC to the inputs of PARAM_FLOW unless it is already there and assert
     896              :    it is the only input.  Used for purposes of transitive parameter
     897              :    splitting.  */
     898              : 
     899              : static void
     900       809997 : set_single_param_flow_source (isra_param_flow *param_flow, int src)
     901              : {
     902       809997 :   gcc_checking_assert (src >= 0 && src <= UCHAR_MAX);
     903       809997 :   if (param_flow->length == 0)
     904              :     {
     905       125375 :       param_flow->inputs[0] = src;
     906       125375 :       param_flow->length = 1;
     907              :     }
     908       684622 :   else if (param_flow->length == 1)
     909       684622 :     gcc_assert (param_flow->inputs[0] == src);
     910              :   else
     911            0 :     gcc_unreachable ();
     912       809997 : }
     913              : 
     914              : /* Assert that there is only a single value in PARAM_FLOW's inputs and return
     915              :    it.  */
     916              : 
     917              : static unsigned
     918      1343958 : get_single_param_flow_source (const isra_param_flow *param_flow)
     919              : {
     920      1343958 :   gcc_assert (param_flow->length == 1);
     921      1343958 :   return param_flow->inputs[0];
     922              : }
     923              : 
     924              : /* Inspect all uses of NAME and simple arithmetic calculations involving NAME
     925              :    in FUN represented with NODE and return a negative number if any of them is
     926              :    used for something else than either an actual call argument, simple return,
     927              :    simple arithmetic operation or debug statement.  If there are no such uses,
     928              :    return the number of actual arguments that this parameter eventually feeds
     929              :    to (or zero if there is none).  If there are any simple return uses, set
     930              :    DESC->remove_only_when_retval_removed.  For any such parameter, mark
     931              :    PARM_NUM as one of its sources.  ANALYZED is a bitmap that tracks which SSA
     932              :    names we have already started investigating.  */
     933              : 
     934              : static int
     935      3538057 : isra_track_scalar_value_uses (function *fun, cgraph_node *node, tree name,
     936              :                               int parm_num, bitmap analyzed,
     937              :                               gensum_param_desc *desc)
     938              : {
     939      3538057 :   int res = 0;
     940      3538057 :   imm_use_iterator imm_iter;
     941      3538057 :   gimple *stmt;
     942              : 
     943      7332623 :   FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
     944              :     {
     945      5502486 :       if (is_gimple_debug (stmt)
     946      5502486 :           || gimple_clobber_p (stmt))
     947       963432 :         continue;
     948              : 
     949              :       /* TODO: We could handle at least const builtin functions like arithmetic
     950              :          operations below.  */
     951      4539054 :       if (is_gimple_call (stmt))
     952              :         {
     953      1379760 :           int all_uses = 0;
     954      1379760 :           use_operand_p use_p;
     955      2773329 :           FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
     956      1393569 :             all_uses++;
     957              : 
     958      1379760 :           gcall *call = as_a <gcall *> (stmt);
     959      1379760 :           unsigned arg_count;
     960      1379760 :           if (gimple_call_internal_p (call)
     961      1379760 :               || (arg_count = gimple_call_num_args (call)) == 0)
     962              :             {
     963              :               res = -1;
     964              :               break;
     965              :             }
     966              : 
     967      1374479 :           cgraph_edge *cs = node->get_edge (stmt);
     968      1374479 :           gcc_checking_assert (cs);
     969      1374479 :           isra_call_summary *csum = call_sums->get_create (cs);
     970      1374479 :           csum->init_inputs (arg_count);
     971              : 
     972      1374479 :           int simple_uses = 0;
     973      7614307 :           for (unsigned i = 0; i < arg_count; i++)
     974      4865351 :             if (gimple_call_arg (call, i) == name)
     975              :               {
     976      1366204 :                 if (!add_src_to_param_flow (&csum->m_arg_flow[i], parm_num))
     977              :                   {
     978              :                     simple_uses = -1;
     979              :                     break;
     980              :                   }
     981      1366202 :                 simple_uses++;
     982              :               }
     983              : 
     984      1374479 :           if (simple_uses < 0
     985      1374479 :               || all_uses != simple_uses)
     986              :             {
     987              :               res = -1;
     988              :               break;
     989              :             }
     990      1352654 :           res += all_uses;
     991              :         }
     992      3159294 :       else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
     993      3159294 :                && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
     994       731351 :                    || gimple_code (stmt) == GIMPLE_PHI))
     995              :         {
     996      2540292 :           tree lhs;
     997      2540292 :           if (gimple_code (stmt) == GIMPLE_PHI)
     998       219939 :             lhs = gimple_phi_result (stmt);
     999              :           else
    1000      2320353 :             lhs = gimple_assign_lhs (stmt);
    1001              : 
    1002      2540292 :           if (TREE_CODE (lhs) != SSA_NAME)
    1003              :             {
    1004              :               res = -1;
    1005              :               break;
    1006              :             }
    1007      2121662 :           gcc_assert (!gimple_vdef (stmt));
    1008      2121662 :           if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs)))
    1009              :             {
    1010      2011173 :               int tmp = isra_track_scalar_value_uses (fun, node, lhs, parm_num,
    1011              :                                                       analyzed, desc);
    1012      2011173 :               if (tmp < 0)
    1013              :                 {
    1014              :                   res = tmp;
    1015              :                   break;
    1016              :                 }
    1017      1194876 :               res += tmp;
    1018              :             }
    1019              :         }
    1020       619002 :       else if (greturn *gr = dyn_cast<greturn *>(stmt))
    1021              :         {
    1022       173115 :           tree rv = gimple_return_retval (gr);
    1023       173115 :           if (rv != name)
    1024              :             {
    1025              :               res = -1;
    1026              :               break;
    1027              :             }
    1028       173115 :           desc->remove_only_when_retval_removed = true;
    1029              :         }
    1030              :       else
    1031              :         {
    1032              :           res = -1;
    1033              :           break;
    1034              :         }
    1035      3538057 :     }
    1036      3538057 :   return res;
    1037              : }
    1038              : 
    1039              : /* Inspect all uses of PARM, which must be a gimple register, in FUN (which is
    1040              :    also described by NODE) and simple arithmetic calculations involving PARM
    1041              :    and return false if any of them is used for something else than either an
    1042              :    actual call argument, simple return, simple arithmetic operation or debug
    1043              :    statement.  If there are no such uses, return true and store the number of
    1044              :    actual arguments that this parameter eventually feeds to (or zero if there
    1045              :    is none) to DESC->call_uses and set DESC->remove_only_when_retval_removed if
    1046              :    there are any uses in return statements.  For any such parameter, mark
    1047              :    PARM_NUM as one of its sources.
    1048              : 
    1049              :    This function is similar to ptr_parm_has_nonarg_uses but its results are
    1050              :    meant for unused parameter removal, as opposed to splitting of parameters
    1051              :    passed by reference or converting them to passed by value.  */
    1052              : 
    1053              : static bool
    1054      2225070 : isra_track_scalar_param_local_uses (function *fun, cgraph_node *node, tree parm,
    1055              :                                     int parm_num, gensum_param_desc *desc)
    1056              : {
    1057      2225070 :   gcc_checking_assert (is_gimple_reg (parm));
    1058              : 
    1059      2225070 :   tree name = ssa_default_def (fun, parm);
    1060      2225070 :   if (!name || has_zero_uses (name))
    1061              :     {
    1062       698186 :       desc->call_uses = 0;
    1063       698186 :       return false;
    1064              :     }
    1065              : 
    1066              :   /* Edge summaries can only handle callers with fewer than 256 parameters.  */
    1067      1526884 :   if (parm_num > UCHAR_MAX)
    1068              :     return true;
    1069              : 
    1070      1526884 :   bitmap analyzed = BITMAP_ALLOC (NULL);
    1071      1526884 :   int call_uses = isra_track_scalar_value_uses (fun, node, name, parm_num,
    1072              :                                                 analyzed, desc);
    1073      1526884 :   BITMAP_FREE (analyzed);
    1074      1526884 :   if (call_uses < 0)
    1075              :     return true;
    1076       635261 :   desc->call_uses = call_uses;
    1077       635261 :   return false;
    1078              : }
    1079              : 
    1080              : /* Scan immediate uses of a default definition SSA name of a parameter PARM and
    1081              :    examine whether there are any nonarg uses that are not actual arguments or
    1082              :    otherwise infeasible uses.  If so, return true, otherwise return false.
    1083              :    Create pass-through IPA flow records for any direct uses as argument calls
    1084              :    and if returning false, store their number into DESC->ptr_pt_count.  If
    1085              :    removal of return value would still allow splitting, return true but set
    1086              :    DESC->split_only_when_retval_removed.  NODE and FUN must represent the
    1087              :    function that is currently analyzed, PARM_NUM must be the index of the
    1088              :    analyzed parameter.
    1089              : 
    1090              :    This function is similar to isra_track_scalar_param_local_uses but its
    1091              :    results are meant for splitting of parameters passed by reference or turning
    1092              :    them into bits passed by value, as opposed to generic unused parameter
    1093              :    removal.  */
    1094              : 
    1095              : static bool
    1096      1175852 : ptr_parm_has_nonarg_uses (cgraph_node *node, function *fun, tree parm,
    1097              :                           int parm_num, gensum_param_desc *desc)
    1098              : {
    1099      1175852 :   imm_use_iterator ui;
    1100      1175852 :   gimple *stmt;
    1101      1175852 :   tree name = ssa_default_def (fun, parm);
    1102      1175852 :   bool ret = false;
    1103      1175852 :   unsigned pt_count = 0;
    1104              : 
    1105      1175852 :   if (!name || has_zero_uses (name))
    1106              :     return false;
    1107              : 
    1108              :   /* Edge summaries can only handle callers with fewer than 256 parameters.  */
    1109      1061587 :   if (parm_num > UCHAR_MAX)
    1110              :     return true;
    1111              : 
    1112      2960934 :   FOR_EACH_IMM_USE_STMT (stmt, ui, name)
    1113              :     {
    1114      2448784 :       unsigned uses_ok = 0;
    1115      2448784 :       use_operand_p use_p;
    1116              : 
    1117      2448784 :       if (is_gimple_debug (stmt)
    1118      2448784 :           || gimple_clobber_p (stmt))
    1119       632043 :         continue;
    1120              : 
    1121      1816741 :       if (gimple_assign_single_p (stmt))
    1122              :         {
    1123       932853 :           tree rhs = gimple_assign_rhs1 (stmt);
    1124       932853 :           if (!TREE_THIS_VOLATILE (rhs))
    1125              :             {
    1126      1773217 :               while (handled_component_p (rhs))
    1127       840448 :                 rhs = TREE_OPERAND (rhs, 0);
    1128       932769 :               if (TREE_CODE (rhs) == MEM_REF
    1129       646910 :                   && TREE_OPERAND (rhs, 0) == name
    1130       642656 :                   && integer_zerop (TREE_OPERAND (rhs, 1))
    1131      1530005 :                   && types_compatible_p (TREE_TYPE (rhs),
    1132       597236 :                                          TREE_TYPE (TREE_TYPE (name))))
    1133              :                 uses_ok++;
    1134              :             }
    1135              :         }
    1136       883888 :       else if (is_gimple_call (stmt))
    1137              :         {
    1138       736863 :           gcall *call = as_a<gcall *> (stmt);
    1139       736863 :           unsigned arg_count = gimple_call_num_args (call);
    1140       736863 :           if ((gimple_call_internal_p (call)
    1141          526 :                && !callback_ifn_assume_call_with_edge_p (node, stmt))
    1142       736874 :               || arg_count == 0)
    1143              :             {
    1144              :               ret = true;
    1145              :               break;
    1146              :             }
    1147              : 
    1148       736074 :           cgraph_edge *cs = node->get_edge (stmt);
    1149       736074 :           gcc_checking_assert (cs);
    1150       736074 :           isra_call_summary *csum = call_sums->get_create (cs);
    1151       736074 :           csum->init_inputs (arg_count);
    1152              : 
    1153      3573242 :           for (unsigned i = 0; i < arg_count; ++i)
    1154              :             {
    1155      2101094 :               tree arg = gimple_call_arg (stmt, i);
    1156              : 
    1157      2101094 :               if (arg == name)
    1158              :                 {
    1159              :                   /* TODO: Allow &MEM_REF[name + offset] here,
    1160              :                      ipa_param_body_adjustments::modify_call_stmt has to be
    1161              :                      adjusted too.  */
    1162       724197 :                   csum->m_arg_flow[i].pointer_pass_through = true;
    1163       724197 :                   set_single_param_flow_source (&csum->m_arg_flow[i], parm_num);
    1164       724197 :                   pt_count++;
    1165       724197 :                   uses_ok++;
    1166       724197 :                   continue;
    1167              :                 }
    1168              : 
    1169      1376897 :               if (!TREE_THIS_VOLATILE (arg))
    1170              :                 {
    1171      1406977 :                   while (handled_component_p (arg))
    1172        30080 :                     arg = TREE_OPERAND (arg, 0);
    1173      1376897 :                   if (TREE_CODE (arg) == MEM_REF
    1174        23115 :                       && TREE_OPERAND (arg, 0) == name
    1175        11861 :                       && integer_zerop (TREE_OPERAND (arg, 1))
    1176      1388756 :                       && types_compatible_p (TREE_TYPE (arg),
    1177        11859 :                                              TREE_TYPE (TREE_TYPE (name))))
    1178         9017 :                     uses_ok++;
    1179              :                 }
    1180              :             }
    1181              :         }
    1182       147025 :       else if (greturn *gr = dyn_cast<greturn *>(stmt))
    1183              :         {
    1184         9390 :           tree rv = gimple_return_retval (gr);
    1185         9390 :           if (rv == name)
    1186              :             {
    1187         9390 :               uses_ok++;
    1188              :               /* Analysis for feasibility of removal must have already reached
    1189              :                  the conclusion that the flag must be set if it completed.  */
    1190         9390 :               gcc_assert (!desc->locally_unused
    1191              :                           || desc->remove_only_when_retval_removed);
    1192         9390 :               desc->split_only_when_retval_removed = true;
    1193              :             }
    1194              :         }
    1195              : 
    1196              :       /* If the number of valid uses does not match the number of
    1197              :          uses in this stmt there is an unhandled use.  */
    1198      1815952 :       unsigned all_uses = 0;
    1199      3639254 :       FOR_EACH_IMM_USE_ON_STMT (use_p, ui)
    1200      1823302 :         all_uses++;
    1201              : 
    1202      1815952 :       gcc_checking_assert (uses_ok <= all_uses);
    1203      1815952 :       if (uses_ok != all_uses)
    1204              :         {
    1205              :           ret = true;
    1206              :           break;
    1207              :         }
    1208      1061587 :     }
    1209              : 
    1210      1061587 :   desc->ptr_pt_count = pt_count;
    1211      1061587 :   return ret;
    1212              : }
    1213              : 
    1214              : /* Initialize vector of parameter descriptors of NODE.  Return true if there
    1215              :    are any candidates for splitting or unused aggregate parameter removal (the
    1216              :    function may return false if there are candidates for removal of register
    1217              :    parameters).  */
    1218              : 
    1219              : static bool
    1220       861037 : create_parameter_descriptors (cgraph_node *node,
    1221              :                               vec<gensum_param_desc> *param_descriptions)
    1222              : {
    1223       861037 :   function *fun = DECL_STRUCT_FUNCTION (node->decl);
    1224       861037 :   bool ret = false;
    1225              : 
    1226       861037 :   int num = 0;
    1227       861037 :   for (tree parm = DECL_ARGUMENTS (node->decl);
    1228      3308445 :        parm;
    1229      2447408 :        parm = DECL_CHAIN (parm), num++)
    1230              :     {
    1231      2447408 :       const char *msg;
    1232      2447408 :       gensum_param_desc *desc = &(*param_descriptions)[num];
    1233              :       /* param_descriptions vector is grown cleared in the caller.  */
    1234      2447408 :       desc->param_number = num;
    1235      2447408 :       decl2desc->put (parm, desc);
    1236              : 
    1237      2447408 :       if (dump_file && (dump_flags & TDF_DETAILS))
    1238           76 :         print_generic_expr (dump_file, parm, TDF_UID);
    1239              : 
    1240      2447408 :       tree type = TREE_TYPE (parm);
    1241      2447408 :       if (TREE_THIS_VOLATILE (parm))
    1242              :         {
    1243           52 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1244            0 :             fprintf (dump_file, " not a candidate, is volatile\n");
    1245      1820117 :           continue;
    1246              :         }
    1247      2447356 :       if (!is_gimple_reg_type (type) && is_va_list_type (type))
    1248              :         {
    1249            0 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1250            0 :             fprintf (dump_file, " not a candidate, is a va_list type\n");
    1251            0 :           continue;
    1252              :         }
    1253      2447356 :       if (TREE_ADDRESSABLE (parm))
    1254              :         {
    1255        36338 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1256            0 :             fprintf (dump_file, " not a candidate, is addressable\n");
    1257        36338 :           continue;
    1258              :         }
    1259      2411018 :       if (TREE_ADDRESSABLE (type))
    1260              :         {
    1261            3 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1262            0 :             fprintf (dump_file, " not a candidate, type cannot be split\n");
    1263            3 :           continue;
    1264              :         }
    1265              : 
    1266      2411015 :       if (is_gimple_reg (parm)
    1267      2411015 :           && !isra_track_scalar_param_local_uses (fun, node, parm, num, desc))
    1268              :         {
    1269      1333447 :           desc->locally_unused = true;
    1270              : 
    1271      1333447 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1272           28 :             fprintf (dump_file, " is a scalar with only %i call uses%s\n",
    1273              :                      desc->call_uses,
    1274           28 :                      desc->remove_only_when_retval_removed
    1275              :                      ? " and return uses" : "");
    1276              :         }
    1277              : 
    1278      2411015 :       if (POINTER_TYPE_P (type))
    1279              :         {
    1280      1182446 :           desc->by_ref = true;
    1281      1182446 :           if (TREE_CODE (type) == REFERENCE_TYPE
    1282      1182446 :               || (num == 0
    1283       570228 :                   && TREE_CODE (TREE_TYPE (node->decl)) == METHOD_TYPE))
    1284       515257 :             desc->safe_ref = true;
    1285              :           else
    1286       667189 :             desc->safe_ref = false;
    1287      1182446 :           type = TREE_TYPE (type);
    1288              : 
    1289      1187519 :           if (TREE_CODE (type) == FUNCTION_TYPE
    1290      1182446 :               || TREE_CODE (type) == METHOD_TYPE)
    1291              :             {
    1292         5073 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1293            0 :                 fprintf (dump_file, " not a candidate, reference to "
    1294              :                          "a function\n");
    1295         5073 :               continue;
    1296              :             }
    1297      1177373 :           if (TYPE_VOLATILE (type))
    1298              :             {
    1299         1130 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1300            0 :                 fprintf (dump_file, " not a candidate, reference to "
    1301              :                          "a volatile type\n");
    1302         1130 :               continue;
    1303              :             }
    1304      1176243 :           if (TREE_CODE (type) == ARRAY_TYPE
    1305      1176243 :               && TYPE_NONALIASED_COMPONENT (type))
    1306              :             {
    1307            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1308            0 :                 fprintf (dump_file, " not a candidate, reference to "
    1309              :                          "a nonaliased component array\n");
    1310            0 :               continue;
    1311              :             }
    1312      1176243 :           if (!is_gimple_reg (parm))
    1313              :             {
    1314            6 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1315            0 :                 fprintf (dump_file, " not a candidate, a reference which is "
    1316              :                          "not a gimple register (probably addressable)\n");
    1317            6 :               continue;
    1318              :             }
    1319      1176237 :           if (is_va_list_type (type))
    1320              :             {
    1321          385 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1322            0 :                 fprintf (dump_file, " not a candidate, reference to "
    1323              :                          "a va list\n");
    1324          385 :               continue;
    1325              :             }
    1326      1175852 :           if (ptr_parm_has_nonarg_uses (node, fun, parm, num, desc))
    1327              :             {
    1328       549437 :               if (dump_file && (dump_flags & TDF_DETAILS))
    1329            7 :                 fprintf (dump_file, " not a candidate, reference has "
    1330              :                          "nonarg uses\n");
    1331       549437 :               continue;
    1332              :             }
    1333              :         }
    1334      1228569 :       else if (!AGGREGATE_TYPE_P (type))
    1335              :         {
    1336              :           /* This is in an else branch because scalars passed by reference are
    1337              :              still candidates to be passed by value.  */
    1338      1042832 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1339           21 :             fprintf (dump_file, " not a candidate, not an aggregate\n");
    1340      1042832 :           continue;
    1341              :         }
    1342              : 
    1343       812152 :       if (!COMPLETE_TYPE_P (type))
    1344              :         {
    1345       171303 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1346            0 :             fprintf (dump_file, " not a candidate, not a complete type\n");
    1347       171303 :           continue;
    1348              :         }
    1349       640849 :       if (!tree_fits_uhwi_p (TYPE_SIZE (type)))
    1350              :         {
    1351            2 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1352            0 :             fprintf (dump_file, " not a candidate, size not representable\n");
    1353            2 :           continue;
    1354              :         }
    1355       640847 :       unsigned HOST_WIDE_INT type_size
    1356       640847 :         = tree_to_uhwi (TYPE_SIZE (type)) / BITS_PER_UNIT;
    1357       645305 :       if (type_size == 0
    1358       640847 :           || type_size >= ISRA_ARG_SIZE_LIMIT)
    1359              :         {
    1360         4458 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1361            0 :             fprintf (dump_file, " not a candidate, has zero or huge size\n");
    1362         4458 :           continue;
    1363              :         }
    1364       636389 :       if (type_internals_preclude_sra_p (type, &msg))
    1365              :         {
    1366         9098 :           if (dump_file && (dump_flags & TDF_DETAILS))
    1367            0 :               fprintf (dump_file, " not a candidate, %s\n", msg);
    1368         9098 :           continue;
    1369              :         }
    1370              : 
    1371       627291 :       if (dump_file && (dump_flags & TDF_DETAILS))
    1372           48 :         fprintf (dump_file, " is a candidate\n");
    1373              : 
    1374       627291 :       ret = true;
    1375       627291 :       desc->split_candidate = true;
    1376       627291 :       if (desc->by_ref && !desc->safe_ref)
    1377       196840 :         desc->deref_index = unsafe_by_ref_count++;
    1378              :     }
    1379       861037 :   return ret;
    1380              : }
    1381              : 
    1382              : /* Return pointer to descriptor of parameter DECL or NULL if it cannot be
    1383              :    found, which happens if DECL is for a static chain.  */
    1384              : 
    1385              : static gensum_param_desc *
    1386      2517067 : get_gensum_param_desc (tree decl)
    1387              : {
    1388      2517067 :   if (!decl2desc)
    1389              :     return NULL;
    1390      2337697 :   gcc_checking_assert (TREE_CODE (decl) == PARM_DECL);
    1391      2337697 :   gensum_param_desc **slot = decl2desc->get (decl);
    1392      2337697 :   if (!slot)
    1393              :     /* This can happen for static chains which we cannot handle so far.  */
    1394              :     return NULL;
    1395      2312255 :   gcc_checking_assert (*slot);
    1396              :   return *slot;
    1397              : }
    1398              : 
    1399              : 
    1400              : /* Remove parameter described by DESC from candidates for IPA-SRA splitting and
    1401              :    write REASON to the dump file if there is one.  */
    1402              : 
    1403              : static void
    1404       168617 : disqualify_split_candidate (gensum_param_desc *desc, const char *reason)
    1405              : {
    1406       168617 :   if (!desc->split_candidate)
    1407              :     return;
    1408              : 
    1409       168588 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1410            0 :     fprintf (dump_file, "! Disqualifying parameter number %i - %s\n",
    1411              :              desc->param_number, reason);
    1412              : 
    1413       168588 :   desc->split_candidate = false;
    1414              : }
    1415              : 
    1416              : /* Remove DECL from candidates for IPA-SRA and write REASON to the dump file if
    1417              :    there is one.  */
    1418              : 
    1419              : static void
    1420          177 : disqualify_split_candidate (tree decl, const char *reason)
    1421              : {
    1422          177 :   gensum_param_desc *desc = get_gensum_param_desc (decl);
    1423          177 :   if (desc)
    1424           29 :     disqualify_split_candidate (desc, reason);
    1425          177 : }
    1426              : 
    1427              : /* Allocate a new access to DESC and fill it in with OFFSET and SIZE.  But
    1428              :    first, check that there are not too many of them already.  If so, do not
    1429              :    allocate anything and return NULL.  */
    1430              : 
    1431              : static gensum_param_access *
    1432       450149 : allocate_access (gensum_param_desc *desc,
    1433              :                  HOST_WIDE_INT offset, HOST_WIDE_INT size)
    1434              : {
    1435       450149 :   if (desc->access_count
    1436       450149 :       == (unsigned) param_ipa_sra_max_replacements)
    1437              :     {
    1438            0 :       disqualify_split_candidate (desc, "Too many replacement candidates");
    1439            0 :       return NULL;
    1440              :     }
    1441              : 
    1442       450149 :   gensum_param_access *access
    1443       450149 :     = (gensum_param_access *) obstack_alloc (&gensum_obstack,
    1444              :                                              sizeof (gensum_param_access));
    1445       450149 :   memset (access, 0, sizeof (*access));
    1446       450149 :   access->offset = offset;
    1447       450149 :   access->size = size;
    1448       450149 :   access->load_count = profile_count::zero ();
    1449       450149 :   return access;
    1450              : }
    1451              : 
    1452              : /* In what context scan_expr_access has been called, whether it deals with a
    1453              :    load, a function argument, or a store.  Please note that in rare
    1454              :    circumstances when it is not clear if the access is a load or store,
    1455              :    ISRA_CTX_STORE is used too.  */
    1456              : 
    1457              : enum isra_scan_context {ISRA_CTX_LOAD, ISRA_CTX_ARG, ISRA_CTX_STORE};
    1458              : 
    1459              : /* Return an access describing memory access to the variable described by DESC
    1460              :    at OFFSET with SIZE in context CTX, starting at pointer to the linked list
    1461              :    at a certain tree level FIRST.  Attempt to create it and put into the
    1462              :    appropriate place in the access tree if does not exist, but fail and return
    1463              :    NULL if there are already too many accesses, if it would create a partially
    1464              :    overlapping access or if an access would end up within a pre-existing
    1465              :    non-call access.  */
    1466              : 
    1467              : static gensum_param_access *
    1468       528348 : get_access_1 (gensum_param_desc *desc, gensum_param_access **first,
    1469              :               HOST_WIDE_INT offset, HOST_WIDE_INT size, isra_scan_context ctx)
    1470              : {
    1471       538488 :   gensum_param_access *access = *first, **ptr = first;
    1472              : 
    1473       538488 :   if (!access)
    1474              :     {
    1475              :       /* No pre-existing access at this level, just create it.  */
    1476       292957 :       gensum_param_access *a = allocate_access (desc, offset, size);
    1477       292957 :       if (!a)
    1478              :         return NULL;
    1479       292957 :       *first = a;
    1480       292957 :       return *first;
    1481              :     }
    1482              : 
    1483       245531 :   if (access->offset >= offset + size)
    1484              :     {
    1485              :       /* We want to squeeze it in front of the very first access, just do
    1486              :          it.  */
    1487        45595 :       gensum_param_access *r = allocate_access (desc, offset, size);
    1488        45595 :       if (!r)
    1489              :         return NULL;
    1490        45595 :       r->next_sibling = access;
    1491        45595 :       *first = r;
    1492        45595 :       return r;
    1493              :     }
    1494              : 
    1495              :   /* Skip all accesses that have to come before us until the next sibling is
    1496              :      already too far.  */
    1497       292907 :   while (offset >= access->offset + access->size
    1498       193078 :          && access->next_sibling
    1499       398084 :          && access->next_sibling->offset < offset + size)
    1500              :     {
    1501        92971 :       ptr = &access->next_sibling;
    1502        92971 :       access = access->next_sibling;
    1503              :     }
    1504              : 
    1505              :   /* At this point we know we do not belong before access.  */
    1506       199936 :   gcc_assert (access->offset < offset + size);
    1507              : 
    1508       199936 :   if (access->offset == offset && access->size == size)
    1509              :     /* We found what we were looking for.  */
    1510              :     return access;
    1511              : 
    1512       126824 :   if (access->offset <= offset
    1513       123687 :       && access->offset + access->size >= offset + size)
    1514              :     {
    1515              :     /* We fit into access which is larger than us.  We need to find/create
    1516              :        something below access.  But we only allow nesting in call
    1517              :        arguments.  */
    1518        11903 :       if (access->nonarg)
    1519              :         return NULL;
    1520              : 
    1521        10140 :       return get_access_1 (desc, &access->first_child, offset, size, ctx);
    1522              :     }
    1523              : 
    1524       114921 :   if (offset <= access->offset
    1525        14814 :       && offset + size  >= access->offset + access->size)
    1526              :     /* We are actually bigger than access, which fully fits into us, take its
    1527              :        place and make all accesses fitting into it its children.  */
    1528              :     {
    1529              :       /* But first, we only allow nesting in call arguments so check if that is
    1530              :          what we are trying to represent.  */
    1531        14814 :       if (ctx != ISRA_CTX_ARG)
    1532              :         return NULL;
    1533              : 
    1534        11490 :       gensum_param_access *r = allocate_access (desc, offset, size);
    1535        11490 :       if (!r)
    1536              :         return NULL;
    1537        11490 :       r->first_child = access;
    1538              : 
    1539        11490 :       while (access->next_sibling
    1540        20202 :              && access->next_sibling->offset < offset + size)
    1541              :         access = access->next_sibling;
    1542        11490 :       if (access->offset + access->size > offset + size)
    1543              :         {
    1544              :           /* This must be a different access, which are sorted, so the
    1545              :              following must be true and this signals a partial overlap.  */
    1546            0 :           gcc_assert (access->offset > offset);
    1547              :           return NULL;
    1548              :         }
    1549              : 
    1550        11490 :       r->next_sibling = access->next_sibling;
    1551        11490 :       access->next_sibling = NULL;
    1552        11490 :       *ptr = r;
    1553        11490 :       return r;
    1554              :     }
    1555              : 
    1556       100107 :   if (offset >= access->offset + access->size)
    1557              :     {
    1558              :       /* We belong after access.  */
    1559       100107 :       gensum_param_access *r = allocate_access (desc, offset, size);
    1560       100107 :       if (!r)
    1561              :         return NULL;
    1562       100107 :       r->next_sibling = access->next_sibling;
    1563       100107 :       access->next_sibling = r;
    1564       100107 :       return r;
    1565              :     }
    1566              : 
    1567            0 :   if (offset < access->offset)
    1568              :     {
    1569              :       /* We know the following, otherwise we would have created a
    1570              :          super-access. */
    1571            0 :       gcc_checking_assert (offset + size < access->offset + access->size);
    1572              :       return NULL;
    1573              :     }
    1574              : 
    1575            0 :   if (offset + size > access->offset + access->size)
    1576              :     {
    1577              :       /* Likewise.  */
    1578            0 :       gcc_checking_assert (offset > access->offset);
    1579              :       return NULL;
    1580              :     }
    1581              : 
    1582            0 :   gcc_unreachable ();
    1583              : }
    1584              : 
    1585              : /* Return an access describing memory access to the variable described by DESC
    1586              :    at OFFSET with SIZE in context CTX, mark it as used in context CTX.  Attempt
    1587              :    to create if it does not exist, but fail and return NULL if there are
    1588              :    already too many accesses, if it would create a partially overlapping access
    1589              :    or if an access would end up in a non-call access.  */
    1590              : 
    1591              : static gensum_param_access *
    1592       528348 : get_access (gensum_param_desc *desc, HOST_WIDE_INT offset, HOST_WIDE_INT size,
    1593              :             isra_scan_context ctx)
    1594              : {
    1595       528348 :   gcc_checking_assert (desc->split_candidate);
    1596              : 
    1597       528348 :   gensum_param_access *access = get_access_1 (desc, &desc->accesses, offset,
    1598              :                                               size, ctx);
    1599       528348 :   if (!access)
    1600              :     {
    1601         5087 :       disqualify_split_candidate (desc,
    1602              :                                   "Bad access overlap or too many accesses");
    1603         5087 :       return NULL;
    1604              :     }
    1605              : 
    1606       523261 :   switch (ctx)
    1607              :     {
    1608        13341 :     case ISRA_CTX_STORE:
    1609        13341 :       gcc_assert (!desc->by_ref);
    1610              :       /* Fall-through */
    1611       434457 :     case ISRA_CTX_LOAD:
    1612       434457 :       access->nonarg = true;
    1613       434457 :       break;
    1614              :     case ISRA_CTX_ARG:
    1615              :       break;
    1616              :     }
    1617              : 
    1618              :   return access;
    1619              : }
    1620              : 
    1621              : /* Verify that parameter access tree starting with ACCESS is in good shape.
    1622              :    PARENT_OFFSET and PARENT_SIZE are the corresponding fields of parent of
    1623              :    ACCESS or zero if there is none.  */
    1624              : 
    1625              : static bool
    1626       944983 : verify_access_tree_1 (gensum_param_access *access, HOST_WIDE_INT parent_offset,
    1627              :                       HOST_WIDE_INT parent_size)
    1628              : {
    1629      1345499 :   while (access)
    1630              :     {
    1631       400516 :       gcc_assert (access->offset >= 0 && access->size >= 0);
    1632              : 
    1633       400516 :       if (parent_size != 0)
    1634              :         {
    1635        22847 :           if (access->offset < parent_offset)
    1636              :             {
    1637            0 :               error ("Access offset before parent offset");
    1638            0 :               return true;
    1639              :             }
    1640        22847 :           if (access->size >= parent_size)
    1641              :             {
    1642            0 :               error ("Access size greater or equal to its parent size");
    1643            0 :               return true;
    1644              :             }
    1645        22847 :           if (access->offset + access->size > parent_offset + parent_size)
    1646              :             {
    1647            0 :               error ("Access terminates outside of its parent");
    1648            0 :               return true;
    1649              :             }
    1650              :         }
    1651              : 
    1652       400516 :       if (verify_access_tree_1 (access->first_child, access->offset,
    1653              :                                 access->size))
    1654              :         return true;
    1655              : 
    1656       400516 :       if (access->next_sibling
    1657       132871 :           && (access->next_sibling->offset < access->offset + access->size))
    1658              :         {
    1659            0 :           error ("Access overlaps with its sibling");
    1660            0 :           return true;
    1661              :         }
    1662              : 
    1663       400516 :       access = access->next_sibling;
    1664              :     }
    1665              :   return false;
    1666              : }
    1667              : 
    1668              : /* Verify that parameter access tree starting with ACCESS is in good shape,
    1669              :    halt compilation and dump the tree to stderr if not.  */
    1670              : 
    1671              : DEBUG_FUNCTION void
    1672       544467 : isra_verify_access_tree (gensum_param_access *access)
    1673              : {
    1674       544467 :   if (verify_access_tree_1 (access, 0, 0))
    1675              :     {
    1676            0 :       for (; access; access = access->next_sibling)
    1677            0 :         dump_gensum_access (stderr, access, 2);
    1678            0 :       internal_error ("IPA-SRA access verification failed");
    1679              :     }
    1680       544467 : }
    1681              : 
    1682              : 
    1683              : /* Callback of walk_stmt_load_store_addr_ops visit_addr used to determine
    1684              :    GIMPLE_ASM operands with memory constrains which cannot be scalarized.  */
    1685              : 
    1686              : static bool
    1687        10867 : asm_visit_addr (gimple *, tree op, tree, void *)
    1688              : {
    1689        10867 :   op = get_base_address (op);
    1690        10867 :   if (op
    1691        10867 :       && TREE_CODE (op) == PARM_DECL)
    1692          177 :     disqualify_split_candidate (op, "Non-scalarizable GIMPLE_ASM operand.");
    1693              : 
    1694        10867 :   return false;
    1695              : }
    1696              : 
    1697              : /* Mark a dereference of parameter identified by DESC of distance DIST in a
    1698              :    basic block BB, unless the BB has already been marked as a potentially
    1699              :    final.  */
    1700              : 
    1701              : static void
    1702       249561 : mark_param_dereference (gensum_param_desc *desc, HOST_WIDE_INT dist,
    1703              :                         basic_block bb)
    1704              : {
    1705       249561 :   gcc_assert (desc->by_ref);
    1706       249561 :   gcc_checking_assert (desc->split_candidate);
    1707              : 
    1708       249561 :   if (desc->safe_ref
    1709       249561 :       || bitmap_bit_p (final_bbs, bb->index))
    1710              :     return;
    1711              : 
    1712        26098 :   int idx = bb->index * unsafe_by_ref_count + desc->deref_index;
    1713        26098 :   if (bb_dereferences[idx] < dist)
    1714        24804 :     bb_dereferences[idx] = dist;
    1715              : }
    1716              : 
    1717              : /* Return true, if any potential replacements should use NEW_TYPE as opposed to
    1718              :    previously recorded OLD_TYPE.  */
    1719              : 
    1720              : static bool
    1721        62786 : type_prevails_p (tree old_type, tree new_type)
    1722              : {
    1723        62786 :   if (old_type == new_type)
    1724              :     return false;
    1725              : 
    1726              :   /* Non-aggregates are always better.  */
    1727          362 :   if (!is_gimple_reg_type (old_type)
    1728          362 :       && is_gimple_reg_type (new_type))
    1729              :     return true;
    1730          362 :   if (is_gimple_reg_type (old_type)
    1731          362 :       && !is_gimple_reg_type (new_type))
    1732              :     return false;
    1733              : 
    1734              :   /* Prefer any complex or vector type over any other scalar type.  */
    1735          296 :   if (TREE_CODE (old_type) != COMPLEX_TYPE
    1736          296 :       && TREE_CODE (old_type) != VECTOR_TYPE
    1737          231 :       && (TREE_CODE (new_type) == COMPLEX_TYPE
    1738          231 :           || VECTOR_TYPE_P (new_type)))
    1739              :     return true;
    1740          296 :   if ((TREE_CODE (old_type) == COMPLEX_TYPE
    1741              :        || VECTOR_TYPE_P (old_type))
    1742           65 :       && TREE_CODE (new_type) != COMPLEX_TYPE
    1743           52 :       && TREE_CODE (new_type) != VECTOR_TYPE)
    1744              :     return false;
    1745              : 
    1746              :   /* Use the integral type with the bigger precision.  */
    1747          296 :   if (INTEGRAL_TYPE_P (old_type)
    1748           12 :       && INTEGRAL_TYPE_P (new_type))
    1749           12 :     return (TYPE_PRECISION (new_type) > TYPE_PRECISION (old_type));
    1750              : 
    1751              :   /* Attempt to disregard any integral type with non-full precision.  */
    1752          284 :   if (INTEGRAL_TYPE_P (old_type)
    1753          284 :       && (TREE_INT_CST_LOW (TYPE_SIZE (old_type))
    1754            0 :           != TYPE_PRECISION (old_type)))
    1755              :     return true;
    1756          284 :   if (INTEGRAL_TYPE_P (new_type)
    1757          284 :       && (TREE_INT_CST_LOW (TYPE_SIZE (new_type))
    1758            6 :           != TYPE_PRECISION (new_type)))
    1759              :     return false;
    1760              :   /* Stabilize the selection.  */
    1761          284 :   return TYPE_UID (old_type) < TYPE_UID (new_type);
    1762              : }
    1763              : 
    1764              : /* When scanning an expression which is a call argument, this structure
    1765              :    specifies the call and the position of the argument.  */
    1766              : 
    1767              : struct scan_call_info
    1768              : {
    1769              :   /* Call graph edge representing the call. */
    1770              :   cgraph_edge *cs;
    1771              :   /* Total number of arguments in the call.  */
    1772              :   unsigned argument_count;
    1773              :   /* Number of the actual argument being scanned.  */
    1774              :   unsigned arg_idx;
    1775              : };
    1776              : 
    1777              : /* Record use of ACCESS which belongs to a parameter described by DESC in a
    1778              :    call argument described by CALL_INFO.  */
    1779              : 
    1780              : static void
    1781        85800 : record_nonregister_call_use (gensum_param_desc *desc,
    1782              :                              scan_call_info *call_info,
    1783              :                              unsigned unit_offset, unsigned unit_size)
    1784              : {
    1785        85800 :   isra_call_summary *csum = call_sums->get_create (call_info->cs);
    1786        85800 :   csum->init_inputs (call_info->argument_count);
    1787              : 
    1788        85800 :   isra_param_flow *param_flow = &csum->m_arg_flow[call_info->arg_idx];
    1789        85800 :   param_flow->aggregate_pass_through = true;
    1790        85800 :   set_single_param_flow_source (param_flow, desc->param_number);
    1791        85800 :   param_flow->unit_offset = unit_offset;
    1792        85800 :   param_flow->unit_size = unit_size;
    1793        85800 :   desc->call_uses++;
    1794        85800 : }
    1795              : 
    1796              : /* Callback of walk_aliased_vdefs, just mark that there was a possible
    1797              :    modification.  */
    1798              : 
    1799              : static bool
    1800        66256 : mark_maybe_modified (ao_ref *, tree, void *data)
    1801              : {
    1802        66256 :   bool *maybe_modified = (bool *) data;
    1803        66256 :   *maybe_modified = true;
    1804        66256 :   return true;
    1805              : }
    1806              : 
    1807              : /* Analyze expression EXPR from GIMPLE for accesses to parameters. CTX
    1808              :    specifies whether EXPR is used in a load, store or as an argument call. BB
    1809              :    must be the basic block in which expr resides.  If CTX specifies call
    1810              :    argument context, CALL_INFO must describe that call and argument position,
    1811              :    otherwise it is ignored.  */
    1812              : 
    1813              : static void
    1814     37148668 : scan_expr_access (tree expr, gimple *stmt, isra_scan_context ctx,
    1815              :                   basic_block bb, scan_call_info *call_info = NULL)
    1816              : {
    1817     37148668 :   poly_int64 poffset, psize, pmax_size;
    1818     37148668 :   HOST_WIDE_INT offset, size, max_size;
    1819     37148668 :   tree base;
    1820     37148668 :   bool deref = false;
    1821     37148668 :   bool reverse;
    1822              : 
    1823     37148668 :   if (TREE_CODE (expr) == ADDR_EXPR)
    1824              :     {
    1825      4606097 :       if (ctx == ISRA_CTX_ARG)
    1826              :         return;
    1827      1265240 :       tree t = get_base_address (TREE_OPERAND (expr, 0));
    1828      1265240 :       if (VAR_P (t) && !TREE_STATIC (t))
    1829       247510 :         loaded_decls->add (t);
    1830              :       return;
    1831              :     }
    1832     32542571 :   if (TREE_CODE (expr) == SSA_NAME
    1833     18074750 :       || CONSTANT_CLASS_P (expr))
    1834              :     return;
    1835              : 
    1836     12971189 :   if (TREE_CODE (expr) == BIT_FIELD_REF
    1837     12971189 :       || TREE_CODE (expr) == IMAGPART_EXPR
    1838     12825890 :       || TREE_CODE (expr) == REALPART_EXPR)
    1839       210837 :     expr = TREE_OPERAND (expr, 0);
    1840              : 
    1841     12971189 :   base = get_ref_base_and_extent (expr, &poffset, &psize, &pmax_size, &reverse);
    1842              : 
    1843     12971189 :   if (TREE_CODE (base) == MEM_REF)
    1844              :     {
    1845      4074818 :       tree op = TREE_OPERAND (base, 0);
    1846      4074818 :       if (TREE_CODE (op) != SSA_NAME
    1847      4074818 :           || !SSA_NAME_IS_DEFAULT_DEF (op))
    1848              :         return;
    1849      2144206 :       base = SSA_NAME_VAR (op);
    1850      2144206 :       if (!base)
    1851              :         return;
    1852              :       deref = true;
    1853              :     }
    1854      8896371 :   else if (VAR_P (base)
    1855      7679713 :            && !TREE_STATIC (base)
    1856      6618087 :            && (ctx == ISRA_CTX_ARG
    1857              :                || ctx == ISRA_CTX_LOAD))
    1858      2476573 :     loaded_decls->add (base);
    1859              : 
    1860     11040577 :   if (TREE_CODE (base) != PARM_DECL)
    1861              :     return;
    1862              : 
    1863      2516890 :   gensum_param_desc *desc = get_gensum_param_desc (base);
    1864      2516890 :   if (!desc || !desc->split_candidate)
    1865              :     return;
    1866              : 
    1867       595750 :   if (storage_order_barrier_p (expr))
    1868              :     {
    1869            0 :       disqualify_split_candidate (desc, "Encountered a storage order barrier.");
    1870            0 :       return;
    1871              :     }
    1872              : 
    1873       595750 :   if (!poffset.is_constant (&offset)
    1874       595750 :       || !psize.is_constant (&size)
    1875       595750 :       || !pmax_size.is_constant (&max_size))
    1876              :     {
    1877              :       disqualify_split_candidate (desc, "Encountered a polynomial-sized "
    1878              :                                   "access.");
    1879              :       return;
    1880              :     }
    1881       595750 :   if (size < 0 || size != max_size)
    1882              :     {
    1883         6583 :       disqualify_split_candidate (desc, "Encountered a variable sized access.");
    1884         6583 :       return;
    1885              :     }
    1886       589167 :   if (TREE_CODE (expr) == COMPONENT_REF
    1887       589167 :       && DECL_BIT_FIELD (TREE_OPERAND (expr, 1)))
    1888              :     {
    1889          939 :       disqualify_split_candidate (desc, "Encountered a bit-field access.");
    1890          939 :       return;
    1891              :     }
    1892       588228 :   if (offset < 0)
    1893              :     {
    1894            4 :       disqualify_split_candidate (desc, "Encountered an access at a "
    1895              :                                   "negative offset.");
    1896            4 :       return;
    1897              :     }
    1898       588224 :   gcc_assert ((offset % BITS_PER_UNIT) == 0);
    1899       588224 :   gcc_assert ((size % BITS_PER_UNIT) == 0);
    1900       588224 :   if ((offset / BITS_PER_UNIT) >= (UINT_MAX - ISRA_ARG_SIZE_LIMIT)
    1901       588224 :       || (size / BITS_PER_UNIT) >= ISRA_ARG_SIZE_LIMIT)
    1902              :     {
    1903            0 :       disqualify_split_candidate (desc, "Encountered an access with too big "
    1904              :                                   "offset or size");
    1905            0 :       return;
    1906              :     }
    1907              : 
    1908       588224 :   tree type = TREE_TYPE (expr);
    1909       588224 :   unsigned int exp_align = get_object_alignment (expr);
    1910              : 
    1911       588224 :   if (exp_align < TYPE_ALIGN (type))
    1912              :     {
    1913         1397 :       disqualify_split_candidate (desc, "Underaligned access.");
    1914         1397 :       return;
    1915              :     }
    1916              : 
    1917       586827 :   if (deref)
    1918              :     {
    1919       308040 :       if (!desc->by_ref)
    1920              :         {
    1921            0 :           disqualify_split_candidate (desc, "Dereferencing a non-reference.");
    1922        58479 :           return;
    1923              :         }
    1924       308040 :       else if (ctx == ISRA_CTX_STORE)
    1925              :         {
    1926            0 :           disqualify_split_candidate (desc, "Storing to data passed by "
    1927              :                                       "reference.");
    1928            0 :           return;
    1929              :         }
    1930              : 
    1931       308040 :       if (!aa_walking_limit)
    1932              :         {
    1933            0 :           disqualify_split_candidate (desc, "Out of alias analysis step "
    1934              :                                       "limit.");
    1935            0 :           return;
    1936              :         }
    1937              : 
    1938       616080 :       gcc_checking_assert (gimple_vuse (stmt));
    1939       308040 :       bool maybe_modified = false;
    1940       308040 :       ao_ref ar;
    1941              : 
    1942       308040 :       ao_ref_init (&ar, expr);
    1943       308040 :       bitmap visited = BITMAP_ALLOC (NULL);
    1944       616080 :       int walked = walk_aliased_vdefs (&ar, gimple_vuse (stmt),
    1945              :                                        mark_maybe_modified, &maybe_modified,
    1946              :                                        &visited, NULL, aa_walking_limit);
    1947       308040 :       BITMAP_FREE (visited);
    1948       308040 :       if (walked > 0)
    1949              :         {
    1950       102440 :           gcc_assert (aa_walking_limit > walked);
    1951       102440 :           aa_walking_limit = aa_walking_limit - walked;
    1952              :         }
    1953       308040 :       if (walked < 0)
    1954            0 :         aa_walking_limit = 0;
    1955       308040 :       if (maybe_modified || walked < 0)
    1956              :         {
    1957        58479 :           disqualify_split_candidate (desc, "Data passed by reference possibly "
    1958              :                                       "modified through an alias.");
    1959        58479 :           return;
    1960              :         }
    1961              :       else
    1962       249561 :         mark_param_dereference (desc, offset + size, bb);
    1963              :     }
    1964              :   else
    1965              :     /* Pointer parameters with direct uses should have been ruled out by
    1966              :        analyzing SSA default def when looking at the parameters.  */
    1967       278787 :     gcc_assert (!desc->by_ref);
    1968              : 
    1969       528348 :   gensum_param_access *access = get_access (desc, offset, size, ctx);
    1970       528348 :   if (!access)
    1971              :     return;
    1972              : 
    1973       523261 :   if (ctx == ISRA_CTX_ARG)
    1974              :     {
    1975        88804 :       gcc_checking_assert (call_info);
    1976              : 
    1977        88804 :       if (desc->param_number > UCHAR_MAX)
    1978              :         {
    1979            3 :           disqualify_split_candidate (desc, "Caller parameter with a large "
    1980              :                                       "index passed in a call.");
    1981            3 :           return;
    1982              :         }
    1983              : 
    1984        88801 :       if (!deref)
    1985        85800 :         record_nonregister_call_use (desc, call_info, offset / BITS_PER_UNIT,
    1986        85800 :                                      size / BITS_PER_UNIT);
    1987              :       else
    1988              :         /* This is not a pass-through of a pointer, this is a use like any
    1989              :            other.  */
    1990         3001 :         access->nonarg = true;
    1991              :     }
    1992       434457 :   else if (ctx == ISRA_CTX_LOAD && bb->count.initialized_p ())
    1993       421028 :     access->load_count += bb->count;
    1994              : 
    1995       523258 :   if (!access->type)
    1996              :     {
    1997       450146 :       access->type = type;
    1998       450146 :       access->alias_ptr_type = reference_alias_ptr_type (expr);
    1999       450146 :       access->reverse = reverse;
    2000              :     }
    2001              :   else
    2002              :     {
    2003        73112 :       if (exp_align < TYPE_ALIGN (access->type))
    2004              :         {
    2005            0 :           disqualify_split_candidate (desc, "Reference has lower alignment "
    2006              :                                       "than a previous one.");
    2007            0 :           return;
    2008              :         }
    2009        73112 :       if (access->alias_ptr_type != reference_alias_ptr_type (expr))
    2010              :         {
    2011         4572 :           disqualify_split_candidate (desc, "Multiple alias pointer types.");
    2012         4572 :           return;
    2013              :         }
    2014        68540 :       if (access->reverse != reverse)
    2015              :         {
    2016            0 :           disqualify_split_candidate (desc, "Both normal and reverse "
    2017              :                                       "scalar storage order.");
    2018            0 :           return;
    2019              :         }
    2020        68540 :       if (!deref
    2021        52587 :           && (AGGREGATE_TYPE_P (type) || AGGREGATE_TYPE_P (access->type))
    2022       107121 :           && (TYPE_MAIN_VARIANT (access->type) != TYPE_MAIN_VARIANT (type)))
    2023              :         {
    2024              :           /* We need the same aggregate type on all accesses to be able to
    2025              :              distinguish transformation spots from pass-through arguments in
    2026              :              the transformation phase.  */
    2027         5754 :           disqualify_split_candidate (desc, "We do not support aggregate "
    2028              :                                       "type punning.");
    2029         5754 :           return;
    2030              :         }
    2031              : 
    2032        62786 :       if (type_prevails_p (access->type, type))
    2033          140 :          access->type = type;
    2034              :     }
    2035              : }
    2036              : 
    2037              : /* Scan body function described by NODE and FUN and create access trees for
    2038              :    parameters.  */
    2039              : 
    2040              : static void
    2041      1304084 : scan_function (cgraph_node *node, struct function *fun)
    2042              : {
    2043      1304084 :   basic_block bb;
    2044              : 
    2045     10565861 :   FOR_EACH_BB_FN (bb, fun)
    2046              :     {
    2047      9261777 :       gimple_stmt_iterator gsi;
    2048     74392693 :       for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    2049              :         {
    2050     55869139 :           gimple *stmt = gsi_stmt (gsi);
    2051              : 
    2052     55869139 :           if (final_bbs && stmt_can_throw_external (fun, stmt))
    2053      1372560 :             bitmap_set_bit (final_bbs, bb->index);
    2054     55869139 :           switch (gimple_code (stmt))
    2055              :             {
    2056      1270821 :             case GIMPLE_RETURN:
    2057      1270821 :               {
    2058      1270821 :                 tree t = gimple_return_retval (as_a <greturn *> (stmt));
    2059      1270821 :                 if (t != NULL_TREE)
    2060       700268 :                   scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
    2061      1270821 :                 if (final_bbs)
    2062       423163 :                   bitmap_set_bit (final_bbs, bb->index);
    2063              :               }
    2064              :               break;
    2065              : 
    2066     18860213 :             case GIMPLE_ASSIGN:
    2067     18860213 :               if (gimple_assign_single_p (stmt)
    2068     18860213 :                   && !gimple_clobber_p (stmt))
    2069              :                 {
    2070     11477179 :                   tree rhs = gimple_assign_rhs1 (stmt);
    2071     11477179 :                   scan_expr_access (rhs, stmt, ISRA_CTX_LOAD, bb);
    2072     11477179 :                   tree lhs = gimple_assign_lhs (stmt);
    2073     11477179 :                   scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
    2074              :                 }
    2075              :               break;
    2076              : 
    2077      5513550 :             case GIMPLE_CALL:
    2078      5513550 :               {
    2079      5513550 :                 unsigned argument_count = gimple_call_num_args (stmt);
    2080      5513550 :                 isra_scan_context ctx = ISRA_CTX_ARG;
    2081      5513550 :                 scan_call_info call_info, *call_info_p = &call_info;
    2082      5513550 :                 if (gimple_call_internal_p (stmt)
    2083      5513550 :                     && !callback_ifn_assume_call_with_edge_p (node, stmt))
    2084              :                   {
    2085       189491 :                     call_info_p = NULL;
    2086       189491 :                     ctx = ISRA_CTX_LOAD;
    2087       189491 :                     internal_fn ifn = gimple_call_internal_fn (stmt);
    2088       189491 :                     if (internal_store_fn_p (ifn))
    2089            0 :                       ctx = ISRA_CTX_STORE;
    2090              :                   }
    2091              :                 else
    2092              :                   {
    2093      5324059 :                     call_info.cs = node->get_edge (stmt);
    2094      5324059 :                     call_info.argument_count = argument_count;
    2095              :                   }
    2096              : 
    2097     16697872 :                 for (unsigned i = 0; i < argument_count; i++)
    2098              :                   {
    2099     11184322 :                     call_info.arg_idx = i;
    2100     11184322 :                     scan_expr_access (gimple_call_arg (stmt, i), stmt,
    2101              :                                       ctx, bb, call_info_p);
    2102              :                   }
    2103              : 
    2104      5513550 :                 tree lhs = gimple_call_lhs (stmt);
    2105      5513550 :                 if (lhs)
    2106      2214369 :                   scan_expr_access (lhs, stmt, ISRA_CTX_STORE, bb);
    2107      5513550 :                 int flags = gimple_call_flags (stmt);
    2108      5513550 :                 if (final_bbs
    2109      2162997 :                     && (((flags & (ECF_CONST | ECF_PURE)) == 0)
    2110       357421 :                         || (flags & ECF_LOOPING_CONST_OR_PURE)))
    2111      1865277 :                   bitmap_set_bit (final_bbs, bb->index);
    2112              :               }
    2113      5513550 :               break;
    2114              : 
    2115        44197 :             case GIMPLE_ASM:
    2116        44197 :               {
    2117        44197 :                 gasm *asm_stmt = as_a <gasm *> (stmt);
    2118        44197 :                 walk_stmt_load_store_addr_ops (asm_stmt, NULL, NULL, NULL,
    2119              :                                                asm_visit_addr);
    2120        44197 :                 if (final_bbs)
    2121         1931 :                   bitmap_set_bit (final_bbs, bb->index);
    2122              : 
    2123        83919 :                 for (unsigned i = 0; i < gimple_asm_ninputs (asm_stmt); i++)
    2124              :                   {
    2125        39722 :                     tree t = TREE_VALUE (gimple_asm_input_op (asm_stmt, i));
    2126        39722 :                     scan_expr_access (t, stmt, ISRA_CTX_LOAD, bb);
    2127              :                   }
    2128        99826 :                 for (unsigned i = 0; i < gimple_asm_noutputs (asm_stmt); i++)
    2129              :                   {
    2130        55629 :                     tree t = TREE_VALUE (gimple_asm_output_op (asm_stmt, i));
    2131        55629 :                     scan_expr_access (t, stmt, ISRA_CTX_STORE, bb);
    2132              :                   }
    2133              :               }
    2134              :               break;
    2135              : 
    2136              :             default:
    2137              :               break;
    2138              :             }
    2139              :         }
    2140              :     }
    2141      1304084 : }
    2142              : 
    2143              : /* Return true if SSA_NAME NAME of function described by FUN is only used in
    2144              :    return statements, or if results of any operations it is involved in are
    2145              :    only used in return statements.  ANALYZED is a bitmap that tracks which SSA
    2146              :    names we have already started investigating.  */
    2147              : 
    2148              : static bool
    2149      2433078 : ssa_name_only_returned_p (function *fun, tree name, bitmap analyzed)
    2150              : {
    2151      2433078 :   bool res = true;
    2152      2433078 :   imm_use_iterator imm_iter;
    2153      2433078 :   gimple *stmt;
    2154              : 
    2155      3088723 :   FOR_EACH_IMM_USE_STMT (stmt, imm_iter, name)
    2156              :     {
    2157      2660843 :       if (is_gimple_debug (stmt))
    2158       185229 :         continue;
    2159              : 
    2160      2475614 :       if (gimple_code (stmt) == GIMPLE_RETURN)
    2161              :         {
    2162       254148 :           tree t = gimple_return_retval (as_a <greturn *> (stmt));
    2163       254148 :           if (t != name)
    2164              :             {
    2165              :               res = false;
    2166              :               break;
    2167              :             }
    2168              :         }
    2169      2221466 :       else if (!stmt_unremovable_because_of_non_call_eh_p (fun, stmt)
    2170      2221466 :                && ((is_gimple_assign (stmt) && !gimple_has_volatile_ops (stmt))
    2171      1213245 :                    || gimple_code (stmt) == GIMPLE_PHI))
    2172              :         {
    2173              :           /* TODO: And perhaps for const function calls too?  */
    2174      1121458 :           tree lhs;
    2175      1121458 :           if (gimple_code (stmt) == GIMPLE_PHI)
    2176       209193 :             lhs = gimple_phi_result (stmt);
    2177              :           else
    2178       912265 :             lhs = gimple_assign_lhs (stmt);
    2179              : 
    2180      1121458 :           if (TREE_CODE (lhs) != SSA_NAME)
    2181              :             {
    2182              :               res = false;
    2183              :               break;
    2184              :             }
    2185       786045 :           gcc_assert (!gimple_vdef (stmt));
    2186       786045 :           if (bitmap_set_bit (analyzed, SSA_NAME_VERSION (lhs))
    2187       786045 :               && !ssa_name_only_returned_p (fun, lhs, analyzed))
    2188              :             {
    2189              :               res = false;
    2190              :               break;
    2191              :             }
    2192              :         }
    2193              :       else
    2194              :         {
    2195              :           res = false;
    2196              :           break;
    2197              :         }
    2198      2433078 :     }
    2199      2433078 :   return res;
    2200              : }
    2201              : 
    2202              : /* Inspect the uses of the return value of the call associated with CS, and if
    2203              :    it is not used or if it is only used to construct the return value of the
    2204              :    caller, mark it as such in call or caller summary.  Also check for
    2205              :    misaligned arguments.  */
    2206              : 
    2207              : static void
    2208      5337640 : isra_analyze_call (cgraph_edge *cs)
    2209              : {
    2210      5337640 :   gcall *call_stmt = cs->call_stmt;
    2211      5337640 :   unsigned count = gimple_call_num_args (call_stmt);
    2212      5337640 :   isra_call_summary *csum = call_sums->get_create (cs);
    2213              : 
    2214     21306966 :   for (unsigned i = 0; i < count; i++)
    2215              :     {
    2216     10631686 :       tree arg = gimple_call_arg (call_stmt, i);
    2217     10631686 :       if (TREE_CODE (arg) == ADDR_EXPR)
    2218              :         {
    2219      3365989 :           poly_int64 poffset, psize, pmax_size;
    2220      3365989 :           bool reverse;
    2221      3365989 :           tree base = get_ref_base_and_extent (TREE_OPERAND (arg, 0), &poffset,
    2222      3365989 :                                                &psize, &pmax_size, &reverse);
    2223      3365989 :           HOST_WIDE_INT offset;
    2224      3365989 :           unsigned HOST_WIDE_INT ds;
    2225      3365989 :           if (DECL_P (base)
    2226      2400664 :               && (poffset.is_constant (&offset))
    2227      2400664 :               && tree_fits_uhwi_p (DECL_SIZE (base))
    2228      5620674 :               && ((ds = tree_to_uhwi (DECL_SIZE (base)) - offset)
    2229              :                   < ISRA_ARG_SIZE_LIMIT * BITS_PER_UNIT))
    2230              :             {
    2231      2121425 :               csum->init_inputs (count);
    2232      2121425 :               gcc_assert (!csum->m_arg_flow[i].aggregate_pass_through);
    2233      2121425 :               csum->m_arg_flow[i].unit_size = ds / BITS_PER_UNIT;
    2234              :             }
    2235              : 
    2236      3365989 :           if (TREE_CODE (base) == VAR_DECL
    2237      2212739 :               && !TREE_STATIC (base)
    2238      4820018 :               && !loaded_decls->contains (base))
    2239              :             {
    2240       966438 :               csum->init_inputs (count);
    2241       966438 :               csum->m_arg_flow[i].constructed_for_calls = true;
    2242              :             }
    2243              :         }
    2244              : 
    2245     10631686 :       if (is_gimple_reg (arg))
    2246      4176977 :         continue;
    2247              : 
    2248      6454709 :       tree offset;
    2249      6454709 :       poly_int64 bitsize, bitpos;
    2250      6454709 :       machine_mode mode;
    2251      6454709 :       int unsignedp, reversep, volatilep = 0;
    2252      6454709 :       get_inner_reference (arg, &bitsize, &bitpos, &offset, &mode,
    2253              :                            &unsignedp, &reversep, &volatilep);
    2254     12909418 :       if (!multiple_p (bitpos, BITS_PER_UNIT))
    2255              :         {
    2256            0 :           csum->m_bit_aligned_arg = true;
    2257            0 :           break;
    2258              :         }
    2259              :     }
    2260              : 
    2261      5337640 :   tree lhs = gimple_call_lhs (call_stmt);
    2262      5337640 :   if (lhs)
    2263              :     {
    2264              :       /* TODO: Also detect aggregates on a LHS of a call that are only returned
    2265              :          from this function (without being read anywhere).  */
    2266      2043125 :       if (TREE_CODE (lhs) == SSA_NAME)
    2267              :         {
    2268      1668393 :           bitmap analyzed = BITMAP_ALLOC (NULL);
    2269      1668393 :           if (ssa_name_only_returned_p (DECL_STRUCT_FUNCTION (cs->caller->decl),
    2270              :                                         lhs, analyzed))
    2271       232972 :             csum->m_return_returned = true;
    2272      1668393 :           BITMAP_FREE (analyzed);
    2273              :         }
    2274              :     }
    2275              : 
    2276              :   /* The return value of an ASSUME body is not ignored, but we cannot detect
    2277              :      this, as the function is never called directly.  */
    2278      3294515 :   else if (cs->callee && cs->callee->assume_function)
    2279          166 :     csum->m_return_ignored = false;
    2280              :   /* Don't set m_return_ignored for musttail calls.  The tailc/musttail passes
    2281              :      compare the returned value against the IPA-VRP return value range if
    2282              :      it is a singleton, but if the call is changed to something which doesn't
    2283              :      return anything, it will always fail.  */
    2284      3294349 :   else if (!gimple_call_must_tail_p (call_stmt))
    2285      3294248 :     csum->m_return_ignored = true;
    2286      5337640 : }
    2287              : 
    2288              : /* Look at all calls going out of NODE, described also by IFS and perform all
    2289              :    analyses necessary for IPA-SRA that are not done at body scan time or done
    2290              :    even when body is not scanned because the function is not a candidate.  */
    2291              : 
    2292              : static void
    2293      1304084 : isra_analyze_all_outgoing_calls (cgraph_node *node)
    2294              : {
    2295      6504870 :   for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
    2296      5200786 :     isra_analyze_call (cs);
    2297      1440938 :   for (cgraph_edge *cs = node->indirect_calls; cs; cs = cs->next_callee)
    2298       136854 :     isra_analyze_call (cs);
    2299      1304084 : }
    2300              : 
    2301              : /* Dump a dereferences table with heading STR to file F.  */
    2302              : 
    2303              : static void
    2304           42 : dump_dereferences_table (FILE *f, struct function *fun, const char *str)
    2305              : {
    2306           42 :   basic_block bb;
    2307              : 
    2308           42 :   fprintf (dump_file, "%s", str);
    2309          196 :   FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun),
    2310              :                   EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
    2311              :     {
    2312          154 :       fprintf (f, "%4i  %i   ", bb->index, bitmap_bit_p (final_bbs, bb->index));
    2313          154 :       if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
    2314              :         {
    2315              :           int i;
    2316          408 :           for (i = 0; i < unsafe_by_ref_count; i++)
    2317              :             {
    2318          254 :               int idx = bb->index * unsafe_by_ref_count + i;
    2319          254 :               fprintf (f, " %4" HOST_WIDE_INT_PRINT "d", bb_dereferences[idx]);
    2320              :             }
    2321              :         }
    2322          154 :       fprintf (f, "\n");
    2323              :     }
    2324           42 :   fprintf (dump_file, "\n");
    2325           42 : }
    2326              : 
    2327              : /* Propagate distances in bb_dereferences in the opposite direction than the
    2328              :    control flow edges, in each step storing the maximum of the current value
    2329              :    and the minimum of all successors.  These steps are repeated until the table
    2330              :    stabilizes.  Note that BBs which might terminate the functions (according to
    2331              :    final_bbs bitmap) never updated in this way.  */
    2332              : 
    2333              : static void
    2334        27106 : propagate_dereference_distances (struct function *fun)
    2335              : {
    2336        27106 :   basic_block bb;
    2337              : 
    2338        27106 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2339           21 :     dump_dereferences_table (dump_file, fun,
    2340              :                              "Dereference table before propagation:\n");
    2341              : 
    2342        27106 :   auto_vec<basic_block> queue (last_basic_block_for_fn (fun));
    2343        27106 :   queue.quick_push (ENTRY_BLOCK_PTR_FOR_FN (fun));
    2344       226902 :   FOR_EACH_BB_FN (bb, fun)
    2345              :     {
    2346       199796 :       queue.quick_push (bb);
    2347       199796 :       bb->aux = bb;
    2348              :     }
    2349              : 
    2350       254901 :   while (!queue.is_empty ())
    2351              :     {
    2352       227795 :       edge_iterator ei;
    2353       227795 :       edge e;
    2354       227795 :       bool change = false;
    2355       227795 :       int i;
    2356              : 
    2357       227795 :       bb = queue.pop ();
    2358       227795 :       bb->aux = NULL;
    2359              : 
    2360       227795 :       if (bitmap_bit_p (final_bbs, bb->index))
    2361        73680 :         continue;
    2362              : 
    2363      6228235 :       for (i = 0; i < unsafe_by_ref_count; i++)
    2364              :         {
    2365      6074120 :           int idx = bb->index * unsafe_by_ref_count + i;
    2366      6074120 :           bool first = true;
    2367      6074120 :           HOST_WIDE_INT inh = 0;
    2368              : 
    2369     14383752 :           FOR_EACH_EDGE (e, ei, bb->succs)
    2370              :           {
    2371      8309632 :             int succ_idx = e->dest->index * unsafe_by_ref_count + i;
    2372              : 
    2373      8309632 :             if (e->dest == EXIT_BLOCK_PTR_FOR_FN (fun))
    2374            0 :               continue;
    2375              : 
    2376      8309632 :             if (first)
    2377              :               {
    2378      6074115 :                 first = false;
    2379      6074115 :                 inh = bb_dereferences [succ_idx];
    2380              :               }
    2381      2235517 :             else if (bb_dereferences [succ_idx] < inh)
    2382              :               inh = bb_dereferences [succ_idx];
    2383              :           }
    2384              : 
    2385      6074120 :           if (!first && bb_dereferences[idx] < inh)
    2386              :             {
    2387        10922 :               bb_dereferences[idx] = inh;
    2388        10922 :               change = true;
    2389              :             }
    2390              :         }
    2391              : 
    2392       154115 :       if (change)
    2393        11875 :         FOR_EACH_EDGE (e, ei, bb->preds)
    2394              :           {
    2395         1882 :             if (e->src->aux)
    2396          989 :               continue;
    2397              : 
    2398          893 :             e->src->aux = e->src;
    2399          893 :             queue.quick_push (e->src);
    2400              :           }
    2401              :     }
    2402              : 
    2403        27106 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2404           21 :     dump_dereferences_table (dump_file, fun,
    2405              :                              "Dereference table after propagation:\n");
    2406        27106 : }
    2407              : 
    2408              : /* Return true if the ACCESS loads happen frequently enough in FUN to risk
    2409              :    moving them to the caller and only pass the result.  */
    2410              : 
    2411              : static bool
    2412       179154 : dereference_probable_p (struct function *fun, gensum_param_access *access)
    2413              : {
    2414       179154 :   int threshold = opt_for_fn (fun->decl, param_ipa_sra_deref_prob_threshold);
    2415       179154 :   return access->load_count
    2416       179154 :     >= ENTRY_BLOCK_PTR_FOR_FN (fun)->count.apply_scale (threshold, 100);
    2417              : }
    2418              : 
    2419              : /* Perform basic checks on ACCESS to PARM (of FUN) described by DESC and all
    2420              :    its children, return true if the parameter cannot be split, otherwise return
    2421              :    false and update *NONARG_ACC_SIZE and *ONLY_CALLS.  ENTRY_BB_INDEX must be
    2422              :    the index of the entry BB in the function of PARM.  */
    2423              : 
    2424              : static bool
    2425       393027 : check_gensum_access (struct function *fun, tree parm, gensum_param_desc *desc,
    2426              :                      gensum_param_access *access,
    2427              :                      HOST_WIDE_INT *nonarg_acc_size, bool *only_calls,
    2428              :                      int entry_bb_index)
    2429              : {
    2430       393027 :   if (access->nonarg)
    2431              :     {
    2432       340606 :       *only_calls = false;
    2433       340606 :       *nonarg_acc_size += access->size;
    2434              : 
    2435       340606 :       if (access->first_child)
    2436              :         {
    2437          384 :           disqualify_split_candidate (desc, "Overlapping non-call uses.");
    2438          384 :           return true;
    2439              :         }
    2440              :     }
    2441              :   /* Do not decompose a non-BLKmode param in a way that would create
    2442              :      BLKmode params.  Especially for by-reference passing (thus,
    2443              :      pointer-type param) this is hardly worthwhile.  */
    2444       392643 :   if (DECL_MODE (parm) != BLKmode
    2445       392643 :       && TYPE_MODE (access->type) == BLKmode)
    2446              :     {
    2447         3586 :       disqualify_split_candidate (desc, "Would convert a non-BLK to a BLK.");
    2448         3586 :       return true;
    2449              :     }
    2450              : 
    2451       389057 :   if (desc->by_ref)
    2452              :     {
    2453       189941 :       if (desc->safe_ref)
    2454              :         {
    2455       154238 :           if (!dereference_probable_p (fun, access))
    2456              :             {
    2457         6417 :               disqualify_split_candidate (desc, "Dereferences in callers "
    2458              :                                           "would happen much more frequently.");
    2459         6417 :               return true;
    2460              :             }
    2461              :         }
    2462              :       else
    2463              :         {
    2464        35703 :           int idx = (entry_bb_index * unsafe_by_ref_count + desc->deref_index);
    2465        35703 :           if ((access->offset + access->size) > bb_dereferences[idx])
    2466              :             {
    2467        24916 :               if (!dereference_probable_p (fun, access))
    2468              :                 {
    2469         2218 :                   disqualify_split_candidate (desc, "Would create a possibly "
    2470              :                                               "illegal dereference in a "
    2471              :                                               "caller.");
    2472         2218 :                   return true;
    2473              :                 }
    2474        22698 :               desc->conditionally_dereferenceable = true;
    2475              :             }
    2476              :         }
    2477              :     }
    2478              : 
    2479       380422 :   for (gensum_param_access *ch = access->first_child;
    2480       402544 :        ch;
    2481        22122 :        ch = ch->next_sibling)
    2482        22124 :     if (check_gensum_access (fun, parm, desc, ch, nonarg_acc_size, only_calls,
    2483              :                              entry_bb_index))
    2484              :       return true;
    2485              : 
    2486              :   return false;
    2487              : }
    2488              : 
    2489              : /* Copy data from FROM and all of its children to a vector of accesses in IPA
    2490              :    descriptor DESC.  */
    2491              : 
    2492              : static void
    2493       450149 : copy_accesses_to_ipa_desc (gensum_param_access *from, isra_param_desc *desc)
    2494              : {
    2495       450149 :   param_access *to = ggc_cleared_alloc<param_access> ();
    2496       450149 :   gcc_checking_assert ((from->offset % BITS_PER_UNIT) == 0);
    2497       450149 :   gcc_checking_assert ((from->size % BITS_PER_UNIT) == 0);
    2498       450149 :   to->unit_offset = from->offset / BITS_PER_UNIT;
    2499       450149 :   to->unit_size = from->size / BITS_PER_UNIT;
    2500       450149 :   to->type = from->type;
    2501       450149 :   to->alias_ptr_type = from->alias_ptr_type;
    2502       450149 :   to->certain = from->nonarg;
    2503       450149 :   to->reverse = from->reverse;
    2504       450149 :   vec_safe_push (desc->accesses, to);
    2505              : 
    2506       450149 :   for (gensum_param_access *ch = from->first_child;
    2507       474398 :        ch;
    2508        24249 :        ch = ch->next_sibling)
    2509        24249 :     copy_accesses_to_ipa_desc (ch, desc);
    2510       450149 : }
    2511              : 
    2512              : /* Analyze function body scan results stored in param_accesses and
    2513              :    param_accesses, detect possible transformations and store information of
    2514              :    those in function summary.  NODE, FUN and IFS are all various structures
    2515              :    describing the currently analyzed function.  */
    2516              : 
    2517              : static void
    2518       861037 : process_scan_results (cgraph_node *node, struct function *fun,
    2519              :                       isra_func_summary *ifs,
    2520              :                       vec<gensum_param_desc> *param_descriptions)
    2521              : {
    2522       861037 :   bool check_pass_throughs = false;
    2523       861037 :   bool dereferences_propagated = false;
    2524       861037 :   tree parm = DECL_ARGUMENTS (node->decl);
    2525       861037 :   unsigned param_count = param_descriptions->length();
    2526              : 
    2527       861037 :   for (unsigned desc_index = 0;
    2528      3308445 :        desc_index < param_count;
    2529      2447408 :        desc_index++, parm = DECL_CHAIN (parm))
    2530              :     {
    2531      2447408 :       gensum_param_desc *desc = &(*param_descriptions)[desc_index];
    2532      2447408 :       if (!desc->split_candidate)
    2533      1915540 :         continue;
    2534              : 
    2535       544473 :       if (flag_checking)
    2536       544467 :         isra_verify_access_tree (desc->accesses);
    2537              : 
    2538       544473 :       if (!dereferences_propagated
    2539       538606 :           && desc->by_ref
    2540       372657 :           && !desc->safe_ref
    2541       166058 :           && desc->accesses)
    2542              :         {
    2543        27106 :           propagate_dereference_distances (fun);
    2544        27106 :           dereferences_propagated = true;
    2545              :         }
    2546              : 
    2547       544473 :       HOST_WIDE_INT nonarg_acc_size = 0;
    2548       544473 :       bool only_calls = true;
    2549       544473 :       bool check_failed = false;
    2550              : 
    2551       544473 :       int entry_bb_index = ENTRY_BLOCK_PTR_FOR_FN (fun)->index;
    2552       544473 :       for (gensum_param_access *acc = desc->accesses;
    2553       902771 :            acc;
    2554       358298 :            acc = acc->next_sibling)
    2555       370903 :         if (check_gensum_access (fun, parm, desc, acc, &nonarg_acc_size,
    2556              :                                  &only_calls, entry_bb_index))
    2557              :           {
    2558              :             check_failed = true;
    2559              :             break;
    2560              :           }
    2561       544473 :       if (check_failed)
    2562        12605 :         continue;
    2563              : 
    2564       531868 :       if (only_calls)
    2565       329515 :         desc->locally_unused = true;
    2566              : 
    2567       531868 :       HOST_WIDE_INT cur_param_size
    2568       531868 :         = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (parm)));
    2569       531868 :       HOST_WIDE_INT param_size_limit, optimistic_limit;
    2570       531868 :       if (!desc->by_ref || optimize_function_for_size_p (fun))
    2571              :         {
    2572              :           param_size_limit = cur_param_size;
    2573              :           optimistic_limit = cur_param_size;
    2574              :         }
    2575              :       else
    2576              :         {
    2577       342878 :           param_size_limit
    2578       342878 :             = opt_for_fn (node->decl,
    2579              :                           param_ipa_sra_ptr_growth_factor) * cur_param_size;
    2580       342878 :           optimistic_limit
    2581       342878 :             = (opt_for_fn (node->decl, param_ipa_sra_ptrwrap_growth_factor)
    2582              :                * param_size_limit);
    2583              :         }
    2584              : 
    2585              :       /* Assume functions have no real replacements, no calls to them will make
    2586              :          it into the binary, so limits don't matter.  */
    2587       531868 :       if (!node->assume_function
    2588       531827 :           && (nonarg_acc_size > optimistic_limit
    2589       529331 :               || (!desc->by_ref && nonarg_acc_size == param_size_limit)))
    2590              :         {
    2591        73165 :           disqualify_split_candidate (desc, "Would result into a too big set "
    2592              :                                       "of replacements even in best "
    2593              :                                       "scenarios.");
    2594              :         }
    2595              :       else
    2596              :         {
    2597              :           /* create_parameter_descriptors makes sure unit sizes of all
    2598              :              candidate parameters fit unsigned integers restricted to
    2599              :              ISRA_ARG_SIZE_LIMIT.  */
    2600       458703 :           desc->param_size_limit = param_size_limit / BITS_PER_UNIT;
    2601       458703 :           desc->nonarg_acc_size = nonarg_acc_size / BITS_PER_UNIT;
    2602       458703 :           if (desc->split_candidate && desc->ptr_pt_count)
    2603              :             {
    2604       190532 :               gcc_assert (desc->by_ref);
    2605              :               check_pass_throughs = true;
    2606              :             }
    2607              :         }
    2608              :     }
    2609              : 
    2610              :   /* When a pointer parameter is passed-through to a callee, in which it is
    2611              :      only used to read only one or a few items, we can attempt to transform it
    2612              :      to obtaining and passing through the items instead of the pointer.  But we
    2613              :      must take extra care that 1) we do not introduce any segfault by moving
    2614              :      dereferences above control flow and that 2) the data is not modified
    2615              :      through an alias in this function.  The IPA analysis must not introduce
    2616              :      any accesses candidates unless it can prove both.
    2617              : 
    2618              :      The current solution is very crude as it consists of ensuring that the
    2619              :      call postdominates entry BB and that the definition of VUSE of the call is
    2620              :      default definition.  TODO: For non-recursive callees in the same
    2621              :      compilation unit we could do better by doing analysis in topological order
    2622              :      an looking into access candidates of callees, using their alias_ptr_types
    2623              :      to attempt real AA.  We could also use the maximum known dereferenced
    2624              :      offset in this function at IPA level.
    2625              : 
    2626              :      TODO: Measure the overhead and the effect of just being pessimistic.
    2627              :      Maybe this is only -O3 material?  */
    2628              : 
    2629       861037 :   hash_map<gimple *, bool> analyzed_stmts;
    2630       861037 :   bitmap always_executed_bbs = NULL;
    2631       861037 :   if (check_pass_throughs)
    2632      1213147 :     for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
    2633              :       {
    2634      1059739 :         gcall *call_stmt = cs->call_stmt;
    2635      1059739 :         tree vuse = gimple_vuse (call_stmt);
    2636              : 
    2637              :         /* If the callee is a const function, we don't get a VUSE.  In such
    2638              :            case there will be no memory accesses in the called function (or the
    2639              :            const attribute is wrong) and then we just don't care.  */
    2640      2068974 :         bool uses_memory_as_obtained = vuse && SSA_NAME_IS_DEFAULT_DEF (vuse);
    2641              : 
    2642      1059739 :         unsigned count = gimple_call_num_args (call_stmt);
    2643      1059739 :         isra_call_summary *csum = call_sums->get_create (cs);
    2644      1059739 :         csum->init_inputs (count);
    2645      1059739 :         csum->m_before_any_store = uses_memory_as_obtained;
    2646      3421565 :         for (unsigned argidx = 0; argidx < count; argidx++)
    2647              :           {
    2648      2361826 :             if (!csum->m_arg_flow[argidx].pointer_pass_through)
    2649      1928131 :               continue;
    2650       433695 :             unsigned pidx
    2651       433695 :               = get_single_param_flow_source (&csum->m_arg_flow[argidx]);
    2652       433695 :             gensum_param_desc *desc = &(*param_descriptions)[pidx];
    2653       433695 :             if (!desc->split_candidate)
    2654              :               {
    2655        36752 :                 csum->m_arg_flow[argidx].pointer_pass_through = false;
    2656        36752 :                 continue;
    2657              :               }
    2658       396943 :             if (!uses_memory_as_obtained)
    2659       150955 :               continue;
    2660              : 
    2661       245988 :             if (desc->safe_ref)
    2662              :               {
    2663        45890 :                 csum->m_arg_flow[argidx].safe_to_import_accesses = true;
    2664        45890 :                 continue;
    2665              :               }
    2666              : 
    2667              :             /* Walk basic block and see if its execution can terminate earlier.
    2668              :                Keep the info for later re-use to avoid quadratic behaviour here.  */
    2669       200098 :             gimple_stmt_iterator gsi = gsi_for_stmt (call_stmt);
    2670       200098 :             bool safe = true;
    2671       200098 :             int n = 0;
    2672       500277 :             for (gsi_prev (&gsi); !gsi_end_p (gsi); gsi_prev (&gsi))
    2673              :               {
    2674       127121 :                 bool *b = analyzed_stmts.get (gsi_stmt (gsi));
    2675       127121 :                 if (b)
    2676              :                   {
    2677         9879 :                     safe = *b;
    2678         9879 :                     gsi_next (&gsi);
    2679         9879 :                     break;
    2680              :                   }
    2681       117242 :                 n++;
    2682       117242 :                 if (stmt_may_terminate_function_p (fun, gsi_stmt (gsi), false))
    2683              :                   {
    2684              :                     safe = false;
    2685              :                     break;
    2686              :                   }
    2687              :               }
    2688       200098 :             if (n)
    2689              :               {
    2690        33261 :                 if (gsi_end_p (gsi))
    2691        59240 :                   gsi = gsi_start_bb (gimple_bb (call_stmt));
    2692       150503 :                 for (; gsi_stmt (gsi) != call_stmt; gsi_next (&gsi))
    2693       117242 :                   analyzed_stmts.get_or_insert (gsi_stmt (gsi)) = safe;
    2694              :               }
    2695              : 
    2696       200098 :             if (safe && !always_executed_bbs)
    2697              :               {
    2698        57558 :                 mark_dfs_back_edges ();
    2699        57558 :                 always_executed_bbs = find_always_executed_bbs (fun, false);
    2700              :               }
    2701       200098 :             if (safe && bitmap_bit_p (always_executed_bbs, gimple_bb (call_stmt)->index))
    2702        46557 :               csum->m_arg_flow[argidx].safe_to_import_accesses = true;
    2703              :           }
    2704              : 
    2705              :       }
    2706       861037 :   BITMAP_FREE (always_executed_bbs);
    2707              : 
    2708              :   /* TODO: Add early exit if we disqualified everything.  This also requires
    2709              :      that we either relax the restriction that
    2710              :      ipa_param_adjustments.m_always_copy_start must be the number of PARM_DECLs
    2711              :      or store the number of parameters to IPA-SRA function summary and use that
    2712              :      when just removing params.  */
    2713              : 
    2714       861037 :   vec_safe_reserve_exact (ifs->m_parameters, param_count);
    2715       861037 :   ifs->m_parameters->quick_grow_cleared (param_count);
    2716      4169482 :   for (unsigned desc_index = 0; desc_index < param_count; desc_index++)
    2717              :     {
    2718      2447408 :       gensum_param_desc *s = &(*param_descriptions)[desc_index];
    2719      2447408 :       isra_param_desc *d = &(*ifs->m_parameters)[desc_index];
    2720              : 
    2721      2447408 :       d->param_size_limit = s->param_size_limit;
    2722      2447408 :       d->size_reached = s->nonarg_acc_size;
    2723      2447408 :       d->locally_unused = s->locally_unused;
    2724      2447408 :       d->split_candidate = s->split_candidate;
    2725      2447408 :       d->by_ref = s->by_ref;
    2726      2447408 :       d->remove_only_when_retval_removed = s->remove_only_when_retval_removed;
    2727      2447408 :       d->split_only_when_retval_removed = s->split_only_when_retval_removed;
    2728      2447408 :       d->conditionally_dereferenceable = s->conditionally_dereferenceable;
    2729              : 
    2730      2447408 :       for (gensum_param_access *acc = s->accesses;
    2731      2873308 :            acc;
    2732       425900 :            acc = acc->next_sibling)
    2733       425900 :         copy_accesses_to_ipa_desc (acc, d);
    2734              :     }
    2735              : 
    2736       861037 :   if (dump_file)
    2737          172 :     dump_isra_param_descriptors (dump_file, node->decl, ifs, false);
    2738       861037 : }
    2739              : 
    2740              : /* Return true if there are any overlaps among certain accesses of DESC.  If
    2741              :    non-NULL, set *CERTAIN_ACCESS_PRESENT_P upon encountering a certain access
    2742              :    too.  DESC is assumed to be a split candidate that is not locally
    2743              :    unused.  */
    2744              : 
    2745              : static bool
    2746       176733 : overlapping_certain_accesses_p (isra_param_desc *desc,
    2747              :                                 bool *certain_access_present_p)
    2748              : {
    2749       176733 :   unsigned pclen = vec_safe_length (desc->accesses);
    2750       463557 :   for (unsigned i = 0; i < pclen; i++)
    2751              :     {
    2752       289043 :       param_access *a1 = (*desc->accesses)[i];
    2753              : 
    2754       289043 :       if (!a1->certain)
    2755         6280 :         continue;
    2756       282763 :       if (certain_access_present_p)
    2757       266712 :         *certain_access_present_p = true;
    2758       460674 :       for (unsigned j = i + 1; j < pclen; j++)
    2759              :         {
    2760       180130 :           param_access *a2 = (*desc->accesses)[j];
    2761       180130 :           if (a2->certain
    2762       180056 :               && a1->unit_offset < a2->unit_offset + a2->unit_size
    2763       179691 :               && a1->unit_offset + a1->unit_size > a2->unit_offset)
    2764              :             return true;
    2765              :         }
    2766              :     }
    2767              :   return false;
    2768              : }
    2769              : 
    2770              : /* Check for any overlaps of certain param accesses among splitting candidates
    2771              :    and signal an ICE if there are any.  If CERTAIN_MUST_EXIST is set, also
    2772              :    check that used splitting candidates have at least one certain access.  */
    2773              : 
    2774              : static void
    2775      2685922 : verify_splitting_accesses (cgraph_node *node, bool certain_must_exist)
    2776              : {
    2777      2685922 :   isra_func_summary *ifs = func_sums->get (node);
    2778      2685922 :   if (!ifs || !ifs->m_candidate)
    2779              :     return;
    2780      1449573 :   unsigned param_count = vec_safe_length (ifs->m_parameters);
    2781      4697540 :   for (unsigned pidx = 0; pidx < param_count; pidx++)
    2782              :     {
    2783      3247967 :       isra_param_desc *desc = &(*ifs->m_parameters)[pidx];
    2784      3247967 :       if (!desc->split_candidate || desc->locally_unused)
    2785      3087216 :         continue;
    2786              : 
    2787       160751 :       bool certain_access_present = !certain_must_exist;
    2788       160751 :       if (overlapping_certain_accesses_p (desc, &certain_access_present))
    2789            0 :         internal_error ("function %qs, parameter %u, has IPA-SRA accesses "
    2790              :                         "which overlap", node->dump_name (), pidx);
    2791       160751 :       if (!certain_access_present)
    2792            0 :         internal_error ("function %qs, parameter %u, is used but does not "
    2793              :                         "have any certain IPA-SRA access",
    2794              :                         node->dump_name (), pidx);
    2795              :     }
    2796              : }
    2797              : 
    2798              : /* Intraprocedural part of IPA-SRA analysis.  Scan bodies of all functions in
    2799              :    this compilation unit and create summary structures describing IPA-SRA
    2800              :    opportunities and constraints in them.  */
    2801              : 
    2802              : static void
    2803       129344 : ipa_sra_generate_summary (void)
    2804              : {
    2805       129344 :   struct cgraph_node *node;
    2806              : 
    2807       129344 :   gcc_checking_assert (!func_sums);
    2808       129344 :   gcc_checking_assert (!call_sums);
    2809       129344 :   func_sums
    2810       129344 :     = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
    2811       129344 :        ipa_sra_function_summaries (symtab, true));
    2812       129344 :   call_sums = new ipa_sra_call_summaries (symtab);
    2813       129344 :   callback_info_sum_t::check_create_info_sum ();
    2814              : 
    2815      1414557 :   FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
    2816      1285213 :     ipa_sra_summarize_function (node);
    2817       129344 :   return;
    2818              : }
    2819              : 
    2820              : /* Write intraprocedural analysis information about E and all of its outgoing
    2821              :    edges into a stream for LTO WPA.  */
    2822              : 
    2823              : static void
    2824       285084 : isra_write_edge_summary (output_block *ob, cgraph_edge *e)
    2825              : {
    2826       285084 :   isra_call_summary *csum = call_sums->get (e);
    2827       285084 :   unsigned input_count = csum->m_arg_flow.length ();
    2828       285084 :   streamer_write_uhwi (ob, input_count);
    2829      1045914 :   for (unsigned i = 0; i < input_count; i++)
    2830              :     {
    2831       475746 :       isra_param_flow *ipf = &csum->m_arg_flow[i];
    2832       475746 :       streamer_write_hwi (ob, ipf->length);
    2833       475746 :       bitpack_d bp = bitpack_create (ob->main_stream);
    2834       551284 :       for (int j = 0; j < ipf->length; j++)
    2835        75538 :         bp_pack_value (&bp, ipf->inputs[j], 8);
    2836       475746 :       bp_pack_value (&bp, ipf->aggregate_pass_through, 1);
    2837       475746 :       bp_pack_value (&bp, ipf->pointer_pass_through, 1);
    2838       475746 :       bp_pack_value (&bp, ipf->safe_to_import_accesses, 1);
    2839       475746 :       bp_pack_value (&bp, ipf->constructed_for_calls, 1);
    2840       475746 :       streamer_write_bitpack (&bp);
    2841       475746 :       streamer_write_uhwi (ob, ipf->unit_offset);
    2842       475746 :       streamer_write_uhwi (ob, ipf->unit_size);
    2843              :     }
    2844       285084 :   bitpack_d bp = bitpack_create (ob->main_stream);
    2845       285084 :   bp_pack_value (&bp, csum->m_return_ignored, 1);
    2846       285084 :   bp_pack_value (&bp, csum->m_return_returned, 1);
    2847       285084 :   bp_pack_value (&bp, csum->m_bit_aligned_arg, 1);
    2848       285084 :   bp_pack_value (&bp, csum->m_before_any_store, 1);
    2849       285084 :   streamer_write_bitpack (&bp);
    2850       285084 : }
    2851              : 
    2852              : /* Write intraprocedural analysis information about NODE and all of its outgoing
    2853              :    edges into a stream for LTO WPA.  */
    2854              : 
    2855              : static void
    2856        50863 : isra_write_node_summary (output_block *ob, cgraph_node *node)
    2857              : {
    2858        50863 :   isra_func_summary *ifs = func_sums->get (node);
    2859        50863 :   lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
    2860        50863 :   int node_ref = lto_symtab_encoder_encode (encoder, node);
    2861        50863 :   streamer_write_uhwi (ob, node_ref);
    2862              : 
    2863        50863 :   unsigned param_desc_count = vec_safe_length (ifs->m_parameters);
    2864        50863 :   streamer_write_uhwi (ob, param_desc_count);
    2865       353426 :   for (unsigned i = 0; i < param_desc_count; i++)
    2866              :     {
    2867       251700 :       isra_param_desc *desc = &(*ifs->m_parameters)[i];
    2868       251700 :       unsigned access_count = vec_safe_length (desc->accesses);
    2869       251700 :       streamer_write_uhwi (ob, access_count);
    2870       506876 :       for (unsigned j = 0; j < access_count; j++)
    2871              :         {
    2872         3476 :           param_access *acc = (*desc->accesses)[j];
    2873         3476 :           stream_write_tree (ob, acc->type, true);
    2874         3476 :           stream_write_tree (ob, acc->alias_ptr_type, true);
    2875         3476 :           streamer_write_uhwi (ob, acc->unit_offset);
    2876         3476 :           streamer_write_uhwi (ob, acc->unit_size);
    2877         3476 :           bitpack_d bp = bitpack_create (ob->main_stream);
    2878         3476 :           bp_pack_value (&bp, acc->certain, 1);
    2879         3476 :           bp_pack_value (&bp, acc->reverse, 1);
    2880         3476 :           streamer_write_bitpack (&bp);
    2881              :         }
    2882       251700 :       streamer_write_uhwi (ob, desc->param_size_limit);
    2883       251700 :       streamer_write_uhwi (ob, desc->size_reached);
    2884       251700 :       gcc_assert (desc->safe_size == 0);
    2885       251700 :       bitpack_d bp = bitpack_create (ob->main_stream);
    2886       251700 :       bp_pack_value (&bp, desc->locally_unused, 1);
    2887       251700 :       bp_pack_value (&bp, desc->split_candidate, 1);
    2888       251700 :       bp_pack_value (&bp, desc->by_ref, 1);
    2889       251700 :       gcc_assert (!desc->not_specially_constructed);
    2890       251700 :       bp_pack_value (&bp, desc->remove_only_when_retval_removed, 1);
    2891       251700 :       bp_pack_value (&bp, desc->split_only_when_retval_removed, 1);
    2892       251700 :       bp_pack_value (&bp, desc->conditionally_dereferenceable, 1);
    2893       251700 :       gcc_assert (!desc->safe_size_set);
    2894       251700 :       streamer_write_bitpack (&bp);
    2895              :     }
    2896        50863 :   bitpack_d bp = bitpack_create (ob->main_stream);
    2897        50863 :   bp_pack_value (&bp, ifs->m_candidate, 1);
    2898        50863 :   bp_pack_value (&bp, ifs->m_returns_value, 1);
    2899        50863 :   bp_pack_value (&bp, ifs->m_return_ignored, 1);
    2900        50863 :   gcc_assert (!ifs->m_queued);
    2901        50863 :   streamer_write_bitpack (&bp);
    2902              : 
    2903       333472 :   for (cgraph_edge *e = node->callees; e; e = e->next_callee)
    2904       282609 :     isra_write_edge_summary (ob, e);
    2905        53338 :   for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
    2906         2475 :     isra_write_edge_summary (ob, e);
    2907        50863 : }
    2908              : 
    2909              : /* Write intraprocedural analysis information into a stream for LTO WPA.  */
    2910              : 
    2911              : static void
    2912        20640 : ipa_sra_write_summary (void)
    2913              : {
    2914        20640 :   if (!func_sums || !call_sums)
    2915            0 :     return;
    2916              : 
    2917        20640 :   struct output_block *ob = create_output_block (LTO_section_ipa_sra);
    2918        20640 :   lto_symtab_encoder_t encoder = ob->decl_state->symtab_node_encoder;
    2919        20640 :   ob->symbol = NULL;
    2920              : 
    2921        20640 :   unsigned int count = 0;
    2922        20640 :   lto_symtab_encoder_iterator lsei;
    2923        20640 :   for (lsei = lsei_start_function_in_partition (encoder);
    2924       116630 :        !lsei_end_p (lsei);
    2925        95990 :        lsei_next_function_in_partition (&lsei))
    2926              :     {
    2927        95990 :       cgraph_node *node = lsei_cgraph_node (lsei);
    2928        95990 :       if (node->has_gimple_body_p ()
    2929        95990 :           && func_sums->get (node) != NULL)
    2930        50863 :         count++;
    2931              :     }
    2932        20640 :   streamer_write_uhwi (ob, count);
    2933              : 
    2934              :   /* Process all of the functions.  */
    2935       116630 :   for (lsei = lsei_start_function_in_partition (encoder); !lsei_end_p (lsei);
    2936        95990 :        lsei_next_function_in_partition (&lsei))
    2937              :     {
    2938        95990 :       cgraph_node *node = lsei_cgraph_node (lsei);
    2939        95990 :       if (node->has_gimple_body_p ()
    2940        95990 :           && func_sums->get (node) != NULL)
    2941        50863 :         isra_write_node_summary (ob, node);
    2942              :     }
    2943        20640 :   streamer_write_char_stream (ob->main_stream, 0);
    2944        20640 :   produce_asm (ob);
    2945        20640 :   destroy_output_block (ob);
    2946              : }
    2947              : 
    2948              : /* Read intraprocedural analysis information about E and all of its outgoing
    2949              :    edges into a stream for LTO WPA.  */
    2950              : 
    2951              : static void
    2952       256499 : isra_read_edge_summary (struct lto_input_block *ib, cgraph_edge *cs)
    2953              : {
    2954       256499 :   isra_call_summary *csum = call_sums->get_create (cs);
    2955       256499 :   unsigned input_count = streamer_read_uhwi (ib);
    2956       256499 :   csum->init_inputs (input_count);
    2957       962641 :   for (unsigned i = 0; i < input_count; i++)
    2958              :     {
    2959       449643 :       isra_param_flow *ipf = &csum->m_arg_flow[i];
    2960       449643 :       ipf->length = streamer_read_hwi (ib);
    2961       449643 :       bitpack_d bp = streamer_read_bitpack (ib);
    2962       517505 :       for (int j = 0; j < ipf->length; j++)
    2963        67862 :         ipf->inputs[j] = bp_unpack_value (&bp, 8);
    2964       449643 :       ipf->aggregate_pass_through = bp_unpack_value (&bp, 1);
    2965       449643 :       ipf->pointer_pass_through = bp_unpack_value (&bp, 1);
    2966       449643 :       ipf->safe_to_import_accesses = bp_unpack_value (&bp, 1);
    2967       449643 :       ipf->constructed_for_calls = bp_unpack_value (&bp, 1);
    2968       449643 :       ipf->unit_offset = streamer_read_uhwi (ib);
    2969       449643 :       ipf->unit_size = streamer_read_uhwi (ib);
    2970              :     }
    2971       256499 :   bitpack_d bp = streamer_read_bitpack (ib);
    2972       256499 :   csum->m_return_ignored = bp_unpack_value (&bp, 1);
    2973       256499 :   csum->m_return_returned = bp_unpack_value (&bp, 1);
    2974       256499 :   csum->m_bit_aligned_arg = bp_unpack_value (&bp, 1);
    2975       256499 :   csum->m_before_any_store = bp_unpack_value (&bp, 1);
    2976       256499 : }
    2977              : 
    2978              : /* Read intraprocedural analysis information about NODE and all of its outgoing
    2979              :    edges into a stream for LTO WPA.  */
    2980              : 
    2981              : static void
    2982        36085 : isra_read_node_info (struct lto_input_block *ib, cgraph_node *node,
    2983              :                      struct data_in *data_in)
    2984              : {
    2985        36085 :   isra_func_summary *ifs = func_sums->get_create (node);
    2986        36085 :   unsigned param_desc_count = streamer_read_uhwi (ib);
    2987        36085 :   if (param_desc_count > 0)
    2988              :     {
    2989        18370 :       vec_safe_reserve_exact (ifs->m_parameters, param_desc_count);
    2990        18370 :       ifs->m_parameters->quick_grow_cleared (param_desc_count);
    2991              :     }
    2992        65350 :   for (unsigned i = 0; i < param_desc_count; i++)
    2993              :     {
    2994        29265 :       isra_param_desc *desc = &(*ifs->m_parameters)[i];
    2995        29265 :       unsigned access_count = streamer_read_uhwi (ib);
    2996        31181 :       for (unsigned j = 0; j < access_count; j++)
    2997              :         {
    2998         1916 :           param_access *acc = ggc_cleared_alloc<param_access> ();
    2999         1916 :           acc->type = stream_read_tree (ib, data_in);
    3000         1916 :           acc->alias_ptr_type = stream_read_tree (ib, data_in);
    3001         1916 :           acc->unit_offset = streamer_read_uhwi (ib);
    3002         1916 :           acc->unit_size = streamer_read_uhwi (ib);
    3003         1916 :           bitpack_d bp = streamer_read_bitpack (ib);
    3004         1916 :           acc->certain = bp_unpack_value (&bp, 1);
    3005         1916 :           acc->reverse = bp_unpack_value (&bp, 1);
    3006         1916 :           vec_safe_push (desc->accesses, acc);
    3007              :         }
    3008        29265 :       desc->param_size_limit = streamer_read_uhwi (ib);
    3009        29265 :       desc->size_reached = streamer_read_uhwi (ib);
    3010        29265 :       desc->safe_size = 0;
    3011        29265 :       bitpack_d bp = streamer_read_bitpack (ib);
    3012        29265 :       desc->locally_unused = bp_unpack_value (&bp, 1);
    3013        29265 :       desc->split_candidate = bp_unpack_value (&bp, 1);
    3014        29265 :       desc->by_ref = bp_unpack_value (&bp, 1);
    3015        29265 :       desc->not_specially_constructed = 0;
    3016        29265 :       desc->remove_only_when_retval_removed = bp_unpack_value (&bp, 1);
    3017        29265 :       desc->split_only_when_retval_removed = bp_unpack_value (&bp, 1);
    3018        29265 :       desc->conditionally_dereferenceable = bp_unpack_value (&bp, 1);
    3019        29265 :       desc->safe_size_set = 0;
    3020              :     }
    3021        36085 :   bitpack_d bp = streamer_read_bitpack (ib);
    3022        36085 :   ifs->m_candidate = bp_unpack_value (&bp, 1);
    3023        36085 :   ifs->m_returns_value = bp_unpack_value (&bp, 1);
    3024        36085 :   ifs->m_return_ignored = bp_unpack_value (&bp, 1);
    3025        36085 :   ifs->m_queued = 0;
    3026              : 
    3027       291241 :   for (cgraph_edge *e = node->callees; e; e = e->next_callee)
    3028       255156 :     isra_read_edge_summary (ib, e);
    3029        37428 :   for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
    3030         1343 :     isra_read_edge_summary (ib, e);
    3031        36085 : }
    3032              : 
    3033              : /* Read IPA-SRA summaries from a section in file FILE_DATA of length LEN with
    3034              :    data DATA.  TODO: This function was copied almost verbatim from ipa-prop.cc,
    3035              :    it should be possible to unify them somehow.  */
    3036              : 
    3037              : static void
    3038        11415 : isra_read_summary_section (struct lto_file_decl_data *file_data,
    3039              :                            const char *data, size_t len)
    3040              : {
    3041        11415 :   const struct lto_function_header *header =
    3042              :     (const struct lto_function_header *) data;
    3043        11415 :   const int cfg_offset = sizeof (struct lto_function_header);
    3044        11415 :   const int main_offset = cfg_offset + header->cfg_size;
    3045        11415 :   const int string_offset = main_offset + header->main_size;
    3046        11415 :   struct data_in *data_in;
    3047        11415 :   unsigned int i;
    3048        11415 :   unsigned int count;
    3049              : 
    3050        11415 :   lto_input_block ib_main ((const char *) data + main_offset,
    3051        11415 :                            header->main_size, file_data);
    3052              : 
    3053        11415 :   data_in =
    3054        22830 :     lto_data_in_create (file_data, (const char *) data + string_offset,
    3055        11415 :                         header->string_size, vNULL);
    3056        11415 :   count = streamer_read_uhwi (&ib_main);
    3057              : 
    3058        47500 :   for (i = 0; i < count; i++)
    3059              :     {
    3060        36085 :       unsigned int index;
    3061        36085 :       struct cgraph_node *node;
    3062        36085 :       lto_symtab_encoder_t encoder;
    3063              : 
    3064        36085 :       index = streamer_read_uhwi (&ib_main);
    3065        36085 :       encoder = file_data->symtab_node_encoder;
    3066        36085 :       node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
    3067              :                                                                 index));
    3068        36085 :       gcc_assert (node->definition);
    3069        36085 :       isra_read_node_info (&ib_main, node, data_in);
    3070              :     }
    3071        11415 :   lto_free_section_data (file_data, LTO_section_ipa_sra, NULL, data,
    3072              :                          len);
    3073        11415 :   lto_data_in_delete (data_in);
    3074        11415 : }
    3075              : 
    3076              : /* Read intraprocedural analysis information into a stream for LTO WPA.  */
    3077              : 
    3078              : static void
    3079        10696 : ipa_sra_read_summary (void)
    3080              : {
    3081        10696 :   struct lto_file_decl_data **file_data_vec = lto_get_file_decl_data ();
    3082        10696 :   struct lto_file_decl_data *file_data;
    3083        10696 :   unsigned int j = 0;
    3084              : 
    3085        10696 :   gcc_checking_assert (!func_sums);
    3086        10696 :   gcc_checking_assert (!call_sums);
    3087        10696 :   func_sums
    3088        10696 :     = (new (ggc_alloc_no_dtor <ipa_sra_function_summaries> ())
    3089        10696 :        ipa_sra_function_summaries (symtab, true));
    3090        10696 :   call_sums = new ipa_sra_call_summaries (symtab);
    3091              : 
    3092        22122 :   while ((file_data = file_data_vec[j++]))
    3093              :     {
    3094        11426 :       size_t len;
    3095        11426 :       const char *data
    3096        11426 :         = lto_get_summary_section_data (file_data, LTO_section_ipa_sra, &len);
    3097        11426 :       if (data)
    3098        11415 :         isra_read_summary_section (file_data, data, len);
    3099              :     }
    3100        10696 : }
    3101              : 
    3102              : /* Dump all IPA-SRA summary data for all cgraph nodes and edges to file F.  If
    3103              :    HINTS is true, also dump IPA-analysis computed hints.  */
    3104              : 
    3105              : static void
    3106           79 : ipa_sra_dump_all_summaries (FILE *f, bool hints)
    3107              : {
    3108           79 :   cgraph_node *node;
    3109          350 :   FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
    3110              :     {
    3111          271 :       fprintf (f, "\nSummary for node %s:\n", node->dump_name ());
    3112              : 
    3113          271 :       isra_func_summary *ifs = func_sums->get (node);
    3114          271 :       if (!ifs)
    3115           10 :         fprintf (f, "  Function does not have any associated IPA-SRA "
    3116              :                  "summary\n");
    3117          261 :       else if (!ifs->m_candidate)
    3118           26 :         fprintf (f, "  Not a candidate function\n");
    3119              :       else
    3120              :         {
    3121          235 :           if (ifs->m_returns_value)
    3122          137 :             fprintf (f, "  Returns value\n");
    3123          235 :           if (vec_safe_is_empty (ifs->m_parameters))
    3124           43 :             fprintf (f, "  No parameter information. \n");
    3125              :           else
    3126          566 :             for (unsigned i = 0; i < ifs->m_parameters->length (); ++i)
    3127              :               {
    3128          374 :                 fprintf (f, "  Descriptor for parameter %i:\n", i);
    3129          374 :                 dump_isra_param_descriptor (f, &(*ifs->m_parameters)[i], hints);
    3130              :               }
    3131          235 :           fprintf (f, "\n");
    3132              :         }
    3133              : 
    3134          271 :       struct cgraph_edge *cs;
    3135          567 :       for (cs = node->callees; cs; cs = cs->next_callee)
    3136              :         {
    3137          296 :           fprintf (f, "  Summary for edge %s->%s:\n", cs->caller->dump_name (),
    3138          296 :                    cs->callee->dump_name ());
    3139          296 :           isra_call_summary *csum = call_sums->get (cs);
    3140          296 :           if (csum)
    3141          296 :             csum->dump (f);
    3142              :           else
    3143            0 :             fprintf (f, "    Call summary is MISSING!\n");
    3144              :         }
    3145              : 
    3146              :     }
    3147           79 :   fprintf (f, "\n\n");
    3148           79 : }
    3149              : 
    3150              : /* Perform function-scope viability tests that can be only made at IPA level
    3151              :    and return false if the function is deemed unsuitable for IPA-SRA.  */
    3152              : 
    3153              : static bool
    3154      1056708 : ipa_sra_ipa_function_checks (cgraph_node *node)
    3155              : {
    3156              :   /* Assume bodies cannot be local, because their address is taken, but we want
    3157              :      to consider them anyway.  */
    3158      1056708 :   if (!node->can_be_local_p () && !node->assume_function)
    3159              :     {
    3160       670180 :       if (dump_file)
    3161          101 :         fprintf (dump_file, "Function %s disqualified because it cannot be "
    3162              :                  "made local.\n", node->dump_name ());
    3163              :       return false;
    3164              :     }
    3165       386528 :   if (!node->can_change_signature)
    3166              :     {
    3167            0 :       if (dump_file)
    3168            0 :         fprintf (dump_file, "Function can not change signature.\n");
    3169              :       return false;
    3170              :     }
    3171              : 
    3172              :   return true;
    3173              : }
    3174              : 
    3175              : /* Issues found out by check_callers_for_issues.  */
    3176              : 
    3177              : struct caller_issues
    3178              : {
    3179              :   /* The candidate being considered.  */
    3180              :   cgraph_node *candidate;
    3181              :   /* There is a thunk among callers.  */
    3182              :   bool thunk;
    3183              :   /* Set if there is at least one caller that is OK.  */
    3184              :   bool there_is_one;
    3185              :   /* Call site with no available information.  */
    3186              :   bool unknown_callsite;
    3187              :   /* Call from outside the candidate's comdat group.  */
    3188              :   bool call_from_outside_comdat;
    3189              :   /* There is a bit-aligned load into one of non-gimple-typed arguments. */
    3190              :   bool bit_aligned_aggregate_argument;
    3191              : };
    3192              : 
    3193              : /* Worker for call_for_symbol_and_aliases, set any flags of passed caller_issues
    3194              :    that apply.  */
    3195              : 
    3196              : static bool
    3197       392214 : check_for_caller_issues (struct cgraph_node *node, void *data)
    3198              : {
    3199       392214 :   struct caller_issues *issues = (struct caller_issues *) data;
    3200              : 
    3201      1499713 :   for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
    3202              :     {
    3203      1107744 :       if (cs->caller->thunk)
    3204              :         {
    3205            0 :           issues->thunk = true;
    3206              :           /* TODO: We should be able to process at least some types of
    3207              :              thunks.  */
    3208            0 :           return true;
    3209              :         }
    3210      1107744 :       if (issues->candidate->calls_comdat_local
    3211           10 :           && issues->candidate->same_comdat_group
    3212      1107754 :           && !issues->candidate->in_same_comdat_group_p (cs->caller))
    3213              :         {
    3214            0 :           issues->call_from_outside_comdat = true;
    3215            0 :           return true;
    3216              :         }
    3217              : 
    3218      1107744 :       isra_call_summary *csum = call_sums->get (cs);
    3219      1107744 :       if (!csum)
    3220              :         {
    3221          245 :           issues->unknown_callsite = true;
    3222          245 :           return true;
    3223              :         }
    3224              : 
    3225      1107499 :       if (csum->m_bit_aligned_arg)
    3226            0 :         issues->bit_aligned_aggregate_argument = true;
    3227              : 
    3228      1107499 :       issues->there_is_one = true;
    3229              :     }
    3230              :   return false;
    3231              : }
    3232              : 
    3233              : /* Look at all incoming edges to NODE, including aliases and thunks and look
    3234              :    for problems.  Return true if NODE type should not be modified at all.  */
    3235              : 
    3236              : static bool
    3237       386528 : check_all_callers_for_issues (cgraph_node *node)
    3238              : {
    3239       386528 :   struct caller_issues issues;
    3240       386528 :   memset (&issues, 0, sizeof (issues));
    3241       386528 :   issues.candidate = node;
    3242              : 
    3243       386528 :   node->call_for_symbol_and_aliases (check_for_caller_issues, &issues, true);
    3244       386528 :   if (issues.unknown_callsite)
    3245              :     {
    3246          245 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3247            0 :         fprintf (dump_file, "A call of %s has not been analyzed.  Disabling "
    3248              :                  "all modifications.\n", node->dump_name ());
    3249              :       return true;
    3250              :     }
    3251              :   /* TODO: We should be able to process at least some types of thunks.  */
    3252       386283 :   if (issues.thunk)
    3253              :     {
    3254            0 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3255            0 :         fprintf (dump_file, "A call of %s is through thunk, which are not"
    3256              :                  " handled yet.  Disabling all modifications.\n",
    3257              :                  node->dump_name ());
    3258              :       return true;
    3259              :     }
    3260       386283 :   if (issues.call_from_outside_comdat)
    3261              :     {
    3262            0 :       if (dump_file)
    3263            0 :         fprintf (dump_file, "Function would become private comdat called "
    3264              :                  "outside of its comdat group.\n");
    3265              :       return true;
    3266              :     }
    3267              : 
    3268       386283 :   if (issues.bit_aligned_aggregate_argument)
    3269              :     {
    3270              :       /* Let's only remove parameters/return values from such functions.
    3271              :          TODO: We could only prevent splitting the problematic parameters if
    3272              :          anybody thinks it is worth it.  */
    3273            0 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3274            0 :         fprintf (dump_file, "A call of %s has bit-aligned aggregate argument,"
    3275              :                  " disabling parameter splitting.\n", node->dump_name ());
    3276              : 
    3277            0 :       isra_func_summary *ifs = func_sums->get (node);
    3278            0 :       gcc_checking_assert (ifs);
    3279            0 :       unsigned param_count = vec_safe_length (ifs->m_parameters);
    3280            0 :       for (unsigned i = 0; i < param_count; i++)
    3281            0 :         (*ifs->m_parameters)[i].split_candidate = false;
    3282              :     }
    3283       386283 :   if (!issues.there_is_one)
    3284              :     {
    3285            1 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3286            0 :         fprintf (dump_file, "There is no call to %s that we can modify.  "
    3287              :                  "Disabling all modifications.\n", node->dump_name ());
    3288              :       return true;
    3289              :     }
    3290              :   return false;
    3291              : }
    3292              : 
    3293              : /* Find the access with corresponding OFFSET and SIZE among accesses in
    3294              :    PARAM_DESC and return it or NULL if such an access is not there.  */
    3295              : 
    3296              : static param_access *
    3297        17148 : find_param_access (isra_param_desc *param_desc, unsigned offset, unsigned size)
    3298              : {
    3299        17148 :   unsigned pclen = vec_safe_length (param_desc->accesses);
    3300              : 
    3301              :   /* The search is linear but the number of stored accesses is bound by
    3302              :      PARAM_IPA_SRA_MAX_REPLACEMENTS, so most probably 8.  */
    3303              : 
    3304        17225 :   for (unsigned i = 0; i < pclen; i++)
    3305        17180 :     if ((*param_desc->accesses)[i]->unit_offset == offset
    3306        17180 :         && (*param_desc->accesses)[i]->unit_size == size)
    3307              :       return (*param_desc->accesses)[i];
    3308              : 
    3309              :   return NULL;
    3310              : }
    3311              : 
    3312              : /* Return iff the total size of definite replacement SIZE would violate the
    3313              :    limit set for it in PARAM.  */
    3314              : 
    3315              : static bool
    3316       229576 : size_would_violate_limit_p (isra_param_desc *desc, unsigned size)
    3317              : {
    3318       229576 :   unsigned limit = desc->param_size_limit;
    3319            0 :   if (size > limit
    3320       226510 :       || (!desc->by_ref && size == limit))
    3321            0 :     return true;
    3322              :   return false;
    3323              : }
    3324              : 
    3325              : /* Increase reached size of DESC by SIZE or disqualify it if it would violate
    3326              :    the set limit.  IDX is the parameter number which is dumped when
    3327              :    disqualifying.  */
    3328              : 
    3329              : static void
    3330        13763 : bump_reached_size (isra_param_desc *desc, unsigned size, unsigned idx)
    3331              : {
    3332        13763 :   unsigned after = desc->size_reached + size;
    3333        13763 :   if (size_would_violate_limit_p (desc, after))
    3334              :     {
    3335        13731 :       if (dump_file && (dump_flags & TDF_DETAILS))
    3336            0 :         fprintf (dump_file, "    ...size limit reached, disqualifying "
    3337              :                  "candidate parameter %u\n", idx);
    3338        13731 :       desc->split_candidate = false;
    3339        13731 :       return;
    3340              :     }
    3341           32 :   desc->size_reached = after;
    3342              : }
    3343              : 
    3344              : /* Take all actions required to deal with an edge CS that represents a call to
    3345              :    an unknown or un-analyzed function, for both parameter removal and
    3346              :    splitting.  */
    3347              : 
    3348              : static void
    3349       546113 : process_edge_to_unknown_caller (cgraph_edge *cs)
    3350              : {
    3351       546113 :   isra_func_summary *from_ifs = func_sums->get (cs->caller);
    3352       546113 :   gcc_checking_assert (from_ifs);
    3353       546113 :   isra_call_summary *csum = call_sums->get (cs);
    3354              : 
    3355       546113 :   if (dump_file && (dump_flags & TDF_DETAILS))
    3356           23 :     fprintf (dump_file, "Processing an edge to an unknown caller from %s:\n",
    3357              :              cs->caller->dump_name ());
    3358              : 
    3359       546113 :   unsigned args_count = csum->m_arg_flow.length ();
    3360      1316901 :   for (unsigned i = 0; i < args_count; i++)
    3361              :     {
    3362       770788 :       isra_param_flow *ipf = &csum->m_arg_flow[i];
    3363              : 
    3364       770788 :      if (ipf->pointer_pass_through)
    3365              :        {
    3366       193948 :          isra_param_desc *param_desc
    3367       193948 :            = &(*from_ifs->m_parameters)[get_single_param_flow_source (ipf)];
    3368       193948 :          param_desc->locally_unused = false;
    3369       193948 :          param_desc->split_candidate = false;
    3370       193948 :         continue;
    3371       193948 :        }
    3372       576840 :       if (ipf->aggregate_pass_through)
    3373              :         {
    3374         6502 :           unsigned idx = get_single_param_flow_source (ipf);
    3375         6502 :           isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
    3376              : 
    3377         6502 :           param_desc->locally_unused = false;
    3378         6502 :           if (!param_desc->split_candidate)
    3379         1786 :             continue;
    3380         4716 :           gcc_assert (!param_desc->by_ref);
    3381         9432 :           param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
    3382         4716 :                                                   ipf->unit_size);
    3383         4716 :           gcc_checking_assert (pacc);
    3384         4716 :           pacc->certain = true;
    3385         4716 :           if (overlapping_certain_accesses_p (param_desc, NULL))
    3386              :             {
    3387          627 :               if (dump_file && (dump_flags & TDF_DETAILS))
    3388            0 :                 fprintf (dump_file, "    ...leading to overlap, "
    3389              :                          " disqualifying candidate parameter %u\n",
    3390              :                          idx);
    3391          627 :               param_desc->split_candidate = false;
    3392              :             }
    3393              :           else
    3394         4089 :             bump_reached_size (param_desc, pacc->unit_size, idx);
    3395         4716 :           ipf->aggregate_pass_through = false;
    3396         4716 :           continue;
    3397         4716 :         }
    3398              : 
    3399       665838 :       for (int j = 0; j < ipf->length; j++)
    3400              :         {
    3401        95500 :           int input_idx = ipf->inputs[j];
    3402        95500 :           (*from_ifs->m_parameters)[input_idx].locally_unused = false;
    3403              :         }
    3404              :     }
    3405       546113 : }
    3406              : 
    3407              : /* Propagate parameter removal information through cross-SCC edge CS,
    3408              :    i.e. decrease the use count in the caller parameter descriptor for each use
    3409              :    in this call.  */
    3410              : 
    3411              : static void
    3412       907024 : param_removal_cross_scc_edge (cgraph_edge *cs)
    3413              : {
    3414       907024 :   enum availability availability;
    3415       907024 :   cgraph_node *callee = cs->callee->function_symbol (&availability);
    3416       907024 :   isra_func_summary *to_ifs = func_sums->get (callee);
    3417       438752 :   if (!to_ifs || !to_ifs->m_candidate
    3418       385656 :       || (availability < AVAIL_AVAILABLE)
    3419      1292680 :       || vec_safe_is_empty (to_ifs->m_parameters))
    3420              :     {
    3421       531956 :       process_edge_to_unknown_caller (cs);
    3422       531956 :       return;
    3423              :     }
    3424       375068 :   isra_func_summary *from_ifs = func_sums->get (cs->caller);
    3425       375068 :   gcc_checking_assert (from_ifs);
    3426              : 
    3427       375068 :   isra_call_summary *csum = call_sums->get (cs);
    3428       375068 :   unsigned args_count = csum->m_arg_flow.length ();
    3429       375068 :   unsigned param_count = vec_safe_length (to_ifs->m_parameters);
    3430              : 
    3431      1295097 :   for (unsigned i = 0; i < args_count; i++)
    3432              :     {
    3433       920029 :       bool unused_in_callee = false;
    3434              : 
    3435       920029 :       isra_param_desc *descriptor = NULL;
    3436       920029 :       if (cs->callback)
    3437              :         {
    3438           14 :           callback_info *ci = callback_info_sum->get (cs);
    3439           14 :           int remapped_idx = ci->inverted_mapping[i];
    3440           14 :           if (remapped_idx != ARG_MAPPING_UNKNOWN_IDX)
    3441            7 :             descriptor = &(*to_ifs->m_parameters)[remapped_idx];
    3442              :         }
    3443       920015 :       else if (i < param_count)
    3444       920011 :         descriptor = &(*to_ifs->m_parameters)[i];
    3445              : 
    3446       920018 :       if (descriptor)
    3447       920018 :         unused_in_callee = descriptor->locally_unused;
    3448              : 
    3449       920018 :       if (!unused_in_callee)
    3450              :         {
    3451       860605 :           isra_param_flow *ipf = &csum->m_arg_flow[i];
    3452      1979595 :           for (int j = 0; j < ipf->length; j++)
    3453              :             {
    3454       258385 :               int input_idx = ipf->inputs[j];
    3455       258385 :               (*from_ifs->m_parameters)[input_idx].locally_unused = false;
    3456              :             }
    3457              :         }
    3458              :     }
    3459              : }
    3460              : 
    3461              : /* Unless it is already there, push NODE which is also described by IFS to
    3462              :    STACK.  */
    3463              : 
    3464              : static void
    3465       161143 : isra_push_node_to_stack (cgraph_node *node, isra_func_summary *ifs,
    3466              :                     vec<cgraph_node *> *stack)
    3467              : {
    3468            0 :   if (!ifs->m_queued)
    3469              :     {
    3470       160721 :       ifs->m_queued = true;
    3471       160721 :       stack->safe_push (node);
    3472              :     }
    3473            0 : }
    3474              : 
    3475              : /* If parameter with index INPUT_IDX is marked as locally unused, mark it as
    3476              :    used and push CALLER on STACK.  */
    3477              : 
    3478              : static void
    3479        13580 : isra_mark_caller_param_used (isra_func_summary *from_ifs, int input_idx,
    3480              :                              cgraph_node *caller, vec<cgraph_node *> *stack)
    3481              : {
    3482        13580 :   if ((*from_ifs->m_parameters)[input_idx].locally_unused)
    3483              :     {
    3484         1153 :       (*from_ifs->m_parameters)[input_idx].locally_unused = false;
    3485         1933 :       isra_push_node_to_stack (caller, from_ifs, stack);
    3486              :     }
    3487        13580 : }
    3488              : 
    3489              : /* Combine safe_size of DESC with SIZE and return true if it has changed.  */
    3490              : 
    3491              : static bool
    3492      2526302 : update_safe_size (isra_param_desc *desc, unsigned size)
    3493              : {
    3494            0 :   if (!desc->safe_size_set)
    3495              :     {
    3496       796597 :       desc->safe_size_set = 1;
    3497       796597 :       desc->safe_size = size;
    3498       796597 :       return true;
    3499              :     }
    3500      1729705 :   if (desc->safe_size <= size)
    3501              :     return false;
    3502        20005 :   desc->safe_size = size;
    3503        20005 :   return true;
    3504              : }
    3505              : 
    3506              : /* Set all param hints in DESC to the pessimistic values.  Return true if any
    3507              :    hints that need to potentially trigger further propagation have changed.  */
    3508              : 
    3509              : static bool
    3510       744375 : flip_all_hints_pessimistic (isra_param_desc *desc)
    3511              : {
    3512       744375 :   desc->not_specially_constructed = true;
    3513            0 :   return update_safe_size (desc, 0);
    3514              : }
    3515              : 
    3516              : /* Because we have not analyzed or otherwise problematic caller, go over all
    3517              :    parameter int flags of IFS describing a call graph node of a callee and
    3518              :    turn them pessimistic.  Return true if any hints that need to potentially
    3519              :    trigger further propagation have changed.  */
    3520              : 
    3521              : static bool
    3522       420567 : flip_all_param_hints_pessimistic (isra_func_summary *ifs)
    3523              : {
    3524       420567 :   if (!ifs || !ifs->m_candidate)
    3525              :     return false;
    3526              : 
    3527        26199 :   bool ret = false;
    3528        26199 :   unsigned param_count = vec_safe_length (ifs->m_parameters);
    3529              : 
    3530        74637 :   for (unsigned i = 0; i < param_count; i++)
    3531        96876 :     ret |= flip_all_hints_pessimistic (&(*ifs->m_parameters)[i]);
    3532              : 
    3533              :   return ret;
    3534              : }
    3535              : 
    3536              : /* Propagate hints across edge CS which ultimately leads to a node described
    3537              :    by TO_IFS.  Return true if any hints of the callee which should potentially
    3538              :    trigger further propagation have changed.  */
    3539              : 
    3540              : static bool
    3541      5021899 : propagate_param_hints_accross_call (cgraph_edge *cs, isra_func_summary *to_ifs)
    3542              : {
    3543      5021899 :   if (!to_ifs || !to_ifs->m_candidate)
    3544              :     return false;
    3545              : 
    3546      1696212 :   isra_call_summary *csum = call_sums->get (cs);
    3547      1696212 :   bool ret = false;
    3548      1696212 :   unsigned args_count = csum->m_arg_flow.length ();
    3549      1696212 :   unsigned param_count = vec_safe_length (to_ifs->m_parameters);
    3550              : 
    3551      5415410 :   for (unsigned i = 0; i < param_count; i++)
    3552              :     {
    3553      3719198 :       isra_param_desc *desc = &(*to_ifs->m_parameters)[i];
    3554      3719198 :       if (i >= args_count)
    3555              :         {
    3556       695937 :           ret |= flip_all_hints_pessimistic (desc);
    3557       695937 :           continue;
    3558              :         }
    3559              : 
    3560      3023261 :       if (desc->by_ref)
    3561              :         {
    3562      1781945 :           unsigned idx = i;
    3563      1781945 :           if (cs->callback)
    3564              :             {
    3565           15 :               gcc_assert (cs->callee->assume_function);
    3566           15 :               callback_info *ci = callback_info_sum->get (cs);
    3567           15 :               int remapped = ci->arg_mapping[i];
    3568           15 :               if (remapped == ARG_MAPPING_UNKNOWN_IDX)
    3569              :                 /* In the case of assume outlined functions, the only skipped
    3570              :                    argument is the address to be called, which is fine.  In
    3571              :                    other cases, if they ever arise, processing of the call to
    3572              :                    callback dispatching functions will need to handle unmapped
    3573              :                    arguments as necessary.  */
    3574            0 :                 continue;
    3575           15 :               idx = (unsigned) remapped;
    3576              :             }
    3577      1781945 :           isra_param_flow *ipf = &csum->m_arg_flow[idx];
    3578              : 
    3579      1781945 :           if (!ipf->constructed_for_calls)
    3580      1310548 :             desc->not_specially_constructed = true;
    3581              : 
    3582      1781945 :           if (ipf->pointer_pass_through)
    3583              :             {
    3584       286928 :               isra_func_summary *from_ifs = func_sums->get (cs->caller);
    3585       286928 :               int srcidx = get_single_param_flow_source (ipf);
    3586       286928 :               if (vec_safe_length (from_ifs->m_parameters) > (unsigned) srcidx)
    3587              :                 {
    3588       178821 :                   isra_param_desc *src_d = &(*from_ifs->m_parameters)[srcidx];
    3589       178821 :                   if (src_d->safe_size_set)
    3590       357606 :                     ret |= update_safe_size (desc, src_d->safe_size);
    3591              :                 }
    3592              :               else
    3593       216214 :                 ret |= update_safe_size (desc, 0);
    3594              :             }
    3595      1495017 :           else if (!ipf->aggregate_pass_through)
    3596      2990032 :             ret |= update_safe_size (desc, ipf->unit_size);
    3597              :           else
    3598              :             /* LTOing type-mismatched programs can end up here.  */
    3599            2 :             ret |= update_safe_size (desc, 0);
    3600              :         }
    3601              :     }
    3602              :   return ret;
    3603              : }
    3604              : 
    3605              : /* Propagate hints from NODE described by FROM_IFS to all its (dorect) callees,
    3606              :    push those that may need re-visiting onto STACK.  */
    3607              : 
    3608              : static void
    3609      1384629 : propagate_hints_to_all_callees (cgraph_node *node, isra_func_summary *from_ifs,
    3610              :                                 vec<cgraph_node *> *stack)
    3611              : {
    3612      6827095 :   for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
    3613              :     {
    3614      5442466 :       enum availability availability;
    3615      5442466 :       cgraph_node *callee = cs->callee->function_symbol (&availability);
    3616      5442466 :       isra_func_summary *to_ifs = func_sums->get (callee);
    3617              :       /* Skip over regular callbacks, as we cannot perform SRA on them.  We
    3618              :          don't zap them early on, because the callback call might get optimized
    3619              :          out, so don't want to miss that opportunity.  Skipping them here is OK,
    3620              :          as they will get disqualified later on in the IPA stage.  */
    3621      5442466 :       if (!from_ifs || (cs->callback && !cs->callee->assume_function))
    3622              :         {
    3623       420567 :           if (flip_all_param_hints_pessimistic (to_ifs)
    3624       420567 :               && ipa_edge_within_scc (cs))
    3625          228 :             isra_push_node_to_stack (callee, to_ifs, stack);
    3626              :         }
    3627      5021899 :       else if (propagate_param_hints_accross_call (cs, to_ifs)
    3628      5021899 :                && ipa_edge_within_scc (cs))
    3629         5333 :         isra_push_node_to_stack (callee, to_ifs, stack);
    3630              :     }
    3631      1384629 : }
    3632              : 
    3633              : /* Propagate information that any parameter is not used only locally within a
    3634              :    SCC across CS to the caller, which must be in the same SCC as the
    3635              :    callee.  Push any callers that need to be re-processed to STACK.  */
    3636              : 
    3637              : static void
    3638        29644 : propagate_used_across_scc_edge (cgraph_edge *cs, vec<cgraph_node *> *stack)
    3639              : {
    3640        29644 :   isra_func_summary *from_ifs = func_sums->get (cs->caller);
    3641        29644 :   if (!from_ifs || vec_safe_is_empty (from_ifs->m_parameters))
    3642        15488 :     return;
    3643              : 
    3644        14156 :   isra_call_summary *csum = call_sums->get (cs);
    3645        14156 :   gcc_checking_assert (csum);
    3646        14156 :   unsigned args_count = csum->m_arg_flow.length ();
    3647        14156 :   enum availability availability;
    3648        14156 :   cgraph_node *callee = cs->callee->function_symbol (&availability);
    3649        14156 :   isra_func_summary *to_ifs = func_sums->get (callee);
    3650              : 
    3651        14156 :   unsigned param_count
    3652        13953 :     = (to_ifs && (availability >= AVAIL_AVAILABLE))
    3653        27843 :     ? vec_safe_length (to_ifs->m_parameters) : 0;
    3654        42084 :   for (unsigned i = 0; i < args_count; i++)
    3655              :     {
    3656        27928 :       unsigned idx = i;
    3657              : 
    3658        27928 :       if (cs->callback)
    3659              :         {
    3660            0 :           gcc_assert (cs->callee->assume_function);
    3661            0 :           callback_info *ci = callback_info_sum->get (cs);
    3662            0 :           int remapped = ci->inverted_mapping[i];
    3663            0 :           if (remapped == ARG_MAPPING_UNKNOWN_IDX)
    3664              :             /* In the case of assume outlined functions, the only skipped
    3665              :                argument is the address to be called, which is fine.  In other
    3666              :                cases, if they ever arise, processing of the call to callback
    3667              :                dispatching functions will need to handle unmapped arguments as
    3668              :                necessary.  */
    3669            0 :             continue;
    3670            0 :           idx = (unsigned) remapped;
    3671              :         }
    3672              : 
    3673        31813 :       if (idx < param_count
    3674        27928 :           && (*to_ifs->m_parameters)[idx].locally_unused)
    3675         3885 :             continue;
    3676              : 
    3677              :       /* The argument is needed in the callee it, we must mark the parameter as
    3678              :          used also in the caller and its callers within this SCC.  */
    3679        24043 :       isra_param_flow *ipf = &csum->m_arg_flow[i];
    3680        61666 :       for (int j = 0; j < ipf->length; j++)
    3681              :         {
    3682        13580 :           int input_idx = ipf->inputs[j];
    3683        13580 :           isra_mark_caller_param_used (from_ifs, input_idx, cs->caller, stack);
    3684              :         }
    3685              :     }
    3686              : }
    3687              : 
    3688              : /* Propagate information that any parameter is not used only locally within a
    3689              :    SCC (i.e. is used also elsewhere) to all callers of NODE that are in the
    3690              :    same SCC.  Push any callers that need to be re-processed to STACK.  */
    3691              : 
    3692              : static bool
    3693      1461308 : propagate_used_to_scc_callers (cgraph_node *node, void *data)
    3694              : {
    3695      1461308 :   vec<cgraph_node *> *stack = (vec<cgraph_node *> *) data;
    3696      1461308 :   cgraph_edge *cs;
    3697      3605638 :   for (cs = node->callers; cs; cs = cs->next_caller)
    3698      2144330 :     if (ipa_edge_within_scc (cs))
    3699        29644 :       propagate_used_across_scc_edge (cs, stack);
    3700      1461308 :   return false;
    3701              : }
    3702              : 
    3703              : /* Return true iff all certain accesses in ARG_DESC are also present as
    3704              :    certain accesses in PARAM_DESC.  */
    3705              : 
    3706              : static bool
    3707           82 : all_callee_accesses_present_p (isra_param_desc *param_desc,
    3708              :                                isra_param_desc *arg_desc)
    3709              : {
    3710           82 :   unsigned aclen = vec_safe_length (arg_desc->accesses);
    3711           86 :   for (unsigned j = 0; j < aclen; j++)
    3712              :     {
    3713           50 :       param_access *argacc = (*arg_desc->accesses)[j];
    3714           50 :       if (!argacc->certain)
    3715            0 :         continue;
    3716          100 :       param_access *pacc = find_param_access (param_desc, argacc->unit_offset,
    3717           50 :                                               argacc->unit_size);
    3718           50 :       if (!pacc
    3719            5 :           || !pacc->certain
    3720           55 :           || !types_compatible_p (argacc->type, pacc->type))
    3721              :         return false;
    3722              :     }
    3723              :   return true;
    3724              : }
    3725              : 
    3726              : /* Type internal to function pull_accesses_from_callee.  Unfortunately gcc 4.8
    3727              :    does not allow instantiating an auto_vec with a type defined within a
    3728              :    function so it is a global type.   */
    3729              : enum acc_prop_kind {ACC_PROP_DONT, ACC_PROP_COPY, ACC_PROP_CERTAIN};
    3730              : 
    3731              : 
    3732              : /* Attempt to propagate all definite accesses from ARG_DESC to PARAM_DESC,
    3733              :    (which belongs to CALLER) if they would not violate some constraint there.
    3734              :    CALLER_IPCP_TS describes the caller, PARAM_IDX is the index of the parameter
    3735              :    described by PARAM_DESC.  If successful, return NULL, otherwise return the
    3736              :    string reason for failure (which can be written to the dump file).
    3737              :    DELTA_OFFSET is the known offset of the actual argument withing the formal
    3738              :    parameter (so of ARG_DESCS within PARAM_DESCS), ARG_SIZE is the size of the
    3739              :    actual argument or zero, if not known. In case of success, set *CHANGE_P to
    3740              :    true if propagation actually changed anything.  */
    3741              : 
    3742              : static const char *
    3743         5525 : pull_accesses_from_callee (cgraph_node *caller,
    3744              :                            ipcp_transformation *caller_ipcp_ts,
    3745              :                            int param_idx,
    3746              :                            isra_param_desc *param_desc,
    3747              :                            isra_param_desc *arg_desc,
    3748              :                            unsigned delta_offset, unsigned arg_size,
    3749              :                            bool *change_p)
    3750              : {
    3751         5525 :   unsigned pclen = vec_safe_length (param_desc->accesses);
    3752         5525 :   unsigned aclen = vec_safe_length (arg_desc->accesses);
    3753         5525 :   unsigned prop_count = 0;
    3754         5525 :   unsigned prop_size = 0;
    3755         5525 :   bool change = false;
    3756              : 
    3757         5525 :   auto_vec <enum acc_prop_kind, 8> prop_kinds (aclen);
    3758        19217 :   for (unsigned j = 0; j < aclen; j++)
    3759              :     {
    3760         8372 :       param_access *argacc = (*arg_desc->accesses)[j];
    3761         8372 :       prop_kinds.safe_push (ACC_PROP_DONT);
    3762              : 
    3763         8372 :       if (arg_size > 0
    3764         2747 :           && argacc->unit_offset + argacc->unit_size > arg_size)
    3765              :         return "callee access outsize size boundary";
    3766              : 
    3767         8372 :       if (!argacc->certain)
    3768          379 :         continue;
    3769              : 
    3770         7993 :       unsigned offset = argacc->unit_offset + delta_offset;
    3771              : 
    3772         7993 :       if (caller_ipcp_ts && !AGGREGATE_TYPE_P (argacc->type))
    3773              :         {
    3774          407 :           ipa_argagg_value_list avl (caller_ipcp_ts);
    3775          407 :           tree value = avl.get_value (param_idx, offset);
    3776          407 :           if (value && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
    3777           51 :                          / BITS_PER_UNIT)
    3778           51 :                         != argacc->unit_size))
    3779            0 :             return " propagated access would conflict with an IPA-CP constant";
    3780              :         }
    3781              : 
    3782              :       /* Given that accesses are initially stored according to increasing
    3783              :          offset and decreasing size in case of equal offsets, the following
    3784              :          searches could be written more efficiently if we kept the ordering
    3785              :          when copying. But the number of accesses is capped at
    3786              :          PARAM_IPA_SRA_MAX_REPLACEMENTS (so most likely 8) and the code gets
    3787              :          messy quickly, so let's improve on that only if necessary.  */
    3788              : 
    3789         7993 :       bool exact_match = false;
    3790        16819 :       for (unsigned i = 0; i < pclen; i++)
    3791              :         {
    3792              :           /* Check for overlaps.  */
    3793         9031 :           param_access *pacc = (*param_desc->accesses)[i];
    3794         9031 :           if (pacc->unit_offset == offset
    3795         4472 :               && pacc->unit_size == argacc->unit_size)
    3796              :             {
    3797         3288 :               if (argacc->alias_ptr_type != pacc->alias_ptr_type
    3798         3084 :                   || !types_compatible_p (argacc->type, pacc->type)
    3799         6372 :                   || argacc->reverse != pacc->reverse)
    3800              :                 return "propagated access types would not match existing ones";
    3801              : 
    3802         3084 :               exact_match = true;
    3803         3084 :               if (!pacc->certain)
    3804              :                 {
    3805            0 :                   prop_kinds[j] = ACC_PROP_CERTAIN;
    3806            0 :                   prop_size += argacc->unit_size;
    3807            0 :                   change = true;
    3808              :                 }
    3809         3084 :               continue;
    3810              :             }
    3811              : 
    3812         5743 :           if (offset < pacc->unit_offset + pacc->unit_size
    3813         3823 :               && offset + argacc->unit_size > pacc->unit_offset)
    3814              :             {
    3815              :               /* None permissible with load accesses, possible to fit into
    3816              :                  argument ones.  */
    3817         2369 :               if (pacc->certain
    3818         2368 :                   || offset < pacc->unit_offset
    3819         2368 :                   || (offset + argacc->unit_size
    3820              :                       > pacc->unit_offset + pacc->unit_size))
    3821              :                 return "a propagated access would conflict in caller";
    3822              :             }
    3823              :         }
    3824              : 
    3825         7788 :       if (!exact_match)
    3826              :         {
    3827         4704 :           prop_kinds[j] = ACC_PROP_COPY;
    3828         4704 :           prop_count++;
    3829         4704 :           prop_size += argacc->unit_size;
    3830         4704 :           change = true;
    3831              :         }
    3832              :     }
    3833              : 
    3834         5320 :     if (!change)
    3835              :       return NULL;
    3836              : 
    3837         3047 :     if ((prop_count + pclen
    3838         3047 :          > (unsigned) opt_for_fn (caller->decl, param_ipa_sra_max_replacements))
    3839         8572 :         || size_would_violate_limit_p (param_desc,
    3840         3047 :                                        param_desc->size_reached + prop_size))
    3841              :       return "propagating accesses would violate the count or size limit";
    3842              : 
    3843         3022 :   *change_p = true;
    3844         8095 :   for (unsigned j = 0; j < aclen; j++)
    3845              :     {
    3846         5073 :       if (prop_kinds[j] == ACC_PROP_COPY)
    3847              :         {
    3848         4659 :           param_access *argacc = (*arg_desc->accesses)[j];
    3849              : 
    3850         4659 :           param_access *copy = ggc_cleared_alloc<param_access> ();
    3851         4659 :           copy->unit_offset = argacc->unit_offset + delta_offset;
    3852         4659 :           copy->unit_size = argacc->unit_size;
    3853         4659 :           copy->type = argacc->type;
    3854         4659 :           copy->alias_ptr_type = argacc->alias_ptr_type;
    3855         4659 :           copy->certain = true;
    3856         4659 :           copy->reverse = argacc->reverse;
    3857         4659 :           vec_safe_push (param_desc->accesses, copy);
    3858              :         }
    3859          414 :       else if (prop_kinds[j] == ACC_PROP_CERTAIN)
    3860              :         {
    3861            0 :           param_access *argacc = (*arg_desc->accesses)[j];
    3862            0 :           param_access *csp
    3863            0 :             = find_param_access (param_desc, argacc->unit_offset + delta_offset,
    3864            0 :                                  argacc->unit_size);
    3865            0 :           csp->certain = true;
    3866              :         }
    3867              :     }
    3868              : 
    3869         3022 :   param_desc->size_reached += prop_size;
    3870              : 
    3871         3022 :   return NULL;
    3872         5525 : }
    3873              : 
    3874              : /* Propagate parameter splitting information through call graph edge CS.
    3875              :    Return true if any changes that might need to be propagated within SCCs have
    3876              :    been made.  The function also clears the aggregate_pass_through and
    3877              :    pointer_pass_through in call summaries which do not need to be processed
    3878              :    again if this CS is revisited when iterating while changes are propagated
    3879              :    within an SCC.  */
    3880              : 
    3881              : static bool
    3882      1704255 : param_splitting_across_edge (cgraph_edge *cs)
    3883              : {
    3884      1704255 :   bool res = false;
    3885      1704255 :   bool cross_scc = !ipa_edge_within_scc (cs);
    3886      1704255 :   enum availability availability;
    3887      1704255 :   cgraph_node *callee = cs->callee->function_symbol (&availability);
    3888      1704255 :   isra_func_summary *from_ifs = func_sums->get (cs->caller);
    3889      1704255 :   gcc_checking_assert (from_ifs && from_ifs->m_parameters);
    3890      1704255 :   ipcp_transformation *caller_ipcp_ts
    3891      1704255 :     = ipcp_get_transformation_summary (cs->caller);
    3892              : 
    3893      1704255 :   isra_call_summary *csum = call_sums->get (cs);
    3894              :   /* The summary may not exist if the edge is a callback edge created after SRA
    3895              :      summaries are generated (when ipa-cp is disabled for example).  Bail if
    3896              :      that happens.  */
    3897      1704255 :   if (!csum)
    3898              :     {
    3899            0 :       gcc_checking_assert (callback_ifn_assume_call_p (cs->call_stmt));
    3900              :       return false;
    3901              :     }
    3902      1704255 :   unsigned args_count = csum->m_arg_flow.length ();
    3903      1704255 :   isra_func_summary *to_ifs = func_sums->get (callee);
    3904      1704255 :   unsigned param_count
    3905       899225 :     = ((to_ifs && to_ifs->m_candidate && (availability >= AVAIL_AVAILABLE))
    3906      2500705 :        ? vec_safe_length (to_ifs->m_parameters)
    3907              :        : 0);
    3908              : 
    3909      1704255 :   if (dump_file && (dump_flags & TDF_DETAILS))
    3910           74 :     fprintf (dump_file, "Splitting across %s->%s:\n",
    3911           37 :              cs->caller->dump_name (), callee->dump_name ());
    3912              : 
    3913      1704255 :   unsigned i;
    3914      3340500 :   for (i = 0; (i < args_count) && (i < param_count); i++)
    3915              :     {
    3916      1636245 :       isra_param_desc *arg_desc = &(*to_ifs->m_parameters)[i];
    3917      1636245 :       unsigned ipf_idx = i;
    3918      1636245 :       if (cs->callback)
    3919              :         {
    3920            8 :           gcc_assert (cs->callee->assume_function);
    3921            8 :           callback_info *ci = callback_info_sum->get (cs);
    3922            8 :           int remapped = ci->arg_mapping[i];
    3923            8 :           if (remapped == ARG_MAPPING_UNKNOWN_IDX)
    3924              :             /* In the case of assume outlined functions, the only skipped
    3925              :                argument is the address to be called, which is fine.  In other
    3926              :                cases, if they ever arise, processing of the call to callback
    3927              :                dispatching functions will need to handle unmapped arguments as
    3928              :                necessary.  */
    3929            0 :             continue;
    3930            8 :           ipf_idx = (unsigned) remapped;
    3931              :         }
    3932      1636245 :       isra_param_flow *ipf = &csum->m_arg_flow[ipf_idx];
    3933              : 
    3934      1636245 :       if (arg_desc->locally_unused)
    3935              :         {
    3936        98295 :           if (dump_file && (dump_flags & TDF_DETAILS))
    3937            0 :             fprintf (dump_file, "    ->%u: unused in callee\n", i);
    3938        98295 :           ipf->pointer_pass_through = false;
    3939        98295 :           continue;
    3940              :         }
    3941              : 
    3942      1537950 :       if (ipf->pointer_pass_through)
    3943              :         {
    3944       185339 :           int idx = get_single_param_flow_source (ipf);
    3945       185339 :           isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
    3946       185339 :           if (!param_desc->split_candidate)
    3947       132684 :             continue;
    3948        52655 :           gcc_assert (param_desc->by_ref);
    3949              : 
    3950        52655 :           if (!arg_desc->split_candidate || !arg_desc->by_ref)
    3951              :             {
    3952        46280 :               if (dump_file && (dump_flags & TDF_DETAILS))
    3953            2 :                 fprintf (dump_file, "  %u->%u: not candidate or not by "
    3954              :                          "reference in callee\n", idx, i);
    3955        46280 :               param_desc->split_candidate = false;
    3956        46280 :               ipf->pointer_pass_through = false;
    3957        46280 :               res = true;
    3958              :             }
    3959         6375 :           else if (!ipf->safe_to_import_accesses)
    3960              :             {
    3961         2177 :               if (!csum->m_before_any_store
    3962         2177 :                   || !all_callee_accesses_present_p (param_desc, arg_desc))
    3963              :                 {
    3964         2141 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    3965            0 :                     fprintf (dump_file, "  %u->%u: cannot import accesses.\n",
    3966              :                              idx, i);
    3967         2141 :                   param_desc->split_candidate = false;
    3968         2141 :                   ipf->pointer_pass_through = false;
    3969         2141 :                   res = true;
    3970              : 
    3971              :                 }
    3972              :               else
    3973              :                 {
    3974           36 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    3975            0 :                     fprintf (dump_file, "  %u->%u: verified callee accesses "
    3976              :                              "present.\n", idx, i);
    3977           36 :                   if (cross_scc)
    3978            0 :                     ipf->pointer_pass_through = false;
    3979              :                 }
    3980              :             }
    3981              :           else
    3982              :             {
    3983         4198 :               const char *pull_failure
    3984         4198 :                 = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
    3985              :                                              param_desc, arg_desc, 0, 0, &res);
    3986         4198 :               if (pull_failure)
    3987              :                 {
    3988           18 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    3989            0 :                     fprintf (dump_file, "  %u->%u: by_ref access pull "
    3990              :                              "failed: %s.\n", idx, i, pull_failure);
    3991           18 :                   param_desc->split_candidate = false;
    3992           18 :                   ipf->pointer_pass_through = false;
    3993           18 :                   res = true;
    3994              :                 }
    3995              :               else
    3996              :                 {
    3997         4180 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    3998            6 :                     fprintf (dump_file, "  %u->%u: by_ref access pull "
    3999              :                              "succeeded.\n", idx, i);
    4000         4180 :                   if (cross_scc)
    4001         4023 :                     ipf->pointer_pass_through = false;
    4002              :                 }
    4003              :             }
    4004              :         }
    4005      1352611 :       else if (ipf->aggregate_pass_through)
    4006              :         {
    4007        34382 :           int idx = get_single_param_flow_source (ipf);
    4008        34382 :           isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
    4009        34382 :           if (!param_desc->split_candidate)
    4010        22000 :             continue;
    4011        12382 :           gcc_assert (!param_desc->by_ref);
    4012        24764 :           param_access *pacc = find_param_access (param_desc, ipf->unit_offset,
    4013        12382 :                                                   ipf->unit_size);
    4014        12382 :           gcc_checking_assert (pacc);
    4015              : 
    4016        12382 :           if (pacc->certain)
    4017              :             {
    4018            1 :               if (dump_file && (dump_flags & TDF_DETAILS))
    4019            0 :                 fprintf (dump_file, "  %u->%u: already certain\n", idx, i);
    4020            1 :               ipf->aggregate_pass_through = false;
    4021              :             }
    4022        12381 :           else if (!arg_desc->split_candidate || arg_desc->by_ref)
    4023              :             {
    4024        11054 :               if (dump_file && (dump_flags & TDF_DETAILS))
    4025            0 :                 fprintf (dump_file, "  %u->%u: not candidate or by "
    4026              :                          "reference in callee\n", idx, i);
    4027              : 
    4028        11054 :               pacc->certain = true;
    4029        11054 :               if (overlapping_certain_accesses_p (param_desc, NULL))
    4030              :                 {
    4031         1380 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    4032            0 :                     fprintf (dump_file, "    ...leading to overlap, "
    4033              :                              " disqualifying candidate parameter %u\n",
    4034              :                              idx);
    4035         1380 :                   param_desc->split_candidate = false;
    4036              :                 }
    4037              :               else
    4038         9674 :                 bump_reached_size (param_desc, pacc->unit_size, idx);
    4039              : 
    4040        11054 :               ipf->aggregate_pass_through = false;
    4041        11054 :               res = true;
    4042              :             }
    4043              :           else
    4044              :             {
    4045         1327 :               const char *pull_failure
    4046         1327 :                 = pull_accesses_from_callee (cs->caller, caller_ipcp_ts, idx,
    4047              :                                              param_desc, arg_desc,
    4048              :                                              ipf->unit_offset,
    4049              :                                              ipf->unit_size, &res);
    4050         1327 :               if (pull_failure)
    4051              :                 {
    4052          212 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    4053            0 :                     fprintf (dump_file, "  %u->%u: arg access pull "
    4054              :                              "failed: %s.\n", idx, i, pull_failure);
    4055              : 
    4056          212 :                   ipf->aggregate_pass_through = false;
    4057          212 :                   pacc->certain = true;
    4058              : 
    4059          212 :                   if (overlapping_certain_accesses_p (param_desc, NULL))
    4060              :                     {
    4061          212 :                       if (dump_file && (dump_flags & TDF_DETAILS))
    4062            0 :                         fprintf (dump_file, "    ...leading to overlap, "
    4063              :                                  " disqualifying candidate parameter %u\n",
    4064              :                                  idx);
    4065          212 :                       param_desc->split_candidate = false;
    4066              :                     }
    4067              :                   else
    4068            0 :                     bump_reached_size (param_desc, pacc->unit_size, idx);
    4069              : 
    4070          212 :                   res = true;
    4071              :                 }
    4072              :               else
    4073              :                 {
    4074         1115 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    4075            2 :                     fprintf (dump_file, "  %u->%u: arg access pull "
    4076              :                              "succeeded.\n", idx, i);
    4077         1115 :                   if (cross_scc)
    4078          980 :                     ipf->aggregate_pass_through = false;
    4079              :                 }
    4080              :             }
    4081              :         }
    4082              :     }
    4083              : 
    4084              :   /* Handle argument-parameter count mismatches. */
    4085      2758658 :   for (; (i < args_count); i++)
    4086              :     {
    4087      1054403 :       isra_param_flow *ipf = &csum->m_arg_flow[i];
    4088              : 
    4089      1054403 :       if (ipf->pointer_pass_through || ipf->aggregate_pass_through)
    4090              :         {
    4091       203164 :           int idx = get_single_param_flow_source (ipf);
    4092       203164 :           ipf->pointer_pass_through = false;
    4093       203164 :           ipf->aggregate_pass_through = false;
    4094       203164 :           isra_param_desc *param_desc = &(*from_ifs->m_parameters)[idx];
    4095       203164 :           if (!param_desc->split_candidate)
    4096       203081 :             continue;
    4097              : 
    4098           83 :           if (dump_file && (dump_flags & TDF_DETAILS))
    4099            0 :             fprintf (dump_file, "  %u->%u: no corresponding formal parameter\n",
    4100              :                      idx, i);
    4101           83 :           param_desc->split_candidate = false;
    4102           83 :           res = true;
    4103              :         }
    4104              :     }
    4105      1704255 :   return res;
    4106              : }
    4107              : 
    4108              : /* Worker for call_for_symbol_and_aliases, look at all callers and if all their
    4109              :    callers ignore the return value, or come from the same SCC and use the
    4110              :    return value only to compute their return value, return false, otherwise
    4111              :    return true.  */
    4112              : 
    4113              : static bool
    4114       191900 : retval_used_p (cgraph_node *node, void *)
    4115              : {
    4116       299921 :   for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
    4117              :     {
    4118       264662 :       isra_call_summary *csum = call_sums->get (cs);
    4119       264662 :       gcc_checking_assert (csum);
    4120       264662 :       if (csum->m_return_ignored)
    4121       102172 :         continue;
    4122       162490 :       if (!csum->m_return_returned)
    4123              :         return true;
    4124              : 
    4125        27628 :       isra_func_summary *from_ifs = func_sums->get (cs->caller);
    4126        27628 :       if (!from_ifs || !from_ifs->m_candidate)
    4127              :         return true;
    4128              : 
    4129        20970 :       if (!ipa_edge_within_scc (cs)
    4130        20970 :           && !from_ifs->m_return_ignored)
    4131              :             return true;
    4132              :     }
    4133              : 
    4134              :   /* Keep the return value, even though it's not used in the code directly,
    4135              :      because it's used by vrp later.  */
    4136        35259 :   if (node->assume_function)
    4137            0 :     return true;
    4138              : 
    4139              :   return false;
    4140              : }
    4141              : 
    4142              : /* Push into NEW_PARAMS all required parameter adjustment entries to copy or
    4143              :    modify parameter which originally had index BASE_INDEX, in the adjustment
    4144              :    vector of parent clone (if any) had PREV_CLONE_INDEX and was described by
    4145              :    PREV_ADJUSTMENT.  If IPA-CP has created a transformation summary for the
    4146              :    original node, it needs to be passed in IPCP_TS, otherwise it should be
    4147              :    NULL.  If the parent clone is the original function, PREV_ADJUSTMENT is NULL
    4148              :    and PREV_CLONE_INDEX is equal to BASE_INDEX.  */
    4149              : 
    4150              : static void
    4151       344372 : push_param_adjustments_for_index (isra_func_summary *ifs, unsigned base_index,
    4152              :                                   unsigned prev_clone_index,
    4153              :                                   ipa_adjusted_param *prev_adjustment,
    4154              :                                   ipcp_transformation *ipcp_ts,
    4155              :                                   vec<ipa_adjusted_param, va_gc> **new_params)
    4156              : {
    4157       344372 :   isra_param_desc *desc = &(*ifs->m_parameters)[base_index];
    4158       344372 :   if (desc->locally_unused)
    4159              :     {
    4160        73621 :       if (dump_file)
    4161           27 :         fprintf (dump_file, "  Will remove parameter %u\n", base_index);
    4162              :       return;
    4163              :     }
    4164              : 
    4165       270751 :   if (!desc->split_candidate)
    4166              :     {
    4167       221061 :       ipa_adjusted_param adj;
    4168       221061 :       if (prev_adjustment)
    4169              :         {
    4170         1678 :           adj = *prev_adjustment;
    4171         1678 :           adj.prev_clone_adjustment = true;
    4172         1678 :           adj.prev_clone_index = prev_clone_index;
    4173              :         }
    4174              :       else
    4175              :         {
    4176       219383 :           memset (&adj, 0, sizeof (adj));
    4177       219383 :           adj.op = IPA_PARAM_OP_COPY;
    4178       219383 :           adj.base_index = base_index;
    4179       219383 :           adj.prev_clone_index = prev_clone_index;
    4180              :         }
    4181       221061 :       vec_safe_push ((*new_params), adj);
    4182       221061 :       return;
    4183              :     }
    4184              : 
    4185        49690 :   if (dump_file)
    4186           62 :     fprintf (dump_file, "  Will split parameter %u\n", base_index);
    4187              : 
    4188        49690 :   gcc_assert (!prev_adjustment || prev_adjustment->op == IPA_PARAM_OP_COPY);
    4189        49690 :   unsigned aclen = vec_safe_length (desc->accesses);
    4190       138587 :   for (unsigned j = 0; j < aclen; j++)
    4191              :     {
    4192        88897 :       param_access *pa = (*desc->accesses)[j];
    4193        88897 :       if (!pa->certain)
    4194         7996 :         continue;
    4195              : 
    4196        88015 :       if (ipcp_ts)
    4197              :         {
    4198        15432 :           ipa_argagg_value_list avl (ipcp_ts);
    4199        15432 :           tree value = avl.get_value (base_index, pa->unit_offset);
    4200        15432 :           if (value && !AGGREGATE_TYPE_P (pa->type))
    4201              :             {
    4202         7114 :               if (dump_file)
    4203           28 :                 fprintf (dump_file, "    - omitting component at byte "
    4204              :                          "offset %u which is known to have a constant value\n ",
    4205              :                          pa->unit_offset);
    4206         7114 :               continue;
    4207              :             }
    4208              :         }
    4209              : 
    4210        80901 :       if (dump_file)
    4211           61 :         fprintf (dump_file, "    - component at byte offset %u, "
    4212           61 :                  "size %u\n", pa->unit_offset, pa->unit_size);
    4213              : 
    4214        80901 :       ipa_adjusted_param adj;
    4215        80901 :       memset (&adj, 0, sizeof (adj));
    4216        80901 :       adj.op = IPA_PARAM_OP_SPLIT;
    4217        80901 :       adj.base_index = base_index;
    4218        80901 :       adj.prev_clone_index = prev_clone_index;
    4219        80901 :       adj.param_prefix_index = IPA_PARAM_PREFIX_ISRA;
    4220        80901 :       adj.reverse = pa->reverse;
    4221        80901 :       adj.type = pa->type;
    4222        80901 :       adj.alias_ptr_type = pa->alias_ptr_type;
    4223        80901 :       adj.unit_offset = pa->unit_offset;
    4224        80901 :       vec_safe_push ((*new_params), adj);
    4225              :     }
    4226              : }
    4227              : 
    4228              : /* Worker for all call_for_symbol_thunks_and_aliases.  Set calls_comdat_local
    4229              :    flag of all callers of NODE.  */
    4230              : 
    4231              : static bool
    4232            0 : mark_callers_calls_comdat_local (struct cgraph_node *node, void *)
    4233              : {
    4234            0 :   for (cgraph_edge *cs = node->callers; cs; cs = cs->next_caller)
    4235            0 :     cs->caller->calls_comdat_local = true;
    4236            0 :   return false;
    4237              : }
    4238              : 
    4239              : /* Remove any IPA-CP results stored in TS that are associated with removed
    4240              :    parameters as marked in IFS. */
    4241              : 
    4242              : static void
    4243         8583 : zap_useless_ipcp_results (const isra_func_summary *ifs, ipcp_transformation *ts)
    4244              : {
    4245         8583 :   ts->remove_argaggs_if ([ifs](const ipa_argagg_value &v)
    4246              :   {
    4247        15944 :     return (*ifs->m_parameters)[v.index].locally_unused;
    4248              :   });
    4249              : 
    4250         8583 :   bool useful_vr = false;
    4251         8583 :   unsigned count = vec_safe_length (ts->m_vr);
    4252        30309 :   for (unsigned i = 0; i < count; i++)
    4253        21726 :     if ((*ts->m_vr)[i].known_p ())
    4254              :       {
    4255        15139 :         const isra_param_desc *desc = &(*ifs->m_parameters)[i];
    4256        15139 :         if (desc->locally_unused)
    4257         1942 :           (*ts->m_vr)[i].set_unknown ();
    4258              :         else
    4259              :           useful_vr = true;
    4260              :       }
    4261         8583 :   if (!useful_vr)
    4262         1224 :     ts->m_vr = NULL;
    4263         8583 : }
    4264              : 
    4265              : /* Do final processing of results of IPA propagation regarding NODE, clone it
    4266              :    if appropriate.  */
    4267              : 
    4268              : static void
    4269      1304448 : process_isra_node_results (cgraph_node *node,
    4270              :                            hash_map<const char *, unsigned> *clone_num_suffixes)
    4271              : {
    4272      1304448 :   isra_func_summary *ifs = func_sums->get (node);
    4273      1304448 :   if (!ifs || !ifs->m_candidate)
    4274      1180368 :     return;
    4275              : 
    4276       386282 :   auto_vec<bool, 16> surviving_params;
    4277       386282 :   bool check_surviving = false;
    4278       386282 :   clone_info *cinfo = clone_info::get (node);
    4279       386282 :   if (cinfo && cinfo->param_adjustments)
    4280              :     {
    4281        13205 :       check_surviving = true;
    4282        13205 :       cinfo->param_adjustments->get_surviving_params (&surviving_params);
    4283              :     }
    4284              : 
    4285       386282 :   unsigned param_count = vec_safe_length (ifs->m_parameters);
    4286       386282 :   bool will_change_function = false;
    4287       386282 :   if (ifs->m_returns_value && ifs->m_return_ignored)
    4288              :     will_change_function = true;
    4289              :   else
    4290       879723 :     for (unsigned i = 0; i < param_count; i++)
    4291              :       {
    4292       617521 :         isra_param_desc *desc = &(*ifs->m_parameters)[i];
    4293       562000 :         if ((desc->locally_unused || desc->split_candidate)
    4294              :             /* Make sure we do not clone just to attempt to remove an already
    4295              :                removed unused argument.  */
    4296       654143 :             && (!check_surviving
    4297       530197 :                 || (i < surviving_params.length ()
    4298         2063 :                     && surviving_params[i])))
    4299              :           {
    4300              :             will_change_function = true;
    4301              :             break;
    4302              :           }
    4303              :       }
    4304       351589 :   if (!will_change_function)
    4305       262202 :     return;
    4306              : 
    4307       124080 :   if (dump_file)
    4308              :     {
    4309           84 :       fprintf (dump_file, "\nEvaluating analysis results for %s\n",
    4310              :                node->dump_name ());
    4311           84 :       if (ifs->m_returns_value && ifs->m_return_ignored)
    4312           15 :         fprintf (dump_file, "  Will remove return value.\n");
    4313              :     }
    4314              : 
    4315       124080 :   ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
    4316       124080 :   if (ipcp_ts)
    4317         8583 :     zap_useless_ipcp_results (ifs, ipcp_ts);
    4318       124080 :   vec<ipa_adjusted_param, va_gc> *new_params = NULL;
    4319       124080 :   if (ipa_param_adjustments *old_adjustments
    4320       124080 :          = cinfo ? cinfo->param_adjustments : NULL)
    4321              :     {
    4322         2496 :       unsigned old_adj_len = vec_safe_length (old_adjustments->m_adj_params);
    4323         5896 :       for (unsigned i = 0; i < old_adj_len; i++)
    4324              :         {
    4325         3400 :           ipa_adjusted_param *old_adj = &(*old_adjustments->m_adj_params)[i];
    4326         3400 :           push_param_adjustments_for_index (ifs, old_adj->base_index, i,
    4327              :                                             old_adj, ipcp_ts, &new_params);
    4328              :         }
    4329              :     }
    4330              :   else
    4331       462556 :     for (unsigned i = 0; i < param_count; i++)
    4332       340972 :       push_param_adjustments_for_index (ifs, i, i, NULL, ipcp_ts, &new_params);
    4333              : 
    4334       124080 :   ipa_param_adjustments *new_adjustments
    4335       124080 :     = (new (ggc_alloc <ipa_param_adjustments> ())
    4336              :        ipa_param_adjustments (new_params, param_count,
    4337       213467 :                               ifs->m_returns_value && ifs->m_return_ignored));
    4338              : 
    4339       124080 :   if (dump_file && (dump_flags & TDF_DETAILS))
    4340              :     {
    4341           32 :       fprintf (dump_file, "\n  Created adjustments:\n");
    4342           32 :       new_adjustments->dump (dump_file);
    4343              :     }
    4344              : 
    4345       372240 :   unsigned &suffix_counter = clone_num_suffixes->get_or_insert (
    4346       124080 :                                IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (
    4347              :                                  node->decl)));
    4348       124080 :   auto_vec<cgraph_edge *> callers = node->collect_callers ();
    4349       124080 :   cgraph_node *new_node
    4350       124080 :     = node->create_virtual_clone (callers, NULL, new_adjustments, "isra",
    4351              :                                   suffix_counter);
    4352       124080 :   suffix_counter++;
    4353       124080 :   if (node->calls_comdat_local && node->same_comdat_group)
    4354              :     {
    4355            0 :       new_node->add_to_same_comdat_group (node);
    4356            0 :       new_node->call_for_symbol_and_aliases (mark_callers_calls_comdat_local,
    4357              :                                              NULL, true);
    4358              :     }
    4359       124080 :   new_node->calls_comdat_local = node->calls_comdat_local;
    4360              : 
    4361       124080 :   if (dump_file)
    4362           84 :     fprintf (dump_file, "  Created new node %s\n", new_node->dump_name ());
    4363       124080 :   callers.release ();
    4364       386282 : }
    4365              : 
    4366              : /* If INDICES is not empty, dump a combination of NODE's dump_name and MSG
    4367              :    followed by the list of numbers in INDICES.  */
    4368              : 
    4369              : static void
    4370       660162 : dump_list_of_param_indices (const cgraph_node *node, const char* msg,
    4371              :                             const vec<unsigned> &indices)
    4372              : {
    4373       660162 :   if (indices.is_empty ())
    4374              :     return;
    4375           13 :   fprintf (dump_file, "The following parameters of %s %s:", node->dump_name (),
    4376              :            msg);
    4377           52 :   for (unsigned i : indices)
    4378           13 :     fprintf (dump_file, " %u", i);
    4379           13 :   fprintf (dump_file, "\n");
    4380              : }
    4381              : 
    4382              : /* Check which parameters of NODE described by IFS have survived until IPA-SRA
    4383              :    and disable transformations for those which have not or which should not
    4384              :    transformed because the associated debug counter reached its limit.  Return
    4385              :    true if none survived or if there were no candidates to begin with.
    4386              :    Additionally, also adjust parameter descriptions based on debug counters and
    4387              :    hints propagated earlier.  */
    4388              : 
    4389              : static bool
    4390       386282 : adjust_parameter_descriptions (cgraph_node *node, isra_func_summary *ifs)
    4391              : {
    4392       386282 :   bool ret = true;
    4393       386282 :   unsigned len = vec_safe_length (ifs->m_parameters);
    4394       330081 :   if (!len)
    4395              :     return true;
    4396              : 
    4397       330081 :   auto_vec<bool, 16> surviving_params;
    4398       330081 :   bool check_surviving = false;
    4399       330081 :   clone_info *cinfo = clone_info::get (node);
    4400       330081 :   if (cinfo && cinfo->param_adjustments)
    4401              :     {
    4402        13205 :       check_surviving = true;
    4403        13205 :       cinfo->param_adjustments->get_surviving_params (&surviving_params);
    4404              :     }
    4405       330081 :   ipcp_transformation *ipcp_ts = ipcp_get_transformation_summary (node);
    4406       330081 :   auto_vec <unsigned> dump_dead_indices;
    4407       330081 :   auto_vec <unsigned> dump_bad_cond_indices;
    4408      1130655 :   for (unsigned i = 0; i < len; i++)
    4409              :     {
    4410       800574 :       isra_param_desc *desc = &(*ifs->m_parameters)[i];
    4411       800574 :       if (!dbg_cnt (ipa_sra_params))
    4412              :         {
    4413            0 :           desc->locally_unused = false;
    4414            0 :           desc->split_candidate = false;
    4415            0 :           continue;
    4416              :         }
    4417              : 
    4418       800574 :       if (desc->split_only_when_retval_removed
    4419         6910 :                && !ifs->m_return_ignored)
    4420              :         {
    4421            0 :           if (dump_file && (dump_flags & TDF_DETAILS)
    4422         1526 :               && (desc->locally_unused || desc->split_candidate))
    4423            0 :             dump_bad_cond_indices.safe_push (i);
    4424              : 
    4425         1526 :           gcc_checking_assert (!desc->locally_unused
    4426              :                                || desc->remove_only_when_retval_removed);
    4427         1526 :           desc->locally_unused = false;
    4428         1526 :           desc->split_candidate = false;
    4429         1526 :           continue;
    4430              :         }
    4431       799048 :       if (desc->remove_only_when_retval_removed
    4432        45964 :                && !ifs->m_return_ignored)
    4433              :         {
    4434           18 :           if (dump_file && (dump_flags & TDF_DETAILS)
    4435        36593 :               && (desc->locally_unused || desc->split_candidate))
    4436           12 :             dump_bad_cond_indices.safe_push (i);
    4437              : 
    4438        36581 :           desc->locally_unused = false;
    4439              :         }
    4440       799048 :       if (check_surviving
    4441       836973 :                && (i >= surviving_params.length ()
    4442        19157 :                    || !surviving_params[i]))
    4443              :         {
    4444              :           /* Even if the parameter was removed by a previous IPA pass, we do
    4445              :              not clear locally_unused because if it really is unused, this
    4446              :              information might be useful in callers.  */
    4447        22726 :           desc->split_candidate = false;
    4448              : 
    4449        22726 :           if (dump_file && (dump_flags & TDF_DETAILS))
    4450            0 :             dump_dead_indices.safe_push (i);
    4451              :         }
    4452              : 
    4453       799048 :       if (desc->split_candidate && desc->conditionally_dereferenceable)
    4454              :         {
    4455          886 :           gcc_assert (desc->safe_size_set);
    4456         1483 :           for (param_access *pa : *desc->accesses)
    4457         1329 :             if ((pa->unit_offset + pa->unit_size) > desc->safe_size)
    4458              :               {
    4459          732 :                 if (dump_file && (dump_flags & TDF_DETAILS))
    4460            1 :                   dump_bad_cond_indices.safe_push (i);
    4461          732 :                 desc->split_candidate = false;
    4462          732 :                 break;
    4463              :               }
    4464              :         }
    4465              : 
    4466       799048 :       if (desc->split_candidate)
    4467              :         {
    4468       212799 :           if (desc->by_ref && !desc->not_specially_constructed)
    4469              :             {
    4470        32498 :               int extra_factor
    4471        32498 :                 = opt_for_fn (node->decl,
    4472              :                               param_ipa_sra_ptrwrap_growth_factor);
    4473        32498 :               desc->param_size_limit = extra_factor * desc->param_size_limit;
    4474              :             }
    4475              :           /* Don't disqualify assume bodies, as they don't make into the final
    4476              :              binary anyway.  */
    4477       212799 :           if (!node->assume_function
    4478       212799 :               && size_would_violate_limit_p (desc, desc->size_reached))
    4479         3049 :             desc->split_candidate = false;
    4480              :         }
    4481              : 
    4482              :       /* Avoid ICEs on size-mismatched VIEW_CONVERT_EXPRs when callers and
    4483              :          callees don't agree on types in aggregates and we try to do both
    4484              :          IPA-CP and IPA-SRA.  */
    4485       799048 :       if (ipcp_ts && desc->split_candidate)
    4486              :         {
    4487        22860 :           ipa_argagg_value_list avl (ipcp_ts);
    4488        58645 :           for (const param_access *pa : desc->accesses)
    4489              :             {
    4490        17787 :               if (!pa->certain)
    4491         1739 :                 continue;
    4492        16048 :               tree value = avl.get_value (i, pa->unit_offset);
    4493        16048 :               if (value
    4494        16048 :                   && ((tree_to_uhwi (TYPE_SIZE (TREE_TYPE (value)))
    4495         7220 :                        / BITS_PER_UNIT)
    4496         7220 :                       != pa->unit_size))
    4497              :                 {
    4498           48 :                   desc->split_candidate = false;
    4499           48 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    4500            0 :                     dump_dead_indices.safe_push (i);
    4501              :                   break;
    4502              :                 }
    4503              :             }
    4504              :         }
    4505              : 
    4506       799048 :       if (desc->locally_unused || desc->split_candidate)
    4507       800574 :         ret = false;
    4508              :     }
    4509              : 
    4510       330081 :   dump_list_of_param_indices (node, "are dead on arrival or have a type "
    4511              :                               "mismatch with IPA-CP", dump_dead_indices);
    4512       330081 :   dump_list_of_param_indices (node, "fail additional requirements ",
    4513              :                               dump_bad_cond_indices);
    4514              : 
    4515       330081 :   return ret;
    4516       330081 : }
    4517              : 
    4518              : 
    4519              : /* Run the interprocedural part of IPA-SRA. */
    4520              : 
    4521              : static unsigned int
    4522       130649 : ipa_sra_analysis (void)
    4523              : {
    4524       130649 :   if (dump_file)
    4525              :     {
    4526           62 :       fprintf (dump_file, "\n========== IPA-SRA IPA stage ==========\n");
    4527           62 :       ipa_sra_dump_all_summaries (dump_file, false);
    4528              :     }
    4529              : 
    4530       130649 :   gcc_checking_assert (func_sums);
    4531       130649 :   gcc_checking_assert (call_sums);
    4532       130649 :   cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
    4533       130649 :   auto_vec <cgraph_node *, 16> stack;
    4534       130649 :   int node_scc_count = ipa_reduced_postorder (order, true, NULL);
    4535              : 
    4536              :   /* One sweep from callers to callees for return value removal.  */
    4537      1506547 :   for (int i = node_scc_count - 1; i >= 0 ; i--)
    4538              :     {
    4539      1375898 :       cgraph_node *scc_rep = order[i];
    4540      1375898 :       vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
    4541              : 
    4542              :       /* Preliminary IPA function level checks.  */
    4543      5509567 :       for (cgraph_node *v : cycle_nodes)
    4544              :         {
    4545      1381873 :           isra_func_summary *ifs = func_sums->get (v);
    4546      1381873 :           if (!ifs || !ifs->m_candidate)
    4547       325165 :             continue;
    4548      1056708 :           if (!ipa_sra_ipa_function_checks (v)
    4549      1056708 :               || check_all_callers_for_issues (v))
    4550       670426 :             ifs->zap ();
    4551              :         }
    4552              : 
    4553      4133669 :       for (cgraph_node *v : cycle_nodes)
    4554              :         {
    4555      1381873 :           isra_func_summary *ifs = func_sums->get (v);
    4556      1381873 :           if (!ifs || !ifs->m_candidate)
    4557       995591 :             continue;
    4558       386282 :           bool return_needed
    4559       386282 :             = (ifs->m_returns_value
    4560       386282 :                && (!dbg_cnt (ipa_sra_retvalues)
    4561       191878 :                    || v->call_for_symbol_and_aliases (retval_used_p,
    4562       386282 :                                                       NULL, true)));
    4563       386282 :           ifs->m_return_ignored = !return_needed;
    4564       386282 :           if (return_needed)
    4565       313282 :             isra_push_node_to_stack (v, ifs, &stack);
    4566              :         }
    4567              : 
    4568      1533083 :       while (!stack.is_empty ())
    4569              :         {
    4570       157185 :           cgraph_node *node = stack.pop ();
    4571       157185 :           isra_func_summary *ifs = func_sums->get (node);
    4572       157185 :           gcc_checking_assert (ifs && ifs->m_queued);
    4573       157185 :           ifs->m_queued = false;
    4574              : 
    4575       873314 :           for (cgraph_edge *cs = node->callees; cs; cs = cs->next_callee)
    4576       716129 :             if (ipa_edge_within_scc (cs)
    4577       716129 :                 && call_sums->get (cs)->m_return_returned)
    4578              :               {
    4579         1253 :                 enum availability av;
    4580         1253 :                 cgraph_node *callee = cs->callee->function_symbol (&av);
    4581         1253 :                 isra_func_summary *to_ifs = func_sums->get (callee);
    4582         1253 :                 if (to_ifs && to_ifs->m_return_ignored)
    4583              :                   {
    4584          544 :                     to_ifs->m_return_ignored = false;
    4585         1088 :                     isra_push_node_to_stack (callee, to_ifs, &stack);
    4586              :                   }
    4587              :               }
    4588              :         }
    4589              : 
    4590              :       /* Parameter hint propagation.  */
    4591      4133669 :       for (cgraph_node *v : cycle_nodes)
    4592              :         {
    4593      1381873 :           isra_func_summary *ifs = func_sums->get (v);
    4594      1381873 :           propagate_hints_to_all_callees (v, ifs, &stack);
    4595              :         }
    4596              : 
    4597      1378654 :       while (!stack.is_empty ())
    4598              :         {
    4599         2756 :           cgraph_node *node = stack.pop ();
    4600         2756 :           isra_func_summary *ifs = func_sums->get (node);
    4601         2756 :           gcc_checking_assert (ifs && ifs->m_queued);
    4602         2756 :           ifs->m_queued = false;
    4603         2756 :           propagate_hints_to_all_callees (node, ifs, &stack);
    4604              :         }
    4605              : 
    4606      1375898 :       cycle_nodes.release ();
    4607              :     }
    4608              : 
    4609              :   /* One sweep from callees to callers for parameter removal and splitting.  */
    4610      1506547 :   for (int i = 0; i < node_scc_count; i++)
    4611              :     {
    4612      1375898 :       cgraph_node *scc_rep = order[i];
    4613      1375898 :       vec<cgraph_node *> cycle_nodes = ipa_get_nodes_in_cycle (scc_rep);
    4614              : 
    4615              :       /* First step of parameter removal.  */
    4616      5509567 :       for (cgraph_node *v : cycle_nodes)
    4617              :         {
    4618      1381873 :           isra_func_summary *ifs = func_sums->get (v);
    4619      1381873 :           if (!ifs || !ifs->m_candidate)
    4620       995591 :             continue;
    4621       386282 :           if (adjust_parameter_descriptions (v, ifs))
    4622       198386 :             continue;
    4623       202053 :           for (cgraph_edge *cs = v->indirect_calls; cs; cs = cs->next_callee)
    4624        14157 :             process_edge_to_unknown_caller (cs);
    4625      1100546 :           for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
    4626       912650 :             if (!ipa_edge_within_scc (cs))
    4627       907024 :               param_removal_cross_scc_edge (cs);
    4628              :         }
    4629              : 
    4630              :       /* Look at edges within the current SCC and propagate used-ness across
    4631              :          them, pushing onto the stack all notes which might need to be
    4632              :          revisited.  */
    4633      4133669 :       for (cgraph_node *v : cycle_nodes)
    4634      1381873 :         v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
    4635              :                                                &stack, true);
    4636              : 
    4637              :       /* Keep revisiting and pushing until nothing changes.  */
    4638      1376678 :       while (!stack.is_empty ())
    4639              :         {
    4640          780 :           cgraph_node *v = stack.pop ();
    4641          780 :           isra_func_summary *ifs = func_sums->get (v);
    4642          780 :           gcc_checking_assert (ifs && ifs->m_queued);
    4643          780 :           ifs->m_queued = false;
    4644              : 
    4645          780 :           v->call_for_symbol_thunks_and_aliases (propagate_used_to_scc_callers,
    4646              :                                                  &stack, true);
    4647              :         }
    4648              : 
    4649              :       /* Parameter splitting.  */
    4650      1424396 :       bool repeat_scc_access_propagation;
    4651      1424396 :       do
    4652              :         {
    4653      1424396 :           repeat_scc_access_propagation = false;
    4654      4282105 :           for (cgraph_node *v : cycle_nodes)
    4655              :             {
    4656      1433313 :               isra_func_summary *ifs = func_sums->get (v);
    4657      2485734 :               if (!ifs
    4658      1111196 :                   || !ifs->m_candidate
    4659      1870406 :                   || vec_safe_is_empty (ifs->m_parameters))
    4660      1052421 :                 continue;
    4661      2085147 :               for (cgraph_edge *cs = v->callees; cs; cs = cs->next_callee)
    4662      1704255 :                 if (param_splitting_across_edge (cs))
    4663        52455 :                   repeat_scc_access_propagation = true;
    4664              :             }
    4665              :         }
    4666              :       while (repeat_scc_access_propagation);
    4667              : 
    4668      1375898 :       if (flag_checking)
    4669      4133618 :         for (cgraph_node *v : cycle_nodes)
    4670      1381856 :           verify_splitting_accesses (v, true);
    4671              : 
    4672      1375898 :       cycle_nodes.release ();
    4673              :     }
    4674              : 
    4675       130649 :   ipa_free_postorder_info ();
    4676       130649 :   free (order);
    4677              : 
    4678       130649 :   if (dump_file)
    4679              :     {
    4680           62 :       if (dump_flags & TDF_DETAILS)
    4681              :         {
    4682           17 :           fprintf (dump_file, "\n========== IPA-SRA propagation final state "
    4683              :                    " ==========\n");
    4684           17 :           ipa_sra_dump_all_summaries (dump_file, true);
    4685              :         }
    4686           62 :       fprintf (dump_file, "\n========== IPA-SRA decisions ==========\n");
    4687              :     }
    4688              : 
    4689       130649 :   hash_map<const char *, unsigned> *clone_num_suffixes
    4690       130649 :     = new hash_map<const char *, unsigned>;
    4691              : 
    4692       130649 :   cgraph_node *node;
    4693      1435097 :   FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
    4694      1304448 :     process_isra_node_results (node, clone_num_suffixes);
    4695              : 
    4696       130649 :   delete clone_num_suffixes;
    4697       130649 :   ggc_delete (func_sums);
    4698       130649 :   func_sums = NULL;
    4699       130649 :   delete call_sums;
    4700       130649 :   call_sums = NULL;
    4701              : 
    4702       130649 :   if (dump_file)
    4703           62 :     fprintf (dump_file, "\n========== IPA SRA IPA analysis done "
    4704              :              "==========\n\n");
    4705       130649 :   return 0;
    4706       130649 : }
    4707              : 
    4708              : 
    4709              : const pass_data pass_data_ipa_sra =
    4710              : {
    4711              :   IPA_PASS, /* type */
    4712              :   "sra", /* name */
    4713              :   OPTGROUP_NONE, /* optinfo_flags */
    4714              :   TV_IPA_SRA, /* tv_id */
    4715              :   0, /* properties_required */
    4716              :   0, /* properties_provided */
    4717              :   0, /* properties_destroyed */
    4718              :   0, /* todo_flags_start */
    4719              :   ( TODO_dump_symtab | TODO_remove_functions ), /* todo_flags_finish */
    4720              : };
    4721              : 
    4722              : class pass_ipa_sra : public ipa_opt_pass_d
    4723              : {
    4724              : public:
    4725       297180 :   pass_ipa_sra (gcc::context *ctxt)
    4726              :     : ipa_opt_pass_d (pass_data_ipa_sra, ctxt,
    4727              :                       ipa_sra_generate_summary, /* generate_summary */
    4728              :                       ipa_sra_write_summary, /* write_summary */
    4729              :                       ipa_sra_read_summary, /* read_summary */
    4730              :                       NULL , /* write_optimization_summary */
    4731              :                       NULL, /* read_optimization_summary */
    4732              :                       NULL, /* stmt_fixup */
    4733              :                       0, /* function_transform_todo_flags_start */
    4734              :                       NULL, /* function_transform */
    4735       297180 :                       NULL) /* variable_transform */
    4736       297180 :   {}
    4737              : 
    4738              :   /* opt_pass methods: */
    4739       609367 :   bool gate (function *) final override
    4740              :     {
    4741              :       /* TODO: We should remove the optimize check after we ensure we never run
    4742              :          IPA passes when not optimizing.  */
    4743       609367 :       return (flag_ipa_sra && optimize);
    4744              :     }
    4745              : 
    4746       130649 :   unsigned int execute (function *)  final override
    4747              :   {
    4748       130649 :     return ipa_sra_analysis ();
    4749              :   }
    4750              : 
    4751              : }; // class pass_ipa_sra
    4752              : 
    4753              : } // anon namespace
    4754              : 
    4755              : /* Intraprocedural part of IPA-SRA analysis.  Scan function body of NODE and
    4756              :    create a summary structure describing IPA-SRA opportunities and constraints
    4757              :    in it.  */
    4758              : 
    4759              : static void
    4760      1304084 : ipa_sra_summarize_function (cgraph_node *node)
    4761              : {
    4762      1304084 :   if (dump_file)
    4763          226 :     fprintf (dump_file, "Creating summary for %s/%i:\n", node->name (),
    4764              :              node->get_uid ());
    4765      1304084 :   gcc_obstack_init (&gensum_obstack);
    4766      1304084 :   loaded_decls = new hash_set<tree>;
    4767              : 
    4768      1304084 :   isra_func_summary *ifs = NULL;
    4769      1304084 :   unsigned count = 0;
    4770      1304084 :   if (ipa_sra_preliminary_function_checks (node))
    4771              :     {
    4772      1063309 :       ifs = func_sums->get_create (node);
    4773      1063309 :       ifs->m_candidate = true;
    4774      1063309 :       tree ret = TREE_TYPE (TREE_TYPE (node->decl));
    4775      1063309 :       ifs->m_returns_value = (TREE_CODE (ret) != VOID_TYPE);
    4776      1063309 :       for (tree parm = DECL_ARGUMENTS (node->decl);
    4777      3510717 :            parm;
    4778      2447408 :            parm = DECL_CHAIN (parm))
    4779      2447408 :         count++;
    4780              :     }
    4781      1304084 :   auto_vec<gensum_param_desc, 16> param_descriptions (count);
    4782              : 
    4783      1304084 :   struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
    4784      1304084 :   bool cfun_pushed = false;
    4785      1304084 :   if (count > 0)
    4786              :     {
    4787       861037 :       decl2desc = new hash_map<tree, gensum_param_desc *>;
    4788       861037 :       param_descriptions.reserve_exact (count);
    4789       861037 :       param_descriptions.quick_grow_cleared (count);
    4790              : 
    4791       861037 :       if (create_parameter_descriptors (node, &param_descriptions))
    4792              :         {
    4793       426790 :           push_cfun (fun);
    4794       426790 :           cfun_pushed = true;
    4795       426790 :           final_bbs = BITMAP_ALLOC (NULL);
    4796       426790 :           bb_dereferences = XCNEWVEC (HOST_WIDE_INT,
    4797              :                                       unsafe_by_ref_count
    4798              :                                       * last_basic_block_for_fn (fun));
    4799       426790 :           aa_walking_limit = opt_for_fn (node->decl, param_ipa_max_aa_steps);
    4800              :         }
    4801              :     }
    4802              :   /* Scan function is run even when there are no removal or splitting
    4803              :      candidates so that we can calculate hints on call edges which can be
    4804              :      useful in callees. */
    4805      1304084 :   scan_function (node, fun);
    4806              : 
    4807      1304084 :   if (count > 0)
    4808              :     {
    4809       861037 :       if (dump_file)
    4810              :         {
    4811          172 :           dump_gensum_param_descriptors (dump_file, node->decl,
    4812              :                                          &param_descriptions);
    4813          172 :           fprintf (dump_file, "----------------------------------------\n");
    4814              :         }
    4815              : 
    4816       861037 :       process_scan_results (node, fun, ifs, &param_descriptions);
    4817              : 
    4818       861037 :       if (cfun_pushed)
    4819       426790 :         pop_cfun ();
    4820       861037 :       if (bb_dereferences)
    4821              :         {
    4822       426790 :           free (bb_dereferences);
    4823       426790 :           bb_dereferences = NULL;
    4824       426790 :           BITMAP_FREE (final_bbs);
    4825       426790 :           final_bbs = NULL;
    4826              :         }
    4827              :     }
    4828      1304084 :   isra_analyze_all_outgoing_calls (node);
    4829              : 
    4830      2608168 :   delete loaded_decls;
    4831      1304084 :   loaded_decls = NULL;
    4832      1304084 :   if (decl2desc)
    4833              :     {
    4834       861037 :       delete decl2desc;
    4835       861037 :       decl2desc = NULL;
    4836              :     }
    4837      1304084 :   obstack_free (&gensum_obstack, NULL);
    4838      1304084 :   if (dump_file)
    4839          226 :     fprintf (dump_file, "\n\n");
    4840      1304084 :   if (flag_checking)
    4841      1304066 :     verify_splitting_accesses (node, false);
    4842      1304084 :   return;
    4843      1304084 : }
    4844              : 
    4845              : ipa_opt_pass_d *
    4846       297180 : make_pass_ipa_sra (gcc::context *ctxt)
    4847              : {
    4848       297180 :   return new pass_ipa_sra (ctxt);
    4849              : }
    4850              : 
    4851              : /* Reset all state within ipa-sra.cc so that we can rerun the compiler
    4852              :    within the same process.  For use by toplev::finalize.  */
    4853              : 
    4854              : void
    4855       266934 : ipa_sra_cc_finalize (void)
    4856              : {
    4857       266934 :   if (func_sums)
    4858         9390 :     ggc_delete (func_sums);
    4859       266934 :   func_sums = NULL;
    4860       266934 :   delete call_sums;
    4861       266934 :   call_sums = NULL;
    4862       266934 : }
    4863              : 
    4864              : #include "gt-ipa-sra.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.