LCOV - code coverage report
Current view: top level - gcc - tree-ssa-sccvn.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 95.8 % 4747 4546
Test Date: 2026-11-14 16:39:50 Functions: 98.5 % 137 135
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* SCC value numbering for trees
       2              :    Copyright (C) 2006-2026 Free Software Foundation, Inc.
       3              :    Contributed by Daniel Berlin <dan@dberlin.org>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify
       8              : it under the terms of the GNU General Public License as published by
       9              : the Free Software Foundation; either version 3, or (at your option)
      10              : any later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful,
      13              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      14              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      15              : GNU General Public License for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "gimple.h"
      28              : #include "ssa.h"
      29              : #include "expmed.h"
      30              : #include "insn-config.h"
      31              : #include "memmodel.h"
      32              : #include "emit-rtl.h"
      33              : #include "cgraph.h"
      34              : #include "gimple-pretty-print.h"
      35              : #include "splay-tree-utils.h"
      36              : #include "alias.h"
      37              : #include "fold-const.h"
      38              : #include "stor-layout.h"
      39              : #include "cfganal.h"
      40              : #include "tree-inline.h"
      41              : #include "internal-fn.h"
      42              : #include "gimple-iterator.h"
      43              : #include "gimple-fold.h"
      44              : #include "tree-eh.h"
      45              : #include "flags.h"
      46              : #include "dojump.h"
      47              : #include "explow.h"
      48              : #include "calls.h"
      49              : #include "varasm.h"
      50              : #include "stmt.h"
      51              : #include "expr.h"
      52              : #include "tree-dfa.h"
      53              : #include "tree-ssa.h"
      54              : #include "dumpfile.h"
      55              : #include "cfgloop.h"
      56              : #include "tree-ssa-propagate.h"
      57              : #include "tree-cfg.h"
      58              : #include "domwalk.h"
      59              : #include "gimple-match.h"
      60              : #include "stringpool.h"
      61              : #include "attribs.h"
      62              : #include "tree-pass.h"
      63              : #include "statistics.h"
      64              : #include "langhooks.h"
      65              : #include "ipa-utils.h"
      66              : #include "dbgcnt.h"
      67              : #include "tree-cfgcleanup.h"
      68              : #include "tree-ssa-loop.h"
      69              : #include "tree-scalar-evolution.h"
      70              : #include "tree-ssa-loop-niter.h"
      71              : #include "builtins.h"
      72              : #include "fold-const-call.h"
      73              : #include "ipa-modref-tree.h"
      74              : #include "ipa-modref.h"
      75              : #include "tree-ssa-sccvn.h"
      76              : #include "alloc-pool.h"
      77              : #include "symbol-summary.h"
      78              : #include "sreal.h"
      79              : #include "ipa-cp.h"
      80              : #include "ipa-prop.h"
      81              : #include "target.h"
      82              : 
      83              : /* This algorithm is based on the SCC algorithm presented by Keith
      84              :    Cooper and L. Taylor Simpson in "SCC-Based Value numbering"
      85              :    (http://citeseer.ist.psu.edu/41805.html).  In
      86              :    straight line code, it is equivalent to a regular hash based value
      87              :    numbering that is performed in reverse postorder.
      88              : 
      89              :    For code with cycles, there are two alternatives, both of which
      90              :    require keeping the hashtables separate from the actual list of
      91              :    value numbers for SSA names.
      92              : 
      93              :    1. Iterate value numbering in an RPO walk of the blocks, removing
      94              :    all the entries from the hashtable after each iteration (but
      95              :    keeping the SSA name->value number mapping between iterations).
      96              :    Iterate until it does not change.
      97              : 
      98              :    2. Perform value numbering as part of an SCC walk on the SSA graph,
      99              :    iterating only the cycles in the SSA graph until they do not change
     100              :    (using a separate, optimistic hashtable for value numbering the SCC
     101              :    operands).
     102              : 
     103              :    The second is not just faster in practice (because most SSA graph
     104              :    cycles do not involve all the variables in the graph), it also has
     105              :    some nice properties.
     106              : 
     107              :    One of these nice properties is that when we pop an SCC off the
     108              :    stack, we are guaranteed to have processed all the operands coming from
     109              :    *outside of that SCC*, so we do not need to do anything special to
     110              :    ensure they have value numbers.
     111              : 
     112              :    Another nice property is that the SCC walk is done as part of a DFS
     113              :    of the SSA graph, which makes it easy to perform combining and
     114              :    simplifying operations at the same time.
     115              : 
     116              :    The code below is deliberately written in a way that makes it easy
     117              :    to separate the SCC walk from the other work it does.
     118              : 
     119              :    In order to propagate constants through the code, we track which
     120              :    expressions contain constants, and use those while folding.  In
     121              :    theory, we could also track expressions whose value numbers are
     122              :    replaced, in case we end up folding based on expression
     123              :    identities.
     124              : 
     125              :    In order to value number memory, we assign value numbers to vuses.
     126              :    This enables us to note that, for example, stores to the same
     127              :    address of the same value from the same starting memory states are
     128              :    equivalent.
     129              :    TODO:
     130              : 
     131              :    1. We can iterate only the changing portions of the SCC's, but
     132              :    I have not seen an SCC big enough for this to be a win.
     133              :    2. If you differentiate between phi nodes for loops and phi nodes
     134              :    for if-then-else, you can properly consider phi nodes in different
     135              :    blocks for equivalence.
     136              :    3. We could value number vuses in more cases, particularly, whole
     137              :    structure copies.
     138              : */
     139              : 
     140              : /* There's no BB_EXECUTABLE but we can use BB_VISITED.  */
     141              : #define BB_EXECUTABLE BB_VISITED
     142              : 
     143              : static vn_lookup_kind default_vn_walk_kind;
     144              : 
     145              : /* vn_nary_op hashtable helpers.  */
     146              : 
     147              : struct vn_nary_op_hasher : nofree_ptr_hash <vn_nary_op_s>
     148              : {
     149              :   typedef vn_nary_op_s *compare_type;
     150              :   static inline hashval_t hash (const vn_nary_op_s *);
     151              :   static inline bool equal (const vn_nary_op_s *, const vn_nary_op_s *);
     152              : };
     153              : 
     154              : /* Return the computed hashcode for nary operation P1.  */
     155              : 
     156              : inline hashval_t
     157    786622695 : vn_nary_op_hasher::hash (const vn_nary_op_s *vno1)
     158              : {
     159    786622695 :   return vno1->hashcode;
     160              : }
     161              : 
     162              : /* Compare nary operations P1 and P2 and return true if they are
     163              :    equivalent.  */
     164              : 
     165              : inline bool
     166    998425999 : vn_nary_op_hasher::equal (const vn_nary_op_s *vno1, const vn_nary_op_s *vno2)
     167              : {
     168    998425999 :   return vno1 == vno2 || vn_nary_op_eq (vno1, vno2);
     169              : }
     170              : 
     171              : typedef hash_table<vn_nary_op_hasher> vn_nary_op_table_type;
     172              : typedef vn_nary_op_table_type::iterator vn_nary_op_iterator_type;
     173              : 
     174              : 
     175              : /* vn_phi hashtable helpers.  */
     176              : 
     177              : static int
     178              : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2);
     179              : 
     180              : struct vn_phi_hasher : nofree_ptr_hash <vn_phi_s>
     181              : {
     182              :   static inline hashval_t hash (const vn_phi_s *);
     183              :   static inline bool equal (const vn_phi_s *, const vn_phi_s *);
     184              : };
     185              : 
     186              : /* Return the computed hashcode for phi operation P1.  */
     187              : 
     188              : inline hashval_t
     189     26597376 : vn_phi_hasher::hash (const vn_phi_s *vp1)
     190              : {
     191     26597376 :   return vp1->hashcode;
     192              : }
     193              : 
     194              : /* Compare two phi entries for equality, ignoring VN_TOP arguments.  */
     195              : 
     196              : inline bool
     197     47361866 : vn_phi_hasher::equal (const vn_phi_s *vp1, const vn_phi_s *vp2)
     198              : {
     199     47361866 :   return vp1 == vp2 || vn_phi_eq (vp1, vp2);
     200              : }
     201              : 
     202              : typedef hash_table<vn_phi_hasher> vn_phi_table_type;
     203              : typedef vn_phi_table_type::iterator vn_phi_iterator_type;
     204              : 
     205              : 
     206              : /* Compare two reference operands P1 and P2 for equality.  Return true if
     207              :    they are equal, and false otherwise.  */
     208              : 
     209              : static int
     210     26330560 : vn_reference_op_eq (const void *p1, const void *p2)
     211              : {
     212     26330560 :   const_vn_reference_op_t const vro1 = (const_vn_reference_op_t) p1;
     213     26330560 :   const_vn_reference_op_t const vro2 = (const_vn_reference_op_t) p2;
     214              : 
     215     26330560 :   return (vro1->opcode == vro2->opcode
     216              :           /* We do not care for differences in type qualification.  */
     217     26328732 :           && (vro1->type == vro2->type
     218      1241660 :               || (vro1->type && vro2->type
     219      1241660 :                   && types_compatible_p (TYPE_MAIN_VARIANT (vro1->type),
     220      1241660 :                                          TYPE_MAIN_VARIANT (vro2->type))))
     221     25325917 :           && expressions_equal_p (vro1->op0, vro2->op0)
     222     25277993 :           && expressions_equal_p (vro1->op1, vro2->op1)
     223     25277993 :           && expressions_equal_p (vro1->op2, vro2->op2)
     224     51608553 :           && (vro1->opcode != CALL_EXPR || vro1->clique == vro2->clique));
     225              : }
     226              : 
     227              : /* Free a reference operation structure VP.  */
     228              : 
     229              : static inline void
     230            0 : free_reference (vn_reference_s *vr)
     231              : {
     232            0 :   vr->operands.release ();
     233              : }
     234              : 
     235              : 
     236              : /* vn_reference hashtable helpers.  */
     237              : 
     238              : struct vn_reference_hasher : nofree_ptr_hash <vn_reference_s>
     239              : {
     240              :   static inline hashval_t hash (const vn_reference_s *);
     241              :   static inline bool equal (const vn_reference_s *, const vn_reference_s *);
     242              : };
     243              : 
     244              : /* Return the hashcode for a given reference operation P1.  */
     245              : 
     246              : inline hashval_t
     247   3787303709 : vn_reference_hasher::hash (const vn_reference_s *vr1)
     248              : {
     249   3787303709 :   return vr1->hashcode;
     250              : }
     251              : 
     252              : inline bool
     253   4510767422 : vn_reference_hasher::equal (const vn_reference_s *v, const vn_reference_s *c)
     254              : {
     255   4510767422 :   return v == c || vn_reference_eq (v, c);
     256              : }
     257              : 
     258              : typedef hash_table<vn_reference_hasher> vn_reference_table_type;
     259              : typedef vn_reference_table_type::iterator vn_reference_iterator_type;
     260              : 
     261              : /* Pretty-print OPS to OUTFILE.  */
     262              : 
     263              : void
     264          287 : print_vn_reference_ops (FILE *outfile, const vec<vn_reference_op_s> ops)
     265              : {
     266          287 :   vn_reference_op_t vro;
     267          287 :   unsigned int i;
     268          287 :   fprintf (outfile, "{");
     269         1591 :   for (i = 0; ops.iterate (i, &vro); i++)
     270              :     {
     271         1017 :       bool closebrace = false;
     272         1017 :       if (vro->opcode != SSA_NAME
     273          803 :           && TREE_CODE_CLASS (vro->opcode) != tcc_declaration)
     274              :         {
     275          803 :           fprintf (outfile, "%s", get_tree_code_name (vro->opcode));
     276          803 :           if (vro->op0 || vro->opcode == CALL_EXPR)
     277              :             {
     278          803 :               fprintf (outfile, "<");
     279          803 :               closebrace = true;
     280              :             }
     281              :         }
     282         1017 :       if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
     283          275 :         fprintf (outfile, "(A%d)", TYPE_ALIGN (vro->type));
     284         1017 :       if (vro->op0 || vro->opcode == CALL_EXPR)
     285              :         {
     286         1017 :           if (!vro->op0)
     287            0 :             fprintf (outfile, internal_fn_name ((internal_fn)vro->clique));
     288              :           else
     289              :             {
     290         1017 :               if (vro->opcode == MEM_REF || vro->opcode == TARGET_MEM_REF)
     291              :                 {
     292          275 :                   fprintf (outfile, "(");
     293          275 :                   print_generic_expr (outfile, TREE_TYPE (vro->op0));
     294          275 :                   fprintf (outfile, ")");
     295              :                 }
     296         1017 :               print_generic_expr (outfile, vro->op0);
     297              :             }
     298         1017 :           if (vro->op1)
     299              :             {
     300          185 :               fprintf (outfile, ",");
     301          185 :               print_generic_expr (outfile, vro->op1);
     302              :             }
     303         1017 :           if (vro->op2)
     304              :             {
     305          185 :               fprintf (outfile, ",");
     306          185 :               print_generic_expr (outfile, vro->op2);
     307              :             }
     308              :         }
     309         1017 :       if (closebrace)
     310          803 :         fprintf (outfile, ">");
     311         1017 :       if (i != ops.length () - 1)
     312          730 :         fprintf (outfile, ",");
     313              :     }
     314          287 :   fprintf (outfile, "}");
     315          287 : }
     316              : 
     317              : DEBUG_FUNCTION void
     318            0 : debug_vn_reference_ops (const vec<vn_reference_op_s> ops)
     319              : {
     320            0 :   print_vn_reference_ops (stderr, ops);
     321            0 :   fputc ('\n', stderr);
     322            0 : }
     323              : 
     324              : /* The set of VN hashtables.  */
     325              : 
     326              : typedef struct vn_tables_s
     327              : {
     328              :   vn_nary_op_table_type *nary;
     329              :   vn_phi_table_type *phis;
     330              :   vn_reference_table_type *references;
     331              :   /* Expressions inserted by vn_nary_build_or_lookup_1, keyed on the
     332              :      operands of the inserted statement rather than their value numbers.
     333              :      This is not unwound on iteration.  Only allocated when iterating.  */
     334              :   vn_nary_op_table_type *inserted_exprs;
     335              : } *vn_tables_t;
     336              : 
     337              : 
     338              : /* vn_constant hashtable helpers.  */
     339              : 
     340              : struct vn_constant_hasher : free_ptr_hash <vn_constant_s>
     341              : {
     342              :   static inline hashval_t hash (const vn_constant_s *);
     343              :   static inline bool equal (const vn_constant_s *, const vn_constant_s *);
     344              : };
     345              : 
     346              : /* Hash table hash function for vn_constant_t.  */
     347              : 
     348              : inline hashval_t
     349     12504905 : vn_constant_hasher::hash (const vn_constant_s *vc1)
     350              : {
     351     12504905 :   return vc1->hashcode;
     352              : }
     353              : 
     354              : /* Hash table equality function for vn_constant_t.  */
     355              : 
     356              : inline bool
     357     15064397 : vn_constant_hasher::equal (const vn_constant_s *vc1, const vn_constant_s *vc2)
     358              : {
     359     15064397 :   if (vc1->hashcode != vc2->hashcode)
     360              :     return false;
     361              : 
     362      2266779 :   return vn_constant_eq_with_type (vc1->constant, vc2->constant);
     363              : }
     364              : 
     365              : static hash_table<vn_constant_hasher> *constant_to_value_id;
     366              : 
     367              : 
     368              : /* Obstack we allocate the vn-tables elements from.  */
     369              : static obstack vn_tables_obstack;
     370              : /* Special obstack we never unwind.  */
     371              : static obstack vn_tables_insert_obstack;
     372              : 
     373              : static vn_reference_t last_inserted_ref;
     374              : static vn_phi_t last_inserted_phi;
     375              : static vn_nary_op_t last_inserted_nary;
     376              : static vn_ssa_aux_t last_pushed_avail;
     377              : 
     378              : /* Valid hashtables storing information we have proven to be
     379              :    correct.  */
     380              : static vn_tables_t valid_info;
     381              : 
     382              : /* Global RPO state for access from hooks.  */
     383              : static class eliminate_dom_walker *rpo_avail;
     384              : basic_block vn_context_bb;
     385              : int *vn_bb_to_rpo;
     386              : static int *vn_exec_idom;
     387              : static int vn_bb_in_region_flag;
     388              : static edge vn_entry_edge;
     389              : 
     390              : 
     391              : /* Valueization hook for simplify_replace_tree.  Valueize NAME if it is
     392              :    an SSA name, otherwise just return it.  */
     393              : tree (*vn_valueize) (tree);
     394              : static tree
     395        81527 : vn_valueize_for_srt (tree t, void* context ATTRIBUTE_UNUSED)
     396              : {
     397        81527 :   basic_block saved_vn_context_bb = vn_context_bb;
     398              :   /* Look for sth available at the definition block of the argument.
     399              :      This avoids inconsistencies between availability there which
     400              :      decides if the stmt can be removed and availability at the
     401              :      use site.  The SSA property ensures that things available
     402              :      at the definition are also available at uses.  */
     403        81527 :   if (!SSA_NAME_IS_DEFAULT_DEF (t))
     404        77598 :     vn_context_bb = gimple_bb (SSA_NAME_DEF_STMT (t));
     405        81527 :   tree res = vn_valueize (t);
     406        81527 :   vn_context_bb = saved_vn_context_bb;
     407        81527 :   return res;
     408              : }
     409              : 
     410              : 
     411              : /* This represents the top of the VN lattice, which is the universal
     412              :    value.  */
     413              : 
     414              : tree VN_TOP;
     415              : 
     416              : /* Unique counter for our value ids.  */
     417              : 
     418              : static unsigned int next_value_id;
     419              : static int next_constant_value_id;
     420              : 
     421              : 
     422              : /* Table of vn_ssa_aux_t's, one per ssa_name.  The vn_ssa_aux_t objects
     423              :    are allocated on an obstack for locality reasons, and to free them
     424              :    without looping over the vec.  */
     425              : 
     426              : struct vn_ssa_aux_hasher : typed_noop_remove <vn_ssa_aux_t>
     427              : {
     428              :   typedef vn_ssa_aux_t value_type;
     429              :   typedef tree compare_type;
     430              :   static inline hashval_t hash (const value_type &);
     431              :   static inline bool equal (const value_type &, const compare_type &);
     432              :   static inline void mark_deleted (value_type &) {}
     433              :   static const bool empty_zero_p = true;
     434            0 :   static inline void mark_empty (value_type &e) { e = NULL; }
     435              :   static inline bool is_deleted (value_type &) { return false; }
     436  >14155*10^7 :   static inline bool is_empty (value_type &e) { return e == NULL; }
     437              : };
     438              : 
     439              : hashval_t
     440  46678699903 : vn_ssa_aux_hasher::hash (const value_type &entry)
     441              : {
     442  46678699903 :   return SSA_NAME_VERSION (entry->name);
     443              : }
     444              : 
     445              : bool
     446  53392076578 : vn_ssa_aux_hasher::equal (const value_type &entry, const compare_type &name)
     447              : {
     448  53392076578 :   return name == entry->name;
     449              : }
     450              : 
     451              : static hash_table<vn_ssa_aux_hasher> *vn_ssa_aux_hash;
     452              : typedef hash_table<vn_ssa_aux_hasher>::iterator vn_ssa_aux_iterator_type;
     453              : static struct obstack vn_ssa_aux_obstack;
     454              : 
     455              : static vn_nary_op_t vn_nary_op_insert_stmt (gimple *, tree);
     456              : static vn_nary_op_t vn_nary_op_insert_into (vn_nary_op_t,
     457              :                                             vn_nary_op_table_type *);
     458              : static void init_vn_nary_op_from_pieces (vn_nary_op_t, unsigned int,
     459              :                                          enum tree_code, tree, tree *);
     460              : static tree vn_lookup_simplify_result (gimple_match_op *);
     461              : static vn_reference_t vn_reference_lookup_or_insert_for_pieces
     462              :           (tree, alias_set_type, alias_set_type, poly_int64, poly_int64, tree,
     463              :            vec<vn_reference_op_s, va_heap>, tree);
     464              : 
     465              : /* Return whether there is value numbering information for a given SSA name.  */
     466              : 
     467              : bool
     468      5266691 : has_VN_INFO (tree name)
     469              : {
     470      5266691 :   return vn_ssa_aux_hash->find_with_hash (name, SSA_NAME_VERSION (name));
     471              : }
     472              : 
     473              : vn_ssa_aux_t
     474   4135973126 : VN_INFO (tree name)
     475              : {
     476   4135973126 :   vn_ssa_aux_t *res
     477   4135973126 :     = vn_ssa_aux_hash->find_slot_with_hash (name, SSA_NAME_VERSION (name),
     478              :                                             INSERT);
     479   4135973126 :   if (*res != NULL)
     480              :     return *res;
     481              : 
     482    178133055 :   vn_ssa_aux_t newinfo = *res = XOBNEW (&vn_ssa_aux_obstack, struct vn_ssa_aux);
     483    178133055 :   memset (newinfo, 0, sizeof (struct vn_ssa_aux));
     484    178133055 :   newinfo->name = name;
     485    178133055 :   newinfo->valnum = VN_TOP;
     486              :   /* We are using the visited flag to handle uses with defs not within the
     487              :      region being value-numbered.  */
     488    178133055 :   newinfo->visited = false;
     489              : 
     490              :   /* Given we create the VN_INFOs on-demand now we have to do initialization
     491              :      different than VN_TOP here.  */
     492    178133055 :   if (SSA_NAME_IS_DEFAULT_DEF (name))
     493      9502088 :     switch (TREE_CODE (SSA_NAME_VAR (name)))
     494              :       {
     495      1709712 :       case VAR_DECL:
     496              :         /* All undefined vars are VARYING.  */
     497      1709712 :         newinfo->valnum = name;
     498      1709712 :         newinfo->visited = true;
     499      1709712 :         break;
     500              : 
     501      7731341 :       case PARM_DECL:
     502              :         /* Parameters are VARYING but we can record a condition
     503              :            if we know it is a non-NULL pointer.  */
     504      7731341 :         newinfo->visited = true;
     505      7731341 :         newinfo->valnum = name;
     506     11918965 :         if (POINTER_TYPE_P (TREE_TYPE (name))
     507      8910167 :             && nonnull_arg_p (SSA_NAME_VAR (name)))
     508              :           {
     509      2392114 :             tree ops[2];
     510      2392114 :             ops[0] = name;
     511      2392114 :             ops[1] = build_int_cst (TREE_TYPE (name), 0);
     512      2392114 :             vn_nary_op_t nary;
     513              :             /* Allocate from non-unwinding stack.  */
     514      2392114 :             nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
     515      2392114 :             init_vn_nary_op_from_pieces (nary, 2, NE_EXPR,
     516              :                                          boolean_type_node, ops);
     517      2392114 :             nary->predicated_values = 0;
     518      2392114 :             nary->u.result = boolean_true_node;
     519      2392114 :             vn_nary_op_insert_into (nary, valid_info->nary);
     520      2392114 :             gcc_assert (nary->unwind_to == NULL);
     521              :             /* Also do not link it into the undo chain.  */
     522      2392114 :             last_inserted_nary = nary->next;
     523      2392114 :             nary->next = (vn_nary_op_t)(void *)-1;
     524      2392114 :             nary = alloc_vn_nary_op_noinit (2, &vn_tables_insert_obstack);
     525      2392114 :             init_vn_nary_op_from_pieces (nary, 2, EQ_EXPR,
     526              :                                          boolean_type_node, ops);
     527      2392114 :             nary->predicated_values = 0;
     528      2392114 :             nary->u.result = boolean_false_node;
     529      2392114 :             vn_nary_op_insert_into (nary, valid_info->nary);
     530      2392114 :             gcc_assert (nary->unwind_to == NULL);
     531      2392114 :             last_inserted_nary = nary->next;
     532      2392114 :             nary->next = (vn_nary_op_t)(void *)-1;
     533      2392114 :             if (dump_file && (dump_flags & TDF_DETAILS))
     534              :               {
     535           38 :                 fprintf (dump_file, "Recording ");
     536           38 :                 print_generic_expr (dump_file, name, TDF_SLIM);
     537           38 :                 fprintf (dump_file, " != 0\n");
     538              :               }
     539              :           }
     540              :         break;
     541              : 
     542        61035 :       case RESULT_DECL:
     543              :         /* If the result is passed by invisible reference the default
     544              :            def is initialized, otherwise it's uninitialized.  Still
     545              :            undefined is varying.  */
     546        61035 :         newinfo->visited = true;
     547        61035 :         newinfo->valnum = name;
     548        61035 :         break;
     549              : 
     550            0 :       default:
     551            0 :         gcc_unreachable ();
     552              :       }
     553              :   return newinfo;
     554              : }
     555              : 
     556              : /* Return the SSA value of X.  */
     557              : 
     558              : inline tree
     559   3531635465 : SSA_VAL (tree x, bool *visited = NULL)
     560              : {
     561   3531635465 :   vn_ssa_aux_t tem = vn_ssa_aux_hash->find_with_hash (x, SSA_NAME_VERSION (x));
     562   3531635465 :   if (visited)
     563   1437879480 :     *visited = tem && tem->visited;
     564   3531635465 :   return tem && tem->visited ? tem->valnum : x;
     565              : }
     566              : 
     567              : /* Return the SSA value of the VUSE x, supporting released VDEFs
     568              :    during elimination which will value-number the VDEF to the
     569              :    associated VUSE (but not substitute in the whole lattice).  */
     570              : 
     571              : static inline tree
     572   1300938243 : vuse_ssa_val (tree x)
     573              : {
     574   1300938243 :   if (!x)
     575              :     return NULL_TREE;
     576              : 
     577   1297478671 :   do
     578              :     {
     579   1297478671 :       x = SSA_VAL (x);
     580   1297478671 :       gcc_assert (x != VN_TOP);
     581              :     }
     582   1297478671 :   while (SSA_NAME_IN_FREE_LIST (x));
     583              : 
     584              :   return x;
     585              : }
     586              : 
     587              : /* Similar to the above but used as callback for walk_non_aliased_vuses
     588              :    and thus should stop at unvisited VUSE to not walk across region
     589              :    boundaries.  */
     590              : 
     591              : static tree
     592   1099103122 : vuse_valueize (tree vuse)
     593              : {
     594   1099103130 :   do
     595              :     {
     596   1099103130 :       bool visited;
     597   1099103130 :       vuse = SSA_VAL (vuse, &visited);
     598   1099103130 :       if (!visited)
     599     16565377 :         return NULL_TREE;
     600   1082537753 :       gcc_assert (vuse != VN_TOP);
     601              :     }
     602   1082537753 :   while (SSA_NAME_IN_FREE_LIST (vuse));
     603              :   return vuse;
     604              : }
     605              : 
     606              : 
     607              : /* Return the vn_kind the expression computed by the stmt should be
     608              :    associated with.  */
     609              : 
     610              : enum vn_kind
     611    105481233 : vn_get_stmt_kind (gimple *stmt)
     612              : {
     613    105481233 :   switch (gimple_code (stmt))
     614              :     {
     615              :     case GIMPLE_CALL:
     616              :       return VN_REFERENCE;
     617              :     case GIMPLE_PHI:
     618              :       return VN_PHI;
     619    105481233 :     case GIMPLE_ASSIGN:
     620    105481233 :       {
     621    105481233 :         enum tree_code code = gimple_assign_rhs_code (stmt);
     622    105481233 :         tree rhs1 = gimple_assign_rhs1 (stmt);
     623    105481233 :         switch (get_gimple_rhs_class (code))
     624              :           {
     625              :           case GIMPLE_UNARY_RHS:
     626              :           case GIMPLE_BINARY_RHS:
     627              :           case GIMPLE_TERNARY_RHS:
     628              :             return VN_NARY;
     629     49445507 :           case GIMPLE_SINGLE_RHS:
     630     49445507 :             switch (TREE_CODE_CLASS (code))
     631              :               {
     632     37286087 :               case tcc_reference:
     633              :                 /* VOP-less references can go through unary case.  */
     634     37286087 :                 if ((code == REALPART_EXPR
     635              :                      || code == IMAGPART_EXPR
     636     37286087 :                      || code == VIEW_CONVERT_EXPR
     637     37286087 :                      || code == BIT_FIELD_REF)
     638     37286087 :                     && (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME
     639       681735 :                         || is_gimple_min_invariant (TREE_OPERAND (rhs1, 0))))
     640              :                   return VN_NARY;
     641              : 
     642              :                 /* Fallthrough.  */
     643              :               case tcc_declaration:
     644              :                 return VN_REFERENCE;
     645              : 
     646              :               case tcc_constant:
     647              :                 return VN_CONSTANT;
     648              : 
     649      6081099 :               default:
     650      6081099 :                 if (code == ADDR_EXPR)
     651      3288711 :                   return (is_gimple_min_invariant (rhs1)
     652      3288711 :                           ? VN_CONSTANT : VN_REFERENCE);
     653      2792388 :                 else if (code == CONSTRUCTOR)
     654              :                   return VN_NARY;
     655              :                 return VN_NONE;
     656              :               }
     657              :           default:
     658              :             return VN_NONE;
     659              :           }
     660              :       }
     661              :     default:
     662              :       return VN_NONE;
     663              :     }
     664              : }
     665              : 
     666              : /* Lookup a value id for CONSTANT and return it.  If it does not
     667              :    exist returns 0.  */
     668              : 
     669              : unsigned int
     670            0 : get_constant_value_id (tree constant)
     671              : {
     672            0 :   vn_constant_s **slot;
     673            0 :   struct vn_constant_s vc;
     674              : 
     675            0 :   vc.hashcode = vn_hash_constant_with_type (constant);
     676            0 :   vc.constant = constant;
     677            0 :   slot = constant_to_value_id->find_slot (&vc, NO_INSERT);
     678            0 :   if (slot)
     679            0 :     return (*slot)->value_id;
     680              :   return 0;
     681              : }
     682              : 
     683              : /* Lookup a value id for CONSTANT, and if it does not exist, create a
     684              :    new one and return it.  If it does exist, return it.  */
     685              : 
     686              : unsigned int
     687     29462202 : get_or_alloc_constant_value_id (tree constant)
     688              : {
     689     29462202 :   vn_constant_s **slot;
     690     29462202 :   struct vn_constant_s vc;
     691     29462202 :   vn_constant_t vcp;
     692              : 
     693              :   /* If the hashtable isn't initialized we're not running from PRE and thus
     694              :      do not need value-ids.  */
     695     29462202 :   if (!constant_to_value_id)
     696              :     return 0;
     697              : 
     698      4841369 :   vc.hashcode = vn_hash_constant_with_type (constant);
     699      4841369 :   vc.constant = constant;
     700      4841369 :   slot = constant_to_value_id->find_slot (&vc, INSERT);
     701      4841369 :   if (*slot)
     702      2248797 :     return (*slot)->value_id;
     703              : 
     704      2592572 :   vcp = XNEW (struct vn_constant_s);
     705      2592572 :   vcp->hashcode = vc.hashcode;
     706      2592572 :   vcp->constant = constant;
     707      2592572 :   vcp->value_id = get_next_constant_value_id ();
     708      2592572 :   *slot = vcp;
     709      2592572 :   return vcp->value_id;
     710              : }
     711              : 
     712              : /* Compute the hash for a reference operand VRO1.  */
     713              : 
     714              : static void
     715    139842087 : vn_reference_op_compute_hash (const vn_reference_op_t vro1, inchash::hash &hstate)
     716              : {
     717    139842087 :   hstate.add_int (vro1->opcode);
     718    139842087 :   if (vro1->opcode == CALL_EXPR && !vro1->op0)
     719       561145 :     hstate.add_int (vro1->clique);
     720    139842087 :   if (vro1->op0)
     721    133328878 :     inchash::add_expr (vro1->op0, hstate);
     722    139842087 :   if (vro1->op1)
     723     12324435 :     inchash::add_expr (vro1->op1, hstate);
     724    139842087 :   if (vro1->op2)
     725     14098269 :     inchash::add_expr (vro1->op2, hstate);
     726    139842087 : }
     727              : 
     728              : /* Compute a hash for the reference operation VR1 and return it.  */
     729              : 
     730              : hashval_t
     731    208540369 : vn_reference_compute_hash (const vn_reference_t vr1)
     732              : {
     733    208540369 :   inchash::hash hstate;
     734    208540369 :   hashval_t result;
     735    208540369 :   int i;
     736    208540369 :   vn_reference_op_t vro;
     737    208540369 :   poly_offset_int off = -1;
     738    208540369 :   bool deref = false;
     739              : 
     740    849080508 :   FOR_EACH_VEC_ELT (vr1->operands, i, vro)
     741              :     {
     742    640540139 :       if (vro->opcode == MEM_REF)
     743              :         deref = true;
     744    442818815 :       else if (vro->opcode != ADDR_EXPR)
     745    310662680 :         deref = false;
     746    640540139 :       if (maybe_ne (vro->off, -1))
     747              :         {
     748    376910294 :           if (known_eq (off, -1))
     749    200128524 :             off = 0;
     750    640540139 :           off += vro->off;
     751              :         }
     752              :       else
     753              :         {
     754    263629845 :           if (maybe_ne (off, -1)
     755    263629845 :               && maybe_ne (off, 0))
     756    106219792 :             hstate.add_poly_hwi (off.force_shwi ());
     757    263629845 :           off = -1;
     758    263629845 :           if (deref
     759    123999531 :               && vro->opcode == ADDR_EXPR)
     760              :             {
     761    123787758 :               if (vro->op0)
     762              :                 {
     763    123787758 :                   tree op = TREE_OPERAND (vro->op0, 0);
     764    123787758 :                   hstate.add_int (TREE_CODE (op));
     765    123787758 :                   inchash::add_expr (op, hstate);
     766              :                 }
     767              :             }
     768              :           else
     769    139842087 :             vn_reference_op_compute_hash (vro, hstate);
     770              :         }
     771              :     }
     772              :   /* Do not hash vr1->offset or vr1->max_size, we want to get collisions
     773              :      to be able to identify compatible results.  */
     774    208540369 :   result = hstate.end ();
     775              :   /* ??? We would ICE later if we hash instead of adding that in. */
     776    208540369 :   if (vr1->vuse)
     777    203507517 :     result += SSA_NAME_VERSION (vr1->vuse);
     778              : 
     779    208540369 :   return result;
     780              : }
     781              : 
     782              : /* Return true if reference operations VR1 and VR2 are equivalent.  This
     783              :    means they have the same set of operands and vuses.  If LEXICAL
     784              :    is true then the full access path has to be the same.  */
     785              : 
     786              : bool
     787   4505511327 : vn_reference_eq (const_vn_reference_t const vr1, const_vn_reference_t const vr2,
     788              :                  bool lexical)
     789              : {
     790   4505511327 :   unsigned i, j;
     791              : 
     792              :   /* Early out if this is not a hash collision.  */
     793   4505511327 :   if (vr1->hashcode != vr2->hashcode)
     794              :     return false;
     795              : 
     796              :   /* The VOP needs to be the same.  */
     797     18253580 :   if (vr1->vuse != vr2->vuse)
     798              :     return false;
     799              : 
     800              :   /* The offset/max_size used for the ao_ref during lookup has to be
     801              :      the same.  */
     802     18253147 :   if (maybe_ne (vr1->offset, vr2->offset)
     803     18253147 :       || maybe_ne (vr1->max_size, vr2->max_size))
     804              :     {
     805              :       /* But nothing known in the prevailing entry is OK to be used.  */
     806      7109495 :       if (maybe_ne (vr1->offset, 0) || known_size_p (vr1->max_size))
     807              :         return false;
     808              :     }
     809              : 
     810              :   /* If the operands are the same we are done.  */
     811     36396910 :   if (vr1->operands == vr2->operands)
     812              :     return true;
     813              : 
     814     18198455 :   if (!vr1->type || !vr2->type)
     815              :     {
     816       555470 :       if (vr1->type != vr2->type)
     817              :         return false;
     818              :     }
     819     17642985 :   else if (vr1->type == vr2->type)
     820              :     ;
     821      2243292 :   else if (COMPLETE_TYPE_P (vr1->type) != COMPLETE_TYPE_P (vr2->type)
     822      2243292 :            || (COMPLETE_TYPE_P (vr1->type)
     823      2243292 :                && !expressions_equal_p (TYPE_SIZE (vr1->type),
     824      2243292 :                                         TYPE_SIZE (vr2->type))))
     825              :     return false;
     826      1494347 :   else if (vr1->operands[0].opcode == CALL_EXPR
     827      1494347 :            && !types_compatible_p (vr1->type, vr2->type))
     828              :     return false;
     829      1494347 :   else if (INTEGRAL_TYPE_P (vr1->type)
     830       594992 :            && INTEGRAL_TYPE_P (vr2->type))
     831              :     {
     832       554858 :       if (TYPE_PRECISION (vr1->type) != TYPE_PRECISION (vr2->type))
     833              :         return false;
     834              :     }
     835       939489 :   else if (INTEGRAL_TYPE_P (vr1->type)
     836       939489 :            && (TYPE_PRECISION (vr1->type)
     837        40134 :                != TREE_INT_CST_LOW (TYPE_SIZE (vr1->type))))
     838              :     return false;
     839       939443 :   else if (INTEGRAL_TYPE_P (vr2->type)
     840       939443 :            && (TYPE_PRECISION (vr2->type)
     841         9033 :                != TREE_INT_CST_LOW (TYPE_SIZE (vr2->type))))
     842              :     return false;
     843        20016 :   else if (VECTOR_BOOLEAN_TYPE_P (vr1->type)
     844       938848 :            && VECTOR_BOOLEAN_TYPE_P (vr2->type))
     845              :     {
     846              :       /* Vector boolean types can have padding, verify we are dealing with
     847              :          the same number of elements, aka the precision of the types.
     848              :          For example, In most architecture the precision_size of vbool*_t
     849              :          types are calculated like below:
     850              :          precision_size = type_size * 8
     851              : 
     852              :          Unfortunately, the RISC-V will adjust the precision_size for the
     853              :          vbool*_t in order to align the ISA as below:
     854              :          type_size      = [1, 1, 1, 1,  2,  4,  8]
     855              :          precision_size = [1, 2, 4, 8, 16, 32, 64]
     856              : 
     857              :          Then the precision_size of RISC-V vbool*_t will not be the multiple
     858              :          of the type_size.  We take care of this case consolidated here.  */
     859            0 :       if (maybe_ne (TYPE_VECTOR_SUBPARTS (vr1->type),
     860            0 :                     TYPE_VECTOR_SUBPARTS (vr2->type)))
     861              :         return false;
     862              :     }
     863       938848 :   else if (TYPE_MODE (vr1->type) != TYPE_MODE (vr2->type)
     864       938848 :            && (!mode_can_transfer_bits (TYPE_MODE (vr1->type))
     865        45247 :                || !mode_can_transfer_bits (TYPE_MODE (vr2->type))))
     866              :     return false;
     867              : 
     868     17332031 :   i = 0;
     869     17332031 :   j = 0;
     870     22397726 :   do
     871              :     {
     872     22397726 :       poly_offset_int off1 = 0, off2 = 0;
     873     22397726 :       vn_reference_op_t vro1, vro2;
     874     22397726 :       vn_reference_op_s tem1, tem2;
     875     22397726 :       bool deref1 = false, deref2 = false;
     876     22397726 :       bool reverse1 = false, reverse2 = false;
     877     72998081 :       for (; vr1->operands.iterate (i, &vro1); i++)
     878              :         {
     879     50600355 :           if (vro1->opcode == MEM_REF)
     880              :             deref1 = true;
     881              :           /* Do not look through a storage order barrier.  */
     882     34623407 :           else if (vro1->opcode == VIEW_CONVERT_EXPR && vro1->reverse)
     883        75470 :             return false;
     884     50600355 :           reverse1 |= vro1->reverse;
     885     50600355 :           if (lexical || known_eq (vro1->off, -1))
     886              :             break;
     887     28202629 :           off1 += vro1->off;
     888              :         }
     889     50794333 :       for (; vr2->operands.iterate (j, &vro2); j++)
     890              :         {
     891     50794333 :           if (vro2->opcode == MEM_REF)
     892              :             deref2 = true;
     893              :           /* Do not look through a storage order barrier.  */
     894     34792607 :           else if (vro2->opcode == VIEW_CONVERT_EXPR && vro2->reverse)
     895              :             return false;
     896     50794333 :           reverse2 |= vro2->reverse;
     897     50794333 :           if (lexical || known_eq (vro2->off, -1))
     898              :             break;
     899     28396607 :           off2 += vro2->off;
     900              :         }
     901     22397726 :       if (maybe_ne (off1, off2) || reverse1 != reverse2)
     902              :         return false;
     903     22397607 :       if (deref1 && vro1->opcode == ADDR_EXPR)
     904              :         {
     905      8460993 :           memset (&tem1, 0, sizeof (tem1));
     906      8460993 :           tem1.op0 = TREE_OPERAND (vro1->op0, 0);
     907      8460993 :           tem1.type = TREE_TYPE (tem1.op0);
     908      8460993 :           tem1.opcode = TREE_CODE (tem1.op0);
     909      8460993 :           vro1 = &tem1;
     910      8460993 :           deref1 = false;
     911              :         }
     912     22397607 :       if (deref2 && vro2->opcode == ADDR_EXPR)
     913              :         {
     914      8460999 :           memset (&tem2, 0, sizeof (tem2));
     915      8460999 :           tem2.op0 = TREE_OPERAND (vro2->op0, 0);
     916      8460999 :           tem2.type = TREE_TYPE (tem2.op0);
     917      8460999 :           tem2.opcode = TREE_CODE (tem2.op0);
     918      8460999 :           vro2 = &tem2;
     919      8460999 :           deref2 = false;
     920              :         }
     921     22397607 :       if (deref1 != deref2)
     922              :         return false;
     923     22338197 :       if (!vn_reference_op_eq (vro1, vro2))
     924              :         return false;
     925              :       /* Both alignment and alias set are not relevant for the produced
     926              :          value but need to be included when doing lexical comparison.
     927              :          We also need to make sure that the access path ends in an
     928              :          access of the same size as otherwise we might assume an access
     929              :          may not trap while in fact it might.  */
     930     22325518 :       if (lexical
     931      2261534 :           && (vro1->opcode == MEM_REF
     932      2261534 :               || vro1->opcode == TARGET_MEM_REF)
     933     23066364 :           && (TYPE_ALIGN (vro1->type) != TYPE_ALIGN (vro2->type)
     934       740639 :               || (TYPE_SIZE (vro1->type) != TYPE_SIZE (vro2->type)
     935            6 :                   && (! TYPE_SIZE (vro1->type)
     936            6 :                       || ! TYPE_SIZE (vro2->type)
     937            6 :                       || ! operand_equal_p (TYPE_SIZE (vro1->type),
     938            6 :                                             TYPE_SIZE (vro2->type))))
     939      2221899 :               || (get_deref_alias_set (vro1->opcode == MEM_REF
     940       740633 :                                        ? TREE_TYPE (vro1->op0)
     941            0 :                                        : TREE_TYPE (vro1->op2))
     942      1481266 :                   != get_deref_alias_set (vro2->opcode == MEM_REF
     943       740633 :                                           ? TREE_TYPE (vro2->op0)
     944            0 :                                           : TREE_TYPE (vro2->op2)))))
     945              :         return false;
     946     22322256 :       ++j;
     947     22322256 :       ++i;
     948              :     }
     949     44644512 :   while (vr1->operands.length () != i
     950     66966768 :          || vr2->operands.length () != j);
     951              : 
     952              :   return true;
     953              : }
     954              : 
     955              : /* Copy the operations present in load/store REF into RESULT, a vector of
     956              :    vn_reference_op_s's.  */
     957              : 
     958              : void
     959    227686917 : copy_reference_ops_from_ref (tree ref, vec<vn_reference_op_s> *result)
     960              : {
     961              :   /* For non-calls, store the information that makes up the address.  */
     962    227686917 :   tree orig = ref;
     963    791231317 :   while (ref)
     964              :     {
     965    563544400 :       vn_reference_op_s temp;
     966              : 
     967    563544400 :       memset (&temp, 0, sizeof (temp));
     968    563544400 :       temp.type = TREE_TYPE (ref);
     969    563544400 :       temp.opcode = TREE_CODE (ref);
     970    563544400 :       temp.off = -1;
     971              : 
     972    563544400 :       switch (temp.opcode)
     973              :         {
     974     15290787 :         case MODIFY_EXPR:
     975     15290787 :           temp.op0 = TREE_OPERAND (ref, 1);
     976     15290787 :           break;
     977          137 :         case WITH_SIZE_EXPR:
     978          137 :           temp.op0 = TREE_OPERAND (ref, 1);
     979          137 :           temp.off = 0;
     980          137 :           break;
     981    120115908 :         case MEM_REF:
     982              :           /* The base address gets its own vn_reference_op_s structure.  */
     983    120115908 :           temp.op0 = TREE_OPERAND (ref, 1);
     984    120115908 :           if (!mem_ref_offset (ref).to_shwi (&temp.off))
     985            0 :             temp.off = -1;
     986    120115908 :           temp.clique = MR_DEPENDENCE_CLIQUE (ref);
     987    120115908 :           temp.base = MR_DEPENDENCE_BASE (ref);
     988    120115908 :           temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
     989    120115908 :           break;
     990      2529076 :         case TARGET_MEM_REF:
     991              :           /* The base address gets its own vn_reference_op_s structure.  */
     992      2529076 :           temp.op0 = TMR_INDEX (ref);
     993      2529076 :           temp.op1 = TMR_STEP (ref);
     994      2529076 :           temp.op2 = TMR_OFFSET (ref);
     995      2529076 :           temp.clique = MR_DEPENDENCE_CLIQUE (ref);
     996      2529076 :           temp.base = MR_DEPENDENCE_BASE (ref);
     997      2529076 :           result->safe_push (temp);
     998      2529076 :           memset (&temp, 0, sizeof (temp));
     999      2529076 :           temp.type = NULL_TREE;
    1000      2529076 :           temp.opcode = ERROR_MARK;
    1001      2529076 :           temp.op0 = TMR_INDEX2 (ref);
    1002      2529076 :           temp.off = -1;
    1003      2529076 :           break;
    1004       818993 :         case BIT_FIELD_REF:
    1005              :           /* Record bits, position and storage order.  */
    1006       818993 :           temp.op0 = TREE_OPERAND (ref, 1);
    1007       818993 :           temp.op1 = TREE_OPERAND (ref, 2);
    1008      1637276 :           if (!multiple_p (bit_field_offset (ref), BITS_PER_UNIT, &temp.off))
    1009          710 :             temp.off = -1;
    1010       818993 :           temp.reverse = REF_REVERSE_STORAGE_ORDER (ref);
    1011       818993 :           break;
    1012    150927999 :         case COMPONENT_REF:
    1013    150927999 :           temp.op0 = TREE_OPERAND (ref, 1);
    1014    150927999 :           temp.op1 = TREE_OPERAND (ref, 2);
    1015    301855415 :           temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
    1016    301853301 :                           && TYPE_REVERSE_STORAGE_ORDER
    1017              :                                (TREE_TYPE (TREE_OPERAND (ref, 0))));
    1018    150927999 :           {
    1019    150927999 :             tree this_offset = component_ref_field_offset (ref);
    1020    150927999 :             if (this_offset
    1021    150927999 :                 && poly_int_tree_p (this_offset))
    1022              :               {
    1023    150925869 :                 tree bit_offset = DECL_FIELD_BIT_OFFSET (TREE_OPERAND (ref, 1));
    1024    150925869 :                 if (TREE_INT_CST_LOW (bit_offset) % BITS_PER_UNIT == 0)
    1025              :                   {
    1026    150433403 :                     poly_offset_int off
    1027    150433403 :                       = (wi::to_poly_offset (this_offset)
    1028    150433403 :                          + (wi::to_offset (bit_offset) >> LOG2_BITS_PER_UNIT));
    1029              :                     /* Prohibit value-numbering zero offset components
    1030              :                        of addresses the same before the pass folding
    1031              :                        __builtin_object_size had a chance to run.  Likewise
    1032              :                        for components of zero size at arbitrary offset.  */
    1033    150433403 :                     if (TREE_CODE (orig) != ADDR_EXPR
    1034      4943784 :                         || (TYPE_SIZE (temp.type)
    1035      4930286 :                             && integer_nonzerop (TYPE_SIZE (temp.type))
    1036    153494082 :                             && maybe_ne (off, 0))
    1037    153509746 :                         || (cfun->curr_properties & PROP_objsz))
    1038    148955317 :                       off.to_shwi (&temp.off);
    1039              :                   }
    1040              :               }
    1041              :           }
    1042              :           break;
    1043     39462713 :         case ARRAY_RANGE_REF:
    1044     39462713 :         case ARRAY_REF:
    1045     39462713 :           {
    1046     39462713 :             tree eltype = TREE_TYPE (TREE_TYPE (TREE_OPERAND (ref, 0)));
    1047              :             /* Record index as operand.  */
    1048     39462713 :             temp.op0 = TREE_OPERAND (ref, 1);
    1049              :             /* Always record lower bounds and element size.  */
    1050     39462713 :             temp.op1 = array_ref_low_bound (ref);
    1051              :             /* But record element size in units of the type alignment.  */
    1052     39462713 :             temp.op2 = TREE_OPERAND (ref, 3);
    1053     39462713 :             temp.align = eltype->type_common.align;
    1054     39462713 :             if (! temp.op2)
    1055     39252057 :               temp.op2 = size_binop (EXACT_DIV_EXPR, TYPE_SIZE_UNIT (eltype),
    1056              :                                      size_int (TYPE_ALIGN_UNIT (eltype)));
    1057              :             /* Prohibit value-numbering addresses of one-after-the-last
    1058              :                element ARRAY_REFs the same as addresses of other components
    1059              :                before the pass folding __builtin_object_size had a chance
    1060              :                to run.  */
    1061     39462713 :             bool avoid_oob = true;
    1062     39462713 :             if (TREE_CODE (orig) != ADDR_EXPR
    1063       487444 :                 || cfun->curr_properties & PROP_objsz)
    1064              :               avoid_oob = false;
    1065       230219 :             else if (poly_int_tree_p (temp.op0))
    1066              :               {
    1067        79144 :                 tree ub = array_ref_up_bound (ref);
    1068        79144 :                 if (ub
    1069        77431 :                     && poly_int_tree_p (ub)
    1070              :                     /* ???  The C frontend for T[0] uses [0:] and the
    1071              :                        C++ frontend [0:-1U].  See layout_type for how
    1072              :                        awkward this is.  */
    1073        68560 :                     && !integer_minus_onep (ub)
    1074       156575 :                     && known_le (wi::to_poly_offset (temp.op0),
    1075              :                                  wi::to_poly_offset (ub)))
    1076        67723 :                   avoid_oob = false;
    1077              :               }
    1078     39462713 :             if (poly_int_tree_p (temp.op0)
    1079     22621315 :                 && poly_int_tree_p (temp.op1)
    1080     22621291 :                 && TREE_CODE (temp.op2) == INTEGER_CST
    1081     62022738 :                 && !avoid_oob)
    1082              :               {
    1083     45099734 :                 poly_offset_int off = ((wi::to_poly_offset (temp.op0)
    1084     67649601 :                                         - wi::to_poly_offset (temp.op1))
    1085     45099734 :                                        * wi::to_offset (temp.op2)
    1086     22549867 :                                        * vn_ref_op_align_unit (&temp));
    1087     22549867 :                 off.to_shwi (&temp.off);
    1088              :               }
    1089     39462713 :             temp.reverse = (AGGREGATE_TYPE_P (TREE_TYPE (TREE_OPERAND (ref, 0)))
    1090     39462713 :                             && TYPE_REVERSE_STORAGE_ORDER
    1091              :                                  (TREE_TYPE (TREE_OPERAND (ref, 0))));
    1092              :           }
    1093     39462713 :           break;
    1094     84000857 :         case VAR_DECL:
    1095     84000857 :           if (DECL_HARD_REGISTER (ref))
    1096              :             {
    1097        20739 :               temp.op0 = ref;
    1098        20739 :               break;
    1099              :             }
    1100              :           /* Fallthru.  */
    1101     87423347 :         case PARM_DECL:
    1102     87423347 :         case CONST_DECL:
    1103     87423347 :         case RESULT_DECL:
    1104              :           /* Canonicalize decls to MEM[&decl] which is what we end up with
    1105              :              when valueizing MEM[ptr] with ptr = &decl.  */
    1106     87423347 :           temp.opcode = MEM_REF;
    1107     87423347 :           temp.op0 = build_int_cst (build_pointer_type (TREE_TYPE (ref)), 0);
    1108     87423347 :           temp.off = 0;
    1109     87423347 :           result->safe_push (temp);
    1110     87423347 :           temp.opcode = ADDR_EXPR;
    1111     87423347 :           temp.op0 = build1 (ADDR_EXPR, TREE_TYPE (temp.op0), ref);
    1112     87423347 :           temp.type = TREE_TYPE (temp.op0);
    1113     87423347 :           temp.off = -1;
    1114     87423347 :           break;
    1115     98306626 :         case STRING_CST:
    1116     98306626 :         case INTEGER_CST:
    1117     98306626 :         case POLY_INT_CST:
    1118     98306626 :         case COMPLEX_CST:
    1119     98306626 :         case VECTOR_CST:
    1120     98306626 :         case REAL_CST:
    1121     98306626 :         case FIXED_CST:
    1122     98306626 :         case CONSTRUCTOR:
    1123     98306626 :         case SSA_NAME:
    1124     98306626 :           temp.op0 = ref;
    1125     98306626 :           break;
    1126     46185119 :         case ADDR_EXPR:
    1127     46185119 :           if (is_gimple_min_invariant (ref))
    1128              :             {
    1129     41936205 :               temp.op0 = ref;
    1130     41936205 :               break;
    1131              :             }
    1132              :           break;
    1133              :           /* These are only interesting for their operands, their
    1134              :              existence, and their type.  They will never be the last
    1135              :              ref in the chain of references (IE they require an
    1136              :              operand), so we don't have to put anything
    1137              :              for op* as it will be handled by the iteration  */
    1138       496564 :         case REALPART_EXPR:
    1139       496564 :           temp.off = 0;
    1140       496564 :           break;
    1141      1463372 :         case VIEW_CONVERT_EXPR:
    1142      1463372 :           temp.off = 0;
    1143      1463372 :           temp.reverse = storage_order_barrier_p (ref);
    1144      1463372 :           break;
    1145       503020 :         case IMAGPART_EXPR:
    1146              :           /* This is only interesting for its constant offset.  */
    1147       503020 :           temp.off = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (ref)));
    1148       503020 :           break;
    1149            0 :         default:
    1150            0 :           gcc_unreachable ();
    1151              :         }
    1152    563544400 :       result->safe_push (temp);
    1153              : 
    1154    563544400 :       if (REFERENCE_CLASS_P (ref)
    1155    247226755 :           || TREE_CODE (ref) == MODIFY_EXPR
    1156    231935968 :           || TREE_CODE (ref) == WITH_SIZE_EXPR
    1157    795480231 :           || (TREE_CODE (ref) == ADDR_EXPR
    1158     46185119 :               && !is_gimple_min_invariant (ref)))
    1159    335857483 :         ref = TREE_OPERAND (ref, 0);
    1160              :       else
    1161              :         ref = NULL_TREE;
    1162              :     }
    1163    227686917 : }
    1164              : 
    1165              : /* Build a alias-oracle reference abstraction in *REF from the vn_reference
    1166              :    operands in *OPS, the reference alias set SET and the reference type TYPE.
    1167              :    Return true if something useful was produced.  */
    1168              : 
    1169              : bool
    1170     14965025 : ao_ref_init_from_vn_reference (ao_ref *ref,
    1171              :                                alias_set_type set, alias_set_type base_set,
    1172              :                                tree type, const vec<vn_reference_op_s> &ops)
    1173              : {
    1174     14965025 :   unsigned i;
    1175     14965025 :   tree base = NULL_TREE;
    1176     14965025 :   tree *op0_p = &base;
    1177     14965025 :   poly_offset_int offset = 0;
    1178     14965025 :   poly_offset_int max_size;
    1179     14965025 :   poly_offset_int size = -1;
    1180     14965025 :   tree size_tree = NULL_TREE;
    1181              : 
    1182              :   /* We don't handle calls.  */
    1183     14965025 :   if (!type)
    1184              :     return false;
    1185              : 
    1186     14965025 :   machine_mode mode = TYPE_MODE (type);
    1187     14965025 :   if (mode == BLKmode)
    1188        42963 :     size_tree = TYPE_SIZE (type);
    1189              :   else
    1190     29844124 :     size = GET_MODE_BITSIZE (mode);
    1191     14922062 :   if (size_tree != NULL_TREE
    1192        42963 :       && poly_int_tree_p (size_tree))
    1193        42963 :     size = wi::to_poly_offset (size_tree);
    1194              : 
    1195              :   /* Lower the final access size from the outermost expression.  */
    1196     14965025 :   const_vn_reference_op_t cst_op = &ops[0];
    1197              :   /* Cast away constness for the sake of the const-unsafe
    1198              :      FOR_EACH_VEC_ELT().  */
    1199     14965025 :   vn_reference_op_t op = const_cast<vn_reference_op_t>(cst_op);
    1200     14965025 :   size_tree = NULL_TREE;
    1201     14965025 :   if (op->opcode == COMPONENT_REF)
    1202      5131932 :     size_tree = DECL_SIZE (op->op0);
    1203      9833093 :   else if (op->opcode == BIT_FIELD_REF)
    1204        74875 :     size_tree = op->op0;
    1205      5206807 :   if (size_tree != NULL_TREE
    1206      5206807 :       && poly_int_tree_p (size_tree)
    1207     10413614 :       && (!known_size_p (size)
    1208     14965025 :           || known_lt (wi::to_poly_offset (size_tree), size)))
    1209        47344 :     size = wi::to_poly_offset (size_tree);
    1210              : 
    1211              :   /* Initially, maxsize is the same as the accessed element size.
    1212              :      In the following it will only grow (or become -1).  */
    1213     14965025 :   max_size = size;
    1214              : 
    1215              :   /* Compute cumulative bit-offset for nested component-refs and array-refs,
    1216              :      and find the ultimate containing object.  */
    1217     57347326 :   FOR_EACH_VEC_ELT (ops, i, op)
    1218              :     {
    1219     42538013 :       switch (op->opcode)
    1220              :         {
    1221              :         case CALL_EXPR:
    1222              :           return false;
    1223              : 
    1224              :         /* Record the base objects.  */
    1225     14511117 :         case MEM_REF:
    1226     14511117 :           *op0_p = build2 (MEM_REF, op->type,
    1227              :                            NULL_TREE, op->op0);
    1228     14511117 :           MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
    1229     14511117 :           MR_DEPENDENCE_BASE (*op0_p) = op->base;
    1230     14511117 :           op0_p = &TREE_OPERAND (*op0_p, 0);
    1231     14511117 :           break;
    1232              : 
    1233       297638 :         case TARGET_MEM_REF:
    1234       892914 :           *op0_p = build5 (TARGET_MEM_REF, op->type,
    1235              :                            NULL_TREE, op->op2, op->op0,
    1236       297638 :                            op->op1, ops[i+1].op0);
    1237       297638 :           MR_DEPENDENCE_CLIQUE (*op0_p) = op->clique;
    1238       297638 :           MR_DEPENDENCE_BASE (*op0_p) = op->base;
    1239       297638 :           op0_p = &TREE_OPERAND (*op0_p, 0);
    1240       297638 :           ++i;
    1241       297638 :           break;
    1242              : 
    1243              :         /* Unwrap some of the wrapped decls.  */
    1244      6692588 :         case ADDR_EXPR:
    1245              :           /* Apart from ADDR_EXPR arguments to MEM_REF.  */
    1246      6692588 :           if (base != NULL_TREE
    1247      6692587 :               && TREE_CODE (base) == MEM_REF
    1248      6660920 :               && op->op0
    1249     13353508 :               && DECL_P (TREE_OPERAND (op->op0, 0)))
    1250              :             {
    1251      6653100 :               const_vn_reference_op_t pop = &ops[i-1];
    1252      6653100 :               base = TREE_OPERAND (op->op0, 0);
    1253      6653100 :               if (known_eq (pop->off, -1))
    1254              :                 {
    1255           27 :                   max_size = -1;
    1256           27 :                   offset = 0;
    1257              :                 }
    1258              :               else
    1259     19959219 :                 offset += poly_offset_int (pop->off) * BITS_PER_UNIT;
    1260              :               op0_p = NULL;
    1261              :               break;
    1262              :             }
    1263              :           /* Fallthru.  */
    1264      8156213 :         case PARM_DECL:
    1265      8156213 :         case CONST_DECL:
    1266      8156213 :         case RESULT_DECL:
    1267              :           /* ???  We shouldn't see these, but un-canonicalize what
    1268              :              copy_reference_ops_from_ref does when visiting MEM_REF.  */
    1269      8156213 :         case VAR_DECL:
    1270              :           /* ???  And for this only have DECL_HARD_REGISTER.  */
    1271      8156213 :         case STRING_CST:
    1272              :           /* This can show up in ARRAY_REF bases.  */
    1273      8156213 :         case INTEGER_CST:
    1274      8156213 :         case SSA_NAME:
    1275      8156213 :           *op0_p = op->op0;
    1276      8156213 :           op0_p = NULL;
    1277      8156213 :           break;
    1278              : 
    1279              :         /* And now the usual component-reference style ops.  */
    1280        74875 :         case BIT_FIELD_REF:
    1281        74875 :           offset += wi::to_poly_offset (op->op1);
    1282        74875 :           break;
    1283              : 
    1284      8346100 :         case COMPONENT_REF:
    1285      8346100 :           {
    1286      8346100 :             tree field = op->op0;
    1287              :             /* We do not have a complete COMPONENT_REF tree here so we
    1288              :                cannot use component_ref_field_offset.  Do the interesting
    1289              :                parts manually.  */
    1290      8346100 :             tree this_offset = DECL_FIELD_OFFSET (field);
    1291              : 
    1292      8346100 :             if (op->op1 || !poly_int_tree_p (this_offset))
    1293          233 :               max_size = -1;
    1294              :             else
    1295              :               {
    1296      8345867 :                 poly_offset_int woffset = (wi::to_poly_offset (this_offset)
    1297      8345867 :                                            << LOG2_BITS_PER_UNIT);
    1298      8345867 :                 woffset += wi::to_offset (DECL_FIELD_BIT_OFFSET (field));
    1299      8345867 :                 offset += woffset;
    1300              :               }
    1301              :             break;
    1302              :           }
    1303              : 
    1304      3144332 :         case ARRAY_RANGE_REF:
    1305      3144332 :         case ARRAY_REF:
    1306              :           /* Use the recorded constant offset.  */
    1307      3144332 :           if (maybe_eq (op->off, -1))
    1308      1218204 :             max_size = -1;
    1309              :           else
    1310      5778384 :             offset += poly_offset_int (op->off) * BITS_PER_UNIT;
    1311              :           break;
    1312              : 
    1313              :         case REALPART_EXPR:
    1314              :           break;
    1315              : 
    1316              :         case IMAGPART_EXPR:
    1317     42382301 :           offset += size;
    1318              :           break;
    1319              : 
    1320              :         case VIEW_CONVERT_EXPR:
    1321              :           break;
    1322              : 
    1323              :         case POLY_INT_CST:
    1324              :         case COMPLEX_CST:
    1325              :         case VECTOR_CST:
    1326              :         case REAL_CST:
    1327              :         case FIXED_CST:
    1328              :         case CONSTRUCTOR:
    1329              :           return false;
    1330              : 
    1331              :         default:
    1332              :           return false;
    1333              :         }
    1334              :     }
    1335              : 
    1336     14809313 :   if (base == NULL_TREE)
    1337              :     return false;
    1338              : 
    1339     14809313 :   ref->ref = NULL_TREE;
    1340     14809313 :   ref->base = base;
    1341     14809313 :   ref->ref_alias_set = set;
    1342     14809313 :   ref->base_alias_set = base_set;
    1343              :   /* We discount volatiles from value-numbering elsewhere.  */
    1344     14809313 :   ref->volatile_p = false;
    1345              : 
    1346     14809313 :   if (!size.to_shwi (&ref->size) || maybe_lt (ref->size, 0))
    1347              :     {
    1348            0 :       ref->offset = 0;
    1349            0 :       ref->size = -1;
    1350            0 :       ref->max_size = -1;
    1351            0 :       return true;
    1352              :     }
    1353              : 
    1354     14809313 :   if (!offset.to_shwi (&ref->offset))
    1355              :     {
    1356           26 :       ref->offset = 0;
    1357           26 :       ref->max_size = -1;
    1358           26 :       return true;
    1359              :     }
    1360              : 
    1361     14809287 :   if (!max_size.to_shwi (&ref->max_size) || maybe_lt (ref->max_size, 0))
    1362      1071776 :     ref->max_size = -1;
    1363              : 
    1364              :   return true;
    1365              : }
    1366              : 
    1367              : /* Copy the operations present in load/store/call REF into RESULT, a vector of
    1368              :    vn_reference_op_s's.  */
    1369              : 
    1370              : static void
    1371      9377348 : copy_reference_ops_from_call (gcall *call,
    1372              :                               vec<vn_reference_op_s> *result)
    1373              : {
    1374      9377348 :   vn_reference_op_s temp;
    1375      9377348 :   unsigned i;
    1376      9377348 :   tree lhs = gimple_call_lhs (call);
    1377      9377348 :   int lr;
    1378              : 
    1379              :   /* If 2 calls have a different non-ssa lhs, vdef value numbers should be
    1380              :      different.  By adding the lhs here in the vector, we ensure that the
    1381              :      hashcode is different, guaranteeing a different value number.  */
    1382      9377348 :   if (lhs && TREE_CODE (lhs) != SSA_NAME)
    1383              :     {
    1384       450223 :       memset (&temp, 0, sizeof (temp));
    1385       450223 :       temp.opcode = MODIFY_EXPR;
    1386       450223 :       temp.type = TREE_TYPE (lhs);
    1387       450223 :       temp.op0 = lhs;
    1388       450223 :       temp.off = -1;
    1389       450223 :       result->safe_push (temp);
    1390              :     }
    1391              : 
    1392              :   /* Copy the type, opcode, function, static chain and EH region, if any.  */
    1393      9377348 :   memset (&temp, 0, sizeof (temp));
    1394      9377348 :   temp.type = gimple_call_fntype (call);
    1395      9377348 :   temp.opcode = CALL_EXPR;
    1396      9377348 :   temp.op0 = gimple_call_fn (call);
    1397      9377348 :   if (gimple_call_internal_p (call))
    1398       545687 :     temp.clique = gimple_call_internal_fn (call);
    1399      9377348 :   temp.op1 = gimple_call_chain (call);
    1400      9377348 :   if (stmt_could_throw_p (cfun, call) && (lr = lookup_stmt_eh_lp (call)) > 0)
    1401       638719 :     temp.op2 = size_int (lr);
    1402      9377348 :   temp.off = -1;
    1403      9377348 :   result->safe_push (temp);
    1404              : 
    1405              :   /* Copy the call arguments.  As they can be references as well,
    1406              :      just chain them together.  */
    1407     37162542 :   for (i = 0; i < gimple_call_num_args (call); ++i)
    1408              :     {
    1409     18407846 :       tree callarg = gimple_call_arg (call, i);
    1410     18407846 :       copy_reference_ops_from_ref (callarg, result);
    1411              :     }
    1412      9377348 : }
    1413              : 
    1414              : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS.  Updates
    1415              :    *I_P to point to the last element of the replacement.  */
    1416              : static bool
    1417    130028956 : vn_reference_fold_indirect (vec<vn_reference_op_s> *ops,
    1418              :                             unsigned int *i_p)
    1419              : {
    1420    130028956 :   unsigned int i = *i_p;
    1421    130028956 :   vn_reference_op_t op = &(*ops)[i];
    1422    130028956 :   vn_reference_op_t mem_op = &(*ops)[i - 1];
    1423    130028956 :   tree addr_base;
    1424    130028956 :   poly_int64 addr_offset = 0;
    1425              : 
    1426              :   /* The only thing we have to do is from &OBJ.foo.bar add the offset
    1427              :      from .foo.bar to the preceding MEM_REF offset and replace the
    1428              :      address with &OBJ.  */
    1429    130028956 :   addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (op->op0, 0),
    1430              :                                                &addr_offset, vn_valueize);
    1431    130028956 :   gcc_checking_assert (addr_base && TREE_CODE (addr_base) != MEM_REF);
    1432    130028956 :   if (addr_base != TREE_OPERAND (op->op0, 0))
    1433              :     {
    1434       707580 :       poly_offset_int off
    1435       707580 :         = (poly_offset_int::from (wi::to_poly_wide (mem_op->op0),
    1436              :                                   SIGNED)
    1437       707580 :            + addr_offset);
    1438       707580 :       mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
    1439       707580 :       op->op0 = build_fold_addr_expr (addr_base);
    1440       707580 :       if (tree_fits_shwi_p (mem_op->op0))
    1441       707503 :         mem_op->off = tree_to_shwi (mem_op->op0);
    1442              :       else
    1443              :         mem_op->off = -1;
    1444       707580 :       return true;
    1445              :     }
    1446              :   return false;
    1447              : }
    1448              : 
    1449              : /* Fold *& at position *I_P in a vn_reference_op_s vector *OPS.  Updates
    1450              :    *I_P to point to the last element of the replacement.  */
    1451              : static bool
    1452     87579171 : vn_reference_maybe_forwprop_address (vec<vn_reference_op_s> *ops,
    1453              :                                      unsigned int *i_p)
    1454              : {
    1455     87579171 :   bool changed = false;
    1456     95387744 :   vn_reference_op_t op;
    1457              : 
    1458     95387744 :   do
    1459              :     {
    1460     95387744 :       unsigned int i = *i_p;
    1461     95387744 :       op = &(*ops)[i];
    1462     95387744 :       vn_reference_op_t mem_op = &(*ops)[i - 1];
    1463     95387744 :       gimple *def_stmt;
    1464     95387744 :       enum tree_code code;
    1465     95387744 :       poly_offset_int off;
    1466              : 
    1467     95387744 :       def_stmt = SSA_NAME_DEF_STMT (op->op0);
    1468     95387744 :       if (!is_gimple_assign (def_stmt))
    1469     87577220 :         return changed;
    1470              : 
    1471     38898662 :       code = gimple_assign_rhs_code (def_stmt);
    1472     38898662 :       if (code != ADDR_EXPR
    1473     38898662 :           && code != POINTER_PLUS_EXPR)
    1474              :         return changed;
    1475              : 
    1476     20771202 :       off = poly_offset_int::from (wi::to_poly_wide (mem_op->op0), SIGNED);
    1477              : 
    1478              :       /* The only thing we have to do is from &OBJ.foo.bar add the offset
    1479              :          from .foo.bar to the preceding MEM_REF offset and replace the
    1480              :          address with &OBJ.  */
    1481     20771202 :       if (code == ADDR_EXPR)
    1482              :         {
    1483       956063 :           tree addr, addr_base;
    1484       956063 :           poly_int64 addr_offset;
    1485              : 
    1486       956063 :           addr = gimple_assign_rhs1 (def_stmt);
    1487       956063 :           addr_base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (addr, 0),
    1488              :                                                        &addr_offset,
    1489              :                                                        vn_valueize);
    1490              :           /* If that didn't work because the address isn't invariant propagate
    1491              :              the reference tree from the address operation in case the current
    1492              :              dereference isn't offsetted.  */
    1493       956063 :           if (!addr_base
    1494       292246 :               && *i_p == ops->length () - 1
    1495       146123 :               && known_eq (off, 0)
    1496              :               /* This makes us disable this transform for PRE where the
    1497              :                  reference ops might be also used for code insertion which
    1498              :                  is invalid.  */
    1499      1043052 :               && default_vn_walk_kind == VN_WALKREWRITE)
    1500              :             {
    1501        86899 :               auto_vec<vn_reference_op_s, 32> tem;
    1502        86899 :               copy_reference_ops_from_ref (TREE_OPERAND (addr, 0), &tem);
    1503              :               /* Make sure to preserve TBAA info.  The only objects not
    1504              :                  wrapped in MEM_REFs that can have their address taken are
    1505              :                  STRING_CSTs.  */
    1506        86899 :               if (tem.length () >= 2
    1507        86899 :                   && tem[tem.length () - 2].opcode == MEM_REF)
    1508              :                 {
    1509        86884 :                   vn_reference_op_t new_mem_op = &tem[tem.length () - 2];
    1510        86884 :                   new_mem_op->op0
    1511        86884 :                       = wide_int_to_tree (TREE_TYPE (mem_op->op0),
    1512       173768 :                                           wi::to_poly_wide (new_mem_op->op0));
    1513              :                 }
    1514              :               /* Do not forward addresses of TARGET_MEM_REF.  */
    1515           15 :               else if (tem[0].opcode == TARGET_MEM_REF)
    1516              :                 return changed;
    1517              :               else
    1518           15 :                 gcc_assert (tem.last ().opcode == STRING_CST);
    1519        86899 :               ops->pop ();
    1520        86899 :               ops->pop ();
    1521        86899 :               ops->safe_splice (tem);
    1522        86899 :               --*i_p;
    1523        86899 :               return true;
    1524        86899 :             }
    1525       869164 :           if (!addr_base
    1526       809940 :               || TREE_CODE (addr_base) != MEM_REF
    1527      1677194 :               || (TREE_CODE (TREE_OPERAND (addr_base, 0)) == SSA_NAME
    1528       806169 :                   && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (addr_base,
    1529              :                                                                     0))))
    1530              :             return changed;
    1531              : 
    1532       808030 :           off += addr_offset;
    1533       808030 :           off += mem_ref_offset (addr_base);
    1534       808030 :           op->op0 = TREE_OPERAND (addr_base, 0);
    1535              :         }
    1536              :       else
    1537              :         {
    1538     19815139 :           tree ptr, ptroff;
    1539     19815139 :           ptr = gimple_assign_rhs1 (def_stmt);
    1540     19815139 :           ptroff = gimple_assign_rhs2 (def_stmt);
    1541     19815139 :           if (TREE_CODE (ptr) != SSA_NAME
    1542     18079352 :               || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ptr)
    1543              :               /* Make sure to not endlessly recurse.
    1544              :                  See gcc.dg/tree-ssa/20040408-1.c for an example.  Can easily
    1545              :                  happen when we value-number a PHI to its backedge value.  */
    1546     18077927 :               || SSA_VAL (ptr) == op->op0
    1547     37893066 :               || !poly_int_tree_p (ptroff))
    1548              :             return changed;
    1549              : 
    1550      7002494 :           off += wi::to_poly_offset (ptroff);
    1551      7002494 :           op->op0 = ptr;
    1552              :         }
    1553              : 
    1554      7810524 :       mem_op->op0 = wide_int_to_tree (TREE_TYPE (mem_op->op0), off);
    1555      7810524 :       if (tree_fits_shwi_p (mem_op->op0))
    1556      7527259 :         mem_op->off = tree_to_shwi (mem_op->op0);
    1557              :       else
    1558              :         mem_op->off = -1;
    1559              :       /* ???  Can end up with endless recursion here!?
    1560              :          gcc.c-torture/execute/strcmp-1.c  */
    1561      7810524 :       if (TREE_CODE (op->op0) == SSA_NAME)
    1562      7808663 :         op->op0 = SSA_VAL (op->op0);
    1563      7810524 :       if (TREE_CODE (op->op0) != SSA_NAME)
    1564         1951 :         op->opcode = TREE_CODE (op->op0);
    1565              : 
    1566      7810524 :       changed = true;
    1567              :     }
    1568              :   /* Tail-recurse.  */
    1569      7810524 :   while (TREE_CODE (op->op0) == SSA_NAME);
    1570              : 
    1571              :   /* Fold a remaining *&.  */
    1572         1951 :   if (TREE_CODE (op->op0) == ADDR_EXPR)
    1573          261 :     vn_reference_fold_indirect (ops, i_p);
    1574              : 
    1575              :   return changed;
    1576              : }
    1577              : 
    1578              : /* Optimize the reference REF to a constant if possible or return
    1579              :    NULL_TREE if not.  */
    1580              : 
    1581              : tree
    1582    112502921 : fully_constant_vn_reference_p (vn_reference_t ref)
    1583              : {
    1584    112502921 :   vec<vn_reference_op_s> operands = ref->operands;
    1585    112502921 :   vn_reference_op_t op;
    1586              : 
    1587              :   /* Try to simplify the translated expression if it is
    1588              :      a call to a builtin function with at most two arguments.  */
    1589    112502921 :   op = &operands[0];
    1590    112502921 :   if (op->opcode == CALL_EXPR
    1591        93101 :       && (!op->op0
    1592        85346 :           || (TREE_CODE (op->op0) == ADDR_EXPR
    1593        85346 :               && TREE_CODE (TREE_OPERAND (op->op0, 0)) == FUNCTION_DECL
    1594        85346 :               && fndecl_built_in_p (TREE_OPERAND (op->op0, 0),
    1595              :                                     BUILT_IN_NORMAL)))
    1596        75346 :       && operands.length () >= 2
    1597    112578235 :       && operands.length () <= 3)
    1598              :     {
    1599        38235 :       vn_reference_op_t arg0, arg1 = NULL;
    1600        38235 :       bool anyconst = false;
    1601        38235 :       arg0 = &operands[1];
    1602        38235 :       if (operands.length () > 2)
    1603         5742 :         arg1 = &operands[2];
    1604        38235 :       if (TREE_CODE_CLASS (arg0->opcode) == tcc_constant
    1605        38235 :           || (arg0->opcode == ADDR_EXPR
    1606        13838 :               && is_gimple_min_invariant (arg0->op0)))
    1607              :         anyconst = true;
    1608        38235 :       if (arg1
    1609        38235 :           && (TREE_CODE_CLASS (arg1->opcode) == tcc_constant
    1610         4233 :               || (arg1->opcode == ADDR_EXPR
    1611          587 :                   && is_gimple_min_invariant (arg1->op0))))
    1612              :         anyconst = true;
    1613        36139 :       if (anyconst)
    1614              :         {
    1615        26127 :           combined_fn fn;
    1616        26127 :           if (op->op0)
    1617        25164 :             fn = as_combined_fn (DECL_FUNCTION_CODE
    1618        25164 :                                         (TREE_OPERAND (op->op0, 0)));
    1619              :           else
    1620          963 :             fn = as_combined_fn ((internal_fn) op->clique);
    1621        26127 :           tree folded;
    1622        26127 :           if (arg1)
    1623         2740 :             folded = fold_const_call (fn, ref->type, arg0->op0, arg1->op0);
    1624              :           else
    1625        23387 :             folded = fold_const_call (fn, ref->type, arg0->op0);
    1626        26127 :           if (folded
    1627        26127 :               && is_gimple_min_invariant (folded))
    1628         1048 :             return folded;
    1629              :         }
    1630              :     }
    1631              : 
    1632              :   /* Simplify reads from constants or constant initializers.  */
    1633    112464686 :   else if (BITS_PER_UNIT == 8
    1634    112464686 :            && ref->type
    1635    112464686 :            && COMPLETE_TYPE_P (ref->type)
    1636    224929330 :            && is_gimple_reg_type (ref->type))
    1637              :     {
    1638    108033619 :       poly_int64 off = 0;
    1639    108033619 :       HOST_WIDE_INT size;
    1640    108033619 :       if (INTEGRAL_TYPE_P (ref->type))
    1641     54995126 :         size = TYPE_PRECISION (ref->type);
    1642     53038493 :       else if (tree_fits_shwi_p (TYPE_SIZE (ref->type)))
    1643     53038493 :         size = tree_to_shwi (TYPE_SIZE (ref->type));
    1644              :       else
    1645              :         return NULL_TREE;
    1646    108033619 :       if (size % BITS_PER_UNIT != 0
    1647    106166828 :           || size > MAX_BITSIZE_MODE_ANY_MODE)
    1648              :         return NULL_TREE;
    1649    106164932 :       size /= BITS_PER_UNIT;
    1650    106164932 :       unsigned i;
    1651    196679396 :       for (i = 0; i < operands.length (); ++i)
    1652              :         {
    1653    196679396 :           if (TREE_CODE_CLASS (operands[i].opcode) == tcc_constant)
    1654              :             {
    1655          307 :               ++i;
    1656          307 :               break;
    1657              :             }
    1658    196679089 :           if (operands[i].reverse)
    1659              :             return NULL_TREE;
    1660    196670288 :           if (known_eq (operands[i].off, -1))
    1661              :             return NULL_TREE;
    1662    182596135 :           off += operands[i].off;
    1663    182596135 :           if (operands[i].opcode == MEM_REF)
    1664              :             {
    1665     92081671 :               ++i;
    1666     92081671 :               break;
    1667              :             }
    1668              :         }
    1669     92081978 :       vn_reference_op_t base = &operands[--i];
    1670     92081978 :       tree ctor = error_mark_node;
    1671     92081978 :       tree decl = NULL_TREE;
    1672     92081978 :       if (TREE_CODE_CLASS (base->opcode) == tcc_constant)
    1673          307 :         ctor = base->op0;
    1674     92081671 :       else if (base->opcode == MEM_REF
    1675     92081671 :                && base[1].opcode == ADDR_EXPR
    1676    151234539 :                && (VAR_P (TREE_OPERAND (base[1].op0, 0))
    1677      3591525 :                    || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == CONST_DECL
    1678      3591465 :                    || TREE_CODE (TREE_OPERAND (base[1].op0, 0)) == STRING_CST))
    1679              :         {
    1680     55567399 :           decl = TREE_OPERAND (base[1].op0, 0);
    1681     55567399 :           if (TREE_CODE (decl) == STRING_CST)
    1682              :             ctor = decl;
    1683              :           else
    1684     55561403 :             ctor = ctor_for_folding (decl);
    1685              :         }
    1686     92075982 :       if (ctor == NULL_TREE)
    1687          386 :         return build_zero_cst (ref->type);
    1688     92081592 :       else if (ctor != error_mark_node)
    1689              :         {
    1690       103548 :           HOST_WIDE_INT const_off;
    1691       103548 :           if (decl)
    1692              :             {
    1693       206482 :               tree res = fold_ctor_reference (ref->type, ctor,
    1694       103241 :                                               off * BITS_PER_UNIT,
    1695       103241 :                                               size * BITS_PER_UNIT, decl);
    1696       103241 :               if (res)
    1697              :                 {
    1698        60695 :                   STRIP_USELESS_TYPE_CONVERSION (res);
    1699        60695 :                   if (is_gimple_min_invariant (res))
    1700        60563 :                     return res;
    1701              :                 }
    1702              :             }
    1703          307 :           else if (off.is_constant (&const_off))
    1704              :             {
    1705          307 :               unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
    1706          307 :               int len = native_encode_expr (ctor, buf, size, const_off);
    1707          307 :               if (len > 0)
    1708          137 :                 return native_interpret_expr (ref->type, buf, len);
    1709              :             }
    1710              :         }
    1711              :     }
    1712              : 
    1713              :   return NULL_TREE;
    1714              : }
    1715              : 
    1716              : /* Return true if OPS contain a storage order barrier.  */
    1717              : 
    1718              : static bool
    1719     60648138 : contains_storage_order_barrier_p (vec<vn_reference_op_s> ops)
    1720              : {
    1721     60648138 :   vn_reference_op_t op;
    1722     60648138 :   unsigned i;
    1723              : 
    1724    237887129 :   FOR_EACH_VEC_ELT (ops, i, op)
    1725    177238991 :     if (op->opcode == VIEW_CONVERT_EXPR && op->reverse)
    1726              :       return true;
    1727              : 
    1728              :   return false;
    1729              : }
    1730              : 
    1731              : /* Return true if OPS represent an access with reverse storage order.  */
    1732              : 
    1733              : static bool
    1734     62711927 : reverse_storage_order_for_component_p (const vec<vn_reference_op_s> &ops)
    1735              : {
    1736     62711927 :   unsigned i = 0;
    1737     62711927 :   if (ops[i].opcode == REALPART_EXPR || ops[i].opcode == IMAGPART_EXPR)
    1738              :     ++i;
    1739     62711927 :   switch (ops[i].opcode)
    1740              :     {
    1741     60038799 :     case ARRAY_REF:
    1742     60038799 :     case COMPONENT_REF:
    1743     60038799 :     case BIT_FIELD_REF:
    1744     60038799 :     case MEM_REF:
    1745     60038799 :       return ops[i].reverse;
    1746              :     default:
    1747              :       return false;
    1748              :     }
    1749              : }
    1750              : 
    1751              : /* Transform any SSA_NAME's in a vector of vn_reference_op_s
    1752              :    structures into their value numbers.  This is done in-place, and
    1753              :    the vector passed in is returned.  *VALUEIZED_ANYTHING will specify
    1754              :    whether any operands were valueized.  */
    1755              : 
    1756              : static void
    1757    224696180 : valueize_refs_1 (vec<vn_reference_op_s> *orig, bool *valueized_anything,
    1758              :                  bool with_avail = false)
    1759              : {
    1760    224696180 :   *valueized_anything = false;
    1761              : 
    1762    907559696 :   for (unsigned i = 0; i < orig->length (); ++i)
    1763              :     {
    1764    682863516 : re_valueize:
    1765    687018868 :       vn_reference_op_t vro = &(*orig)[i];
    1766    687018868 :       if (vro->opcode == SSA_NAME
    1767    587277780 :           || (vro->op0 && TREE_CODE (vro->op0) == SSA_NAME))
    1768              :         {
    1769    124694972 :           tree tem = with_avail ? vn_valueize (vro->op0) : SSA_VAL (vro->op0);
    1770    124694972 :           if (tem != vro->op0)
    1771              :             {
    1772     18592587 :               *valueized_anything = true;
    1773     18592587 :               vro->op0 = tem;
    1774              :             }
    1775              :           /* If it transforms from an SSA_NAME to a constant, update
    1776              :              the opcode.  */
    1777    124694972 :           if (TREE_CODE (vro->op0) != SSA_NAME && vro->opcode == SSA_NAME)
    1778      2216089 :             vro->opcode = TREE_CODE (vro->op0);
    1779              :         }
    1780    687018868 :       if (vro->op1 && TREE_CODE (vro->op1) == SSA_NAME)
    1781              :         {
    1782        26284 :           tree tem = with_avail ? vn_valueize (vro->op1) : SSA_VAL (vro->op1);
    1783        26284 :           if (tem != vro->op1)
    1784              :             {
    1785          586 :               *valueized_anything = true;
    1786          586 :               vro->op1 = tem;
    1787              :             }
    1788              :         }
    1789    687018868 :       if (vro->op2 && TREE_CODE (vro->op2) == SSA_NAME)
    1790              :         {
    1791       205104 :           tree tem = with_avail ? vn_valueize (vro->op2) : SSA_VAL (vro->op2);
    1792       205104 :           if (tem != vro->op2)
    1793              :             {
    1794       120020 :               *valueized_anything = true;
    1795       120020 :               vro->op2 = tem;
    1796              :             }
    1797              :         }
    1798              :       /* If it transforms from an SSA_NAME to an address, fold with
    1799              :          a preceding indirect reference.  */
    1800    687018868 :       if (i > 0
    1801    462242218 :           && vro->op0
    1802    458697901 :           && TREE_CODE (vro->op0) == ADDR_EXPR
    1803    823034565 :           && (*orig)[i - 1].opcode == MEM_REF)
    1804              :         {
    1805    130028695 :           if (vn_reference_fold_indirect (orig, &i))
    1806       707580 :             *valueized_anything = true;
    1807              :         }
    1808    556990173 :       else if (i > 0
    1809    332213523 :                && vro->opcode == SSA_NAME
    1810    654515172 :                && (*orig)[i - 1].opcode == MEM_REF)
    1811              :         {
    1812     87579171 :           if (vn_reference_maybe_forwprop_address (orig, &i))
    1813              :             {
    1814      4155352 :               *valueized_anything = true;
    1815              :               /* Re-valueize the current operand.  */
    1816      4155352 :               goto re_valueize;
    1817              :             }
    1818              :         }
    1819              :       /* If it transforms a non-constant ARRAY_REF into a constant
    1820              :          one, adjust the constant offset.  */
    1821    469411002 :       else if ((vro->opcode == ARRAY_REF
    1822    469411002 :                 || vro->opcode == ARRAY_RANGE_REF)
    1823     40593203 :                && known_eq (vro->off, -1)
    1824     17653317 :                && poly_int_tree_p (vro->op0)
    1825      5052085 :                && poly_int_tree_p (vro->op1)
    1826    474463087 :                && TREE_CODE (vro->op2) == INTEGER_CST)
    1827              :         {
    1828              :             /* Prohibit value-numbering addresses of one-after-the-last
    1829              :                element ARRAY_REFs the same as addresses of other components
    1830              :                before the pass folding __builtin_object_size had a chance
    1831              :                to run.  */
    1832      4917698 :           if (!(cfun->curr_properties & PROP_objsz)
    1833      6160738 :               && (*orig)[0].opcode == ADDR_EXPR)
    1834              :             {
    1835        36195 :               tree dom = TYPE_DOMAIN ((*orig)[i + 1].type);
    1836        55257 :               if (!dom
    1837        36045 :                   || !TYPE_MAX_VALUE (dom)
    1838        25800 :                   || !poly_int_tree_p (TYPE_MAX_VALUE (dom))
    1839        53404 :                   || integer_minus_onep (TYPE_MAX_VALUE (dom)))
    1840        19869 :                 continue;
    1841        17133 :               if (!known_le (wi::to_poly_offset (vro->op0),
    1842              :                              wi::to_poly_offset (TYPE_MAX_VALUE (dom))))
    1843          807 :                 continue;
    1844              :             }
    1845              : 
    1846      9795658 :           poly_offset_int off = ((wi::to_poly_offset (vro->op0)
    1847     14693487 :                                   - wi::to_poly_offset (vro->op1))
    1848      9795658 :                                  * wi::to_offset (vro->op2)
    1849      4897829 :                                  * vn_ref_op_align_unit (vro));
    1850      4897829 :           off.to_shwi (&vro->off);
    1851              :         }
    1852              :     }
    1853    224696180 : }
    1854              : 
    1855              : static void
    1856     12955447 : valueize_refs (vec<vn_reference_op_s> *orig)
    1857              : {
    1858     12955447 :   bool tem;
    1859            0 :   valueize_refs_1 (orig, &tem);
    1860            0 : }
    1861              : 
    1862              : static vec<vn_reference_op_s> shared_lookup_references;
    1863              : 
    1864              : /* Create a vector of vn_reference_op_s structures from REF, a
    1865              :    REFERENCE_CLASS_P tree.  The vector is shared among all callers of
    1866              :    this function.  *VALUEIZED_ANYTHING will specify whether any
    1867              :    operands were valueized.  */
    1868              : 
    1869              : static vec<vn_reference_op_s>
    1870    185333203 : valueize_shared_reference_ops_from_ref (tree ref, bool *valueized_anything)
    1871              : {
    1872    185333203 :   if (!ref)
    1873            0 :     return vNULL;
    1874    185333203 :   shared_lookup_references.truncate (0);
    1875    185333203 :   copy_reference_ops_from_ref (ref, &shared_lookup_references);
    1876    185333203 :   valueize_refs_1 (&shared_lookup_references, valueized_anything);
    1877    185333203 :   return shared_lookup_references;
    1878              : }
    1879              : 
    1880              : /* Create a vector of vn_reference_op_s structures from CALL, a
    1881              :    call statement.  The vector is shared among all callers of
    1882              :    this function.  */
    1883              : 
    1884              : static vec<vn_reference_op_s>
    1885      9377348 : valueize_shared_reference_ops_from_call (gcall *call)
    1886              : {
    1887      9377348 :   if (!call)
    1888            0 :     return vNULL;
    1889      9377348 :   shared_lookup_references.truncate (0);
    1890      9377348 :   copy_reference_ops_from_call (call, &shared_lookup_references);
    1891      9377348 :   valueize_refs (&shared_lookup_references);
    1892      9377348 :   return shared_lookup_references;
    1893              : }
    1894              : 
    1895              : /* Lookup a SCCVN reference operation VR in the current hash table.
    1896              :    Returns the resulting value number if it exists in the hash table,
    1897              :    NULL_TREE otherwise.  VNRESULT will be filled in with the actual
    1898              :    vn_reference_t stored in the hashtable if something is found.  */
    1899              : 
    1900              : static tree
    1901     67057850 : vn_reference_lookup_1 (vn_reference_t vr, vn_reference_t *vnresult)
    1902              : {
    1903     67057850 :   vn_reference_s **slot;
    1904     67057850 :   hashval_t hash;
    1905              : 
    1906     67057850 :   hash = vr->hashcode;
    1907     67057850 :   slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
    1908     67057850 :   if (slot)
    1909              :     {
    1910      8368991 :       if (vnresult)
    1911      8368991 :         *vnresult = (vn_reference_t)*slot;
    1912      8368991 :       return ((vn_reference_t)*slot)->result;
    1913              :     }
    1914              : 
    1915              :   return NULL_TREE;
    1916              : }
    1917              : 
    1918              : 
    1919              : /* Partial definition tracking support.  */
    1920              : 
    1921              : struct pd_range
    1922              : {
    1923              :   HOST_WIDE_INT offset;
    1924              :   HOST_WIDE_INT size;
    1925              :   pd_range *m_children[2];
    1926              : };
    1927              : 
    1928              : struct pd_data
    1929              : {
    1930              :   tree rhs;
    1931              :   HOST_WIDE_INT rhs_off;
    1932              :   HOST_WIDE_INT offset;
    1933              :   HOST_WIDE_INT size;
    1934              : };
    1935              : 
    1936              : /* Context for alias walking.  */
    1937              : 
    1938              : struct vn_walk_cb_data
    1939              : {
    1940     62827636 :   vn_walk_cb_data (vn_reference_t vr_, tree orig_ref_, tree *last_vuse_ptr_,
    1941              :                    vn_lookup_kind vn_walk_kind_, bool tbaa_p_, tree mask_,
    1942              :                    bool redundant_store_removal_p_)
    1943     62827636 :     : vr (vr_), last_vuse_ptr (last_vuse_ptr_), last_vuse (NULL_TREE),
    1944     62827636 :       mask (mask_), masked_result (NULL_TREE), same_val (NULL_TREE),
    1945     62827636 :       vn_walk_kind (vn_walk_kind_),
    1946     62827636 :       tbaa_p (tbaa_p_), redundant_store_removal_p (redundant_store_removal_p_),
    1947    125655272 :       saved_operands (vNULL), first_range (), first_set (-2),
    1948    125655272 :       first_base_set (-2)
    1949              :   {
    1950     62827636 :     if (!last_vuse_ptr)
    1951     29092487 :       last_vuse_ptr = &last_vuse;
    1952     62827636 :     ao_ref_init (&orig_ref, orig_ref_);
    1953     62827636 :     if (mask)
    1954              :       {
    1955       319285 :         wide_int w = wi::to_wide (mask);
    1956       319285 :         unsigned int pos = 0, prec = w.get_precision ();
    1957       319285 :         pd_data pd;
    1958       319285 :         pd.rhs = build_constructor (NULL_TREE, NULL);
    1959       319285 :         pd.rhs_off = 0;
    1960              :         /* When bitwise and with a constant is done on a memory load,
    1961              :            we don't really need all the bits to be defined or defined
    1962              :            to constants, we don't really care what is in the position
    1963              :            corresponding to 0 bits in the mask.
    1964              :            So, push the ranges of those 0 bits in the mask as artificial
    1965              :            zero stores and let the partial def handling code do the
    1966              :            rest.  */
    1967       684072 :         while (pos < prec)
    1968              :           {
    1969       662136 :             int tz = wi::ctz (w);
    1970       662136 :             if (pos + tz > prec)
    1971       297349 :               tz = prec - pos;
    1972       662136 :             if (tz)
    1973              :               {
    1974       504317 :                 if (BYTES_BIG_ENDIAN)
    1975              :                   pd.offset = prec - pos - tz;
    1976              :                 else
    1977       504317 :                   pd.offset = pos;
    1978       504317 :                 pd.size = tz;
    1979       504317 :                 void *r = push_partial_def (pd, 0, 0, 0, prec);
    1980       504317 :                 gcc_assert (r == NULL_TREE);
    1981              :               }
    1982       662136 :             pos += tz;
    1983       662136 :             if (pos == prec)
    1984              :               break;
    1985       364787 :             w = wi::lrshift (w, tz);
    1986       364787 :             tz = wi::ctz (wi::bit_not (w));
    1987       364787 :             if (pos + tz > prec)
    1988            0 :               tz = prec - pos;
    1989       364787 :             pos += tz;
    1990       364787 :             w = wi::lrshift (w, tz);
    1991              :           }
    1992       319285 :       }
    1993     62827636 :   }
    1994              :   ~vn_walk_cb_data ();
    1995              :   void *finish (alias_set_type, alias_set_type, tree);
    1996              :   void *push_partial_def (pd_data pd,
    1997              :                           alias_set_type, alias_set_type, HOST_WIDE_INT,
    1998              :                           HOST_WIDE_INT);
    1999              : 
    2000              :   vn_reference_t vr;
    2001              :   ao_ref orig_ref;
    2002              :   tree *last_vuse_ptr;
    2003              :   tree last_vuse;
    2004              :   tree mask;
    2005              :   tree masked_result;
    2006              :   tree same_val;
    2007              :   vn_lookup_kind vn_walk_kind;
    2008              :   bool tbaa_p;
    2009              :   bool redundant_store_removal_p;
    2010              :   vec<vn_reference_op_s> saved_operands;
    2011              : 
    2012              :   /* The VDEFs of partial defs we come along.  */
    2013              :   auto_vec<pd_data, 2> partial_defs;
    2014              :   /* The first defs range to avoid splay tree setup in most cases.  */
    2015              :   pd_range first_range;
    2016              :   alias_set_type first_set;
    2017              :   alias_set_type first_base_set;
    2018              :   default_splay_tree<pd_range *> known_ranges;
    2019              :   obstack ranges_obstack;
    2020              :   static constexpr HOST_WIDE_INT bufsize = 64;
    2021              : };
    2022              : 
    2023     62827636 : vn_walk_cb_data::~vn_walk_cb_data ()
    2024              : {
    2025     62827636 :   if (known_ranges)
    2026       180031 :     obstack_free (&ranges_obstack, NULL);
    2027     62827636 :   saved_operands.release ();
    2028     62827636 : }
    2029              : 
    2030              : void *
    2031      1647030 : vn_walk_cb_data::finish (alias_set_type set, alias_set_type base_set, tree val)
    2032              : {
    2033      1647030 :   if (first_set != -2)
    2034              :     {
    2035       487603 :       set = first_set;
    2036       487603 :       base_set = first_base_set;
    2037              :     }
    2038      1647030 :   if (mask)
    2039              :     {
    2040          447 :       masked_result = val;
    2041          447 :       return (void *) -1;
    2042              :     }
    2043      1646583 :   if (same_val && !operand_equal_p (val, same_val))
    2044              :     return (void *) -1;
    2045      1642445 :   vec<vn_reference_op_s> &operands
    2046      1642445 :     = saved_operands.exists () ? saved_operands : vr->operands;
    2047      1642445 :   return vn_reference_lookup_or_insert_for_pieces (last_vuse, set, base_set,
    2048              :                                                    vr->offset, vr->max_size,
    2049      1642445 :                                                    vr->type, operands, val);
    2050              : }
    2051              : 
    2052              : /* Push PD to the vector of partial definitions returning a
    2053              :    value when we are ready to combine things with VUSE, SET and MAXSIZEI,
    2054              :    NULL when we want to continue looking for partial defs or -1
    2055              :    on failure.  */
    2056              : 
    2057              : void *
    2058       589826 : vn_walk_cb_data::push_partial_def (pd_data pd,
    2059              :                                    alias_set_type set, alias_set_type base_set,
    2060              :                                    HOST_WIDE_INT offseti,
    2061              :                                    HOST_WIDE_INT maxsizei)
    2062              : {
    2063              :   /* We're using a fixed buffer for encoding so fail early if the object
    2064              :      we want to interpret is bigger.  */
    2065       589826 :   if (maxsizei > bufsize * BITS_PER_UNIT
    2066              :       || CHAR_BIT != 8
    2067              :       || BITS_PER_UNIT != 8
    2068              :       /* Not prepared to handle PDP endian.  */
    2069              :       || BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
    2070              :     return (void *)-1;
    2071              : 
    2072              :   /* Turn too large constant stores into non-constant stores.  */
    2073       589751 :   if (CONSTANT_CLASS_P (pd.rhs) && pd.size > bufsize * BITS_PER_UNIT)
    2074            0 :     pd.rhs = error_mark_node;
    2075              : 
    2076              :   /* And for non-constant or CONSTRUCTOR stores shrink them to only keep at
    2077              :      most a partial byte before and/or after the region.  */
    2078       589751 :   if (!CONSTANT_CLASS_P (pd.rhs))
    2079              :     {
    2080       548394 :       if (pd.offset < offseti)
    2081              :         {
    2082         8479 :           HOST_WIDE_INT o = ROUND_DOWN (offseti - pd.offset, BITS_PER_UNIT);
    2083         8479 :           gcc_assert (pd.size > o);
    2084         8479 :           pd.size -= o;
    2085         8479 :           pd.offset += o;
    2086              :         }
    2087       548394 :       if (pd.size + pd.offset > offseti + maxsizei)
    2088         7714 :         pd.size = ((offseti - pd.offset + maxsizei)
    2089         7714 :                    + ((pd.size + pd.offset - (offseti + maxsizei))
    2090         7714 :                       % BITS_PER_UNIT));
    2091              :     }
    2092              : 
    2093       589751 :   pd.offset -= offseti;
    2094              : 
    2095      1179502 :   bool pd_constant_p = (TREE_CODE (pd.rhs) == CONSTRUCTOR
    2096       589751 :                         || CONSTANT_CLASS_P (pd.rhs));
    2097       589751 :   pd_range *r;
    2098       589751 :   if (partial_defs.is_empty ())
    2099              :     {
    2100              :       /* If we get a clobber upfront, fail.  */
    2101       377002 :       if (TREE_CLOBBER_P (pd.rhs))
    2102              :         return (void *)-1;
    2103       376642 :       if (!pd_constant_p)
    2104              :         return (void *)-1;
    2105       343584 :       partial_defs.safe_push (pd);
    2106       343584 :       first_range.offset = pd.offset;
    2107       343584 :       first_range.size = pd.size;
    2108       343584 :       first_set = set;
    2109       343584 :       first_base_set = base_set;
    2110       343584 :       last_vuse_ptr = NULL;
    2111       343584 :       r = &first_range;
    2112              :       /* Go check if the first partial definition was a full one in case
    2113              :          the caller didn't optimize for this.  */
    2114              :     }
    2115              :   else
    2116              :     {
    2117       212749 :       if (!known_ranges)
    2118              :         {
    2119              :           /* ???  Optimize the case where the 2nd partial def completes
    2120              :              things.  */
    2121       180031 :           gcc_obstack_init (&ranges_obstack);
    2122       180031 :           known_ranges.insert_max_node (&first_range);
    2123              :         }
    2124              :       /* Lookup the offset and see if we need to merge.  */
    2125       212749 :       int comparison = known_ranges.lookup_le
    2126       429708 :         ([&] (pd_range *r) { return pd.offset < r->offset; },
    2127       191232 :          [&] (pd_range *r) { return pd.offset > r->offset; });
    2128       212749 :       r = known_ranges.root ();
    2129       212749 :       if (comparison >= 0
    2130       212749 :           && ranges_known_overlap_p (r->offset, r->size + 1,
    2131              :                                      pd.offset, pd.size))
    2132              :         {
    2133              :           /* Ignore partial defs already covered.  Here we also drop shadowed
    2134              :              clobbers arriving here at the floor.  */
    2135         5832 :           if (known_subrange_p (pd.offset, pd.size, r->offset, r->size))
    2136              :             return NULL;
    2137         5005 :           r->size = MAX (r->offset + r->size, pd.offset + pd.size) - r->offset;
    2138              :         }
    2139              :       else
    2140              :         {
    2141              :           /* pd.offset wasn't covered yet, insert the range.  */
    2142       206917 :           void *addr = XOBNEW (&ranges_obstack, pd_range);
    2143       206917 :           r = new (addr) pd_range { pd.offset, pd.size, {} };
    2144       206917 :           known_ranges.insert_relative (comparison, r);
    2145              :         }
    2146              :       /* Merge r which now contains pd's range and is a member of the splay
    2147              :          tree with adjacent overlapping ranges.  */
    2148       211922 :       if (known_ranges.splay_next_node ())
    2149        23570 :         do
    2150              :           {
    2151        23570 :             pd_range *rafter = known_ranges.root ();
    2152        23570 :             if (!ranges_known_overlap_p (r->offset, r->size + 1,
    2153        23570 :                                          rafter->offset, rafter->size))
    2154              :               break;
    2155        23314 :             r->size = MAX (r->offset + r->size,
    2156        23314 :                            rafter->offset + rafter->size) - r->offset;
    2157              :           }
    2158        23314 :         while (known_ranges.remove_root_and_splay_next ());
    2159              :       /* If we get a clobber, fail.  */
    2160       211922 :       if (TREE_CLOBBER_P (pd.rhs))
    2161              :         return (void *)-1;
    2162              :       /* Non-constants are OK as long as they are shadowed by a constant.  */
    2163       209854 :       if (!pd_constant_p)
    2164              :         return (void *)-1;
    2165       203391 :       partial_defs.safe_push (pd);
    2166              :     }
    2167              : 
    2168              :   /* Now we have merged pd's range into the range tree.  When we have covered
    2169              :      [offseti, sizei] then the tree will contain exactly one node which has
    2170              :      the desired properties and it will be 'r'.  */
    2171       546975 :   if (!known_subrange_p (0, maxsizei, r->offset, r->size))
    2172              :     /* Continue looking for partial defs.  */
    2173              :     return NULL;
    2174              : 
    2175              :   /* Now simply native encode all partial defs in reverse order.  */
    2176         9375 :   unsigned ndefs = partial_defs.length ();
    2177              :   /* We support up to 512-bit values (for V8DFmode).  */
    2178         9375 :   unsigned char buffer[bufsize + 1];
    2179         9375 :   unsigned char this_buffer[bufsize + 1];
    2180         9375 :   int len;
    2181              : 
    2182         9375 :   memset (buffer, 0, bufsize + 1);
    2183         9375 :   unsigned needed_len = ROUND_UP (maxsizei, BITS_PER_UNIT) / BITS_PER_UNIT;
    2184        36627 :   while (!partial_defs.is_empty ())
    2185              :     {
    2186        27252 :       pd_data pd = partial_defs.pop ();
    2187        27252 :       unsigned int amnt;
    2188        27252 :       if (TREE_CODE (pd.rhs) == CONSTRUCTOR)
    2189              :         {
    2190              :           /* Empty CONSTRUCTOR.  */
    2191         2256 :           if (pd.size >= needed_len * BITS_PER_UNIT)
    2192         2256 :             len = needed_len;
    2193              :           else
    2194         1972 :             len = ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT;
    2195         2256 :           memset (this_buffer, 0, len);
    2196              :         }
    2197        24996 :       else if (pd.rhs_off >= 0)
    2198              :         {
    2199        49992 :           len = native_encode_expr (pd.rhs, this_buffer, bufsize,
    2200        24996 :                                     (MAX (0, -pd.offset)
    2201        24996 :                                      + pd.rhs_off) / BITS_PER_UNIT);
    2202        24996 :           if (len <= 0
    2203        24996 :               || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
    2204        24996 :                         - MAX (0, -pd.offset) / BITS_PER_UNIT))
    2205              :             {
    2206            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    2207            0 :                 fprintf (dump_file, "Failed to encode %u "
    2208              :                          "partial definitions\n", ndefs);
    2209       589826 :               return (void *)-1;
    2210              :             }
    2211              :         }
    2212              :       else /* negative pd.rhs_off indicates we want to chop off first bits */
    2213              :         {
    2214            0 :           if (-pd.rhs_off >= bufsize)
    2215              :             return (void *)-1;
    2216            0 :           len = native_encode_expr (pd.rhs,
    2217            0 :                                     this_buffer + -pd.rhs_off / BITS_PER_UNIT,
    2218            0 :                                     bufsize - -pd.rhs_off / BITS_PER_UNIT,
    2219            0 :                                     MAX (0, -pd.offset) / BITS_PER_UNIT);
    2220            0 :           if (len <= 0
    2221            0 :               || len < (ROUND_UP (pd.size, BITS_PER_UNIT) / BITS_PER_UNIT
    2222            0 :                         - MAX (0, -pd.offset) / BITS_PER_UNIT))
    2223              :             {
    2224            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    2225            0 :                 fprintf (dump_file, "Failed to encode %u "
    2226              :                          "partial definitions\n", ndefs);
    2227              :               return (void *)-1;
    2228              :             }
    2229              :         }
    2230              : 
    2231        27252 :       unsigned char *p = buffer;
    2232        27252 :       HOST_WIDE_INT size = pd.size;
    2233        27252 :       if (pd.offset < 0)
    2234          364 :         size -= ROUND_DOWN (-pd.offset, BITS_PER_UNIT);
    2235        27252 :       this_buffer[len] = 0;
    2236        27252 :       if (BYTES_BIG_ENDIAN)
    2237              :         {
    2238              :           /* LSB of this_buffer[len - 1] byte should be at
    2239              :              pd.offset + pd.size - 1 bits in buffer.  */
    2240              :           amnt = ((unsigned HOST_WIDE_INT) pd.offset
    2241              :                   + pd.size) % BITS_PER_UNIT;
    2242              :           if (amnt)
    2243              :             shift_bytes_in_array_right (this_buffer, len + 1, amnt);
    2244              :           unsigned char *q = this_buffer;
    2245              :           unsigned int off = 0;
    2246              :           if (pd.offset >= 0)
    2247              :             {
    2248              :               unsigned int msk;
    2249              :               off = pd.offset / BITS_PER_UNIT;
    2250              :               gcc_assert (off < needed_len);
    2251              :               p = buffer + off;
    2252              :               if (size <= amnt)
    2253              :                 {
    2254              :                   msk = ((1 << size) - 1) << (BITS_PER_UNIT - amnt);
    2255              :                   *p = (*p & ~msk) | (this_buffer[len] & msk);
    2256              :                   size = 0;
    2257              :                 }
    2258              :               else
    2259              :                 {
    2260              :                   if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
    2261              :                     q = (this_buffer + len
    2262              :                          - (ROUND_UP (size - amnt, BITS_PER_UNIT)
    2263              :                             / BITS_PER_UNIT));
    2264              :                   if (pd.offset % BITS_PER_UNIT)
    2265              :                     {
    2266              :                       msk = -1U << (BITS_PER_UNIT
    2267              :                                     - (pd.offset % BITS_PER_UNIT));
    2268              :                       *p = (*p & msk) | (*q & ~msk);
    2269              :                       p++;
    2270              :                       q++;
    2271              :                       off++;
    2272              :                       size -= BITS_PER_UNIT - (pd.offset % BITS_PER_UNIT);
    2273              :                       gcc_assert (size >= 0);
    2274              :                     }
    2275              :                 }
    2276              :             }
    2277              :           else if (TREE_CODE (pd.rhs) != CONSTRUCTOR)
    2278              :             {
    2279              :               q = (this_buffer + len
    2280              :                    - (ROUND_UP (size - amnt, BITS_PER_UNIT)
    2281              :                       / BITS_PER_UNIT));
    2282              :               if (pd.offset % BITS_PER_UNIT)
    2283              :                 {
    2284              :                   q++;
    2285              :                   size -= BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) pd.offset
    2286              :                                            % BITS_PER_UNIT);
    2287              :                   gcc_assert (size >= 0);
    2288              :                 }
    2289              :             }
    2290              :           if ((unsigned HOST_WIDE_INT) size / BITS_PER_UNIT + off
    2291              :               > needed_len)
    2292              :             size = (needed_len - off) * BITS_PER_UNIT;
    2293              :           memcpy (p, q, size / BITS_PER_UNIT);
    2294              :           if (size % BITS_PER_UNIT)
    2295              :             {
    2296              :               unsigned int msk
    2297              :                 = -1U << (BITS_PER_UNIT - (size % BITS_PER_UNIT));
    2298              :               p += size / BITS_PER_UNIT;
    2299              :               q += size / BITS_PER_UNIT;
    2300              :               *p = (*q & msk) | (*p & ~msk);
    2301              :             }
    2302              :         }
    2303              :       else
    2304              :         {
    2305        27252 :           if (pd.offset >= 0)
    2306              :             {
    2307              :               /* LSB of this_buffer[0] byte should be at pd.offset bits
    2308              :                  in buffer.  */
    2309        26888 :               unsigned int msk;
    2310        26888 :               size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
    2311        26888 :               amnt = pd.offset % BITS_PER_UNIT;
    2312        26888 :               if (amnt)
    2313         1519 :                 shift_bytes_in_array_left (this_buffer, len + 1, amnt);
    2314        26888 :               unsigned int off = pd.offset / BITS_PER_UNIT;
    2315        26888 :               gcc_assert (off < needed_len);
    2316        26888 :               size = MIN (size,
    2317              :                           (HOST_WIDE_INT) (needed_len - off) * BITS_PER_UNIT);
    2318        26888 :               p = buffer + off;
    2319        26888 :               if (amnt + size < BITS_PER_UNIT)
    2320              :                 {
    2321              :                   /* Low amnt bits come from *p, then size bits
    2322              :                      from this_buffer[0] and the remaining again from
    2323              :                      *p.  */
    2324         1159 :                   msk = ((1 << size) - 1) << amnt;
    2325         1159 :                   *p = (*p & ~msk) | (this_buffer[0] & msk);
    2326         1159 :                   size = 0;
    2327              :                 }
    2328        25729 :               else if (amnt)
    2329              :                 {
    2330         1143 :                   msk = -1U << amnt;
    2331         1143 :                   *p = (*p & ~msk) | (this_buffer[0] & msk);
    2332         1143 :                   p++;
    2333         1143 :                   size -= (BITS_PER_UNIT - amnt);
    2334              :                 }
    2335              :             }
    2336              :           else
    2337              :             {
    2338          364 :               amnt = (unsigned HOST_WIDE_INT) pd.offset % BITS_PER_UNIT;
    2339          364 :               if (amnt)
    2340           45 :                 size -= BITS_PER_UNIT - amnt;
    2341          364 :               size = MIN (size, (HOST_WIDE_INT) needed_len * BITS_PER_UNIT);
    2342          364 :               if (amnt)
    2343           45 :                 shift_bytes_in_array_left (this_buffer, len + 1, amnt);
    2344              :             }
    2345        27252 :           memcpy (p, this_buffer + (amnt != 0), size / BITS_PER_UNIT);
    2346        27252 :           p += size / BITS_PER_UNIT;
    2347        27252 :           if (size % BITS_PER_UNIT)
    2348              :             {
    2349          648 :               unsigned int msk = -1U << (size % BITS_PER_UNIT);
    2350          648 :               *p = (this_buffer[(amnt != 0) + size / BITS_PER_UNIT]
    2351          648 :                     & ~msk) | (*p & msk);
    2352              :             }
    2353              :         }
    2354              :     }
    2355              : 
    2356         9375 :   tree type = vr->type;
    2357              :   /* Make sure to interpret in a type that has a range covering the whole
    2358              :      access size.  */
    2359         9375 :   if (INTEGRAL_TYPE_P (vr->type) && maxsizei != TYPE_PRECISION (vr->type))
    2360              :     {
    2361            0 :       if (BITINT_TYPE_P (vr->type)
    2362           26 :           && maxsizei > MAX_FIXED_MODE_SIZE)
    2363           13 :         type = build_bitint_type (maxsizei, TYPE_UNSIGNED (type));
    2364              :       else
    2365            0 :         type = build_nonstandard_integer_type (maxsizei, TYPE_UNSIGNED (type));
    2366              :     }
    2367         9375 :   tree val;
    2368         9375 :   if (BYTES_BIG_ENDIAN)
    2369              :     {
    2370              :       unsigned sz = needed_len;
    2371              :       if (maxsizei % BITS_PER_UNIT)
    2372              :         shift_bytes_in_array_right (buffer, needed_len,
    2373              :                                     BITS_PER_UNIT
    2374              :                                     - (maxsizei % BITS_PER_UNIT));
    2375              :       if (INTEGRAL_TYPE_P (type))
    2376              :         {
    2377              :           if (TYPE_MODE (type) != BLKmode)
    2378              :             sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
    2379              :           else
    2380              :             sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
    2381              :         }
    2382              :       if (sz > needed_len)
    2383              :         {
    2384              :           memcpy (this_buffer + (sz - needed_len), buffer, needed_len);
    2385              :           val = native_interpret_expr (type, this_buffer, sz);
    2386              :         }
    2387              :       else
    2388              :         val = native_interpret_expr (type, buffer, needed_len);
    2389              :     }
    2390              :   else
    2391         9375 :     val = native_interpret_expr (type, buffer, bufsize);
    2392              :   /* If we chop off bits because the types precision doesn't match the memory
    2393              :      access size this is ok when optimizing reads but not when called from
    2394              :      the DSE code during elimination.  */
    2395         9375 :   if (val && type != vr->type)
    2396              :     {
    2397           13 :       if (! int_fits_type_p (val, vr->type))
    2398              :         val = NULL_TREE;
    2399              :       else
    2400           13 :         val = fold_convert (vr->type, val);
    2401              :     }
    2402              : 
    2403         9371 :   if (val)
    2404              :     {
    2405         9371 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2406            0 :         fprintf (dump_file,
    2407              :                  "Successfully combined %u partial definitions\n", ndefs);
    2408              :       /* We are using the alias-set of the first store we encounter which
    2409              :          should be appropriate here.  */
    2410         9371 :       return finish (first_set, first_base_set, val);
    2411              :     }
    2412              :   else
    2413              :     {
    2414            4 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2415            0 :         fprintf (dump_file,
    2416              :                  "Failed to interpret %u encoded partial definitions\n", ndefs);
    2417              :       return (void *)-1;
    2418              :     }
    2419              : }
    2420              : 
    2421              : /* Callback for walk_non_aliased_vuses.  Adjusts the vn_reference_t VR_
    2422              :    with the current VUSE and performs the expression lookup.  */
    2423              : 
    2424              : static void *
    2425   1107239823 : vn_reference_lookup_2 (ao_ref *op, tree vuse, void *data_)
    2426              : {
    2427   1107239823 :   vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
    2428   1107239823 :   vn_reference_t vr = data->vr;
    2429   1107239823 :   vn_reference_s **slot;
    2430   1107239823 :   hashval_t hash;
    2431              : 
    2432              :   /* If we have partial definitions recorded we have to go through
    2433              :      vn_reference_lookup_3.  */
    2434   1107239823 :   if (!data->partial_defs.is_empty ())
    2435              :     return NULL;
    2436              : 
    2437   1106416114 :   if (data->last_vuse_ptr)
    2438              :     {
    2439   1082061709 :       *data->last_vuse_ptr = vuse;
    2440   1082061709 :       data->last_vuse = vuse;
    2441              :     }
    2442              : 
    2443              :   /* Fixup vuse and hash.  */
    2444   1106416114 :   if (vr->vuse)
    2445   1106416114 :     vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
    2446   1106416114 :   vr->vuse = vuse_ssa_val (vuse);
    2447   1106416114 :   if (vr->vuse)
    2448   1106416114 :     vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
    2449              : 
    2450   1106416114 :   hash = vr->hashcode;
    2451   1106416114 :   slot = valid_info->references->find_slot_with_hash (vr, hash, NO_INSERT);
    2452   1106416114 :   if (slot)
    2453              :     {
    2454      8135495 :       if ((*slot)->result && data->saved_operands.exists ())
    2455       471701 :         return data->finish (vr->set, vr->base_set, (*slot)->result);
    2456              :       return *slot;
    2457              :     }
    2458              : 
    2459   1098280619 :   if (SSA_NAME_IS_DEFAULT_DEF (vuse))
    2460              :     {
    2461     18623352 :       HOST_WIDE_INT op_offset, op_size;
    2462     18623352 :       tree v = NULL_TREE;
    2463     18623352 :       tree base = ao_ref_base (op);
    2464              : 
    2465     18623352 :       if (base
    2466     18623352 :           && op->offset.is_constant (&op_offset)
    2467     18623352 :           && op->size.is_constant (&op_size)
    2468     18623352 :           && op->max_size_known_p ()
    2469     36771575 :           && known_eq (op->size, op->max_size))
    2470              :         {
    2471     17828358 :           if (TREE_CODE (base) == PARM_DECL)
    2472       676252 :             v = ipcp_get_aggregate_const (cfun, base, false, op_offset,
    2473              :                                           op_size);
    2474     17152106 :           else if (TREE_CODE (base) == MEM_REF
    2475      7133116 :                    && integer_zerop (TREE_OPERAND (base, 1))
    2476      5730712 :                    && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
    2477      5725507 :                    && SSA_NAME_IS_DEFAULT_DEF (TREE_OPERAND (base, 0))
    2478     20953126 :                    && (TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (base, 0)))
    2479              :                        == PARM_DECL))
    2480      3749050 :             v = ipcp_get_aggregate_const (cfun,
    2481      3749050 :                                           SSA_NAME_VAR (TREE_OPERAND (base, 0)),
    2482              :                                           true, op_offset, op_size);
    2483              :         }
    2484      4425302 :       if (v)
    2485         1206 :         return data->finish (vr->set, vr->base_set, v);
    2486              :     }
    2487              : 
    2488              :   return NULL;
    2489              : }
    2490              : 
    2491              : /* Lookup an existing or insert a new vn_reference entry into the
    2492              :    value table for the VUSE, SET, TYPE, OPERANDS reference which
    2493              :    has the value VALUE which is either a constant or an SSA name.  */
    2494              : 
    2495              : static vn_reference_t
    2496      1642445 : vn_reference_lookup_or_insert_for_pieces (tree vuse,
    2497              :                                           alias_set_type set,
    2498              :                                           alias_set_type base_set,
    2499              :                                           poly_int64 offset,
    2500              :                                           poly_int64 max_size,
    2501              :                                           tree type,
    2502              :                                           vec<vn_reference_op_s,
    2503              :                                                 va_heap> operands,
    2504              :                                           tree value)
    2505              : {
    2506      1642445 :   vn_reference_s vr1;
    2507      1642445 :   vn_reference_t result;
    2508      1642445 :   unsigned value_id;
    2509      1642445 :   vr1.vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
    2510      1642445 :   vr1.operands = operands;
    2511      1642445 :   vr1.type = type;
    2512      1642445 :   vr1.set = set;
    2513      1642445 :   vr1.base_set = base_set;
    2514      1642445 :   vr1.offset = offset;
    2515      1642445 :   vr1.max_size = max_size;
    2516      1642445 :   vr1.hashcode = vn_reference_compute_hash (&vr1);
    2517      1642445 :   if (vn_reference_lookup_1 (&vr1, &result))
    2518        14877 :     return result;
    2519              : 
    2520      1627568 :   if (TREE_CODE (value) == SSA_NAME)
    2521       384192 :     value_id = VN_INFO (value)->value_id;
    2522              :   else
    2523      1243376 :     value_id = get_or_alloc_constant_value_id (value);
    2524      1627568 :   return vn_reference_insert_pieces (vuse, set, base_set, offset, max_size,
    2525      1627568 :                                      type, operands.copy (), value, value_id);
    2526              : }
    2527              : 
    2528              : /* Return true if RES_OP is a CONSTRUCTOR in GENERIC form.  ???  We're
    2529              :    arriving here with SCCVNs view, decomposed CONSTRUCTOR and GIMPLEs /
    2530              :    match-and-simplifies, CONSTRUCTOR as GENERIC tree.  */
    2531              : 
    2532              : static bool
    2533     72220521 : match_op_generic_constructor_p (gimple_match_op *res_op)
    2534              : {
    2535            0 :   return (res_op->code == CONSTRUCTOR
    2536     72220521 :           && TREE_CODE (res_op->ops[0]) == CONSTRUCTOR);
    2537              : }
    2538              : 
    2539              : /* Return the number of operands for a vn_nary ops structure from RES_OP.  */
    2540              : 
    2541              : static unsigned int
    2542     48147014 : vn_nary_length_from_match_op (gimple_match_op *res_op)
    2543              : {
    2544     48147014 :   if (match_op_generic_constructor_p (res_op))
    2545         2042 :     return CONSTRUCTOR_NELTS (res_op->ops[0]);
    2546     48144972 :   return res_op->num_ops;
    2547              : }
    2548              : 
    2549              : /* Initialize VNO from RES_OP.  The operands are taken as-is, they are
    2550              :    not valueized.  */
    2551              : 
    2552              : static void
    2553     24073507 : init_vn_nary_op_from_match_op (vn_nary_op_t vno, gimple_match_op *res_op)
    2554              : {
    2555     24073507 :   vno->opcode = (tree_code) res_op->code;
    2556     24073507 :   vno->length = vn_nary_length_from_match_op (res_op);
    2557     24073507 :   vno->type = res_op->type;
    2558     24073507 :   if (match_op_generic_constructor_p (res_op))
    2559         4595 :     for (unsigned i = 0; i < vno->length; ++i)
    2560         3574 :       vno->op[i] = CONSTRUCTOR_ELT (res_op->ops[0], i)->value;
    2561              :   else
    2562     24072486 :     memcpy (&vno->op[0], res_op->ops, sizeof (tree) * vno->length);
    2563     24073507 :   vno->predicated_values = 0;
    2564     24073507 : }
    2565              : 
    2566              : /* Return a value-number for RCODE OPS... either by looking up an existing
    2567              :    value-number for the possibly simplified result or by inserting the
    2568              :    operation if INSERT is true.  If SIMPLIFY is false, return a value
    2569              :    number for the unsimplified expression.  */
    2570              : 
    2571              : static tree
    2572     19224061 : vn_nary_build_or_lookup_1 (gimple_match_op *res_op, bool insert,
    2573              :                            bool simplify)
    2574              : {
    2575     19224061 :   tree result = NULL_TREE;
    2576              :   /* We will be creating a value number for
    2577              :        RCODE (OPS...).
    2578              :      So first simplify and lookup this expression to see if it
    2579              :      is already available.  */
    2580              :   /* For simplification valueize.  */
    2581     19224061 :   unsigned i = 0;
    2582     19224061 :   if (simplify)
    2583     44593948 :     for (i = 0; i < res_op->num_ops; ++i)
    2584     25377234 :       if (TREE_CODE (res_op->ops[i]) == SSA_NAME)
    2585              :         {
    2586     16253513 :           tree tem = vn_valueize (res_op->ops[i]);
    2587     16253513 :           if (!tem)
    2588              :             break;
    2589     16253513 :           res_op->ops[i] = tem;
    2590              :         }
    2591              :   /* If valueization of an operand fails (it is not available), skip
    2592              :      simplification.  */
    2593     19224061 :   bool res = false;
    2594     19224061 :   if (i == res_op->num_ops)
    2595              :     {
    2596              :       /* Do not leak not available operands into the simplified expression
    2597              :          when called from PRE context.  */
    2598     19216714 :       if (rpo_avail)
    2599     11523717 :         mprts_hook = vn_lookup_simplify_result;
    2600     19216714 :       res = res_op->resimplify (NULL, vn_valueize);
    2601     19216714 :       mprts_hook = NULL;
    2602              :     }
    2603     33102660 :   gimple *new_stmt = NULL;
    2604     33102660 :   bool reused = false;
    2605     19216714 :   if (res
    2606     19216714 :       && gimple_simplified_result_is_gimple_val (res_op))
    2607              :     {
    2608              :       /* The expression is already available.  */
    2609      5338115 :       result = res_op->ops[0];
    2610              :       /* Valueize it, simplification returns sth in AVAIL only.  */
    2611      5338115 :       if (TREE_CODE (result) == SSA_NAME)
    2612       296156 :         result = SSA_VAL (result);
    2613              :     }
    2614              :   else
    2615              :     {
    2616     13885946 :       tree val = vn_lookup_simplify_result (res_op);
    2617              :       /* ???  In weird cases we can end up with internal-fn calls,
    2618              :          but this isn't expected so throw the result away.  See
    2619              :          PR123040 for an example.  Likewise we can end up with
    2620              :          &MEM[ptr_1 + CST] which would be a vn_reference (PR127000).  */
    2621     13885946 :       if (!val
    2622     13885946 :           && insert
    2623       179690 :           && res_op->code.is_tree_code ()
    2624     14065636 :           && (tree_code) res_op->code != ADDR_EXPR)
    2625              :         {
    2626              :           /* When iterating, re-use the SSA name we created for the same
    2627              :              expression in a previous iteration instead of creating a new
    2628              :              one each time.  */
    2629       179690 :           vn_nary_op_t vno1 = NULL;
    2630       179690 :           vn_nary_op_s **slot = NULL;
    2631       179690 :           if (valid_info->inserted_exprs)
    2632              :             {
    2633       129059 :               vno1 = XALLOCAVAR (struct vn_nary_op_s,
    2634              :                                  sizeof_vn_nary_op
    2635              :                                    (vn_nary_length_from_match_op (res_op)));
    2636       129059 :               init_vn_nary_op_from_match_op (vno1, res_op);
    2637       129059 :               vno1->hashcode = vn_nary_op_compute_hash (vno1);
    2638       129059 :               slot = valid_info->inserted_exprs->find_slot_with_hash
    2639       129059 :                        (vno1, vno1->hashcode, NO_INSERT);
    2640              :             }
    2641       129059 :           if (slot)
    2642              :             {
    2643        28111 :               result = (*slot)->u.result;
    2644        28111 :               new_stmt = gimple_seq_first_stmt (VN_INFO (result)->expr);
    2645        28111 :               reused = true;
    2646              :             }
    2647              :           else
    2648              :             {
    2649       151579 :               gimple_seq stmts = NULL;
    2650       151579 :               result = maybe_push_res_to_seq (res_op, &stmts);
    2651       151579 :               if (result)
    2652              :                 {
    2653       151573 :                   gcc_assert (gimple_seq_singleton_p (stmts));
    2654       151573 :                   new_stmt = gimple_seq_first_stmt (stmts);
    2655       151573 :                   if (vno1)
    2656              :                     {
    2657       100943 :                       slot = valid_info->inserted_exprs->find_slot_with_hash
    2658       100943 :                                (vno1, vno1->hashcode, INSERT);
    2659       100943 :                       vn_nary_op_t entry
    2660       100943 :                         = alloc_vn_nary_op_noinit (vno1->length,
    2661              :                                                    &vn_tables_insert_obstack);
    2662       100943 :                       memcpy (entry, vno1, sizeof_vn_nary_op (vno1->length));
    2663       100943 :                       entry->u.result = result;
    2664       100943 :                       *slot = entry;
    2665              :                     }
    2666              :                 }
    2667              :             }
    2668              :         }
    2669              :       else
    2670              :         /* The expression is already available.  */
    2671              :         result = val;
    2672              :     }
    2673       475846 :   if (new_stmt)
    2674              :     {
    2675              :       /* The expression is not yet available, value-number lhs to
    2676              :          the new SSA_NAME we created.  */
    2677       179684 :       if (!reused)
    2678              :         {
    2679              :           /* Initialize value-number information properly.  */
    2680       151573 :           vn_ssa_aux_t result_info = VN_INFO (result);
    2681       151573 :           result_info->valnum = result;
    2682       151573 :           result_info->value_id = get_next_value_id ();
    2683       151573 :           result_info->visited = 1;
    2684       151573 :           gimple_seq_add_stmt_without_update (&VN_INFO (result)->expr,
    2685              :                                               new_stmt);
    2686       151573 :           result_info->needs_insertion = true;
    2687              :         }
    2688              :       /* ???  PRE phi-translation inserts NARYs without corresponding
    2689              :          SSA name result.  Re-use those but set their result according
    2690              :          to the stmt we just built.  */
    2691       179684 :       vn_nary_op_t nary = NULL;
    2692       179684 :       vn_nary_op_lookup_stmt (new_stmt, &nary);
    2693       179684 :       if (nary)
    2694              :         {
    2695            0 :           gcc_assert (! nary->predicated_values && nary->u.result == NULL_TREE);
    2696            0 :           nary->u.result = gimple_assign_lhs (new_stmt);
    2697              :         }
    2698              :       else
    2699              :         {
    2700              :           /* Record the expression in the hashtable.  The entry is keyed
    2701              :              on the current, possibly optimistic, value numbers of the
    2702              :              operands, so unlike the inserted_exprs entry it has to be
    2703              :              unwound on iteration like any other entry.  */
    2704       179684 :           vn_nary_op_insert_stmt (new_stmt, result);
    2705              :         }
    2706       179684 :       if (dump_file && (dump_flags & TDF_DETAILS))
    2707              :         {
    2708         1190 :           fprintf (dump_file, reused ? "Re-using name " : "Inserting name ");
    2709          595 :           print_generic_expr (dump_file, result);
    2710          595 :           fprintf (dump_file, " for expression ");
    2711          595 :           print_gimple_expr (dump_file, new_stmt, 0, TDF_SLIM);
    2712          595 :           fprintf (dump_file, "\n");
    2713              :         }
    2714              :     }
    2715     19224061 :   return result;
    2716              : }
    2717              : 
    2718              : /* Return a value-number for RCODE OPS... either by looking up an existing
    2719              :    value-number for the simplified result or by inserting the operation.  */
    2720              : 
    2721              : static tree
    2722       198078 : vn_nary_build_or_lookup (gimple_match_op *res_op)
    2723              : {
    2724            0 :   return vn_nary_build_or_lookup_1 (res_op, true, true);
    2725              : }
    2726              : 
    2727              : /* Try to simplify the expression RCODE OPS... of type TYPE and return
    2728              :    its value if present.  Update NARY with a simplified expression if
    2729              :    it fits.  */
    2730              : 
    2731              : tree
    2732      7689644 : vn_nary_simplify (vn_nary_op_t nary)
    2733              : {
    2734      7689644 :   if (nary->length > gimple_match_op::MAX_NUM_OPS
    2735              :       /* For CONSTRUCTOR the vn_nary_op_t and gimple_match_op representation
    2736              :          does not match.  */
    2737      7689096 :       || nary->opcode == CONSTRUCTOR)
    2738              :     return NULL_TREE;
    2739      7686601 :   gimple_match_op op (gimple_match_cond::UNCOND, nary->opcode,
    2740      7686601 :                       nary->type, nary->length);
    2741      7686601 :   memcpy (op.ops, nary->op, sizeof (tree) * nary->length);
    2742      7686601 :   tree res = vn_nary_build_or_lookup_1 (&op, false, true);
    2743              :   /* Do not update *NARY with a simplified result that contains abnormals.
    2744              :      This matches what maybe_push_res_to_seq does when requesting insertion.  */
    2745     27841464 :   for (unsigned i = 0; i < op.num_ops; ++i)
    2746     12468344 :     if (TREE_CODE (op.ops[i]) == SSA_NAME
    2747     12468344 :         && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op.ops[i]))
    2748              :       return res;
    2749      7686519 :   if (op.code.is_tree_code ()
    2750      7686519 :       && op.num_ops <= nary->length
    2751     15372204 :       && (tree_code) op.code != CONSTRUCTOR)
    2752              :     {
    2753      7685684 :       nary->opcode = (tree_code) op.code;
    2754      7685684 :       nary->length = op.num_ops;
    2755     20152234 :       for (unsigned i = 0; i < op.num_ops; ++i)
    2756     12466550 :         nary->op[i] = op.ops[i];
    2757              :     }
    2758              :   return res;
    2759              : }
    2760              : 
    2761              : /* Elimination engine.  */
    2762              : 
    2763              : class eliminate_dom_walker : public dom_walker
    2764              : {
    2765              : public:
    2766              :   eliminate_dom_walker (cdi_direction, bitmap);
    2767              :   virtual ~eliminate_dom_walker ();
    2768              : 
    2769              :   edge before_dom_children (basic_block) final override;
    2770              :   void after_dom_children (basic_block) final override;
    2771              : 
    2772              :   virtual tree eliminate_avail (basic_block, tree op);
    2773              :   virtual void eliminate_push_avail (basic_block, tree op);
    2774              :   tree eliminate_insert (basic_block, gimple_stmt_iterator *gsi, tree val);
    2775              : 
    2776              :   void eliminate_stmt (basic_block, gimple_stmt_iterator *);
    2777              : 
    2778              :   unsigned eliminate_cleanup (bool region_p = false);
    2779              : 
    2780              :   bool do_pre;
    2781              :   unsigned int el_todo;
    2782              :   unsigned int eliminations;
    2783              :   unsigned int insertions;
    2784              : 
    2785              :   /* SSA names that had their defs inserted by PRE if do_pre.  */
    2786              :   bitmap inserted_exprs;
    2787              : 
    2788              :   /* Blocks with statements that have had their EH properties changed.  */
    2789              :   bitmap need_eh_cleanup;
    2790              : 
    2791              :   /* Blocks with statements that have had their AB properties changed.  */
    2792              :   bitmap need_ab_cleanup;
    2793              : 
    2794              :   /* Local state for the eliminate domwalk.  */
    2795              :   auto_vec<gimple *> to_remove;
    2796              :   auto_vec<gimple *> to_fixup;
    2797              :   auto_vec<tree> avail;
    2798              :   auto_vec<tree> avail_stack;
    2799              : };
    2800              : 
    2801              : /* Adaptor to the elimination engine using RPO availability.  */
    2802              : 
    2803              : class rpo_elim : public eliminate_dom_walker
    2804              : {
    2805              : public:
    2806      6308971 :   rpo_elim(basic_block entry_)
    2807      6308971 :     : eliminate_dom_walker (CDI_DOMINATORS, NULL), entry (entry_),
    2808     12617942 :       m_avail_freelist (NULL) {}
    2809              : 
    2810              :   tree eliminate_avail (basic_block, tree op) final override;
    2811              : 
    2812              :   void eliminate_push_avail (basic_block, tree) final override;
    2813              : 
    2814              :   basic_block entry;
    2815              :   /* Freelist of avail entries which are allocated from the vn_ssa_aux
    2816              :      obstack.  */
    2817              :   vn_avail *m_avail_freelist;
    2818              : };
    2819              : 
    2820              : /* Return true if BASE1 and BASE2 can be adjusted so they have the
    2821              :    same address and adjust *OFFSET1 and *OFFSET2 accordingly.
    2822              :    Otherwise return false.  */
    2823              : 
    2824              : static bool
    2825      7053873 : adjust_offsets_for_equal_base_address (tree base1, poly_int64 *offset1,
    2826              :                                        tree base2, poly_int64 *offset2)
    2827              : {
    2828      7053873 :   poly_int64 soff;
    2829      7053873 :   if (TREE_CODE (base1) == MEM_REF
    2830      3213951 :       && TREE_CODE (base2) == MEM_REF)
    2831              :     {
    2832      2584199 :       if (mem_ref_offset (base1).to_shwi (&soff))
    2833              :         {
    2834      2584199 :           base1 = TREE_OPERAND (base1, 0);
    2835      2584199 :           *offset1 += soff * BITS_PER_UNIT;
    2836              :         }
    2837      2584199 :       if (mem_ref_offset (base2).to_shwi (&soff))
    2838              :         {
    2839      2584199 :           base2 = TREE_OPERAND (base2, 0);
    2840      2584199 :           *offset2 += soff * BITS_PER_UNIT;
    2841              :         }
    2842      2584199 :       return operand_equal_p (base1, base2, 0);
    2843              :     }
    2844      4469674 :   return operand_equal_p (base1, base2, OEP_ADDRESS_OF);
    2845              : }
    2846              : 
    2847              : /* Callback for walk_non_aliased_vuses.  Tries to perform a lookup
    2848              :    from the statement defining VUSE and if not successful tries to
    2849              :    translate *REFP and VR_ through an aggregate copy at the definition
    2850              :    of VUSE.  If *DISAMBIGUATE_ONLY is true then do not perform translation
    2851              :    of *REF and *VR.  If only disambiguation was performed then
    2852              :    *DISAMBIGUATE_ONLY is set to true.  */
    2853              : 
    2854              : static void *
    2855     43653424 : vn_reference_lookup_3 (ao_ref *ref, tree vuse, void *data_,
    2856              :                        translate_flags *disambiguate_only)
    2857              : {
    2858     43653424 :   vn_walk_cb_data *data = (vn_walk_cb_data *)data_;
    2859     43653424 :   vn_reference_t vr = data->vr;
    2860     43653424 :   gimple *def_stmt = SSA_NAME_DEF_STMT (vuse);
    2861     43653424 :   tree base = ao_ref_base (ref);
    2862     43653424 :   HOST_WIDE_INT offseti = 0, maxsizei, sizei = 0;
    2863     43653424 :   static vec<vn_reference_op_s> lhs_ops;
    2864     43653424 :   ao_ref lhs_ref;
    2865     43653424 :   bool lhs_ref_ok = false;
    2866     43653424 :   poly_int64 copy_size;
    2867              : 
    2868              :   /* First try to disambiguate after value-replacing in the definitions LHS.  */
    2869     43653424 :   if (is_gimple_assign (def_stmt))
    2870              :     {
    2871     20961884 :       tree lhs = gimple_assign_lhs (def_stmt);
    2872     20961884 :       bool valueized_anything = false;
    2873              :       /* Avoid re-allocation overhead.  */
    2874     20961884 :       lhs_ops.truncate (0);
    2875     20961884 :       basic_block saved_rpo_bb = vn_context_bb;
    2876     20961884 :       vn_context_bb = gimple_bb (def_stmt);
    2877     20961884 :       if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE)
    2878              :         {
    2879     13596327 :           copy_reference_ops_from_ref (lhs, &lhs_ops);
    2880     13596327 :           valueize_refs_1 (&lhs_ops, &valueized_anything, true);
    2881              :         }
    2882     20961884 :       vn_context_bb = saved_rpo_bb;
    2883     20961884 :       ao_ref_init (&lhs_ref, lhs);
    2884     20961884 :       lhs_ref_ok = true;
    2885     20961884 :       if (valueized_anything
    2886      2176543 :           && ao_ref_init_from_vn_reference
    2887      2176543 :                (&lhs_ref, ao_ref_alias_set (&lhs_ref),
    2888      2176543 :                 ao_ref_base_alias_set (&lhs_ref), TREE_TYPE (lhs), lhs_ops)
    2889     23138427 :           && !refs_may_alias_p_1 (ref, &lhs_ref, data->tbaa_p))
    2890              :         {
    2891      1893155 :           *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
    2892      6854106 :           return NULL;
    2893              :         }
    2894              : 
    2895              :       /* When the def is a CLOBBER we can optimistically disambiguate
    2896              :          against it since any overlap it would be undefined behavior.
    2897              :          Avoid this for obvious must aliases to save compile-time though.
    2898              :          We also may not do this when the query is used for redundant
    2899              :          store removal.  */
    2900     19068729 :       if (!data->redundant_store_removal_p
    2901     10643222 :           && gimple_clobber_p (def_stmt)
    2902     19582532 :           && !operand_equal_p (ao_ref_base (&lhs_ref), base, OEP_ADDRESS_OF))
    2903              :         {
    2904       486583 :           *disambiguate_only = TR_DISAMBIGUATE;
    2905       486583 :           return NULL;
    2906              :         }
    2907              : 
    2908              :       /* Besides valueizing the LHS we can also use access-path based
    2909              :          disambiguation on the original non-valueized ref.  */
    2910     18582146 :       if (!ref->ref
    2911              :           && lhs_ref_ok
    2912      2778391 :           && data->orig_ref.ref)
    2913              :         {
    2914              :           /* We want to use the non-valueized LHS for this, but avoid redundant
    2915              :              work.  */
    2916      1912706 :           ao_ref *lref = &lhs_ref;
    2917      1912706 :           ao_ref lref_alt;
    2918      1912706 :           if (valueized_anything)
    2919              :             {
    2920       112981 :               ao_ref_init (&lref_alt, lhs);
    2921       112981 :               lref = &lref_alt;
    2922              :             }
    2923      1912706 :           if (!refs_may_alias_p_1 (&data->orig_ref, lref, data->tbaa_p))
    2924              :             {
    2925       324332 :               *disambiguate_only = (valueized_anything
    2926       162166 :                                     ? TR_VALUEIZE_AND_DISAMBIGUATE
    2927              :                                     : TR_DISAMBIGUATE);
    2928       162166 :               return NULL;
    2929              :             }
    2930              :         }
    2931              : 
    2932              :       /* If we reach a clobbering statement try to skip it and see if
    2933              :          we find a VN result with exactly the same value as the
    2934              :          possible clobber.  In this case we can ignore the clobber
    2935              :          and return the found value.  */
    2936     18419980 :       if (!gimple_has_volatile_ops (def_stmt)
    2937     17008684 :           && ((is_gimple_reg_type (TREE_TYPE (lhs))
    2938     12843401 :                && types_compatible_p (TREE_TYPE (lhs), vr->type)
    2939     10039193 :                && !storage_order_barrier_p (lhs)
    2940     10039193 :                && !reverse_storage_order_for_component_p (lhs))
    2941      6969581 :               || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == CONSTRUCTOR)
    2942     11133490 :           && (ref->ref || data->orig_ref.ref)
    2943     10643376 :           && !data->mask
    2944     10622874 :           && data->partial_defs.is_empty ()
    2945     10620482 :           && multiple_p (get_object_alignment
    2946              :                            (ref->ref ? ref->ref : data->orig_ref.ref),
    2947              :                            ref->size)
    2948     41332410 :           && multiple_p (get_object_alignment (lhs), ref->size))
    2949              :         {
    2950     10215948 :           HOST_WIDE_INT offset2i, size2i;
    2951     10215948 :           poly_int64 offset = ref->offset;
    2952     10215948 :           poly_int64 maxsize = ref->max_size;
    2953              : 
    2954     10215948 :           gcc_assert (lhs_ref_ok);
    2955     10215948 :           tree base2 = ao_ref_base (&lhs_ref);
    2956     10215948 :           poly_int64 offset2 = lhs_ref.offset;
    2957     10215948 :           poly_int64 size2 = lhs_ref.size;
    2958     10215948 :           poly_int64 maxsize2 = lhs_ref.max_size;
    2959              : 
    2960     10215948 :           tree rhs = gimple_assign_rhs1 (def_stmt);
    2961     10215948 :           if (TREE_CODE (rhs) == CONSTRUCTOR)
    2962      1061403 :             rhs = integer_zero_node;
    2963              :           /* ???  We may not compare to ahead values which might be from
    2964              :              a different loop iteration but only to loop invariants.  Use
    2965              :              CONSTANT_CLASS_P (unvalueized!) as conservative approximation.
    2966              :              The one-hop lookup below doesn't have this issue since there's
    2967              :              a virtual PHI before we ever reach a backedge to cross.
    2968              :              We can skip multiple defs as long as they are from the same
    2969              :              value though.  */
    2970     10215948 :           if (data->same_val
    2971     10215948 :               && !operand_equal_p (data->same_val, rhs))
    2972              :             ;
    2973              :           /* When this is a (partial) must-def, leave it to handling
    2974              :              below in case we are interested in the value.  */
    2975      9923472 :           else if (!(*disambiguate_only > TR_TRANSLATE)
    2976      3427084 :                    && base2
    2977      3427084 :                    && known_eq (maxsize2, size2)
    2978      2423043 :                    && adjust_offsets_for_equal_base_address (base, &offset,
    2979              :                                                              base2, &offset2)
    2980      1188699 :                    && offset2.is_constant (&offset2i)
    2981      1188699 :                    && size2.is_constant (&size2i)
    2982      1188699 :                    && maxsize.is_constant (&maxsizei)
    2983      1188699 :                    && offset.is_constant (&offseti)
    2984     11112171 :                    && ranges_known_overlap_p (offseti, maxsizei, offset2i,
    2985              :                                               size2i))
    2986              :             ;
    2987      8830917 :           else if (CONSTANT_CLASS_P (rhs))
    2988              :             {
    2989      4244030 :               if (dump_file && (dump_flags & TDF_DETAILS))
    2990              :                 {
    2991         2330 :                   fprintf (dump_file,
    2992              :                            "Skipping possible redundant definition ");
    2993         2330 :                   print_gimple_stmt (dump_file, def_stmt, 0);
    2994              :                 }
    2995              :               /* Delay the actual compare of the values to the end of the walk
    2996              :                  but do not update last_vuse from here.  */
    2997      4244030 :               data->last_vuse_ptr = NULL;
    2998      4244030 :               data->same_val = rhs;
    2999      4312202 :               return NULL;
    3000              :             }
    3001              :           else
    3002              :             {
    3003      4586887 :               tree saved_vuse = vr->vuse;
    3004      4586887 :               hashval_t saved_hashcode = vr->hashcode;
    3005      4586887 :               if (vr->vuse)
    3006      4586887 :                 vr->hashcode = vr->hashcode - SSA_NAME_VERSION (vr->vuse);
    3007      9173774 :               vr->vuse = vuse_ssa_val (gimple_vuse (def_stmt));
    3008      4586887 :               if (vr->vuse)
    3009      4586887 :                 vr->hashcode = vr->hashcode + SSA_NAME_VERSION (vr->vuse);
    3010      4586887 :               vn_reference_t vnresult = NULL;
    3011              :               /* Do not use vn_reference_lookup_2 since that might perform
    3012              :                  expression hashtable insertion but this lookup crosses
    3013              :                  a possible may-alias making such insertion conditionally
    3014              :                  invalid.  */
    3015      4586887 :               vn_reference_lookup_1 (vr, &vnresult);
    3016              :               /* Need to restore vr->vuse and vr->hashcode.  */
    3017      4586887 :               vr->vuse = saved_vuse;
    3018      4586887 :               vr->hashcode = saved_hashcode;
    3019      4586887 :               if (vnresult)
    3020              :                 {
    3021       255222 :                   if (TREE_CODE (rhs) == SSA_NAME)
    3022       253866 :                     rhs = SSA_VAL (rhs);
    3023       255222 :                   if (vnresult->result
    3024       255222 :                       && operand_equal_p (vnresult->result, rhs, 0))
    3025        68172 :                     return vnresult;
    3026              :                 }
    3027              :             }
    3028              :         }
    3029              :     }
    3030     22691540 :   else if (*disambiguate_only <= TR_VALUEIZE_AND_DISAMBIGUATE
    3031     20340801 :            && gimple_call_builtin_p (def_stmt, BUILT_IN_NORMAL)
    3032     24826551 :            && gimple_call_num_args (def_stmt) <= 4)
    3033              :     {
    3034              :       /* For builtin calls valueize its arguments and call the
    3035              :          alias oracle again.  Valueization may improve points-to
    3036              :          info of pointers and constify size and position arguments.
    3037              :          Originally this was motivated by PR61034 which has
    3038              :          conditional calls to free falsely clobbering ref because
    3039              :          of imprecise points-to info of the argument.  */
    3040              :       tree oldargs[4];
    3041              :       bool valueized_anything = false;
    3042      5028710 :       for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
    3043              :         {
    3044      3458279 :           oldargs[i] = gimple_call_arg (def_stmt, i);
    3045      3458279 :           tree val = vn_valueize (oldargs[i]);
    3046      3458279 :           if (val != oldargs[i])
    3047              :             {
    3048       136763 :               gimple_call_set_arg (def_stmt, i, val);
    3049       136763 :               valueized_anything = true;
    3050              :             }
    3051              :         }
    3052      1570431 :       if (valueized_anything)
    3053              :         {
    3054       106199 :           bool res = call_may_clobber_ref_p_1 (as_a <gcall *> (def_stmt),
    3055              :                                                ref, data->tbaa_p);
    3056       487587 :           for (unsigned i = 0; i < gimple_call_num_args (def_stmt); ++i)
    3057       275189 :             gimple_call_set_arg (def_stmt, i, oldargs[i]);
    3058       106199 :           if (!res)
    3059              :             {
    3060        30104 :               *disambiguate_only = TR_VALUEIZE_AND_DISAMBIGUATE;
    3061        30104 :               return NULL;
    3062              :             }
    3063              :         }
    3064              :     }
    3065              : 
    3066     36769214 :   if (*disambiguate_only > TR_TRANSLATE)
    3067              :     return (void *)-1;
    3068              : 
    3069              :   /* If we cannot constrain the size of the reference we cannot
    3070              :      test if anything kills it.  */
    3071     24505670 :   if (!ref->max_size_known_p ())
    3072              :     return (void *)-1;
    3073              : 
    3074     24063969 :   poly_int64 offset = ref->offset;
    3075     24063969 :   poly_int64 maxsize = ref->max_size;
    3076              : 
    3077              :   /* def_stmt may-defs *ref.  See if we can derive a value for *ref
    3078              :      from that definition.
    3079              :      1) Memset.  */
    3080     24063969 :   if (is_gimple_reg_type (vr->type)
    3081     24054579 :       && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET)
    3082     23963629 :           || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMSET_CHK))
    3083        91484 :       && (integer_zerop (gimple_call_arg (def_stmt, 1))
    3084        33649 :           || ((TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST
    3085         8948 :                || (INTEGRAL_TYPE_P (vr->type) && known_eq (ref->size, 8)))
    3086              :               && CHAR_BIT == 8
    3087              :               && BITS_PER_UNIT == 8
    3088              :               && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
    3089        32316 :               && offset.is_constant (&offseti)
    3090        32316 :               && ref->size.is_constant (&sizei)
    3091        32316 :               && (offseti % BITS_PER_UNIT == 0
    3092          183 :                   || TREE_CODE (gimple_call_arg (def_stmt, 1)) == INTEGER_CST)))
    3093        90151 :       && (poly_int_tree_p (gimple_call_arg (def_stmt, 2))
    3094        36657 :           || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
    3095        36657 :               && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)))))
    3096     24118047 :       && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
    3097        30631 :           || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME))
    3098              :     {
    3099        54037 :       tree base2;
    3100        54037 :       poly_int64 offset2, size2, maxsize2;
    3101        54037 :       bool reverse;
    3102        54037 :       tree ref2 = gimple_call_arg (def_stmt, 0);
    3103        54037 :       if (TREE_CODE (ref2) == SSA_NAME)
    3104              :         {
    3105        30590 :           ref2 = SSA_VAL (ref2);
    3106        30590 :           if (TREE_CODE (ref2) == SSA_NAME
    3107        30590 :               && (TREE_CODE (base) != MEM_REF
    3108        19198 :                   || TREE_OPERAND (base, 0) != ref2))
    3109              :             {
    3110        24143 :               gimple *def_stmt = SSA_NAME_DEF_STMT (ref2);
    3111        24143 :               if (gimple_assign_single_p (def_stmt)
    3112        24143 :                   && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
    3113          829 :                 ref2 = gimple_assign_rhs1 (def_stmt);
    3114              :             }
    3115              :         }
    3116        54037 :       if (TREE_CODE (ref2) == ADDR_EXPR)
    3117              :         {
    3118        27201 :           ref2 = TREE_OPERAND (ref2, 0);
    3119        27201 :           base2 = get_ref_base_and_extent (ref2, &offset2, &size2, &maxsize2,
    3120              :                                            &reverse);
    3121        27201 :           if (!known_size_p (maxsize2)
    3122        27161 :               || !known_eq (maxsize2, size2)
    3123        54294 :               || !operand_equal_p (base, base2, OEP_ADDRESS_OF))
    3124        53673 :             return (void *)-1;
    3125              :         }
    3126        26836 :       else if (TREE_CODE (ref2) == SSA_NAME)
    3127              :         {
    3128        26836 :           poly_int64 soff;
    3129        26836 :           if (TREE_CODE (base) != MEM_REF
    3130        45421 :               || !(mem_ref_offset (base)
    3131        45421 :                    << LOG2_BITS_PER_UNIT).to_shwi (&soff))
    3132        22711 :             return (void *)-1;
    3133        18585 :           offset += soff;
    3134        18585 :           offset2 = 0;
    3135        18585 :           if (TREE_OPERAND (base, 0) != ref2)
    3136              :             {
    3137        15063 :               gimple *def = SSA_NAME_DEF_STMT (ref2);
    3138        15063 :               if (is_gimple_assign (def)
    3139        13624 :                   && gimple_assign_rhs_code (def) == POINTER_PLUS_EXPR
    3140        11773 :                   && gimple_assign_rhs1 (def) == TREE_OPERAND (base, 0)
    3141        15697 :                   && poly_int_tree_p (gimple_assign_rhs2 (def)))
    3142              :                 {
    3143          603 :                   tree rhs2 = gimple_assign_rhs2 (def);
    3144          603 :                   if (!(poly_offset_int::from (wi::to_poly_wide (rhs2),
    3145              :                                                SIGNED)
    3146          603 :                         << LOG2_BITS_PER_UNIT).to_shwi (&offset2))
    3147              :                     return (void *)-1;
    3148          603 :                   ref2 = gimple_assign_rhs1 (def);
    3149          603 :                   if (TREE_CODE (ref2) == SSA_NAME)
    3150          603 :                     ref2 = SSA_VAL (ref2);
    3151              :                 }
    3152              :               else
    3153              :                 return (void *)-1;
    3154              :             }
    3155              :         }
    3156              :       else
    3157              :         return (void *)-1;
    3158        27325 :       tree len = gimple_call_arg (def_stmt, 2);
    3159        27325 :       HOST_WIDE_INT leni, offset2i;
    3160        27325 :       if (TREE_CODE (len) == SSA_NAME)
    3161          322 :         len = SSA_VAL (len);
    3162              :       /* Sometimes the above trickery is smarter than alias analysis.  Take
    3163              :          advantage of that.  */
    3164        27325 :       if (!ranges_maybe_overlap_p (offset, maxsize, offset2,
    3165        54650 :                                    (wi::to_poly_offset (len)
    3166        27325 :                                     << LOG2_BITS_PER_UNIT)))
    3167              :         return NULL;
    3168        54592 :       if (data->partial_defs.is_empty ()
    3169        27267 :           && known_subrange_p (offset, maxsize, offset2,
    3170        27267 :                                wi::to_poly_offset (len) << LOG2_BITS_PER_UNIT))
    3171              :         {
    3172        26750 :           tree val;
    3173        26750 :           if (integer_zerop (gimple_call_arg (def_stmt, 1)))
    3174        21843 :             val = build_zero_cst (vr->type);
    3175         4907 :           else if (INTEGRAL_TYPE_P (vr->type)
    3176         3863 :                    && known_eq (ref->size, 8)
    3177         7893 :                    && offseti % BITS_PER_UNIT == 0)
    3178              :             {
    3179         2986 :               gimple_match_op res_op (gimple_match_cond::UNCOND, NOP_EXPR,
    3180         2986 :                                       vr->type, gimple_call_arg (def_stmt, 1));
    3181         2986 :               val = vn_nary_build_or_lookup (&res_op);
    3182         2986 :               if (!val
    3183         2986 :                   || (TREE_CODE (val) == SSA_NAME
    3184          616 :                       && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
    3185            0 :                 return (void *)-1;
    3186              :             }
    3187              :           else
    3188              :             {
    3189         1921 :               unsigned buflen
    3190         1921 :                 = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (vr->type)) + 1;
    3191         1921 :               if (INTEGRAL_TYPE_P (vr->type)
    3192         1921 :                   && TYPE_MODE (vr->type) != BLKmode)
    3193         1752 :                 buflen = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (vr->type)) + 1;
    3194         1921 :               unsigned char *buf = XALLOCAVEC (unsigned char, buflen);
    3195         1921 :               memset (buf, TREE_INT_CST_LOW (gimple_call_arg (def_stmt, 1)),
    3196              :                       buflen);
    3197         1921 :               if (BYTES_BIG_ENDIAN)
    3198              :                 {
    3199              :                   unsigned int amnt
    3200              :                     = (((unsigned HOST_WIDE_INT) offseti + sizei)
    3201              :                        % BITS_PER_UNIT);
    3202              :                   if (amnt)
    3203              :                     {
    3204              :                       shift_bytes_in_array_right (buf, buflen,
    3205              :                                                   BITS_PER_UNIT - amnt);
    3206              :                       buf++;
    3207              :                       buflen--;
    3208              :                     }
    3209              :                 }
    3210         1921 :               else if (offseti % BITS_PER_UNIT != 0)
    3211              :                 {
    3212          121 :                   unsigned int amnt
    3213              :                     = BITS_PER_UNIT - ((unsigned HOST_WIDE_INT) offseti
    3214          121 :                                        % BITS_PER_UNIT);
    3215          121 :                   shift_bytes_in_array_left (buf, buflen, amnt);
    3216          121 :                   buf++;
    3217          121 :                   buflen--;
    3218              :                 }
    3219         1921 :               val = native_interpret_expr (vr->type, buf, buflen);
    3220         1921 :               if (!val)
    3221              :                 return (void *)-1;
    3222              :             }
    3223        26750 :           return data->finish (0, 0, val);
    3224              :         }
    3225              :       /* For now handle clearing memory with partial defs.  */
    3226          575 :       else if (known_eq (ref->size, maxsize)
    3227          498 :                && integer_zerop (gimple_call_arg (def_stmt, 1))
    3228          215 :                && tree_fits_poly_int64_p (len)
    3229          211 :                && tree_to_poly_int64 (len).is_constant (&leni)
    3230          211 :                && leni <= INTTYPE_MAXIMUM (HOST_WIDE_INT) / BITS_PER_UNIT
    3231          211 :                && offset.is_constant (&offseti)
    3232          211 :                && offset2.is_constant (&offset2i)
    3233          211 :                && maxsize.is_constant (&maxsizei)
    3234          575 :                && ranges_known_overlap_p (offseti, maxsizei, offset2i,
    3235          575 :                                           leni << LOG2_BITS_PER_UNIT))
    3236              :         {
    3237          211 :           pd_data pd;
    3238          211 :           pd.rhs = build_constructor (NULL_TREE, NULL);
    3239          211 :           pd.rhs_off = 0;
    3240          211 :           pd.offset = offset2i;
    3241          211 :           pd.size = leni << LOG2_BITS_PER_UNIT;
    3242          211 :           return data->push_partial_def (pd, 0, 0, offseti, maxsizei);
    3243              :         }
    3244              :     }
    3245              : 
    3246              :   /* 2) Assignment from an empty CONSTRUCTOR.  */
    3247     24009932 :   else if (is_gimple_reg_type (vr->type)
    3248     24000542 :            && gimple_assign_single_p (def_stmt)
    3249      7951657 :            && gimple_assign_rhs_code (def_stmt) == CONSTRUCTOR
    3250      2014296 :            && CONSTRUCTOR_NELTS (gimple_assign_rhs1 (def_stmt)) == 0
    3251     26024228 :            && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt)))
    3252              :     {
    3253      2014264 :       tree base2;
    3254      2014264 :       poly_int64 offset2, size2, maxsize2;
    3255      2014264 :       HOST_WIDE_INT offset2i, size2i;
    3256      2014264 :       gcc_assert (lhs_ref_ok);
    3257      2014264 :       base2 = ao_ref_base (&lhs_ref);
    3258      2014264 :       offset2 = lhs_ref.offset;
    3259      2014264 :       size2 = lhs_ref.size;
    3260      2014264 :       maxsize2 = lhs_ref.max_size;
    3261      2014264 :       if (known_size_p (maxsize2)
    3262      2014218 :           && known_eq (maxsize2, size2)
    3263      4028430 :           && adjust_offsets_for_equal_base_address (base, &offset,
    3264              :                                                     base2, &offset2))
    3265              :         {
    3266      1983519 :           if (data->partial_defs.is_empty ()
    3267      1980011 :               && known_subrange_p (offset, maxsize, offset2, size2))
    3268              :             {
    3269              :               /* While technically undefined behavior do not optimize
    3270              :                  a full read from a clobber.  */
    3271      1979219 :               if (gimple_clobber_p (def_stmt))
    3272      1983469 :                 return (void *)-1;
    3273      1022807 :               tree val = build_zero_cst (vr->type);
    3274      1022807 :               return data->finish (ao_ref_alias_set (&lhs_ref),
    3275      1022807 :                                    ao_ref_base_alias_set (&lhs_ref), val);
    3276              :             }
    3277         4300 :           else if (known_eq (ref->size, maxsize)
    3278         4250 :                    && maxsize.is_constant (&maxsizei)
    3279         4250 :                    && offset.is_constant (&offseti)
    3280         4250 :                    && offset2.is_constant (&offset2i)
    3281         4250 :                    && size2.is_constant (&size2i)
    3282         4300 :                    && ranges_known_overlap_p (offseti, maxsizei,
    3283              :                                               offset2i, size2i))
    3284              :             {
    3285              :               /* Let clobbers be consumed by the partial-def tracker
    3286              :                  which can choose to ignore them if they are shadowed
    3287              :                  by a later def.  */
    3288         4250 :               pd_data pd;
    3289         4250 :               pd.rhs = gimple_assign_rhs1 (def_stmt);
    3290         4250 :               pd.rhs_off = 0;
    3291         4250 :               pd.offset = offset2i;
    3292         4250 :               pd.size = size2i;
    3293         4250 :               return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
    3294              :                                              ao_ref_base_alias_set (&lhs_ref),
    3295              :                                              offseti, maxsizei);
    3296              :             }
    3297              :         }
    3298              :     }
    3299              : 
    3300              :   /* 3) Assignment from a constant.  We can use folds native encode/interpret
    3301              :      routines to extract the assigned bits.  */
    3302     21995668 :   else if (known_eq (ref->size, maxsize)
    3303     21457878 :            && is_gimple_reg_type (vr->type)
    3304     21448488 :            && !reverse_storage_order_for_component_p (vr->operands)
    3305     21445650 :            && !contains_storage_order_barrier_p (vr->operands)
    3306     21445650 :            && gimple_assign_single_p (def_stmt)
    3307      5605407 :            && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
    3308              :            && CHAR_BIT == 8
    3309              :            && BITS_PER_UNIT == 8
    3310              :            && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
    3311              :            /* native_encode and native_decode operate on arrays of bytes
    3312              :               and so fundamentally need a compile-time size and offset.  */
    3313      5601692 :            && maxsize.is_constant (&maxsizei)
    3314      5601692 :            && offset.is_constant (&offseti)
    3315     27597360 :            && (is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt))
    3316      4726434 :                || (TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME
    3317      1921050 :                    && is_gimple_min_invariant (SSA_VAL (gimple_assign_rhs1 (def_stmt))))))
    3318              :     {
    3319       892504 :       tree lhs = gimple_assign_lhs (def_stmt);
    3320       892504 :       tree base2;
    3321       892504 :       poly_int64 offset2, size2, maxsize2;
    3322       892504 :       HOST_WIDE_INT offset2i, size2i;
    3323       892504 :       bool reverse;
    3324       892504 :       gcc_assert (lhs_ref_ok);
    3325       892504 :       base2 = ao_ref_base (&lhs_ref);
    3326       892504 :       offset2 = lhs_ref.offset;
    3327       892504 :       size2 = lhs_ref.size;
    3328       892504 :       maxsize2 = lhs_ref.max_size;
    3329       892504 :       reverse = reverse_storage_order_for_component_p (lhs);
    3330       892504 :       if (base2
    3331       892504 :           && !reverse
    3332       891676 :           && !storage_order_barrier_p (lhs)
    3333       891676 :           && known_eq (maxsize2, size2)
    3334       858510 :           && adjust_offsets_for_equal_base_address (base, &offset,
    3335              :                                                     base2, &offset2)
    3336        86700 :           && offset.is_constant (&offseti)
    3337        86700 :           && offset2.is_constant (&offset2i)
    3338       892504 :           && size2.is_constant (&size2i))
    3339              :         {
    3340        86700 :           if (data->partial_defs.is_empty ()
    3341        69023 :               && (maxsizei
    3342        69023 :                   <= (HOST_WIDE_INT) vn_walk_cb_data::bufsize * BITS_PER_UNIT)
    3343        69004 :               && known_subrange_p (offseti, maxsizei, offset2, size2))
    3344              :             {
    3345              :               /* We support up to 512-bit values (for V8DFmode).  An
    3346              :                  unaligned access can need one more byte from the
    3347              :                  encoding, plus one for the shift carry-over.  */
    3348        45236 :               unsigned char buffer[vn_walk_cb_data::bufsize + 2];
    3349        45236 :               int len;
    3350              : 
    3351        45236 :               tree rhs = gimple_assign_rhs1 (def_stmt);
    3352        45236 :               if (TREE_CODE (rhs) == SSA_NAME)
    3353         1781 :                 rhs = SSA_VAL (rhs);
    3354        90472 :               len = native_encode_expr (rhs,
    3355              :                                         buffer, sizeof (buffer) - 1,
    3356        45236 :                                         (offseti - offset2i) / BITS_PER_UNIT);
    3357        45236 :               if (len > 0
    3358        42214 :                   && (len * BITS_PER_UNIT
    3359        42214 :                       >= ((offseti - offset2i) % BITS_PER_UNIT) + maxsizei))
    3360              :                 {
    3361        42214 :                   tree type = vr->type;
    3362        42214 :                   unsigned char *buf = buffer;
    3363        42214 :                   unsigned int amnt = 0;
    3364              :                   /* Make sure to interpret in a type that has a range
    3365              :                      covering the whole access size.  */
    3366        42214 :                   if (INTEGRAL_TYPE_P (vr->type)
    3367        42214 :                       && maxsizei != TYPE_PRECISION (vr->type))
    3368              :                     {
    3369          980 :                       bool uns = TYPE_UNSIGNED (type);
    3370          979 :                       if (BITINT_TYPE_P (vr->type)
    3371          981 :                           && maxsizei > MAX_FIXED_MODE_SIZE)
    3372            1 :                         type = build_bitint_type (maxsizei, uns);
    3373              :                       else
    3374          979 :                         type = build_nonstandard_integer_type (maxsizei, uns);
    3375              :                     }
    3376        42214 :                   if (BYTES_BIG_ENDIAN)
    3377              :                     {
    3378              :                       /* For big-endian native_encode_expr stored the rhs
    3379              :                          such that the LSB of it is the LSB of buffer[len - 1].
    3380              :                          That bit is stored into memory at position
    3381              :                          offset2 + size2 - 1, i.e. in byte
    3382              :                          base + (offset2 + size2 - 1) / BITS_PER_UNIT.
    3383              :                          E.g. for offset2 1 and size2 14, rhs -1 and memory
    3384              :                          previously cleared that is:
    3385              :                          0        1
    3386              :                          01111111|11111110
    3387              :                          Now, if we want to extract offset 2 and size 12 from
    3388              :                          it using native_interpret_expr (which actually works
    3389              :                          for integral bitfield types in terms of byte size of
    3390              :                          the mode), the native_encode_expr stored the value
    3391              :                          into buffer as
    3392              :                          XX111111|11111111
    3393              :                          and returned len 2 (the X bits are outside of
    3394              :                          precision).
    3395              :                          Let sz be maxsize / BITS_PER_UNIT if not extracting
    3396              :                          a bitfield, and GET_MODE_SIZE otherwise.
    3397              :                          We need to align the LSB of the value we want to
    3398              :                          extract as the LSB of buf[sz - 1].
    3399              :                          The LSB from memory we need to read is at position
    3400              :                          offset + maxsize - 1.  */
    3401              :                       HOST_WIDE_INT sz = maxsizei / BITS_PER_UNIT;
    3402              :                       if (INTEGRAL_TYPE_P (type))
    3403              :                         {
    3404              :                           if (TYPE_MODE (type) != BLKmode)
    3405              :                             sz = GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (type));
    3406              :                           else
    3407              :                             sz = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (type));
    3408              :                         }
    3409              :                       amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
    3410              :                               - offseti - maxsizei) % BITS_PER_UNIT;
    3411              :                       if (amnt)
    3412              :                         shift_bytes_in_array_right (buffer, len, amnt);
    3413              :                       amnt = ((unsigned HOST_WIDE_INT) offset2i + size2i
    3414              :                               - offseti - maxsizei - amnt) / BITS_PER_UNIT;
    3415              :                       if ((unsigned HOST_WIDE_INT) sz + amnt > (unsigned) len)
    3416              :                         len = 0;
    3417              :                       else
    3418              :                         {
    3419              :                           buf = buffer + len - sz - amnt;
    3420              :                           len -= (buf - buffer);
    3421              :                         }
    3422              :                     }
    3423              :                   else
    3424              :                     {
    3425        42214 :                       amnt = ((unsigned HOST_WIDE_INT) offset2i
    3426        42214 :                               - offseti) % BITS_PER_UNIT;
    3427        42214 :                       if (amnt)
    3428              :                         {
    3429          392 :                           buffer[len] = 0;
    3430          392 :                           shift_bytes_in_array_left (buffer, len + 1, amnt);
    3431          392 :                           buf = buffer + 1;
    3432              :                         }
    3433              :                     }
    3434        42214 :                   tree val = native_interpret_expr (type, buf, len);
    3435              :                   /* If we chop off bits because the types precision doesn't
    3436              :                      match the memory access size this is ok when optimizing
    3437              :                      reads but not when called from the DSE code during
    3438              :                      elimination.  */
    3439        42214 :                   if (val
    3440        42212 :                       && type != vr->type)
    3441              :                     {
    3442          980 :                       if (! int_fits_type_p (val, vr->type))
    3443              :                         val = NULL_TREE;
    3444              :                       else
    3445          980 :                         val = fold_convert (vr->type, val);
    3446              :                     }
    3447              : 
    3448        42212 :                   if (val)
    3449        42212 :                     return data->finish (ao_ref_alias_set (&lhs_ref),
    3450        42212 :                                          ao_ref_base_alias_set (&lhs_ref), val);
    3451              :                 }
    3452              :             }
    3453        41464 :           else if (ranges_known_overlap_p (offseti, maxsizei, offset2i,
    3454              :                                            size2i))
    3455              :             {
    3456        41464 :               pd_data pd;
    3457        41464 :               tree rhs = gimple_assign_rhs1 (def_stmt);
    3458        41464 :               if (TREE_CODE (rhs) == SSA_NAME)
    3459         2598 :                 rhs = SSA_VAL (rhs);
    3460        41464 :               pd.rhs = rhs;
    3461        41464 :               pd.rhs_off = 0;
    3462        41464 :               pd.offset = offset2i;
    3463        41464 :               pd.size = size2i;
    3464        41464 :               return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
    3465              :                                              ao_ref_base_alias_set (&lhs_ref),
    3466              :                                              offseti, maxsizei);
    3467              :             }
    3468              :         }
    3469              :     }
    3470              : 
    3471              :   /* 4) Assignment from an SSA name which definition we may be able
    3472              :      to access pieces from or we can combine to a larger entity.  */
    3473     21103164 :   else if (known_eq (ref->size, maxsize)
    3474     20565374 :            && is_gimple_reg_type (vr->type)
    3475     20555984 :            && !reverse_storage_order_for_component_p (vr->operands)
    3476     20553146 :            && !contains_storage_order_barrier_p (vr->operands)
    3477     20553146 :            && gimple_assign_single_p (def_stmt)
    3478      4712903 :            && !TREE_THIS_VOLATILE (gimple_assign_lhs (def_stmt))
    3479     25812352 :            && TREE_CODE (gimple_assign_rhs1 (def_stmt)) == SSA_NAME)
    3480              :     {
    3481      1903804 :       tree lhs = gimple_assign_lhs (def_stmt);
    3482      1903804 :       tree base2;
    3483      1903804 :       poly_int64 offset2, size2, maxsize2;
    3484      1903804 :       HOST_WIDE_INT offset2i, size2i, offseti;
    3485      1903804 :       bool reverse;
    3486      1903804 :       gcc_assert (lhs_ref_ok);
    3487      1903804 :       base2 = ao_ref_base (&lhs_ref);
    3488      1903804 :       offset2 = lhs_ref.offset;
    3489      1903804 :       size2 = lhs_ref.size;
    3490      1903804 :       maxsize2 = lhs_ref.max_size;
    3491      1903804 :       reverse = reverse_storage_order_for_component_p (lhs);
    3492      1903804 :       tree def_rhs = gimple_assign_rhs1 (def_stmt);
    3493      1903804 :       if (!reverse
    3494      1903592 :           && !storage_order_barrier_p (lhs)
    3495      1903592 :           && known_size_p (maxsize2)
    3496      1876802 :           && known_eq (maxsize2, size2)
    3497      3661944 :           && adjust_offsets_for_equal_base_address (base, &offset,
    3498              :                                                     base2, &offset2))
    3499              :         {
    3500        87147 :           if (data->partial_defs.is_empty ()
    3501        80671 :               && known_subrange_p (offset, maxsize, offset2, size2)
    3502              :               /* ???  We can't handle bitfield precision extracts without
    3503              :                  either using an alternate type for the BIT_FIELD_REF and
    3504              :                  then doing a conversion or possibly adjusting the offset
    3505              :                  according to endianness.  */
    3506        56160 :               && (! INTEGRAL_TYPE_P (vr->type)
    3507        42088 :                   || known_eq (ref->size, TYPE_PRECISION (vr->type)))
    3508        98582 :               && multiple_p (ref->size, BITS_PER_UNIT))
    3509              :             {
    3510        47582 :               tree val = NULL_TREE;
    3511        95158 :               if (! INTEGRAL_TYPE_P (TREE_TYPE (def_rhs))
    3512        52520 :                   || type_has_mode_precision_p (TREE_TYPE (def_rhs)))
    3513              :                 {
    3514        92682 :                   gimple_match_op op (gimple_match_cond::UNCOND,
    3515        46341 :                                       BIT_FIELD_REF, vr->type,
    3516              :                                       SSA_VAL (def_rhs),
    3517              :                                       bitsize_int (ref->size),
    3518        46341 :                                       bitsize_int (offset - offset2));
    3519        46341 :                   val = vn_nary_build_or_lookup (&op);
    3520              :                 }
    3521         1241 :               else if (known_eq (ref->size, size2))
    3522              :                 {
    3523         1142 :                   gimple_match_op op (gimple_match_cond::UNCOND,
    3524         1142 :                                       VIEW_CONVERT_EXPR, vr->type,
    3525         1142 :                                       SSA_VAL (def_rhs));
    3526         1142 :                   val = vn_nary_build_or_lookup (&op);
    3527              :                 }
    3528        47483 :               if (val
    3529        47483 :                   && (TREE_CODE (val) != SSA_NAME
    3530        46689 :                       || ! SSA_NAME_OCCURS_IN_ABNORMAL_PHI (val)))
    3531        47464 :                 return data->finish (ao_ref_alias_set (&lhs_ref),
    3532        87029 :                                      ao_ref_base_alias_set (&lhs_ref), val);
    3533              :             }
    3534        39565 :           else if (maxsize.is_constant (&maxsizei)
    3535        39565 :                    && offset.is_constant (&offseti)
    3536        39565 :                    && offset2.is_constant (&offset2i)
    3537        39565 :                    && size2.is_constant (&size2i)
    3538        39565 :                    && ranges_known_overlap_p (offset, maxsize, offset2, size2))
    3539              :             {
    3540        39565 :               pd_data pd;
    3541        39565 :               pd.rhs = SSA_VAL (def_rhs);
    3542        39565 :               pd.rhs_off = 0;
    3543        39565 :               pd.offset = offset2i;
    3544        39565 :               pd.size = size2i;
    3545        39565 :               return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
    3546              :                                              ao_ref_base_alias_set (&lhs_ref),
    3547              :                                              offseti, maxsizei);
    3548              :             }
    3549              :         }
    3550              :     }
    3551              : 
    3552              :   /* 4b) Assignment done via one of the vectorizer internal store
    3553              :      functions where we may be able to access pieces from or we can
    3554              :      combine to a larger entity.  */
    3555     19199360 :   else if (known_eq (ref->size, maxsize)
    3556     18661570 :            && is_gimple_reg_type (vr->type)
    3557     18652180 :            && !reverse_storage_order_for_component_p (vr->operands)
    3558     18649342 :            && !contains_storage_order_barrier_p (vr->operands)
    3559     18649342 :            && is_gimple_call (def_stmt)
    3560     15014156 :            && gimple_call_internal_p (def_stmt)
    3561     19512463 :            && internal_store_fn_p (gimple_call_internal_fn (def_stmt)))
    3562              :     {
    3563           36 :       gcall *call = as_a <gcall *> (def_stmt);
    3564           36 :       internal_fn fn = gimple_call_internal_fn (call);
    3565              : 
    3566           36 :       tree mask = NULL_TREE, len = NULL_TREE, bias = NULL_TREE;
    3567           36 :       switch (fn)
    3568              :         {
    3569           36 :         case IFN_MASK_STORE:
    3570           36 :           mask = gimple_call_arg (call, internal_fn_mask_index (fn));
    3571           36 :           mask = vn_valueize (mask);
    3572           36 :           if (TREE_CODE (mask) != VECTOR_CST)
    3573           28 :             return (void *)-1;
    3574              :           break;
    3575            0 :         case IFN_LEN_STORE:
    3576            0 :           {
    3577            0 :             int len_index = internal_fn_len_index (fn);
    3578            0 :             len = gimple_call_arg (call, len_index);
    3579            0 :             bias = gimple_call_arg (call, len_index + 1);
    3580            0 :             if (!tree_fits_uhwi_p (len) || !tree_fits_shwi_p (bias))
    3581              :               return (void *) -1;
    3582              :             break;
    3583              :           }
    3584              :         default:
    3585              :           return (void *)-1;
    3586              :         }
    3587           14 :       tree def_rhs = gimple_call_arg (call,
    3588           14 :                                       internal_fn_stored_value_index (fn));
    3589           14 :       def_rhs = vn_valueize (def_rhs);
    3590           14 :       if (TREE_CODE (def_rhs) != VECTOR_CST)
    3591              :         return (void *)-1;
    3592              : 
    3593           14 :       ao_ref_init_from_ptr_and_size (&lhs_ref,
    3594              :                                      vn_valueize (gimple_call_arg (call, 0)),
    3595           14 :                                      TYPE_SIZE_UNIT (TREE_TYPE (def_rhs)));
    3596           14 :       tree base2;
    3597           14 :       poly_int64 offset2, size2, maxsize2;
    3598           14 :       HOST_WIDE_INT offset2i, size2i, offseti;
    3599           14 :       base2 = ao_ref_base (&lhs_ref);
    3600           14 :       offset2 = lhs_ref.offset;
    3601           14 :       size2 = lhs_ref.size;
    3602           14 :       maxsize2 = lhs_ref.max_size;
    3603           14 :       if (known_size_p (maxsize2)
    3604           14 :           && known_eq (maxsize2, size2)
    3605           14 :           && adjust_offsets_for_equal_base_address (base, &offset,
    3606              :                                                     base2, &offset2)
    3607            6 :           && maxsize.is_constant (&maxsizei)
    3608            6 :           && offset.is_constant (&offseti)
    3609            6 :           && offset2.is_constant (&offset2i)
    3610           14 :           && size2.is_constant (&size2i))
    3611              :         {
    3612            6 :           if (!ranges_maybe_overlap_p (offset, maxsize, offset2, size2))
    3613              :             /* Poor-mans disambiguation.  */
    3614              :             return NULL;
    3615            6 :           else if (ranges_known_overlap_p (offset, maxsize, offset2, size2))
    3616              :             {
    3617            6 :               pd_data pd;
    3618            6 :               pd.rhs = def_rhs;
    3619            6 :               tree aa = gimple_call_arg (call, 1);
    3620            6 :               alias_set_type set = get_deref_alias_set (TREE_TYPE (aa));
    3621            6 :               tree vectype = TREE_TYPE (def_rhs);
    3622            6 :               unsigned HOST_WIDE_INT elsz
    3623            6 :                 = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (vectype)));
    3624            6 :               if (mask)
    3625              :                 {
    3626              :                   HOST_WIDE_INT start = 0, length = 0;
    3627              :                   unsigned mask_idx = 0;
    3628           48 :                   do
    3629              :                     {
    3630           48 :                       if (integer_zerop (VECTOR_CST_ELT (mask, mask_idx)))
    3631              :                         {
    3632           24 :                           if (length != 0)
    3633              :                             {
    3634           18 :                               pd.rhs_off = start;
    3635           18 :                               pd.offset = offset2i + start;
    3636           18 :                               pd.size = length;
    3637           18 :                               if (ranges_known_overlap_p
    3638           18 :                                     (offset, maxsize, pd.offset, pd.size))
    3639              :                                 {
    3640            0 :                                   void *res = data->push_partial_def
    3641            0 :                                               (pd, set, set, offseti, maxsizei);
    3642            0 :                                   if (res != NULL)
    3643            6 :                                     return res;
    3644              :                                 }
    3645              :                             }
    3646           24 :                           start = (mask_idx + 1) * elsz;
    3647           24 :                           length = 0;
    3648              :                         }
    3649              :                       else
    3650           24 :                         length += elsz;
    3651           48 :                       mask_idx++;
    3652              :                     }
    3653           48 :                   while (known_lt (mask_idx, TYPE_VECTOR_SUBPARTS (vectype)));
    3654            6 :                   if (length != 0)
    3655              :                     {
    3656            6 :                       pd.rhs_off = start;
    3657            6 :                       pd.offset = offset2i + start;
    3658            6 :                       pd.size = length;
    3659            6 :                       if (ranges_known_overlap_p (offset, maxsize,
    3660              :                                                   pd.offset, pd.size))
    3661            2 :                         return data->push_partial_def (pd, set, set,
    3662            2 :                                                        offseti, maxsizei);
    3663              :                     }
    3664              :                 }
    3665            0 :               else if (fn == IFN_LEN_STORE)
    3666              :                 {
    3667            0 :                   pd.offset = offset2i;
    3668            0 :                   pd.size = (tree_to_uhwi (len)
    3669            0 :                              + -tree_to_shwi (bias)) * BITS_PER_UNIT;
    3670            0 :                   if (BYTES_BIG_ENDIAN)
    3671              :                     pd.rhs_off = pd.size - tree_to_uhwi (TYPE_SIZE (vectype));
    3672              :                   else
    3673            0 :                     pd.rhs_off = 0;
    3674            0 :                   if (ranges_known_overlap_p (offset, maxsize,
    3675              :                                               pd.offset, pd.size))
    3676            0 :                     return data->push_partial_def (pd, set, set,
    3677            0 :                                                    offseti, maxsizei);
    3678              :                 }
    3679              :               else
    3680            0 :                 gcc_unreachable ();
    3681              :               return NULL;
    3682              :             }
    3683              :         }
    3684              :     }
    3685              : 
    3686              :   /* 5) For aggregate copies translate the reference through them if
    3687              :      the copy kills ref.  */
    3688     19199324 :   else if (data->vn_walk_kind == VN_WALKREWRITE
    3689     15511145 :            && gimple_assign_single_p (def_stmt)
    3690      2604823 :            && !gimple_has_volatile_ops (def_stmt)
    3691     21801814 :            && (DECL_P (gimple_assign_rhs1 (def_stmt))
    3692      2000324 :                || TREE_CODE (gimple_assign_rhs1 (def_stmt)) == MEM_REF
    3693      1577690 :                || handled_component_p (gimple_assign_rhs1 (def_stmt))))
    3694              :     {
    3695      2383737 :       tree base2;
    3696      2383737 :       int i, j, k;
    3697      2383737 :       auto_vec<vn_reference_op_s> rhs;
    3698      2383737 :       vn_reference_op_t vro;
    3699      2383737 :       ao_ref r;
    3700              : 
    3701      2383737 :       gcc_assert (lhs_ref_ok);
    3702              : 
    3703              :       /* See if the assignment kills REF.  */
    3704      2383737 :       base2 = ao_ref_base (&lhs_ref);
    3705      2383737 :       if (!lhs_ref.max_size_known_p ()
    3706      2383128 :           || (base != base2
    3707        92366 :               && (TREE_CODE (base) != MEM_REF
    3708        76612 :                   || TREE_CODE (base2) != MEM_REF
    3709        57531 :                   || TREE_OPERAND (base, 0) != TREE_OPERAND (base2, 0)
    3710        19995 :                   || !tree_int_cst_equal (TREE_OPERAND (base, 1),
    3711        19995 :                                           TREE_OPERAND (base2, 1))))
    3712      4692532 :           || !stmt_kills_ref_p (def_stmt, ref))
    3713              :         return (void *)-1;
    3714              : 
    3715              :       /* Find the common base of ref and the lhs.  lhs_ops already
    3716              :          contains valueized operands for the lhs.  */
    3717      1982020 :       poly_int64 extra_off = 0;
    3718      1982020 :       i = vr->operands.length () - 1;
    3719      1982020 :       j = lhs_ops.length () - 1;
    3720              : 
    3721              :       /* The base should be always equal due to the above check.  */
    3722      1982020 :       if (! vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
    3723              :         return (void *)-1;
    3724      1981757 :       i--, j--;
    3725              : 
    3726              :       /* The 2nd component should always exist and be a MEM_REF.  */
    3727      1981757 :       if (!(i >= 0 && j >= 0))
    3728              :         ;
    3729      1981757 :       else if (vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
    3730       944086 :         i--, j--;
    3731      1037671 :       else if (vr->operands[i].opcode == MEM_REF
    3732      1036233 :                && lhs_ops[j].opcode == MEM_REF
    3733      1036233 :                && known_ne (lhs_ops[j].off, -1)
    3734      2073904 :                && known_ne (vr->operands[i].off, -1))
    3735              :         {
    3736      1036233 :           bool found = false;
    3737              :           /* When we ge a mismatch at a MEM_REF that is not the sole component
    3738              :              try finding a match in one of the outer components and continue
    3739              :              stripping there.  This happens when addresses of components get
    3740              :              forwarded into dereferences.  */
    3741      1036233 :           if (i > 0)
    3742              :             {
    3743       114850 :               int temi = i - 1;
    3744       114850 :               poly_int64 tem_extra_off = extra_off + vr->operands[i].off;
    3745       114850 :               while (temi >= 0
    3746       256515 :                      && known_ne (vr->operands[temi].off, -1))
    3747              :                 {
    3748       143313 :                   if (vr->operands[temi].type
    3749       143313 :                       && lhs_ops[j].type
    3750       286626 :                       && (TYPE_MAIN_VARIANT (vr->operands[temi].type)
    3751       143313 :                           == TYPE_MAIN_VARIANT (lhs_ops[j].type)))
    3752              :                     {
    3753         1648 :                       i = temi;
    3754              :                       /* Strip the component that was type matched to
    3755              :                          the MEM_REF.  */
    3756         1648 :                       extra_off = (tem_extra_off
    3757         1648 :                                    + vr->operands[i].off - lhs_ops[j].off);
    3758         1648 :                       i--, j--;
    3759              :                       /* Strip further equal components.  */
    3760         1648 :                       found = true;
    3761         1648 :                       break;
    3762              :                     }
    3763       141665 :                   tem_extra_off += vr->operands[temi].off;
    3764       141665 :                   temi--;
    3765              :                 }
    3766              :             }
    3767      1036233 :           if (!found && j > 0)
    3768              :             {
    3769        34199 :               int temj = j - 1;
    3770        34199 :               poly_int64 tem_extra_off = extra_off - lhs_ops[j].off;
    3771        34199 :               while (temj >= 0
    3772        65309 :                      && known_ne (lhs_ops[temj].off, -1))
    3773              :                 {
    3774        36601 :                   if (vr->operands[i].type
    3775        36601 :                       && lhs_ops[temj].type
    3776        73202 :                       && (TYPE_MAIN_VARIANT (vr->operands[i].type)
    3777        36601 :                           == TYPE_MAIN_VARIANT (lhs_ops[temj].type)))
    3778              :                     {
    3779         5491 :                       j = temj;
    3780              :                       /* Strip the component that was type matched to
    3781              :                          the MEM_REF.  */
    3782         5491 :                       extra_off = (tem_extra_off
    3783         5491 :                                    + vr->operands[i].off - lhs_ops[j].off);
    3784         5491 :                       i--, j--;
    3785              :                       /* Strip further equal components.  */
    3786         5491 :                       found = true;
    3787         5491 :                       break;
    3788              :                     }
    3789        31110 :                   tem_extra_off += -lhs_ops[temj].off;
    3790        31110 :                   temj--;
    3791              :                 }
    3792              :             }
    3793              :           /* When we cannot find a common base to reconstruct the full
    3794              :              reference instead try to reduce the lookup to the new
    3795              :              base plus a constant offset.  */
    3796      1036233 :           if (!found)
    3797              :             {
    3798              :               while (j >= 0
    3799      2088814 :                      && known_ne (lhs_ops[j].off, -1))
    3800              :                 {
    3801      1059720 :                   extra_off += -lhs_ops[j].off;
    3802      1059720 :                   j--;
    3803              :                 }
    3804      1029094 :               if (j != -1)
    3805              :                 return (void *)-1;
    3806              :               while (i >= 0
    3807      2192035 :                      && known_ne (vr->operands[i].off, -1))
    3808              :                 {
    3809              :                   /* Punt if the additional ops contain a storage order
    3810              :                      barrier.  */
    3811      1162941 :                   if (vr->operands[i].opcode == VIEW_CONVERT_EXPR
    3812      1162941 :                       && vr->operands[i].reverse)
    3813              :                     break;
    3814      1162941 :                   extra_off += vr->operands[i].off;
    3815      1162941 :                   i--;
    3816              :                 }
    3817      1029094 :               if (i != -1)
    3818              :                 return (void *)-1;
    3819              :               found = true;
    3820              :             }
    3821              :           /* If we did find a match we'd eventually append a MEM_REF
    3822              :              as component.  Don't.  */
    3823              :           if (!found)
    3824              :             return (void *)-1;
    3825              :         }
    3826              :       else
    3827              :         return (void *)-1;
    3828              : 
    3829              :       /* Strip further common components, attempting to consume lhs_ops
    3830              :          in full.  */
    3831      1980338 :       while (j >= 0 && i >= 0
    3832      1980338 :              && vn_reference_op_eq (&vr->operands[i], &lhs_ops[j]))
    3833              :         {
    3834        26632 :           i--;
    3835        26632 :           j--;
    3836              :         }
    3837              : 
    3838              :       /* i now points to the first additional op.
    3839              :          ???  LHS may not be completely contained in VR, one or more
    3840              :          VIEW_CONVERT_EXPRs could be in its way.  We could at least
    3841              :          try handling outermost VIEW_CONVERT_EXPRs.  */
    3842      1953706 :       if (j != -1)
    3843              :         return (void *)-1;
    3844              : 
    3845              :       /* Punt if the additional ops contain a storage order barrier.  */
    3846      3068032 :       for (k = i; k >= 0; k--)
    3847              :         {
    3848      1117391 :           vro = &vr->operands[k];
    3849      1117391 :           if (vro->opcode == VIEW_CONVERT_EXPR && vro->reverse)
    3850              :             return (void *)-1;
    3851              :         }
    3852              : 
    3853              :       /* Now re-write REF to be based on the rhs of the assignment.  */
    3854      1950641 :       tree rhs1 = gimple_assign_rhs1 (def_stmt);
    3855      1950641 :       copy_reference_ops_from_ref (rhs1, &rhs);
    3856              : 
    3857              :       /* When none of the original operands survives the storage order of
    3858              :          the translated reference is the one of the RHS of the copy.  The
    3859              :          operands we folded into a constant offset above may well have
    3860              :          specified a reverse storage order, which is a property of the
    3861              :          component and not of its position, so it is not recoverable from
    3862              :          that offset.  Punt unless both accesses are in natural order.  */
    3863      1950641 :       if (i < 0
    3864      1950641 :           && (reverse_storage_order_for_component_p (vr->operands)
    3865      1027592 :               || reverse_storage_order_for_component_p (rhs)))
    3866              :         return (void *)-1;
    3867              : 
    3868              :       /* Apply an extra offset to the inner MEM_REF of the RHS.  */
    3869      1950550 :       bool force_no_tbaa = false;
    3870      1950550 :       if (maybe_ne (extra_off, 0))
    3871              :         {
    3872       744382 :           if (rhs.length () < 2)
    3873              :             return (void *)-1;
    3874       744382 :           int ix = rhs.length () - 2;
    3875       744382 :           if (rhs[ix].opcode != MEM_REF
    3876       744382 :               || known_eq (rhs[ix].off, -1))
    3877              :             return (void *)-1;
    3878       744363 :           rhs[ix].off += extra_off;
    3879       744363 :           rhs[ix].op0 = int_const_binop (PLUS_EXPR, rhs[ix].op0,
    3880       744363 :                                          build_int_cst (TREE_TYPE (rhs[ix].op0),
    3881              :                                                         extra_off));
    3882              :           /* When we have offsetted the RHS, reading only parts of it,
    3883              :              we can no longer use the original TBAA type, force alias-set
    3884              :              zero.  */
    3885       744363 :           force_no_tbaa = true;
    3886              :         }
    3887              : 
    3888              :       /* Save the operands since we need to use the original ones for
    3889              :          the hash entry we use.  */
    3890      1950531 :       if (!data->saved_operands.exists ())
    3891      1823722 :         data->saved_operands = vr->operands.copy ();
    3892              : 
    3893              :       /* We need to pre-pend vr->operands[0..i] to rhs.  */
    3894      1950531 :       vec<vn_reference_op_s> old = vr->operands;
    3895      5851593 :       if (i + 1 + rhs.length () > vr->operands.length ())
    3896      1144818 :         vr->operands.safe_grow (i + 1 + rhs.length (), true);
    3897              :       else
    3898       805713 :         vr->operands.truncate (i + 1 + rhs.length ());
    3899      7042298 :       FOR_EACH_VEC_ELT (rhs, j, vro)
    3900      5091767 :         vr->operands[i + 1 + j] = *vro;
    3901      1950531 :       valueize_refs (&vr->operands);
    3902      3901062 :       if (old == shared_lookup_references)
    3903      1950531 :         shared_lookup_references = vr->operands;
    3904      1950531 :       vr->hashcode = vn_reference_compute_hash (vr);
    3905              : 
    3906              :       /* Try folding the new reference to a constant.  */
    3907      1950531 :       tree val = fully_constant_vn_reference_p (vr);
    3908      1950531 :       if (val)
    3909              :         {
    3910        22440 :           if (data->partial_defs.is_empty ())
    3911        22423 :             return data->finish (ao_ref_alias_set (&lhs_ref),
    3912        22423 :                                  ao_ref_base_alias_set (&lhs_ref), val);
    3913              :           /* This is the only interesting case for partial-def handling
    3914              :              coming from targets that like to gimplify init-ctors as
    3915              :              aggregate copies from constant data like aarch64 for
    3916              :              PR83518.  */
    3917           17 :           if (maxsize.is_constant (&maxsizei) && known_eq (ref->size, maxsize))
    3918              :             {
    3919           17 :               pd_data pd;
    3920           17 :               pd.rhs = val;
    3921           17 :               pd.rhs_off = 0;
    3922           17 :               pd.offset = 0;
    3923           17 :               pd.size = maxsizei;
    3924           17 :               return data->push_partial_def (pd, ao_ref_alias_set (&lhs_ref),
    3925              :                                              ao_ref_base_alias_set (&lhs_ref),
    3926              :                                              0, maxsizei);
    3927              :             }
    3928              :         }
    3929              : 
    3930              :       /* Continuing with partial defs isn't easily possible here, we
    3931              :          have to find a full def from further lookups from here.  Probably
    3932              :          not worth the special-casing everywhere.  */
    3933      4311828 :       if (!data->partial_defs.is_empty ())
    3934              :         return (void *)-1;
    3935              : 
    3936              :       /* Adjust *ref from the new operands.  */
    3937      1921560 :       ao_ref rhs1_ref;
    3938      1921560 :       ao_ref_init (&rhs1_ref, rhs1);
    3939      3113547 :       if (!ao_ref_init_from_vn_reference (&r,
    3940              :                                           force_no_tbaa ? 0
    3941      1191987 :                                           : ao_ref_alias_set (&rhs1_ref),
    3942              :                                           force_no_tbaa ? 0
    3943      1191987 :                                           : ao_ref_base_alias_set (&rhs1_ref),
    3944              :                                           vr->type, vr->operands))
    3945              :         return (void *)-1;
    3946              :       /* This can happen with bitfields.  */
    3947      1921560 :       if (maybe_ne (ref->size, r.size))
    3948              :         {
    3949              :           /* If the access lacks some subsetting simply apply that by
    3950              :              shortening it.  That in the end can only be successful
    3951              :              if we can pun the lookup result which in turn requires
    3952              :              exact offsets.  */
    3953         1581 :           if (known_eq (r.size, r.max_size)
    3954         1581 :               && known_lt (ref->size, r.size))
    3955         1581 :             r.size = r.max_size = ref->size;
    3956              :           else
    3957              :             return (void *)-1;
    3958              :         }
    3959      1921560 :       *ref = r;
    3960      1921560 :       vr->offset = r.offset;
    3961      1921560 :       vr->max_size = r.max_size;
    3962              : 
    3963              :       /* Do not update last seen VUSE after translating.  */
    3964      1921560 :       data->last_vuse_ptr = NULL;
    3965              :       /* Invalidate the original access path since it now contains
    3966              :          the wrong base.  */
    3967      1921560 :       data->orig_ref.ref = NULL_TREE;
    3968              :       /* Use the alias-set of this LHS for recording an eventual result.  */
    3969      1921560 :       if (data->first_set == -2)
    3970              :         {
    3971      1796412 :           data->first_set = ao_ref_alias_set (&lhs_ref);
    3972      1796412 :           data->first_base_set = ao_ref_base_alias_set (&lhs_ref);
    3973              :         }
    3974              : 
    3975              :       /* Keep looking for the adjusted *REF / VR pair.  */
    3976              :       return NULL;
    3977      2383737 :     }
    3978              : 
    3979              :   /* 6) For memcpy copies translate the reference through them if the copy
    3980              :      kills ref.  But we cannot (easily) do this translation if the memcpy is
    3981              :      a storage order barrier, i.e. is equivalent to a VIEW_CONVERT_EXPR that
    3982              :      can modify the storage order of objects (see storage_order_barrier_p).  */
    3983     16815587 :   else if (data->vn_walk_kind == VN_WALKREWRITE
    3984     13127408 :            && is_gimple_reg_type (vr->type)
    3985              :            /* ???  Handle BCOPY as well.  */
    3986     13118018 :            && (gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY)
    3987     13048990 :                || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMCPY_CHK)
    3988     13048567 :                || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY)
    3989     13047391 :                || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMPCPY_CHK)
    3990     13047149 :                || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE)
    3991     13021130 :                || gimple_call_builtin_p (def_stmt, BUILT_IN_MEMMOVE_CHK))
    3992        97216 :            && (TREE_CODE (gimple_call_arg (def_stmt, 0)) == ADDR_EXPR
    3993        85441 :                || TREE_CODE (gimple_call_arg (def_stmt, 0)) == SSA_NAME)
    3994        97180 :            && (TREE_CODE (gimple_call_arg (def_stmt, 1)) == ADDR_EXPR
    3995        68625 :                || TREE_CODE (gimple_call_arg (def_stmt, 1)) == SSA_NAME)
    3996        97165 :            && (poly_int_tree_p (gimple_call_arg (def_stmt, 2), &copy_size)
    3997        55383 :                || (TREE_CODE (gimple_call_arg (def_stmt, 2)) == SSA_NAME
    3998        55383 :                    && poly_int_tree_p (SSA_VAL (gimple_call_arg (def_stmt, 2)),
    3999              :                                        &copy_size)))
    4000              :            /* Handling this is more complicated, give up for now.  */
    4001     16859978 :            && data->partial_defs.is_empty ())
    4002              :     {
    4003        43698 :       tree lhs, rhs;
    4004        43698 :       ao_ref r;
    4005        43698 :       poly_int64 rhs_offset, lhs_offset;
    4006        43698 :       vn_reference_op_s op;
    4007        43698 :       poly_uint64 mem_offset;
    4008        43698 :       poly_int64 at, byte_maxsize;
    4009              : 
    4010              :       /* Only handle non-variable, addressable refs.  */
    4011        43698 :       if (maybe_ne (ref->size, maxsize)
    4012        43176 :           || !multiple_p (offset, BITS_PER_UNIT, &at)
    4013        86938 :           || !multiple_p (maxsize, BITS_PER_UNIT, &byte_maxsize))
    4014              :         return (void *)-1;
    4015              : 
    4016              :       /* Extract a pointer base and an offset for the destination.  */
    4017        43176 :       lhs = gimple_call_arg (def_stmt, 0);
    4018        43176 :       lhs_offset = 0;
    4019        43176 :       if (TREE_CODE (lhs) == SSA_NAME)
    4020              :         {
    4021        33127 :           lhs = vn_valueize (lhs);
    4022        33127 :           if (TREE_CODE (lhs) == SSA_NAME)
    4023              :             {
    4024        32798 :               gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
    4025        32798 :               if (gimple_assign_single_p (def_stmt)
    4026        32798 :                   && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
    4027         2514 :                 lhs = gimple_assign_rhs1 (def_stmt);
    4028              :             }
    4029              :         }
    4030        43176 :       if (TREE_CODE (lhs) == ADDR_EXPR)
    4031              :         {
    4032        18119 :           if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (lhs)))
    4033        17822 :               && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (lhs))))
    4034              :             return (void *)-1;
    4035        12752 :           tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (lhs, 0),
    4036              :                                                     &lhs_offset);
    4037        12752 :           if (!tem)
    4038              :             return (void *)-1;
    4039        12014 :           if (TREE_CODE (tem) == MEM_REF
    4040        12014 :               && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
    4041              :             {
    4042         1776 :               lhs = TREE_OPERAND (tem, 0);
    4043         1776 :               if (TREE_CODE (lhs) == SSA_NAME)
    4044         1776 :                 lhs = vn_valueize (lhs);
    4045         1776 :               lhs_offset += mem_offset;
    4046              :             }
    4047        10238 :           else if (DECL_P (tem))
    4048        10238 :             lhs = build_fold_addr_expr (tem);
    4049              :           else
    4050              :             return (void *)-1;
    4051              :         }
    4052        42298 :       if (TREE_CODE (lhs) != SSA_NAME
    4053        10240 :           && TREE_CODE (lhs) != ADDR_EXPR)
    4054              :         return (void *)-1;
    4055              : 
    4056              :       /* Extract a pointer base and an offset for the source.  */
    4057        42298 :       rhs = gimple_call_arg (def_stmt, 1);
    4058        42298 :       rhs_offset = 0;
    4059        42298 :       if (TREE_CODE (rhs) == SSA_NAME)
    4060        19854 :         rhs = vn_valueize (rhs);
    4061        42298 :       if (TREE_CODE (rhs) == ADDR_EXPR)
    4062              :         {
    4063        35265 :           if (AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (rhs)))
    4064        24712 :               && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (TREE_TYPE (rhs))))
    4065              :             return (void *)-1;
    4066        24080 :           tree tem = get_addr_base_and_unit_offset (TREE_OPERAND (rhs, 0),
    4067              :                                                     &rhs_offset);
    4068        24080 :           if (!tem)
    4069              :             return (void *)-1;
    4070        24080 :           if (TREE_CODE (tem) == MEM_REF
    4071        24080 :               && poly_int_tree_p (TREE_OPERAND (tem, 1), &mem_offset))
    4072              :             {
    4073            0 :               rhs = TREE_OPERAND (tem, 0);
    4074            0 :               rhs_offset += mem_offset;
    4075              :             }
    4076        24080 :           else if (DECL_P (tem)
    4077        17553 :                    || TREE_CODE (tem) == STRING_CST)
    4078        24080 :             rhs = build_fold_addr_expr (tem);
    4079              :           else
    4080              :             return (void *)-1;
    4081              :         }
    4082        42298 :       if (TREE_CODE (rhs) == SSA_NAME)
    4083        18218 :         rhs = SSA_VAL (rhs);
    4084        24080 :       else if (TREE_CODE (rhs) != ADDR_EXPR)
    4085              :         return (void *)-1;
    4086              : 
    4087              :       /* The bases of the destination and the references have to agree.  */
    4088        42298 :       if (TREE_CODE (base) == MEM_REF)
    4089              :         {
    4090        16106 :           if (TREE_OPERAND (base, 0) != lhs
    4091        16106 :               || !poly_int_tree_p (TREE_OPERAND (base, 1), &mem_offset))
    4092              :             return (void *) -1;
    4093        12839 :           at += mem_offset;
    4094              :         }
    4095        26192 :       else if (!DECL_P (base)
    4096        25199 :                || TREE_CODE (lhs) != ADDR_EXPR
    4097        35250 :                || TREE_OPERAND (lhs, 0) != base)
    4098              :         return (void *)-1;
    4099              : 
    4100              :       /* If the access is completely outside of the memcpy destination
    4101              :          area there is no aliasing.  */
    4102        12839 :       if (!ranges_maybe_overlap_p (lhs_offset, copy_size, at, byte_maxsize))
    4103              :         return NULL;
    4104              :       /* And the access has to be contained within the memcpy destination.  */
    4105        12804 :       if (!known_subrange_p (at, byte_maxsize, lhs_offset, copy_size))
    4106              :         return (void *)-1;
    4107              : 
    4108              :       /* Save the operands since we need to use the original ones for
    4109              :          the hash entry we use.  */
    4110        12187 :       if (!data->saved_operands.exists ())
    4111        11743 :         data->saved_operands = vr->operands.copy ();
    4112              : 
    4113              :       /* Make room for 2 operands in the new reference.  */
    4114        12187 :       if (vr->operands.length () < 2)
    4115              :         {
    4116            0 :           vec<vn_reference_op_s> old = vr->operands;
    4117            0 :           vr->operands.safe_grow_cleared (2, true);
    4118            0 :           if (old == shared_lookup_references)
    4119            0 :             shared_lookup_references = vr->operands;
    4120              :         }
    4121              :       else
    4122        12187 :         vr->operands.truncate (2);
    4123              : 
    4124              :       /* The looked-through reference is a simple MEM_REF.  */
    4125        12187 :       memset (&op, 0, sizeof (op));
    4126        12187 :       op.type = vr->type;
    4127        12187 :       op.opcode = MEM_REF;
    4128        12187 :       op.op0 = build_int_cst (ptr_type_node, at - lhs_offset + rhs_offset);
    4129        12187 :       op.off = at - lhs_offset + rhs_offset;
    4130        12187 :       vr->operands[0] = op;
    4131        12187 :       op.type = TREE_TYPE (rhs);
    4132        12187 :       op.opcode = TREE_CODE (rhs);
    4133        12187 :       op.op0 = rhs;
    4134        12187 :       op.off = -1;
    4135        12187 :       vr->operands[1] = op;
    4136        12187 :       vr->hashcode = vn_reference_compute_hash (vr);
    4137              : 
    4138              :       /* Try folding the new reference to a constant.  */
    4139        12187 :       tree val = fully_constant_vn_reference_p (vr);
    4140        12187 :       if (val)
    4141         3096 :         return data->finish (0, 0, val);
    4142              : 
    4143              :       /* Adjust *ref from the new operands.  */
    4144         9091 :       if (!ao_ref_init_from_vn_reference (&r, 0, 0, vr->type, vr->operands))
    4145              :         return (void *)-1;
    4146              :       /* This can happen with bitfields.  */
    4147         9091 :       if (maybe_ne (ref->size, r.size))
    4148              :         return (void *)-1;
    4149         9091 :       *ref = r;
    4150         9091 :       vr->offset = r.offset;
    4151         9091 :       vr->max_size = r.max_size;
    4152              : 
    4153              :       /* Do not update last seen VUSE after translating.  */
    4154         9091 :       data->last_vuse_ptr = NULL;
    4155              :       /* Invalidate the original access path since it now contains
    4156              :          the wrong base.  */
    4157         9091 :       data->orig_ref.ref = NULL_TREE;
    4158              :       /* Use the alias-set of this stmt for recording an eventual result.  */
    4159         9091 :       if (data->first_set == -2)
    4160              :         {
    4161         8702 :           data->first_set = 0;
    4162         8702 :           data->first_base_set = 0;
    4163              :         }
    4164              : 
    4165              :       /* Keep looking for the adjusted *REF / VR pair.  */
    4166              :       return NULL;
    4167              :     }
    4168              : 
    4169              :   /* Bail out and stop walking.  */
    4170              :   return (void *)-1;
    4171              : }
    4172              : 
    4173              : /* Return true if E is a backedge with respect to our CFG walk order.  */
    4174              : 
    4175              : static bool
    4176    213667999 : vn_is_backedge (edge e, void *)
    4177              : {
    4178              :   /* During PRE elimination and ANTIC compute we no longer have
    4179              :      vn_bb_to_rpo and the region is no longer marked.  */
    4180    213667999 :   if (!vn_bb_to_rpo)
    4181              :     return true;
    4182              : 
    4183    174414581 :   gcc_assert (vn_bb_in_region_flag != 0);
    4184              : 
    4185              :   /* The entry edge is a forward edge.  */
    4186    174414581 :   if (e == vn_entry_edge)
    4187              :     return false;
    4188              :   /* For out-of region edges conservatively return true.  */
    4189    174222059 :   if (!(e->src->flags & vn_bb_in_region_flag))
    4190              :     return true;
    4191              :   /* We should never ask for region exit edges.  */
    4192    174222059 :   gcc_checking_assert (e->dest->flags & vn_bb_in_region_flag);
    4193    174222059 :   return vn_bb_to_rpo[e->dest->index] <= vn_bb_to_rpo[e->src->index];
    4194              : }
    4195              : 
    4196              : /* Return a reference op vector from OP that can be used for
    4197              :    vn_reference_lookup_pieces.  The caller is responsible for releasing
    4198              :    the vector.  */
    4199              : 
    4200              : vec<vn_reference_op_s>
    4201      4844072 : vn_reference_operands_for_lookup (tree op)
    4202              : {
    4203      4844072 :   bool valueized;
    4204      4844072 :   return valueize_shared_reference_ops_from_ref (op, &valueized).copy ();
    4205              : }
    4206              : 
    4207              : /* Lookup a reference operation by it's parts, in the current hash table.
    4208              :    Returns the resulting value number if it exists in the hash table,
    4209              :    NULL_TREE otherwise.  VNRESULT will be filled in with the actual
    4210              :    vn_reference_t stored in the hashtable if something is found.  */
    4211              : 
    4212              : tree
    4213      7952764 : vn_reference_lookup_pieces (tree vuse, alias_set_type set,
    4214              :                             alias_set_type base_set, tree type,
    4215              :                             vec<vn_reference_op_s> operands,
    4216              :                             vn_reference_t *vnresult, vn_lookup_kind kind)
    4217              : {
    4218      7952764 :   struct vn_reference_s vr1;
    4219      7952764 :   vn_reference_t tmp;
    4220      7952764 :   tree cst;
    4221              : 
    4222      7952764 :   if (!vnresult)
    4223            0 :     vnresult = &tmp;
    4224      7952764 :   *vnresult = NULL;
    4225              : 
    4226      7952764 :   vr1.vuse = vuse_ssa_val (vuse);
    4227      7952764 :   shared_lookup_references.truncate (0);
    4228     15905528 :   shared_lookup_references.safe_grow (operands.length (), true);
    4229      7952764 :   memcpy (shared_lookup_references.address (),
    4230      7952764 :           operands.address (),
    4231              :           sizeof (vn_reference_op_s)
    4232      7952764 :           * operands.length ());
    4233      7952764 :   bool valueized_p;
    4234      7952764 :   valueize_refs_1 (&shared_lookup_references, &valueized_p);
    4235      7952764 :   vr1.operands = shared_lookup_references;
    4236      7952764 :   vr1.type = type;
    4237      7952764 :   vr1.set = set;
    4238      7952764 :   vr1.base_set = base_set;
    4239              :   /* We can pretend there's no extra info fed in since the ao_refs offset
    4240              :      and max_size are computed only from the VN reference ops.  */
    4241      7952764 :   vr1.offset = 0;
    4242      7952764 :   vr1.max_size = -1;
    4243      7952764 :   vr1.hashcode = vn_reference_compute_hash (&vr1);
    4244      7952764 :   if ((cst = fully_constant_vn_reference_p (&vr1)))
    4245              :     return cst;
    4246              : 
    4247      7932424 :   vn_reference_lookup_1 (&vr1, vnresult);
    4248      7932424 :   if (!*vnresult
    4249      3066475 :       && kind != VN_NOWALK
    4250      3066475 :       && vr1.vuse)
    4251              :     {
    4252      3036069 :       ao_ref r;
    4253      3036069 :       unsigned limit = param_sccvn_max_alias_queries_per_access;
    4254      3036069 :       vn_walk_cb_data data (&vr1, NULL_TREE, NULL, kind, true, NULL_TREE,
    4255      3036069 :                             false);
    4256      3036069 :       vec<vn_reference_op_s> ops_for_ref;
    4257      3036069 :       if (!valueized_p)
    4258      2943497 :         ops_for_ref = vr1.operands;
    4259              :       else
    4260              :         {
    4261              :           /* For ao_ref_from_mem we have to ensure only available SSA names
    4262              :              end up in base and the only convenient way to make this work
    4263              :              for PRE is to re-valueize with that in mind.  */
    4264       185144 :           ops_for_ref.create (operands.length ());
    4265       185144 :           ops_for_ref.quick_grow (operands.length ());
    4266        92572 :           memcpy (ops_for_ref.address (),
    4267        92572 :                   operands.address (),
    4268              :                   sizeof (vn_reference_op_s)
    4269        92572 :                   * operands.length ());
    4270        92572 :           valueize_refs_1 (&ops_for_ref, &valueized_p, true);
    4271              :         }
    4272      3036069 :       if (ao_ref_init_from_vn_reference (&r, set, base_set, type,
    4273              :                                          ops_for_ref))
    4274      2963410 :         *vnresult
    4275      2963410 :           = ((vn_reference_t)
    4276      2963410 :              walk_non_aliased_vuses (&r, vr1.vuse, true, vn_reference_lookup_2,
    4277              :                                      vn_reference_lookup_3, vn_is_backedge,
    4278              :                                      vuse_valueize, limit, &data));
    4279      6072138 :       if (ops_for_ref != shared_lookup_references)
    4280        92572 :         ops_for_ref.release ();
    4281      6072138 :       gcc_checking_assert (vr1.operands == shared_lookup_references);
    4282      3036069 :       if (*vnresult
    4283       426939 :           && data.same_val
    4284      3036069 :           && (!(*vnresult)->result
    4285            0 :               || !operand_equal_p ((*vnresult)->result, data.same_val)))
    4286              :         {
    4287            0 :           *vnresult = NULL;
    4288            0 :           return NULL_TREE;
    4289              :         }
    4290      3036069 :     }
    4291              : 
    4292      7932424 :   if (*vnresult)
    4293      5292888 :      return (*vnresult)->result;
    4294              : 
    4295              :   return NULL_TREE;
    4296              : }
    4297              : 
    4298              : /* When OPERANDS is an ADDR_EXPR that can be possibly expressed as a
    4299              :    POINTER_PLUS_EXPR return true and fill in its operands in OPS.  */
    4300              : 
    4301              : bool
    4302      2231594 : vn_pp_nary_for_addr (const vec<vn_reference_op_s>& operands, tree ops[2])
    4303              : {
    4304      4463188 :   gcc_assert (operands[0].opcode == ADDR_EXPR
    4305              :               && operands.last ().opcode == SSA_NAME);
    4306              :   poly_int64 off = 0;
    4307              :   vn_reference_op_t vro;
    4308              :   unsigned i;
    4309      7211569 :   for (i = 1; operands.iterate (i, &vro); ++i)
    4310              :     {
    4311      7211569 :       if (vro->opcode == SSA_NAME)
    4312              :         break;
    4313      5031268 :       else if (known_eq (vro->off, -1))
    4314              :         break;
    4315      4979975 :       off += vro->off;
    4316              :     }
    4317      2231594 :   if (i == operands.length () - 1
    4318      2180301 :       && maybe_ne (off, 0)
    4319              :       /* Make sure we the offset we accumulated in a 64bit int
    4320              :          fits the address computation carried out in target
    4321              :          offset precision.  */
    4322      3665172 :       && (off.coeffs[0]
    4323      1433578 :           == sext_hwi (off.coeffs[0], TYPE_PRECISION (sizetype))))
    4324              :     {
    4325      1433038 :       gcc_assert (operands[i-1].opcode == MEM_REF);
    4326      1433038 :       ops[0] = operands[i].op0;
    4327      1433038 :       ops[1] = wide_int_to_tree (sizetype, off);
    4328      1433038 :       return true;
    4329              :     }
    4330              :   return false;
    4331              : }
    4332              : 
    4333              : /* Lookup OP in the current hash table, and return the resulting value
    4334              :    number if it exists in the hash table.  Return NULL_TREE if it does
    4335              :    not exist in the hash table or if the result field of the structure
    4336              :    was NULL..  VNRESULT will be filled in with the vn_reference_t
    4337              :    stored in the hashtable if one exists.  When TBAA_P is false assume
    4338              :    we are looking up a store and treat it as having alias-set zero.
    4339              :    *LAST_VUSE_PTR will be updated with the VUSE the value lookup succeeded.
    4340              :    MASK is either NULL_TREE, or can be an INTEGER_CST if the result of the
    4341              :    load is bitwise anded with MASK and so we are only interested in a subset
    4342              :    of the bits and can ignore if the other bits are uninitialized or
    4343              :    not initialized with constants.  When doing redundant store removal
    4344              :    the caller has to set REDUNDANT_STORE_REMOVAL_P.  */
    4345              : 
    4346              : tree
    4347    103605764 : vn_reference_lookup (tree op, tree vuse, vn_lookup_kind kind,
    4348              :                      vn_reference_t *vnresult, bool tbaa_p,
    4349              :                      tree *last_vuse_ptr, tree mask,
    4350              :                      bool redundant_store_removal_p)
    4351              : {
    4352    103605764 :   vec<vn_reference_op_s> operands;
    4353    103605764 :   struct vn_reference_s vr1;
    4354    103605764 :   bool valueized_anything;
    4355              : 
    4356    103605764 :   if (vnresult)
    4357    103211828 :     *vnresult = NULL;
    4358              : 
    4359    103605764 :   vr1.vuse = vuse_ssa_val (vuse);
    4360    207211528 :   vr1.operands = operands
    4361    103605764 :     = valueize_shared_reference_ops_from_ref (op, &valueized_anything);
    4362              : 
    4363              :   /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR.  Avoid doing
    4364              :      this before the pass folding __builtin_object_size had a chance to run.  */
    4365    103605764 :   if ((cfun->curr_properties & PROP_objsz)
    4366     75047708 :       && operands[0].opcode == ADDR_EXPR
    4367    104726777 :       && operands.last ().opcode == SSA_NAME)
    4368              :     {
    4369      1086077 :       tree ops[2];
    4370      1086077 :       if (vn_pp_nary_for_addr (operands, ops))
    4371              :         {
    4372       699040 :           tree res = vn_nary_op_lookup_pieces (2, POINTER_PLUS_EXPR,
    4373       699040 :                                                TREE_TYPE (op), ops, NULL);
    4374       699040 :           if (res)
    4375       699040 :             return res;
    4376       699040 :           return NULL_TREE;
    4377              :         }
    4378              :     }
    4379              : 
    4380    102906724 :   vr1.type = TREE_TYPE (op);
    4381    102906724 :   ao_ref op_ref;
    4382    102906724 :   ao_ref_init (&op_ref, op);
    4383    102906724 :   vr1.set = ao_ref_alias_set (&op_ref);
    4384    102906724 :   vr1.base_set = ao_ref_base_alias_set (&op_ref);
    4385    102906724 :   vr1.offset = 0;
    4386    102906724 :   vr1.max_size = -1;
    4387    102906724 :   vr1.hashcode = vn_reference_compute_hash (&vr1);
    4388    102906724 :   if (mask == NULL_TREE)
    4389    102587439 :     if (tree cst = fully_constant_vn_reference_p (&vr1))
    4390              :       return cst;
    4391              : 
    4392    102890466 :   if (kind != VN_NOWALK && vr1.vuse)
    4393              :     {
    4394     59791567 :       vn_reference_t wvnresult;
    4395     59791567 :       ao_ref r;
    4396     59791567 :       unsigned limit = param_sccvn_max_alias_queries_per_access;
    4397     59791567 :       auto_vec<vn_reference_op_s> ops_for_ref;
    4398     59791567 :       if (valueized_anything)
    4399              :         {
    4400      4765867 :           copy_reference_ops_from_ref (op, &ops_for_ref);
    4401      4765867 :           bool tem;
    4402      4765867 :           valueize_refs_1 (&ops_for_ref, &tem, true);
    4403              :         }
    4404              :       /* Make sure to use a valueized reference if we valueized anything.
    4405              :          Otherwise preserve the full reference for advanced TBAA.  */
    4406     59791567 :       if (!valueized_anything
    4407     59791567 :           || !ao_ref_init_from_vn_reference (&r, vr1.set, vr1.base_set,
    4408              :                                              vr1.type, ops_for_ref))
    4409              :         {
    4410     55025700 :           ao_ref_init (&r, op);
    4411              :           /* Record the extra info we're getting from the full ref.  */
    4412     55025700 :           ao_ref_base (&r);
    4413     55025700 :           vr1.offset = r.offset;
    4414     55025700 :           vr1.max_size = r.max_size;
    4415              :         }
    4416     59791567 :       vn_walk_cb_data data (&vr1, r.ref ? NULL_TREE : op,
    4417              :                             last_vuse_ptr, kind, tbaa_p, mask,
    4418    114817267 :                             redundant_store_removal_p);
    4419              : 
    4420     59791567 :       wvnresult
    4421              :         = ((vn_reference_t)
    4422     59791567 :            walk_non_aliased_vuses (&r, vr1.vuse, tbaa_p, vn_reference_lookup_2,
    4423              :                                    vn_reference_lookup_3, vn_is_backedge,
    4424              :                                    vuse_valueize, limit, &data));
    4425    119583134 :       gcc_checking_assert (vr1.operands == shared_lookup_references);
    4426     59791567 :       if (wvnresult)
    4427              :         {
    4428      8942074 :           gcc_assert (mask == NULL_TREE);
    4429      8942074 :           if (data.same_val
    4430      8942074 :               && (!wvnresult->result
    4431        69450 :                   || !operand_equal_p (wvnresult->result, data.same_val)))
    4432              :             return NULL_TREE;
    4433      8892852 :           if (vnresult)
    4434      8890889 :             *vnresult = wvnresult;
    4435      8892852 :           return wvnresult->result;
    4436              :         }
    4437     50849493 :       else if (mask)
    4438       319285 :         return data.masked_result;
    4439              : 
    4440              :       return NULL_TREE;
    4441     59791567 :     }
    4442              : 
    4443     43098899 :   if (last_vuse_ptr)
    4444      1487445 :     *last_vuse_ptr = vr1.vuse;
    4445     43098899 :   if (mask)
    4446              :     return NULL_TREE;
    4447     43098899 :   return vn_reference_lookup_1 (&vr1, vnresult);
    4448              : }
    4449              : 
    4450              : /* Lookup CALL in the current hash table and return the entry in
    4451              :    *VNRESULT if found.  Populates *VR for the hashtable lookup.  */
    4452              : 
    4453              : void
    4454      9377348 : vn_reference_lookup_call (gcall *call, vn_reference_t *vnresult,
    4455              :                           vn_reference_t vr)
    4456              : {
    4457      9377348 :   if (vnresult)
    4458      9377348 :     *vnresult = NULL;
    4459              : 
    4460      9377348 :   tree vuse = gimple_vuse (call);
    4461              : 
    4462      9377348 :   vr->vuse = vuse ? SSA_VAL (vuse) : NULL_TREE;
    4463      9377348 :   vr->operands = valueize_shared_reference_ops_from_call (call);
    4464      9377348 :   tree lhs = gimple_call_lhs (call);
    4465              :   /* For non-SSA return values the reference ops contain the LHS.  */
    4466      5101239 :   vr->type = ((lhs && TREE_CODE (lhs) == SSA_NAME)
    4467     14028364 :               ? TREE_TYPE (lhs) : NULL_TREE);
    4468      9377348 :   vr->punned = false;
    4469      9377348 :   vr->set = 0;
    4470      9377348 :   vr->base_set = 0;
    4471      9377348 :   vr->offset = 0;
    4472      9377348 :   vr->max_size = -1;
    4473      9377348 :   vr->hashcode = vn_reference_compute_hash (vr);
    4474      9377348 :   vn_reference_lookup_1 (vr, vnresult);
    4475      9377348 : }
    4476              : 
    4477              : /* Insert OP into the current hash table with a value number of RESULT.  */
    4478              : 
    4479              : static void
    4480     76883367 : vn_reference_insert (tree op, tree result, tree vuse, tree vdef)
    4481              : {
    4482     76883367 :   vn_reference_s **slot;
    4483     76883367 :   vn_reference_t vr1;
    4484     76883367 :   bool tem;
    4485              : 
    4486     76883367 :   vec<vn_reference_op_s> operands
    4487     76883367 :     = valueize_shared_reference_ops_from_ref (op, &tem);
    4488              :   /* Handle &MEM[ptr + 5].b[1].c as POINTER_PLUS_EXPR.  Avoid doing this
    4489              :      before the pass folding __builtin_object_size had a chance to run.  */
    4490     76883367 :   if ((cfun->curr_properties & PROP_objsz)
    4491     57593894 :       && operands[0].opcode == ADDR_EXPR
    4492     77801260 :       && operands.last ().opcode == SSA_NAME)
    4493              :     {
    4494       885395 :       tree ops[2];
    4495       885395 :       if (vn_pp_nary_for_addr (operands, ops))
    4496              :         {
    4497       560869 :           vn_nary_op_insert_pieces (2, POINTER_PLUS_EXPR,
    4498       560869 :                                     TREE_TYPE (op), ops, result,
    4499       560869 :                                     VN_INFO (result)->value_id);
    4500       560869 :           return;
    4501              :         }
    4502              :     }
    4503              : 
    4504     76322498 :   vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
    4505     76322498 :   if (TREE_CODE (result) == SSA_NAME)
    4506     52779798 :     vr1->value_id = VN_INFO (result)->value_id;
    4507              :   else
    4508     23542700 :     vr1->value_id = get_or_alloc_constant_value_id (result);
    4509     76322498 :   vr1->vuse = vuse_ssa_val (vuse);
    4510     76322498 :   vr1->operands = operands.copy ();
    4511     76322498 :   vr1->type = TREE_TYPE (op);
    4512     76322498 :   vr1->punned = false;
    4513     76322498 :   ao_ref op_ref;
    4514     76322498 :   ao_ref_init (&op_ref, op);
    4515     76322498 :   vr1->set = ao_ref_alias_set (&op_ref);
    4516     76322498 :   vr1->base_set = ao_ref_base_alias_set (&op_ref);
    4517              :   /* Specifically use an unknown extent here, we're not doing any lookup
    4518              :      and assume the caller didn't either (or it went VARYING).  */
    4519     76322498 :   vr1->offset = 0;
    4520     76322498 :   vr1->max_size = -1;
    4521     76322498 :   vr1->hashcode = vn_reference_compute_hash (vr1);
    4522     76322498 :   vr1->result = TREE_CODE (result) == SSA_NAME ? SSA_VAL (result) : result;
    4523     76322498 :   vr1->result_vdef = vdef;
    4524              : 
    4525     76322498 :   slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
    4526              :                                                       INSERT);
    4527              : 
    4528              :   /* Because IL walking on reference lookup can end up visiting
    4529              :      a def that is only to be visited later in iteration order
    4530              :      when we are about to make an irreducible region reducible
    4531              :      the def can be effectively processed and its ref being inserted
    4532              :      by vn_reference_lookup_3 already.  So we cannot assert (!*slot)
    4533              :      but save a lookup if we deal with already inserted refs here.  */
    4534     76322498 :   if (*slot)
    4535              :     {
    4536              :       /* We cannot assert that we have the same value either because
    4537              :          when disentangling an irreducible region we may end up visiting
    4538              :          a use before the corresponding def.  That's a missed optimization
    4539              :          only though.  See gcc.dg/tree-ssa/pr87126.c for example.  */
    4540            0 :       if (dump_file && (dump_flags & TDF_DETAILS)
    4541            0 :           && !operand_equal_p ((*slot)->result, vr1->result, 0))
    4542              :         {
    4543            0 :           fprintf (dump_file, "Keeping old value ");
    4544            0 :           print_generic_expr (dump_file, (*slot)->result);
    4545            0 :           fprintf (dump_file, " because of collision\n");
    4546              :         }
    4547            0 :       free_reference (vr1);
    4548            0 :       obstack_free (&vn_tables_obstack, vr1);
    4549              :       return;
    4550              :     }
    4551              : 
    4552     76322498 :   *slot = vr1;
    4553     76322498 :   vr1->next = last_inserted_ref;
    4554     76322498 :   last_inserted_ref = vr1;
    4555              : }
    4556              : 
    4557              : /* Insert a reference by it's pieces into the current hash table with
    4558              :    a value number of RESULT.  Return the resulting reference
    4559              :    structure we created.  */
    4560              : 
    4561              : vn_reference_t
    4562      1627568 : vn_reference_insert_pieces (tree vuse, alias_set_type set,
    4563              :                             alias_set_type base_set,
    4564              :                             poly_int64 offset, poly_int64 max_size, tree type,
    4565              :                             vec<vn_reference_op_s> operands,
    4566              :                             tree result, unsigned int value_id)
    4567              : 
    4568              : {
    4569      1627568 :   vn_reference_s **slot;
    4570      1627568 :   vn_reference_t vr1;
    4571              : 
    4572      1627568 :   vr1 = XOBNEW (&vn_tables_obstack, vn_reference_s);
    4573      1627568 :   vr1->value_id = value_id;
    4574      1627568 :   vr1->vuse = vuse_ssa_val (vuse);
    4575      1627568 :   vr1->operands = operands;
    4576      1627568 :   valueize_refs (&vr1->operands);
    4577      1627568 :   vr1->type = type;
    4578      1627568 :   vr1->punned = false;
    4579      1627568 :   vr1->set = set;
    4580      1627568 :   vr1->base_set = base_set;
    4581      1627568 :   vr1->offset = offset;
    4582      1627568 :   vr1->max_size = max_size;
    4583      1627568 :   vr1->hashcode = vn_reference_compute_hash (vr1);
    4584      1627568 :   if (result && TREE_CODE (result) == SSA_NAME)
    4585       384192 :     result = SSA_VAL (result);
    4586      1627568 :   vr1->result = result;
    4587      1627568 :   vr1->result_vdef = NULL_TREE;
    4588              : 
    4589      1627568 :   slot = valid_info->references->find_slot_with_hash (vr1, vr1->hashcode,
    4590              :                                                       INSERT);
    4591              : 
    4592              :   /* At this point we should have all the things inserted that we have
    4593              :      seen before, and we should never try inserting something that
    4594              :      already exists.  */
    4595      1627568 :   gcc_assert (!*slot);
    4596              : 
    4597      1627568 :   *slot = vr1;
    4598      1627568 :   vr1->next = last_inserted_ref;
    4599      1627568 :   last_inserted_ref = vr1;
    4600      1627568 :   return vr1;
    4601              : }
    4602              : 
    4603              : /* Compute and return the hash value for nary operation VBO1.  */
    4604              : 
    4605              : hashval_t
    4606    312172361 : vn_nary_op_compute_hash (const vn_nary_op_t vno1)
    4607              : {
    4608    312172361 :   inchash::hash hstate;
    4609    312172361 :   unsigned i;
    4610              : 
    4611    312172361 :   if (((vno1->length == 2
    4612    262389244 :         && commutative_tree_code (vno1->opcode))
    4613    142884273 :        || (vno1->length == 3
    4614      1701120 :            && commutative_ternary_tree_code (vno1->opcode)))
    4615    481462712 :       && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
    4616      2508606 :     std::swap (vno1->op[0], vno1->op[1]);
    4617    309663755 :   else if (TREE_CODE_CLASS (vno1->opcode) == tcc_comparison
    4618    309663755 :            && tree_swap_operands_p (vno1->op[0], vno1->op[1]))
    4619              :     {
    4620       468095 :       std::swap (vno1->op[0], vno1->op[1]);
    4621       468095 :       vno1->opcode = swap_tree_comparison  (vno1->opcode);
    4622              :     }
    4623              : 
    4624    312172361 :   hstate.add_int (vno1->opcode);
    4625    890969239 :   for (i = 0; i < vno1->length; ++i)
    4626    578796878 :     inchash::add_expr (vno1->op[i], hstate);
    4627              : 
    4628    312172361 :   return hstate.end ();
    4629              : }
    4630              : 
    4631              : /* Compare nary operations VNO1 and VNO2 and return true if they are
    4632              :    equivalent.  */
    4633              : 
    4634              : bool
    4635    987975240 : vn_nary_op_eq (const_vn_nary_op_t const vno1, const_vn_nary_op_t const vno2)
    4636              : {
    4637    987975240 :   unsigned i;
    4638              : 
    4639    987975240 :   if (vno1->hashcode != vno2->hashcode)
    4640              :     return false;
    4641              : 
    4642     51673447 :   if (vno1->length != vno2->length)
    4643              :     return false;
    4644              : 
    4645     51673447 :   if (vno1->opcode != vno2->opcode
    4646     51673447 :       || !types_compatible_p (vno1->type, vno2->type))
    4647              :     return false;
    4648              : 
    4649    145859530 :   for (i = 0; i < vno1->length; ++i)
    4650     95484688 :     if (!expressions_equal_p (vno1->op[i], vno2->op[i]))
    4651              :       return false;
    4652              : 
    4653              :   /* BIT_INSERT_EXPR has an implicit operand as the type precision
    4654              :      of op1.  Need to check to make sure they are the same.  */
    4655     50374842 :   if (vno1->opcode == BIT_INSERT_EXPR
    4656          581 :       && TREE_CODE (vno1->op[1]) == INTEGER_CST
    4657     50374970 :       && TYPE_PRECISION (TREE_TYPE (vno1->op[1]))
    4658          128 :          != TYPE_PRECISION (TREE_TYPE (vno2->op[1])))
    4659            0 :     return false;
    4660              : 
    4661              :   return true;
    4662              : }
    4663              : 
    4664              : /* Initialize VNO from the pieces provided.  */
    4665              : 
    4666              : static void
    4667    169252347 : init_vn_nary_op_from_pieces (vn_nary_op_t vno, unsigned int length,
    4668              :                              enum tree_code code, tree type, tree *ops)
    4669              : {
    4670    169252347 :   vno->opcode = code;
    4671    169252347 :   vno->length = length;
    4672    169252347 :   vno->type = type;
    4673      4784228 :   memcpy (&vno->op[0], ops, sizeof (tree) * length);
    4674            0 : }
    4675              : 
    4676              : /* Return the number of operands for a vn_nary ops structure from STMT.  */
    4677              : 
    4678              : unsigned int
    4679    112621656 : vn_nary_length_from_stmt (gimple *stmt)
    4680              : {
    4681    112621656 :   switch (gimple_assign_rhs_code (stmt))
    4682              :     {
    4683              :     case REALPART_EXPR:
    4684              :     case IMAGPART_EXPR:
    4685              :     case VIEW_CONVERT_EXPR:
    4686              :       return 1;
    4687              : 
    4688       670508 :     case BIT_FIELD_REF:
    4689       670508 :       return 3;
    4690              : 
    4691       554947 :     case CONSTRUCTOR:
    4692       554947 :       return CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
    4693              : 
    4694    107696144 :     default:
    4695    107696144 :       return gimple_num_ops (stmt) - 1;
    4696              :     }
    4697              : }
    4698              : 
    4699              : /* Initialize VNO from STMT.  */
    4700              : 
    4701              : void
    4702    112621656 : init_vn_nary_op_from_stmt (vn_nary_op_t vno, gassign *stmt)
    4703              : {
    4704    112621656 :   unsigned i;
    4705              : 
    4706    112621656 :   vno->opcode = gimple_assign_rhs_code (stmt);
    4707    112621656 :   vno->type = TREE_TYPE (gimple_assign_lhs (stmt));
    4708    112621656 :   switch (vno->opcode)
    4709              :     {
    4710      3700057 :     case REALPART_EXPR:
    4711      3700057 :     case IMAGPART_EXPR:
    4712      3700057 :     case VIEW_CONVERT_EXPR:
    4713      3700057 :       vno->length = 1;
    4714      3700057 :       vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
    4715      3700057 :       break;
    4716              : 
    4717       670508 :     case BIT_FIELD_REF:
    4718       670508 :       vno->length = 3;
    4719       670508 :       vno->op[0] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 0);
    4720       670508 :       vno->op[1] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 1);
    4721       670508 :       vno->op[2] = TREE_OPERAND (gimple_assign_rhs1 (stmt), 2);
    4722       670508 :       break;
    4723              : 
    4724       554947 :     case CONSTRUCTOR:
    4725       554947 :       vno->length = CONSTRUCTOR_NELTS (gimple_assign_rhs1 (stmt));
    4726      2222628 :       for (i = 0; i < vno->length; ++i)
    4727      1667681 :         vno->op[i] = CONSTRUCTOR_ELT (gimple_assign_rhs1 (stmt), i)->value;
    4728              :       break;
    4729              : 
    4730    107696144 :     default:
    4731    107696144 :       gcc_checking_assert (!gimple_assign_single_p (stmt));
    4732    107696144 :       vno->length = gimple_num_ops (stmt) - 1;
    4733    295015503 :       for (i = 0; i < vno->length; ++i)
    4734    187319359 :         vno->op[i] = gimple_op (stmt, i + 1);
    4735              :     }
    4736    112621656 : }
    4737              : 
    4738              : /* Compute the hashcode for VNO and look for it in the hash table;
    4739              :    return the resulting value number if it exists in the hash table.
    4740              :    Return NULL_TREE if it does not exist in the hash table or if the
    4741              :    result field of the operation is NULL.  VNRESULT will contain the
    4742              :    vn_nary_op_t from the hashtable if it exists.  */
    4743              : 
    4744              : static tree
    4745    135881093 : vn_nary_op_lookup_1 (vn_nary_op_t vno, vn_nary_op_t *vnresult)
    4746              : {
    4747    135881093 :   vn_nary_op_s **slot;
    4748              : 
    4749    135881093 :   if (vnresult)
    4750    127840568 :     *vnresult = NULL;
    4751              : 
    4752    377742694 :   for (unsigned i = 0; i < vno->length; ++i)
    4753    241861601 :     if (TREE_CODE (vno->op[i]) == SSA_NAME)
    4754    171204747 :       vno->op[i] = SSA_VAL (vno->op[i]);
    4755              : 
    4756    135881093 :   vno->hashcode = vn_nary_op_compute_hash (vno);
    4757    135881093 :   slot = valid_info->nary->find_slot_with_hash (vno, vno->hashcode, NO_INSERT);
    4758    135881093 :   if (!slot)
    4759              :     return NULL_TREE;
    4760     18183492 :   if (vnresult)
    4761     17719419 :     *vnresult = *slot;
    4762     18183492 :   return (*slot)->predicated_values ? NULL_TREE : (*slot)->u.result;
    4763              : }
    4764              : 
    4765              : /* Lookup a n-ary operation by its pieces and return the resulting value
    4766              :    number if it exists in the hash table.  Return NULL_TREE if it does
    4767              :    not exist in the hash table or if the result field of the operation
    4768              :    is NULL. VNRESULT will contain the vn_nary_op_t from the hashtable
    4769              :    if it exists.  */
    4770              : 
    4771              : tree
    4772     53626316 : vn_nary_op_lookup_pieces (unsigned int length, enum tree_code code,
    4773              :                           tree type, tree *ops, vn_nary_op_t *vnresult)
    4774              : {
    4775     53626316 :   vn_nary_op_t vno1 = XALLOCAVAR (struct vn_nary_op_s,
    4776              :                                   sizeof_vn_nary_op (length));
    4777     53626316 :   init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
    4778     53626316 :   return vn_nary_op_lookup_1 (vno1, vnresult);
    4779              : }
    4780              : 
    4781              : /* Lookup the rhs of STMT in the current hash table, and return the resulting
    4782              :    value number if it exists in the hash table.  Return NULL_TREE if
    4783              :    it does not exist in the hash table.  VNRESULT will contain the
    4784              :    vn_nary_op_t from the hashtable if it exists.  */
    4785              : 
    4786              : tree
    4787     58310329 : vn_nary_op_lookup_stmt (gimple *stmt, vn_nary_op_t *vnresult)
    4788              : {
    4789     58310329 :   vn_nary_op_t vno1
    4790     58310329 :     = XALLOCAVAR (struct vn_nary_op_s,
    4791              :                   sizeof_vn_nary_op (vn_nary_length_from_stmt (stmt)));
    4792     58310329 :   init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
    4793     58310329 :   return vn_nary_op_lookup_1 (vno1, vnresult);
    4794              : }
    4795              : 
    4796              : /* Allocate a vn_nary_op_t with LENGTH operands on STACK.  */
    4797              : 
    4798              : vn_nary_op_t
    4799    175264039 : alloc_vn_nary_op_noinit (unsigned int length, struct obstack *stack)
    4800              : {
    4801    175264039 :   return (vn_nary_op_t) obstack_alloc (stack, sizeof_vn_nary_op (length));
    4802              : }
    4803              : 
    4804              : /* Allocate and initialize a vn_nary_op_t on CURRENT_INFO's
    4805              :    obstack.  */
    4806              : 
    4807              : static vn_nary_op_t
    4808    157555704 : alloc_vn_nary_op (unsigned int length, tree result, unsigned int value_id)
    4809              : {
    4810            0 :   vn_nary_op_t vno1 = alloc_vn_nary_op_noinit (length, &vn_tables_obstack);
    4811              : 
    4812    157555704 :   vno1->value_id = value_id;
    4813    157555704 :   vno1->length = length;
    4814    157555704 :   vno1->predicated_values = 0;
    4815    157555704 :   vno1->u.result = result;
    4816              : 
    4817    157555704 :   return vno1;
    4818              : }
    4819              : 
    4820              : /* Insert VNO into TABLE.  */
    4821              : 
    4822              : static vn_nary_op_t
    4823    162339932 : vn_nary_op_insert_into (vn_nary_op_t vno, vn_nary_op_table_type *table)
    4824              : {
    4825    162339932 :   vn_nary_op_s **slot;
    4826              : 
    4827    162339932 :   gcc_assert (! vno->predicated_values
    4828              :               || (! vno->u.values->next
    4829              :                   && vno->u.values->n == 1));
    4830              : 
    4831    474946219 :   for (unsigned i = 0; i < vno->length; ++i)
    4832    312606287 :     if (TREE_CODE (vno->op[i]) == SSA_NAME)
    4833    204014925 :       vno->op[i] = SSA_VAL (vno->op[i]);
    4834              : 
    4835    162339932 :   vno->hashcode = vn_nary_op_compute_hash (vno);
    4836    162339932 :   slot = table->find_slot_with_hash (vno, vno->hashcode, INSERT);
    4837    162339932 :   vno->unwind_to = *slot;
    4838    162339932 :   if (*slot)
    4839              :     {
    4840              :       /* Prefer non-predicated values.
    4841              :          ???  Only if those are constant, otherwise, with constant predicated
    4842              :          value, turn them into predicated values with entry-block validity
    4843              :          (???  but we always find the first valid result currently).  */
    4844     31164126 :       if ((*slot)->predicated_values
    4845     30424900 :           && ! vno->predicated_values)
    4846              :         {
    4847              :           /* ???  We cannot remove *slot from the unwind stack list.
    4848              :              For the moment we deal with this by skipping not found
    4849              :              entries but this isn't ideal ...  */
    4850        73224 :           *slot = vno;
    4851              :           /* ???  Maintain a stack of states we can unwind in
    4852              :              vn_nary_op_s?  But how far do we unwind?  In reality
    4853              :              we need to push change records somewhere...  Or not
    4854              :              unwind vn_nary_op_s and linking them but instead
    4855              :              unwind the results "list", linking that, which also
    4856              :              doesn't move on hashtable resize.  */
    4857              :           /* We can also have a ->unwind_to recording *slot there.
    4858              :              That way we can make u.values a fixed size array with
    4859              :              recording the number of entries but of course we then
    4860              :              have always N copies for each unwind_to-state.  Or we
    4861              :              make sure to only ever append and each unwinding will
    4862              :              pop off one entry (but how to deal with predicated
    4863              :              replaced with non-predicated here?)  */
    4864        73224 :           vno->next = last_inserted_nary;
    4865        73224 :           last_inserted_nary = vno;
    4866        73224 :           return vno;
    4867              :         }
    4868     31090902 :       else if (vno->predicated_values
    4869     31090550 :                && ! (*slot)->predicated_values)
    4870              :         return *slot;
    4871     30352028 :       else if (vno->predicated_values
    4872     30351676 :                && (*slot)->predicated_values)
    4873              :         {
    4874              :           /* ???  Factor this all into a insert_single_predicated_value
    4875              :              routine.  */
    4876     30351676 :           gcc_assert (!vno->u.values->next && vno->u.values->n == 1);
    4877     30351676 :           basic_block vno_bb
    4878     30351676 :             = BASIC_BLOCK_FOR_FN (cfun, vno->u.values->valid_dominated_by_p[0]);
    4879     30351676 :           vn_pval *nval = vno->u.values;
    4880     30351676 :           vn_pval **next = &vno->u.values;
    4881     30351676 :           vn_pval *ins = NULL;
    4882     30351676 :           vn_pval *ins_at = NULL;
    4883              :           /* Find an existing value to append to.  */
    4884     57034533 :           for (vn_pval *val = (*slot)->u.values; val; val = val->next)
    4885              :             {
    4886     31376951 :               if (expressions_equal_p (val->result, nval->result))
    4887              :                 {
    4888              :                   /* Limit the number of places we register a predicate
    4889              :                      as valid.  */
    4890      4694094 :                   if (val->n > 8)
    4891       131461 :                     return *slot;
    4892     11570137 :                   for (unsigned i = 0; i < val->n; ++i)
    4893              :                     {
    4894      7267730 :                       basic_block val_bb
    4895      7267730 :                         = BASIC_BLOCK_FOR_FN (cfun,
    4896              :                                               val->valid_dominated_by_p[i]);
    4897      7267730 :                       if (dominated_by_p (CDI_DOMINATORS, vno_bb, val_bb))
    4898              :                         /* Value registered with more generic predicate.  */
    4899       260226 :                         return *slot;
    4900      7007504 :                       else if (flag_checking)
    4901              :                         /* Shouldn't happen, we insert in RPO order.  */
    4902      7007504 :                         gcc_assert (!dominated_by_p (CDI_DOMINATORS,
    4903              :                                                      val_bb, vno_bb));
    4904              :                     }
    4905              :                   /* Append the location.  */
    4906      4302407 :                   ins_at = val;
    4907      4302407 :                   ins = (vn_pval *) obstack_alloc (&vn_tables_obstack,
    4908              :                                                    sizeof (vn_pval)
    4909              :                                                    + val->n * sizeof (int));
    4910      4302407 :                   ins->next = NULL;
    4911      4302407 :                   ins->result = val->result;
    4912      4302407 :                   ins->n = val->n + 1;
    4913      4302407 :                   memcpy (ins->valid_dominated_by_p,
    4914      4302407 :                           val->valid_dominated_by_p,
    4915      4302407 :                           val->n * sizeof (int));
    4916      4302407 :                   ins->valid_dominated_by_p[val->n] = vno_bb->index;
    4917      4302407 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    4918            4 :                     fprintf (dump_file, "Appending predicate to value.\n");
    4919              :                   break;
    4920              :                 }
    4921              :             }
    4922              :           /* Copy the rest of the value chain.  */
    4923     61847705 :           for (vn_pval *val = (*slot)->u.values; val; val = val->next)
    4924              :             {
    4925     31887716 :               if (val == ins_at)
    4926              :                 /* Replace the node we appended to.  */
    4927      4302407 :                 *next = ins;
    4928              :               else
    4929              :                 {
    4930              :                   /* Copy other predicated values.  */
    4931     27585309 :                   *next = (vn_pval *) obstack_alloc (&vn_tables_obstack,
    4932              :                                                      sizeof (vn_pval)
    4933              :                                                      + ((val->n-1)
    4934              :                                                         * sizeof (int)));
    4935     27585309 :                   memcpy (*next, val,
    4936     27585309 :                           sizeof (vn_pval) + (val->n-1) * sizeof (int));
    4937     27585309 :                   (*next)->next = NULL;
    4938              :                 }
    4939     31887716 :               next = &(*next)->next;
    4940              :             }
    4941              :           /* Append the value if we didn't find it.  */
    4942     29959989 :           if (!ins_at)
    4943     25657582 :             *next = nval;
    4944     29959989 :           *slot = vno;
    4945     29959989 :           vno->next = last_inserted_nary;
    4946     29959989 :           last_inserted_nary = vno;
    4947     29959989 :           return vno;
    4948              :         }
    4949              : 
    4950              :       /* While we do not want to insert things twice it's awkward to
    4951              :          avoid it in the case where visit_nary_op pattern-matches stuff
    4952              :          and ends up simplifying the replacement to itself.  We then
    4953              :          get two inserts, one from visit_nary_op and one from
    4954              :          vn_nary_build_or_lookup.
    4955              :          So allow inserts with the same value number.  */
    4956          352 :       if ((*slot)->u.result == vno->u.result)
    4957              :         return *slot;
    4958              :     }
    4959              : 
    4960              :   /* ???  There's also optimistic vs. previous committed state merging
    4961              :      that is problematic for the case of unwinding.  */
    4962              : 
    4963              :   /* ???  We should return NULL if we do not use 'vno' and have the
    4964              :      caller release it.  */
    4965    131175806 :   gcc_assert (!*slot);
    4966              : 
    4967    131175806 :   *slot = vno;
    4968    131175806 :   vno->next = last_inserted_nary;
    4969    131175806 :   last_inserted_nary = vno;
    4970    131175806 :   return vno;
    4971              : }
    4972              : 
    4973              : /* Insert a n-ary operation into the current hash table using it's
    4974              :    pieces.  Return the vn_nary_op_t structure we created and put in
    4975              :    the hashtable.  */
    4976              : 
    4977              : vn_nary_op_t
    4978       560869 : vn_nary_op_insert_pieces (unsigned int length, enum tree_code code,
    4979              :                           tree type, tree *ops,
    4980              :                           tree result, unsigned int value_id)
    4981              : {
    4982       560869 :   vn_nary_op_t vno1 = alloc_vn_nary_op (length, result, value_id);
    4983       560869 :   init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
    4984       560869 :   return vn_nary_op_insert_into (vno1, valid_info->nary);
    4985              : }
    4986              : 
    4987              : /* Return whether we can track a predicate valid when PRED_E is executed.  */
    4988              : 
    4989              : static bool
    4990    155521696 : can_track_predicate_on_edge (edge pred_e)
    4991              : {
    4992              :   /* ???  As we are currently recording the destination basic-block index in
    4993              :      vn_pval.valid_dominated_by_p and using dominance for the
    4994              :      validity check we cannot track predicates on all edges.  */
    4995    155521696 :   if (single_pred_p (pred_e->dest))
    4996              :     return true;
    4997              :   /* Never record for backedges.  */
    4998     12210967 :   if (pred_e->flags & EDGE_DFS_BACK)
    4999              :     return false;
    5000              :   /* When there's more than one predecessor we cannot track
    5001              :      predicate validity based on the destination block.  The
    5002              :      exception is when all other incoming edges sources are
    5003              :      dominated by the destination block.  */
    5004     11663536 :   edge_iterator ei;
    5005     11663536 :   edge e;
    5006     19938502 :   FOR_EACH_EDGE (e, ei, pred_e->dest->preds)
    5007     18024702 :     if (e != pred_e && ! dominated_by_p (CDI_DOMINATORS, e->src, e->dest))
    5008              :       return false;
    5009              :   return true;
    5010              : }
    5011              : 
    5012              : static vn_nary_op_t
    5013    110280934 : vn_nary_op_insert_pieces_predicated (unsigned int length, enum tree_code code,
    5014              :                                      tree type, tree *ops,
    5015              :                                      tree result, unsigned int value_id,
    5016              :                                      edge pred_e)
    5017              : {
    5018    110280934 :   if (flag_checking)
    5019    110280024 :     gcc_assert (can_track_predicate_on_edge (pred_e));
    5020              : 
    5021        78236 :   if (dump_file && (dump_flags & TDF_DETAILS)
    5022              :       /* ???  Fix dumping, but currently we only get comparisons.  */
    5023    110354972 :       && TREE_CODE_CLASS (code) == tcc_comparison)
    5024              :     {
    5025        74038 :       fprintf (dump_file, "Recording on edge %d->%d ", pred_e->src->index,
    5026        74038 :                pred_e->dest->index);
    5027        74038 :       print_generic_expr (dump_file, ops[0], TDF_SLIM);
    5028        74038 :       fprintf (dump_file, " %s ", get_tree_code_name (code));
    5029        74038 :       print_generic_expr (dump_file, ops[1], TDF_SLIM);
    5030       110688 :       fprintf (dump_file, " == %s\n",
    5031        74038 :                integer_zerop (result) ? "false" : "true");
    5032              :     }
    5033    110280934 :   vn_nary_op_t vno1 = alloc_vn_nary_op (length, NULL_TREE, value_id);
    5034    110280934 :   init_vn_nary_op_from_pieces (vno1, length, code, type, ops);
    5035    110280934 :   vno1->predicated_values = 1;
    5036    110280934 :   vno1->u.values = (vn_pval *) obstack_alloc (&vn_tables_obstack,
    5037              :                                               sizeof (vn_pval));
    5038    110280934 :   vno1->u.values->next = NULL;
    5039    110280934 :   vno1->u.values->result = result;
    5040    110280934 :   vno1->u.values->n = 1;
    5041    110280934 :   vno1->u.values->valid_dominated_by_p[0] = pred_e->dest->index;
    5042    110280934 :   return vn_nary_op_insert_into (vno1, valid_info->nary);
    5043              : }
    5044              : 
    5045              : static bool
    5046              : dominated_by_p_w_unex (basic_block bb1, basic_block bb2);
    5047              : 
    5048              : static tree
    5049      1773430 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, basic_block bb,
    5050              :                                  edge e = NULL)
    5051              : {
    5052      1773430 :   if (! vno->predicated_values)
    5053            0 :     return vno->u.result;
    5054      3707559 :   for (vn_pval *val = vno->u.values; val; val = val->next)
    5055      5653761 :     for (unsigned i = 0; i < val->n; ++i)
    5056              :       {
    5057      3719632 :         basic_block cand
    5058      3719632 :           = BASIC_BLOCK_FOR_FN (cfun, val->valid_dominated_by_p[i]);
    5059              :         /* When asking for predicated values on an edge avoid looking
    5060              :            at edge executability for edges forward in our iteration
    5061              :            as well.  */
    5062      3719632 :         if (e && (e->flags & EDGE_DFS_BACK))
    5063              :           {
    5064        21511 :             if (dominated_by_p (CDI_DOMINATORS, bb, cand))
    5065         8325 :               return val->result;
    5066              :           }
    5067      3698121 :         else if (dominated_by_p_w_unex (bb, cand))
    5068       536971 :           return val->result;
    5069              :       }
    5070              :   return NULL_TREE;
    5071              : }
    5072              : 
    5073              : static tree
    5074       204344 : vn_nary_op_get_predicated_value (vn_nary_op_t vno, edge e)
    5075              : {
    5076            0 :   return vn_nary_op_get_predicated_value (vno, e->src, e);
    5077              : }
    5078              : 
    5079              : /* Insert the rhs of STMT into the current hash table with a value number of
    5080              :    RESULT.  */
    5081              : 
    5082              : static vn_nary_op_t
    5083     46713901 : vn_nary_op_insert_stmt (gimple *stmt, tree result)
    5084              : {
    5085     46713901 :   vn_nary_op_t vno1
    5086     46713901 :     = alloc_vn_nary_op (vn_nary_length_from_stmt (stmt),
    5087     46713901 :                         result, VN_INFO (result)->value_id);
    5088     46713901 :   init_vn_nary_op_from_stmt (vno1, as_a <gassign *> (stmt));
    5089     46713901 :   return vn_nary_op_insert_into (vno1, valid_info->nary);
    5090              : }
    5091              : 
    5092              : /* Compute a hashcode for PHI operation VP1 and return it.  */
    5093              : 
    5094              : static inline hashval_t
    5095     51274343 : vn_phi_compute_hash (vn_phi_t vp1)
    5096              : {
    5097     51274343 :   inchash::hash hstate;
    5098     51274343 :   tree phi1op;
    5099     51274343 :   tree type;
    5100     51274343 :   edge e;
    5101     51274343 :   edge_iterator ei;
    5102              : 
    5103    102548686 :   hstate.add_int (EDGE_COUNT (vp1->block->preds));
    5104     51274343 :   switch (EDGE_COUNT (vp1->block->preds))
    5105              :     {
    5106              :     case 1:
    5107              :       break;
    5108     44512773 :     case 2:
    5109              :       /* When this is a PHI node subject to CSE for different blocks
    5110              :          avoid hashing the block index.  */
    5111     44512773 :       if (vp1->cclhs)
    5112              :         break;
    5113              :       /* Fallthru.  */
    5114     34712526 :     default:
    5115     34712526 :       hstate.add_int (vp1->block->index);
    5116              :     }
    5117              : 
    5118              :   /* If all PHI arguments are constants we need to distinguish
    5119              :      the PHI node via its type.  */
    5120     51274343 :   type = vp1->type;
    5121     51274343 :   hstate.merge_hash (vn_hash_type (type));
    5122              : 
    5123    177359156 :   FOR_EACH_EDGE (e, ei, vp1->block->preds)
    5124              :     {
    5125              :       /* Don't hash backedge values they need to be handled as VN_TOP
    5126              :          for optimistic value-numbering.  */
    5127    126084813 :       if (e->flags & EDGE_DFS_BACK)
    5128     27413937 :         continue;
    5129              : 
    5130     98670876 :       phi1op = vp1->phiargs[e->dest_idx];
    5131     98670876 :       if (phi1op == VN_TOP)
    5132       243701 :         continue;
    5133     98427175 :       inchash::add_expr (phi1op, hstate);
    5134              :     }
    5135              : 
    5136     51274343 :   return hstate.end ();
    5137              : }
    5138              : 
    5139              : 
    5140              : /* Return true if COND1 and COND2 represent the same condition, set
    5141              :    *INVERTED_P if one needs to be inverted to make it the same as
    5142              :    the other.  */
    5143              : 
    5144              : static bool
    5145      3551142 : cond_stmts_equal_p (gcond *cond1, tree lhs1, tree rhs1,
    5146              :                     gcond *cond2, tree lhs2, tree rhs2, bool *inverted_p)
    5147              : {
    5148      3551142 :   enum tree_code code1 = gimple_cond_code (cond1);
    5149      3551142 :   enum tree_code code2 = gimple_cond_code (cond2);
    5150              : 
    5151      3551142 :   *inverted_p = false;
    5152      3551142 :   if (code1 == code2)
    5153              :     ;
    5154       287860 :   else if (code1 == swap_tree_comparison (code2))
    5155              :     std::swap (lhs2, rhs2);
    5156       250813 :   else if (code1 == invert_tree_comparison (code2, HONOR_NANS (lhs2)))
    5157       116085 :     *inverted_p = true;
    5158       134728 :   else if (code1 == invert_tree_comparison
    5159       134728 :                       (swap_tree_comparison (code2), HONOR_NANS (lhs2)))
    5160              :     {
    5161        10278 :       std::swap (lhs2, rhs2);
    5162        10278 :       *inverted_p = true;
    5163              :     }
    5164              :   else
    5165              :     return false;
    5166              : 
    5167      3426692 :   return ((expressions_equal_p (lhs1, lhs2)
    5168       108396 :            && expressions_equal_p (rhs1, rhs2))
    5169      3453646 :           || (commutative_tree_code (code1)
    5170      1769391 :               && expressions_equal_p (lhs1, rhs2)
    5171         3145 :               && expressions_equal_p (rhs1, lhs2)));
    5172              : }
    5173              : 
    5174              : /* Compare two phi entries for equality, ignoring VN_TOP arguments.  */
    5175              : 
    5176              : static int
    5177     41626638 : vn_phi_eq (const_vn_phi_t const vp1, const_vn_phi_t const vp2)
    5178              : {
    5179     41626638 :   if (vp1->hashcode != vp2->hashcode)
    5180              :     return false;
    5181              : 
    5182     12471404 :   if (vp1->block != vp2->block)
    5183              :     {
    5184     10676979 :       if (EDGE_COUNT (vp1->block->preds) != EDGE_COUNT (vp2->block->preds))
    5185              :         return false;
    5186              : 
    5187      3558993 :       switch (EDGE_COUNT (vp1->block->preds))
    5188              :         {
    5189              :         case 1:
    5190              :           /* Single-arg PHIs are just copies.  */
    5191              :           break;
    5192              : 
    5193      3558993 :         case 2:
    5194      3558993 :           {
    5195              :             /* Make sure both PHIs are classified as CSEable.  */
    5196      3558993 :             if (! vp1->cclhs || ! vp2->cclhs)
    5197              :               return false;
    5198              : 
    5199              :             /* Rule out backedges into the PHI.  */
    5200      3558993 :             gcc_checking_assert
    5201              :               (vp1->block->loop_father->header != vp1->block
    5202              :                && vp2->block->loop_father->header != vp2->block);
    5203              : 
    5204              :             /* If the PHI nodes do not have compatible types
    5205              :                they are not the same.  */
    5206      3558993 :             if (!types_compatible_p (vp1->type, vp2->type))
    5207              :               return false;
    5208              : 
    5209              :             /* If the immediate dominator end in switch stmts multiple
    5210              :                values may end up in the same PHI arg via intermediate
    5211              :                CFG merges.  */
    5212      3551142 :             basic_block idom1
    5213      3551142 :               = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
    5214      3551142 :             basic_block idom2
    5215      3551142 :               = get_immediate_dominator (CDI_DOMINATORS, vp2->block);
    5216      3551142 :             gcc_checking_assert (EDGE_COUNT (idom1->succs) == 2
    5217              :                                  && EDGE_COUNT (idom2->succs) == 2);
    5218              : 
    5219              :             /* Verify the controlling stmt is the same.  */
    5220      7102284 :             gcond *last1 = as_a <gcond *> (*gsi_last_bb (idom1));
    5221      7102284 :             gcond *last2 = as_a <gcond *> (*gsi_last_bb (idom2));
    5222      3551142 :             bool inverted_p;
    5223      3551142 :             if (! cond_stmts_equal_p (last1, vp1->cclhs, vp1->ccrhs,
    5224      3551142 :                                       last2, vp2->cclhs, vp2->ccrhs,
    5225              :                                       &inverted_p))
    5226              :               return false;
    5227              : 
    5228              :             /* Get at true/false controlled edges into the PHI.  */
    5229        81612 :             edge te1, te2, fe1, fe2;
    5230        81612 :             if (! extract_true_false_controlled_edges (idom1, vp1->block,
    5231              :                                                        &te1, &fe1)
    5232        81612 :                 || ! extract_true_false_controlled_edges (idom2, vp2->block,
    5233              :                                                           &te2, &fe2))
    5234              :               return false;
    5235              : 
    5236              :             /* Swap edges if the second condition is the inverted of the
    5237              :                first.  */
    5238        46820 :             if (inverted_p)
    5239         1846 :               std::swap (te2, fe2);
    5240              : 
    5241              :             /* Since we do not know which edge will be executed we have
    5242              :                to be careful when matching VN_TOP.  Be conservative and
    5243              :                only match VN_TOP == VN_TOP for now, we could allow
    5244              :                VN_TOP on the not prevailing PHI though.  See for example
    5245              :                PR102920.  */
    5246        46820 :             if (! expressions_equal_p (vp1->phiargs[te1->dest_idx],
    5247        46820 :                                        vp2->phiargs[te2->dest_idx], false)
    5248        91921 :                 || ! expressions_equal_p (vp1->phiargs[fe1->dest_idx],
    5249        45101 :                                           vp2->phiargs[fe2->dest_idx], false))
    5250         1719 :               return false;
    5251              : 
    5252              :             return true;
    5253              :           }
    5254              : 
    5255              :         default:
    5256              :           return false;
    5257              :         }
    5258              :     }
    5259              : 
    5260              :   /* If the PHI nodes do not have compatible types
    5261              :      they are not the same.  */
    5262      8912411 :   if (!types_compatible_p (vp1->type, vp2->type))
    5263              :     return false;
    5264              : 
    5265              :   /* Any phi in the same block will have it's arguments in the
    5266              :      same edge order, because of how we store phi nodes.  */
    5267      8911505 :   unsigned nargs = EDGE_COUNT (vp1->block->preds);
    5268     20011956 :   for (unsigned i = 0; i < nargs; ++i)
    5269              :     {
    5270     15875759 :       tree phi1op = vp1->phiargs[i];
    5271     15875759 :       tree phi2op = vp2->phiargs[i];
    5272     15875759 :       if (phi1op == phi2op)
    5273     11007310 :         continue;
    5274      4868449 :       if (!expressions_equal_p (phi1op, phi2op, false))
    5275              :         return false;
    5276              :     }
    5277              : 
    5278              :   return true;
    5279              : }
    5280              : 
    5281              : /* Lookup PHI in the current hash table, and return the resulting
    5282              :    value number if it exists in the hash table.  Return NULL_TREE if
    5283              :    it does not exist in the hash table. */
    5284              : 
    5285              : static tree
    5286     28048491 : vn_phi_lookup (gimple *phi, bool backedges_varying_p)
    5287              : {
    5288     28048491 :   vn_phi_s **slot;
    5289     28048491 :   struct vn_phi_s *vp1;
    5290     28048491 :   edge e;
    5291     28048491 :   edge_iterator ei;
    5292              : 
    5293     28048491 :   vp1 = XALLOCAVAR (struct vn_phi_s,
    5294              :                     sizeof (struct vn_phi_s)
    5295              :                     + (gimple_phi_num_args (phi) - 1) * sizeof (tree));
    5296              : 
    5297              :   /* Canonicalize the SSA_NAME's to their value number.  */
    5298     96113957 :   FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
    5299              :     {
    5300     68065466 :       tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
    5301     68065466 :       if (TREE_CODE (def) == SSA_NAME
    5302     56581788 :           && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
    5303              :         {
    5304     53965523 :           if (!virtual_operand_p (def)
    5305     53965523 :               && ssa_undefined_value_p (def, false))
    5306       134747 :             def = VN_TOP;
    5307              :           else
    5308     53830776 :             def = SSA_VAL (def);
    5309              :         }
    5310     68065466 :       vp1->phiargs[e->dest_idx] = def;
    5311              :     }
    5312     28048491 :   vp1->type = TREE_TYPE (gimple_phi_result (phi));
    5313     28048491 :   vp1->block = gimple_bb (phi);
    5314              :   /* Extract values of the controlling condition.  */
    5315     28048491 :   vp1->cclhs = NULL_TREE;
    5316     28048491 :   vp1->ccrhs = NULL_TREE;
    5317     28048491 :   if (EDGE_COUNT (vp1->block->preds) == 2
    5318     28048491 :       && vp1->block->loop_father->header != vp1->block)
    5319              :     {
    5320      8768826 :       basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
    5321      8768826 :       if (EDGE_COUNT (idom1->succs) == 2)
    5322     17435466 :         if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
    5323              :           {
    5324              :             /* ???  We want to use SSA_VAL here.  But possibly not
    5325              :                allow VN_TOP.  */
    5326      8468720 :             vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
    5327      8468720 :             vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
    5328              :           }
    5329              :     }
    5330     28048491 :   vp1->hashcode = vn_phi_compute_hash (vp1);
    5331     28048491 :   slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, NO_INSERT);
    5332     28048491 :   if (!slot)
    5333              :     return NULL_TREE;
    5334      4181298 :   return (*slot)->result;
    5335              : }
    5336              : 
    5337              : /* Insert PHI into the current hash table with a value number of
    5338              :    RESULT.  */
    5339              : 
    5340              : static vn_phi_t
    5341     23225852 : vn_phi_insert (gimple *phi, tree result, bool backedges_varying_p)
    5342              : {
    5343     23225852 :   vn_phi_s **slot;
    5344     23225852 :   vn_phi_t vp1 = (vn_phi_t) obstack_alloc (&vn_tables_obstack,
    5345              :                                            sizeof (vn_phi_s)
    5346              :                                            + ((gimple_phi_num_args (phi) - 1)
    5347              :                                               * sizeof (tree)));
    5348     23225852 :   edge e;
    5349     23225852 :   edge_iterator ei;
    5350              : 
    5351              :   /* Canonicalize the SSA_NAME's to their value number.  */
    5352     81245199 :   FOR_EACH_EDGE (e, ei, gimple_bb (phi)->preds)
    5353              :     {
    5354     58019347 :       tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
    5355     58019347 :       if (TREE_CODE (def) == SSA_NAME
    5356     47579280 :           && (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK)))
    5357              :         {
    5358     44963512 :           if (!virtual_operand_p (def)
    5359     44963512 :               && ssa_undefined_value_p (def, false))
    5360       109190 :             def = VN_TOP;
    5361              :           else
    5362     44854322 :             def = SSA_VAL (def);
    5363              :         }
    5364     58019347 :       vp1->phiargs[e->dest_idx] = def;
    5365              :     }
    5366     23225852 :   vp1->value_id = VN_INFO (result)->value_id;
    5367     23225852 :   vp1->type = TREE_TYPE (gimple_phi_result (phi));
    5368     23225852 :   vp1->block = gimple_bb (phi);
    5369              :   /* Extract values of the controlling condition.  */
    5370     23225852 :   vp1->cclhs = NULL_TREE;
    5371     23225852 :   vp1->ccrhs = NULL_TREE;
    5372     23225852 :   if (EDGE_COUNT (vp1->block->preds) == 2
    5373     23225852 :       && vp1->block->loop_father->header != vp1->block)
    5374              :     {
    5375      8388087 :       basic_block idom1 = get_immediate_dominator (CDI_DOMINATORS, vp1->block);
    5376      8388087 :       if (EDGE_COUNT (idom1->succs) == 2)
    5377     16678344 :         if (gcond *last1 = safe_dyn_cast <gcond *> (*gsi_last_bb (idom1)))
    5378              :           {
    5379              :             /* ???  We want to use SSA_VAL here.  But possibly not
    5380              :                allow VN_TOP.  */
    5381      8093097 :             vp1->cclhs = vn_valueize (gimple_cond_lhs (last1));
    5382      8093097 :             vp1->ccrhs = vn_valueize (gimple_cond_rhs (last1));
    5383              :           }
    5384              :     }
    5385     23225852 :   vp1->result = result;
    5386     23225852 :   vp1->hashcode = vn_phi_compute_hash (vp1);
    5387              : 
    5388     23225852 :   slot = valid_info->phis->find_slot_with_hash (vp1, vp1->hashcode, INSERT);
    5389     23225852 :   gcc_assert (!*slot);
    5390              : 
    5391     23225852 :   *slot = vp1;
    5392     23225852 :   vp1->next = last_inserted_phi;
    5393     23225852 :   last_inserted_phi = vp1;
    5394     23225852 :   return vp1;
    5395              : }
    5396              : 
    5397              : 
    5398              : /* Find the nearest common dominator of R1 and R2 in the IDOMS tree
    5399              :    using Reverse Postorder (RPO) numbers.  Both walks terminate latest
    5400              :    at the root (RPO number 0, whose IDOMS entry is -1) because callers
    5401              :    maintain the invariant IDOMS[x] < x for every x they store into
    5402              :    IDOMS.  */
    5403              : 
    5404              : static int
    5405     17848278 : exec_idom_intersect (int *idoms, int r1, int r2)
    5406              : {
    5407    473253443 :   while (r1 != r2)
    5408              :     {
    5409    455405165 :       if (r1 > r2)
    5410     13307922 :         r1 = idoms[r1];
    5411              :       else
    5412    442097243 :         r2 = idoms[r2];
    5413              :     }
    5414            0 :   return r1;
    5415              : }
    5416              : 
    5417              : /* Update the immediate dominator of BB based on its executable predecessors.
    5418              : 
    5419              :    This performs one step of a Cooper-Harvey-Kennedy style dominator
    5420              :    computation, but unlike the textbook algorithm we do not have the
    5421              :    luxury of iterating over a fixed graph to a fixpoint: the executable
    5422              :    subgraph is discovered incrementally as the RPO walk proceeds (edges
    5423              :    only ever turn executable, never back), and BB can be visited before
    5424              :    every one of its predecessors has reached its final executable state.
    5425              : 
    5426              :    We can only trust an edges executable state if it is not a backedge
    5427              :    in the RPO order.  Any other predecessor edge is folded in as BB's
    5428              :    immediate dominator in the static (whole-graph, CDI_DOMINATORS)
    5429              :    dominator tree.  */
    5430              : 
    5431              : static void
    5432     63271639 : update_exec_idom (basic_block bb)
    5433              : {
    5434     63271639 :   int rpo = vn_bb_to_rpo[bb->index];
    5435              :   /* Skip the entry block (RPO number zero) as its predecessors are
    5436              :      not covered by the region.  */
    5437     63271639 :   if (rpo == 0)
    5438     11718325 :     return;
    5439              : 
    5440     56960649 :   edge_iterator ei;
    5441     56960649 :   edge e;
    5442     56960649 :   int idom = -1;
    5443    130370677 :   FOR_EACH_EDGE (e, ei, bb->preds)
    5444              :     {
    5445     78817363 :       int pred_rpo = vn_bb_to_rpo[e->src->index];
    5446              :       /* For backedges fall back to the static immediate dominator.  */
    5447     78817363 :       if (pred_rpo >= rpo)
    5448              :         /* We initialize to the static immediate dominator and this is
    5449              :            never overwritten to another value since we either visit each
    5450              :            block exactly once or iterate in the same order.  */
    5451              :         return;
    5452              : 
    5453     73410028 :       if (e->flags & EDGE_EXECUTABLE)
    5454              :         {
    5455     72718636 :           if (idom == -1)
    5456              :             idom = pred_rpo;
    5457              :           else
    5458     91258306 :             idom = exec_idom_intersect (vn_exec_idom, idom, pred_rpo);
    5459              :         }
    5460              :     }
    5461              : 
    5462     51553314 :   gcc_checking_assert (idom != -1);
    5463     51553314 :   vn_exec_idom[rpo] = idom;
    5464              : }
    5465              : 
    5466              : /* Return true if BB1 is dominated by BB2 taking into account edges
    5467              :    that are known executable, including those that are not executable
    5468              :    only because they are backedges not yet proven to be taken (see
    5469              :    the commentary on update_exec_idom for why folding those in
    5470              :    unconditionally is always sound, and why it used to be handled by
    5471              :    a second, separate optimistic/pessimistic pair of trees here).
    5472              : 
    5473              :    Note: It is assumed that both BB1 and BB2 are inside the region
    5474              :    currently being value-numbered.  */
    5475              : 
    5476              : static bool
    5477     77571287 : dominated_by_p_w_unex (basic_block bb1, basic_block bb2)
    5478              : {
    5479     77571287 :   if (dominated_by_p (CDI_DOMINATORS, bb1, bb2))
    5480              :     return true;
    5481              : 
    5482     22054414 :   if (!vn_exec_idom)
    5483              :     return false;
    5484              : 
    5485     22054414 :   int rpo2 = vn_bb_to_rpo[bb2->index];
    5486     22054414 :   int rpo1 = vn_bb_to_rpo[bb1->index];
    5487              : 
    5488     22054414 :   int curr = rpo1;
    5489     22054414 :   int limit = 32;
    5490    124589625 :   while (curr > rpo2 && limit-- > 0)
    5491    102535211 :     curr = vn_exec_idom[curr];
    5492              : 
    5493     22054414 :   return curr == rpo2;
    5494              : }
    5495              : 
    5496              : /* Set the value number of FROM to TO, return true if it has changed
    5497              :    as a result.  */
    5498              : 
    5499              : static inline bool
    5500    211660487 : set_ssa_val_to (tree from, tree to)
    5501              : {
    5502    211660487 :   vn_ssa_aux_t from_info = VN_INFO (from);
    5503    211660487 :   tree currval = from_info->valnum; // SSA_VAL (from)
    5504    211660487 :   poly_int64 toff, coff;
    5505    211660487 :   bool curr_undefined = false;
    5506    211660487 :   bool curr_invariant = false;
    5507              : 
    5508              :   /* The only thing we allow as value numbers are ssa_names
    5509              :      and invariants.  So assert that here.  We don't allow VN_TOP
    5510              :      as visiting a stmt should produce a value-number other than
    5511              :      that.
    5512              :      ???  Still VN_TOP can happen for unreachable code, so force
    5513              :      it to varying in that case.  Not all code is prepared to
    5514              :      get VN_TOP on valueization.  */
    5515    211660487 :   if (to == VN_TOP)
    5516              :     {
    5517              :       /* ???  When iterating and visiting PHI <undef, backedge-value>
    5518              :          for the first time we rightfully get VN_TOP and we need to
    5519              :          preserve that to optimize for example gcc.dg/tree-ssa/ssa-sccvn-2.c.
    5520              :          With SCCVN we were simply lucky we iterated the other PHI
    5521              :          cycles first and thus visited the backedge-value DEF.  */
    5522            0 :       if (currval == VN_TOP)
    5523            0 :         goto set_and_exit;
    5524            0 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5525            0 :         fprintf (dump_file, "Forcing value number to varying on "
    5526              :                  "receiving VN_TOP\n");
    5527              :       to = from;
    5528              :     }
    5529              : 
    5530    211660487 :   gcc_checking_assert (to != NULL_TREE
    5531              :                        && ((TREE_CODE (to) == SSA_NAME
    5532              :                             && (to == from || SSA_VAL (to) == to))
    5533              :                            || is_gimple_min_invariant (to)));
    5534              : 
    5535    211660487 :   if (from != to)
    5536              :     {
    5537     33902761 :       if (currval == from)
    5538              :         {
    5539        13363 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5540              :             {
    5541            0 :               fprintf (dump_file, "Not changing value number of ");
    5542            0 :               print_generic_expr (dump_file, from);
    5543            0 :               fprintf (dump_file, " from VARYING to ");
    5544            0 :               print_generic_expr (dump_file, to);
    5545            0 :               fprintf (dump_file, "\n");
    5546              :             }
    5547              :           return false;
    5548              :         }
    5549     33889398 :       curr_invariant = is_gimple_min_invariant (currval);
    5550     67778796 :       curr_undefined = (TREE_CODE (currval) == SSA_NAME
    5551      3922848 :                         && !virtual_operand_p (currval)
    5552     37583060 :                         && ssa_undefined_value_p (currval, false));
    5553     33889398 :       if (currval != VN_TOP
    5554              :           && !curr_invariant
    5555      5472536 :           && !curr_undefined
    5556     37800440 :           && is_gimple_min_invariant (to))
    5557              :         {
    5558          202 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5559              :             {
    5560            0 :               fprintf (dump_file, "Forcing VARYING instead of changing "
    5561              :                        "value number of ");
    5562            0 :               print_generic_expr (dump_file, from);
    5563            0 :               fprintf (dump_file, " from ");
    5564            0 :               print_generic_expr (dump_file, currval);
    5565            0 :               fprintf (dump_file, " (non-constant) to ");
    5566            0 :               print_generic_expr (dump_file, to);
    5567            0 :               fprintf (dump_file, " (constant)\n");
    5568              :             }
    5569              :           to = from;
    5570              :         }
    5571     33889196 :       else if (currval != VN_TOP
    5572      5472334 :                && !curr_undefined
    5573      5460528 :                && TREE_CODE (to) == SSA_NAME
    5574      4614425 :                && !virtual_operand_p (to)
    5575     38274435 :                && ssa_undefined_value_p (to, false))
    5576              :         {
    5577            6 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5578              :             {
    5579            0 :               fprintf (dump_file, "Forcing VARYING instead of changing "
    5580              :                        "value number of ");
    5581            0 :               print_generic_expr (dump_file, from);
    5582            0 :               fprintf (dump_file, " from ");
    5583            0 :               print_generic_expr (dump_file, currval);
    5584            0 :               fprintf (dump_file, " (non-undefined) to ");
    5585            0 :               print_generic_expr (dump_file, to);
    5586            0 :               fprintf (dump_file, " (undefined)\n");
    5587              :             }
    5588              :           to = from;
    5589              :         }
    5590     33889190 :       else if (TREE_CODE (to) == SSA_NAME
    5591     33889190 :                && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (to))
    5592              :         to = from;
    5593              :     }
    5594              : 
    5595    177757726 : set_and_exit:
    5596    211647124 :   if (dump_file && (dump_flags & TDF_DETAILS))
    5597              :     {
    5598       412578 :       fprintf (dump_file, "Setting value number of ");
    5599       412578 :       print_generic_expr (dump_file, from);
    5600       412578 :       fprintf (dump_file, " to ");
    5601       412578 :       print_generic_expr (dump_file, to);
    5602              :     }
    5603              : 
    5604    211647124 :   if (currval != to
    5605    172817810 :       && !operand_equal_p (currval, to, 0)
    5606              :       /* Different undefined SSA names are not actually different.  See
    5607              :          PR82320 for a testcase were we'd otherwise not terminate iteration.  */
    5608    172748051 :       && !(curr_undefined
    5609         3550 :            && TREE_CODE (to) == SSA_NAME
    5610          723 :            && !virtual_operand_p (to)
    5611          723 :            && ssa_undefined_value_p (to, false))
    5612              :       /* ???  For addresses involving volatile objects or types operand_equal_p
    5613              :          does not reliably detect ADDR_EXPRs as equal.  We know we are only
    5614              :          getting invariant gimple addresses here, so can use
    5615              :          get_addr_base_and_unit_offset to do this comparison.  */
    5616    384394383 :       && !(TREE_CODE (currval) == ADDR_EXPR
    5617       473733 :            && TREE_CODE (to) == ADDR_EXPR
    5618           12 :            && (get_addr_base_and_unit_offset (TREE_OPERAND (currval, 0), &coff)
    5619            6 :                == get_addr_base_and_unit_offset (TREE_OPERAND (to, 0), &toff))
    5620            6 :            && known_eq (coff, toff)))
    5621              :     {
    5622    172747253 :       if (to != from
    5623     29475750 :           && currval != VN_TOP
    5624      1062666 :           && !curr_undefined
    5625              :           /* We do not want to allow lattice transitions from one value
    5626              :              to another since that may lead to not terminating iteration
    5627              :              (see PR95049).  Since there's no convenient way to check
    5628              :              for the allowed transition of VAL -> PHI (loop entry value,
    5629              :              same on two PHIs, to same PHI result) we restrict the check
    5630              :              to invariants.  */
    5631      1062666 :           && curr_invariant
    5632    173450832 :           && is_gimple_min_invariant (to))
    5633              :         {
    5634            0 :           if (dump_file && (dump_flags & TDF_DETAILS))
    5635            0 :             fprintf (dump_file, " forced VARYING");
    5636              :           to = from;
    5637              :         }
    5638    172747253 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5639       412262 :         fprintf (dump_file, " (changed)\n");
    5640    172747253 :       from_info->valnum = to;
    5641    172747253 :       return true;
    5642              :     }
    5643     38899871 :   if (dump_file && (dump_flags & TDF_DETAILS))
    5644          316 :     fprintf (dump_file, "\n");
    5645              :   return false;
    5646              : }
    5647              : 
    5648              : /* Set all definitions in STMT to value number to themselves.
    5649              :    Return true if a value number changed. */
    5650              : 
    5651              : static bool
    5652    308880588 : defs_to_varying (gimple *stmt)
    5653              : {
    5654    308880588 :   bool changed = false;
    5655    308880588 :   ssa_op_iter iter;
    5656    308880588 :   def_operand_p defp;
    5657              : 
    5658    339981801 :   FOR_EACH_SSA_DEF_OPERAND (defp, stmt, iter, SSA_OP_ALL_DEFS)
    5659              :     {
    5660     31101213 :       tree def = DEF_FROM_PTR (defp);
    5661     31101213 :       changed |= set_ssa_val_to (def, def);
    5662              :     }
    5663    308880588 :   return changed;
    5664              : }
    5665              : 
    5666              : /* Visit a copy between LHS and RHS, return true if the value number
    5667              :    changed.  */
    5668              : 
    5669              : static bool
    5670      8252622 : visit_copy (tree lhs, tree rhs)
    5671              : {
    5672              :   /* Valueize.  */
    5673      8252622 :   rhs = SSA_VAL (rhs);
    5674              : 
    5675      8252622 :   return set_ssa_val_to (lhs, rhs);
    5676              : }
    5677              : 
    5678              : /* Lookup a value for OP in type WIDE_TYPE where the value in type of OP
    5679              :    is the same.  */
    5680              : 
    5681              : static tree
    5682      2550702 : valueized_wider_op (tree wide_type, tree op, bool allow_truncate)
    5683              : {
    5684      2550702 :   if (TREE_CODE (op) == SSA_NAME)
    5685      2241138 :     op = vn_valueize (op);
    5686              : 
    5687              :   /* Either we have the op widened available.  */
    5688      2550702 :   tree ops[3] = {};
    5689      2550702 :   ops[0] = op;
    5690      2550702 :   tree tem = vn_nary_op_lookup_pieces (1, NOP_EXPR,
    5691              :                                        wide_type, ops, NULL);
    5692      2550702 :   if (tem)
    5693              :     return tem;
    5694              : 
    5695              :   /* Or the op is truncated from some existing value.  */
    5696      2255123 :   if (allow_truncate && TREE_CODE (op) == SSA_NAME)
    5697              :     {
    5698       574791 :       gimple *def = SSA_NAME_DEF_STMT (op);
    5699       574791 :       if (is_gimple_assign (def)
    5700       574791 :           && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
    5701              :         {
    5702       314050 :           tem = gimple_assign_rhs1 (def);
    5703       314050 :           if (useless_type_conversion_p (wide_type, TREE_TYPE (tem)))
    5704              :             {
    5705       214602 :               if (TREE_CODE (tem) == SSA_NAME)
    5706       214602 :                 tem = vn_valueize (tem);
    5707              :               return tem;
    5708              :             }
    5709              :         }
    5710              :     }
    5711              : 
    5712              :   /* For constants simply extend it.  */
    5713      2040521 :   if (TREE_CODE (op) == INTEGER_CST)
    5714       343061 :     return wide_int_to_tree (wide_type, wi::to_widest (op));
    5715              : 
    5716              :   return NULL_TREE;
    5717              : }
    5718              : 
    5719              : /* Return true if RESULT, the result of a value-number lookup, may be
    5720              :    used at the statement being visited.  A result of wrapping type can
    5721              :    be inserted for code hoisting without introducing undefined
    5722              :    overflow; anything else has to be available.  See PR86554.  */
    5723              : 
    5724              : static bool
    5725        22976 : vn_nary_result_avail_or_insertable_p (tree result)
    5726              : {
    5727        22976 :   return (TYPE_OVERFLOW_WRAPS (TREE_TYPE (result))
    5728        14540 :           || (rpo_avail && vn_context_bb
    5729        14540 :               && rpo_avail->eliminate_avail (vn_context_bb, result)));
    5730              : }
    5731              : 
    5732              : /* If OP is an SSA name defined by a conversion from an integral type,
    5733              :    return the valueized source of the conversion, otherwise return
    5734              :    NULL_TREE.  */
    5735              : 
    5736              : static tree
    5737     14311015 : ssa_integral_conversion_op (tree op)
    5738              : {
    5739     14311015 :   if (TREE_CODE (op) != SSA_NAME)
    5740              :     return NULL_TREE;
    5741     13994967 :   gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op));
    5742     11705506 :   if (!def || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
    5743              :     return NULL_TREE;
    5744      1509326 :   const tree src = gimple_assign_rhs1 (def);
    5745      1509326 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (src)))
    5746              :     return NULL_TREE;
    5747      1253488 :   return vn_valueize (src);
    5748              : }
    5749              : 
    5750              : /* Visit a nary operator RHS, value number it, and return true if the
    5751              :    value number of LHS has changed as a result.  */
    5752              : 
    5753              : static bool
    5754     50420182 : visit_nary_op (tree lhs, gassign *stmt)
    5755              : {
    5756     50420182 :   vn_nary_op_t vnresult;
    5757     50420182 :   tree result = vn_nary_op_lookup_stmt (stmt, &vnresult);
    5758     50420182 :   if (! result && vnresult)
    5759       143086 :     result = vn_nary_op_get_predicated_value (vnresult, gimple_bb (stmt));
    5760     46606263 :   if (result)
    5761      3883781 :     return set_ssa_val_to (lhs, result);
    5762              : 
    5763              :   /* Do some special pattern matching for redundancies of operations
    5764              :      in different types.  */
    5765     46536401 :   enum tree_code code = gimple_assign_rhs_code (stmt);
    5766     46536401 :   tree type = TREE_TYPE (lhs);
    5767     46536401 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    5768     46536401 :   switch (code)
    5769              :     {
    5770     10343269 :     CASE_CONVERT:
    5771              :       /* Match arithmetic done in a different type where we can easily
    5772              :          substitute the result from some earlier sign-changed or widened
    5773              :          operation.  */
    5774     10343269 :       if (INTEGRAL_TYPE_P (type)
    5775      9280349 :           && TREE_CODE (rhs1) == SSA_NAME
    5776              :           /* We only handle sign-changes, zero-extension -> & mask or
    5777              :              sign-extension if we know the inner operation doesn't
    5778              :              overflow.  */
    5779     19381647 :           && (((TYPE_UNSIGNED (TREE_TYPE (rhs1))
    5780      5456114 :                 || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    5781      5455317 :                     && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
    5782      8308836 :                && TYPE_PRECISION (type) > TYPE_PRECISION (TREE_TYPE (rhs1)))
    5783      6142769 :               || TYPE_PRECISION (type) == TYPE_PRECISION (TREE_TYPE (rhs1))))
    5784              :         {
    5785      7890430 :           gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
    5786      5754016 :           if (def
    5787      5754016 :               && (gimple_assign_rhs_code (def) == PLUS_EXPR
    5788      4485302 :                   || gimple_assign_rhs_code (def) == MINUS_EXPR
    5789      4320341 :                   || gimple_assign_rhs_code (def) == MULT_EXPR))
    5790              :             {
    5791      2052503 :               tree ops[3] = {};
    5792              :               /* When requiring a sign-extension we cannot model a
    5793              :                  previous truncation with a single op so don't bother.  */
    5794      2052503 :               bool allow_truncate = TYPE_UNSIGNED (TREE_TYPE (rhs1));
    5795              :               /* Either we have the op widened available.  */
    5796      2052503 :               ops[0] = valueized_wider_op (type, gimple_assign_rhs1 (def),
    5797              :                                            allow_truncate);
    5798      2052503 :               if (ops[0])
    5799       996398 :                 ops[1] = valueized_wider_op (type, gimple_assign_rhs2 (def),
    5800              :                                              allow_truncate);
    5801      2052503 :               if (ops[0] && ops[1])
    5802              :                 {
    5803       355043 :                   ops[0] = vn_nary_op_lookup_pieces
    5804       355043 :                       (2, gimple_assign_rhs_code (def), type, ops, NULL);
    5805              :                   /* We have wider operation available.  */
    5806       355043 :                   if (ops[0] && vn_nary_result_avail_or_insertable_p (ops[0]))
    5807              :                     {
    5808         8372 :                       unsigned lhs_prec = TYPE_PRECISION (type);
    5809         8372 :                       unsigned rhs_prec = TYPE_PRECISION (TREE_TYPE (rhs1));
    5810         8372 :                       if (lhs_prec == rhs_prec
    5811         8372 :                           || (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    5812          814 :                               && TYPE_OVERFLOW_UNDEFINED (TREE_TYPE (rhs1))))
    5813              :                         {
    5814         7731 :                           gimple_match_op match_op (gimple_match_cond::UNCOND,
    5815         7731 :                                                     NOP_EXPR, type, ops[0]);
    5816         7731 :                           result = vn_nary_build_or_lookup (&match_op);
    5817         7731 :                           if (result)
    5818              :                             {
    5819         7731 :                               bool changed = set_ssa_val_to (lhs, result);
    5820         7731 :                               if (TREE_CODE (result) == SSA_NAME)
    5821         7731 :                                 vn_nary_op_insert_stmt (stmt, result);
    5822         7731 :                               return changed;
    5823              :                             }
    5824              :                         }
    5825              :                       else
    5826              :                         {
    5827          641 :                           tree mask = wide_int_to_tree
    5828          641 :                             (type, wi::mask (rhs_prec, false, lhs_prec));
    5829          641 :                           gimple_match_op match_op (gimple_match_cond::UNCOND,
    5830          641 :                                                     BIT_AND_EXPR,
    5831          641 :                                                     TREE_TYPE (lhs),
    5832          641 :                                                     ops[0], mask);
    5833          641 :                           result = vn_nary_build_or_lookup (&match_op);
    5834          641 :                           if (result)
    5835              :                             {
    5836          641 :                               bool changed = set_ssa_val_to (lhs, result);
    5837          641 :                               if (TREE_CODE (result) == SSA_NAME)
    5838          641 :                                 vn_nary_op_insert_stmt (stmt, result);
    5839          641 :                               return changed;
    5840              :                             }
    5841              :                         }
    5842              :                     }
    5843              :                 }
    5844              :             }
    5845              :         }
    5846              :       break;
    5847     13955051 :     case PLUS_EXPR:
    5848     13955051 :     case MINUS_EXPR:
    5849     13955051 :       {
    5850              :         /* Match (T)A +- B against an existing (T)(A +- B'), the inverse
    5851              :            of the conversion case above, so the redundancy is detected
    5852              :            regardless of the order the two forms appear in the IL.
    5853              :            See PR124545.  The narrow operation is only ever looked up,
    5854              :            never created: assuming no overflow is only valid for
    5855              :            operations the program actually executes, so the narrow
    5856              :            leader has to be available.  Creating the narrow operation
    5857              :            instead is wrong-code, see PR126415.  */
    5858     13955051 :         const tree narrow1 = ssa_integral_conversion_op (vn_valueize (rhs1));
    5859     13955051 :         if (!INTEGRAL_TYPE_P (type) || !narrow1)
    5860              :           break;
    5861      1099297 :         const tree ntype = TREE_TYPE (narrow1);
    5862              :         /* A sign-change keeps the value bit-identical; a widening is
    5863              :            only handled when the narrow operation cannot wrap.  */
    5864      1099297 :         const bool sign_change_p
    5865      1099297 :           = TYPE_PRECISION (ntype) == TYPE_PRECISION (type);
    5866      1099297 :         const bool nowrap_widening_p
    5867      1099297 :           = (TYPE_PRECISION (ntype) < TYPE_PRECISION (type)
    5868      1099297 :              && TYPE_OVERFLOW_UNDEFINED (ntype));
    5869      1099297 :         if (!sign_change_p && !nowrap_widening_p)
    5870              :           break;
    5871              :         /* Determine the narrow variant of the second operand: a
    5872              :            constant that narrows and extends back unchanged, or a
    5873              :            conversion from the same narrow type.  */
    5874       833111 :         const tree rhs2 = gimple_assign_rhs2 (stmt);
    5875       833111 :         tree narrow2 = NULL_TREE;
    5876       833111 :         if (TREE_CODE (rhs2) == INTEGER_CST)
    5877              :           {
    5878       477147 :             const widest_int cst = wi::to_widest (rhs2);
    5879       477147 :             const widest_int narrowed
    5880       477147 :               = wi::ext (cst, TYPE_PRECISION (ntype), TYPE_SIGN (ntype));
    5881       477147 :             const widest_int extended
    5882       477147 :               = wi::ext (narrowed, TYPE_PRECISION (type), TYPE_SIGN (type));
    5883       477147 :             if (cst == extended)
    5884       474303 :               narrow2 = fold_convert (ntype, rhs2);
    5885       477153 :           }
    5886       355964 :         else if (TREE_CODE (rhs2) == SSA_NAME)
    5887              :           {
    5888       355964 :             const tree op = ssa_integral_conversion_op (vn_valueize (rhs2));
    5889       355964 :             if (op && types_compatible_p (TREE_TYPE (op), ntype))
    5890              :               narrow2 = op;
    5891              :           }
    5892       620009 :         if (!narrow2)
    5893              :           break;
    5894       617165 :         tree ops[3] = { narrow1, narrow2 };
    5895       617165 :         const tree narrow_val
    5896       617165 :           = vn_nary_op_lookup_pieces (2, code, ntype, ops, NULL);
    5897              :         /* We have a narrower or sign-changed operation available.  */
    5898       617165 :         if (narrow_val && vn_nary_result_avail_or_insertable_p (narrow_val))
    5899              :           {
    5900        11883 :             gimple_match_op match_op (gimple_match_cond::UNCOND,
    5901        11883 :                                       NOP_EXPR, type, narrow_val);
    5902        11883 :             result = vn_nary_build_or_lookup (&match_op);
    5903        11883 :             if (result)
    5904              :               {
    5905        11883 :                 const bool changed = set_ssa_val_to (lhs, result);
    5906        11883 :                 if (TREE_CODE (result) == SSA_NAME)
    5907        11883 :                   vn_nary_op_insert_stmt (stmt, result);
    5908        11883 :                 return changed;
    5909              :               }
    5910              :           }
    5911              :       }
    5912       605282 :       break;
    5913      1564156 :     case BIT_AND_EXPR:
    5914      1564156 :       if (INTEGRAL_TYPE_P (type)
    5915      1519722 :           && TREE_CODE (rhs1) == SSA_NAME
    5916      1519722 :           && TREE_CODE (gimple_assign_rhs2 (stmt)) == INTEGER_CST
    5917       931339 :           && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)
    5918       931219 :           && default_vn_walk_kind != VN_NOWALK
    5919              :           && CHAR_BIT == 8
    5920              :           && BITS_PER_UNIT == 8
    5921              :           && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN
    5922       931010 :           && TYPE_PRECISION (type) <= vn_walk_cb_data::bufsize * BITS_PER_UNIT
    5923       931008 :           && !integer_all_onesp (gimple_assign_rhs2 (stmt))
    5924      2495164 :           && !integer_zerop (gimple_assign_rhs2 (stmt)))
    5925              :         {
    5926       931008 :           gassign *ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs1));
    5927       682501 :           if (ass
    5928       682501 :               && !gimple_has_volatile_ops (ass)
    5929       680907 :               && vn_get_stmt_kind (ass) == VN_REFERENCE)
    5930              :             {
    5931       319285 :               tree last_vuse = gimple_vuse (ass);
    5932       319285 :               tree op = gimple_assign_rhs1 (ass);
    5933       957855 :               tree result = vn_reference_lookup (op, gimple_vuse (ass),
    5934              :                                                  default_vn_walk_kind,
    5935              :                                                  NULL, true, &last_vuse,
    5936              :                                                  gimple_assign_rhs2 (stmt));
    5937       319285 :               if (result
    5938       319732 :                   && useless_type_conversion_p (TREE_TYPE (result),
    5939          447 :                                                 TREE_TYPE (op)))
    5940          447 :                 return set_ssa_val_to (lhs, result);
    5941              :             }
    5942              :         }
    5943              :       break;
    5944       260793 :     case BIT_FIELD_REF:
    5945       260793 :       if (TREE_CODE (TREE_OPERAND (rhs1, 0)) == SSA_NAME)
    5946              :         {
    5947       260765 :           tree op0 = vn_valueize (TREE_OPERAND (rhs1, 0));
    5948       260765 :           gassign *ass;
    5949       260765 :           if (TREE_CODE (op0) == SSA_NAME
    5950       260765 :               && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (op0)))
    5951       211223 :               && !gimple_has_volatile_ops (ass)
    5952       471905 :               && vn_get_stmt_kind (ass) == VN_REFERENCE)
    5953              :             {
    5954        81645 :               tree last_vuse = gimple_vuse (ass);
    5955        81645 :               tree op = gimple_assign_rhs1 (ass);
    5956              :               /* Avoid building invalid and unexpected refs.  */
    5957        81645 :               if (TREE_CODE (op) != TARGET_MEM_REF
    5958              :                   && TREE_CODE (op) != BIT_FIELD_REF
    5959              :                   && TREE_CODE (op) != REALPART_EXPR
    5960              :                   && TREE_CODE (op) != IMAGPART_EXPR)
    5961              :                 {
    5962        74651 :                   tree op = build3 (BIT_FIELD_REF, TREE_TYPE (rhs1),
    5963              :                                     gimple_assign_rhs1 (ass),
    5964        74651 :                                     TREE_OPERAND (rhs1, 1),
    5965        74651 :                                     TREE_OPERAND (rhs1, 2));
    5966       149302 :                   tree result = vn_reference_lookup (op, gimple_vuse (ass),
    5967              :                                                      default_vn_walk_kind,
    5968              :                                                      NULL, true, &last_vuse);
    5969        74651 :                   if (result
    5970        74651 :                       && useless_type_conversion_p (type, TREE_TYPE (result)))
    5971         1963 :                     return set_ssa_val_to (lhs, result);
    5972        72927 :                   else if (result
    5973          239 :                            && TYPE_SIZE (type)
    5974          239 :                            && TYPE_SIZE (TREE_TYPE (result))
    5975        73166 :                            && operand_equal_p (TYPE_SIZE (type),
    5976          239 :                                                TYPE_SIZE (TREE_TYPE (result))))
    5977              :                     {
    5978          239 :                       gimple_match_op match_op (gimple_match_cond::UNCOND,
    5979          239 :                                                 VIEW_CONVERT_EXPR,
    5980          239 :                                                 type, result);
    5981          239 :                       result = vn_nary_build_or_lookup (&match_op);
    5982          239 :                       if (result)
    5983              :                         {
    5984          239 :                           bool changed = set_ssa_val_to (lhs, result);
    5985          239 :                           if (TREE_CODE (result) == SSA_NAME)
    5986          227 :                             vn_nary_op_insert_stmt (stmt, result);
    5987          239 :                           return changed;
    5988              :                         }
    5989              :                     }
    5990              :                 }
    5991              :             }
    5992              :         }
    5993              :       break;
    5994       350277 :     case TRUNC_DIV_EXPR:
    5995       350277 :       if (TYPE_UNSIGNED (type))
    5996              :         break;
    5997              :       /* Fallthru.  */
    5998      5696243 :     case RDIV_EXPR:
    5999      5696243 :     case MULT_EXPR:
    6000              :       /* Match up ([-]a){/,*}([-])b with v=a{/,*}b, replacing it with -v.  */
    6001      5696243 :       if (! HONOR_SIGN_DEPENDENT_ROUNDING (type)
    6002      5696243 :           && (! COMPLEX_FLOAT_TYPE_P (type)
    6003        65788 :               || ! HONOR_SIGNED_ZEROS (type)))
    6004              :         {
    6005      5666765 :           tree rhs[2];
    6006      5666765 :           rhs[0] = rhs1;
    6007      5666765 :           rhs[1] = gimple_assign_rhs2 (stmt);
    6008     16991453 :           for (unsigned i = 0; i <= 1; ++i)
    6009              :             {
    6010     11332035 :               unsigned j = i == 0 ? 1 : 0;
    6011     11332035 :               tree ops[2];
    6012     11332035 :               gimple_match_op match_op (gimple_match_cond::UNCOND,
    6013     11332035 :                                         NEGATE_EXPR, type, rhs[i]);
    6014     11332035 :               ops[i] = vn_nary_build_or_lookup_1 (&match_op, false, true);
    6015     11332035 :               ops[j] = rhs[j];
    6016     11332035 :               if (ops[i]
    6017     11332035 :                   && (ops[0] = vn_nary_op_lookup_pieces (2, code,
    6018              :                                                          type, ops, NULL)))
    6019              :                 {
    6020         7347 :                   gimple_match_op match_op (gimple_match_cond::UNCOND,
    6021         7347 :                                             NEGATE_EXPR, type, ops[0]);
    6022         7347 :                   result = vn_nary_build_or_lookup_1 (&match_op, true, false);
    6023         7347 :                   if (result)
    6024              :                     {
    6025         7347 :                       bool changed = set_ssa_val_to (lhs, result);
    6026         7347 :                       if (TREE_CODE (result) == SSA_NAME)
    6027         7347 :                         vn_nary_op_insert_stmt (stmt, result);
    6028         7347 :                       return changed;
    6029              :                     }
    6030              :                 }
    6031              :             }
    6032              :         }
    6033              :       break;
    6034       373535 :     case LSHIFT_EXPR:
    6035              :       /* For X << C, use the value number of X * (1 << C).  */
    6036       373535 :       if (INTEGRAL_TYPE_P (type)
    6037       357251 :           && TYPE_OVERFLOW_WRAPS (type)
    6038       565855 :           && !TYPE_SATURATING (type))
    6039              :         {
    6040       192320 :           tree rhs2 = gimple_assign_rhs2 (stmt);
    6041       192320 :           if (TREE_CODE (rhs2) == INTEGER_CST
    6042       112432 :               && tree_fits_uhwi_p (rhs2)
    6043       304752 :               && tree_to_uhwi (rhs2) < TYPE_PRECISION (type))
    6044              :             {
    6045       224864 :               wide_int w = wi::set_bit_in_zero (tree_to_uhwi (rhs2),
    6046       112432 :                                                 TYPE_PRECISION (type));
    6047       224864 :               gimple_match_op match_op (gimple_match_cond::UNCOND,
    6048       112432 :                                         MULT_EXPR, type, rhs1,
    6049       112432 :                                         wide_int_to_tree (type, w));
    6050       112432 :               result = vn_nary_build_or_lookup (&match_op);
    6051       112432 :               if (result)
    6052              :                 {
    6053       112432 :                   bool changed = set_ssa_val_to (lhs, result);
    6054       112432 :                   if (TREE_CODE (result) == SSA_NAME)
    6055       112431 :                     vn_nary_op_insert_stmt (stmt, result);
    6056       112432 :                   return changed;
    6057              :                 }
    6058       112432 :             }
    6059              :         }
    6060              :       break;
    6061              :     default:
    6062              :       break;
    6063              :     }
    6064              : 
    6065     46393957 :   bool changed = set_ssa_val_to (lhs, lhs);
    6066     46393957 :   vn_nary_op_insert_stmt (stmt, lhs);
    6067     46393957 :   return changed;
    6068              : }
    6069              : 
    6070              : /* Visit a call STMT storing into LHS.  Return true if the value number
    6071              :    of the LHS has changed as a result.  */
    6072              : 
    6073              : static bool
    6074      8812461 : visit_reference_op_call (tree lhs, gcall *stmt)
    6075              : {
    6076      8812461 :   bool changed = false;
    6077      8812461 :   struct vn_reference_s vr1;
    6078      8812461 :   vn_reference_t vnresult = NULL;
    6079      8812461 :   tree vdef = gimple_vdef (stmt);
    6080      8812461 :   modref_summary *summary;
    6081              : 
    6082              :   /* Non-ssa lhs is handled in copy_reference_ops_from_call.  */
    6083      8812461 :   if (lhs && TREE_CODE (lhs) != SSA_NAME)
    6084      4696577 :     lhs = NULL_TREE;
    6085              : 
    6086      8812461 :   vn_reference_lookup_call (stmt, &vnresult, &vr1);
    6087              : 
    6088              :   /* If the lookup did not succeed for pure functions try to use
    6089              :      modref info to find a candidate to CSE to.  */
    6090      8812461 :   const unsigned accesses_limit = 8;
    6091      8812461 :   if (!vnresult
    6092      8140551 :       && !vdef
    6093      8140551 :       && lhs
    6094      2839650 :       && gimple_vuse (stmt)
    6095     10396112 :       && (((summary = get_modref_function_summary (stmt, NULL))
    6096       219343 :            && !summary->global_memory_read
    6097        86144 :            && summary->load_accesses < accesses_limit)
    6098      1497868 :           || gimple_call_flags (stmt) & ECF_CONST))
    6099              :     {
    6100              :       /* First search if we can do something useful and build a
    6101              :          vector of all loads we have to check.  */
    6102        86577 :       bool unknown_memory_access = false;
    6103        86577 :       auto_vec<ao_ref, accesses_limit> accesses;
    6104        86577 :       unsigned load_accesses = summary ? summary->load_accesses : 0;
    6105        86577 :       if (!unknown_memory_access)
    6106              :         /* Add loads done as part of setting up the call arguments.
    6107              :            That's also necessary for CONST functions which will
    6108              :            not have a modref summary.  */
    6109       264577 :         for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
    6110              :           {
    6111       178008 :             tree arg = gimple_call_arg (stmt, i);
    6112       178008 :             if (TREE_CODE (arg) != SSA_NAME
    6113       178008 :                 && !is_gimple_min_invariant (arg))
    6114              :               {
    6115        64790 :                 if (accesses.length () >= accesses_limit - load_accesses)
    6116              :                   {
    6117              :                     unknown_memory_access = true;
    6118              :                     break;
    6119              :                   }
    6120        32387 :                 accesses.quick_grow (accesses.length () + 1);
    6121        32387 :                 ao_ref_init (&accesses.last (), arg);
    6122              :               }
    6123              :           }
    6124        86577 :       if (summary && !unknown_memory_access)
    6125              :         {
    6126              :           /* Add loads as analyzed by IPA modref.  */
    6127       308708 :           for (auto base_node : summary->loads->bases)
    6128        78676 :             if (unknown_memory_access)
    6129              :               break;
    6130       320867 :             else for (auto ref_node : base_node->refs)
    6131        85790 :               if (unknown_memory_access)
    6132              :                 break;
    6133       360877 :               else for (auto access_node : ref_node->accesses)
    6134              :                 {
    6135       242558 :                   accesses.quick_grow (accesses.length () + 1);
    6136       121279 :                   ao_ref *r = &accesses.last ();
    6137       121279 :                   if (!access_node.get_ao_ref (stmt, r))
    6138              :                     {
    6139              :                       /* Initialize a ref based on the argument and
    6140              :                          unknown offset if possible.  */
    6141        17736 :                       tree arg = access_node.get_call_arg (stmt);
    6142        17736 :                       if (arg && TREE_CODE (arg) == SSA_NAME)
    6143         4264 :                         arg = SSA_VAL (arg);
    6144         4264 :                       if (arg
    6145        17726 :                           && TREE_CODE (arg) == ADDR_EXPR
    6146        13468 :                           && (arg = get_base_address (arg))
    6147        17732 :                           && DECL_P (arg))
    6148              :                         {
    6149            0 :                           ao_ref_init (r, arg);
    6150            0 :                           r->ref = NULL_TREE;
    6151            0 :                           r->base = arg;
    6152              :                         }
    6153              :                       else
    6154              :                         {
    6155              :                           unknown_memory_access = true;
    6156              :                           break;
    6157              :                         }
    6158              :                     }
    6159       103543 :                   r->base_alias_set = base_node->base;
    6160       103543 :                   r->ref_alias_set = ref_node->ref;
    6161              :                 }
    6162              :         }
    6163              : 
    6164              :       /* Walk the VUSE->VDEF chain optimistically trying to find an entry
    6165              :          for the call in the hashtable.  */
    6166        86577 :       unsigned limit = (unknown_memory_access
    6167        86577 :                         ? 0
    6168        68833 :                         : (param_sccvn_max_alias_queries_per_access
    6169        68833 :                            / (accesses.length () + 1)));
    6170        86577 :       tree saved_vuse = vr1.vuse;
    6171        86577 :       hashval_t saved_hashcode = vr1.hashcode;
    6172       506424 :       while (limit > 0 && !vnresult && !SSA_NAME_IS_DEFAULT_DEF (vr1.vuse))
    6173              :         {
    6174       447709 :           vr1.hashcode = vr1.hashcode - SSA_NAME_VERSION (vr1.vuse);
    6175       447709 :           gimple *def = SSA_NAME_DEF_STMT (vr1.vuse);
    6176              :           /* ???  We could use fancy stuff like in walk_non_aliased_vuses, but
    6177              :              do not bother for now.  */
    6178       447709 :           if (is_a <gphi *> (def))
    6179              :             break;
    6180       839694 :           vr1.vuse = vuse_ssa_val (gimple_vuse (def));
    6181       419847 :           vr1.hashcode = vr1.hashcode + SSA_NAME_VERSION (vr1.vuse);
    6182       419847 :           vn_reference_lookup_1 (&vr1, &vnresult);
    6183       419847 :           limit--;
    6184              :         }
    6185              : 
    6186              :       /* If we found a candidate to CSE to verify it is valid.  */
    6187        86577 :       if (vnresult && !accesses.is_empty ())
    6188              :         {
    6189         1980 :           tree vuse = vuse_ssa_val (gimple_vuse (stmt));
    6190         7440 :           while (vnresult && vuse != vr1.vuse)
    6191              :             {
    6192         3480 :               gimple *def = SSA_NAME_DEF_STMT (vuse);
    6193        18718 :               for (auto &ref : accesses)
    6194              :                 {
    6195              :                   /* ???  stmt_may_clobber_ref_p_1 does per stmt constant
    6196              :                      analysis overhead that we might be able to cache.  */
    6197        10038 :                   if (stmt_may_clobber_ref_p_1 (def, &ref, true))
    6198              :                     {
    6199         1760 :                       vnresult = NULL;
    6200         1760 :                       break;
    6201              :                     }
    6202              :                 }
    6203         6960 :               vuse = vuse_ssa_val (gimple_vuse (def));
    6204              :             }
    6205              :         }
    6206        86577 :       vr1.vuse = saved_vuse;
    6207        86577 :       vr1.hashcode = saved_hashcode;
    6208        86577 :     }
    6209              : 
    6210      8812461 :   if (vnresult)
    6211              :     {
    6212       672158 :       if (vdef)
    6213              :         {
    6214       177417 :           if (vnresult->result_vdef)
    6215       177417 :             changed |= set_ssa_val_to (vdef, vnresult->result_vdef);
    6216            0 :           else if (!lhs && gimple_call_lhs (stmt))
    6217              :             /* If stmt has non-SSA_NAME lhs, value number the vdef to itself,
    6218              :                as the call still acts as a lhs store.  */
    6219            0 :             changed |= set_ssa_val_to (vdef, vdef);
    6220              :           else
    6221              :             /* If the call was discovered to be pure or const reflect
    6222              :                that as far as possible.  */
    6223            0 :             changed |= set_ssa_val_to (vdef,
    6224              :                                        vuse_ssa_val (gimple_vuse (stmt)));
    6225              :         }
    6226              : 
    6227       672158 :       if (!vnresult->result && lhs)
    6228            0 :         vnresult->result = lhs;
    6229              : 
    6230       672158 :       if (vnresult->result && lhs)
    6231       125903 :         changed |= set_ssa_val_to (lhs, vnresult->result);
    6232              :     }
    6233              :   else
    6234              :     {
    6235      8140303 :       vn_reference_t vr2;
    6236      8140303 :       vn_reference_s **slot;
    6237      8140303 :       tree vdef_val = vdef;
    6238      8140303 :       if (vdef)
    6239              :         {
    6240              :           /* If we value numbered an indirect functions function to
    6241              :              one not clobbering memory value number its VDEF to its
    6242              :              VUSE.  */
    6243      4974523 :           tree fn = gimple_call_fn (stmt);
    6244      4974523 :           if (fn && TREE_CODE (fn) == SSA_NAME)
    6245              :             {
    6246       130806 :               fn = SSA_VAL (fn);
    6247       130806 :               if (TREE_CODE (fn) == ADDR_EXPR
    6248         2037 :                   && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL
    6249         2037 :                   && (flags_from_decl_or_type (TREE_OPERAND (fn, 0))
    6250         2037 :                       & (ECF_CONST | ECF_PURE))
    6251              :                   /* If stmt has non-SSA_NAME lhs, value number the
    6252              :                      vdef to itself, as the call still acts as a lhs
    6253              :                      store.  */
    6254       132218 :                   && (lhs || gimple_call_lhs (stmt) == NULL_TREE))
    6255         2682 :                 vdef_val = vuse_ssa_val (gimple_vuse (stmt));
    6256              :             }
    6257      4974523 :           changed |= set_ssa_val_to (vdef, vdef_val);
    6258              :         }
    6259      8140303 :       if (lhs)
    6260      3989981 :         changed |= set_ssa_val_to (lhs, lhs);
    6261      8140303 :       vr2 = XOBNEW (&vn_tables_obstack, vn_reference_s);
    6262      8140303 :       vr2->vuse = vr1.vuse;
    6263              :       /* As we are not walking the virtual operand chain we know the
    6264              :          shared_lookup_references are still original so we can re-use
    6265              :          them here.  */
    6266      8140303 :       vr2->operands = vr1.operands.copy ();
    6267      8140303 :       vr2->type = vr1.type;
    6268      8140303 :       vr2->punned = vr1.punned;
    6269      8140303 :       vr2->set = vr1.set;
    6270      8140303 :       vr2->offset = vr1.offset;
    6271      8140303 :       vr2->max_size = vr1.max_size;
    6272      8140303 :       vr2->base_set = vr1.base_set;
    6273      8140303 :       vr2->hashcode = vr1.hashcode;
    6274      8140303 :       vr2->result = lhs;
    6275      8140303 :       vr2->result_vdef = vdef_val;
    6276      8140303 :       vr2->value_id = 0;
    6277      8140303 :       slot = valid_info->references->find_slot_with_hash (vr2, vr2->hashcode,
    6278              :                                                           INSERT);
    6279      8140303 :       gcc_assert (!*slot);
    6280      8140303 :       *slot = vr2;
    6281      8140303 :       vr2->next = last_inserted_ref;
    6282      8140303 :       last_inserted_ref = vr2;
    6283              :     }
    6284              : 
    6285      8812461 :   return changed;
    6286              : }
    6287              : 
    6288              : /* Visit a load from a reference operator RHS, part of STMT, value number it,
    6289              :    and return true if the value number of the LHS has changed as a result.  */
    6290              : 
    6291              : static bool
    6292     35543631 : visit_reference_op_load (tree lhs, tree op, gimple *stmt)
    6293              : {
    6294     35543631 :   bool changed = false;
    6295     35543631 :   tree result;
    6296     35543631 :   vn_reference_t res;
    6297              : 
    6298     35543631 :   tree vuse = gimple_vuse (stmt);
    6299     35543631 :   tree last_vuse = vuse;
    6300     35543631 :   result = vn_reference_lookup (op, vuse, default_vn_walk_kind, &res, true, &last_vuse);
    6301              : 
    6302              :   /* We handle type-punning through unions by value-numbering based
    6303              :      on offset and size of the access.  Be prepared to handle a
    6304              :      type-mismatch here via creating a VIEW_CONVERT_EXPR.  */
    6305     35543631 :   if (result
    6306     35543631 :       && !useless_type_conversion_p (TREE_TYPE (result), TREE_TYPE (op)))
    6307              :     {
    6308        18948 :       if (CONSTANT_CLASS_P (result))
    6309         4265 :         result = const_unop (VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
    6310              :       else
    6311              :         {
    6312              :           /* We will be setting the value number of lhs to the value number
    6313              :              of VIEW_CONVERT_EXPR <TREE_TYPE (result)> (result).
    6314              :              So first simplify and lookup this expression to see if it
    6315              :              is already available.  */
    6316        14683 :           gimple_match_op res_op (gimple_match_cond::UNCOND,
    6317        14683 :                                   VIEW_CONVERT_EXPR, TREE_TYPE (op), result);
    6318        14683 :           result = vn_nary_build_or_lookup (&res_op);
    6319        14683 :           if (result
    6320        14677 :               && TREE_CODE (result) == SSA_NAME
    6321        27701 :               && VN_INFO (result)->needs_insertion)
    6322              :             /* Track whether this is the canonical expression for different
    6323              :                typed loads.  We use that as a stopgap measure for code
    6324              :                hoisting when dealing with floating point loads.  */
    6325        11365 :             res->punned = true;
    6326              :         }
    6327              : 
    6328              :       /* When building the conversion fails avoid inserting the reference
    6329              :          again.  */
    6330        18948 :       if (!result)
    6331            6 :         return set_ssa_val_to (lhs, lhs);
    6332              :     }
    6333              : 
    6334     35524683 :   if (result)
    6335      5799334 :     changed = set_ssa_val_to (lhs, result);
    6336              :   else
    6337              :     {
    6338     29744291 :       changed = set_ssa_val_to (lhs, lhs);
    6339     29744291 :       vn_reference_insert (op, lhs, last_vuse, NULL_TREE);
    6340     29744291 :       if (vuse && SSA_VAL (last_vuse) != SSA_VAL (vuse))
    6341              :         {
    6342      9312832 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6343              :             {
    6344        23641 :               fprintf (dump_file, "Using extra use virtual operand ");
    6345        23641 :               print_generic_expr (dump_file, last_vuse);
    6346        23641 :               fprintf (dump_file, "\n");
    6347              :             }
    6348      9312832 :           vn_reference_insert (op, lhs, vuse, NULL_TREE);
    6349              :         }
    6350              :     }
    6351              : 
    6352              :   return changed;
    6353              : }
    6354              : 
    6355              : 
    6356              : /* Visit a store to a reference operator LHS, part of STMT, value number it,
    6357              :    and return true if the value number of the LHS has changed as a result.  */
    6358              : 
    6359              : static bool
    6360     33945481 : visit_reference_op_store (tree lhs, tree op, gimple *stmt)
    6361              : {
    6362     33945481 :   bool changed = false;
    6363     33945481 :   vn_reference_t vnresult = NULL;
    6364     33945481 :   tree assign;
    6365     33945481 :   bool resultsame = false;
    6366     33945481 :   tree vuse = gimple_vuse (stmt);
    6367     33945481 :   tree vdef = gimple_vdef (stmt);
    6368              : 
    6369     33945481 :   if (TREE_CODE (op) == SSA_NAME)
    6370     15446001 :     op = SSA_VAL (op);
    6371              : 
    6372              :   /* First we want to lookup using the *vuses* from the store and see
    6373              :      if there the last store to this location with the same address
    6374              :      had the same value.
    6375              : 
    6376              :      The vuses represent the memory state before the store.  If the
    6377              :      memory state, address, and value of the store is the same as the
    6378              :      last store to this location, then this store will produce the
    6379              :      same memory state as that store.
    6380              : 
    6381              :      In this case the vdef versions for this store are value numbered to those
    6382              :      vuse versions, since they represent the same memory state after
    6383              :      this store.
    6384              : 
    6385              :      Otherwise, the vdefs for the store are used when inserting into
    6386              :      the table, since the store generates a new memory state.  */
    6387              : 
    6388     33945481 :   vn_reference_lookup (lhs, vuse, VN_NOWALK, &vnresult, false);
    6389     33945481 :   if (vnresult
    6390      1731579 :       && vnresult->result)
    6391              :     {
    6392      1731579 :       tree result = vnresult->result;
    6393      1731579 :       gcc_checking_assert (TREE_CODE (result) != SSA_NAME
    6394              :                            || result == SSA_VAL (result));
    6395      1731579 :       resultsame = expressions_equal_p (result, op);
    6396      1731579 :       if (resultsame)
    6397              :         {
    6398              :           /* If the TBAA state isn't compatible for downstream reads
    6399              :              we cannot value-number the VDEFs the same.  */
    6400        56702 :           ao_ref lhs_ref;
    6401        56702 :           ao_ref_init (&lhs_ref, lhs);
    6402        56702 :           alias_set_type set = ao_ref_alias_set (&lhs_ref);
    6403        56702 :           alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
    6404        56702 :           if ((vnresult->set != set
    6405          927 :                && ! alias_set_subset_of (set, vnresult->set))
    6406        57324 :               || (vnresult->base_set != base_set
    6407         8275 :                   && ! alias_set_subset_of (base_set, vnresult->base_set)))
    6408         2712 :             resultsame = false;
    6409              :         }
    6410              :     }
    6411              : 
    6412         2712 :   if (!resultsame)
    6413              :     {
    6414     33891491 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6415              :         {
    6416        20949 :           fprintf (dump_file, "No store match\n");
    6417        20949 :           fprintf (dump_file, "Value numbering store ");
    6418        20949 :           print_generic_expr (dump_file, lhs);
    6419        20949 :           fprintf (dump_file, " to ");
    6420        20949 :           print_generic_expr (dump_file, op);
    6421        20949 :           fprintf (dump_file, "\n");
    6422              :         }
    6423              :       /* Have to set value numbers before insert, since insert is
    6424              :          going to valueize the references in-place.  */
    6425     33891491 :       if (vdef)
    6426     33891491 :         changed |= set_ssa_val_to (vdef, vdef);
    6427              : 
    6428              :       /* Do not insert structure copies into the tables.  */
    6429     33891491 :       if (is_gimple_min_invariant (op)
    6430     33891491 :           || is_gimple_reg (op))
    6431     30201603 :         vn_reference_insert (lhs, op, vdef, NULL);
    6432              : 
    6433              :       /* Only perform the following when being called from PRE
    6434              :          which embeds tail merging.  */
    6435     33891491 :       if (default_vn_walk_kind == VN_WALK)
    6436              :         {
    6437      7666146 :           assign = build2 (MODIFY_EXPR, TREE_TYPE (lhs), lhs, op);
    6438      7666146 :           vn_reference_lookup (assign, vuse, VN_NOWALK, &vnresult, false);
    6439      7666146 :           if (!vnresult)
    6440      7624641 :             vn_reference_insert (assign, lhs, vuse, vdef);
    6441              :         }
    6442              :     }
    6443              :   else
    6444              :     {
    6445              :       /* We had a match, so value number the vdef to have the value
    6446              :          number of the vuse it came from.  */
    6447              : 
    6448        53990 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6449            9 :         fprintf (dump_file, "Store matched earlier value, "
    6450              :                  "value numbering store vdefs to matching vuses.\n");
    6451              : 
    6452        53990 :       changed |= set_ssa_val_to (vdef, SSA_VAL (vuse));
    6453              :     }
    6454              : 
    6455     33945481 :   return changed;
    6456              : }
    6457              : 
    6458              : /* Visit and value number PHI, return true if the value number
    6459              :    changed.  When BACKEDGES_VARYING_P is true then assume all
    6460              :    backedge values are varying.  When INSERTED is not NULL then
    6461              :    this is just a ahead query for a possible iteration, set INSERTED
    6462              :    to true if we'd insert into the hashtable.  */
    6463              : 
    6464              : static bool
    6465     35153346 : visit_phi (gimple *phi, bool *inserted, bool backedges_varying_p)
    6466              : {
    6467     35153346 :   tree result, sameval = VN_TOP, seen_undef = NULL_TREE;
    6468     35153346 :   bool seen_undef_visited = false;
    6469     35153346 :   tree backedge_val = NULL_TREE;
    6470     35153346 :   bool seen_non_backedge = false;
    6471     35153346 :   tree sameval_base = NULL_TREE;
    6472     35153346 :   poly_int64 soff, doff;
    6473     35153346 :   unsigned n_executable = 0;
    6474     35153346 :   edge sameval_e = NULL;
    6475              : 
    6476              :   /* TODO: We could check for this in initialization, and replace this
    6477              :      with a gcc_assert.  */
    6478     35153346 :   if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)))
    6479        31672 :     return set_ssa_val_to (PHI_RESULT (phi), PHI_RESULT (phi));
    6480              : 
    6481              :   /* We track whether a PHI was CSEd to avoid excessive iterations
    6482              :      that would be necessary only because the PHI changed arguments
    6483              :      but not value.  */
    6484     35121674 :   if (!inserted)
    6485     27531737 :     gimple_set_plf (phi, GF_PLF_1, false);
    6486              : 
    6487     35121674 :   basic_block bb = gimple_bb (phi);
    6488              : 
    6489              :   /* For the equivalence handling below make sure to first process an
    6490              :      edge with a non-constant.  */
    6491     35121674 :   auto_vec<edge, 2> preds;
    6492     70243348 :   preds.reserve_exact (EDGE_COUNT (bb->preds));
    6493     35121674 :   bool seen_nonconstant = false;
    6494    150208377 :   for (unsigned i = 0; i < EDGE_COUNT (bb->preds); ++i)
    6495              :     {
    6496     79965029 :       edge e = EDGE_PRED (bb, i);
    6497     79965029 :       preds.quick_push (e);
    6498     79965029 :       if (!seen_nonconstant)
    6499              :         {
    6500     42946664 :           tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
    6501     42946664 :           if (TREE_CODE (def) == SSA_NAME)
    6502              :             {
    6503     33340742 :               seen_nonconstant = true;
    6504     33340742 :               if (i != 0)
    6505      5805419 :                 std::swap (preds[0], preds[i]);
    6506              :             }
    6507              :         }
    6508              :     }
    6509              : 
    6510              :   /* See if all non-TOP arguments have the same value.  TOP is
    6511              :      equivalent to everything, so we can ignore it.  */
    6512    147365381 :   for (edge e : preds)
    6513     69267734 :     if (e->flags & EDGE_EXECUTABLE)
    6514              :       {
    6515     64443444 :         tree def = PHI_ARG_DEF_FROM_EDGE (phi, e);
    6516              : 
    6517     64443444 :         if (def == PHI_RESULT (phi))
    6518       153654 :           continue;
    6519     64314691 :         ++n_executable;
    6520     64314691 :         bool visited = true;
    6521     64314691 :         if (TREE_CODE (def) == SSA_NAME)
    6522              :           {
    6523     51919927 :             tree val = SSA_VAL (def, &visited);
    6524     51919927 :             if (SSA_NAME_IS_DEFAULT_DEF (def))
    6525      2705964 :               visited = true;
    6526     51919927 :             if (!backedges_varying_p || !(e->flags & EDGE_DFS_BACK))
    6527     49308752 :               def = val;
    6528     51919927 :             if (e->flags & EDGE_DFS_BACK)
    6529     15725920 :               backedge_val = def;
    6530              :           }
    6531     64314691 :         if (!(e->flags & EDGE_DFS_BACK))
    6532     48455008 :           seen_non_backedge = true;
    6533     64314691 :         if (def == VN_TOP)
    6534              :           ;
    6535              :         /* Ignore undefined defs for sameval but record one.  */
    6536     64314691 :         else if (TREE_CODE (def) == SSA_NAME
    6537     48448889 :                  && ! virtual_operand_p (def)
    6538     88986355 :                  && ssa_undefined_value_p (def, false))
    6539              :           {
    6540       230782 :             if (!seen_undef
    6541              :                 /* Avoid having not visited undefined defs if we also have
    6542              :                    a visited one.  */
    6543        35102 :                 || (!seen_undef_visited && visited))
    6544              :               {
    6545       195684 :                 seen_undef = def;
    6546       195684 :                 seen_undef_visited = visited;
    6547              :               }
    6548              :           }
    6549     64083909 :         else if (sameval == VN_TOP)
    6550              :           {
    6551              :             sameval = def;
    6552              :             sameval_e = e;
    6553              :           }
    6554     29008228 :         else if (expressions_equal_p (def, sameval))
    6555              :           sameval_e = NULL;
    6556     45587097 :         else if (virtual_operand_p (def))
    6557              :           {
    6558              :             sameval = NULL_TREE;
    6559     27267375 :             break;
    6560              :           }
    6561              :         else
    6562              :           {
    6563              :             /* We know we're arriving only with invariant addresses here,
    6564              :                try harder comparing them.  We can do some caching here
    6565              :                which we cannot do in expressions_equal_p.  */
    6566     17056802 :             if (TREE_CODE (def) == ADDR_EXPR
    6567       394257 :                 && TREE_CODE (sameval) == ADDR_EXPR
    6568       109382 :                 && sameval_base != (void *)-1)
    6569              :               {
    6570       109382 :                 if (!sameval_base)
    6571       109380 :                   sameval_base = get_addr_base_and_unit_offset
    6572       109380 :                                    (TREE_OPERAND (sameval, 0), &soff);
    6573       109380 :                 if (!sameval_base)
    6574              :                   sameval_base = (tree)(void *)-1;
    6575       109387 :                 else if ((get_addr_base_and_unit_offset
    6576       109382 :                             (TREE_OPERAND (def, 0), &doff) == sameval_base)
    6577       109382 :                          && known_eq (soff, doff))
    6578            5 :                   continue;
    6579              :               }
    6580              :             /* There's also the possibility to use equivalences.  */
    6581     33012357 :             if (!FLOAT_TYPE_P (TREE_TYPE (def))
    6582              :                 /* But only do this if we didn't force any of sameval or
    6583              :                    val to VARYING because of backedge processing rules.  */
    6584     15849360 :                 && (TREE_CODE (sameval) != SSA_NAME
    6585     12518129 :                     || SSA_VAL (sameval) == sameval)
    6586     32906096 :                 && (TREE_CODE (def) != SSA_NAME || SSA_VAL (def) == def))
    6587              :               {
    6588     15849287 :                 vn_nary_op_t vnresult;
    6589     15849287 :                 tree ops[2];
    6590     15849287 :                 ops[0] = def;
    6591     15849287 :                 ops[1] = sameval;
    6592              :                 /* Canonicalize the operands order for eq below. */
    6593     15849287 :                 if (tree_swap_operands_p (ops[0], ops[1]))
    6594      9446984 :                   std::swap (ops[0], ops[1]);
    6595     15849287 :                 tree val = vn_nary_op_lookup_pieces (2, EQ_EXPR,
    6596              :                                                      boolean_type_node,
    6597              :                                                      ops, &vnresult);
    6598     15849287 :                 if (! val && vnresult && vnresult->predicated_values)
    6599              :                   {
    6600       204344 :                     val = vn_nary_op_get_predicated_value (vnresult, e);
    6601       112365 :                     if (val && integer_truep (val)
    6602       229360 :                         && !(sameval_e && (sameval_e->flags & EDGE_DFS_BACK)))
    6603              :                       {
    6604        24896 :                         if (dump_file && (dump_flags & TDF_DETAILS))
    6605              :                           {
    6606            2 :                             fprintf (dump_file, "Predication says ");
    6607            2 :                             print_generic_expr (dump_file, def, TDF_NONE);
    6608            2 :                             fprintf (dump_file, " and ");
    6609            2 :                             print_generic_expr (dump_file, sameval, TDF_NONE);
    6610            2 :                             fprintf (dump_file, " are equal on edge %d -> %d\n",
    6611            2 :                                      e->src->index, e->dest->index);
    6612              :                           }
    6613        24896 :                         continue;
    6614              :                       }
    6615              :                   }
    6616              :               }
    6617              :             sameval = NULL_TREE;
    6618              :             break;
    6619              :           }
    6620              :       }
    6621              : 
    6622              :   /* If the value we want to use is flowing over the backedge and we
    6623              :      should take it as VARYING but it has a non-VARYING value drop to
    6624              :      VARYING.
    6625              :      If we value-number a virtual operand never value-number to the
    6626              :      value from the backedge as that confuses the alias-walking code.
    6627              :      See gcc.dg/torture/pr87176.c.  If the value is the same on a
    6628              :      non-backedge everything is OK though.  */
    6629     35121674 :   bool visited_p;
    6630     35121674 :   if ((backedge_val
    6631     35121674 :        && !seen_non_backedge
    6632          702 :        && TREE_CODE (backedge_val) == SSA_NAME
    6633          491 :        && sameval == backedge_val
    6634          328 :        && (SSA_NAME_IS_VIRTUAL_OPERAND (backedge_val)
    6635           46 :            || SSA_VAL (backedge_val) != backedge_val))
    6636              :       /* Do not value-number a virtual operand to sth not visited though
    6637              :          given that allows us to escape a region in alias walking.  */
    6638     35122094 :       || (sameval
    6639      7854017 :           && TREE_CODE (sameval) == SSA_NAME
    6640      4668410 :           && !SSA_NAME_IS_DEFAULT_DEF (sameval)
    6641      3954196 :           && SSA_NAME_IS_VIRTUAL_OPERAND (sameval)
    6642      2004979 :           && (SSA_VAL (sameval, &visited_p), !visited_p)))
    6643              :     /* Note this just drops to VARYING without inserting the PHI into
    6644              :        the hashes.  */
    6645       305604 :     result = PHI_RESULT (phi);
    6646              :   /* If none of the edges was executable keep the value-number at VN_TOP,
    6647              :      if only a single edge is executable use its value.  */
    6648     34816070 :   else if (n_executable <= 1)
    6649      6762567 :     result = seen_undef ? seen_undef : sameval;
    6650              :   /* If we saw only undefined values and VN_TOP use one of the
    6651              :      undefined values.  */
    6652     28053503 :   else if (sameval == VN_TOP)
    6653         5012 :     result = (seen_undef && seen_undef_visited) ? seen_undef : sameval;
    6654              :   /* First see if it is equivalent to a phi node in this block.  We prefer
    6655              :      this as it allows IV elimination - see PRs 66502 and 67167.  */
    6656     28048491 :   else if ((result = vn_phi_lookup (phi, backedges_varying_p)))
    6657              :     {
    6658      4181298 :       if (!inserted
    6659        70670 :           && TREE_CODE (result) == SSA_NAME
    6660      4251968 :           && gimple_code (SSA_NAME_DEF_STMT (result)) == GIMPLE_PHI)
    6661              :         {
    6662        70670 :           gimple_set_plf (SSA_NAME_DEF_STMT (result), GF_PLF_1, true);
    6663        70670 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6664              :             {
    6665            6 :               fprintf (dump_file, "Marking CSEd to PHI node ");
    6666            6 :               print_gimple_expr (dump_file, SSA_NAME_DEF_STMT (result),
    6667              :                                  0, TDF_SLIM);
    6668            6 :               fprintf (dump_file, "\n");
    6669              :             }
    6670              :         }
    6671              :     }
    6672              :   /* If all values are the same use that, unless we've seen undefined
    6673              :      values as well and the value isn't constant.
    6674              :      CCP/copyprop have the same restriction to not remove uninit warnings.  */
    6675     23867193 :   else if (sameval
    6676     23867193 :            && (! seen_undef || is_gimple_min_invariant (sameval)))
    6677              :     result = sameval;
    6678              :   else
    6679              :     {
    6680     23225852 :       result = PHI_RESULT (phi);
    6681              :       /* Only insert PHIs that are varying, for constant value numbers
    6682              :          we mess up equivalences otherwise as we are only comparing
    6683              :          the immediate controlling predicates.  */
    6684     23225852 :       vn_phi_insert (phi, result, backedges_varying_p);
    6685     23225852 :       if (inserted)
    6686      3347818 :         *inserted = true;
    6687              :     }
    6688              : 
    6689     35121674 :   return set_ssa_val_to (PHI_RESULT (phi), result);
    6690     35121674 : }
    6691              : 
    6692              : /* Try to simplify RHS using equivalences and constant folding.  */
    6693              : 
    6694              : static tree
    6695    130986428 : try_to_simplify (gassign *stmt)
    6696              : {
    6697    130986428 :   enum tree_code code = gimple_assign_rhs_code (stmt);
    6698    130986428 :   tree tem;
    6699              : 
    6700              :   /* For stores we can end up simplifying a SSA_NAME rhs.  Just return
    6701              :      in this case, there is no point in doing extra work.  */
    6702    130986428 :   if (code == SSA_NAME)
    6703              :     return NULL_TREE;
    6704              : 
    6705              :   /* First try constant folding based on our current lattice.  */
    6706    115540085 :   mprts_hook = vn_lookup_simplify_result;
    6707    115540085 :   tem = gimple_fold_stmt_to_constant_1 (stmt, vn_valueize, vn_valueize);
    6708    115540085 :   mprts_hook = NULL;
    6709    115540085 :   if (tem
    6710    115540085 :       && (TREE_CODE (tem) == SSA_NAME
    6711     25710463 :           || is_gimple_min_invariant (tem)))
    6712     25787191 :     return tem;
    6713              : 
    6714              :   return NULL_TREE;
    6715              : }
    6716              : 
    6717              : /* Visit and value number STMT, return true if the value number
    6718              :    changed.  */
    6719              : 
    6720              : static bool
    6721    481383944 : visit_stmt (gimple *stmt, bool backedges_varying_p = false)
    6722              : {
    6723    481383944 :   bool changed = false;
    6724              : 
    6725    481383944 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6726              :     {
    6727       425620 :       fprintf (dump_file, "Value numbering stmt = ");
    6728       425620 :       print_gimple_stmt (dump_file, stmt, 0);
    6729              :     }
    6730              : 
    6731    481383944 :   if (gimple_code (stmt) == GIMPLE_PHI)
    6732     27553542 :     changed = visit_phi (stmt, NULL, backedges_varying_p);
    6733    631003041 :   else if (gimple_has_volatile_ops (stmt))
    6734      9664801 :     changed = defs_to_varying (stmt);
    6735    444165601 :   else if (gassign *ass = dyn_cast <gassign *> (stmt))
    6736              :     {
    6737    136180450 :       enum tree_code code = gimple_assign_rhs_code (ass);
    6738    136180450 :       tree lhs = gimple_assign_lhs (ass);
    6739    136180450 :       tree rhs1 = gimple_assign_rhs1 (ass);
    6740    136180450 :       tree simplified;
    6741              : 
    6742              :       /* Shortcut for copies. Simplifying copies is pointless,
    6743              :          since we copy the expression and value they represent.  */
    6744    136180450 :       if (code == SSA_NAME
    6745     20640365 :           && TREE_CODE (lhs) == SSA_NAME)
    6746              :         {
    6747      5194022 :           changed = visit_copy (lhs, rhs1);
    6748      5194022 :           goto done;
    6749              :         }
    6750    130986428 :       simplified = try_to_simplify (ass);
    6751    130986428 :       if (simplified)
    6752              :         {
    6753     25787191 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6754              :             {
    6755        15109 :               fprintf (dump_file, "RHS ");
    6756        15109 :               print_gimple_expr (dump_file, ass, 0);
    6757        15109 :               fprintf (dump_file, " simplified to ");
    6758        15109 :               print_generic_expr (dump_file, simplified);
    6759        15109 :               fprintf (dump_file, "\n");
    6760              :             }
    6761              :         }
    6762              :       /* Setting value numbers to constants will occasionally
    6763              :          screw up phi congruence because constants are not
    6764              :          uniquely associated with a single ssa name that can be
    6765              :          looked up.  */
    6766     25787191 :       if (simplified
    6767     25787191 :           && is_gimple_min_invariant (simplified)
    6768     22728900 :           && TREE_CODE (lhs) == SSA_NAME)
    6769              :         {
    6770      7913437 :           changed = set_ssa_val_to (lhs, simplified);
    6771      7913437 :           goto done;
    6772              :         }
    6773    123072991 :       else if (simplified
    6774     17873754 :                && TREE_CODE (simplified) == SSA_NAME
    6775      3058291 :                && TREE_CODE (lhs) == SSA_NAME)
    6776              :         {
    6777      3058291 :           changed = visit_copy (lhs, simplified);
    6778      3058291 :           goto done;
    6779              :         }
    6780              : 
    6781    120014700 :       if ((TREE_CODE (lhs) == SSA_NAME
    6782              :            /* We can substitute SSA_NAMEs that are live over
    6783              :               abnormal edges with their constant value.  */
    6784     86068911 :            && !(gimple_assign_copy_p (ass)
    6785           34 :                 && is_gimple_min_invariant (rhs1))
    6786     86068877 :            && !(simplified
    6787            0 :                 && is_gimple_min_invariant (simplified))
    6788     86068877 :            && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
    6789              :           /* Stores or copies from SSA_NAMEs that are live over
    6790              :              abnormal edges are a problem.  */
    6791    206082167 :           || (code == SSA_NAME
    6792     15446343 :               && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
    6793         1752 :         changed = defs_to_varying (ass);
    6794    120012948 :       else if (REFERENCE_CLASS_P (lhs)
    6795    120012948 :                || DECL_P (lhs))
    6796     33945481 :         changed = visit_reference_op_store (lhs, rhs1, ass);
    6797     86067467 :       else if (TREE_CODE (lhs) == SSA_NAME)
    6798              :         {
    6799     86067467 :           if ((gimple_assign_copy_p (ass)
    6800           34 :                && is_gimple_min_invariant (rhs1))
    6801     86067501 :               || (simplified
    6802            0 :                   && is_gimple_min_invariant (simplified)))
    6803              :             {
    6804            0 :               if (simplified)
    6805            0 :                 changed = set_ssa_val_to (lhs, simplified);
    6806              :               else
    6807            0 :                 changed = set_ssa_val_to (lhs, rhs1);
    6808              :             }
    6809              :           else
    6810              :             {
    6811              :               /* Visit the original statement.  */
    6812     86067467 :               switch (vn_get_stmt_kind (ass))
    6813              :                 {
    6814     50420182 :                 case VN_NARY:
    6815     50420182 :                   changed = visit_nary_op (lhs, ass);
    6816     50420182 :                   break;
    6817     35543631 :                 case VN_REFERENCE:
    6818     35543631 :                   changed = visit_reference_op_load (lhs, rhs1, ass);
    6819     35543631 :                   break;
    6820       103654 :                 default:
    6821       103654 :                   changed = defs_to_varying (ass);
    6822       103654 :                   break;
    6823              :                 }
    6824              :             }
    6825              :         }
    6826              :       else
    6827            0 :         changed = defs_to_varying (ass);
    6828              :     }
    6829    307985151 :   else if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
    6830              :     {
    6831     25525201 :       tree lhs = gimple_call_lhs (call_stmt);
    6832     25525201 :       if (lhs && TREE_CODE (lhs) == SSA_NAME)
    6833              :         {
    6834              :           /* Try constant folding based on our current lattice.  */
    6835      8595495 :           tree simplified = gimple_fold_stmt_to_constant_1 (call_stmt,
    6836              :                                                             vn_valueize);
    6837      8595495 :           if (simplified)
    6838              :             {
    6839        68533 :               if (dump_file && (dump_flags & TDF_DETAILS))
    6840              :                 {
    6841            1 :                   fprintf (dump_file, "call ");
    6842            1 :                   print_gimple_expr (dump_file, call_stmt, 0);
    6843            1 :                   fprintf (dump_file, " simplified to ");
    6844            1 :                   print_generic_expr (dump_file, simplified);
    6845            1 :                   fprintf (dump_file, "\n");
    6846              :                 }
    6847              :             }
    6848              :           /* Setting value numbers to constants will occasionally
    6849              :              screw up phi congruence because constants are not
    6850              :              uniquely associated with a single ssa name that can be
    6851              :              looked up.  */
    6852        68533 :           if (simplified
    6853        68533 :               && is_gimple_min_invariant (simplified))
    6854              :             {
    6855        62000 :               changed = set_ssa_val_to (lhs, simplified);
    6856       124000 :               if (gimple_vdef (call_stmt))
    6857          751 :                 changed |= set_ssa_val_to (gimple_vdef (call_stmt),
    6858              :                                            SSA_VAL (gimple_vuse (call_stmt)));
    6859        62000 :               goto done;
    6860              :             }
    6861      8533495 :           else if (simplified
    6862         6533 :                    && TREE_CODE (simplified) == SSA_NAME)
    6863              :             {
    6864          309 :               changed = visit_copy (lhs, simplified);
    6865          618 :               if (gimple_vdef (call_stmt))
    6866            0 :                 changed |= set_ssa_val_to (gimple_vdef (call_stmt),
    6867              :                                            SSA_VAL (gimple_vuse (call_stmt)));
    6868          309 :               goto done;
    6869              :             }
    6870      8533186 :           else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
    6871              :             {
    6872          414 :               changed = defs_to_varying (call_stmt);
    6873          414 :               goto done;
    6874              :             }
    6875              :         }
    6876              : 
    6877              :       /* Pick up flags from a devirtualization target.  */
    6878     25462478 :       tree fn = gimple_call_fn (stmt);
    6879     25462478 :       int extra_fnflags = 0;
    6880     25462478 :       if (fn && TREE_CODE (fn) == SSA_NAME)
    6881              :         {
    6882       543105 :           fn = SSA_VAL (fn);
    6883       543105 :           if (TREE_CODE (fn) == ADDR_EXPR
    6884       543105 :               && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL)
    6885         5454 :             extra_fnflags = flags_from_decl_or_type (TREE_OPERAND (fn, 0));
    6886              :         }
    6887     25462478 :       if ((/* Calls to the same function with the same vuse
    6888              :               and the same operands do not necessarily return the same
    6889              :               value, unless they're pure or const.  */
    6890     25462478 :            ((gimple_call_flags (call_stmt) | extra_fnflags)
    6891     25462478 :             & (ECF_PURE | ECF_CONST))
    6892              :            /* If calls have a vdef, subsequent calls won't have
    6893              :               the same incoming vuse.  So, if 2 calls with vdef have the
    6894              :               same vuse, we know they're not subsequent.
    6895              :               We can value number 2 calls to the same function with the
    6896              :               same vuse and the same operands which are not subsequent
    6897              :               the same, because there is no code in the program that can
    6898              :               compare the 2 values...  */
    6899     21462345 :            || (gimple_vdef (call_stmt)
    6900              :                /* ... unless the call returns a pointer which does
    6901              :                   not alias with anything else.  In which case the
    6902              :                   information that the values are distinct are encoded
    6903              :                   in the IL.  */
    6904     21427465 :                && !(gimple_call_return_flags (call_stmt) & ERF_NOALIAS)
    6905              :                /* Only perform the following when being called from PRE
    6906              :                   which embeds tail merging.  */
    6907     20849435 :                && default_vn_walk_kind == VN_WALK))
    6908              :           /* Do not process .DEFERRED_INIT since that confuses uninit
    6909              :              analysis.  */
    6910     30551275 :           && !gimple_call_internal_p (call_stmt, IFN_DEFERRED_INIT))
    6911      8812461 :         changed = visit_reference_op_call (lhs, call_stmt);
    6912              :       else
    6913     16650017 :         changed = defs_to_varying (call_stmt);
    6914              :     }
    6915              :   else
    6916    282459950 :     changed = defs_to_varying (stmt);
    6917    481383944 :  done:
    6918    481383944 :   return changed;
    6919              : }
    6920              : 
    6921              : 
    6922              : /* Allocate a value number table.  */
    6923              : 
    6924              : static void
    6925      6308971 : allocate_vn_table (vn_tables_t table, unsigned size)
    6926              : {
    6927      6308971 :   table->phis = new vn_phi_table_type (size);
    6928      6308971 :   table->nary = new vn_nary_op_table_type (size);
    6929      6308971 :   table->references = new vn_reference_table_type (size);
    6930      6308971 :   table->inserted_exprs = NULL;
    6931      6308971 : }
    6932              : 
    6933              : /* Free a value number table.  */
    6934              : 
    6935              : static void
    6936      6308971 : free_vn_table (vn_tables_t table)
    6937              : {
    6938              :   /* Walk over elements and release vectors.  */
    6939      6308971 :   vn_reference_iterator_type hir;
    6940      6308971 :   vn_reference_t vr;
    6941     78630583 :   FOR_EACH_HASH_TABLE_ELEMENT (*table->references, vr, vn_reference_t, hir)
    6942     72321612 :     vr->operands.release ();
    6943      6308971 :   delete table->phis;
    6944      6308971 :   table->phis = NULL;
    6945      6308971 :   delete table->nary;
    6946      6308971 :   table->nary = NULL;
    6947      6308971 :   delete table->references;
    6948      6308971 :   table->references = NULL;
    6949      6308971 :   delete table->inserted_exprs;
    6950      6308971 :   table->inserted_exprs = NULL;
    6951      6308971 : }
    6952              : 
    6953              : /* Set *ID according to RESULT.  */
    6954              : 
    6955              : static void
    6956     35346982 : set_value_id_for_result (tree result, unsigned int *id)
    6957              : {
    6958     35346982 :   if (result && TREE_CODE (result) == SSA_NAME)
    6959     22006699 :     *id = VN_INFO (result)->value_id;
    6960      9976556 :   else if (result && is_gimple_min_invariant (result))
    6961      3768340 :     *id = get_or_alloc_constant_value_id (result);
    6962              :   else
    6963      9571943 :     *id = get_next_value_id ();
    6964     35346982 : }
    6965              : 
    6966              : /* Set the value ids in the valid hash tables.  */
    6967              : 
    6968              : static void
    6969       982970 : set_hashtable_value_ids (void)
    6970              : {
    6971       982970 :   vn_nary_op_iterator_type hin;
    6972       982970 :   vn_phi_iterator_type hip;
    6973       982970 :   vn_reference_iterator_type hir;
    6974       982970 :   vn_nary_op_t vno;
    6975       982970 :   vn_reference_t vr;
    6976       982970 :   vn_phi_t vp;
    6977              : 
    6978              :   /* Now set the value ids of the things we had put in the hash
    6979              :      table.  */
    6980              : 
    6981     25235892 :   FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->nary, vno, vn_nary_op_t, hin)
    6982     24252922 :     if (! vno->predicated_values)
    6983      7955608 :       set_value_id_for_result (vno->u.result, &vno->value_id);
    6984              : 
    6985      5084112 :   FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->phis, vp, vn_phi_t, hip)
    6986      4101142 :     set_value_id_for_result (vp->result, &vp->value_id);
    6987              : 
    6988     24273202 :   FOR_EACH_HASH_TABLE_ELEMENT (*valid_info->references, vr, vn_reference_t,
    6989              :                                hir)
    6990     23290232 :     set_value_id_for_result (vr->result, &vr->value_id);
    6991       982970 : }
    6992              : 
    6993              : /* Return the maximum value id we have ever seen.  */
    6994              : 
    6995              : unsigned int
    6996      1965940 : get_max_value_id (void)
    6997              : {
    6998      1965940 :   return next_value_id;
    6999              : }
    7000              : 
    7001              : /* Return the maximum constant value id we have ever seen.  */
    7002              : 
    7003              : unsigned int
    7004      1965940 : get_max_constant_value_id (void)
    7005              : {
    7006      1965940 :   return -next_constant_value_id;
    7007              : }
    7008              : 
    7009              : /* Return the next unique value id.  */
    7010              : 
    7011              : unsigned int
    7012     50412310 : get_next_value_id (void)
    7013              : {
    7014     50412310 :   gcc_checking_assert ((int)next_value_id > 0);
    7015     50412310 :   return next_value_id++;
    7016              : }
    7017              : 
    7018              : /* Return the next unique value id for constants.  */
    7019              : 
    7020              : unsigned int
    7021      2592572 : get_next_constant_value_id (void)
    7022              : {
    7023      2592572 :   gcc_checking_assert (next_constant_value_id < 0);
    7024      2592572 :   return next_constant_value_id--;
    7025              : }
    7026              : 
    7027              : 
    7028              : /* Compare two expressions E1 and E2 and return true if they are equal.
    7029              :    If match_vn_top_optimistically is true then VN_TOP is equal to anything,
    7030              :    otherwise VN_TOP only matches VN_TOP.  */
    7031              : 
    7032              : bool
    7033    253080682 : expressions_equal_p (tree e1, tree e2, bool match_vn_top_optimistically)
    7034              : {
    7035              :   /* The obvious case.  */
    7036    253080682 :   if (e1 == e2)
    7037              :     return true;
    7038              : 
    7039              :   /* If either one is VN_TOP consider them equal.  */
    7040     71919247 :   if (match_vn_top_optimistically
    7041     67047229 :       && (e1 == VN_TOP || e2 == VN_TOP))
    7042              :     return true;
    7043              : 
    7044              :   /* If only one of them is null, they cannot be equal.  While in general
    7045              :      this should not happen for operations like TARGET_MEM_REF some
    7046              :      operands are optional and an identity value we could substitute
    7047              :      has differing semantics.  */
    7048     71919247 :   if (!e1 || !e2)
    7049              :     return false;
    7050              : 
    7051              :   /* SSA_NAME compare pointer equal.  */
    7052     71919247 :   if (TREE_CODE (e1) == SSA_NAME || TREE_CODE (e2) == SSA_NAME)
    7053              :     return false;
    7054              : 
    7055              :   /* Now perform the actual comparison.  */
    7056     36117612 :   if (TREE_CODE (e1) == TREE_CODE (e2)
    7057     36117612 :       && operand_equal_p (e1, e2, OEP_PURE_SAME))
    7058              :     return true;
    7059              : 
    7060              :   return false;
    7061              : }
    7062              : 
    7063              : 
    7064              : /* Return true if the nary operation NARY may trap.  This is a copy
    7065              :    of stmt_could_throw_1_p adjusted to the SCCVN IL.  */
    7066              : 
    7067              : bool
    7068      5734563 : vn_nary_may_trap (vn_nary_op_t nary)
    7069              : {
    7070      5734563 :   tree type;
    7071      5734563 :   tree rhs2 = NULL_TREE;
    7072      5734563 :   bool honor_nans = false;
    7073      5734563 :   bool honor_snans = false;
    7074      5734563 :   bool fp_operation = false;
    7075      5734563 :   bool honor_trapv = false;
    7076      5734563 :   bool handled, ret;
    7077      5734563 :   unsigned i;
    7078              : 
    7079      5734563 :   if (nary->opcode == COND_EXPR || nary->opcode == VEC_COND_EXPR)
    7080              :     return false;
    7081              : 
    7082      5728735 :   if (TREE_CODE_CLASS (nary->opcode) == tcc_comparison
    7083              :       || TREE_CODE_CLASS (nary->opcode) == tcc_unary
    7084      5728735 :       || TREE_CODE_CLASS (nary->opcode) == tcc_binary)
    7085              :     {
    7086      5615177 :       if (TREE_CODE_CLASS (nary->opcode) == tcc_comparison
    7087      5478605 :           || nary->opcode == FIX_TRUNC_EXPR)
    7088       146173 :         type = TREE_TYPE (nary->op[0]);
    7089              :       else
    7090      5469004 :         type = nary->type;
    7091      5615177 :       fp_operation = FLOAT_TYPE_P (type);
    7092      5465669 :       if (fp_operation)
    7093              :         {
    7094       149508 :           honor_nans = flag_trapping_math && !flag_finite_math_only;
    7095       149508 :           honor_snans = flag_signaling_nans != 0;
    7096              :         }
    7097      5465669 :       else if (INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type))
    7098              :         honor_trapv = true;
    7099              :     }
    7100      5728735 :   if (nary->length >= 2)
    7101      2301268 :     rhs2 = nary->op[1];
    7102      5728735 :   ret = operation_could_trap_helper_p (nary->opcode, fp_operation,
    7103              :                                        honor_trapv, honor_nans, honor_snans,
    7104              :                                        rhs2, &handled);
    7105      5728735 :   if (handled && ret)
    7106              :     return true;
    7107              : 
    7108     13435528 :   for (i = 0; i < nary->length; ++i)
    7109      7829881 :     if (tree_could_trap_p (nary->op[i]))
    7110              :       return true;
    7111              : 
    7112              :   return false;
    7113              : }
    7114              : 
    7115              : /* Return true if the reference operation REF may trap.  */
    7116              : 
    7117              : bool
    7118       938790 : vn_reference_may_trap (vn_reference_t ref)
    7119              : {
    7120       938790 :   switch (ref->operands[0].opcode)
    7121              :     {
    7122              :     case MODIFY_EXPR:
    7123              :     case CALL_EXPR:
    7124              :       /* We do not handle calls.  */
    7125              :       return true;
    7126              :     case ADDR_EXPR:
    7127              :       /* And toplevel address computations never trap.  */
    7128              :       return false;
    7129              :     default:;
    7130              :     }
    7131              : 
    7132              :   vn_reference_op_t op;
    7133              :   unsigned i;
    7134      2571032 :   FOR_EACH_VEC_ELT (ref->operands, i, op)
    7135              :     {
    7136      2570776 :       switch (op->opcode)
    7137              :         {
    7138              :         case WITH_SIZE_EXPR:
    7139              :         case TARGET_MEM_REF:
    7140              :           /* Always variable.  */
    7141              :           return true;
    7142       703872 :         case COMPONENT_REF:
    7143       703872 :           if (op->op1 && TREE_CODE (op->op1) == SSA_NAME)
    7144              :             return true;
    7145              :           break;
    7146            0 :         case ARRAY_RANGE_REF:
    7147            0 :           if (TREE_CODE (op->op0) == SSA_NAME)
    7148              :             return true;
    7149              :           break;
    7150       209028 :         case ARRAY_REF:
    7151       209028 :           {
    7152       209028 :             if (TREE_CODE (op->op0) != INTEGER_CST)
    7153              :               return true;
    7154              : 
    7155              :             /* !in_array_bounds   */
    7156       187873 :             tree domain_type = TYPE_DOMAIN (ref->operands[i+1].type);
    7157       187873 :             if (!domain_type)
    7158              :               return true;
    7159              : 
    7160       187827 :             tree min = op->op1;
    7161       187827 :             tree max = TYPE_MAX_VALUE (domain_type);
    7162       187827 :             if (!min
    7163       187827 :                 || !max
    7164       174589 :                 || TREE_CODE (min) != INTEGER_CST
    7165       174589 :                 || TREE_CODE (max) != INTEGER_CST)
    7166              :               return true;
    7167              : 
    7168       171792 :             if (tree_int_cst_lt (op->op0, min)
    7169       171792 :                 || tree_int_cst_lt (max, op->op0))
    7170              :               return true;
    7171              : 
    7172              :             break;
    7173              :           }
    7174              :         case MEM_REF:
    7175              :           /* Nothing interesting in itself, the base is separate.  */
    7176              :           break;
    7177              :         /* The following are the address bases.  */
    7178              :         case SSA_NAME:
    7179              :           return true;
    7180       534281 :         case ADDR_EXPR:
    7181       534281 :           if (op->op0)
    7182       534281 :             return tree_could_trap_p (TREE_OPERAND (op->op0, 0));
    7183              :           return false;
    7184      1717408 :         default:;
    7185              :         }
    7186              :     }
    7187              :   return false;
    7188              : }
    7189              : 
    7190     10689987 : eliminate_dom_walker::eliminate_dom_walker (cdi_direction direction,
    7191              :                                             bitmap inserted_exprs_)
    7192     10689987 :   : dom_walker (direction), do_pre (inserted_exprs_ != NULL),
    7193     10689987 :     el_todo (0), eliminations (0), insertions (0),
    7194     10689987 :     inserted_exprs (inserted_exprs_)
    7195              : {
    7196     10689987 :   need_eh_cleanup = BITMAP_ALLOC (NULL);
    7197     10689987 :   need_ab_cleanup = BITMAP_ALLOC (NULL);
    7198     10689987 : }
    7199              : 
    7200     14088033 : eliminate_dom_walker::~eliminate_dom_walker ()
    7201              : {
    7202     10689987 :   BITMAP_FREE (need_eh_cleanup);
    7203     10689987 :   BITMAP_FREE (need_ab_cleanup);
    7204     14088033 : }
    7205              : 
    7206              : /* Return a leader for OP that is available at the current point of the
    7207              :    eliminate domwalk.  */
    7208              : 
    7209              : tree
    7210    187020378 : eliminate_dom_walker::eliminate_avail (basic_block, tree op)
    7211              : {
    7212    187020378 :   tree valnum = VN_INFO (op)->valnum;
    7213    187020378 :   if (TREE_CODE (valnum) == SSA_NAME)
    7214              :     {
    7215    181771727 :       if (SSA_NAME_IS_DEFAULT_DEF (valnum))
    7216              :         return valnum;
    7217    316473281 :       if (avail.length () > SSA_NAME_VERSION (valnum))
    7218              :         {
    7219    142628412 :           tree av = avail[SSA_NAME_VERSION (valnum)];
    7220              :           /* When PRE discovers a new redundancy there's no way to unite
    7221              :              the value classes so it instead inserts a copy old-val = new-val.
    7222              :              Look through such copies here, providing one more level of
    7223              :              simplification at elimination time.  */
    7224    142628412 :           gassign *ass;
    7225    250698331 :           if (av && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (av))))
    7226     76898357 :             if (gimple_assign_rhs_class (ass) == GIMPLE_SINGLE_RHS)
    7227              :               {
    7228     40686737 :                 tree rhs1 = gimple_assign_rhs1 (ass);
    7229     40686737 :                 if (CONSTANT_CLASS_P (rhs1)
    7230     40686737 :                     || (TREE_CODE (rhs1) == SSA_NAME
    7231        27486 :                         && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
    7232              :                   av = rhs1;
    7233              :               }
    7234              :           return av;
    7235              :         }
    7236              :     }
    7237      5248651 :   else if (is_gimple_min_invariant (valnum))
    7238      5248651 :     return valnum;
    7239              :   return NULL_TREE;
    7240              : }
    7241              : 
    7242              : /* At the current point of the eliminate domwalk make OP available.  */
    7243              : 
    7244              : void
    7245     51541459 : eliminate_dom_walker::eliminate_push_avail (basic_block, tree op)
    7246              : {
    7247     51541459 :   tree valnum = VN_INFO (op)->valnum;
    7248     51541459 :   if (TREE_CODE (valnum) == SSA_NAME)
    7249              :     {
    7250     99621340 :       if (avail.length () <= SSA_NAME_VERSION (valnum))
    7251     17351976 :         avail.safe_grow_cleared (SSA_NAME_VERSION (valnum) + 1, true);
    7252     51541459 :       tree pushop = op;
    7253     51541459 :       if (avail[SSA_NAME_VERSION (valnum)])
    7254        46608 :         pushop = avail[SSA_NAME_VERSION (valnum)];
    7255     51541459 :       avail_stack.safe_push (pushop);
    7256     51541459 :       avail[SSA_NAME_VERSION (valnum)] = op;
    7257              :     }
    7258     51541459 : }
    7259              : 
    7260              : /* Insert the expression recorded by SCCVN for VAL at *GSI.  Returns
    7261              :    the leader for the expression if insertion was successful.  */
    7262              : 
    7263              : tree
    7264       141383 : eliminate_dom_walker::eliminate_insert (basic_block bb,
    7265              :                                         gimple_stmt_iterator *gsi, tree val)
    7266              : {
    7267              :   /* We can insert a sequence with a single assignment only.  */
    7268       141383 :   gimple_seq stmts = VN_INFO (val)->expr;
    7269       141383 :   if (!gimple_seq_singleton_p (stmts))
    7270              :     return NULL_TREE;
    7271       141383 :   gassign *stmt = dyn_cast <gassign *> (gimple_seq_first_stmt (stmts));
    7272       141383 :   if (!stmt
    7273       141383 :       || (!CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
    7274              :           && gimple_assign_rhs_code (stmt) != VIEW_CONVERT_EXPR
    7275              :           && gimple_assign_rhs_code (stmt) != NEGATE_EXPR
    7276              :           && gimple_assign_rhs_code (stmt) != BIT_FIELD_REF
    7277              :           && (gimple_assign_rhs_code (stmt) != BIT_AND_EXPR
    7278          111 :               || TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST)))
    7279              :     return NULL_TREE;
    7280              : 
    7281        46613 :   tree op = gimple_assign_rhs1 (stmt);
    7282        46613 :   if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
    7283        46613 :       || gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
    7284        20493 :     op = TREE_OPERAND (op, 0);
    7285        46613 :   tree leader = TREE_CODE (op) == SSA_NAME ? eliminate_avail (bb, op) : op;
    7286        46567 :   if (!leader)
    7287              :     return NULL_TREE;
    7288              : 
    7289        36462 :   tree res;
    7290        36462 :   stmts = NULL;
    7291        55965 :   if (gimple_assign_rhs_code (stmt) == BIT_FIELD_REF)
    7292        34616 :     res = gimple_build (&stmts, BIT_FIELD_REF,
    7293        17308 :                         TREE_TYPE (val), leader,
    7294        17308 :                         TREE_OPERAND (gimple_assign_rhs1 (stmt), 1),
    7295        17308 :                         TREE_OPERAND (gimple_assign_rhs1 (stmt), 2));
    7296        19154 :   else if (gimple_assign_rhs_code (stmt) == BIT_AND_EXPR)
    7297          214 :     res = gimple_build (&stmts, BIT_AND_EXPR,
    7298          107 :                         TREE_TYPE (val), leader, gimple_assign_rhs2 (stmt));
    7299              :   else
    7300        19047 :     res = gimple_build (&stmts, gimple_assign_rhs_code (stmt),
    7301        19047 :                         TREE_TYPE (val), leader);
    7302        36462 :   if (TREE_CODE (res) != SSA_NAME
    7303        36461 :       || SSA_NAME_IS_DEFAULT_DEF (res)
    7304        72923 :       || gimple_bb (SSA_NAME_DEF_STMT (res)))
    7305              :     {
    7306            4 :       gimple_seq_discard (stmts);
    7307              : 
    7308              :       /* During propagation we have to treat SSA info conservatively
    7309              :          and thus we can end up simplifying the inserted expression
    7310              :          at elimination time to sth not defined in stmts.  */
    7311              :       /* But then this is a redundancy we failed to detect.  Which means
    7312              :          res now has two values.  That doesn't play well with how
    7313              :          we track availability here, so give up.  */
    7314            4 :       if (dump_file && (dump_flags & TDF_DETAILS))
    7315              :         {
    7316            0 :           if (TREE_CODE (res) == SSA_NAME)
    7317            0 :             res = eliminate_avail (bb, res);
    7318            0 :           if (res)
    7319              :             {
    7320            0 :               fprintf (dump_file, "Failed to insert expression for value ");
    7321            0 :               print_generic_expr (dump_file, val);
    7322            0 :               fprintf (dump_file, " which is really fully redundant to ");
    7323            0 :               print_generic_expr (dump_file, res);
    7324            0 :               fprintf (dump_file, "\n");
    7325              :             }
    7326              :         }
    7327              : 
    7328              :       return NULL_TREE;
    7329              :     }
    7330              :   else
    7331              :     {
    7332        36458 :       gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    7333        36458 :       vn_ssa_aux_t vn_info = VN_INFO (res);
    7334        36458 :       vn_info->valnum = val;
    7335        36458 :       vn_info->visited = true;
    7336              :     }
    7337              : 
    7338        36458 :   insertions++;
    7339        36458 :   if (dump_file && (dump_flags & TDF_DETAILS))
    7340              :     {
    7341          501 :       fprintf (dump_file, "Inserted ");
    7342          501 :       print_gimple_stmt (dump_file, SSA_NAME_DEF_STMT (res), 0);
    7343              :     }
    7344              : 
    7345              :   return res;
    7346              : }
    7347              : 
    7348              : void
    7349    373184350 : eliminate_dom_walker::eliminate_stmt (basic_block b, gimple_stmt_iterator *gsi)
    7350              : {
    7351    373184350 :   tree sprime = NULL_TREE;
    7352    373184350 :   gimple *stmt = gsi_stmt (*gsi);
    7353    373184350 :   tree lhs = gimple_get_lhs (stmt);
    7354    124502301 :   if (lhs && TREE_CODE (lhs) == SSA_NAME
    7355    171760394 :       && !gimple_has_volatile_ops (stmt)
    7356              :       /* See PR43491.  Do not replace a global register variable when
    7357              :          it is a the RHS of an assignment.  Do replace local register
    7358              :          variables since gcc does not guarantee a local variable will
    7359              :          be allocated in register.
    7360              :          ???  The fix isn't effective here.  This should instead
    7361              :          be ensured by not value-numbering them the same but treating
    7362              :          them like volatiles?  */
    7363    457988939 :       && !(gimple_assign_single_p (stmt)
    7364     36570650 :            && (TREE_CODE (gimple_assign_rhs1 (stmt)) == VAR_DECL
    7365      2531825 :                && DECL_HARD_REGISTER (gimple_assign_rhs1 (stmt))
    7366         4187 :                && is_global_var (gimple_assign_rhs1 (stmt)))))
    7367              :     {
    7368     84804341 :       sprime = eliminate_avail (b, lhs);
    7369     84804341 :       if (!sprime)
    7370              :         {
    7371              :           /* If there is no existing usable leader but SCCVN thinks
    7372              :              it has an expression it wants to use as replacement,
    7373              :              insert that.  */
    7374     71220076 :           tree val = VN_INFO (lhs)->valnum;
    7375     71220076 :           vn_ssa_aux_t vn_info;
    7376     71220076 :           if (val != VN_TOP
    7377     71220076 :               && TREE_CODE (val) == SSA_NAME
    7378     71220076 :               && (vn_info = VN_INFO (val), true)
    7379     71220076 :               && vn_info->needs_insertion
    7380       345506 :               && vn_info->expr != NULL
    7381     71361459 :               && (sprime = eliminate_insert (b, gsi, val)) != NULL_TREE)
    7382        36458 :             eliminate_push_avail (b, sprime);
    7383              :         }
    7384              : 
    7385              :       /* If this now constitutes a copy duplicate points-to
    7386              :          and range info appropriately.  This is especially
    7387              :          important for inserted code.  */
    7388     71220076 :       if (sprime
    7389     13620723 :           && TREE_CODE (sprime) == SSA_NAME)
    7390      9328244 :         maybe_duplicate_ssa_info_at_copy (lhs, sprime);
    7391              : 
    7392              :       /* Inhibit the use of an inserted PHI on a loop header when
    7393              :          the address of the memory reference is a simple induction
    7394              :          variable.  In other cases the vectorizer won't do anything
    7395              :          anyway (either it's loop invariant or a complicated
    7396              :          expression).  */
    7397      9328244 :       if (sprime
    7398     13620723 :           && TREE_CODE (sprime) == SSA_NAME
    7399      9328244 :           && do_pre
    7400       942147 :           && (flag_tree_loop_vectorize || flag_tree_parallelize_loops > 1)
    7401       923491 :           && loop_outer (b->loop_father)
    7402       395017 :           && has_zero_uses (sprime)
    7403       196022 :           && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))
    7404       195820 :           && gimple_assign_load_p (stmt))
    7405              :         {
    7406       106741 :           gimple *def_stmt = SSA_NAME_DEF_STMT (sprime);
    7407       106741 :           basic_block def_bb = gimple_bb (def_stmt);
    7408       106741 :           if (gimple_code (def_stmt) == GIMPLE_PHI
    7409       106741 :               && def_bb->loop_father->header == def_bb)
    7410              :             {
    7411        68243 :               loop_p loop = def_bb->loop_father;
    7412        68243 :               ssa_op_iter iter;
    7413        68243 :               tree op;
    7414        68243 :               bool found = false;
    7415        86274 :               FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
    7416              :                 {
    7417        64595 :                   affine_iv iv;
    7418        64595 :                   def_bb = gimple_bb (SSA_NAME_DEF_STMT (op));
    7419        64595 :                   if (def_bb
    7420        58280 :                       && flow_bb_inside_loop_p (loop, def_bb)
    7421       117435 :                       && simple_iv (loop, loop, op, &iv, true))
    7422              :                     {
    7423        46564 :                       found = true;
    7424        46564 :                       break;
    7425              :                     }
    7426              :                 }
    7427        21679 :               if (found)
    7428              :                 {
    7429        46564 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    7430              :                     {
    7431            3 :                       fprintf (dump_file, "Not replacing ");
    7432            3 :                       print_gimple_expr (dump_file, stmt, 0);
    7433            3 :                       fprintf (dump_file, " with ");
    7434            3 :                       print_generic_expr (dump_file, sprime);
    7435            3 :                       fprintf (dump_file, " which would add a loop"
    7436              :                                " carried dependence to loop %d\n",
    7437              :                                loop->num);
    7438              :                     }
    7439              :                   /* Don't keep sprime available.  */
    7440              :                   sprime = NULL_TREE;
    7441              :                 }
    7442              :             }
    7443              :         }
    7444              : 
    7445     84804341 :       if (sprime)
    7446              :         {
    7447              :           /* If we can propagate the value computed for LHS into
    7448              :              all uses don't bother doing anything with this stmt.  */
    7449     13574159 :           if (may_propagate_copy (lhs, sprime))
    7450              :             {
    7451              :               /* Mark it for removal.  */
    7452     13572072 :               to_remove.safe_push (stmt);
    7453              : 
    7454              :               /* ???  Don't count copy/constant propagations.  */
    7455     13572072 :               if (gimple_assign_single_p (stmt)
    7456     13572072 :                   && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
    7457      4748960 :                       || gimple_assign_rhs1 (stmt) == sprime))
    7458     14449502 :                 return;
    7459              : 
    7460      8267808 :               if (dump_file && (dump_flags & TDF_DETAILS))
    7461              :                 {
    7462        19412 :                   fprintf (dump_file, "Replaced ");
    7463        19412 :                   print_gimple_expr (dump_file, stmt, 0);
    7464        19412 :                   fprintf (dump_file, " with ");
    7465        19412 :                   print_generic_expr (dump_file, sprime);
    7466        19412 :                   fprintf (dump_file, " in all uses of ");
    7467        19412 :                   print_gimple_stmt (dump_file, stmt, 0);
    7468              :                 }
    7469              : 
    7470      8267808 :               eliminations++;
    7471      8267808 :               return;
    7472              :             }
    7473              : 
    7474              :           /* If this is an assignment from our leader (which
    7475              :              happens in the case the value-number is a constant)
    7476              :              then there is nothing to do.  Likewise if we run into
    7477              :              inserted code that needed a conversion because of
    7478              :              our type-agnostic value-numbering of loads.  */
    7479         2087 :           if ((gimple_assign_single_p (stmt)
    7480            1 :                || (is_gimple_assign (stmt)
    7481            1 :                    && (CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
    7482            0 :                        || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)))
    7483         2088 :               && sprime == gimple_assign_rhs1 (stmt))
    7484              :             return;
    7485              : 
    7486              :           /* Else replace its RHS.  */
    7487          719 :           if (dump_file && (dump_flags & TDF_DETAILS))
    7488              :             {
    7489            0 :               fprintf (dump_file, "Replaced ");
    7490            0 :               print_gimple_expr (dump_file, stmt, 0);
    7491            0 :               fprintf (dump_file, " with ");
    7492            0 :               print_generic_expr (dump_file, sprime);
    7493            0 :               fprintf (dump_file, " in ");
    7494            0 :               print_gimple_stmt (dump_file, stmt, 0);
    7495              :             }
    7496          719 :           eliminations++;
    7497              : 
    7498          719 :           bool can_make_abnormal_goto = (is_gimple_call (stmt)
    7499          719 :                                          && stmt_can_make_abnormal_goto (stmt));
    7500          719 :           gimple *orig_stmt = stmt;
    7501          719 :           if (!useless_type_conversion_p (TREE_TYPE (lhs),
    7502          719 :                                           TREE_TYPE (sprime)))
    7503              :             {
    7504              :               /* We preserve conversions to but not from function or method
    7505              :                  types.  This asymmetry makes it necessary to re-instantiate
    7506              :                  conversions here.  */
    7507          717 :               if (POINTER_TYPE_P (TREE_TYPE (lhs))
    7508          717 :                   && FUNC_OR_METHOD_TYPE_P (TREE_TYPE (TREE_TYPE (lhs))))
    7509          717 :                 sprime = fold_convert (TREE_TYPE (lhs), sprime);
    7510              :               else
    7511            0 :                 gcc_unreachable ();
    7512              :             }
    7513          719 :           tree vdef = gimple_vdef (stmt);
    7514          719 :           tree vuse = gimple_vuse (stmt);
    7515          719 :           propagate_tree_value_into_stmt (gsi, sprime);
    7516          719 :           stmt = gsi_stmt (*gsi);
    7517          719 :           update_stmt (stmt);
    7518              :           /* In case the VDEF on the original stmt was released, value-number
    7519              :              it to the VUSE.  This is to make vuse_ssa_val able to skip
    7520              :              released virtual operands.  */
    7521         1438 :           if (vdef != gimple_vdef (stmt))
    7522              :             {
    7523            0 :               gcc_assert (SSA_NAME_IN_FREE_LIST (vdef));
    7524            0 :               VN_INFO (vdef)->valnum = vuse;
    7525              :             }
    7526              : 
    7527              :           /* If we removed EH side-effects from the statement, clean
    7528              :              its EH information.  */
    7529          719 :           if (maybe_clean_or_replace_eh_stmt (orig_stmt, stmt))
    7530              :             {
    7531            0 :               bitmap_set_bit (need_eh_cleanup,
    7532            0 :                               gimple_bb (stmt)->index);
    7533            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    7534            0 :                 fprintf (dump_file, "  Removed EH side-effects.\n");
    7535              :             }
    7536              : 
    7537              :           /* Likewise for AB side-effects.  */
    7538          719 :           if (can_make_abnormal_goto
    7539          719 :               && !stmt_can_make_abnormal_goto (stmt))
    7540              :             {
    7541            0 :               bitmap_set_bit (need_ab_cleanup,
    7542            0 :                               gimple_bb (stmt)->index);
    7543            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    7544            0 :                 fprintf (dump_file, "  Removed AB side-effects.\n");
    7545              :             }
    7546              : 
    7547              :           return;
    7548              :         }
    7549              :     }
    7550              : 
    7551              :   /* If the statement is a scalar store, see if the expression
    7552              :      has the same value number as its rhs.  If so, the store is
    7553              :      dead.  */
    7554    359610191 :   if (gimple_assign_single_p (stmt)
    7555    131622314 :       && !gimple_has_volatile_ops (stmt)
    7556     56962247 :       && !is_gimple_reg (gimple_assign_lhs (stmt))
    7557     29353962 :       && (TREE_CODE (gimple_assign_lhs (stmt)) != VAR_DECL
    7558      2883387 :           || !DECL_HARD_REGISTER (gimple_assign_lhs (stmt)))
    7559    388960070 :       && (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
    7560     16739206 :           || is_gimple_min_invariant (gimple_assign_rhs1 (stmt))))
    7561              :     {
    7562     26192075 :       tree rhs = gimple_assign_rhs1 (stmt);
    7563     26192075 :       vn_reference_t vnresult;
    7564              :       /* ???  gcc.dg/torture/pr91445.c shows that we lookup a boolean
    7565              :          typed load of a byte known to be 0x11 as 1 so a store of
    7566              :          a boolean 1 is detected as redundant.  Because of this we
    7567              :          have to make sure to lookup with a ref where its size
    7568              :          matches the precision.  */
    7569     26192075 :       tree lookup_lhs = lhs;
    7570     52121366 :       if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    7571     13674736 :           && (TREE_CODE (lhs) != COMPONENT_REF
    7572      8258511 :               || !DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
    7573     39660662 :           && !type_has_mode_precision_p (TREE_TYPE (lhs)))
    7574              :         {
    7575       865104 :           if (BITINT_TYPE_P (TREE_TYPE (lhs))
    7576       453519 :               && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
    7577              :             lookup_lhs = NULL_TREE;
    7578       432020 :           else if (TREE_CODE (lhs) == COMPONENT_REF
    7579       432020 :                    || TREE_CODE (lhs) == MEM_REF)
    7580              :             {
    7581       304032 :               tree ltype = build_nonstandard_integer_type
    7582       304032 :                                 (TREE_INT_CST_LOW (TYPE_SIZE (TREE_TYPE (lhs))),
    7583       304032 :                                  TYPE_UNSIGNED (TREE_TYPE (lhs)));
    7584       304032 :               if (TREE_CODE (lhs) == COMPONENT_REF)
    7585              :                 {
    7586       234192 :                   tree bpos, foff = component_ref_field_offset (lhs);
    7587       234192 :                   tree f = TREE_OPERAND (lhs, 1);
    7588       234192 :                   if (!poly_int_tree_p (foff)
    7589       234192 :                       || !tree_fits_poly_uint64_p
    7590       234192 :                            (bpos = bit_from_pos (foff,
    7591       234192 :                                                  DECL_FIELD_BIT_OFFSET (f))))
    7592              :                     lookup_lhs = NULL_TREE;
    7593              :                   else
    7594       234192 :                     lookup_lhs = build3 (BIT_FIELD_REF, ltype,
    7595       234192 :                                          TREE_OPERAND (lhs, 0),
    7596       234192 :                                          TYPE_SIZE (TREE_TYPE (lhs)), bpos);
    7597              :                 }
    7598              :               else
    7599        69840 :                 lookup_lhs = build2 (MEM_REF, ltype,
    7600        69840 :                                      TREE_OPERAND (lhs, 0),
    7601        69840 :                                      TREE_OPERAND (lhs, 1));
    7602              :             }
    7603              :           else
    7604              :             lookup_lhs = NULL_TREE;
    7605              :         }
    7606     26056570 :       tree val = NULL_TREE, tem;
    7607     26056570 :       if (lookup_lhs)
    7608     52113140 :         val = vn_reference_lookup (lookup_lhs, gimple_vuse (stmt),
    7609              :                                    VN_WALKREWRITE, &vnresult, false,
    7610              :                                    NULL, NULL_TREE, true);
    7611     26192075 :       if (TREE_CODE (rhs) == SSA_NAME)
    7612     12610673 :         rhs = VN_INFO (rhs)->valnum;
    7613     26192075 :       gassign *ass;
    7614     26192075 :       if (val
    7615     26192075 :           && (operand_equal_p (val, rhs, 0)
    7616              :               /* Due to the bitfield lookups above we can get bit
    7617              :                  interpretations of the same RHS as values here.  Those
    7618              :                  are redundant as well.  */
    7619      3232212 :               || (TREE_CODE (val) == SSA_NAME
    7620      1966648 :                   && gimple_assign_single_p (SSA_NAME_DEF_STMT (val))
    7621      1795958 :                   && (tem = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (val)))
    7622      1795958 :                   && TREE_CODE (tem) == VIEW_CONVERT_EXPR
    7623         3650 :                   && TREE_OPERAND (tem, 0) == rhs)
    7624      3232210 :               || (TREE_CODE (rhs) == SSA_NAME
    7625     26677193 :                   && (ass = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (rhs)))
    7626      1539412 :                   && gimple_assign_rhs1 (ass) == val
    7627       719078 :                   && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (ass))
    7628            9 :                   && tree_nop_conversion_p (TREE_TYPE (rhs), TREE_TYPE (val)))))
    7629              :         {
    7630              :           /* We can only remove the later store if the former aliases
    7631              :              at least all accesses the later one does or if the store
    7632              :              was to readonly memory storing the same value.  */
    7633       258374 :           ao_ref lhs_ref;
    7634       258374 :           ao_ref_init (&lhs_ref, lhs);
    7635       258374 :           alias_set_type set = ao_ref_alias_set (&lhs_ref);
    7636       258374 :           alias_set_type base_set = ao_ref_base_alias_set (&lhs_ref);
    7637       258374 :           if (! vnresult
    7638       258374 :               || ((vnresult->set == set
    7639        57213 :                    || alias_set_subset_of (set, vnresult->set))
    7640       239307 :                   && (vnresult->base_set == base_set
    7641        26598 :                       || alias_set_subset_of (base_set, vnresult->base_set))))
    7642              :             {
    7643       233960 :               if (dump_file && (dump_flags & TDF_DETAILS))
    7644              :                 {
    7645           17 :                   fprintf (dump_file, "Deleted redundant store ");
    7646           17 :                   print_gimple_stmt (dump_file, stmt, 0);
    7647              :                 }
    7648              : 
    7649              :               /* Queue stmt for removal.  */
    7650       233960 :               to_remove.safe_push (stmt);
    7651       233960 :               return;
    7652              :             }
    7653              :         }
    7654              :     }
    7655              : 
    7656              :   /* If this is a control statement value numbering left edges
    7657              :      unexecuted on force the condition in a way consistent with
    7658              :      that.  */
    7659    359376231 :   if (gcond *cond = dyn_cast <gcond *> (stmt))
    7660              :     {
    7661     19478546 :       if ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE)
    7662     19478546 :           ^ (EDGE_SUCC (b, 1)->flags & EDGE_EXECUTABLE))
    7663              :         {
    7664       641383 :           if (dump_file && (dump_flags & TDF_DETAILS))
    7665              :             {
    7666           15 :               fprintf (dump_file, "Removing unexecutable edge from ");
    7667           15 :               print_gimple_stmt (dump_file, stmt, 0);
    7668              :             }
    7669       641383 :           if (((EDGE_SUCC (b, 0)->flags & EDGE_TRUE_VALUE) != 0)
    7670       641383 :               == ((EDGE_SUCC (b, 0)->flags & EDGE_EXECUTABLE) != 0))
    7671       256516 :             gimple_cond_make_true (cond);
    7672              :           else
    7673       384867 :             gimple_cond_make_false (cond);
    7674       641383 :           update_stmt (cond);
    7675       641383 :           el_todo |= TODO_cleanup_cfg;
    7676       641383 :           return;
    7677              :         }
    7678              :     }
    7679              : 
    7680    358734848 :   bool can_make_abnormal_goto = stmt_can_make_abnormal_goto (stmt);
    7681    358734848 :   bool was_noreturn = (is_gimple_call (stmt)
    7682    358734848 :                        && gimple_call_noreturn_p (stmt));
    7683    358734848 :   tree vdef = gimple_vdef (stmt);
    7684    358734848 :   tree vuse = gimple_vuse (stmt);
    7685              : 
    7686              :   /* If we didn't replace the whole stmt (or propagate the result
    7687              :      into all uses), replace all uses on this stmt with their
    7688              :      leaders.  */
    7689    358734848 :   bool modified = false;
    7690    358734848 :   use_operand_p use_p;
    7691    358734848 :   ssa_op_iter iter;
    7692    529157331 :   FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
    7693              :     {
    7694    170422483 :       tree use = USE_FROM_PTR (use_p);
    7695              :       /* ???  The call code above leaves stmt operands un-updated.  */
    7696    170422483 :       if (TREE_CODE (use) != SSA_NAME)
    7697            0 :         continue;
    7698    170422483 :       tree sprime;
    7699    170422483 :       if (SSA_NAME_IS_DEFAULT_DEF (use))
    7700              :         /* ???  For default defs BB shouldn't matter, but we have to
    7701              :            solve the inconsistency between rpo eliminate and
    7702              :            dom eliminate avail valueization first.  */
    7703     27276810 :         sprime = eliminate_avail (b, use);
    7704              :       else
    7705              :         /* Look for sth available at the definition block of the argument.
    7706              :            This avoids inconsistencies between availability there which
    7707              :            decides if the stmt can be removed and availability at the
    7708              :            use site.  The SSA property ensures that things available
    7709              :            at the definition are also available at uses.  */
    7710    143145673 :         sprime = eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (use)), use);
    7711    170422483 :       if (sprime && sprime != use
    7712     13573674 :           && may_propagate_copy (use, sprime, true)
    7713              :           /* We substitute into debug stmts to avoid excessive
    7714              :              debug temporaries created by removed stmts, but we need
    7715              :              to avoid doing so for inserted sprimes as we never want
    7716              :              to create debug temporaries for them.  */
    7717    183995440 :           && (!inserted_exprs
    7718      1201250 :               || TREE_CODE (sprime) != SSA_NAME
    7719      1183647 :               || !is_gimple_debug (stmt)
    7720       366217 :               || !bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (sprime))))
    7721              :         {
    7722     13241917 :           propagate_value (use_p, sprime);
    7723     13241917 :           modified = true;
    7724              :         }
    7725              :     }
    7726              : 
    7727              :   /* Fold the stmt if modified, this canonicalizes MEM_REFs we propagated
    7728              :      into which is a requirement for the IPA devirt machinery.  */
    7729    358734848 :   gimple *old_stmt = stmt;
    7730    358734848 :   if (modified)
    7731              :     {
    7732              :       /* If a formerly non-invariant ADDR_EXPR is turned into an
    7733              :          invariant one it was on a separate stmt.  */
    7734     12310186 :       if (gimple_assign_single_p (stmt)
    7735     12310186 :           && TREE_CODE (gimple_assign_rhs1 (stmt)) == ADDR_EXPR)
    7736       247394 :         recompute_tree_invariant_for_addr_expr (gimple_assign_rhs1 (stmt));
    7737     12310186 :       gimple_stmt_iterator prev = *gsi;
    7738     12310186 :       gsi_prev (&prev);
    7739     12310186 :       if (fold_stmt (gsi, follow_all_ssa_edges))
    7740              :         {
    7741              :           /* fold_stmt may have created new stmts in between
    7742              :              the previous stmt and the folded stmt.  Mark
    7743              :              all defs created there as varying to not confuse
    7744              :              the SCCVN machinery as we're using that even during
    7745              :              elimination.  */
    7746      1058772 :           if (gsi_end_p (prev))
    7747       231810 :             prev = gsi_start_bb (b);
    7748              :           else
    7749       942867 :             gsi_next (&prev);
    7750      1058772 :           if (gsi_stmt (prev) != gsi_stmt (*gsi))
    7751       106847 :             do
    7752              :               {
    7753        67248 :                 tree def;
    7754        67248 :                 ssa_op_iter dit;
    7755       130575 :                 FOR_EACH_SSA_TREE_OPERAND (def, gsi_stmt (prev),
    7756              :                                            dit, SSA_OP_ALL_DEFS)
    7757              :                     /* As existing DEFs may move between stmts
    7758              :                        only process new ones.  */
    7759        63327 :                     if (! has_VN_INFO (def))
    7760              :                       {
    7761        39497 :                         vn_ssa_aux_t vn_info = VN_INFO (def);
    7762        39497 :                         vn_info->valnum = def;
    7763        39497 :                         vn_info->visited = true;
    7764              :                       }
    7765        67248 :                 if (gsi_stmt (prev) == gsi_stmt (*gsi))
    7766              :                   break;
    7767        39599 :                 gsi_next (&prev);
    7768        39599 :               }
    7769              :             while (1);
    7770              :         }
    7771     12310186 :       stmt = gsi_stmt (*gsi);
    7772              :       /* In case we folded the stmt away schedule the NOP for removal.  */
    7773     12310186 :       if (gimple_nop_p (stmt))
    7774          849 :         to_remove.safe_push (stmt);
    7775              :     }
    7776              : 
    7777              :   /* Visit indirect calls and turn them into direct calls if
    7778              :      possible using the devirtualization machinery.  Do this before
    7779              :      checking for required EH/abnormal/noreturn cleanup as devird
    7780              :      may expose more of those.  */
    7781    358734848 :   if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
    7782              :     {
    7783     23011494 :       tree fn = gimple_call_fn (call_stmt);
    7784     23011494 :       if (fn
    7785     22185507 :           && flag_devirtualize
    7786     44443561 :           && virtual_method_call_p (fn))
    7787              :         {
    7788       184007 :           tree otr_type = obj_type_ref_class (fn);
    7789       184007 :           unsigned HOST_WIDE_INT otr_tok
    7790       184007 :               = tree_to_uhwi (OBJ_TYPE_REF_TOKEN (fn));
    7791       184007 :           tree instance;
    7792       184007 :           ipa_polymorphic_call_context context (current_function_decl,
    7793       184007 :                                                 fn, stmt, &instance);
    7794       184007 :           context.get_dynamic_type (instance, OBJ_TYPE_REF_OBJECT (fn),
    7795              :                                     otr_type, stmt, NULL);
    7796       184007 :           bool final;
    7797       184007 :           vec <cgraph_node *> targets
    7798       184007 :               = possible_polymorphic_call_targets (obj_type_ref_class (fn),
    7799              :                                                    otr_tok, context, &final);
    7800       184007 :           if (dump_file)
    7801           22 :             dump_possible_polymorphic_call_targets (dump_file,
    7802              :                                                     obj_type_ref_class (fn),
    7803              :                                                     otr_tok, context);
    7804       184296 :           if (final && targets.length () <= 1 && dbg_cnt (devirt))
    7805              :             {
    7806           67 :               tree fn;
    7807           67 :               if (targets.length () == 1)
    7808           67 :                 fn = targets[0]->decl;
    7809              :               else
    7810            0 :                 fn = builtin_decl_unreachable ();
    7811           67 :               if (dump_enabled_p ())
    7812              :                 {
    7813            9 :                   dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
    7814              :                                    "converting indirect call to "
    7815              :                                    "function %s\n",
    7816            9 :                                    lang_hooks.decl_printable_name (fn, 2));
    7817              :                 }
    7818           67 :               gimple_call_set_fndecl (call_stmt, fn);
    7819              :               /* If changing the call to __builtin_unreachable
    7820              :                  or similar noreturn function, adjust gimple_call_fntype
    7821              :                  too.  */
    7822           67 :               if (gimple_call_noreturn_p (call_stmt)
    7823            0 :                   && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fn)))
    7824            0 :                   && TYPE_ARG_TYPES (TREE_TYPE (fn))
    7825           67 :                   && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fn)))
    7826            0 :                       == void_type_node))
    7827            0 :                 gimple_call_set_fntype (call_stmt, TREE_TYPE (fn));
    7828           67 :               maybe_remove_unused_call_args (cfun, call_stmt);
    7829           67 :               modified = true;
    7830              :             }
    7831              :         }
    7832              :     }
    7833              : 
    7834    358734848 :   if (modified)
    7835              :     {
    7836              :       /* When changing a call into a noreturn call, cfg cleanup
    7837              :          is needed to fix up the noreturn call.  */
    7838     12310207 :       if (!was_noreturn
    7839     12310207 :           && is_gimple_call (stmt) && gimple_call_noreturn_p (stmt))
    7840           56 :         to_fixup.safe_push  (stmt);
    7841              :       /* When changing a condition or switch into one we know what
    7842              :          edge will be executed, schedule a cfg cleanup.  */
    7843     12310207 :       if ((gimple_code (stmt) == GIMPLE_COND
    7844      1555044 :            && (gimple_cond_true_p (as_a <gcond *> (stmt))
    7845      1549198 :                || gimple_cond_false_p (as_a <gcond *> (stmt))))
    7846     13856513 :           || (gimple_code (stmt) == GIMPLE_SWITCH
    7847         8044 :               && TREE_CODE (gimple_switch_index
    7848              :                             (as_a <gswitch *> (stmt))) == INTEGER_CST))
    7849        10598 :         el_todo |= TODO_cleanup_cfg;
    7850              :       /* If we removed EH side-effects from the statement, clean
    7851              :          its EH information.  */
    7852     12310207 :       if (maybe_clean_or_replace_eh_stmt (old_stmt, stmt))
    7853              :         {
    7854         1909 :           bitmap_set_bit (need_eh_cleanup,
    7855         1909 :                           gimple_bb (stmt)->index);
    7856         1909 :           if (dump_file && (dump_flags & TDF_DETAILS))
    7857            0 :             fprintf (dump_file, "  Removed EH side-effects.\n");
    7858              :         }
    7859              :       /* Likewise for AB side-effects.  */
    7860     12310207 :       if (can_make_abnormal_goto
    7861     12310207 :           && !stmt_can_make_abnormal_goto (stmt))
    7862              :         {
    7863            0 :           bitmap_set_bit (need_ab_cleanup,
    7864            0 :                           gimple_bb (stmt)->index);
    7865            0 :           if (dump_file && (dump_flags & TDF_DETAILS))
    7866            0 :             fprintf (dump_file, "  Removed AB side-effects.\n");
    7867              :         }
    7868     12310207 :       update_stmt (stmt);
    7869              :       /* In case the VDEF on the original stmt was released, value-number
    7870              :          it to the VUSE.  This is to make vuse_ssa_val able to skip
    7871              :          released virtual operands.  */
    7872     15666520 :       if (vdef && SSA_NAME_IN_FREE_LIST (vdef))
    7873         2178 :         VN_INFO (vdef)->valnum = vuse;
    7874              :     }
    7875              : 
    7876              :   /* Make new values available - for fully redundant LHS we
    7877              :      continue with the next stmt above and skip this.
    7878              :      But avoid picking up dead defs.  */
    7879    358734848 :   tree def;
    7880    431310656 :   FOR_EACH_SSA_TREE_OPERAND (def, stmt, iter, SSA_OP_DEF)
    7881     72575808 :     if (! has_zero_uses (def)
    7882     72575808 :         || (inserted_exprs
    7883       214286 :             && bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (def))))
    7884     71130722 :       eliminate_push_avail (b, def);
    7885              : }
    7886              : 
    7887              : /* Perform elimination for the basic-block B during the domwalk.  */
    7888              : 
    7889              : edge
    7890     42420000 : eliminate_dom_walker::before_dom_children (basic_block b)
    7891              : {
    7892              :   /* Mark new bb.  */
    7893     42420000 :   avail_stack.safe_push (NULL_TREE);
    7894              : 
    7895              :   /* Skip unreachable blocks marked unreachable during the SCCVN domwalk.  */
    7896     42420000 :   if (!(b->flags & BB_EXECUTABLE))
    7897              :     return NULL;
    7898              : 
    7899     37467312 :   vn_context_bb = b;
    7900              : 
    7901     49235173 :   for (gphi_iterator gsi = gsi_start_phis (b); !gsi_end_p (gsi);)
    7902              :     {
    7903     11767861 :       gphi *phi = gsi.phi ();
    7904     11767861 :       tree res = PHI_RESULT (phi);
    7905              : 
    7906     23535722 :       if (virtual_operand_p (res))
    7907              :         {
    7908      5416479 :           gsi_next (&gsi);
    7909      5416479 :           continue;
    7910              :         }
    7911              : 
    7912      6351382 :       tree sprime = eliminate_avail (b, res);
    7913      6351382 :       if (sprime
    7914      6351382 :           && sprime != res)
    7915              :         {
    7916       439706 :           if (dump_file && (dump_flags & TDF_DETAILS))
    7917              :             {
    7918           20 :               fprintf (dump_file, "Replaced redundant PHI node defining ");
    7919           20 :               print_generic_expr (dump_file, res);
    7920           20 :               fprintf (dump_file, " with ");
    7921           20 :               print_generic_expr (dump_file, sprime);
    7922           20 :               fprintf (dump_file, "\n");
    7923              :             }
    7924              : 
    7925              :           /* If we inserted this PHI node ourself, it's not an elimination.  */
    7926       439706 :           if (! inserted_exprs
    7927       560925 :               || ! bitmap_bit_p (inserted_exprs, SSA_NAME_VERSION (res)))
    7928       413009 :             eliminations++;
    7929              : 
    7930              :           /* If we will propagate into all uses don't bother to do
    7931              :              anything.  */
    7932       439706 :           if (may_propagate_copy (res, sprime))
    7933              :             {
    7934              :               /* Mark the PHI for removal.  */
    7935       439706 :               to_remove.safe_push (phi);
    7936       439706 :               gsi_next (&gsi);
    7937       439706 :               continue;
    7938              :             }
    7939              : 
    7940            0 :           remove_phi_node (&gsi, false);
    7941              : 
    7942            0 :           if (!useless_type_conversion_p (TREE_TYPE (res), TREE_TYPE (sprime)))
    7943            0 :             sprime = fold_convert (TREE_TYPE (res), sprime);
    7944            0 :           gimple *stmt = gimple_build_assign (res, sprime);
    7945            0 :           gimple_stmt_iterator gsi2 = gsi_after_labels (b);
    7946            0 :           gsi_insert_before (&gsi2, stmt, GSI_NEW_STMT);
    7947            0 :           continue;
    7948            0 :         }
    7949              : 
    7950      5911676 :       eliminate_push_avail (b, res);
    7951      5911676 :       gsi_next (&gsi);
    7952              :     }
    7953              : 
    7954     74934624 :   for (gimple_stmt_iterator gsi = gsi_start_bb (b);
    7955    296672947 :        !gsi_end_p (gsi);
    7956    259205635 :        gsi_next (&gsi))
    7957    259205635 :     eliminate_stmt (b, &gsi);
    7958              : 
    7959              :   /* Replace destination PHI arguments.  */
    7960     37467312 :   edge_iterator ei;
    7961     37467312 :   edge e;
    7962     88457864 :   FOR_EACH_EDGE (e, ei, b->succs)
    7963     50990552 :     if (e->flags & EDGE_EXECUTABLE)
    7964     50417196 :       for (gphi_iterator gsi = gsi_start_phis (e->dest);
    7965     80600787 :            !gsi_end_p (gsi);
    7966     30183591 :            gsi_next (&gsi))
    7967              :         {
    7968     30183591 :           gphi *phi = gsi.phi ();
    7969     30183591 :           use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
    7970     30183591 :           tree arg = USE_FROM_PTR (use_p);
    7971     49995514 :           if (TREE_CODE (arg) != SSA_NAME
    7972     30183591 :               || virtual_operand_p (arg))
    7973     19811923 :             continue;
    7974     10371668 :           tree sprime = eliminate_avail (b, arg);
    7975     20743336 :           if (sprime && may_propagate_copy (arg, sprime,
    7976     10371668 :                                             !(e->flags & EDGE_ABNORMAL)))
    7977     10359043 :             propagate_value (use_p, sprime);
    7978              :         }
    7979              : 
    7980     37467312 :   vn_context_bb = NULL;
    7981              : 
    7982     37467312 :   return NULL;
    7983              : }
    7984              : 
    7985              : /* Make no longer available leaders no longer available.  */
    7986              : 
    7987              : void
    7988     42420000 : eliminate_dom_walker::after_dom_children (basic_block)
    7989              : {
    7990     42420000 :   tree entry;
    7991     93961459 :   while ((entry = avail_stack.pop ()) != NULL_TREE)
    7992              :     {
    7993     51541459 :       tree valnum = VN_INFO (entry)->valnum;
    7994     51541459 :       tree old = avail[SSA_NAME_VERSION (valnum)];
    7995     51541459 :       if (old == entry)
    7996              :         avail[SSA_NAME_VERSION (valnum)] = NULL_TREE;
    7997              :       else
    7998        46608 :         avail[SSA_NAME_VERSION (valnum)] = entry;
    7999              :     }
    8000     42420000 : }
    8001              : 
    8002              : /* Remove queued stmts and perform delayed cleanups.  */
    8003              : 
    8004              : unsigned
    8005      6289356 : eliminate_dom_walker::eliminate_cleanup (bool region_p)
    8006              : {
    8007      6289356 :   statistics_counter_event (cfun, "Eliminated", eliminations);
    8008      6289356 :   statistics_counter_event (cfun, "Insertions", insertions);
    8009              : 
    8010              :   /* We cannot remove stmts during BB walk, especially not release SSA
    8011              :      names there as this confuses the VN machinery.  The stmts ending
    8012              :      up in to_remove are either stores or simple copies.
    8013              :      Remove stmts in reverse order to make debug stmt creation possible.  */
    8014     34470748 :   while (!to_remove.is_empty ())
    8015              :     {
    8016     15602624 :       bool do_release_defs = true;
    8017     15602624 :       gimple *stmt = to_remove.pop ();
    8018              : 
    8019              :       /* When we are value-numbering a region we do not require exit PHIs to
    8020              :          be present so we have to make sure to deal with uses outside of the
    8021              :          region of stmts that we thought are eliminated.
    8022              :          ??? Note we may be confused by uses in dead regions we didn't run
    8023              :          elimination on.  Rather than checking individual uses we accept
    8024              :          dead copies to be generated here (gcc.c-torture/execute/20060905-1.c
    8025              :          contains such example).  */
    8026     15602624 :       if (region_p)
    8027              :         {
    8028      1842444 :           if (gphi *phi = dyn_cast <gphi *> (stmt))
    8029              :             {
    8030      1148532 :               tree lhs = gimple_phi_result (phi);
    8031      1148532 :               if (!has_zero_uses (lhs))
    8032              :                 {
    8033        24309 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    8034            3 :                     fprintf (dump_file, "Keeping eliminated stmt live "
    8035              :                              "as copy because of out-of-region uses\n");
    8036        24309 :                   tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
    8037        24309 :                   gimple *copy = gimple_build_assign (lhs, sprime);
    8038        24309 :                   gimple_stmt_iterator gsi
    8039        24309 :                     = gsi_after_labels (gimple_bb (stmt));
    8040        24309 :                   gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
    8041        24309 :                   do_release_defs = false;
    8042              :                 }
    8043              :             }
    8044       693912 :           else if (tree lhs = gimple_get_lhs (stmt))
    8045       693912 :             if (TREE_CODE (lhs) == SSA_NAME
    8046       693912 :                 && !has_zero_uses (lhs))
    8047              :               {
    8048         2128 :                 if (dump_file && (dump_flags & TDF_DETAILS))
    8049            0 :                   fprintf (dump_file, "Keeping eliminated stmt live "
    8050              :                            "as copy because of out-of-region uses\n");
    8051         2128 :                 tree sprime = eliminate_avail (gimple_bb (stmt), lhs);
    8052         2128 :                 gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    8053         2128 :                 if (is_gimple_assign (stmt))
    8054              :                   {
    8055         2128 :                     gimple_assign_set_rhs_from_tree (&gsi, sprime);
    8056         2128 :                     stmt = gsi_stmt (gsi);
    8057         2128 :                     update_stmt (stmt);
    8058         2128 :                     if (maybe_clean_or_replace_eh_stmt (stmt, stmt))
    8059            0 :                       bitmap_set_bit (need_eh_cleanup, gimple_bb (stmt)->index);
    8060         2128 :                     continue;
    8061              :                   }
    8062              :                 else
    8063              :                   {
    8064            0 :                     gimple *copy = gimple_build_assign (lhs, sprime);
    8065            0 :                     gsi_insert_before (&gsi, copy, GSI_SAME_STMT);
    8066            0 :                     do_release_defs = false;
    8067              :                   }
    8068              :               }
    8069              :         }
    8070              : 
    8071     15600496 :       if (dump_file && (dump_flags & TDF_DETAILS))
    8072              :         {
    8073        22070 :           fprintf (dump_file, "Removing dead stmt ");
    8074        22070 :           print_gimple_stmt (dump_file, stmt, 0, TDF_NONE);
    8075              :         }
    8076              : 
    8077     15600496 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    8078     15600496 :       if (gimple_code (stmt) == GIMPLE_PHI)
    8079      1795743 :         remove_phi_node (&gsi, do_release_defs);
    8080              :       else
    8081              :         {
    8082     13804753 :           basic_block bb = gimple_bb (stmt);
    8083     13804753 :           unlink_stmt_vdef (stmt);
    8084     13804753 :           if (gsi_remove (&gsi, true))
    8085        26747 :             bitmap_set_bit (need_eh_cleanup, bb->index);
    8086     13804753 :           if (is_gimple_call (stmt) && stmt_can_make_abnormal_goto (stmt))
    8087            2 :             bitmap_set_bit (need_ab_cleanup, bb->index);
    8088     13804753 :           if (do_release_defs)
    8089     13804753 :             release_defs (stmt);
    8090              :         }
    8091              : 
    8092              :       /* Removing a stmt may expose a forwarder block.  */
    8093     15600496 :       el_todo |= TODO_cleanup_cfg;
    8094              :     }
    8095              : 
    8096              :   /* Fixup stmts that became noreturn calls.  This may require splitting
    8097              :      blocks and thus isn't possible during the dominator walk.  Do this
    8098              :      in reverse order so we don't inadvertently remove a stmt we want to
    8099              :      fixup by visiting a dominating now noreturn call first.  */
    8100      6289412 :   while (!to_fixup.is_empty ())
    8101              :     {
    8102           56 :       gimple *stmt = to_fixup.pop ();
    8103              : 
    8104           56 :       if (dump_file && (dump_flags & TDF_DETAILS))
    8105              :         {
    8106            0 :           fprintf (dump_file, "Fixing up noreturn call ");
    8107            0 :           print_gimple_stmt (dump_file, stmt, 0);
    8108              :         }
    8109              : 
    8110           56 :       if (fixup_noreturn_call (stmt))
    8111           56 :         el_todo |= TODO_cleanup_cfg;
    8112              :     }
    8113              : 
    8114      6289356 :   bool do_eh_cleanup = !bitmap_empty_p (need_eh_cleanup);
    8115      6289356 :   bool do_ab_cleanup = !bitmap_empty_p (need_ab_cleanup);
    8116              : 
    8117      6289356 :   if (do_eh_cleanup)
    8118        10755 :     gimple_purge_all_dead_eh_edges (need_eh_cleanup);
    8119              : 
    8120      6289356 :   if (do_ab_cleanup)
    8121            2 :     gimple_purge_all_dead_abnormal_call_edges (need_ab_cleanup);
    8122              : 
    8123      6289356 :   if (do_eh_cleanup || do_ab_cleanup)
    8124        10757 :     el_todo |= TODO_cleanup_cfg;
    8125              : 
    8126      6289356 :   return el_todo;
    8127              : }
    8128              : 
    8129              : /* Eliminate fully redundant computations.  */
    8130              : 
    8131              : static unsigned
    8132       982970 : eliminate_with_rpo_vn (bitmap inserted_exprs)
    8133              : {
    8134       982970 :   eliminate_dom_walker walker (CDI_DOMINATORS, inserted_exprs);
    8135              : 
    8136       982970 :   eliminate_dom_walker *saved_rpo_avail = rpo_avail;
    8137       982970 :   rpo_avail = &walker;
    8138       982970 :   walker.walk (cfun->cfg->x_entry_block_ptr);
    8139       982970 :   rpo_avail = saved_rpo_avail;
    8140              : 
    8141       982970 :   return walker.eliminate_cleanup ();
    8142       982970 : }
    8143              : 
    8144              : /* Give every value a value id, for PRE.  */
    8145              : 
    8146              : static void
    8147       982970 : assign_value_ids ()
    8148              : {
    8149              :   /* ???  Prune requirement of these.  */
    8150       982970 :   constant_to_value_id = new hash_table<vn_constant_hasher> (23);
    8151              : 
    8152              :   /* Initialize the value ids and prune out remaining VN_TOPs
    8153              :      from dead code.  */
    8154       982970 :   tree name;
    8155       982970 :   unsigned i;
    8156     48390666 :   FOR_EACH_SSA_NAME (i, name, cfun)
    8157              :     {
    8158     34501625 :       vn_ssa_aux_t info = VN_INFO (name);
    8159     34501625 :       if (!info->visited
    8160     34425749 :           || info->valnum == VN_TOP)
    8161        75876 :         info->valnum = name;
    8162     34501625 :       if (info->valnum == name)
    8163     33318091 :         info->value_id = get_next_value_id ();
    8164      1183534 :       else if (is_gimple_min_invariant (info->valnum))
    8165        41751 :         info->value_id = get_or_alloc_constant_value_id (info->valnum);
    8166              :     }
    8167              : 
    8168              :   /* Propagate.  */
    8169     48390666 :   FOR_EACH_SSA_NAME (i, name, cfun)
    8170              :     {
    8171     34501625 :       vn_ssa_aux_t info = VN_INFO (name);
    8172     34501625 :       if (TREE_CODE (info->valnum) == SSA_NAME
    8173     34459874 :           && info->valnum != name
    8174     35643408 :           && info->value_id != VN_INFO (info->valnum)->value_id)
    8175      1141783 :         info->value_id = VN_INFO (info->valnum)->value_id;
    8176              :     }
    8177              : 
    8178       982970 :   set_hashtable_value_ids ();
    8179              : 
    8180       982970 :   if (dump_file && (dump_flags & TDF_DETAILS))
    8181              :     {
    8182           14 :       fprintf (dump_file, "Value numbers:\n");
    8183          406 :       FOR_EACH_SSA_NAME (i, name, cfun)
    8184              :         {
    8185          307 :           if (VN_INFO (name)->visited
    8186          307 :               && SSA_VAL (name) != name)
    8187              :             {
    8188           33 :               print_generic_expr (dump_file, name);
    8189           33 :               fprintf (dump_file, " = ");
    8190           33 :               print_generic_expr (dump_file, SSA_VAL (name));
    8191           33 :               fprintf (dump_file, " (%04d)\n", VN_INFO (name)->value_id);
    8192              :             }
    8193              :         }
    8194              :     }
    8195       982970 : }
    8196              : 
    8197              : /* Free VN associated data structures.  */
    8198              : 
    8199              : static void
    8200      6308971 : free_rpo_vn (void)
    8201              : {
    8202      6308971 :   free_vn_table (valid_info);
    8203      6308971 :   XDELETE (valid_info);
    8204      6308971 :   obstack_free (&vn_tables_obstack, NULL);
    8205      6308971 :   obstack_free (&vn_tables_insert_obstack, NULL);
    8206              : 
    8207      6308971 :   vn_ssa_aux_iterator_type it;
    8208      6308971 :   vn_ssa_aux_t info;
    8209    184442026 :   FOR_EACH_HASH_TABLE_ELEMENT (*vn_ssa_aux_hash, info, vn_ssa_aux_t, it)
    8210    178133055 :     if (info->needs_insertion)
    8211      4275214 :       release_ssa_name (info->name);
    8212      6308971 :   obstack_free (&vn_ssa_aux_obstack, NULL);
    8213      6308971 :   delete vn_ssa_aux_hash;
    8214              : 
    8215      6308971 :   delete constant_to_value_id;
    8216      6308971 :   constant_to_value_id = NULL;
    8217      6308971 : }
    8218              : 
    8219              : /* Hook for maybe_push_res_to_seq, lookup the expression in the VN tables.  */
    8220              : 
    8221              : static tree
    8222     23947675 : vn_lookup_simplify_result (gimple_match_op *res_op)
    8223              : {
    8224     23947675 :   if (!res_op->code.is_tree_code ())
    8225              :     return NULL_TREE;
    8226     23944448 :   vn_nary_op_t vno1 = XALLOCAVAR (struct vn_nary_op_s,
    8227              :                                   sizeof_vn_nary_op
    8228              :                                     (vn_nary_length_from_match_op (res_op)));
    8229     23944448 :   init_vn_nary_op_from_match_op (vno1, res_op);
    8230     23944448 :   vn_nary_op_t vnresult = NULL;
    8231     23944448 :   tree res = vn_nary_op_lookup_1 (vno1, &vnresult);
    8232              :   /* If this is used from expression simplification make sure to
    8233              :      return an available expression.  */
    8234     23944448 :   if (res && TREE_CODE (res) == SSA_NAME && mprts_hook && rpo_avail)
    8235      2327768 :     res = rpo_avail->eliminate_avail (vn_context_bb, res);
    8236              :   return res;
    8237              : }
    8238              : 
    8239              : /* Return a leader for OPs value that is valid at BB.  */
    8240              : 
    8241              : tree
    8242    284851444 : rpo_elim::eliminate_avail (basic_block bb, tree op)
    8243              : {
    8244    284851444 :   bool visited;
    8245    284851444 :   tree valnum = SSA_VAL (op, &visited);
    8246              :   /* If we didn't visit OP then it must be defined outside of the
    8247              :      region we process and also dominate it.  So it is available.  */
    8248    284851444 :   if (!visited)
    8249              :     return op;
    8250    282640039 :   if (TREE_CODE (valnum) == SSA_NAME)
    8251              :     {
    8252    267860840 :       if (SSA_NAME_IS_DEFAULT_DEF (valnum))
    8253              :         return valnum;
    8254    260870908 :       vn_ssa_aux_t valnum_info = VN_INFO (valnum);
    8255    260870908 :       vn_avail *av = valnum_info->avail;
    8256    260870908 :       if (!av)
    8257              :         {
    8258              :           /* See above.  But when there's availability info prefer
    8259              :              what we recorded there for example to preserve LC SSA.  */
    8260     86109725 :           if (!valnum_info->visited)
    8261              :             return valnum;
    8262     85214355 :           return NULL_TREE;
    8263              :         }
    8264    174761183 :       if (av->location == bb->index)
    8265              :         /* On tramp3d 90% of the cases are here.  */
    8266    115218172 :         return ssa_name (av->leader);
    8267     73873166 :       do
    8268              :         {
    8269     73873166 :           basic_block abb = BASIC_BLOCK_FOR_FN (cfun, av->location);
    8270              :           /* ???  During elimination we have to use availability at the
    8271              :              definition site of a use we try to replace.  This
    8272              :              is required to not run into inconsistencies because
    8273              :              of dominated_by_p_w_unex behavior and removing a definition
    8274              :              while not replacing all uses.
    8275              :              ???  We could try to consistently walk dominators
    8276              :              ignoring non-executable regions.  The nearest common
    8277              :              dominator of bb and abb is where we can stop walking.  We
    8278              :              may also be able to "pre-compute" (bits of) the next immediate
    8279              :              (non-)dominator during the RPO walk when marking edges as
    8280              :              executable.  */
    8281     73873166 :           if (dominated_by_p_w_unex (bb, abb))
    8282              :             {
    8283     55582138 :               tree leader = ssa_name (av->leader);
    8284              :               /* Prevent eliminations that break loop-closed SSA.  */
    8285     55582138 :               if (loops_state_satisfies_p (LOOP_CLOSED_SSA)
    8286      3793629 :                   && ! SSA_NAME_IS_DEFAULT_DEF (leader)
    8287     59375767 :                   && ! flow_bb_inside_loop_p (gimple_bb (SSA_NAME_DEF_STMT
    8288      3793629 :                                                          (leader))->loop_father,
    8289              :                                               bb))
    8290              :                 return NULL_TREE;
    8291     55498220 :               if (dump_file && (dump_flags & TDF_DETAILS))
    8292              :                 {
    8293         3951 :                   print_generic_expr (dump_file, leader);
    8294         3951 :                   fprintf (dump_file, " is available for ");
    8295         3951 :                   print_generic_expr (dump_file, valnum);
    8296         3951 :                   fprintf (dump_file, "\n");
    8297              :                 }
    8298              :               /* On tramp3d 99% of the _remaining_ cases succeed at
    8299              :                  the first enty.  */
    8300              :               return leader;
    8301              :             }
    8302              :           /* ???  Can we somehow skip to the immediate dominator
    8303              :              RPO index (bb_to_rpo)?  Again, maybe not worth, on
    8304              :              tramp3d the worst number of elements in the vector is 9.  */
    8305     18291028 :           av = av->next;
    8306              :         }
    8307     18291028 :       while (av);
    8308              :       /* While we prefer avail we have to fallback to using the value
    8309              :          directly if defined outside of the region when none of the
    8310              :          available defs suit.  */
    8311      3960873 :       if (!valnum_info->visited)
    8312            2 :         return valnum;
    8313              :     }
    8314     14779199 :   else if (valnum != VN_TOP)
    8315              :     /* valnum is is_gimple_min_invariant.  */
    8316     14779199 :     return valnum;
    8317              :   return NULL_TREE;
    8318              : }
    8319              : 
    8320              : /* Make LEADER a leader for its value at BB.  */
    8321              : 
    8322              : void
    8323     99981384 : rpo_elim::eliminate_push_avail (basic_block bb, tree leader)
    8324              : {
    8325     99981384 :   tree valnum = VN_INFO (leader)->valnum;
    8326     99981384 :   if (valnum == VN_TOP
    8327     99981384 :       || is_gimple_min_invariant (valnum))
    8328              :     return;
    8329     99981384 :   if (dump_file && (dump_flags & TDF_DETAILS))
    8330              :     {
    8331       336601 :       fprintf (dump_file, "Making available beyond BB%d ", bb->index);
    8332       336601 :       print_generic_expr (dump_file, leader);
    8333       336601 :       fprintf (dump_file, " for value ");
    8334       336601 :       print_generic_expr (dump_file, valnum);
    8335       336601 :       fprintf (dump_file, "\n");
    8336              :     }
    8337     99981384 :   vn_ssa_aux_t value = VN_INFO (valnum);
    8338     99981384 :   vn_avail *av;
    8339     99981384 :   if (m_avail_freelist)
    8340              :     {
    8341     19291337 :       av = m_avail_freelist;
    8342     19291337 :       m_avail_freelist = m_avail_freelist->next;
    8343              :     }
    8344              :   else
    8345     80690047 :     av = XOBNEW (&vn_ssa_aux_obstack, vn_avail);
    8346     99981384 :   av->location = bb->index;
    8347     99981384 :   av->leader = SSA_NAME_VERSION (leader);
    8348     99981384 :   av->next = value->avail;
    8349     99981384 :   av->next_undo = last_pushed_avail;
    8350     99981384 :   last_pushed_avail = value;
    8351     99981384 :   value->avail = av;
    8352              : }
    8353              : 
    8354              : /* Valueization hook for RPO VN plus required state.  */
    8355              : 
    8356              : tree
    8357   2286038250 : rpo_vn_valueize (tree name)
    8358              : {
    8359   2286038250 :   if (TREE_CODE (name) == SSA_NAME)
    8360              :     {
    8361   2238774909 :       vn_ssa_aux_t val = VN_INFO (name);
    8362   2238774909 :       if (val)
    8363              :         {
    8364   2238774909 :           tree tem = val->valnum;
    8365   2238774909 :           if (tem != VN_TOP && tem != name)
    8366              :             {
    8367    120260325 :               if (TREE_CODE (tem) != SSA_NAME)
    8368              :                 return tem;
    8369              :               /* For all values we only valueize to an available leader
    8370              :                  which means we can use SSA name info without restriction.  */
    8371    102388353 :               tem = rpo_avail->eliminate_avail (vn_context_bb, tem);
    8372    102388353 :               if (tem)
    8373    102334668 :                 return tem;
    8374              :             }
    8375              :         }
    8376              :     }
    8377              :   return name;
    8378              : }
    8379              : 
    8380              : /* Insert on PRED_E predicates derived from CODE OPS being true besides the
    8381              :    inverted condition.  */
    8382              : 
    8383              : static void
    8384     28083225 : insert_related_predicates_on_edge (enum tree_code code, tree *ops, edge pred_e)
    8385              : {
    8386     28083225 :   switch (code)
    8387              :     {
    8388      1381234 :     case LT_EXPR:
    8389              :       /* a < b -> a {!,<}= b */
    8390      1381234 :       vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
    8391              :                                            ops, boolean_true_node, 0, pred_e);
    8392      1381234 :       vn_nary_op_insert_pieces_predicated (2, LE_EXPR, boolean_type_node,
    8393              :                                            ops, boolean_true_node, 0, pred_e);
    8394              :       /* a < b -> ! a {>,=} b */
    8395      1381234 :       vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
    8396              :                                            ops, boolean_false_node, 0, pred_e);
    8397      1381234 :       vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
    8398              :                                            ops, boolean_false_node, 0, pred_e);
    8399      1381234 :       break;
    8400      3558681 :     case GT_EXPR:
    8401              :       /* a > b -> a {!,>}= b */
    8402      3558681 :       vn_nary_op_insert_pieces_predicated (2, NE_EXPR, boolean_type_node,
    8403              :                                            ops, boolean_true_node, 0, pred_e);
    8404      3558681 :       vn_nary_op_insert_pieces_predicated (2, GE_EXPR, boolean_type_node,
    8405              :                                            ops, boolean_true_node, 0, pred_e);
    8406              :       /* a > b -> ! a {<,=} b */
    8407      3558681 :       vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
    8408              :                                            ops, boolean_false_node, 0, pred_e);
    8409      3558681 :       vn_nary_op_insert_pieces_predicated (2, EQ_EXPR, boolean_type_node,
    8410              :                                            ops, boolean_false_node, 0, pred_e);
    8411      3558681 :       break;
    8412      9618366 :     case EQ_EXPR:
    8413              :       /* a == b -> ! a {<,>} b */
    8414      9618366 :       vn_nary_op_insert_pieces_predicated (2, LT_EXPR, boolean_type_node,
    8415              :                                            ops, boolean_false_node, 0, pred_e);
    8416      9618366 :       vn_nary_op_insert_pieces_predicated (2, GT_EXPR, boolean_type_node,
    8417              :                                            ops, boolean_false_node, 0, pred_e);
    8418      9618366 :       break;
    8419              :     case LE_EXPR:
    8420              :     case GE_EXPR:
    8421              :     case NE_EXPR:
    8422              :       /* Nothing besides inverted condition.  */
    8423              :       break;
    8424     28083225 :     default:;
    8425              :     }
    8426     28083225 : }
    8427              : 
    8428              : /* Insert on the TRUE_E true and FALSE_E false predicates
    8429              :    derived from LHS CODE RHS.  */
    8430              : 
    8431              : static void
    8432     23878717 : insert_predicates_for_cond (tree_code code, tree lhs, tree rhs,
    8433              :                             edge true_e, edge false_e)
    8434              : {
    8435              :   /* If both edges are null, then there is nothing to be done. */
    8436     23878717 :   if (!true_e && !false_e)
    8437      1349160 :     return;
    8438              : 
    8439              :   /* Canonicalize the comparison if needed, putting
    8440              :      the constant in the rhs.  */
    8441     22533043 :   if (tree_swap_operands_p (lhs, rhs))
    8442              :     {
    8443            0 :       std::swap (lhs, rhs);
    8444            0 :       code = swap_tree_comparison (code);
    8445              :     }
    8446              : 
    8447              :   /* If the lhs is not a ssa name, don't record anything. */
    8448     22533043 :   if (TREE_CODE (lhs) != SSA_NAME)
    8449              :     return;
    8450              : 
    8451     22529557 :   tree_code icode = invert_tree_comparison (code, HONOR_NANS (lhs));
    8452     22529557 :   tree ops[2];
    8453     22529557 :   ops[0] = lhs;
    8454     22529557 :   ops[1] = rhs;
    8455     22529557 :   if (true_e)
    8456     18434302 :     vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
    8457              :                                          boolean_true_node, 0, true_e);
    8458     22529557 :   if (false_e)
    8459     17395997 :     vn_nary_op_insert_pieces_predicated (2, code, boolean_type_node, ops,
    8460              :                                          boolean_false_node, 0, false_e);
    8461     22529557 :   if (icode != ERROR_MARK)
    8462              :     {
    8463     22268492 :       if (true_e)
    8464     18269109 :         vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
    8465              :                                              boolean_false_node, 0, true_e);
    8466     22268492 :       if (false_e)
    8467     17185134 :         vn_nary_op_insert_pieces_predicated (2, icode, boolean_type_node, ops,
    8468              :                                              boolean_true_node, 0, false_e);
    8469              :     }
    8470              :   /* Relax for non-integers, inverted condition handled
    8471              :      above.  */
    8472     22529557 :   if (INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
    8473              :     {
    8474     17648908 :       if (true_e)
    8475     14515632 :         insert_related_predicates_on_edge (code, ops, true_e);
    8476     17648908 :       if (false_e)
    8477     13567593 :         insert_related_predicates_on_edge (icode, ops, false_e);
    8478              :   }
    8479     22529557 :   if (integer_zerop (rhs)
    8480     22529557 :       && (code == NE_EXPR || code == EQ_EXPR))
    8481              :     {
    8482      9357914 :       gimple *def_stmt = SSA_NAME_DEF_STMT (lhs);
    8483              :       /* (A CMP B) != 0 is the same as (A CMP B).
    8484              :          (A CMP B) == 0 is just (A CMP B) with the edges swapped.  */
    8485      9357914 :       if (is_gimple_assign (def_stmt)
    8486      9357914 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt)) == tcc_comparison)
    8487              :           {
    8488       427106 :             tree_code nc = gimple_assign_rhs_code (def_stmt);
    8489       427106 :             tree nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
    8490       427106 :             tree nrhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
    8491              :             // Canonicalize the comparison before the check below,
    8492              :             // it might be the case where nlhs is a constant now.
    8493       427106 :             if (tree_swap_operands_p (nlhs, nrhs))
    8494              :               {
    8495        16860 :                 std::swap (nlhs, nrhs);
    8496        16860 :                 nc = swap_tree_comparison (nc);
    8497              :               }
    8498       427106 :             edge nt = true_e;
    8499       427106 :             edge nf = false_e;
    8500       427106 :             if (code == EQ_EXPR)
    8501       300156 :               std::swap (nt, nf);
    8502       427106 :             if (lhs != nlhs)
    8503       427106 :               insert_predicates_for_cond (nc, nlhs, nrhs, nt, nf);
    8504              :           }
    8505              :       /* (a | b) == 0 ->
    8506              :             on true edge assert: a == 0 & b == 0. */
    8507              :       /* (a | b) != 0 ->
    8508              :             on false edge assert: a == 0 & b == 0. */
    8509      9357914 :       if (is_gimple_assign (def_stmt)
    8510      9357914 :           && gimple_assign_rhs_code (def_stmt) == BIT_IOR_EXPR)
    8511              :         {
    8512       249310 :           edge e = code == EQ_EXPR ? true_e : false_e;
    8513       249310 :           tree nlhs;
    8514              : 
    8515       249310 :           nlhs = vn_valueize (gimple_assign_rhs1 (def_stmt));
    8516              :           /* A valueization of the `a` might return the old lhs
    8517              :              which is already handled above. */
    8518       249310 :           if (nlhs != lhs)
    8519       249310 :             insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
    8520              : 
    8521              :           /* A valueization of the `b` might return the old lhs
    8522              :              which is already handled above. */
    8523       249310 :           nlhs = vn_valueize (gimple_assign_rhs2 (def_stmt));
    8524       249310 :           if (nlhs != lhs)
    8525       249310 :             insert_predicates_for_cond (EQ_EXPR, nlhs, rhs, e, nullptr);
    8526              :         }
    8527              :     }
    8528              : }
    8529              : 
    8530              : /* Main stmt worker for RPO VN, process BB.  */
    8531              : 
    8532              : static unsigned
    8533     63271639 : process_bb (rpo_elim &avail, basic_block bb,
    8534              :             bool bb_visited, bool iterate_phis, bool iterate, bool eliminate,
    8535              :             bool do_region, bitmap exit_bbs, bool skip_phis)
    8536              : {
    8537     63271639 :   unsigned todo = 0;
    8538     63271639 :   edge_iterator ei;
    8539     63271639 :   edge e;
    8540              : 
    8541     63271639 :   vn_context_bb = bb;
    8542              : 
    8543     63271639 :   update_exec_idom (bb);
    8544              : 
    8545              :   /* If we are in loop-closed SSA preserve this state.  This is
    8546              :      relevant when called on regions from outside of FRE/PRE.  */
    8547     63271639 :   bool lc_phi_nodes = false;
    8548     63271639 :   if (!skip_phis
    8549     63271639 :       && loops_state_satisfies_p (LOOP_CLOSED_SSA))
    8550      3954403 :     FOR_EACH_EDGE (e, ei, bb->preds)
    8551      2392140 :       if (e->src->loop_father != e->dest->loop_father
    8552      2392140 :           && flow_loop_nested_p (e->dest->loop_father,
    8553              :                                  e->src->loop_father))
    8554              :         {
    8555              :           lc_phi_nodes = true;
    8556              :           break;
    8557              :         }
    8558              : 
    8559              :   /* When we visit a loop header substitute into loop info.  */
    8560     63271639 :   if (!iterate && eliminate && bb->loop_father->header == bb)
    8561              :     {
    8562              :       /* Keep fields in sync with substitute_in_loop_info.  */
    8563       965233 :       if (bb->loop_father->nb_iterations)
    8564       159967 :         bb->loop_father->nb_iterations
    8565       159967 :           = simplify_replace_tree (bb->loop_father->nb_iterations,
    8566              :                                    NULL_TREE, NULL_TREE, &vn_valueize_for_srt);
    8567              :     }
    8568              : 
    8569              :   /* Value-number all defs in the basic-block.  */
    8570     63271639 :   if (!skip_phis)
    8571     90799201 :     for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
    8572     27553542 :          gsi_next (&gsi))
    8573              :       {
    8574     27553542 :         gphi *phi = gsi.phi ();
    8575     27553542 :         tree res = PHI_RESULT (phi);
    8576     27553542 :         vn_ssa_aux_t res_info = VN_INFO (res);
    8577     27553542 :         if (!bb_visited)
    8578              :           {
    8579     19464532 :             gcc_assert (!res_info->visited);
    8580     19464532 :             res_info->valnum = VN_TOP;
    8581     19464532 :             res_info->visited = true;
    8582              :           }
    8583              : 
    8584              :         /* When not iterating force backedge values to varying.  */
    8585     27553542 :         visit_stmt (phi, !iterate_phis);
    8586     55107084 :         if (virtual_operand_p (res))
    8587     10923944 :           continue;
    8588              : 
    8589              :         /* Eliminate */
    8590              :         /* The interesting case is gcc.dg/tree-ssa/pr22230.c for correctness
    8591              :            how we handle backedges and availability.
    8592              :            And gcc.dg/tree-ssa/ssa-sccvn-2.c for optimization.  */
    8593     16629598 :         tree val = res_info->valnum;
    8594     16629598 :         if (res != val && !iterate && eliminate)
    8595              :           {
    8596      1481941 :             if (tree leader = avail.eliminate_avail (bb, res))
    8597              :               {
    8598      1356568 :                 if (leader != res
    8599              :                     /* Preserve loop-closed SSA form.  */
    8600      1356568 :                     && (! lc_phi_nodes
    8601         6888 :                         || is_gimple_min_invariant (leader)))
    8602              :                   {
    8603      1356037 :                     if (dump_file && (dump_flags & TDF_DETAILS))
    8604              :                       {
    8605          241 :                         fprintf (dump_file, "Replaced redundant PHI node "
    8606              :                                  "defining ");
    8607          241 :                         print_generic_expr (dump_file, res);
    8608          241 :                         fprintf (dump_file, " with ");
    8609          241 :                         print_generic_expr (dump_file, leader);
    8610          241 :                         fprintf (dump_file, "\n");
    8611              :                       }
    8612      1356037 :                     avail.eliminations++;
    8613              : 
    8614      1356037 :                     if (may_propagate_copy (res, leader))
    8615              :                       {
    8616              :                         /* Schedule for removal.  */
    8617      1356037 :                         avail.to_remove.safe_push (phi);
    8618      1356037 :                         continue;
    8619              :                       }
    8620              :                     /* ???  Else generate a copy stmt.  */
    8621              :                   }
    8622              :               }
    8623              :           }
    8624              :         /* Only make defs available that not already are.  But make
    8625              :            sure loop-closed SSA PHI node defs are picked up for
    8626              :            downstream uses.  */
    8627     15273561 :         if (lc_phi_nodes
    8628     15273561 :             || res == val
    8629     15273561 :             || ! avail.eliminate_avail (bb, res))
    8630     11699921 :           avail.eliminate_push_avail (bb, res);
    8631              :       }
    8632              : 
    8633              :   /* For empty BBs mark outgoing edges executable.  For non-empty BBs
    8634              :      we do this when processing the last stmt as we have to do this
    8635              :      before elimination which otherwise forces GIMPLE_CONDs to
    8636              :      if (1 != 0) style when seeing non-executable edges.  */
    8637    126543278 :   if (gsi_end_p (gsi_start_bb (bb)))
    8638              :     {
    8639     14570354 :       FOR_EACH_EDGE (e, ei, bb->succs)
    8640              :         {
    8641      7285177 :           if (!(e->flags & EDGE_EXECUTABLE))
    8642              :             {
    8643      4924234 :               if (dump_file && (dump_flags & TDF_DETAILS))
    8644         6478 :                 fprintf (dump_file,
    8645              :                          "marking outgoing edge %d -> %d executable\n",
    8646         6478 :                          e->src->index, e->dest->index);
    8647      4924234 :               e->flags |= EDGE_EXECUTABLE;
    8648      4924234 :               e->dest->flags |= BB_EXECUTABLE;
    8649              :             }
    8650      2360943 :           else if (!(e->dest->flags & BB_EXECUTABLE))
    8651              :             {
    8652            0 :               if (dump_file && (dump_flags & TDF_DETAILS))
    8653            0 :                 fprintf (dump_file,
    8654              :                          "marking destination block %d reachable\n",
    8655              :                          e->dest->index);
    8656            0 :               e->dest->flags |= BB_EXECUTABLE;
    8657              :             }
    8658              :         }
    8659              :     }
    8660    126543278 :   for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    8661    517102041 :        !gsi_end_p (gsi); gsi_next (&gsi))
    8662              :     {
    8663    453830402 :       ssa_op_iter i;
    8664    453830402 :       tree op;
    8665    453830402 :       if (!bb_visited)
    8666              :         {
    8667    515429778 :           FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_ALL_DEFS)
    8668              :             {
    8669    142596864 :               vn_ssa_aux_t op_info = VN_INFO (op);
    8670    142596864 :               gcc_assert (!op_info->visited);
    8671    142596864 :               op_info->valnum = VN_TOP;
    8672    142596864 :               op_info->visited = true;
    8673              :             }
    8674              : 
    8675              :           /* We somehow have to deal with uses that are not defined
    8676              :              in the processed region.  Forcing unvisited uses to
    8677              :              varying here doesn't play well with def-use following during
    8678              :              expression simplification, so we deal with this by checking
    8679              :              the visited flag in SSA_VAL.  */
    8680              :         }
    8681              : 
    8682    453830402 :       visit_stmt (gsi_stmt (gsi));
    8683              : 
    8684    453830402 :       gimple *last = gsi_stmt (gsi);
    8685    453830402 :       e = NULL;
    8686    453830402 :       switch (gimple_code (last))
    8687              :         {
    8688       120717 :         case GIMPLE_SWITCH:
    8689       120717 :           e = find_taken_edge (bb, vn_valueize (gimple_switch_index
    8690       120717 :                                                 (as_a <gswitch *> (last))));
    8691       120717 :           break;
    8692     25262989 :         case GIMPLE_COND:
    8693     25262989 :           {
    8694     25262989 :             tree lhs = vn_valueize (gimple_cond_lhs (last));
    8695     25262989 :             tree rhs = vn_valueize (gimple_cond_rhs (last));
    8696     25262989 :             tree_code cmpcode = gimple_cond_code (last);
    8697              :             /* Canonicalize the comparison if needed, putting
    8698              :                the constant in the rhs.  */
    8699     25262989 :             if (tree_swap_operands_p (lhs, rhs))
    8700              :               {
    8701       861797 :                 std::swap (lhs, rhs);
    8702       861797 :                 cmpcode = swap_tree_comparison (cmpcode);
    8703              :                }
    8704     25262989 :             tree val = gimple_simplify (cmpcode,
    8705              :                                         boolean_type_node, lhs, rhs,
    8706              :                                         NULL, vn_valueize);
    8707              :             /* If the condition didn't simplify see if we have recorded
    8708              :                an expression from sofar taken edges.  */
    8709     25262989 :             if (! val || TREE_CODE (val) != INTEGER_CST)
    8710              :               {
    8711     23316086 :                 vn_nary_op_t vnresult;
    8712     23316086 :                 tree ops[2];
    8713     23316086 :                 ops[0] = lhs;
    8714     23316086 :                 ops[1] = rhs;
    8715     23316086 :                 val = vn_nary_op_lookup_pieces (2, cmpcode,
    8716              :                                                 boolean_type_node, ops,
    8717              :                                                 &vnresult);
    8718              :                 /* Got back a ssa name, then try looking up `val != 0`
    8719              :                    as it might have been recorded that way.  */
    8720     23316086 :                 if (val && TREE_CODE (val) == SSA_NAME)
    8721              :                   {
    8722       163442 :                     ops[0] = val;
    8723       163442 :                     ops[1] = build_zero_cst (TREE_TYPE (val));
    8724       163442 :                     val = vn_nary_op_lookup_pieces (2, NE_EXPR,
    8725              :                                                     boolean_type_node, ops,
    8726              :                                                     &vnresult);
    8727              :                   }
    8728              :                 /* Did we get a predicated value?  */
    8729     23316060 :                 if (! val && vnresult && vnresult->predicated_values)
    8730              :                   {
    8731      1426000 :                     val = vn_nary_op_get_predicated_value (vnresult, bb);
    8732      1426000 :                     if (val && dump_file && (dump_flags & TDF_DETAILS))
    8733              :                       {
    8734            2 :                         fprintf (dump_file, "Got predicated value ");
    8735            2 :                         print_generic_expr (dump_file, val, TDF_NONE);
    8736            2 :                         fprintf (dump_file, " for ");
    8737            2 :                         print_gimple_stmt (dump_file, last, TDF_SLIM);
    8738              :                       }
    8739              :                   }
    8740              :               }
    8741     23316086 :             if (val)
    8742      2309998 :               e = find_taken_edge (bb, val);
    8743     25262989 :             if (! e)
    8744              :               {
    8745              :                 /* If we didn't manage to compute the taken edge then
    8746              :                    push predicated expressions for the condition itself
    8747              :                    and related conditions to the hashtables.  This allows
    8748              :                    simplification of redundant conditions which is
    8749              :                    important as early cleanup.  */
    8750     22952991 :                 edge true_e, false_e;
    8751     22952991 :                 extract_true_false_edges_from_block (bb, &true_e, &false_e);
    8752       558964 :                 if ((do_region && bitmap_bit_p (exit_bbs, true_e->dest->index))
    8753     23190985 :                     || !can_track_predicate_on_edge (true_e))
    8754      5014059 :                   true_e = NULL;
    8755       558964 :                 if ((do_region && bitmap_bit_p (exit_bbs, false_e->dest->index))
    8756     23168615 :                     || !can_track_predicate_on_edge (false_e))
    8757      5947418 :                   false_e = NULL;
    8758     22952991 :                 insert_predicates_for_cond (cmpcode, lhs, rhs, true_e, false_e);
    8759              :               }
    8760              :             break;
    8761              :           }
    8762         1457 :         case GIMPLE_GOTO:
    8763         1457 :           e = find_taken_edge (bb, vn_valueize (gimple_goto_dest (last)));
    8764         1457 :           break;
    8765              :         default:
    8766              :           e = NULL;
    8767              :         }
    8768    453830402 :       if (e)
    8769              :         {
    8770      2313853 :           todo = TODO_cleanup_cfg;
    8771      2313853 :           if (!(e->flags & EDGE_EXECUTABLE))
    8772              :             {
    8773      1823410 :               if (dump_file && (dump_flags & TDF_DETAILS))
    8774           89 :                 fprintf (dump_file,
    8775              :                          "marking known outgoing %sedge %d -> %d executable\n",
    8776           89 :                          e->flags & EDGE_DFS_BACK ? "back-" : "",
    8777           89 :                          e->src->index, e->dest->index);
    8778      1823410 :               e->flags |= EDGE_EXECUTABLE;
    8779      1823410 :               e->dest->flags |= BB_EXECUTABLE;
    8780              :             }
    8781       490443 :           else if (!(e->dest->flags & BB_EXECUTABLE))
    8782              :             {
    8783        27858 :               if (dump_file && (dump_flags & TDF_DETAILS))
    8784            1 :                 fprintf (dump_file,
    8785              :                          "marking destination block %d reachable\n",
    8786              :                          e->dest->index);
    8787        27858 :               e->dest->flags |= BB_EXECUTABLE;
    8788              :             }
    8789              :         }
    8790    451516549 :       else if (gsi_one_before_end_p (gsi))
    8791              :         {
    8792    131693387 :           FOR_EACH_EDGE (e, ei, bb->succs)
    8793              :             {
    8794     78020778 :               if (!(e->flags & EDGE_EXECUTABLE))
    8795              :                 {
    8796     57277811 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    8797        19267 :                     fprintf (dump_file,
    8798              :                              "marking outgoing edge %d -> %d executable\n",
    8799        19267 :                              e->src->index, e->dest->index);
    8800     57277811 :                   e->flags |= EDGE_EXECUTABLE;
    8801     57277811 :                   e->dest->flags |= BB_EXECUTABLE;
    8802              :                 }
    8803     20742967 :               else if (!(e->dest->flags & BB_EXECUTABLE))
    8804              :                 {
    8805      2616492 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    8806         6366 :                     fprintf (dump_file,
    8807              :                              "marking destination block %d reachable\n",
    8808              :                              e->dest->index);
    8809      2616492 :                   e->dest->flags |= BB_EXECUTABLE;
    8810              :                 }
    8811              :             }
    8812              :         }
    8813              : 
    8814              :       /* Eliminate.  That also pushes to avail.  */
    8815    453830402 :       if (eliminate && ! iterate)
    8816    113978715 :         avail.eliminate_stmt (bb, &gsi);
    8817              :       else
    8818              :         /* If not eliminating, make all not already available defs
    8819              :            available.  But avoid picking up dead defs.  */
    8820    422377082 :         FOR_EACH_SSA_TREE_OPERAND (op, gsi_stmt (gsi), i, SSA_OP_DEF)
    8821     82525395 :           if (! has_zero_uses (op)
    8822     82525395 :               && ! avail.eliminate_avail (bb, op))
    8823     62744066 :             avail.eliminate_push_avail (bb, op);
    8824              :     }
    8825              : 
    8826              :   /* Eliminate in destination PHI arguments.  Always substitute in dest
    8827              :      PHIs, even for non-executable edges.  This handles region
    8828              :      exits PHIs.  */
    8829     63271639 :   if (!iterate && eliminate)
    8830     33940641 :     FOR_EACH_EDGE (e, ei, bb->succs)
    8831     20219892 :       for (gphi_iterator gsi = gsi_start_phis (e->dest);
    8832     39158399 :            !gsi_end_p (gsi); gsi_next (&gsi))
    8833              :         {
    8834     18938507 :           gphi *phi = gsi.phi ();
    8835     18938507 :           use_operand_p use_p = PHI_ARG_DEF_PTR_FROM_EDGE (phi, e);
    8836     18938507 :           tree arg = USE_FROM_PTR (use_p);
    8837     28847936 :           if (TREE_CODE (arg) != SSA_NAME
    8838     18938507 :               || virtual_operand_p (arg))
    8839      9909429 :             continue;
    8840      9029078 :           tree sprime;
    8841      9029078 :           if (SSA_NAME_IS_DEFAULT_DEF (arg))
    8842              :             {
    8843       121780 :               sprime = SSA_VAL (arg);
    8844       121780 :               gcc_assert (TREE_CODE (sprime) != SSA_NAME
    8845              :                           || SSA_NAME_IS_DEFAULT_DEF (sprime));
    8846              :             }
    8847              :           else
    8848              :             /* Look for sth available at the definition block of the argument.
    8849              :                This avoids inconsistencies between availability there which
    8850              :                decides if the stmt can be removed and availability at the
    8851              :                use site.  The SSA property ensures that things available
    8852              :                at the definition are also available at uses.  */
    8853      8907298 :             sprime = avail.eliminate_avail (gimple_bb (SSA_NAME_DEF_STMT (arg)),
    8854              :                                             arg);
    8855      9029078 :           if (sprime
    8856      9029078 :               && sprime != arg
    8857      9029078 :               && may_propagate_copy (arg, sprime, !(e->flags & EDGE_ABNORMAL)))
    8858      1587570 :             propagate_value (use_p, sprime);
    8859              :         }
    8860              : 
    8861     63271639 :   vn_context_bb = NULL;
    8862     63271639 :   return todo;
    8863              : }
    8864              : 
    8865              : /* Unwind state per basic-block.  */
    8866              : 
    8867              : struct unwind_state
    8868              : {
    8869              :   /* Times this block has been visited.  */
    8870              :   unsigned visited;
    8871              :   /* Whether to handle this as iteration point or whether to treat
    8872              :      incoming backedge PHI values as varying.  */
    8873              :   bool iterate;
    8874              :   /* Maximum RPO index this block is reachable from.  */
    8875              :   int max_rpo;
    8876              :   /* Unwind state.  */
    8877              :   void *ob_top;
    8878              :   vn_reference_t ref_top;
    8879              :   vn_phi_t phi_top;
    8880              :   vn_nary_op_t nary_top;
    8881              :   vn_avail *avail_top;
    8882              : };
    8883              : 
    8884              : /* Record in TO the RPO VN state do_unwind unwinds to.  */
    8885              : 
    8886              : static void
    8887      4513268 : record_unwind_state (unwind_state *to)
    8888              : {
    8889      4513268 :   to->ob_top = obstack_alloc (&vn_tables_obstack, 0);
    8890      4513268 :   to->ref_top = last_inserted_ref;
    8891      4513268 :   to->phi_top = last_inserted_phi;
    8892      4513268 :   to->nary_top = last_inserted_nary;
    8893      4513268 :   to->avail_top = last_pushed_avail ? last_pushed_avail->avail : NULL;
    8894      4513268 : }
    8895              : 
    8896              : /* Unwind the RPO VN state for iteration.  */
    8897              : 
    8898              : static void
    8899      1950527 : do_unwind (unwind_state *to, rpo_elim &avail)
    8900              : {
    8901      1950527 :   gcc_assert (to->iterate);
    8902     35917023 :   for (; last_inserted_nary != to->nary_top;
    8903     33966496 :        last_inserted_nary = last_inserted_nary->next)
    8904              :     {
    8905     33966496 :       vn_nary_op_t *slot;
    8906     33966496 :       slot = valid_info->nary->find_slot_with_hash
    8907     33966496 :         (last_inserted_nary, last_inserted_nary->hashcode, NO_INSERT);
    8908              :       /* Predication causes the need to restore previous state.  */
    8909     33966496 :       if ((*slot)->unwind_to)
    8910      6942161 :         *slot = (*slot)->unwind_to;
    8911              :       else
    8912     27024335 :         valid_info->nary->clear_slot (slot);
    8913              :     }
    8914      7685755 :   for (; last_inserted_phi != to->phi_top;
    8915      5735228 :        last_inserted_phi = last_inserted_phi->next)
    8916              :     {
    8917      5735228 :       vn_phi_t *slot;
    8918      5735228 :       slot = valid_info->phis->find_slot_with_hash
    8919      5735228 :         (last_inserted_phi, last_inserted_phi->hashcode, NO_INSERT);
    8920      5735228 :       valid_info->phis->clear_slot (slot);
    8921              :     }
    8922     15719284 :   for (; last_inserted_ref != to->ref_top;
    8923     13768757 :        last_inserted_ref = last_inserted_ref->next)
    8924              :     {
    8925     13768757 :       vn_reference_t *slot;
    8926     13768757 :       slot = valid_info->references->find_slot_with_hash
    8927     13768757 :         (last_inserted_ref, last_inserted_ref->hashcode, NO_INSERT);
    8928     13768757 :       (*slot)->operands.release ();
    8929     13768757 :       valid_info->references->clear_slot (slot);
    8930              :     }
    8931      1950527 :   obstack_free (&vn_tables_obstack, to->ob_top);
    8932              : 
    8933              :   /* Prune [rpo_idx, ] from avail.  */
    8934     21241864 :   for (; last_pushed_avail && last_pushed_avail->avail != to->avail_top;)
    8935              :     {
    8936     19291337 :       vn_ssa_aux_t val = last_pushed_avail;
    8937     19291337 :       vn_avail *av = val->avail;
    8938     19291337 :       val->avail = av->next;
    8939     19291337 :       last_pushed_avail = av->next_undo;
    8940     19291337 :       av->next = avail.m_avail_freelist;
    8941     19291337 :       avail.m_avail_freelist = av;
    8942              :     }
    8943      1950527 : }
    8944              : 
    8945              : /* Do VN on a SEME region specified by ENTRY and EXIT_BBS in FN.
    8946              :    If ITERATE is true then treat backedges optimistically as not
    8947              :    executed and iterate.  If ELIMINATE is true then perform
    8948              :    elimination, otherwise leave that to the caller.  If SKIP_ENTRY_PHIS
    8949              :    is true then force PHI nodes in ENTRY->dest to VARYING.  AVAIL,
    8950              :    BB_TO_RPO, EXEC_IDOM and ELIMINATOR belong to the caller, which can
    8951              :    keep them past the walk.  Elimination stops short of eliminate_cleanup, which
    8952              :    is left to the caller.  */
    8953              : 
    8954              : static unsigned
    8955      6308971 : do_rpo_vn_1 (function *fn, edge entry, bitmap exit_bbs,
    8956              :              bool iterate, bool eliminate, bool skip_entry_phis,
    8957              :              vn_lookup_kind kind, rpo_elim &avail, int *bb_to_rpo,
    8958              :              int *exec_idom, eliminate_dom_walker *eliminator)
    8959              : {
    8960      6308971 :   unsigned todo = 0;
    8961      6308971 :   default_vn_walk_kind = kind;
    8962              : 
    8963              :   /* We currently do not support region-based iteration when
    8964              :      elimination is requested.  */
    8965      6308971 :   gcc_assert (!entry || !iterate || !eliminate);
    8966              :   /* When iterating we need loop info up-to-date.  */
    8967      6308971 :   gcc_assert (!iterate || !loops_state_satisfies_p (LOOPS_NEED_FIXUP));
    8968              : 
    8969      6308971 :   bool do_region = entry != NULL;
    8970      6308971 :   if (!do_region)
    8971              :     {
    8972      5606180 :       entry = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fn));
    8973      5606180 :       exit_bbs = BITMAP_ALLOC (NULL);
    8974      5606180 :       bitmap_set_bit (exit_bbs, EXIT_BLOCK);
    8975              :     }
    8976              : 
    8977              :   /* Clear EDGE_DFS_BACK on "all" entry edges, RPO order compute will
    8978              :      re-mark those that are contained in the region.  */
    8979      6308971 :   edge_iterator ei;
    8980      6308971 :   edge e;
    8981     12675355 :   FOR_EACH_EDGE (e, ei, entry->dest->preds)
    8982      6366384 :     e->flags &= ~EDGE_DFS_BACK;
    8983              : 
    8984      6308971 :   int *rpo = XNEWVEC (int, n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS);
    8985      6308971 :   auto_vec<std::pair<int, int> > toplevel_scc_extents;
    8986      6308971 :   int n = rev_post_order_and_mark_dfs_back_seme
    8987      8236926 :     (fn, entry, exit_bbs, true, rpo, !iterate ? &toplevel_scc_extents : NULL);
    8988              : 
    8989      6308971 :   if (!do_region)
    8990      5606180 :     BITMAP_FREE (exit_bbs);
    8991              : 
    8992              :   /* If there are any non-DFS_BACK edges into entry->dest skip
    8993              :      processing PHI nodes for that block.  This supports
    8994              :      value-numbering loop bodies w/o the actual loop.  */
    8995     12675354 :   FOR_EACH_EDGE (e, ei, entry->dest->preds)
    8996      6366384 :     if (e != entry
    8997        57413 :         && !(e->flags & EDGE_DFS_BACK))
    8998              :       break;
    8999      6308971 :   if (e != NULL && dump_file && (dump_flags & TDF_DETAILS))
    9000            0 :     fprintf (dump_file, "Region does not contain all edges into "
    9001              :              "the entry block, skipping its PHIs.\n");
    9002      6308971 :   skip_entry_phis |= e != NULL;
    9003              : 
    9004     58232852 :   for (int i = 0; i < n; ++i)
    9005     51923881 :     bb_to_rpo[rpo[i]] = i;
    9006      6308971 :   vn_bb_to_rpo = bb_to_rpo;
    9007              : 
    9008              :   /* When not iterating we can end up with executable edges from unvisited
    9009              :      but backward reachable blocks, so we cannot rely on the idom being
    9010              :      up-to-date.  Initialize it with a conservative default.  */
    9011      6308971 :   vn_exec_idom = exec_idom;
    9012      6308971 :   vn_exec_idom[0] = -1;
    9013     51923881 :   for (int i = 1; i < n; ++i)
    9014              :     {
    9015     45614910 :       basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
    9016     91229820 :       vn_exec_idom[i]
    9017     45614910 :         = bb_to_rpo[get_immediate_dominator (CDI_DOMINATORS, bb)->index];
    9018              :     }
    9019              : 
    9020      6308971 :   unwind_state *rpo_state = XNEWVEC (unwind_state, n);
    9021              : 
    9022      6308971 :   rpo_avail = &avail;
    9023              : 
    9024              :   /* Create the VN state.  For the initial size of the various hashtables
    9025              :      use a heuristic based on region size and number of SSA names.  */
    9026      6308971 :   unsigned region_size = (((unsigned HOST_WIDE_INT)n * num_ssa_names)
    9027      6308971 :                           / (n_basic_blocks_for_fn (fn) - NUM_FIXED_BLOCKS));
    9028      6308971 :   VN_TOP = create_tmp_var_raw (void_type_node, "vn_top");
    9029      6308971 :   next_value_id = 1;
    9030      6308971 :   next_constant_value_id = -1;
    9031              : 
    9032      6308971 :   vn_ssa_aux_hash = new hash_table <vn_ssa_aux_hasher> (region_size * 2);
    9033      6308971 :   gcc_obstack_init (&vn_ssa_aux_obstack);
    9034              : 
    9035      6308971 :   gcc_obstack_init (&vn_tables_obstack);
    9036      6308971 :   gcc_obstack_init (&vn_tables_insert_obstack);
    9037      6308971 :   valid_info = XCNEW (struct vn_tables_s);
    9038      6308971 :   allocate_vn_table (valid_info, region_size);
    9039              :   /* Without iteration nothing is unwound so there's no need to remember
    9040              :      inserted expressions for re-use.  */
    9041      6308971 :   if (iterate)
    9042      4381016 :     valid_info->inserted_exprs = new vn_nary_op_table_type (16);
    9043      6308971 :   last_inserted_ref = NULL;
    9044      6308971 :   last_inserted_phi = NULL;
    9045      6308971 :   last_inserted_nary = NULL;
    9046      6308971 :   last_pushed_avail = NULL;
    9047              : 
    9048      6308971 :   vn_valueize = rpo_vn_valueize;
    9049              : 
    9050              :   /* The blocks in the value-numbered region are marked with bb_in_region.  */
    9051      6308971 :   auto_bb_flag bb_in_region (fn);
    9052      6308971 :   vn_bb_in_region_flag = bb_in_region;
    9053      6308971 :   vn_entry_edge = entry;
    9054              : 
    9055              :   /* Initialize the unwind state and edge/BB executable state.  */
    9056      6308971 :   unsigned curr_scc = 0;
    9057     58232852 :   for (int i = 0; i < n; ++i)
    9058              :     {
    9059     51923881 :       basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
    9060     51923881 :       bb->flags |= bb_in_region;
    9061     51923881 :       rpo_state[i].visited = 0;
    9062     51923881 :       rpo_state[i].max_rpo = i;
    9063     60677474 :       if (!iterate && curr_scc < toplevel_scc_extents.length ())
    9064              :         {
    9065      7250966 :           if (i >= toplevel_scc_extents[curr_scc].first
    9066      7250966 :               && i <= toplevel_scc_extents[curr_scc].second)
    9067      3977074 :             rpo_state[i].max_rpo = toplevel_scc_extents[curr_scc].second;
    9068      7250966 :           if (i == toplevel_scc_extents[curr_scc].second)
    9069       744964 :             curr_scc++;
    9070              :         }
    9071     51923881 :       bb->flags &= ~BB_EXECUTABLE;
    9072     51923881 :       bool has_backedges = false;
    9073     51923881 :       edge e;
    9074     51923881 :       edge_iterator ei;
    9075    123236849 :       FOR_EACH_EDGE (e, ei, bb->preds)
    9076              :         {
    9077     71312968 :           if (e->flags & EDGE_DFS_BACK)
    9078      2900536 :             has_backedges = true;
    9079     71312968 :           e->flags &= ~EDGE_EXECUTABLE;
    9080     71312968 :           if (iterate || e == entry || (skip_entry_phis && bb == entry->dest))
    9081     71312968 :             continue;
    9082              :         }
    9083     51923881 :       rpo_state[i].iterate = iterate && has_backedges;
    9084              :     }
    9085      6308971 :   entry->flags |= EDGE_EXECUTABLE;
    9086      6308971 :   entry->dest->flags |= BB_EXECUTABLE;
    9087              : 
    9088              :   /* Verify we have no extra entries into the region.  */
    9089      6308971 :   if (flag_checking && do_region)
    9090              :     {
    9091              :       /* We can't merge the first two loops because we cannot rely
    9092              :          on EDGE_DFS_BACK for edges not within the region.  */
    9093      2156296 :       for (int i = 0; i < n; ++i)
    9094              :         {
    9095      1453511 :           basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
    9096      1453511 :           edge e;
    9097      1453511 :           edge_iterator ei;
    9098      3179338 :           FOR_EACH_EDGE (e, ei, bb->preds)
    9099      1725827 :             gcc_assert (e == entry
    9100              :                         || (skip_entry_phis && bb == entry->dest)
    9101              :                         || (e->src->flags & bb_in_region));
    9102              :         }
    9103              :     }
    9104              : 
    9105              :   /* As heuristic to improve compile-time we handle only the N innermost
    9106              :      loops and the outermost one optimistically.  */
    9107      6308971 :   if (iterate)
    9108              :     {
    9109      4381016 :       unsigned max_depth = param_rpo_vn_max_loop_depth;
    9110     14733527 :       for (auto loop : loops_list (cfun, LI_ONLY_INNERMOST))
    9111      1592951 :         if (loop_depth (loop) > max_depth)
    9112         2108 :           for (unsigned i = 2;
    9113         9160 :                i < loop_depth (loop) - max_depth; ++i)
    9114              :             {
    9115         2108 :               basic_block header = superloop_at_depth (loop, i)->header;
    9116         2108 :               bool non_latch_backedge = false;
    9117         2108 :               edge e;
    9118         2108 :               edge_iterator ei;
    9119         6336 :               FOR_EACH_EDGE (e, ei, header->preds)
    9120         4228 :                 if (e->flags & EDGE_DFS_BACK)
    9121              :                   {
    9122              :                     /* There can be a non-latch backedge into the header
    9123              :                        which is part of an outer irreducible region.  We
    9124              :                        cannot avoid iterating this block then.  */
    9125         2120 :                     if (!dominated_by_p (CDI_DOMINATORS,
    9126         2120 :                                          e->src, e->dest))
    9127              :                       {
    9128           12 :                         if (dump_file && (dump_flags & TDF_DETAILS))
    9129            0 :                           fprintf (dump_file, "non-latch backedge %d -> %d "
    9130              :                                    "forces iteration of loop %d\n",
    9131            0 :                                    e->src->index, e->dest->index, loop->num);
    9132              :                         non_latch_backedge = true;
    9133              :                       }
    9134              :                     else
    9135         2108 :                       e->flags |= EDGE_EXECUTABLE;
    9136              :                   }
    9137         2108 :               rpo_state[bb_to_rpo[header->index]].iterate = non_latch_backedge;
    9138      4381016 :             }
    9139              :     }
    9140              : 
    9141      6308971 :   uint64_t nblk = 0;
    9142      6308971 :   int idx = 0;
    9143      4381016 :   if (iterate)
    9144              :     /* Go and process all blocks, iterating as necessary.  */
    9145     50481600 :     do
    9146              :       {
    9147     50481600 :         basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
    9148              : 
    9149              :         /* If the block has incoming backedges remember unwind state.  This
    9150              :            is required even for non-executable blocks since in irreducible
    9151              :            regions we might reach them via the backedge and re-start iterating
    9152              :            from there.
    9153              :            Note we can individually mark blocks with incoming backedges to
    9154              :            not iterate where we then handle PHIs conservatively.  We do that
    9155              :            heuristically to reduce compile-time for degenerate cases.  */
    9156     50481600 :         if (rpo_state[idx].iterate)
    9157      4513268 :           record_unwind_state (&rpo_state[idx]);
    9158              : 
    9159     50481600 :         if (!(bb->flags & BB_EXECUTABLE))
    9160              :           {
    9161      1039788 :             if (dump_file && (dump_flags & TDF_DETAILS))
    9162            2 :               fprintf (dump_file, "Block %d: BB%d found not executable\n",
    9163              :                        idx, bb->index);
    9164      1039788 :             idx++;
    9165      2990315 :             continue;
    9166              :           }
    9167              : 
    9168     49441812 :         if (dump_file && (dump_flags & TDF_DETAILS))
    9169          338 :           fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
    9170     49441812 :         nblk++;
    9171     98883624 :         todo |= process_bb (avail, bb,
    9172     49441812 :                             rpo_state[idx].visited != 0,
    9173              :                             rpo_state[idx].iterate,
    9174              :                             iterate, eliminate, do_region, exit_bbs, false);
    9175     49441812 :         rpo_state[idx].visited++;
    9176              : 
    9177              :         /* Verify if changed values flow over executable outgoing backedges
    9178              :            and those change destination PHI values (that's the thing we
    9179              :            can easily verify).  Reduce over all such edges to the farthest
    9180              :            away PHI.  */
    9181     49441812 :         int iterate_to = -1;
    9182     49441812 :         edge_iterator ei;
    9183     49441812 :         edge e;
    9184    119038555 :         FOR_EACH_EDGE (e, ei, bb->succs)
    9185     69596743 :           if ((e->flags & (EDGE_DFS_BACK|EDGE_EXECUTABLE))
    9186              :               == (EDGE_DFS_BACK|EDGE_EXECUTABLE)
    9187      4478002 :               && rpo_state[bb_to_rpo[e->dest->index]].iterate)
    9188              :             {
    9189      4475274 :               int destidx = bb_to_rpo[e->dest->index];
    9190      4475274 :               if (!rpo_state[destidx].visited)
    9191              :                 {
    9192          138 :                   if (dump_file && (dump_flags & TDF_DETAILS))
    9193            0 :                     fprintf (dump_file, "Unvisited destination %d\n",
    9194              :                              e->dest->index);
    9195          138 :                   if (iterate_to == -1 || destidx < iterate_to)
    9196          138 :                     iterate_to = destidx;
    9197          138 :                   continue;
    9198              :                 }
    9199      4475136 :               if (dump_file && (dump_flags & TDF_DETAILS))
    9200           53 :                 fprintf (dump_file, "Looking for changed values of backedge"
    9201              :                          " %d->%d destination PHIs\n",
    9202           53 :                          e->src->index, e->dest->index);
    9203      4475136 :               vn_context_bb = e->dest;
    9204      4475136 :               gphi_iterator gsi;
    9205      4475136 :               for (gsi = gsi_start_phis (e->dest);
    9206     10124525 :                    !gsi_end_p (gsi); gsi_next (&gsi))
    9207              :                 {
    9208      7599804 :                   bool inserted = false;
    9209              :                   /* While we'd ideally just iterate on value changes
    9210              :                      we CSE PHIs and do that even across basic-block
    9211              :                      boundaries.  So even hashtable state changes can
    9212              :                      be important (which is roughly equivalent to
    9213              :                      PHI argument value changes).  To not excessively
    9214              :                      iterate because of that we track whether a PHI
    9215              :                      was CSEd to with GF_PLF_1.  */
    9216      7599804 :                   bool phival_changed;
    9217      7599804 :                   if ((phival_changed = visit_phi (gsi.phi (),
    9218              :                                                    &inserted, false))
    9219      9002132 :                       || (inserted && gimple_plf (gsi.phi (), GF_PLF_1)))
    9220              :                     {
    9221      1950415 :                       if (!phival_changed
    9222      1950415 :                           && dump_file && (dump_flags & TDF_DETAILS))
    9223            0 :                         fprintf (dump_file, "PHI was CSEd and hashtable "
    9224              :                                  "state (changed)\n");
    9225      1950415 :                       if (iterate_to == -1 || destidx < iterate_to)
    9226      1950392 :                         iterate_to = destidx;
    9227      1950415 :                       break;
    9228              :                     }
    9229              :                 }
    9230      4475136 :               vn_context_bb = NULL;
    9231              :             }
    9232     49441812 :         if (iterate_to != -1)
    9233              :           {
    9234      1950527 :             do_unwind (&rpo_state[iterate_to], avail);
    9235      1950527 :             idx = iterate_to;
    9236      1950527 :             if (dump_file && (dump_flags & TDF_DETAILS))
    9237           20 :               fprintf (dump_file, "Iterating to %d BB%d\n",
    9238           20 :                        iterate_to, rpo[iterate_to]);
    9239      1950527 :             continue;
    9240              :           }
    9241              : 
    9242     47491285 :         idx++;
    9243              :       }
    9244     50481600 :     while (idx < n);
    9245              : 
    9246              :   else /* !iterate */
    9247              :     {
    9248              :       /* Process all blocks greedily with a worklist that enforces RPO
    9249              :          processing of reachable blocks.  */
    9250      1927955 :       auto_bitmap worklist;
    9251      1927955 :       bitmap_set_bit (worklist, 0);
    9252     17685737 :       while (!bitmap_empty_p (worklist))
    9253              :         {
    9254     13829827 :           int idx = bitmap_clear_first_set_bit (worklist);
    9255     13829827 :           basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[idx]);
    9256     13829827 :           gcc_assert ((bb->flags & BB_EXECUTABLE)
    9257              :                       && !rpo_state[idx].visited);
    9258              : 
    9259     13829827 :           if (dump_file && (dump_flags & TDF_DETAILS))
    9260        36721 :             fprintf (dump_file, "Processing block %d: BB%d\n", idx, bb->index);
    9261              : 
    9262              :           /* When we run into predecessor edges where we cannot trust its
    9263              :              executable state mark them executable so PHI processing will
    9264              :              be conservative.
    9265              :              ???  Do we need to force arguments flowing over that edge
    9266              :              to be varying or will they even always be?  */
    9267     13829827 :           edge_iterator ei;
    9268     13829827 :           edge e;
    9269     33525346 :           FOR_EACH_EDGE (e, ei, bb->preds)
    9270     19695519 :             if (!(e->flags & EDGE_EXECUTABLE)
    9271      1044539 :                 && (bb == entry->dest
    9272       989816 :                     || (!rpo_state[bb_to_rpo[e->src->index]].visited
    9273       950769 :                         && (rpo_state[bb_to_rpo[e->src->index]].max_rpo
    9274              :                             >= (int)idx))))
    9275              :               {
    9276       981704 :                 if (dump_file && (dump_flags & TDF_DETAILS))
    9277        11800 :                   fprintf (dump_file, "Cannot trust state of predecessor "
    9278              :                            "edge %d -> %d, marking executable\n",
    9279        11800 :                            e->src->index, e->dest->index);
    9280       981704 :                 e->flags |= EDGE_EXECUTABLE;
    9281              :               }
    9282              : 
    9283     13829827 :           nblk++;
    9284     27659654 :           todo |= process_bb (avail, bb, false, false, false, eliminate,
    9285              :                               do_region, exit_bbs,
    9286        52570 :                               skip_entry_phis && bb == entry->dest);
    9287     13829827 :           rpo_state[idx].visited++;
    9288              : 
    9289     34182393 :           FOR_EACH_EDGE (e, ei, bb->succs)
    9290     20352566 :             if ((e->flags & EDGE_EXECUTABLE)
    9291     20271486 :                 && e->dest->index != EXIT_BLOCK
    9292     19076209 :                 && (!do_region || !bitmap_bit_p (exit_bbs, e->dest->index))
    9293     38051785 :                 && !rpo_state[bb_to_rpo[e->dest->index]].visited)
    9294     16723025 :               bitmap_set_bit (worklist, bb_to_rpo[e->dest->index]);
    9295              :         }
    9296      1927955 :     }
    9297              : 
    9298              :   /* If statistics or dump file active.  */
    9299      6308971 :   int nex = 0;
    9300      6308971 :   unsigned max_visited = 1;
    9301     58232852 :   for (int i = 0; i < n; ++i)
    9302              :     {
    9303     51923881 :       basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
    9304     51923881 :       if (bb->flags & BB_EXECUTABLE)
    9305     51285587 :         nex++;
    9306     51923881 :       statistics_histogram_event (cfun, "RPO block visited times",
    9307     51923881 :                                   rpo_state[i].visited);
    9308     51923881 :       if (rpo_state[i].visited > max_visited)
    9309              :         max_visited = rpo_state[i].visited;
    9310              :     }
    9311      6308971 :   unsigned nvalues = 0, navail = 0;
    9312    175478254 :   for (hash_table<vn_ssa_aux_hasher>::iterator i = vn_ssa_aux_hash->begin ();
    9313    175478254 :        i != vn_ssa_aux_hash->end (); ++i)
    9314              :     {
    9315    169169283 :       nvalues++;
    9316    169169283 :       vn_avail *av = (*i)->avail;
    9317    249859330 :       while (av)
    9318              :         {
    9319     80690047 :           navail++;
    9320     80690047 :           av = av->next;
    9321              :         }
    9322              :     }
    9323      6308971 :   statistics_counter_event (cfun, "RPO blocks", n);
    9324      6308971 :   statistics_counter_event (cfun, "RPO blocks visited", nblk);
    9325      6308971 :   statistics_counter_event (cfun, "RPO blocks executable", nex);
    9326      6308971 :   statistics_histogram_event (cfun, "RPO iterations", 10*nblk / nex);
    9327      6308971 :   statistics_histogram_event (cfun, "RPO num values", nvalues);
    9328      6308971 :   statistics_histogram_event (cfun, "RPO num avail", navail);
    9329      6308971 :   statistics_histogram_event (cfun, "RPO num lattice",
    9330      6308971 :                               vn_ssa_aux_hash->elements ());
    9331      6308971 :   if (dump_file && (dump_flags & (TDF_DETAILS|TDF_STATS)))
    9332              :     {
    9333        11706 :       fprintf (dump_file, "RPO iteration over %d blocks visited %" PRIu64
    9334              :                " blocks in total discovering %d executable blocks iterating "
    9335              :                "%d.%d times, a block was visited max. %u times\n",
    9336              :                n, nblk, nex,
    9337        11706 :                (int)((10*nblk / nex)/10), (int)((10*nblk / nex)%10),
    9338              :                max_visited);
    9339        11706 :       fprintf (dump_file, "RPO tracked %d values available at %d locations "
    9340              :                "and %" PRIu64 " lattice elements\n",
    9341        11706 :                nvalues, navail, (uint64_t) vn_ssa_aux_hash->elements ());
    9342              :     }
    9343              : 
    9344      6308971 :   if (eliminate)
    9345              :     {
    9346              :       /* When !iterate we already performed elimination during the RPO
    9347              :          walk.  */
    9348      5306386 :       if (iterate)
    9349              :         {
    9350              :           /* Elimination for region-based VN needs to be done within the
    9351              :              RPO walk.  */
    9352      3398046 :           gcc_assert (! do_region);
    9353              :           /* Note we can't use avail.walk here because that gets confused
    9354              :              by the existing availability and it will be less efficient
    9355              :              as well.  */
    9356      3398046 :           rpo_avail = eliminator;
    9357      3398046 :           eliminator->walk (ENTRY_BLOCK_PTR_FOR_FN (fn));
    9358              :         }
    9359              :     }
    9360              : 
    9361              :   /* Clear the bb_in_region flag.  */
    9362     58232852 :   for (int i = 0; i < n; ++i)
    9363              :     {
    9364     51923881 :       basic_block bb = BASIC_BLOCK_FOR_FN (fn, rpo[i]);
    9365     51923881 :       bb->flags &= ~bb_in_region;
    9366              :     }
    9367              : 
    9368      6308971 :   vn_valueize = NULL;
    9369      6308971 :   rpo_avail = NULL;
    9370      6308971 :   vn_bb_to_rpo = NULL;
    9371      6308971 :   vn_bb_in_region_flag = 0;
    9372      6308971 :   vn_entry_edge = NULL;
    9373              : 
    9374      6308971 :   vn_exec_idom = NULL;
    9375              : 
    9376      6308971 :   XDELETEVEC (rpo);
    9377      6308971 :   XDELETEVEC (rpo_state);
    9378              : 
    9379      6308971 :   return todo;
    9380      6308971 : }
    9381              : 
    9382              : /* Region-based entry for RPO VN.  Performs value-numbering and elimination
    9383              :    on the SEME region specified by ENTRY and EXIT_BBS.  If ENTRY is not
    9384              :    the only edge into the region at ENTRY->dest PHI nodes in ENTRY->dest
    9385              :    are not considered.
    9386              :    If ITERATE is true then treat backedges optimistically as not
    9387              :    executed and iterate.  If ELIMINATE is true then perform
    9388              :    elimination, otherwise leave that to the caller.
    9389              :    If SKIP_ENTRY_PHIS is true then force PHI nodes in ENTRY->dest to VARYING.
    9390              :    KIND specifies the amount of work done for handling memory operations.  */
    9391              : 
    9392              : unsigned
    9393       722406 : do_rpo_vn (function *fn, edge entry, bitmap exit_bbs,
    9394              :            bool iterate, bool eliminate, bool skip_entry_phis,
    9395              :            vn_lookup_kind kind)
    9396              : {
    9397       722406 :   auto_timevar tv (TV_TREE_RPO_VN);
    9398              : 
    9399       722406 :   vn_driver vn (fn, entry, exit_bbs, iterate, eliminate,
    9400       722406 :                 skip_entry_phis, kind);
    9401       722406 :   return vn.finalize ();
    9402       722406 : }
    9403              : 
    9404      5586565 : vn_driver::vn_driver (function *fn, bool iterate, bool eliminate,
    9405              :                       bool want_value_ids, vn_lookup_kind kind)
    9406      5586565 :   : vn_driver (fn, NULL, NULL, iterate, eliminate, false, kind)
    9407              : {
    9408      5586565 :   if (want_value_ids)
    9409       982970 :     assign_value_ids ();
    9410      5586565 : }
    9411              : 
    9412              : /* Value-number the region ENTRY leads into, which exits to EXIT_BBS, or all of
    9413              :    FN if ENTRY is NULL.  */
    9414              : 
    9415      6308971 : vn_driver::vn_driver (function *fn, edge entry, bitmap exit_bbs,
    9416              :                       bool iterate, bool eliminate, bool skip_entry_phis,
    9417              :                       vn_lookup_kind kind)
    9418      6308971 :   : m_eliminate (eliminate), m_region_p (entry != NULL)
    9419              : {
    9420      6308971 :   basic_block entry_bb
    9421      6308971 :     = entry ? entry->dest : single_succ (ENTRY_BLOCK_PTR_FOR_FN (fn));
    9422      6308971 :   m_avail = new rpo_elim (entry_bb);
    9423      6308971 :   m_bb_to_rpo = XNEWVEC (int, last_basic_block_for_fn (fn));
    9424      6308971 :   m_exec_idom = XNEWVEC (int, n_basic_blocks_for_fn (fn));
    9425              :   /* When iterating, elimination is a walk of its own.  */
    9426     12617942 :   m_eliminator = (iterate && eliminate
    9427      6308971 :                   ? new eliminate_dom_walker (CDI_DOMINATORS, NULL) : m_avail);
    9428     12617942 :   m_todo = do_rpo_vn_1 (fn, entry, exit_bbs, iterate, eliminate,
    9429      6308971 :                         skip_entry_phis, kind, *m_avail, m_bb_to_rpo,
    9430              :                         m_exec_idom, m_eliminator);
    9431      6308971 : }
    9432              : 
    9433      6308971 : vn_driver::~vn_driver ()
    9434              : {
    9435      6308971 :   free_rpo_vn ();
    9436      6308971 :   XDELETEVEC (m_bb_to_rpo);
    9437      6308971 :   XDELETEVEC (m_exec_idom);
    9438      6308971 :   if (m_eliminator != m_avail)
    9439      3398046 :     delete m_eliminator;
    9440      6308971 :   delete m_avail;
    9441      6308971 : }
    9442              : 
    9443              : /* Finish elimination and return the TODO flags.  */
    9444              : 
    9445              : unsigned
    9446      6308971 : vn_driver::finalize (bitmap inserted_exprs)
    9447              : {
    9448              :   /* If ELIMINATE was passed to the constructor, finish the elimination.  */
    9449      6308971 :   if (m_eliminate)
    9450              :     {
    9451      5306386 :       vn_bb_to_rpo = m_bb_to_rpo;
    9452      5306386 :       vn_exec_idom = m_exec_idom;
    9453      5306386 :       unsigned todo = m_todo | m_eliminator->eliminate_cleanup (m_region_p);
    9454      5306386 :       vn_bb_to_rpo = NULL;
    9455      5306386 :       vn_exec_idom = NULL;
    9456      5306386 :       return todo;
    9457              :     }
    9458              : 
    9459              :   /* PRE eliminates after its insertions, when the RPO map no longer
    9460              :      describes the IL.  Return only its flags, as PRE always has, to avoid
    9461              :      an extra CFG cleanup.  */
    9462      1002585 :   if (inserted_exprs)
    9463       982970 :     return eliminate_with_rpo_vn (inserted_exprs);
    9464              : 
    9465        19615 :   return m_todo;
    9466              : }
    9467              : 
    9468              : 
    9469              : namespace {
    9470              : 
    9471              : const pass_data pass_data_fre =
    9472              : {
    9473              :   GIMPLE_PASS, /* type */
    9474              :   "fre", /* name */
    9475              :   OPTGROUP_NONE, /* optinfo_flags */
    9476              :   TV_TREE_FRE, /* tv_id */
    9477              :   ( PROP_cfg | PROP_ssa ), /* properties_required */
    9478              :   0, /* properties_provided */
    9479              :   0, /* properties_destroyed */
    9480              :   0, /* todo_flags_start */
    9481              :   0, /* todo_flags_finish */
    9482              : };
    9483              : 
    9484              : class pass_fre : public gimple_opt_pass
    9485              : {
    9486              : public:
    9487      1485900 :   pass_fre (gcc::context *ctxt)
    9488      2971800 :     : gimple_opt_pass (pass_data_fre, ctxt), may_iterate (true)
    9489              :   {}
    9490              : 
    9491              :   /* opt_pass methods: */
    9492      1188720 :   opt_pass * clone () final override { return new pass_fre (m_ctxt); }
    9493      1485900 :   void set_pass_param (unsigned int n, bool param) final override
    9494              :     {
    9495      1485900 :       gcc_assert (n == 0);
    9496      1485900 :       may_iterate = param;
    9497      1485900 :     }
    9498      4685217 :   bool gate (function *) final override
    9499              :     {
    9500      4685217 :       return flag_tree_fre != 0 && (may_iterate || optimize > 1);
    9501              :     }
    9502              :   unsigned int execute (function *) final override;
    9503              : 
    9504              : private:
    9505              :   bool may_iterate;
    9506              : }; // class pass_fre
    9507              : 
    9508              : unsigned int
    9509      4603595 : pass_fre::execute (function *fun)
    9510              : {
    9511      4603595 :   unsigned todo = 0;
    9512              : 
    9513              :   /* At -O[1g] use the cheap non-iterating mode.  */
    9514      4603595 :   bool iterate_p = may_iterate && (optimize > 1);
    9515      4603595 :   calculate_dominance_info (CDI_DOMINATORS);
    9516      4603595 :   if (iterate_p)
    9517              :     /* When iterating make sure to minimize the number of backedges.  */
    9518      3398046 :     loop_optimizer_init (LOOPS_HAVE_SIMPLE_LATCHES);
    9519              : 
    9520      4603595 :   {
    9521      4603595 :     vn_driver vn (fun, iterate_p, true);
    9522      4603595 :     todo = vn.finalize ();
    9523      4603595 :   }
    9524              : 
    9525      4603595 :   if (iterate_p)
    9526      3398046 :     loop_optimizer_finalize ();
    9527              : 
    9528      4603595 :   if (scev_initialized_p ())
    9529        33272 :     scev_reset_htab ();
    9530              : 
    9531              :   /* For late FRE after IVOPTs and unrolling, see if we can
    9532              :      remove some TREE_ADDRESSABLE and rewrite stuff into SSA.  */
    9533      4603595 :   if (!may_iterate)
    9534      1016032 :     todo |= TODO_update_address_taken;
    9535              : 
    9536      4603595 :   return todo;
    9537              : }
    9538              : 
    9539              : } // anon namespace
    9540              : 
    9541              : gimple_opt_pass *
    9542       297180 : make_pass_fre (gcc::context *ctxt)
    9543              : {
    9544       297180 :   return new pass_fre (ctxt);
    9545              : }
    9546              : 
    9547              : #undef BB_EXECUTABLE
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.