LCOV - code coverage report
Current view: top level - gcc - gimple-fold.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 92.4 % 5601 5176
Test Date: 2026-11-14 16:39:50 Functions: 96.7 % 153 148
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Statement simplification on GIMPLE.
       2              :    Copyright (C) 2010-2026 Free Software Foundation, Inc.
       3              :    Split out from tree-ssa-ccp.cc.
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it
       8              : under the terms of the GNU General Public License as published by the
       9              : Free Software Foundation; either version 3, or (at your option) any
      10              : later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT
      13              : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "gimple.h"
      29              : #include "predict.h"
      30              : #include "ssa.h"
      31              : #include "cgraph.h"
      32              : #include "gimple-pretty-print.h"
      33              : #include "gimple-ssa-warn-access.h"
      34              : #include "gimple-ssa-warn-restrict.h"
      35              : #include "fold-const.h"
      36              : #include "stmt.h"
      37              : #include "expr.h"
      38              : #include "stor-layout.h"
      39              : #include "dumpfile.h"
      40              : #include "gimple-iterator.h"
      41              : #include "tree-pass.h"
      42              : #include "gimple-fold.h"
      43              : #include "gimplify.h"
      44              : #include "tree-into-ssa.h"
      45              : #include "tree-dfa.h"
      46              : #include "tree-object-size.h"
      47              : #include "tree-ssa.h"
      48              : #include "tree-ssa-propagate.h"
      49              : #include "ipa-utils.h"
      50              : #include "tree-ssa-address.h"
      51              : #include "langhooks.h"
      52              : #include "gimplify-me.h"
      53              : #include "dbgcnt.h"
      54              : #include "builtins.h"
      55              : #include "tree-eh.h"
      56              : #include "gimple-match.h"
      57              : #include "gomp-constants.h"
      58              : #include "optabs-query.h"
      59              : #include "omp-general.h"
      60              : #include "tree-cfg.h"
      61              : #include "fold-const-call.h"
      62              : #include "stringpool.h"
      63              : #include "attribs.h"
      64              : #include "asan.h"
      65              : #include "diagnostic-core.h"
      66              : #include "intl.h"
      67              : #include "calls.h"
      68              : #include "tree-vector-builder.h"
      69              : #include "tree-ssa-strlen.h"
      70              : #include "varasm.h"
      71              : #include "internal-fn.h"
      72              : #include "gimple-range.h"
      73              : 
      74              : enum strlen_range_kind {
      75              :   /* Compute the exact constant string length.  */
      76              :   SRK_STRLEN,
      77              :   /* Compute the maximum constant string length.  */
      78              :   SRK_STRLENMAX,
      79              :   /* Compute a range of string lengths bounded by object sizes.  When
      80              :      the length of a string cannot be determined, consider as the upper
      81              :      bound the size of the enclosing object the string may be a member
      82              :      or element of.  Also determine the size of the largest character
      83              :      array the string may refer to.  */
      84              :   SRK_LENRANGE,
      85              :   /* Determine the integer value of the argument (not string length).  */
      86              :   SRK_INT_VALUE
      87              : };
      88              : 
      89              : static bool
      90              : get_range_strlen (tree, bitmap, strlen_range_kind, c_strlen_data *, unsigned);
      91              : 
      92              : /* Return true when DECL can be referenced from current unit.
      93              :    FROM_DECL (if non-null) specify constructor of variable DECL was taken from.
      94              :    We can get declarations that are not possible to reference for various
      95              :    reasons:
      96              : 
      97              :      1) When analyzing C++ virtual tables.
      98              :         C++ virtual tables do have known constructors even
      99              :         when they are keyed to other compilation unit.
     100              :         Those tables can contain pointers to methods and vars
     101              :         in other units.  Those methods have both STATIC and EXTERNAL
     102              :         set.
     103              :      2) In WHOPR mode devirtualization might lead to reference
     104              :         to method that was partitioned elsewhere.
     105              :         In this case we have static VAR_DECL or FUNCTION_DECL
     106              :         that has no corresponding callgraph/varpool node
     107              :         declaring the body.
     108              :      3) COMDAT functions referred by external vtables that
     109              :         we devirtualize only during final compilation stage.
     110              :         At this time we already decided that we will not output
     111              :         the function body and thus we can't reference the symbol
     112              :         directly.  */
     113              : 
     114              : static bool
     115      4443238 : can_refer_decl_in_current_unit_p (tree decl, tree from_decl)
     116              : {
     117      4443238 :   varpool_node *vnode;
     118      4443238 :   struct cgraph_node *node;
     119      4443238 :   symtab_node *snode;
     120              : 
     121      4443238 :   if (DECL_ABSTRACT_P (decl))
     122              :     return false;
     123              : 
     124              :   /* We are concerned only about static/external vars and functions.  */
     125      1502014 :   if ((!TREE_STATIC (decl) && !DECL_EXTERNAL (decl))
     126      5540445 :       || !VAR_OR_FUNCTION_DECL_P (decl))
     127              :     return true;
     128              : 
     129              :   /* Static objects can be referred only if they are defined and not optimized
     130              :      out yet.  */
     131      4038431 :   if (!TREE_PUBLIC (decl))
     132              :     {
     133      1101075 :       if (DECL_EXTERNAL (decl))
     134              :         return false;
     135              :       /* Before we start optimizing unreachable code we can be sure all
     136              :          static objects are defined.  */
     137      1101030 :       if (symtab->function_flags_ready)
     138              :         return true;
     139      1063737 :       snode = symtab_node::get (decl);
     140      1063737 :       if (!snode || !snode->definition)
     141              :         return false;
     142      1063682 :       node = dyn_cast <cgraph_node *> (snode);
     143         8910 :       return !node || !node->inlined_to;
     144              :     }
     145              : 
     146              :   /* We will later output the initializer, so we can refer to it.
     147              :      So we are concerned only when DECL comes from initializer of
     148              :      external var or var that has been optimized out.  */
     149      2937356 :   if (!from_decl
     150       475475 :       || !VAR_P (from_decl)
     151       474346 :       || (!DECL_EXTERNAL (from_decl)
     152       210651 :           && (vnode = varpool_node::get (from_decl)) != NULL
     153       145858 :           && vnode->definition)
     154      3265850 :       || (flag_ltrans
     155            3 :           && (vnode = varpool_node::get (from_decl)) != NULL
     156            3 :           && vnode->in_other_partition))
     157              :     return true;
     158              :   /* We are folding reference from external vtable.  The vtable may refer
     159              :      to a symbol keyed to other compilation unit.  The other compilation
     160              :      unit may be in separate DSO and the symbol may be hidden.  */
     161       328491 :   if (DECL_VISIBILITY_SPECIFIED (decl)
     162       321417 :       && DECL_EXTERNAL (decl)
     163       256244 :       && DECL_VISIBILITY (decl) != VISIBILITY_DEFAULT
     164       499712 :       && (!(snode = symtab_node::get (decl)) || !snode->in_other_partition))
     165              :     return false;
     166              :   /* When function is public, we always can introduce new reference.
     167              :      Exception are the COMDAT functions where introducing a direct
     168              :      reference imply need to include function body in the current unit.  */
     169       157270 :   if (TREE_PUBLIC (decl) && !DECL_COMDAT (decl))
     170              :     return true;
     171              :   /* We have COMDAT.  We are going to check if we still have definition
     172              :      or if the definition is going to be output in other partition.
     173              :      Bypass this when gimplifying; all needed functions will be produced.
     174              : 
     175              :      As observed in PR20991 for already optimized out comdat virtual functions
     176              :      it may be tempting to not necessarily give up because the copy will be
     177              :      output elsewhere when corresponding vtable is output.
     178              :      This is however not possible - ABI specify that COMDATs are output in
     179              :      units where they are used and when the other unit was compiled with LTO
     180              :      it is possible that vtable was kept public while the function itself
     181              :      was privatized. */
     182       110841 :   if (!symtab->function_flags_ready)
     183              :     return true;
     184              : 
     185        97993 :   snode = symtab_node::get (decl);
     186        97993 :   if (!snode
     187        97993 :       || ((!snode->definition || DECL_EXTERNAL (decl))
     188        12216 :           && (!snode->in_other_partition
     189            0 :               || (!snode->forced_by_abi && !snode->force_output))))
     190              :     return false;
     191        60061 :   node = dyn_cast <cgraph_node *> (snode);
     192        60061 :   return !node || !node->inlined_to;
     193              : }
     194              : 
     195              : /* CVAL is value taken from DECL_INITIAL of variable.  Try to transform it into
     196              :    acceptable form for is_gimple_min_invariant.
     197              :    FROM_DECL (if non-NULL) specify variable whose constructor contains CVAL.  */
     198              : 
     199              : tree
     200     15603161 : canonicalize_constructor_val (tree cval, tree from_decl)
     201              : {
     202     15603161 :   if (CONSTANT_CLASS_P (cval))
     203              :     return cval;
     204              : 
     205      9431970 :   tree orig_cval = cval;
     206      9431970 :   STRIP_NOPS (cval);
     207      9431970 :   if (TREE_CODE (cval) == POINTER_PLUS_EXPR
     208      9431970 :       && TREE_CODE (TREE_OPERAND (cval, 1)) == INTEGER_CST)
     209              :     {
     210        71935 :       tree ptr = TREE_OPERAND (cval, 0);
     211        71935 :       if (is_gimple_min_invariant (ptr))
     212       215094 :         cval = build1_loc (EXPR_LOCATION (cval),
     213        71698 :                            ADDR_EXPR, TREE_TYPE (ptr),
     214       143396 :                            fold_build2 (MEM_REF, TREE_TYPE (TREE_TYPE (ptr)),
     215              :                                         ptr,
     216              :                                         fold_convert (ptr_type_node,
     217              :                                                       TREE_OPERAND (cval, 1))));
     218              :     }
     219      9431970 :   if (TREE_CODE (cval) == ADDR_EXPR)
     220              :     {
     221      5162771 :       tree base = NULL_TREE;
     222      5162771 :       if (TREE_CODE (TREE_OPERAND (cval, 0)) == COMPOUND_LITERAL_EXPR)
     223              :         {
     224          193 :           base = COMPOUND_LITERAL_EXPR_DECL (TREE_OPERAND (cval, 0));
     225          193 :           if (base)
     226          193 :             TREE_OPERAND (cval, 0) = base;
     227              :         }
     228              :       else
     229      5162578 :         base = get_base_address (TREE_OPERAND (cval, 0));
     230      5162771 :       if (!base)
     231              :         return NULL_TREE;
     232              : 
     233      2297114 :       if (VAR_OR_FUNCTION_DECL_P (base)
     234      6461598 :           && !can_refer_decl_in_current_unit_p (base, from_decl))
     235              :         return NULL_TREE;
     236      4990768 :       if (TREE_TYPE (base) == error_mark_node)
     237              :         return NULL_TREE;
     238      4990768 :       if (VAR_P (base))
     239              :         /* ???  We should be able to assert that TREE_ADDRESSABLE is set,
     240              :            but since the use can be in a debug stmt we can't.  */
     241              :         ;
     242      2296229 :       else if (TREE_CODE (base) == FUNCTION_DECL)
     243              :         {
     244              :           /* Make sure we create a cgraph node for functions we'll reference.
     245              :              They can be non-existent if the reference comes from an entry
     246              :              of an external vtable for example.  */
     247      1297942 :           cgraph_node::get_create (base);
     248              :         }
     249              :       /* Fixup types in global initializers.  */
     250      4990768 :       if (TREE_TYPE (TREE_TYPE (cval)) != TREE_TYPE (TREE_OPERAND (cval, 0)))
     251        39996 :         cval = build_fold_addr_expr (TREE_OPERAND (cval, 0));
     252              : 
     253      4990768 :       if (!useless_type_conversion_p (TREE_TYPE (orig_cval), TREE_TYPE (cval)))
     254       213963 :         cval = fold_convert (TREE_TYPE (orig_cval), cval);
     255              :       return cval;
     256              :     }
     257              :   /* In CONSTRUCTORs we may see unfolded constants like (int (*) ()) 0.  */
     258      4269199 :   if (TREE_CODE (cval) == INTEGER_CST)
     259              :     {
     260        65685 :       if (TREE_OVERFLOW_P (cval))
     261            0 :         cval = drop_tree_overflow (cval);
     262        65685 :       if (!useless_type_conversion_p (TREE_TYPE (orig_cval), TREE_TYPE (cval)))
     263        62484 :         cval = fold_convert (TREE_TYPE (orig_cval), cval);
     264              :       return cval;
     265              :     }
     266              :   return orig_cval;
     267              : }
     268              : 
     269              : /* If SYM is a constant variable with known value, return the value.
     270              :    NULL_TREE is returned otherwise.  */
     271              : 
     272              : tree
     273     20645139 : get_symbol_constant_value (tree sym)
     274              : {
     275     20645139 :   tree val = ctor_for_folding (sym);
     276     20645139 :   if (val != error_mark_node)
     277              :     {
     278        39867 :       if (val)
     279              :         {
     280        37569 :           val = canonicalize_constructor_val (unshare_expr (val), sym);
     281        37569 :           if (val
     282        37569 :               && is_gimple_min_invariant (val)
     283        66317 :               && useless_type_conversion_p (TREE_TYPE (sym), TREE_TYPE (val)))
     284              :             return val;
     285              :           else
     286         8927 :             return NULL_TREE;
     287              :         }
     288              :       /* Variables declared 'const' without an initializer
     289              :          have zero as the initializer if they may not be
     290              :          overridden at link or run time.  */
     291         2298 :       if (!val
     292         2298 :           && is_gimple_reg_type (TREE_TYPE (sym)))
     293         1961 :         return build_zero_cst (TREE_TYPE (sym));
     294              :     }
     295              : 
     296              :   return NULL_TREE;
     297              : }
     298              : 
     299              : 
     300              : 
     301              : /* Subroutine of fold_stmt.  We perform constant folding of the
     302              :    memory reference tree EXPR.  */
     303              : 
     304              : static tree
     305     66203625 : maybe_fold_reference (tree expr)
     306              : {
     307     66203625 :   tree result = NULL_TREE;
     308              : 
     309              :   /* Avoid expensive fold_const_aggregate_ref early on aggregate loads
     310              :      and esp. replacing STRING_CSTs inline.  */
     311     66203625 :   if (!is_gimple_reg_type (TREE_TYPE (expr)))
     312              :     return NULL_TREE;
     313              : 
     314     63050257 :   if ((TREE_CODE (expr) == VIEW_CONVERT_EXPR
     315     61711603 :        || TREE_CODE (expr) == REALPART_EXPR
     316     60912562 :        || TREE_CODE (expr) == IMAGPART_EXPR)
     317     64785137 :       && CONSTANT_CLASS_P (TREE_OPERAND (expr, 0)))
     318         5831 :     result = fold_unary_loc (EXPR_LOCATION (expr),
     319              :                              TREE_CODE (expr),
     320         5831 :                              TREE_TYPE (expr),
     321         5831 :                              TREE_OPERAND (expr, 0));
     322     63044426 :   else if (TREE_CODE (expr) == BIT_FIELD_REF
     323     63044426 :            && CONSTANT_CLASS_P (TREE_OPERAND (expr, 0)))
     324           49 :     result = fold_ternary_loc (EXPR_LOCATION (expr),
     325              :                                TREE_CODE (expr),
     326           49 :                                TREE_TYPE (expr),
     327           49 :                                TREE_OPERAND (expr, 0),
     328           49 :                                TREE_OPERAND (expr, 1),
     329           49 :                                TREE_OPERAND (expr, 2));
     330              :   else
     331     63044377 :     result = fold_const_aggregate_ref (expr);
     332              : 
     333     63050257 :   if (result && is_gimple_min_invariant (result))
     334       108896 :     return result;
     335              : 
     336              :   return NULL_TREE;
     337              : }
     338              : 
     339              : /* Return true if EXPR is an acceptable right-hand-side for a
     340              :    GIMPLE assignment.  We validate the entire tree, not just
     341              :    the root node, thus catching expressions that embed complex
     342              :    operands that are not permitted in GIMPLE.  This function
     343              :    is needed because the folding routines in fold-const.cc
     344              :    may return such expressions in some cases, e.g., an array
     345              :    access with an embedded index addition.  It may make more
     346              :    sense to have folding routines that are sensitive to the
     347              :    constraints on GIMPLE operands, rather than abandoning any
     348              :    any attempt to fold if the usual folding turns out to be too
     349              :    aggressive.  */
     350              : 
     351              : bool
     352            0 : valid_gimple_rhs_p (tree expr)
     353              : {
     354            0 :   enum tree_code code = TREE_CODE (expr);
     355              : 
     356            0 :   switch (TREE_CODE_CLASS (code))
     357              :     {
     358            0 :     case tcc_declaration:
     359            0 :       if (!is_gimple_variable (expr))
     360              :         return false;
     361              :       break;
     362              : 
     363              :     case tcc_constant:
     364              :       /* All constants are ok.  */
     365              :       break;
     366              : 
     367            0 :     case tcc_comparison:
     368              :       /* GENERIC allows comparisons with non-boolean types, reject
     369              :          those for GIMPLE.  Let vector-typed comparisons pass - rules
     370              :          for GENERIC and GIMPLE are the same here.  */
     371            0 :       if (!(INTEGRAL_TYPE_P (TREE_TYPE (expr))
     372            0 :             && (TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE
     373            0 :                 || TYPE_PRECISION (TREE_TYPE (expr)) == 1))
     374            0 :           && ! VECTOR_TYPE_P (TREE_TYPE (expr)))
     375              :         return false;
     376              : 
     377              :       /* Fallthru.  */
     378            0 :     case tcc_binary:
     379            0 :       if (!is_gimple_val (TREE_OPERAND (expr, 0))
     380            0 :           || !is_gimple_val (TREE_OPERAND (expr, 1)))
     381              :         return false;
     382              :       break;
     383              : 
     384            0 :     case tcc_unary:
     385            0 :       if (!is_gimple_val (TREE_OPERAND (expr, 0)))
     386              :         return false;
     387              :       break;
     388              : 
     389            0 :     case tcc_expression:
     390            0 :       switch (code)
     391              :         {
     392            0 :         case ADDR_EXPR:
     393            0 :           {
     394            0 :             tree t;
     395            0 :             if (is_gimple_min_invariant (expr))
     396              :               return true;
     397            0 :             t = TREE_OPERAND (expr, 0);
     398            0 :             while (handled_component_p (t))
     399              :               {
     400              :                 /* ??? More checks needed, see the GIMPLE verifier.  */
     401            0 :                 if ((TREE_CODE (t) == ARRAY_REF
     402            0 :                      || TREE_CODE (t) == ARRAY_RANGE_REF)
     403            0 :                     && !is_gimple_val (TREE_OPERAND (t, 1)))
     404              :                   return false;
     405            0 :                 t = TREE_OPERAND (t, 0);
     406              :               }
     407            0 :             if (!is_gimple_id (t))
     408              :               return false;
     409              :           }
     410              :           break;
     411              : 
     412            0 :         default:
     413            0 :           if (get_gimple_rhs_class (code) == GIMPLE_TERNARY_RHS)
     414              :             {
     415            0 :               if (!is_gimple_val (TREE_OPERAND (expr, 0))
     416            0 :                   || !is_gimple_val (TREE_OPERAND (expr, 1))
     417            0 :                   || !is_gimple_val (TREE_OPERAND (expr, 2)))
     418              :                 return false;
     419              :               break;
     420              :             }
     421              :           return false;
     422              :         }
     423              :       break;
     424              : 
     425              :     case tcc_vl_exp:
     426              :       return false;
     427              : 
     428            0 :     case tcc_exceptional:
     429            0 :       if (code == CONSTRUCTOR)
     430              :         {
     431              :           unsigned i;
     432              :           tree elt;
     433            0 :           FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (expr), i, elt)
     434            0 :             if (!is_gimple_val (elt))
     435              :               return false;
     436              :           return true;
     437              :         }
     438            0 :       if (code != SSA_NAME)
     439              :         return false;
     440              :       break;
     441              : 
     442            0 :     case tcc_reference:
     443            0 :       if (code == BIT_FIELD_REF)
     444            0 :         return is_gimple_val (TREE_OPERAND (expr, 0));
     445              :       return false;
     446              : 
     447              :     default:
     448              :       return false;
     449              :     }
     450              : 
     451              :   return true;
     452              : }
     453              : 
     454              : 
     455              : /* Attempt to fold an assignment statement pointed-to by SI.  Returns a
     456              :    replacement rhs for the statement or NULL_TREE if no simplification
     457              :    could be made.  It is assumed that the operands have been previously
     458              :    folded.  */
     459              : 
     460              : static tree
     461    264393960 : fold_gimple_assign (gimple_stmt_iterator *si)
     462              : {
     463    264393960 :   gimple *stmt = gsi_stmt (*si);
     464    264393960 :   enum tree_code subcode = gimple_assign_rhs_code (stmt);
     465    264393960 :   location_t loc = gimple_location (stmt);
     466              : 
     467    264393960 :   tree result = NULL_TREE;
     468              : 
     469    264393960 :   switch (get_gimple_rhs_class (subcode))
     470              :     {
     471    175474944 :     case GIMPLE_SINGLE_RHS:
     472    175474944 :       {
     473    175474944 :         tree rhs = gimple_assign_rhs1 (stmt);
     474              : 
     475    175474944 :         if (TREE_CLOBBER_P (rhs))
     476              :           return NULL_TREE;
     477              : 
     478    161144786 :         if (REFERENCE_CLASS_P (rhs))
     479     63516042 :           return maybe_fold_reference (rhs);
     480              : 
     481     97628744 :         else if (TREE_CODE (rhs) == OBJ_TYPE_REF)
     482              :           {
     483       168337 :             tree val = OBJ_TYPE_REF_EXPR (rhs);
     484       168337 :             if (is_gimple_min_invariant (val))
     485              :               return val;
     486       168314 :             else if (flag_devirtualize && virtual_method_call_p (rhs))
     487              :               {
     488       168274 :                 bool final;
     489       168274 :                 vec <cgraph_node *>targets
     490       168274 :                   = possible_polymorphic_call_targets (rhs, stmt, &final);
     491       168546 :                 if (final && targets.length () <= 1 && dbg_cnt (devirt))
     492              :                   {
     493           42 :                     if (dump_enabled_p ())
     494              :                       {
     495            0 :                         dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
     496              :                                          "resolving virtual function address "
     497              :                                          "reference to function %s\n",
     498            0 :                                          targets.length () == 1
     499            0 :                                          ? targets[0]->name ()
     500              :                                          : "NULL");
     501              :                       }
     502           42 :                     if (targets.length () == 1)
     503              :                       {
     504           33 :                         val = fold_convert (TREE_TYPE (val),
     505              :                                             build_fold_addr_expr_loc
     506              :                                               (loc, targets[0]->decl));
     507           33 :                         STRIP_USELESS_TYPE_CONVERSION (val);
     508              :                       }
     509              :                     else
     510              :                       /* We cannot use __builtin_unreachable here because it
     511              :                          cannot have address taken.  */
     512            9 :                       val = build_int_cst (TREE_TYPE (val), 0);
     513           42 :                     return val;
     514              :                   }
     515              :               }
     516              :           }
     517              : 
     518     97460407 :         else if (TREE_CODE (rhs) == ADDR_EXPR)
     519              :           {
     520     15316611 :             tree ref = TREE_OPERAND (rhs, 0);
     521     15316611 :             if (TREE_CODE (ref) == MEM_REF
     522     15316611 :                 && integer_zerop (TREE_OPERAND (ref, 1)))
     523              :               {
     524         2676 :                 result = TREE_OPERAND (ref, 0);
     525         2676 :                 if (!useless_type_conversion_p (TREE_TYPE (rhs),
     526         2676 :                                                 TREE_TYPE (result)))
     527            0 :                   result = build1 (NOP_EXPR, TREE_TYPE (rhs), result);
     528              :                 return result;
     529              :               }
     530              :           }
     531              : 
     532     82143796 :         else if (TREE_CODE (rhs) == CONSTRUCTOR
     533     82143796 :                  && TREE_CODE (TREE_TYPE (rhs)) == VECTOR_TYPE)
     534              :           {
     535              :             /* Fold a constant vector CONSTRUCTOR to VECTOR_CST.  */
     536              :             unsigned i;
     537              :             tree val;
     538              : 
     539       471750 :             FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), i, val)
     540       467378 :               if (! CONSTANT_CLASS_P (val))
     541              :                 return NULL_TREE;
     542              : 
     543         4372 :             return build_vector_from_ctor (TREE_TYPE (rhs),
     544         8744 :                                            CONSTRUCTOR_ELTS (rhs));
     545              :           }
     546              : 
     547     81770133 :         else if (DECL_P (rhs)
     548     81770133 :                  && is_gimple_reg_type (TREE_TYPE (rhs)))
     549     12384010 :           return get_symbol_constant_value (rhs);
     550              :       }
     551              :       break;
     552              : 
     553              :     case GIMPLE_UNARY_RHS:
     554              :       break;
     555              : 
     556              :     case GIMPLE_BINARY_RHS:
     557              :       break;
     558              : 
     559       567439 :     case GIMPLE_TERNARY_RHS:
     560      1134878 :       result = fold_ternary_loc (loc, subcode,
     561       567439 :                                  TREE_TYPE (gimple_assign_lhs (stmt)),
     562              :                                  gimple_assign_rhs1 (stmt),
     563              :                                  gimple_assign_rhs2 (stmt),
     564              :                                  gimple_assign_rhs3 (stmt));
     565              : 
     566       567439 :       if (result)
     567              :         {
     568            0 :           STRIP_USELESS_TYPE_CONVERSION (result);
     569            0 :           if (valid_gimple_rhs_p (result))
     570            0 :             return result;
     571              :         }
     572              :       break;
     573              : 
     574            0 :     case GIMPLE_INVALID_RHS:
     575            0 :       gcc_unreachable ();
     576              :     }
     577              : 
     578              :   return NULL_TREE;
     579              : }
     580              : 
     581              : 
     582              : /* Replace a statement at *SI_P with a sequence of statements in STMTS,
     583              :    adjusting the replacement stmts location and virtual operands.
     584              : 
     585              :    If the statements has an lhs, either:
     586              : 
     587              :    - the last statement of the new sequence must assign to the same lhs or
     588              : 
     589              :    - the caller must ensure that all uses of the old lhs have been
     590              :      removed before calling this function.  This includes removing
     591              :      all debug uses.  */
     592              : 
     593              : void
     594       129456 : gsi_replace_with_seq_vops (gimple_stmt_iterator *si_p, gimple_seq stmts)
     595              : {
     596       129456 :   gimple *stmt = gsi_stmt (*si_p);
     597              : 
     598       129456 :   if (gimple_has_location (stmt))
     599       107291 :     annotate_all_with_location (stmts, gimple_location (stmt));
     600              : 
     601              :   /* First iterate over the replacement statements backward, assigning
     602              :      virtual operands to their defining statements.  */
     603       129456 :   gimple *laststore = NULL;
     604       258912 :   for (gimple_stmt_iterator i = gsi_last (stmts);
     605       459265 :        !gsi_end_p (i); gsi_prev (&i))
     606              :     {
     607       200353 :       gimple *new_stmt = gsi_stmt (i);
     608       200353 :       if ((gimple_assign_single_p (new_stmt)
     609       126567 :            && !is_gimple_reg (gimple_assign_lhs (new_stmt)))
     610       326650 :           || (is_gimple_call (new_stmt)
     611        13897 :               && (gimple_call_flags (new_stmt)
     612        13897 :                   & (ECF_NOVOPS | ECF_PURE | ECF_CONST | ECF_NORETURN)) == 0))
     613              :         {
     614         2322 :           tree vdef;
     615         2322 :           if (!laststore)
     616         2316 :             vdef = gimple_vdef (stmt);
     617              :           else
     618            6 :             vdef = make_ssa_name (gimple_vop (cfun), new_stmt);
     619         2322 :           gimple_set_vdef (new_stmt, vdef);
     620         2322 :           if (vdef && TREE_CODE (vdef) == SSA_NAME)
     621         1394 :             SSA_NAME_DEF_STMT (vdef) = new_stmt;
     622              :           laststore = new_stmt;
     623              :         }
     624              :     }
     625              : 
     626              :   /* Second iterate over the statements forward, assigning virtual
     627              :      operands to their uses.  */
     628       129456 :   tree reaching_vuse = gimple_vuse (stmt);
     629       129456 :   for (gimple_stmt_iterator i = gsi_start (stmts);
     630       329809 :        !gsi_end_p (i); gsi_next (&i))
     631              :     {
     632       200353 :       gimple *new_stmt = gsi_stmt (i);
     633              :       /* If the new statement possibly has a VUSE, update it with exact SSA
     634              :          name we know will reach this one.  */
     635       200353 :       if (gimple_has_mem_ops (new_stmt))
     636       200351 :         gimple_set_vuse (new_stmt, reaching_vuse);
     637       200353 :       gimple_set_modified (new_stmt, true);
     638       599657 :       if (gimple_vdef (new_stmt))
     639              :         reaching_vuse = gimple_vdef (new_stmt);
     640              :     }
     641              : 
     642              :   /* If the new sequence does not do a store release the virtual
     643              :      definition of the original statement.  */
     644       129456 :   if (reaching_vuse
     645       215285 :       && reaching_vuse == gimple_vuse (stmt))
     646              :     {
     647        84441 :       tree vdef = gimple_vdef (stmt);
     648        84441 :       if (vdef
     649         1528 :           && TREE_CODE (vdef) == SSA_NAME)
     650              :         {
     651         1472 :           unlink_stmt_vdef (stmt);
     652         1472 :           release_ssa_name (vdef);
     653              :         }
     654              :     }
     655              : 
     656              :   /* Finally replace the original statement with the sequence.  */
     657       129456 :   gsi_replace_with_seq (si_p, stmts, false);
     658       129456 : }
     659              : 
     660              : /* Helper function for update_gimple_call and
     661              :    gimplify_and_update_call_from_tree.  A GIMPLE_CALL STMT is being replaced
     662              :    with GIMPLE_CALL NEW_STMT.  */
     663              : 
     664              : static void
     665         2455 : finish_update_gimple_call (gimple_stmt_iterator *si_p, gimple *new_stmt,
     666              :                            gimple *stmt)
     667              : {
     668         2455 :   tree lhs = gimple_call_lhs (stmt);
     669         2455 :   gimple_call_set_lhs (new_stmt, lhs);
     670         2455 :   if (lhs && TREE_CODE (lhs) == SSA_NAME)
     671          800 :     SSA_NAME_DEF_STMT (lhs) = new_stmt;
     672         2455 :   gimple_move_vops (new_stmt, stmt);
     673         2455 :   gimple_set_location (new_stmt, gimple_location (stmt));
     674         2455 :   if (gimple_block (new_stmt) == NULL_TREE)
     675            1 :     gimple_set_block (new_stmt, gimple_block (stmt));
     676         2455 :   gsi_replace (si_p, new_stmt, false);
     677         2455 : }
     678              : 
     679              : /* Update a GIMPLE_CALL statement at iterator *SI_P to call to FN
     680              :    with number of arguments NARGS, where the arguments in GIMPLE form
     681              :    follow NARGS argument.  */
     682              : 
     683              : bool
     684         2452 : update_gimple_call (gimple_stmt_iterator *si_p, tree fn, int nargs, ...)
     685              : {
     686         2452 :   va_list ap;
     687         2452 :   gcall *new_stmt, *stmt = as_a <gcall *> (gsi_stmt (*si_p));
     688              : 
     689         2452 :   gcc_assert (is_gimple_call (stmt));
     690         2452 :   va_start (ap, nargs);
     691         2452 :   new_stmt = gimple_build_call_valist (fn, nargs, ap);
     692         2452 :   finish_update_gimple_call (si_p, new_stmt, stmt);
     693         2452 :   va_end (ap);
     694         2452 :   return true;
     695              : }
     696              : 
     697              : /* Return true if EXPR is a CALL_EXPR suitable for representation
     698              :    as a single GIMPLE_CALL statement.  If the arguments require
     699              :    further gimplification, return false.  */
     700              : 
     701              : static bool
     702        60373 : valid_gimple_call_p (tree expr)
     703              : {
     704        60373 :   unsigned i, nargs;
     705              : 
     706        60373 :   if (TREE_CODE (expr) != CALL_EXPR)
     707              :     return false;
     708              : 
     709            3 :   nargs = call_expr_nargs (expr);
     710            6 :   for (i = 0; i < nargs; i++)
     711              :     {
     712            3 :       tree arg = CALL_EXPR_ARG (expr, i);
     713            3 :       if (is_gimple_reg_type (TREE_TYPE (arg)))
     714              :         {
     715            3 :           if (!is_gimple_val (arg))
     716              :             return false;
     717              :         }
     718              :       else
     719            0 :         if (!is_gimple_lvalue (arg))
     720              :           return false;
     721              :     }
     722              : 
     723              :   return true;
     724              : }
     725              : 
     726              : /* Convert EXPR into a GIMPLE value suitable for substitution on the
     727              :    RHS of an assignment.  Insert the necessary statements before
     728              :    iterator *SI_P.  The statement at *SI_P, which must be a GIMPLE_CALL
     729              :    is replaced.  If the call is expected to produces a result, then it
     730              :    is replaced by an assignment of the new RHS to the result variable.
     731              :    If the result is to be ignored, then the call is replaced by a
     732              :    GIMPLE_NOP.  A proper VDEF chain is retained by making the first
     733              :    VUSE and the last VDEF of the whole sequence be the same as the replaced
     734              :    statement and using new SSA names for stores in between.  */
     735              : 
     736              : void
     737        60373 : gimplify_and_update_call_from_tree (gimple_stmt_iterator *si_p, tree expr)
     738              : {
     739        60373 :   tree lhs;
     740        60373 :   gimple *stmt, *new_stmt;
     741        60373 :   gimple_stmt_iterator i;
     742        60373 :   gimple_seq stmts = NULL;
     743              : 
     744        60373 :   stmt = gsi_stmt (*si_p);
     745              : 
     746        60373 :   gcc_assert (is_gimple_call (stmt));
     747              : 
     748        60373 :   if (valid_gimple_call_p (expr))
     749              :     {
     750              :       /* The call has simplified to another call.  */
     751            3 :       tree fn = CALL_EXPR_FN (expr);
     752            3 :       unsigned i;
     753            3 :       unsigned nargs = call_expr_nargs (expr);
     754            3 :       vec<tree> args = vNULL;
     755            3 :       gcall *new_stmt;
     756              : 
     757            3 :       if (nargs > 0)
     758              :         {
     759            3 :           args.create (nargs);
     760            3 :           args.safe_grow_cleared (nargs, true);
     761              : 
     762            9 :           for (i = 0; i < nargs; i++)
     763            3 :             args[i] = CALL_EXPR_ARG (expr, i);
     764              :         }
     765              : 
     766            3 :       new_stmt = gimple_build_call_vec (fn, args);
     767            3 :       finish_update_gimple_call (si_p, new_stmt, stmt);
     768            3 :       args.release ();
     769            3 :       return;
     770              :     }
     771              : 
     772        60370 :   lhs = gimple_call_lhs (stmt);
     773        60370 :   if (lhs == NULL_TREE)
     774              :     {
     775         2516 :       push_gimplify_context (gimple_in_ssa_p (cfun));
     776         1258 :       gimplify_and_add (expr, &stmts);
     777         1258 :       pop_gimplify_context (NULL);
     778              : 
     779              :       /* We can end up with folding a memcpy of an empty class assignment
     780              :          which gets optimized away by C++ gimplification.  */
     781         1258 :       if (gimple_seq_empty_p (stmts))
     782              :         {
     783         1123 :           if (gimple_in_ssa_p (cfun))
     784              :             {
     785         1123 :               unlink_stmt_vdef (stmt);
     786         1123 :               release_defs (stmt);
     787              :             }
     788         1123 :           gsi_replace (si_p, gimple_build_nop (), false);
     789         1123 :           return;
     790              :         }
     791              :     }
     792              :   else
     793              :     {
     794        59112 :       tree tmp = force_gimple_operand (expr, &stmts, false, NULL_TREE);
     795        59112 :       new_stmt = gimple_build_assign (lhs, tmp);
     796        59112 :       i = gsi_last (stmts);
     797        59112 :       gsi_insert_after_without_update (&i, new_stmt,
     798              :                                        GSI_CONTINUE_LINKING);
     799              :     }
     800              : 
     801        59247 :   gsi_replace_with_seq_vops (si_p, stmts);
     802              : }
     803              : 
     804              : /* Print a message in the dump file recording transformation of FROM to TO.  */
     805              : 
     806              : static void
     807        40555 : dump_transformation (gcall *from, gcall *to)
     808              : {
     809        40555 :   if (dump_enabled_p ())
     810           12 :     dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, from, "simplified %T to %T\n",
     811              :                      gimple_call_fn (from), gimple_call_fn (to));
     812        40555 : }
     813              : 
     814              : /* Replace the call at *GSI with the gimple value VAL.  */
     815              : 
     816              : void
     817        87092 : replace_call_with_value (gimple_stmt_iterator *gsi, tree val)
     818              : {
     819        87092 :   gimple *stmt = gsi_stmt (*gsi);
     820        87092 :   tree lhs = gimple_call_lhs (stmt);
     821        87092 :   gimple *repl;
     822        87092 :   if (lhs)
     823              :     {
     824        81974 :       if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (val)))
     825         2210 :         val = fold_convert (TREE_TYPE (lhs), val);
     826        81974 :       repl = gimple_build_assign (lhs, val);
     827              :     }
     828              :   else
     829         5118 :     repl = gimple_build_nop ();
     830        87092 :   tree vdef = gimple_vdef (stmt);
     831        87092 :   if (vdef && TREE_CODE (vdef) == SSA_NAME)
     832              :     {
     833         5688 :       unlink_stmt_vdef (stmt);
     834         5688 :       release_ssa_name (vdef);
     835              :     }
     836        87092 :   gsi_replace (gsi, repl, false);
     837        87092 : }
     838              : 
     839              : /* Replace the call at *GSI with the new call REPL and fold that
     840              :    again.  */
     841              : 
     842              : static void
     843        40555 : replace_call_with_call_and_fold (gimple_stmt_iterator *gsi, gimple *repl)
     844              : {
     845        40555 :   gimple *stmt = gsi_stmt (*gsi);
     846        40555 :   dump_transformation (as_a <gcall *> (stmt), as_a <gcall *> (repl));
     847        40555 :   gimple_call_set_lhs (repl, gimple_call_lhs (stmt));
     848        40555 :   gimple_set_location (repl, gimple_location (stmt));
     849        40555 :   gimple_move_vops (repl, stmt);
     850        40555 :   gsi_replace (gsi, repl, false);
     851        40555 :   fold_stmt (gsi);
     852        40555 : }
     853              : 
     854              : /* Return true if VAR is a VAR_DECL or a component thereof.  */
     855              : 
     856              : static bool
     857       402716 : var_decl_component_p (tree var)
     858              : {
     859       402716 :   tree inner = var;
     860       570683 :   while (handled_component_p (inner))
     861       167967 :     inner = TREE_OPERAND (inner, 0);
     862       402716 :   return (DECL_P (inner)
     863       402716 :           || (TREE_CODE (inner) == MEM_REF
     864        43568 :               && TREE_CODE (TREE_OPERAND (inner, 0)) == ADDR_EXPR));
     865              : }
     866              : 
     867              : /* Return TRUE if the SIZE argument, representing the size of an
     868              :    object, is in a range of values of which exactly zero is valid.  */
     869              : 
     870              : static bool
     871      1030843 : size_must_be_zero_p (tree size)
     872              : {
     873      1030843 :   if (integer_zerop (size))
     874              :     return true;
     875              : 
     876      1028200 :   if (TREE_CODE (size) != SSA_NAME || !INTEGRAL_TYPE_P (TREE_TYPE (size)))
     877              :     return false;
     878              : 
     879       625177 :   tree type = TREE_TYPE (size);
     880       625177 :   int prec = TYPE_PRECISION (type);
     881              : 
     882              :   /* Compute the value of SSIZE_MAX, the largest positive value that
     883              :      can be stored in ssize_t, the signed counterpart of size_t.  */
     884       625177 :   wide_int ssize_max = wi::lshift (wi::one (prec), prec - 1) - 1;
     885       625177 :   wide_int zero = wi::zero (TYPE_PRECISION (type));
     886       625177 :   int_range_max valid_range (type, zero, ssize_max);
     887       625177 :   int_range_max vr;
     888      1250354 :   get_range_query (cfun)->range_of_expr (vr, size);
     889              : 
     890       625177 :   if (vr.undefined_p ())
     891           86 :     vr.set_varying (TREE_TYPE (size));
     892       625177 :   vr.intersect (valid_range);
     893       625177 :   return vr.zero_p ();
     894       625177 : }
     895              : 
     896              : /* Fold function call to builtin mem{{,p}cpy,move}.  Try to detect and
     897              :    diagnose (otherwise undefined) overlapping copies without preventing
     898              :    folding.  When folded, GCC guarantees that overlapping memcpy has
     899              :    the same semantics as memmove.  Call to the library memcpy need not
     900              :    provide the same guarantee.  Return false if no simplification can
     901              :    be made.  */
     902              : 
     903              : static bool
     904      1030843 : gimple_fold_builtin_memory_op (gimple_stmt_iterator *gsi,
     905              :                                tree dest, tree src, enum built_in_function code)
     906              : {
     907      1030843 :   gimple *stmt = gsi_stmt (*gsi);
     908      1030843 :   tree lhs = gimple_call_lhs (stmt);
     909      1030843 :   tree len = gimple_call_arg (stmt, 2);
     910      1030843 :   location_t loc = gimple_location (stmt);
     911              : 
     912              :   /* If the LEN parameter is a constant zero or in range where
     913              :      the only valid value is zero, return DEST.  */
     914      1030843 :   if (size_must_be_zero_p (len))
     915              :     {
     916         2695 :       gimple *repl;
     917         2695 :       if (gimple_call_lhs (stmt))
     918           60 :         repl = gimple_build_assign (gimple_call_lhs (stmt), dest);
     919              :       else
     920         2635 :         repl = gimple_build_nop ();
     921         2695 :       tree vdef = gimple_vdef (stmt);
     922         2695 :       if (vdef && TREE_CODE (vdef) == SSA_NAME)
     923              :         {
     924          487 :           unlink_stmt_vdef (stmt);
     925          487 :           release_ssa_name (vdef);
     926              :         }
     927         2695 :       gsi_replace (gsi, repl, false);
     928         2695 :       return true;
     929              :     }
     930              : 
     931              :   /* If SRC and DEST are the same (and not volatile), return
     932              :      DEST{,+LEN,+LEN-1}.  */
     933      1028148 :   if (operand_equal_p (src, dest, 0))
     934              :     {
     935              :       /* Avoid diagnosing exact overlap in calls to __builtin_memcpy.
     936              :          It's safe and may even be emitted by GCC itself (see bug
     937              :          32667).  */
     938           85 :       unlink_stmt_vdef (stmt);
     939          170 :       if (gimple_vdef (stmt) && TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
     940           48 :         release_ssa_name (gimple_vdef (stmt));
     941           85 :       if (!lhs)
     942              :         {
     943           64 :           gsi_replace (gsi, gimple_build_nop (), false);
     944           64 :           return true;
     945              :         }
     946           21 :       goto done;
     947              :     }
     948      2056126 :   else if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
     949              :     return false;
     950              :   else
     951              :     {
     952              :       /* We cannot (easily) change the type of the copy if it is a storage
     953              :          order barrier, i.e. is equivalent to a VIEW_CONVERT_EXPR that can
     954              :          modify the storage order of objects (see storage_order_barrier_p).  */
     955      1028063 :       tree srctype
     956      1041959 :         = POINTER_TYPE_P (TREE_TYPE (src))
     957      1041959 :           ? TREE_TYPE (TREE_TYPE (src)) : NULL_TREE;
     958      1028063 :       tree desttype
     959      1049040 :         = POINTER_TYPE_P (TREE_TYPE (dest))
     960      1049040 :           ? TREE_TYPE (TREE_TYPE (dest)) : NULL_TREE;
     961      1028063 :       tree destvar, srcvar, srcoff;
     962      1028063 :       unsigned int src_align, dest_align;
     963      1028063 :       unsigned HOST_WIDE_INT tmp_len;
     964      1028063 :       const char *tmp_str;
     965              : 
     966              :       /* Build accesses at offset zero with a ref-all character type.  */
     967      1028063 :       tree off0
     968      1028063 :         = build_int_cst (build_pointer_type_for_mode (char_type_node,
     969              :                                                       ptr_mode, true), 0);
     970              : 
     971              :       /* If we can perform the copy efficiently with first doing all loads
     972              :          and then all stores inline it that way.  Currently efficiently
     973              :          means that we can load all the memory into a single integer
     974              :          register which is what MOVE_MAX gives us.  */
     975      1028063 :       src_align = get_pointer_alignment (src);
     976      1028063 :       dest_align = get_pointer_alignment (dest);
     977      1028063 :       if (tree_fits_uhwi_p (len)
     978       392859 :           && compare_tree_int (len, MOVE_MAX) <= 0
     979              :           /* FIXME: Don't transform copies from strings with known length.
     980              :              Until GCC 9 this prevented a case in gcc.dg/strlenopt-8.c
     981              :              from being handled, and the case was XFAILed for that reason.
     982              :              Now that it is handled and the XFAIL removed, as soon as other
     983              :              strlenopt tests that rely on it for passing are adjusted, this
     984              :              hack can be removed.  */
     985       293995 :           && !c_strlen (src, 1)
     986       196395 :           && !((tmp_str = getbyterep (src, &tmp_len)) != NULL
     987        82736 :                && memchr (tmp_str, 0, tmp_len) == NULL)
     988       127477 :           && !(srctype
     989       127477 :                && AGGREGATE_TYPE_P (srctype)
     990        60436 :                && TYPE_REVERSE_STORAGE_ORDER (srctype))
     991      1155407 :           && !(desttype
     992       127344 :                && AGGREGATE_TYPE_P (desttype)
     993        69847 :                && TYPE_REVERSE_STORAGE_ORDER (desttype)))
     994              :         {
     995       127311 :           unsigned ilen = tree_to_uhwi (len);
     996       127311 :           if (pow2p_hwi (ilen))
     997              :             {
     998              :               /* Detect out-of-bounds accesses without issuing warnings.
     999              :                  Avoid folding out-of-bounds copies but to avoid false
    1000              :                  positives for unreachable code defer warning until after
    1001              :                  DCE has worked its magic.
    1002              :                  -Wrestrict is still diagnosed.  */
    1003        25794 :               if (int warning = check_bounds_or_overlap (as_a <gcall *>(stmt),
    1004              :                                                          dest, src, len, len,
    1005        25794 :                                                          false, false))
    1006          981 :                 if (warning != OPT_Wrestrict)
    1007      1024516 :                   return false;
    1008              : 
    1009        24872 :               scalar_int_mode imode;
    1010        24872 :               machine_mode mode;
    1011        24872 :               if (int_mode_for_size (ilen * BITS_PER_UNIT, 0).exists (&imode)
    1012        49744 :                   && bitwise_mode_for_size (ilen
    1013        24872 :                                             * BITS_PER_UNIT).exists (&mode)
    1014        49744 :                   && known_eq (GET_MODE_BITSIZE (mode), ilen * BITS_PER_UNIT)
    1015              :                   /* If the destination pointer is not aligned we must be able
    1016              :                      to emit an unaligned store.  */
    1017        24872 :                   && (dest_align >= GET_MODE_ALIGNMENT (mode)
    1018        13420 :                       || !targetm.slow_unaligned_access (mode, dest_align)
    1019            0 :                       || (optab_handler (movmisalign_optab, mode)
    1020              :                           != CODE_FOR_nothing)))
    1021              :                 {
    1022        24872 :                   tree type = bitwise_type_for_mode (mode);
    1023        24872 :                   tree srctype = type;
    1024        24872 :                   tree desttype = type;
    1025        24872 :                   if (src_align < GET_MODE_ALIGNMENT (mode))
    1026        12869 :                     srctype = build_aligned_type (type, src_align);
    1027        24872 :                   tree srcmem = fold_build2 (MEM_REF, srctype, src, off0);
    1028        24872 :                   tree tem = fold_const_aggregate_ref (srcmem);
    1029        24872 :                   if (tem)
    1030              :                     srcmem = tem;
    1031        23771 :                   else if (src_align < GET_MODE_ALIGNMENT (mode)
    1032        12606 :                            && targetm.slow_unaligned_access (mode, src_align)
    1033        23771 :                            && (optab_handler (movmisalign_optab, mode)
    1034              :                                == CODE_FOR_nothing))
    1035              :                     srcmem = NULL_TREE;
    1036        23771 :                   if (srcmem)
    1037              :                     {
    1038        24872 :                       gimple *new_stmt;
    1039        24872 :                       if (is_gimple_reg_type (TREE_TYPE (srcmem)))
    1040              :                         {
    1041        24872 :                           new_stmt = gimple_build_assign (NULL_TREE, srcmem);
    1042        24872 :                           srcmem
    1043        24872 :                             = make_ssa_name (TREE_TYPE (srcmem), new_stmt);
    1044        24872 :                           gimple_assign_set_lhs (new_stmt, srcmem);
    1045        49744 :                           gimple_set_vuse (new_stmt, gimple_vuse (stmt));
    1046        24872 :                           gimple_set_location (new_stmt, loc);
    1047        24872 :                           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    1048              :                         }
    1049        24872 :                       if (dest_align < GET_MODE_ALIGNMENT (mode))
    1050        13420 :                         desttype = build_aligned_type (type, dest_align);
    1051        24872 :                       new_stmt
    1052        24872 :                         = gimple_build_assign (fold_build2 (MEM_REF, desttype,
    1053              :                                                             dest, off0),
    1054              :                                                srcmem);
    1055        24872 :                       gimple_move_vops (new_stmt, stmt);
    1056        24872 :                       if (!lhs)
    1057              :                         {
    1058        21509 :                           gsi_replace (gsi, new_stmt, false);
    1059        21509 :                           return true;
    1060              :                         }
    1061         3363 :                       gimple_set_location (new_stmt, loc);
    1062         3363 :                       gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    1063         3363 :                       goto done;
    1064              :                     }
    1065              :                 }
    1066              :             }
    1067              :         }
    1068              : 
    1069      1002269 :       if (code == BUILT_IN_MEMMOVE)
    1070              :         {
    1071              :           /* Both DEST and SRC must be pointer types.
    1072              :              ??? This is what old code did.  Is the testing for pointer types
    1073              :              really mandatory?
    1074              : 
    1075              :              If either SRC is readonly or length is 1, we can use memcpy.  */
    1076       208961 :           if (!dest_align || !src_align)
    1077              :             return false;
    1078       208961 :           if (readonly_data_expr (src)
    1079       208961 :               || (tree_fits_uhwi_p (len)
    1080        33403 :                   && (MIN (src_align, dest_align) / BITS_PER_UNIT
    1081        33403 :                       >= tree_to_uhwi (len))))
    1082              :             {
    1083      1045981 :               tree fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    1084        21465 :               if (!fn)
    1085              :                 return false;
    1086        21465 :               gimple_call_set_fndecl (stmt, fn);
    1087        21465 :               gimple_call_set_arg (stmt, 0, dest);
    1088        21465 :               gimple_call_set_arg (stmt, 1, src);
    1089        21465 :               fold_stmt (gsi);
    1090        21465 :               return true;
    1091              :             }
    1092              : 
    1093              :           /* If *src and *dest can't overlap, optimize into memcpy as well.  */
    1094       187496 :           if (TREE_CODE (src) == ADDR_EXPR
    1095         5537 :               && TREE_CODE (dest) == ADDR_EXPR)
    1096              :             {
    1097         1848 :               tree src_base, dest_base, fn;
    1098         1848 :               poly_int64 src_offset = 0, dest_offset = 0;
    1099         1848 :               poly_uint64 maxsize;
    1100              : 
    1101         1848 :               srcvar = TREE_OPERAND (src, 0);
    1102         1848 :               src_base = get_addr_base_and_unit_offset (srcvar, &src_offset);
    1103         1848 :               if (src_base == NULL)
    1104            0 :                 src_base = srcvar;
    1105         1848 :               destvar = TREE_OPERAND (dest, 0);
    1106         1848 :               dest_base = get_addr_base_and_unit_offset (destvar,
    1107              :                                                          &dest_offset);
    1108         1848 :               if (dest_base == NULL)
    1109            0 :                 dest_base = destvar;
    1110         1848 :               if (!poly_int_tree_p (len, &maxsize))
    1111          239 :                 maxsize = -1;
    1112         1848 :               if (SSA_VAR_P (src_base)
    1113         1848 :                   && SSA_VAR_P (dest_base))
    1114              :                 {
    1115         1848 :                   if (operand_equal_p (src_base, dest_base, 0)
    1116         1848 :                       && ranges_maybe_overlap_p (src_offset, maxsize,
    1117              :                                                  dest_offset, maxsize))
    1118              :                     return false;
    1119              :                 }
    1120            0 :               else if (TREE_CODE (src_base) == MEM_REF
    1121            0 :                        && TREE_CODE (dest_base) == MEM_REF)
    1122              :                 {
    1123            0 :                   if (! operand_equal_p (TREE_OPERAND (src_base, 0),
    1124            0 :                                          TREE_OPERAND (dest_base, 0), 0))
    1125            0 :                     return false;
    1126            0 :                   poly_offset_int full_src_offset
    1127            0 :                     = mem_ref_offset (src_base) + src_offset;
    1128            0 :                   poly_offset_int full_dest_offset
    1129            0 :                     = mem_ref_offset (dest_base) + dest_offset;
    1130            0 :                   if (ranges_maybe_overlap_p (full_src_offset, maxsize,
    1131              :                                               full_dest_offset, maxsize))
    1132              :                     return false;
    1133            0 :                 }
    1134              :               else
    1135              :                 return false;
    1136              : 
    1137         3261 :               fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    1138         1413 :               if (!fn)
    1139              :                 return false;
    1140         1413 :               gimple_call_set_fndecl (stmt, fn);
    1141         1413 :               gimple_call_set_arg (stmt, 0, dest);
    1142         1413 :               gimple_call_set_arg (stmt, 1, src);
    1143         1413 :               fold_stmt (gsi);
    1144         1413 :               return true;
    1145              :             }
    1146              : 
    1147              :           /* If the destination and source do not alias optimize into
    1148              :              memcpy as well.  */
    1149       185648 :           if ((is_gimple_min_invariant (dest)
    1150       181834 :                || TREE_CODE (dest) == SSA_NAME)
    1151       348414 :               && (is_gimple_min_invariant (src)
    1152       162918 :                   || TREE_CODE (src) == SSA_NAME))
    1153              :             {
    1154       166008 :               ao_ref destr, srcr;
    1155       166008 :               ao_ref_init_from_ptr_and_size (&destr, dest, len);
    1156       166008 :               ao_ref_init_from_ptr_and_size (&srcr, src, len);
    1157       166008 :               if (!refs_may_alias_p_1 (&destr, &srcr, false))
    1158              :                 {
    1159        10399 :                   tree fn;
    1160        10399 :                   fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    1161        10399 :                   if (!fn)
    1162        10399 :                     return false;
    1163        10399 :                   gimple_call_set_fndecl (stmt, fn);
    1164        10399 :                   gimple_call_set_arg (stmt, 0, dest);
    1165        10399 :                   gimple_call_set_arg (stmt, 1, src);
    1166        10399 :                   fold_stmt (gsi);
    1167        10399 :                   return true;
    1168              :                 }
    1169              :             }
    1170              : 
    1171              :           return false;
    1172              :         }
    1173              : 
    1174       793308 :       if (!tree_fits_shwi_p (len))
    1175              :         return false;
    1176       316191 :       if (!srctype
    1177       316191 :           || (AGGREGATE_TYPE_P (srctype)
    1178       192941 :               && TYPE_REVERSE_STORAGE_ORDER (srctype)))
    1179              :         return false;
    1180       316058 :       if (!desttype
    1181       316058 :           || (AGGREGATE_TYPE_P (desttype)
    1182       184023 :               && TYPE_REVERSE_STORAGE_ORDER (desttype)))
    1183              :         return false;
    1184              :       /* In the following try to find a type that is most natural to be
    1185              :          used for the memcpy source and destination and that allows
    1186              :          the most optimization when memcpy is turned into a plain assignment
    1187              :          using that type.  In theory we could always use a char[len] type
    1188              :          but that only gains us that the destination and source possibly
    1189              :          no longer will have their address taken.  */
    1190       316025 :       if (TREE_CODE (srctype) == ARRAY_TYPE
    1191       316025 :           && !tree_int_cst_equal (TYPE_SIZE_UNIT (srctype), len))
    1192       118316 :         srctype = TREE_TYPE (srctype);
    1193       316025 :       if (TREE_CODE (desttype) == ARRAY_TYPE
    1194       316025 :           && !tree_int_cst_equal (TYPE_SIZE_UNIT (desttype), len))
    1195        96075 :         desttype = TREE_TYPE (desttype);
    1196       316025 :       if (TREE_ADDRESSABLE (srctype)
    1197       315936 :           || TREE_ADDRESSABLE (desttype))
    1198              :         return false;
    1199              : 
    1200              :       /* Make sure we are not copying using a floating-point mode or
    1201              :          a type whose size possibly does not match its precision.  */
    1202       631173 :       if (FLOAT_MODE_P (TYPE_MODE (desttype))
    1203       315006 :           || TREE_CODE (desttype) == BOOLEAN_TYPE
    1204       630888 :           || TREE_CODE (desttype) == ENUMERAL_TYPE)
    1205          940 :         desttype = bitwise_type_for_mode (TYPE_MODE (desttype));
    1206       631245 :       if (FLOAT_MODE_P (TYPE_MODE (srctype))
    1207       315229 :           || TREE_CODE (srctype) == BOOLEAN_TYPE
    1208       631116 :           || TREE_CODE (srctype) == ENUMERAL_TYPE)
    1209          712 :         srctype = bitwise_type_for_mode (TYPE_MODE (srctype));
    1210       315904 :       if (!srctype)
    1211          120 :         srctype = desttype;
    1212       315904 :       if (!desttype)
    1213            0 :         desttype = srctype;
    1214       315904 :       if (!srctype)
    1215              :         return false;
    1216              : 
    1217       315904 :       src_align = get_pointer_alignment (src);
    1218       315904 :       dest_align = get_pointer_alignment (dest);
    1219              : 
    1220              :       /* Choose between src and destination type for the access based
    1221              :          on alignment, whether the access constitutes a register access
    1222              :          and whether it may actually expose a declaration for SSA rewrite
    1223              :          or SRA decomposition.  Also try to expose a string constant, we
    1224              :          might be able to concatenate several of them later into a single
    1225              :          string store.  */
    1226       315904 :       destvar = NULL_TREE;
    1227       315904 :       srcvar = NULL_TREE;
    1228       315904 :       if (TREE_CODE (dest) == ADDR_EXPR
    1229       107455 :           && var_decl_component_p (TREE_OPERAND (dest, 0))
    1230       107451 :           && tree_int_cst_equal (TYPE_SIZE_UNIT (desttype), len)
    1231        25224 :           && dest_align >= TYPE_ALIGN (desttype)
    1232       341128 :           && (is_gimple_reg_type (desttype)
    1233        24796 :               || src_align >= TYPE_ALIGN (desttype)))
    1234        20020 :         destvar = fold_build2 (MEM_REF, desttype, dest, off0);
    1235       295884 :       else if (TREE_CODE (src) == ADDR_EXPR
    1236       230562 :                && var_decl_component_p (TREE_OPERAND (src, 0))
    1237        47365 :                && tree_int_cst_equal (TYPE_SIZE_UNIT (srctype), len)
    1238         9070 :                && src_align >= TYPE_ALIGN (srctype)
    1239       304938 :                && (is_gimple_reg_type (srctype)
    1240         8891 :                    || dest_align >= TYPE_ALIGN (srctype)))
    1241         3144 :         srcvar = fold_build2 (MEM_REF, srctype, src, off0);
    1242              :       /* FIXME: Don't transform copies from strings with known original length.
    1243              :          As soon as strlenopt tests that rely on it for passing are adjusted,
    1244              :          this hack can be removed.  */
    1245       292740 :       else if (gimple_call_alloca_for_var_p (stmt)
    1246          115 :                && (srcvar = string_constant (src, &srcoff, NULL, NULL))
    1247            3 :                && integer_zerop (srcoff)
    1248            3 :                && tree_int_cst_equal (TYPE_SIZE_UNIT (TREE_TYPE (srcvar)), len)
    1249       292743 :                && dest_align >= TYPE_ALIGN (TREE_TYPE (srcvar)))
    1250            3 :         srctype = TREE_TYPE (srcvar);
    1251              :       else
    1252              :         return false;
    1253              : 
    1254              :       /* Now that we chose an access type express the other side in
    1255              :          terms of it if the target allows that with respect to alignment
    1256              :          constraints.  */
    1257        23167 :       if (srcvar == NULL_TREE)
    1258              :         {
    1259        20020 :           if (src_align >= TYPE_ALIGN (desttype))
    1260        20002 :             srcvar = fold_build2 (MEM_REF, desttype, src, off0);
    1261              :           else
    1262              :             {
    1263           18 :               enum machine_mode mode = TYPE_MODE (desttype);
    1264           18 :               if ((mode == BLKmode && STRICT_ALIGNMENT)
    1265           18 :                   || (targetm.slow_unaligned_access (mode, src_align)
    1266           18 :                       && (optab_handler (movmisalign_optab, mode)
    1267              :                           == CODE_FOR_nothing)))
    1268              :                 return false;
    1269           18 :               srctype = build_aligned_type (TYPE_MAIN_VARIANT (desttype),
    1270              :                                             src_align);
    1271           18 :               srcvar = fold_build2 (MEM_REF, srctype, src, off0);
    1272              :             }
    1273              :         }
    1274         3147 :       else if (destvar == NULL_TREE)
    1275              :         {
    1276         3147 :           if (dest_align >= TYPE_ALIGN (srctype))
    1277         3147 :             destvar = fold_build2 (MEM_REF, srctype, dest, off0);
    1278              :           else
    1279              :             {
    1280            0 :               enum machine_mode mode = TYPE_MODE (srctype);
    1281            0 :               if ((mode == BLKmode && STRICT_ALIGNMENT)
    1282            0 :                   || (targetm.slow_unaligned_access (mode, dest_align)
    1283            0 :                       && (optab_handler (movmisalign_optab, mode)
    1284              :                           == CODE_FOR_nothing)))
    1285              :                 return false;
    1286            0 :               desttype = build_aligned_type (TYPE_MAIN_VARIANT (srctype),
    1287              :                                              dest_align);
    1288            0 :               destvar = fold_build2 (MEM_REF, desttype, dest, off0);
    1289              :             }
    1290              :         }
    1291              : 
    1292              :       /* Same as above, detect out-of-bounds accesses without issuing
    1293              :          warnings.  Avoid folding out-of-bounds copies but to avoid
    1294              :          false positives for unreachable code defer warning until
    1295              :          after DCE has worked its magic.
    1296              :          -Wrestrict is still diagnosed.  */
    1297        23167 :       if (int warning = check_bounds_or_overlap (as_a <gcall *>(stmt),
    1298              :                                                  dest, src, len, len,
    1299        23167 :                                                  false, false))
    1300         1263 :         if (warning != OPT_Wrestrict)
    1301              :           return false;
    1302              : 
    1303        21912 :       gimple *new_stmt;
    1304        21912 :       if (is_gimple_reg_type (TREE_TYPE (srcvar)))
    1305              :         {
    1306          551 :           tree tem = fold_const_aggregate_ref (srcvar);
    1307          551 :           if (tem)
    1308          530 :             srcvar = tem;
    1309          551 :           if (! is_gimple_min_invariant (srcvar))
    1310              :             {
    1311           21 :               new_stmt = gimple_build_assign (NULL_TREE, srcvar);
    1312           21 :               srcvar = make_ssa_name (TREE_TYPE (srcvar), new_stmt);
    1313           21 :               gimple_assign_set_lhs (new_stmt, srcvar);
    1314           42 :               gimple_set_vuse (new_stmt, gimple_vuse (stmt));
    1315           21 :               gimple_set_location (new_stmt, loc);
    1316           21 :               gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    1317              :             }
    1318          551 :           new_stmt = gimple_build_assign (destvar, srcvar);
    1319          551 :           goto set_vop_and_replace;
    1320              :         }
    1321              : 
    1322              :       /* We get an aggregate copy.  If the source is a STRING_CST, then
    1323              :          directly use its type to perform the copy.  */
    1324        21361 :       if (TREE_CODE (srcvar) == STRING_CST)
    1325              :           desttype = srctype;
    1326              : 
    1327              :       /* Or else, use an unsigned char[] type to perform the copy in order
    1328              :          to preserve padding and to avoid any issues with TREE_ADDRESSABLE
    1329              :          types or float modes behavior on copying.  */
    1330              :       else
    1331              :         {
    1332        42716 :           desttype = build_array_type_nelts (unsigned_char_type_node,
    1333        21358 :                                              tree_to_uhwi (len));
    1334        21358 :           srctype = desttype;
    1335        21358 :           if (src_align > TYPE_ALIGN (srctype))
    1336        12841 :             srctype = build_aligned_type (srctype, src_align);
    1337        21358 :           srcvar = fold_build2 (MEM_REF, srctype, src, off0);
    1338              :         }
    1339              : 
    1340        21361 :       if (dest_align > TYPE_ALIGN (desttype))
    1341        13443 :         desttype = build_aligned_type (desttype, dest_align);
    1342        21361 :       destvar = fold_build2 (MEM_REF, desttype, dest, off0);
    1343        21361 :       new_stmt = gimple_build_assign (destvar, srcvar);
    1344              : 
    1345        21912 : set_vop_and_replace:
    1346        21912 :       gimple_move_vops (new_stmt, stmt);
    1347        21912 :       if (!lhs)
    1348              :         {
    1349        21728 :           gsi_replace (gsi, new_stmt, false);
    1350        21728 :           return true;
    1351              :         }
    1352          184 :       gimple_set_location (new_stmt, loc);
    1353          184 :       gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    1354              :     }
    1355              : 
    1356         3568 : done:
    1357         3568 :   gimple_seq stmts = NULL;
    1358         3568 :   if (code == BUILT_IN_MEMCPY || code == BUILT_IN_MEMMOVE)
    1359         3568 :     len = NULL_TREE;
    1360          200 :   else if (code == BUILT_IN_MEMPCPY)
    1361              :     {
    1362          200 :       len = gimple_convert_to_ptrofftype (&stmts, loc, len);
    1363          200 :       dest = gimple_build (&stmts, loc, POINTER_PLUS_EXPR,
    1364          200 :                            TREE_TYPE (dest), dest, len);
    1365              :     }
    1366              :   else
    1367            0 :     gcc_unreachable ();
    1368              : 
    1369         3568 :   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    1370         3568 :   gimple *repl = gimple_build_assign (lhs, dest);
    1371         3568 :   gsi_replace (gsi, repl, false);
    1372         3568 :   return true;
    1373              : }
    1374              : 
    1375              : /* Transform a call to built-in bcmp(a, b, len) at *GSI into one
    1376              :    to built-in memcmp (a, b, len).  */
    1377              : 
    1378              : static bool
    1379          149 : gimple_fold_builtin_bcmp (gimple_stmt_iterator *gsi)
    1380              : {
    1381          149 :   tree fn = builtin_decl_implicit (BUILT_IN_MEMCMP);
    1382              : 
    1383          149 :   if (!fn)
    1384              :     return false;
    1385              : 
    1386              :   /* Transform bcmp (a, b, len) into memcmp (a, b, len).  */
    1387              : 
    1388          149 :   gimple *stmt = gsi_stmt (*gsi);
    1389          298 :   if (!gimple_vuse (stmt) && gimple_in_ssa_p (cfun))
    1390              :     return false;
    1391          149 :   tree a = gimple_call_arg (stmt, 0);
    1392          149 :   tree b = gimple_call_arg (stmt, 1);
    1393          149 :   tree len = gimple_call_arg (stmt, 2);
    1394              : 
    1395          149 :   gimple *repl = gimple_build_call (fn, 3, a, b, len);
    1396          149 :   replace_call_with_call_and_fold (gsi, repl);
    1397              : 
    1398          149 :   return true;
    1399              : }
    1400              : 
    1401              : /* Transform a call to built-in bcopy (src, dest, len) at *GSI into one
    1402              :    to built-in memmove (dest, src, len).  */
    1403              : 
    1404              : static bool
    1405          367 : gimple_fold_builtin_bcopy (gimple_stmt_iterator *gsi)
    1406              : {
    1407          367 :   tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
    1408              : 
    1409          367 :   if (!fn)
    1410              :     return false;
    1411              : 
    1412              :   /* bcopy has been removed from POSIX in Issue 7 but Issue 6 specifies
    1413              :      it's equivalent to memmove (not memcpy).  Transform bcopy (src, dest,
    1414              :      len) into memmove (dest, src, len).  */
    1415              : 
    1416          367 :   gimple *stmt = gsi_stmt (*gsi);
    1417          734 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    1418              :     return false;
    1419          367 :   tree src = gimple_call_arg (stmt, 0);
    1420          367 :   tree dest = gimple_call_arg (stmt, 1);
    1421          367 :   tree len = gimple_call_arg (stmt, 2);
    1422              : 
    1423          367 :   gimple *repl = gimple_build_call (fn, 3, dest, src, len);
    1424          367 :   gimple_call_set_fntype (as_a <gcall *> (stmt), TREE_TYPE (fn));
    1425          367 :   replace_call_with_call_and_fold (gsi, repl);
    1426              : 
    1427          367 :   return true;
    1428              : }
    1429              : 
    1430              : /* Transform a call to built-in bzero (dest, len) at *GSI into one
    1431              :    to built-in memset (dest, 0, len).  */
    1432              : 
    1433              : static bool
    1434          250 : gimple_fold_builtin_bzero (gimple_stmt_iterator *gsi)
    1435              : {
    1436          250 :   tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
    1437              : 
    1438          250 :   if (!fn)
    1439              :     return false;
    1440              : 
    1441              :   /* Transform bzero (dest, len) into memset (dest, 0, len).  */
    1442              : 
    1443          250 :   gimple *stmt = gsi_stmt (*gsi);
    1444          500 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    1445              :     return false;
    1446          250 :   tree dest = gimple_call_arg (stmt, 0);
    1447          250 :   tree len = gimple_call_arg (stmt, 1);
    1448              : 
    1449          250 :   gimple_seq seq = NULL;
    1450          250 :   gimple *repl = gimple_build_call (fn, 3, dest, integer_zero_node, len);
    1451          250 :   gimple_seq_add_stmt_without_update (&seq, repl);
    1452          250 :   gsi_replace_with_seq_vops (gsi, seq);
    1453          250 :   fold_stmt (gsi);
    1454              : 
    1455          250 :   return true;
    1456              : }
    1457              : 
    1458              : /* Fold function call to builtin memset or bzero at *GSI setting the
    1459              :    memory of size LEN to VAL.  Return whether a simplification was made.  */
    1460              : 
    1461              : static bool
    1462       318809 : gimple_fold_builtin_memset (gimple_stmt_iterator *gsi, tree c, tree len)
    1463              : {
    1464       318809 :   gimple *stmt = gsi_stmt (*gsi);
    1465       318809 :   tree etype;
    1466       318809 :   unsigned HOST_WIDE_INT length, cval;
    1467              : 
    1468              :   /* If the LEN parameter is zero, return DEST.  */
    1469       318809 :   if (integer_zerop (len))
    1470              :     {
    1471          841 :       replace_call_with_value (gsi, gimple_call_arg (stmt, 0));
    1472          841 :       return true;
    1473              :     }
    1474              : 
    1475       635936 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    1476              :     return false;
    1477              : 
    1478       317890 :   if (! tree_fits_uhwi_p (len))
    1479              :     return false;
    1480              : 
    1481       209591 :   length = tree_to_uhwi (len);
    1482              : 
    1483       209591 :   tree dest = gimple_call_arg (stmt, 0);
    1484       209591 :   if (length == 1
    1485       209591 :       && POINTER_TYPE_P (TREE_TYPE (dest)))
    1486              :     {
    1487              :       /* Keep the original call until object-size analysis has inspected it.  */
    1488         3513 :       if (!(cfun->curr_properties & PROP_objsz))
    1489              :         return false;
    1490              : 
    1491              :       /* Detect out-of-bounds accesses without issuing warnings.
    1492              :          Avoid folding out-of-bounds accesses but to avoid false
    1493              :          positives for unreachable code defer warning until after
    1494              :          DCE has worked its magic.
    1495              :          -Wrestrict is still diagnosed.  */
    1496          600 :       if (int warning = check_bounds_or_overlap (as_a <gcall *>(stmt),
    1497              :                                                  dest, NULL_TREE, len,
    1498          600 :                                                  NULL_TREE, false, false))
    1499           20 :         if (warning != OPT_Wrestrict)
    1500              :           return false;
    1501              : 
    1502          580 :       etype = unsigned_char_type_node;
    1503          580 :       tree ptype = TREE_TYPE (TREE_TYPE (dest));
    1504          580 :       if (TYPE_VOLATILE (ptype))
    1505            0 :         etype = build_qualified_type (etype, TYPE_QUAL_VOLATILE);
    1506              : 
    1507          580 :       location_t loc = gimple_location (stmt);
    1508          580 :       tree cval_tree;
    1509          580 :       if (TREE_CODE (c) == INTEGER_CST)
    1510          535 :         cval_tree = fold_convert (etype, c);
    1511              :       else
    1512           45 :         cval_tree = gimple_convert (gsi, true, GSI_SAME_STMT, loc, etype, c);
    1513              : 
    1514              :       /* Build accesses at offset zero with a ref-all character type.  */
    1515          580 :       tree off0
    1516          580 :         = build_int_cst (build_pointer_type_for_mode (char_type_node,
    1517              :                                                       ptr_mode, true), 0);
    1518          580 :       tree var = fold_build2_loc (loc, MEM_REF, etype, dest, off0);
    1519          580 :       gimple *store = gimple_build_assign (var, cval_tree);
    1520          580 :       gimple_move_vops (store, stmt);
    1521          580 :       gimple_set_location (store, loc);
    1522          580 :       copy_warning (store, stmt);
    1523              : 
    1524          580 :       tree lhs = gimple_call_lhs (stmt);
    1525          580 :       if (!lhs)
    1526              :         {
    1527          471 :           gsi_replace (gsi, store, false);
    1528          471 :           return true;
    1529              :         }
    1530              : 
    1531          109 :       gsi_insert_before (gsi, store, GSI_SAME_STMT);
    1532          109 :       tree ret = dest;
    1533          109 :       if (!useless_type_conversion_p (TREE_TYPE (lhs), TREE_TYPE (dest)))
    1534            0 :         ret = gimple_convert (gsi, true, GSI_SAME_STMT, loc,
    1535            0 :                               TREE_TYPE (lhs), dest);
    1536          109 :       gimple *asgn = gimple_build_assign (lhs, ret);
    1537          109 :       gsi_replace (gsi, asgn, false);
    1538              : 
    1539          109 :       return true;
    1540              :     }
    1541              : 
    1542       206078 :   if (TREE_CODE (c) != INTEGER_CST)
    1543              :     return false;
    1544              : 
    1545       200392 :   tree var = dest;
    1546       200392 :   if (TREE_CODE (var) != ADDR_EXPR)
    1547              :     return false;
    1548              : 
    1549       162067 :   var = TREE_OPERAND (var, 0);
    1550       162067 :   if (TREE_THIS_VOLATILE (var))
    1551              :     return false;
    1552              : 
    1553       162024 :   etype = TREE_TYPE (var);
    1554       162024 :   if (TREE_CODE (etype) == ARRAY_TYPE)
    1555        85909 :     etype = TREE_TYPE (etype);
    1556              : 
    1557       162024 :   if ((!INTEGRAL_TYPE_P (etype)
    1558        96429 :        && !POINTER_TYPE_P (etype))
    1559       162456 :       || BITINT_TYPE_P (etype))
    1560              :     return false;
    1561              : 
    1562        64699 :   if (! var_decl_component_p (var))
    1563              :     return false;
    1564              : 
    1565        64699 :   if (GET_MODE_SIZE (SCALAR_INT_TYPE_MODE (etype)) != length
    1566         1414 :       || (GET_MODE_PRECISION (SCALAR_INT_TYPE_MODE (etype))
    1567        66113 :           != GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (etype)))
    1568        66113 :       || get_pointer_alignment (dest) / BITS_PER_UNIT < length)
    1569              :     return false;
    1570              : 
    1571         1414 :   if (length > HOST_BITS_PER_WIDE_INT / BITS_PER_UNIT)
    1572              :     return false;
    1573              : 
    1574         1414 :   if (!type_has_mode_precision_p (etype))
    1575            7 :     etype = lang_hooks.types.type_for_mode (SCALAR_INT_TYPE_MODE (etype),
    1576            7 :                                             TYPE_UNSIGNED (etype));
    1577              : 
    1578         1414 :   if (integer_zerop (c))
    1579              :     cval = 0;
    1580              :   else
    1581              :     {
    1582           66 :       if (CHAR_BIT != 8 || BITS_PER_UNIT != 8 || HOST_BITS_PER_WIDE_INT > 64)
    1583              :         return NULL_TREE;
    1584              : 
    1585           66 :       cval = TREE_INT_CST_LOW (c);
    1586           66 :       cval &= 0xff;
    1587           66 :       cval |= cval << 8;
    1588           66 :       cval |= cval << 16;
    1589           66 :       cval |= (cval << 31) << 1;
    1590              :     }
    1591              : 
    1592         1414 :   var = fold_build2 (MEM_REF, etype, dest, build_int_cst (ptr_type_node, 0));
    1593         1414 :   gimple *store = gimple_build_assign (var, build_int_cst_type (etype, cval));
    1594         1414 :   gimple_move_vops (store, stmt);
    1595         1414 :   gimple_set_location (store, gimple_location (stmt));
    1596         1414 :   gsi_insert_before (gsi, store, GSI_SAME_STMT);
    1597         1414 :   if (gimple_call_lhs (stmt))
    1598              :     {
    1599            0 :       gimple *asgn = gimple_build_assign (gimple_call_lhs (stmt), dest);
    1600            0 :       gsi_replace (gsi, asgn, false);
    1601              :     }
    1602              :   else
    1603              :     {
    1604         1414 :       gimple_stmt_iterator gsi2 = *gsi;
    1605         1414 :       gsi_prev (gsi);
    1606         1414 :       gsi_remove (&gsi2, true);
    1607              :     }
    1608              : 
    1609              :   return true;
    1610              : }
    1611              : 
    1612              : /* Helper of get_range_strlen for ARG that is not an SSA_NAME.  */
    1613              : 
    1614              : static bool
    1615       448573 : get_range_strlen_tree (tree arg, bitmap visited, strlen_range_kind rkind,
    1616              :                        c_strlen_data *pdata, unsigned eltsize)
    1617              : {
    1618       448573 :   gcc_assert (TREE_CODE (arg) != SSA_NAME);
    1619              : 
    1620              :   /* The length computed by this invocation of the function.  */
    1621       448573 :   tree val = NULL_TREE;
    1622              : 
    1623              :   /* True if VAL is an optimistic (tight) bound determined from
    1624              :      the size of the character array in which the string may be
    1625              :      stored.  In that case, the computed VAL is used to set
    1626              :      PDATA->MAXBOUND.  */
    1627       448573 :   bool tight_bound = false;
    1628              : 
    1629              :   /* We can end up with &(*iftmp_1)[0] here as well, so handle it.  */
    1630       448573 :   if (TREE_CODE (arg) == ADDR_EXPR
    1631       448573 :       && TREE_CODE (TREE_OPERAND (arg, 0)) == ARRAY_REF)
    1632              :     {
    1633        28812 :       tree op = TREE_OPERAND (arg, 0);
    1634        28812 :       if (integer_zerop (TREE_OPERAND (op, 1)))
    1635              :         {
    1636        12562 :           tree aop0 = TREE_OPERAND (op, 0);
    1637        12562 :           if (TREE_CODE (aop0) == INDIRECT_REF
    1638        12562 :               && TREE_CODE (TREE_OPERAND (aop0, 0)) == SSA_NAME)
    1639            0 :             return get_range_strlen (TREE_OPERAND (aop0, 0), visited, rkind,
    1640            0 :                                      pdata, eltsize);
    1641              :         }
    1642        16250 :       else if (TREE_CODE (TREE_OPERAND (op, 0)) == COMPONENT_REF
    1643        16250 :                && rkind == SRK_LENRANGE)
    1644              :         {
    1645              :           /* Fail if an array is the last member of a struct object
    1646              :              since it could be treated as a (fake) flexible array
    1647              :              member.  */
    1648         4800 :           tree idx = TREE_OPERAND (op, 1);
    1649              : 
    1650         4800 :           arg = TREE_OPERAND (op, 0);
    1651         4800 :           tree optype = TREE_TYPE (arg);
    1652         4800 :           if (tree dom = TYPE_DOMAIN (optype))
    1653         4800 :             if (tree bound = TYPE_MAX_VALUE (dom))
    1654         4800 :               if (TREE_CODE (bound) == INTEGER_CST
    1655         4800 :                   && TREE_CODE (idx) == INTEGER_CST
    1656         8030 :                   && tree_int_cst_lt (bound, idx))
    1657              :                 return false;
    1658              :         }
    1659              :     }
    1660              : 
    1661       448365 :   if (rkind == SRK_INT_VALUE)
    1662              :     {
    1663              :       /* We are computing the maximum value (not string length).  */
    1664         3083 :       val = arg;
    1665         3083 :       if (TREE_CODE (val) != INTEGER_CST
    1666         3083 :           || tree_int_cst_sgn (val) < 0)
    1667              :         return false;
    1668              :     }
    1669              :   else
    1670              :     {
    1671       445282 :       c_strlen_data lendata = { };
    1672       445282 :       val = c_strlen (arg, 1, &lendata, eltsize);
    1673              : 
    1674       445282 :       if (!val && lendata.decl)
    1675              :         {
    1676              :           /* ARG refers to an unterminated const character array.
    1677              :              DATA.DECL with size DATA.LEN.  */
    1678         4226 :           val = lendata.minlen;
    1679         4226 :           pdata->decl = lendata.decl;
    1680              :         }
    1681              :     }
    1682              : 
    1683              :   /* Set if VAL represents the maximum length based on array size (set
    1684              :      when exact length cannot be determined).  */
    1685       445792 :   bool maxbound = false;
    1686              : 
    1687       445792 :   if (!val && rkind == SRK_LENRANGE)
    1688              :     {
    1689       244160 :       if (TREE_CODE (arg) == ADDR_EXPR)
    1690        87504 :         return get_range_strlen (TREE_OPERAND (arg, 0), visited, rkind,
    1691        87504 :                                  pdata, eltsize);
    1692              : 
    1693       156656 :       if (TREE_CODE (arg) == ARRAY_REF)
    1694              :         {
    1695        18041 :           tree optype = TREE_TYPE (TREE_OPERAND (arg, 0));
    1696              : 
    1697              :           /* Determine the "innermost" array type.  */
    1698        18041 :           while (TREE_CODE (optype) == ARRAY_TYPE
    1699        24998 :                  && TREE_CODE (TREE_TYPE (optype)) == ARRAY_TYPE)
    1700         6957 :             optype = TREE_TYPE (optype);
    1701              : 
    1702              :           /* Avoid arrays of pointers.  */
    1703        18041 :           tree eltype = TREE_TYPE (optype);
    1704        18041 :           if (TREE_CODE (optype) != ARRAY_TYPE
    1705        18041 :               || !INTEGRAL_TYPE_P (eltype))
    1706              :             return false;
    1707              : 
    1708              :           /* Fail when the array bound is unknown or zero.  */
    1709        13679 :           val = TYPE_SIZE_UNIT (optype);
    1710        13679 :           if (!val
    1711        13607 :               || TREE_CODE (val) != INTEGER_CST
    1712        27258 :               || integer_zerop (val))
    1713              :             return false;
    1714              : 
    1715        13574 :           val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
    1716              :                               integer_one_node);
    1717              : 
    1718              :           /* Set the minimum size to zero since the string in
    1719              :              the array could have zero length.  */
    1720        13574 :           pdata->minlen = ssize_int (0);
    1721              : 
    1722        13574 :           tight_bound = true;
    1723              :         }
    1724       138615 :       else if (TREE_CODE (arg) == COMPONENT_REF
    1725       138615 :                && (TREE_CODE (TREE_TYPE (TREE_OPERAND (arg, 1)))
    1726              :                    == ARRAY_TYPE))
    1727              :         {
    1728              :           /* Use the type of the member array to determine the upper
    1729              :              bound on the length of the array.  This may be overly
    1730              :              optimistic if the array itself isn't NUL-terminated and
    1731              :              the caller relies on the subsequent member to contain
    1732              :              the NUL but that would only be considered valid if
    1733              :              the array were the last member of a struct.  */
    1734              : 
    1735        10009 :           tree fld = TREE_OPERAND (arg, 1);
    1736              : 
    1737        10009 :           tree optype = TREE_TYPE (fld);
    1738              : 
    1739              :           /* Determine the "innermost" array type.  */
    1740        10009 :           while (TREE_CODE (optype) == ARRAY_TYPE
    1741        10571 :                  && TREE_CODE (TREE_TYPE (optype)) == ARRAY_TYPE)
    1742          562 :             optype = TREE_TYPE (optype);
    1743              : 
    1744              :           /* Fail when the array bound is unknown or zero.  */
    1745        10009 :           val = TYPE_SIZE_UNIT (optype);
    1746        10009 :           if (!val
    1747         9773 :               || TREE_CODE (val) != INTEGER_CST
    1748        19747 :               || integer_zerop (val))
    1749              :             return false;
    1750         9659 :           val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
    1751              :                              integer_one_node);
    1752              : 
    1753              :           /* Set the minimum size to zero since the string in
    1754              :              the array could have zero length.  */
    1755         9659 :           pdata->minlen = ssize_int (0);
    1756              : 
    1757              :           /* The array size determined above is an optimistic bound
    1758              :              on the length.  If the array isn't nul-terminated the
    1759              :              length computed by the library function would be greater.
    1760              :              Even though using strlen to cross the subobject boundary
    1761              :              is undefined, avoid drawing conclusions from the member
    1762              :              type about the length here.  */
    1763         9659 :           tight_bound = true;
    1764              :         }
    1765       128606 :       else if (TREE_CODE (arg) == MEM_REF
    1766        34396 :                && TREE_CODE (TREE_TYPE (arg)) == ARRAY_TYPE
    1767        10137 :                && TREE_CODE (TREE_TYPE (TREE_TYPE (arg))) == INTEGER_TYPE
    1768       138305 :                && TREE_CODE (TREE_OPERAND (arg, 0)) == ADDR_EXPR)
    1769              :         {
    1770              :           /* Handle a MEM_REF into a DECL accessing an array of integers,
    1771              :              being conservative about references to extern structures with
    1772              :              flexible array members that can be initialized to arbitrary
    1773              :              numbers of elements as an extension (static structs are okay).  */
    1774         9699 :           tree ref = TREE_OPERAND (TREE_OPERAND (arg, 0), 0);
    1775         9699 :           if ((TREE_CODE (ref) == PARM_DECL || VAR_P (ref))
    1776        19385 :               && (decl_binds_to_current_def_p (ref)
    1777          438 :                   || !array_ref_flexible_size_p (arg)))
    1778              :             {
    1779              :               /* Fail if the offset is out of bounds.  Such accesses
    1780              :                  should be diagnosed at some point.  */
    1781         9573 :               val = DECL_SIZE_UNIT (ref);
    1782         9573 :               if (!val
    1783         9401 :                   || TREE_CODE (val) != INTEGER_CST
    1784        18974 :                   || integer_zerop (val))
    1785          371 :                 return false;
    1786              : 
    1787         9399 :               poly_offset_int psiz = wi::to_offset (val);
    1788         9399 :               poly_offset_int poff = mem_ref_offset (arg);
    1789         9399 :               if (known_le (psiz, poff))
    1790              :                 return false;
    1791              : 
    1792         9202 :               pdata->minlen = ssize_int (0);
    1793              : 
    1794              :               /* Subtract the offset and one for the terminating nul.  */
    1795         9202 :               psiz -= poff;
    1796         9202 :               psiz -= 1;
    1797         9202 :               val = wide_int_to_tree (TREE_TYPE (val), psiz);
    1798              :               /* Since VAL reflects the size of a declared object
    1799              :                  rather the type of the access it is not a tight bound.  */
    1800              :             }
    1801              :         }
    1802       118907 :       else if (TREE_CODE (arg) == PARM_DECL || VAR_P (arg))
    1803              :         {
    1804              :           /* Avoid handling pointers to arrays.  GCC might misuse
    1805              :              a pointer to an array of one bound to point to an array
    1806              :              object of a greater bound.  */
    1807        71121 :           tree argtype = TREE_TYPE (arg);
    1808        71121 :           if (TREE_CODE (argtype) == ARRAY_TYPE)
    1809              :             {
    1810        43006 :               val = TYPE_SIZE_UNIT (argtype);
    1811        43006 :               if (!val
    1812        42240 :                   || TREE_CODE (val) != INTEGER_CST
    1813        85246 :                   || integer_zerop (val))
    1814              :                 return false;
    1815        42125 :               val = wide_int_to_tree (TREE_TYPE (val),
    1816        42125 :                                       wi::sub (wi::to_wide (val), 1));
    1817              : 
    1818              :               /* Set the minimum size to zero since the string in
    1819              :                  the array could have zero length.  */
    1820        42125 :               pdata->minlen = ssize_int (0);
    1821              :             }
    1822              :         }
    1823              :       maxbound = true;
    1824              :     }
    1825              : 
    1826       352219 :   if (!val)
    1827              :     return false;
    1828              : 
    1829              :   /* Adjust the lower bound on the string length as necessary.  */
    1830       251765 :   if (!pdata->minlen
    1831       251765 :       || (rkind != SRK_STRLEN
    1832        78994 :           && TREE_CODE (pdata->minlen) == INTEGER_CST
    1833        78994 :           && TREE_CODE (val) == INTEGER_CST
    1834        78989 :           && tree_int_cst_lt (val, pdata->minlen)))
    1835       172790 :     pdata->minlen = val;
    1836              : 
    1837       251765 :   if (pdata->maxbound && TREE_CODE (pdata->maxbound) == INTEGER_CST)
    1838              :     {
    1839              :       /* Adjust the tighter (more optimistic) string length bound
    1840              :          if necessary and proceed to adjust the more conservative
    1841              :          bound.  */
    1842         7588 :       if (TREE_CODE (val) == INTEGER_CST)
    1843              :         {
    1844         7588 :           if (tree_int_cst_lt (pdata->maxbound, val))
    1845          668 :             pdata->maxbound = val;
    1846              :         }
    1847              :       else
    1848            0 :         pdata->maxbound = val;
    1849              :     }
    1850       244177 :   else if (pdata->maxbound || maxbound)
    1851              :     /* Set PDATA->MAXBOUND only if it either isn't INTEGER_CST or
    1852              :        if VAL corresponds to the maximum length determined based
    1853              :        on the type of the object.  */
    1854        71395 :     pdata->maxbound = val;
    1855              : 
    1856       251765 :   if (tight_bound)
    1857              :     {
    1858              :       /* VAL computed above represents an optimistically tight bound
    1859              :          on the length of the string based on the referenced object's
    1860              :          or subobject's type.  Determine the conservative upper bound
    1861              :          based on the enclosing object's size if possible.  */
    1862        23233 :       if (rkind == SRK_LENRANGE)
    1863              :         {
    1864        23233 :           poly_int64 offset;
    1865        23233 :           tree base = get_addr_base_and_unit_offset (arg, &offset);
    1866        23233 :           if (!base)
    1867              :             {
    1868              :               /* When the call above fails due to a non-constant offset
    1869              :                  assume the offset is zero and use the size of the whole
    1870              :                  enclosing object instead.  */
    1871         7846 :               base = get_base_address (arg);
    1872         7846 :               offset = 0;
    1873              :             }
    1874              :           /* If the base object is a pointer no upper bound on the length
    1875              :              can be determined.  Otherwise the maximum length is equal to
    1876              :              the size of the enclosing object minus the offset of
    1877              :              the referenced subobject minus 1 (for the terminating nul).  */
    1878        23233 :           tree type = TREE_TYPE (base);
    1879        23233 :           if (POINTER_TYPE_P (type)
    1880        23229 :               || (TREE_CODE (base) != PARM_DECL && !VAR_P (base))
    1881        41895 :               || !(val = DECL_SIZE_UNIT (base)))
    1882         5818 :             val = build_all_ones_cst (size_type_node);
    1883              :           else
    1884              :             {
    1885        17415 :               val = DECL_SIZE_UNIT (base);
    1886        17415 :               val = fold_build2 (MINUS_EXPR, TREE_TYPE (val), val,
    1887              :                                  size_int (offset + 1));
    1888              :             }
    1889              :         }
    1890              :       else
    1891              :         return false;
    1892              :     }
    1893              : 
    1894       251765 :   if (pdata->maxlen)
    1895              :     {
    1896              :       /* Adjust the more conservative bound if possible/necessary
    1897              :          and fail otherwise.  */
    1898        14775 :       if (rkind != SRK_STRLEN)
    1899              :         {
    1900        13844 :           if (TREE_CODE (pdata->maxlen) != INTEGER_CST
    1901        13844 :               || TREE_CODE (val) != INTEGER_CST)
    1902              :             return false;
    1903              : 
    1904        13839 :           if (tree_int_cst_lt (pdata->maxlen, val))
    1905         1371 :             pdata->maxlen = val;
    1906              :           return true;
    1907              :         }
    1908          931 :       else if (simple_cst_equal (val, pdata->maxlen) != 1)
    1909              :         {
    1910              :           /* Fail if the length of this ARG is different from that
    1911              :              previously determined from another ARG.  */
    1912              :           return false;
    1913              :         }
    1914              :     }
    1915              : 
    1916       237114 :   pdata->maxlen = val;
    1917       237114 :   return rkind == SRK_LENRANGE || !integer_all_onesp (val);
    1918              : }
    1919              : 
    1920              : /* For an ARG referencing one or more strings, try to obtain the range
    1921              :    of their lengths, or the size of the largest array ARG refers to if
    1922              :    the range of lengths cannot be determined, and store all in *PDATA.
    1923              :    For an integer ARG (when RKIND == SRK_INT_VALUE), try to determine
    1924              :    the maximum constant value.
    1925              :    If ARG is an SSA_NAME, follow its use-def chains.  When RKIND ==
    1926              :    SRK_STRLEN, then if PDATA->MAXLEN is not equal to the determined
    1927              :    length or if we are unable to determine the length, return false.
    1928              :    VISITED is a bitmap of visited variables.
    1929              :    RKIND determines the kind of value or range to obtain (see
    1930              :    strlen_range_kind).
    1931              :    Set PDATA->DECL if ARG refers to an unterminated constant array.
    1932              :    On input, set ELTSIZE to 1 for normal single byte character strings,
    1933              :    and either 2 or 4 for wide character strings (the size of wchar_t).
    1934              :    Return true if *PDATA was successfully populated and false otherwise.  */
    1935              : 
    1936              : static bool
    1937      1358196 : get_range_strlen (tree arg, bitmap visited,
    1938              :                   strlen_range_kind rkind,
    1939              :                   c_strlen_data *pdata, unsigned eltsize)
    1940              : {
    1941              : 
    1942      1439049 :   if (TREE_CODE (arg) != SSA_NAME)
    1943       448573 :     return get_range_strlen_tree (arg, visited, rkind, pdata, eltsize);
    1944              : 
    1945              :   /* If ARG is registered for SSA update we cannot look at its defining
    1946              :      statement.  */
    1947       990476 :   if (name_registered_for_update_p (arg))
    1948              :     return false;
    1949              : 
    1950              :   /* If we were already here, break the infinite cycle.  */
    1951       990476 :   if (!bitmap_set_bit (visited, SSA_NAME_VERSION (arg)))
    1952              :     return true;
    1953              : 
    1954       984941 :   tree var = arg;
    1955       984941 :   gimple *def_stmt = SSA_NAME_DEF_STMT (var);
    1956              : 
    1957       984941 :   switch (gimple_code (def_stmt))
    1958              :     {
    1959       123996 :       case GIMPLE_ASSIGN:
    1960              :         /* The RHS of the statement defining VAR must either have a
    1961              :            constant length or come from another SSA_NAME with a constant
    1962              :            length.  */
    1963       123996 :         if (gimple_assign_single_p (def_stmt)
    1964       123996 :             || gimple_assign_unary_nop_p (def_stmt))
    1965              :           {
    1966        80853 :             tree rhs = gimple_assign_rhs1 (def_stmt);
    1967        80853 :             return get_range_strlen (rhs, visited, rkind, pdata, eltsize);
    1968              :           }
    1969        43143 :         else if (gimple_assign_rhs_code (def_stmt) == COND_EXPR)
    1970              :           {
    1971          246 :             tree ops[2] = { gimple_assign_rhs2 (def_stmt),
    1972          246 :                             gimple_assign_rhs3 (def_stmt) };
    1973              : 
    1974          738 :             for (unsigned int i = 0; i < 2; i++)
    1975          492 :               if (!get_range_strlen (ops[i], visited, rkind, pdata, eltsize))
    1976              :                 {
    1977           28 :                   if (rkind != SRK_LENRANGE)
    1978              :                     return false;
    1979              :                   /* Set the upper bound to the maximum to prevent
    1980              :                      it from being adjusted in the next iteration but
    1981              :                      leave MINLEN and the more conservative MAXBOUND
    1982              :                      determined so far alone (or leave them null if
    1983              :                      they haven't been set yet).  That the MINLEN is
    1984              :                      in fact zero can be determined from MAXLEN being
    1985              :                      unbounded but the discovered minimum is used for
    1986              :                      diagnostics.  */
    1987           28 :                   pdata->maxlen = build_all_ones_cst (size_type_node);
    1988              :                 }
    1989              :             return true;
    1990              :           }
    1991              :         return false;
    1992              : 
    1993              :       case GIMPLE_PHI:
    1994              :         /* Unless RKIND == SRK_LENRANGE, all arguments of the PHI node
    1995              :            must have a constant length.  */
    1996        82664 :         for (unsigned i = 0; i < gimple_phi_num_args (def_stmt); i++)
    1997              :           {
    1998        59993 :             tree arg = gimple_phi_arg (def_stmt, i)->def;
    1999              : 
    2000              :             /* If this PHI has itself as an argument, we cannot
    2001              :                determine the string length of this argument.  However,
    2002              :                if we can find a constant string length for the other
    2003              :                PHI args then we can still be sure that this is a
    2004              :                constant string length.  So be optimistic and just
    2005              :                continue with the next argument.  */
    2006        59993 :             if (arg == gimple_phi_result (def_stmt))
    2007            0 :               continue;
    2008              : 
    2009        59993 :             if (!get_range_strlen (arg, visited, rkind, pdata, eltsize))
    2010              :               {
    2011        28267 :                 if (rkind != SRK_LENRANGE)
    2012              :                   return false;
    2013              :                 /* Set the upper bound to the maximum to prevent
    2014              :                    it from being adjusted in the next iteration but
    2015              :                    leave MINLEN and the more conservative MAXBOUND
    2016              :                    determined so far alone (or leave them null if
    2017              :                    they haven't been set yet).  That the MINLEN is
    2018              :                    in fact zero can be determined from MAXLEN being
    2019              :                    unbounded but the discovered minimum is used for
    2020              :                    diagnostics.  */
    2021        26496 :                 pdata->maxlen = build_all_ones_cst (size_type_node);
    2022              :               }
    2023              :           }
    2024              :         return true;
    2025              : 
    2026              :       default:
    2027              :         return false;
    2028              :     }
    2029              : }
    2030              : 
    2031              : /* Try to obtain the range of the lengths of the string(s) referenced
    2032              :    by ARG, or the size of the largest array ARG refers to if the range
    2033              :    of lengths cannot be determined, and store all in *PDATA which must
    2034              :    be zero-initialized on input except PDATA->MAXBOUND may be set to
    2035              :    a non-null tree node other than INTEGER_CST to request to have it
    2036              :    set to the length of the longest string in a PHI.  ELTSIZE is
    2037              :    the expected size of the string element in bytes: 1 for char and
    2038              :    some power of 2 for wide characters.
    2039              :    Return true if the range [PDATA->MINLEN, PDATA->MAXLEN] is suitable
    2040              :    for optimization.  Returning false means that a nonzero PDATA->MINLEN
    2041              :    doesn't reflect the true lower bound of the range when PDATA->MAXLEN
    2042              :    is -1 (in that case, the actual range is indeterminate, i.e.,
    2043              :    [0, PTRDIFF_MAX - 2].  */
    2044              : 
    2045              : bool
    2046      1137163 : get_range_strlen (tree arg, c_strlen_data *pdata, unsigned eltsize)
    2047              : {
    2048      1137163 :   auto_bitmap visited;
    2049      1137163 :   tree maxbound = pdata->maxbound;
    2050              : 
    2051      1137163 :   if (!get_range_strlen (arg, visited, SRK_LENRANGE, pdata, eltsize))
    2052              :     {
    2053              :       /* On failure extend the length range to an impossible maximum
    2054              :          (a valid MAXLEN must be less than PTRDIFF_MAX - 1).  Other
    2055              :          members can stay unchanged regardless.  */
    2056       913077 :       pdata->minlen = ssize_int (0);
    2057       913077 :       pdata->maxlen = build_all_ones_cst (size_type_node);
    2058              :     }
    2059       224086 :   else if (!pdata->minlen)
    2060         8462 :     pdata->minlen = ssize_int (0);
    2061              : 
    2062              :   /* If it's unchanged from it initial non-null value, set the conservative
    2063              :      MAXBOUND to SIZE_MAX.  Otherwise leave it null (if it is null).  */
    2064      1137163 :   if (maxbound && pdata->maxbound == maxbound)
    2065       650724 :     pdata->maxbound = build_all_ones_cst (size_type_node);
    2066              : 
    2067      1137163 :   return !integer_all_onesp (pdata->maxlen);
    2068      1137163 : }
    2069              : 
    2070              : /* Return the maximum value for ARG given RKIND (see strlen_range_kind).
    2071              :    For ARG of pointer types, NONSTR indicates if the caller is prepared
    2072              :    to handle unterminated strings.   For integer ARG and when RKIND ==
    2073              :    SRK_INT_VALUE, NONSTR must be null.
    2074              : 
    2075              :    If an unterminated array is discovered and our caller handles
    2076              :    unterminated arrays, then bubble up the offending DECL and
    2077              :    return the maximum size.  Otherwise return NULL.  */
    2078              : 
    2079              : static tree
    2080        95872 : get_maxval_strlen (tree arg, strlen_range_kind rkind, tree *nonstr = NULL)
    2081              : {
    2082              :   /* A non-null NONSTR is meaningless when determining the maximum
    2083              :      value of an integer ARG.  */
    2084        95872 :   gcc_assert (rkind != SRK_INT_VALUE || nonstr == NULL);
    2085              : 
    2086              :   // If arg is already a constant, simply return it.
    2087        95872 :   if (TREE_CODE (arg) == INTEGER_CST && rkind == SRK_INT_VALUE)
    2088              :     return arg;
    2089              : 
    2090              :   /* ARG must have an integral type when RKIND says so.  */
    2091        72945 :   gcc_assert (rkind != SRK_INT_VALUE || INTEGRAL_TYPE_P (TREE_TYPE (arg)));
    2092              : 
    2093        73044 :   auto_bitmap visited;
    2094              : 
    2095              :   /* Reset DATA.MAXLEN if the call fails or when DATA.MAXLEN
    2096              :      is unbounded.  */
    2097        73044 :   c_strlen_data lendata = { };
    2098        73044 :   if (!get_range_strlen (arg, visited, rkind, &lendata, /* eltsize = */1))
    2099        49915 :     lendata.maxlen = NULL_TREE;
    2100        23129 :   else if (lendata.maxlen && integer_all_onesp (lendata.maxlen))
    2101            0 :     lendata.maxlen = NULL_TREE;
    2102              : 
    2103        73044 :   if (nonstr)
    2104              :     {
    2105              :       /* For callers prepared to handle unterminated arrays set
    2106              :          *NONSTR to point to the declaration of the array and return
    2107              :          the maximum length/size. */
    2108        24005 :       *nonstr = lendata.decl;
    2109        24005 :       return lendata.maxlen;
    2110              :     }
    2111              : 
    2112              :   /* Fail if the constant array isn't nul-terminated.  */
    2113        49039 :   return lendata.decl ? NULL_TREE : lendata.maxlen;
    2114        73044 : }
    2115              : 
    2116              : /* Return true if LEN is known to be less than or equal to (or if STRICT is
    2117              :    true, strictly less than) the lower bound of SIZE at compile time and false
    2118              :    otherwise.  */
    2119              : 
    2120              : static bool
    2121        63260 : known_lower (gimple *stmt, tree len, tree size, bool strict = false)
    2122              : {
    2123        63260 :   if (len == NULL_TREE)
    2124              :     return false;
    2125              : 
    2126       236290 :   wide_int size_range[2];
    2127       236290 :   wide_int len_range[2];
    2128        47258 :   if (get_range (len, stmt, len_range) && get_range (size, stmt, size_range))
    2129              :     {
    2130        16555 :       if (strict)
    2131         1867 :         return wi::ltu_p (len_range[1], size_range[0]);
    2132              :       else
    2133        14688 :        return wi::leu_p (len_range[1], size_range[0]);
    2134              :     }
    2135              : 
    2136              :   return false;
    2137       283548 : }
    2138              : 
    2139              : /* Fold function call to builtin strcpy with arguments DEST and SRC.
    2140              :    If LEN is not NULL, it represents the length of the string to be
    2141              :    copied.  Return NULL_TREE if no simplification can be made.  */
    2142              : 
    2143              : static bool
    2144        26067 : gimple_fold_builtin_strcpy (gimple_stmt_iterator *gsi,
    2145              :                             tree dest, tree src)
    2146              : {
    2147        26067 :   gimple *stmt = gsi_stmt (*gsi);
    2148        26067 :   location_t loc = gimple_location (stmt);
    2149        26067 :   tree fn;
    2150              : 
    2151              :   /* If SRC and DEST are the same (and not volatile), return DEST.  */
    2152        26067 :   if (operand_equal_p (src, dest, 0))
    2153              :     {
    2154              :       /* Issue -Wrestrict unless the pointers are null (those do
    2155              :          not point to objects and so do not indicate an overlap;
    2156              :          such calls could be the result of sanitization and jump
    2157              :          threading).  */
    2158           86 :       if (!integer_zerop (dest) && !warning_suppressed_p (stmt, OPT_Wrestrict))
    2159              :         {
    2160           51 :           tree func = gimple_call_fndecl (stmt);
    2161              : 
    2162           51 :           warning_at (loc, OPT_Wrestrict,
    2163              :                       "%qD source argument is the same as destination",
    2164              :                       func);
    2165              :         }
    2166              : 
    2167           86 :       replace_call_with_value (gsi, dest);
    2168           86 :       return true;
    2169              :     }
    2170              : 
    2171        25981 :   if (optimize_function_for_size_p (cfun))
    2172              :     return false;
    2173              : 
    2174        24005 :   fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    2175        24005 :   if (!fn)
    2176              :     return false;
    2177              : 
    2178              :   /* Set to non-null if ARG refers to an unterminated array.  */
    2179        24005 :   tree nonstr = NULL;
    2180        24005 :   tree len = get_maxval_strlen (src, SRK_STRLEN, &nonstr);
    2181              : 
    2182        24005 :   if (nonstr)
    2183              :     {
    2184              :       /* Avoid folding calls with unterminated arrays.  */
    2185          531 :       if (!warning_suppressed_p (stmt, OPT_Wstringop_overread))
    2186           69 :         warn_string_no_nul (loc, stmt, "strcpy", src, nonstr);
    2187          531 :       suppress_warning (stmt, OPT_Wstringop_overread);
    2188          531 :       return false;
    2189              :     }
    2190              : 
    2191        26373 :   if (!len || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    2192              :     return false;
    2193              : 
    2194         2899 :   len = fold_convert_loc (loc, size_type_node, len);
    2195         2899 :   len = size_binop_loc (loc, PLUS_EXPR, len, build_int_cst (size_type_node, 1));
    2196         2899 :   len = force_gimple_operand_gsi (gsi, len, true,
    2197              :                                   NULL_TREE, true, GSI_SAME_STMT);
    2198         2899 :   gimple *repl = gimple_build_call (fn, 3, dest, src, len);
    2199         2899 :   replace_call_with_call_and_fold (gsi, repl);
    2200         2899 :   return true;
    2201              : }
    2202              : 
    2203              : /* Fold function call to builtin strncpy with arguments DEST, SRC, and LEN.
    2204              :    If SLEN is not NULL, it represents the length of the source string.
    2205              :    Return NULL_TREE if no simplification can be made.  */
    2206              : 
    2207              : static bool
    2208        17276 : gimple_fold_builtin_strncpy (gimple_stmt_iterator *gsi,
    2209              :                              tree dest, tree src, tree len)
    2210              : {
    2211        17276 :   gimple *stmt = gsi_stmt (*gsi);
    2212        17276 :   location_t loc = gimple_location (stmt);
    2213        17276 :   bool nonstring = get_attr_nonstring_decl (dest) != NULL_TREE;
    2214              : 
    2215              :   /* If the LEN parameter is zero, return DEST.  */
    2216        17276 :   if (integer_zerop (len))
    2217              :     {
    2218              :       /* Avoid warning if the destination refers to an array/pointer
    2219              :          decorate with attribute nonstring.  */
    2220          167 :       if (!nonstring)
    2221              :         {
    2222          155 :           tree fndecl = gimple_call_fndecl (stmt);
    2223              : 
    2224              :           /* Warn about the lack of nul termination: the result is not
    2225              :              a (nul-terminated) string.  */
    2226          155 :           tree slen = get_maxval_strlen (src, SRK_STRLEN);
    2227          155 :           if (slen && !integer_zerop (slen))
    2228           24 :             warning_at (loc, OPT_Wstringop_truncation,
    2229              :                         "%qD destination unchanged after copying no bytes "
    2230              :                         "from a string of length %E",
    2231              :                         fndecl, slen);
    2232              :           else
    2233          131 :             warning_at (loc, OPT_Wstringop_truncation,
    2234              :                         "%qD destination unchanged after copying no bytes",
    2235              :                         fndecl);
    2236              :         }
    2237              : 
    2238          167 :       replace_call_with_value (gsi, dest);
    2239          167 :       return true;
    2240              :     }
    2241              : 
    2242              :   /* We can't compare slen with len as constants below if len is not a
    2243              :      constant.  */
    2244        17109 :   if (TREE_CODE (len) != INTEGER_CST)
    2245              :     return false;
    2246              : 
    2247              :   /* Now, we must be passed a constant src ptr parameter.  */
    2248        10750 :   tree slen = get_maxval_strlen (src, SRK_STRLEN);
    2249        10750 :   if (!slen || TREE_CODE (slen) != INTEGER_CST)
    2250              :     return false;
    2251              : 
    2252              :   /* The size of the source string including the terminating nul.  */
    2253         1780 :   tree ssize = size_binop_loc (loc, PLUS_EXPR, slen, ssize_int (1));
    2254              : 
    2255              :   /* We do not support simplification of this case, though we do
    2256              :      support it when expanding trees into RTL.  */
    2257              :   /* FIXME: generate a call to __builtin_memset.  */
    2258         1780 :   if (tree_int_cst_lt (ssize, len))
    2259              :     return false;
    2260              : 
    2261              :   /* Diagnose truncation that leaves the copy unterminated.  */
    2262          695 :   maybe_diag_stxncpy_trunc (*gsi, src, len);
    2263              : 
    2264              :   /* OK transform into builtin memcpy.  */
    2265          695 :   tree fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    2266         1390 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    2267              :     return false;
    2268              : 
    2269          695 :   len = fold_convert_loc (loc, size_type_node, len);
    2270          695 :   len = force_gimple_operand_gsi (gsi, len, true,
    2271              :                                   NULL_TREE, true, GSI_SAME_STMT);
    2272          695 :   gimple *repl = gimple_build_call (fn, 3, dest, src, len);
    2273          695 :   replace_call_with_call_and_fold (gsi, repl);
    2274              : 
    2275          695 :   return true;
    2276              : }
    2277              : 
    2278              : /* Fold function call to builtin strchr or strrchr.
    2279              :    If both arguments are constant, evaluate and fold the result,
    2280              :    otherwise simplify str(r)chr (str, 0) into str + strlen (str).
    2281              :    In general strlen is significantly faster than strchr
    2282              :    due to being a simpler operation.  */
    2283              : static bool
    2284         5472 : gimple_fold_builtin_strchr (gimple_stmt_iterator *gsi, bool is_strrchr)
    2285              : {
    2286         5472 :   gimple *stmt = gsi_stmt (*gsi);
    2287         5472 :   tree str = gimple_call_arg (stmt, 0);
    2288         5472 :   tree c = gimple_call_arg (stmt, 1);
    2289         5472 :   location_t loc = gimple_location (stmt);
    2290         5472 :   const char *p;
    2291         5472 :   char ch;
    2292              : 
    2293         5472 :   if (!gimple_call_lhs (stmt))
    2294              :     return false;
    2295              : 
    2296              :   /* Avoid folding if the first argument is not a nul-terminated array.
    2297              :      Defer warning until later.  */
    2298         5462 :   if (!check_nul_terminated_array (NULL_TREE, str))
    2299              :     return false;
    2300              : 
    2301         5378 :   if ((p = c_getstr (str)) && target_char_cst_p (c, &ch))
    2302              :     {
    2303           41 :       const char *p1 = is_strrchr ? strrchr (p, ch) : strchr (p, ch);
    2304              : 
    2305           41 :       if (p1 == NULL)
    2306              :         {
    2307            1 :           replace_call_with_value (gsi, integer_zero_node);
    2308            1 :           return true;
    2309              :         }
    2310              : 
    2311           40 :       tree len = build_int_cst (size_type_node, p1 - p);
    2312           40 :       gimple_seq stmts = NULL;
    2313           40 :       gimple *new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
    2314              :                                               POINTER_PLUS_EXPR, str, len);
    2315           40 :       gimple_seq_add_stmt_without_update (&stmts, new_stmt);
    2316           40 :       gsi_replace_with_seq_vops (gsi, stmts);
    2317           40 :       return true;
    2318              :     }
    2319              : 
    2320         5419 :   if (!integer_zerop (c) || (!gimple_vuse (stmt) && gimple_in_ssa_p (cfun)))
    2321              :     return false;
    2322              : 
    2323              :   /* Transform strrchr (s, 0) to strchr (s, 0) when optimizing for size.  */
    2324           82 :   if (is_strrchr && optimize_function_for_size_p (cfun))
    2325              :     {
    2326            3 :       tree strchr_fn = builtin_decl_implicit (BUILT_IN_STRCHR);
    2327              : 
    2328            3 :       if (strchr_fn)
    2329              :         {
    2330            3 :           gimple *repl = gimple_build_call (strchr_fn, 2, str, c);
    2331            3 :           replace_call_with_call_and_fold (gsi, repl);
    2332            3 :           return true;
    2333              :         }
    2334              : 
    2335              :       return false;
    2336              :     }
    2337              : 
    2338           79 :   tree len;
    2339           79 :   tree strlen_fn = builtin_decl_implicit (BUILT_IN_STRLEN);
    2340              : 
    2341           79 :   if (!strlen_fn)
    2342              :     return false;
    2343              : 
    2344              :   /* Create newstr = strlen (str).  */
    2345           79 :   gimple_seq stmts = NULL;
    2346           79 :   gimple *new_stmt = gimple_build_call (strlen_fn, 1, str);
    2347           79 :   gimple_set_location (new_stmt, loc);
    2348           79 :   len = make_ssa_name (size_type_node);
    2349           79 :   gimple_call_set_lhs (new_stmt, len);
    2350           79 :   gimple_seq_add_stmt_without_update (&stmts, new_stmt);
    2351              : 
    2352              :   /* Create (str p+ strlen (str)).  */
    2353           79 :   new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
    2354              :                                   POINTER_PLUS_EXPR, str, len);
    2355           79 :   gimple_seq_add_stmt_without_update (&stmts, new_stmt);
    2356           79 :   gsi_replace_with_seq_vops (gsi, stmts);
    2357              :   /* gsi now points at the assignment to the lhs, get a
    2358              :      stmt iterator to the strlen.
    2359              :      ???  We can't use gsi_for_stmt as that doesn't work when the
    2360              :      CFG isn't built yet.  */
    2361           79 :   gimple_stmt_iterator gsi2 = *gsi;
    2362           79 :   gsi_prev (&gsi2);
    2363           79 :   fold_stmt (&gsi2);
    2364           79 :   return true;
    2365              : }
    2366              : 
    2367              : /* Fold function call to builtin strstr.
    2368              :    If both arguments are constant, evaluate and fold the result,
    2369              :    additionally fold strstr (x, "") into x and strstr (x, "c")
    2370              :    into strchr (x, 'c').  */
    2371              : static bool
    2372         4250 : gimple_fold_builtin_strstr (gimple_stmt_iterator *gsi)
    2373              : {
    2374         4250 :   gimple *stmt = gsi_stmt (*gsi);
    2375         4250 :   if (!gimple_call_lhs (stmt))
    2376              :     return false;
    2377              : 
    2378         4247 :   tree haystack = gimple_call_arg (stmt, 0);
    2379         4247 :   tree needle = gimple_call_arg (stmt, 1);
    2380              : 
    2381              :   /* Avoid folding if either argument is not a nul-terminated array.
    2382              :      Defer warning until later.  */
    2383         4247 :   if (!check_nul_terminated_array (NULL_TREE, haystack)
    2384         4247 :       || !check_nul_terminated_array (NULL_TREE, needle))
    2385              :     return false;
    2386              : 
    2387         4228 :   const char *q = c_getstr (needle);
    2388         4228 :   if (q == NULL)
    2389              :     return false;
    2390              : 
    2391         3081 :   if (const char *p = c_getstr (haystack))
    2392              :     {
    2393           14 :       const char *r = strstr (p, q);
    2394              : 
    2395           14 :       if (r == NULL)
    2396              :         {
    2397            1 :           replace_call_with_value (gsi, integer_zero_node);
    2398            1 :           return true;
    2399              :         }
    2400              : 
    2401           13 :       tree len = build_int_cst (size_type_node, r - p);
    2402           13 :       gimple_seq stmts = NULL;
    2403           13 :       gimple *new_stmt
    2404           13 :         = gimple_build_assign (gimple_call_lhs (stmt), POINTER_PLUS_EXPR,
    2405              :                                haystack, len);
    2406           13 :       gimple_seq_add_stmt_without_update (&stmts, new_stmt);
    2407           13 :       gsi_replace_with_seq_vops (gsi, stmts);
    2408           13 :       return true;
    2409              :     }
    2410              : 
    2411              :   /* For strstr (x, "") return x.  */
    2412         3067 :   if (q[0] == '\0')
    2413              :     {
    2414            6 :       replace_call_with_value (gsi, haystack);
    2415            6 :       return true;
    2416              :     }
    2417              : 
    2418         6122 :   if (!gimple_vuse (stmt) && gimple_in_ssa_p (cfun))
    2419              :     return false;
    2420              : 
    2421              :   /* Transform strstr (x, "c") into strchr (x, 'c').  */
    2422         3061 :   if (q[1] == '\0')
    2423              :     {
    2424           22 :       tree strchr_fn = builtin_decl_implicit (BUILT_IN_STRCHR);
    2425           22 :       if (strchr_fn)
    2426              :         {
    2427           22 :           tree c = build_int_cst (integer_type_node, q[0]);
    2428           22 :           gimple *repl = gimple_build_call (strchr_fn, 2, haystack, c);
    2429           22 :           replace_call_with_call_and_fold (gsi, repl);
    2430           22 :           return true;
    2431              :         }
    2432              :     }
    2433              : 
    2434              :   return false;
    2435              : }
    2436              : 
    2437              : /* Simplify a call to the strcat builtin.  DST and SRC are the arguments
    2438              :    to the call.
    2439              : 
    2440              :    Return NULL_TREE if no simplification was possible, otherwise return the
    2441              :    simplified form of the call as a tree.
    2442              : 
    2443              :    The simplified form may be a constant or other expression which
    2444              :    computes the same value, but in a more efficient manner (including
    2445              :    calls to other builtin functions).
    2446              : 
    2447              :    The call may contain arguments which need to be evaluated, but
    2448              :    which are not useful to determine the result of the call.  In
    2449              :    this case we return a chain of COMPOUND_EXPRs.  The LHS of each
    2450              :    COMPOUND_EXPR will be an argument which must be evaluated.
    2451              :    COMPOUND_EXPRs are chained through their RHS.  The RHS of the last
    2452              :    COMPOUND_EXPR in the chain will contain the tree for the simplified
    2453              :    form of the builtin function call.  */
    2454              : 
    2455              : static bool
    2456         7330 : gimple_fold_builtin_strcat (gimple_stmt_iterator *gsi, tree dst, tree src)
    2457              : {
    2458         7330 :   gimple *stmt = gsi_stmt (*gsi);
    2459         7330 :   location_t loc = gimple_location (stmt);
    2460              : 
    2461         7330 :   const char *p = c_getstr (src);
    2462              : 
    2463              :   /* If the string length is zero, return the dst parameter.  */
    2464         7330 :   if (p && *p == '\0')
    2465              :     {
    2466           72 :       replace_call_with_value (gsi, dst);
    2467           72 :       return true;
    2468              :     }
    2469              : 
    2470         7258 :   if (!optimize_bb_for_speed_p (gimple_bb (stmt)))
    2471              :     return false;
    2472              : 
    2473        13342 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    2474              :     return false;
    2475              : 
    2476              :   /* See if we can store by pieces into (dst + strlen(dst)).  */
    2477         6671 :   tree newdst;
    2478         6671 :   tree strlen_fn = builtin_decl_implicit (BUILT_IN_STRLEN);
    2479         6671 :   tree memcpy_fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    2480              : 
    2481         6671 :   if (!strlen_fn || !memcpy_fn)
    2482              :     return false;
    2483              : 
    2484              :   /* If the length of the source string isn't computable don't
    2485              :      split strcat into strlen and memcpy.  */
    2486         6671 :   tree len = get_maxval_strlen (src, SRK_STRLEN);
    2487         6671 :   if (! len)
    2488              :     return false;
    2489              : 
    2490              :   /* Create strlen (dst).  */
    2491          736 :   gimple_seq stmts = NULL, stmts2;
    2492          736 :   gimple *repl = gimple_build_call (strlen_fn, 1, dst);
    2493          736 :   gimple_set_location (repl, loc);
    2494          736 :   newdst = make_ssa_name (size_type_node);
    2495          736 :   gimple_call_set_lhs (repl, newdst);
    2496          736 :   gimple_seq_add_stmt_without_update (&stmts, repl);
    2497              : 
    2498              :   /* Create (dst p+ strlen (dst)).  */
    2499          736 :   newdst = fold_build_pointer_plus_loc (loc, dst, newdst);
    2500          736 :   newdst = force_gimple_operand (newdst, &stmts2, true, NULL_TREE);
    2501          736 :   gimple_seq_add_seq_without_update (&stmts, stmts2);
    2502              : 
    2503          736 :   len = fold_convert_loc (loc, size_type_node, len);
    2504          736 :   len = size_binop_loc (loc, PLUS_EXPR, len,
    2505              :                         build_int_cst (size_type_node, 1));
    2506          736 :   len = force_gimple_operand (len, &stmts2, true, NULL_TREE);
    2507          736 :   gimple_seq_add_seq_without_update (&stmts, stmts2);
    2508              : 
    2509          736 :   repl = gimple_build_call (memcpy_fn, 3, newdst, src, len);
    2510          736 :   gimple_seq_add_stmt_without_update (&stmts, repl);
    2511          736 :   if (gimple_call_lhs (stmt))
    2512              :     {
    2513          165 :       repl = gimple_build_assign (gimple_call_lhs (stmt), dst);
    2514          165 :       gimple_seq_add_stmt_without_update (&stmts, repl);
    2515          165 :       gsi_replace_with_seq_vops (gsi, stmts);
    2516              :       /* gsi now points at the assignment to the lhs, get a
    2517              :          stmt iterator to the memcpy call.
    2518              :          ???  We can't use gsi_for_stmt as that doesn't work when the
    2519              :          CFG isn't built yet.  */
    2520          165 :       gimple_stmt_iterator gsi2 = *gsi;
    2521          165 :       gsi_prev (&gsi2);
    2522          165 :       fold_stmt (&gsi2);
    2523              :     }
    2524              :   else
    2525              :     {
    2526          571 :       gsi_replace_with_seq_vops (gsi, stmts);
    2527          571 :       fold_stmt (gsi);
    2528              :     }
    2529              :   return true;
    2530              : }
    2531              : 
    2532              : /* Fold a call to the __strcat_chk builtin FNDECL.  DEST, SRC, and SIZE
    2533              :    are the arguments to the call.  */
    2534              : 
    2535              : static bool
    2536         1493 : gimple_fold_builtin_strcat_chk (gimple_stmt_iterator *gsi)
    2537              : {
    2538         1493 :   gimple *stmt = gsi_stmt (*gsi);
    2539         1493 :   tree dest = gimple_call_arg (stmt, 0);
    2540         1493 :   tree src = gimple_call_arg (stmt, 1);
    2541         1493 :   tree size = gimple_call_arg (stmt, 2);
    2542         1493 :   tree fn;
    2543         1493 :   const char *p;
    2544              : 
    2545         1493 :   p = c_getstr (src);
    2546              :   /* If the SRC parameter is "", return DEST.  */
    2547         1493 :   if (p && *p == '\0')
    2548              :     {
    2549           60 :       replace_call_with_value (gsi, dest);
    2550           60 :       return true;
    2551              :     }
    2552              : 
    2553         1433 :   if (! tree_fits_uhwi_p (size) || ! integer_all_onesp (size))
    2554              :     return false;
    2555              : 
    2556          164 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    2557              :     return false;
    2558              : 
    2559              :   /* If __builtin_strcat_chk is used, assume strcat is available.  */
    2560           82 :   fn = builtin_decl_explicit (BUILT_IN_STRCAT);
    2561           82 :   if (!fn)
    2562              :     return false;
    2563              : 
    2564           82 :   gimple *repl = gimple_build_call (fn, 2, dest, src);
    2565           82 :   replace_call_with_call_and_fold (gsi, repl);
    2566           82 :   return true;
    2567              : }
    2568              : 
    2569              : /* Simplify a call to the strncat builtin.  */
    2570              : 
    2571              : static bool
    2572         6786 : gimple_fold_builtin_strncat (gimple_stmt_iterator *gsi)
    2573              : {
    2574         6786 :   gimple *stmt = gsi_stmt (*gsi);
    2575         6786 :   tree dst = gimple_call_arg (stmt, 0);
    2576         6786 :   tree src = gimple_call_arg (stmt, 1);
    2577         6786 :   tree len = gimple_call_arg (stmt, 2);
    2578         6786 :   tree src_len = c_strlen (src, 1);
    2579              : 
    2580              :   /* If the requested length is zero, or the src parameter string
    2581              :      length is zero, return the dst parameter.  */
    2582         6786 :   if (integer_zerop (len) || (src_len && integer_zerop (src_len)))
    2583              :     {
    2584          119 :       replace_call_with_value (gsi, dst);
    2585          119 :       return true;
    2586              :     }
    2587              : 
    2588              :   /* Return early if the requested len is less than the string length.
    2589              :      Warnings will be issued elsewhere later.  */
    2590         6667 :   if (!src_len || known_lower (stmt, len, src_len, true))
    2591              :     return false;
    2592              : 
    2593              :   /* Warn on constant LEN.  */
    2594          568 :   if (TREE_CODE (len) == INTEGER_CST)
    2595              :     {
    2596          131 :       bool nowarn = warning_suppressed_p (stmt, OPT_Wstringop_overflow_);
    2597          131 :       tree dstsize;
    2598              : 
    2599          131 :       if (!nowarn && compute_builtin_object_size (dst, 1, &dstsize)
    2600          175 :           && TREE_CODE (dstsize) == INTEGER_CST)
    2601              :         {
    2602           44 :           int cmpdst = tree_int_cst_compare (len, dstsize);
    2603              : 
    2604           44 :           if (cmpdst >= 0)
    2605              :             {
    2606           19 :               tree fndecl = gimple_call_fndecl (stmt);
    2607              : 
    2608              :               /* Strncat copies (at most) LEN bytes and always appends
    2609              :                  the terminating NUL so the specified bound should never
    2610              :                  be equal to (or greater than) the size of the destination.
    2611              :                  If it is, the copy could overflow.  */
    2612           19 :               location_t loc = gimple_location (stmt);
    2613           37 :               nowarn = warning_at (loc, OPT_Wstringop_overflow_,
    2614              :                                    cmpdst == 0
    2615              :                                    ? G_("%qD specified bound %E equals "
    2616              :                                         "destination size")
    2617              :                                    : G_("%qD specified bound %E exceeds "
    2618              :                                         "destination size %E"),
    2619              :                                    fndecl, len, dstsize);
    2620           19 :               if (nowarn)
    2621            0 :                 suppress_warning (stmt, OPT_Wstringop_overflow_);
    2622              :             }
    2623              :         }
    2624              : 
    2625          131 :       if (!nowarn && TREE_CODE (src_len) == INTEGER_CST
    2626          243 :           && tree_int_cst_compare (src_len, len) == 0)
    2627              :         {
    2628           20 :           tree fndecl = gimple_call_fndecl (stmt);
    2629           20 :           location_t loc = gimple_location (stmt);
    2630              : 
    2631              :           /* To avoid possible overflow the specified bound should also
    2632              :              not be equal to the length of the source, even when the size
    2633              :              of the destination is unknown (it's not an uncommon mistake
    2634              :              to specify as the bound to strncpy the length of the source).  */
    2635           20 :           if (warning_at (loc, OPT_Wstringop_overflow_,
    2636              :                           "%qD specified bound %E equals source length",
    2637              :                           fndecl, len))
    2638            6 :             suppress_warning (stmt, OPT_Wstringop_overflow_);
    2639              :         }
    2640              :     }
    2641              : 
    2642          568 :   if (!known_lower (stmt, src_len, len))
    2643              :     return false;
    2644              : 
    2645          136 :   tree fn = builtin_decl_implicit (BUILT_IN_STRCAT);
    2646              : 
    2647              :   /* If the replacement _DECL isn't initialized, don't do the
    2648              :      transformation.  */
    2649          272 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    2650              :     return false;
    2651              : 
    2652              :   /* Otherwise, emit a call to strcat.  */
    2653          136 :   gcall *repl = gimple_build_call (fn, 2, dst, src);
    2654          136 :   replace_call_with_call_and_fold (gsi, repl);
    2655          136 :   return true;
    2656              : }
    2657              : 
    2658              : /* Fold a call to the __strncat_chk builtin with arguments DEST, SRC,
    2659              :    LEN, and SIZE.  */
    2660              : 
    2661              : static bool
    2662         1116 : gimple_fold_builtin_strncat_chk (gimple_stmt_iterator *gsi)
    2663              : {
    2664         1116 :   gimple *stmt = gsi_stmt (*gsi);
    2665         1116 :   tree dest = gimple_call_arg (stmt, 0);
    2666         1116 :   tree src = gimple_call_arg (stmt, 1);
    2667         1116 :   tree len = gimple_call_arg (stmt, 2);
    2668         1116 :   tree size = gimple_call_arg (stmt, 3);
    2669         1116 :   tree fn;
    2670         1116 :   const char *p;
    2671              : 
    2672         1116 :   p = c_getstr (src);
    2673              :   /* If the SRC parameter is "" or if LEN is 0, return DEST.  */
    2674          276 :   if ((p && *p == '\0')
    2675         1341 :       || integer_zerop (len))
    2676              :     {
    2677           78 :       replace_call_with_value (gsi, dest);
    2678           78 :       return true;
    2679              :     }
    2680              : 
    2681         2076 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    2682              :     return false;
    2683              : 
    2684         1038 :   if (! integer_all_onesp (size))
    2685              :     {
    2686          951 :       tree src_len = c_strlen (src, 1);
    2687          951 :       if (known_lower (stmt, src_len, len))
    2688              :         {
    2689              :           /* If LEN >= strlen (SRC), optimize into __strcat_chk.  */
    2690           39 :           fn = builtin_decl_explicit (BUILT_IN_STRCAT_CHK);
    2691           39 :           if (!fn)
    2692              :             return false;
    2693              : 
    2694           39 :           gimple *repl = gimple_build_call (fn, 3, dest, src, size);
    2695           39 :           replace_call_with_call_and_fold (gsi, repl);
    2696           39 :           return true;
    2697              :         }
    2698              :       return false;
    2699              :     }
    2700              : 
    2701              :   /* If __builtin_strncat_chk is used, assume strncat is available.  */
    2702           87 :   fn = builtin_decl_explicit (BUILT_IN_STRNCAT);
    2703           87 :   if (!fn)
    2704              :     return false;
    2705              : 
    2706           87 :   gimple *repl = gimple_build_call (fn, 3, dest, src, len);
    2707           87 :   replace_call_with_call_and_fold (gsi, repl);
    2708           87 :   return true;
    2709              : }
    2710              : 
    2711              : /* Build and append gimple statements to STMTS that would load a first
    2712              :    character of a memory location identified by STR.  LOC is location
    2713              :    of the statement.  */
    2714              : 
    2715              : static tree
    2716          482 : gimple_load_first_char (location_t loc, tree str, gimple_seq *stmts)
    2717              : {
    2718          482 :   tree var;
    2719              : 
    2720          482 :   tree cst_uchar_node = build_type_variant (unsigned_char_type_node, 1, 0);
    2721          482 :   tree cst_uchar_ptr_node
    2722          482 :     = build_pointer_type_for_mode (cst_uchar_node, ptr_mode, true);
    2723          482 :   tree off0 = build_int_cst (cst_uchar_ptr_node, 0);
    2724              : 
    2725          482 :   tree temp = fold_build2_loc (loc, MEM_REF, cst_uchar_node, str, off0);
    2726          482 :   gassign *stmt = gimple_build_assign (NULL_TREE, temp);
    2727          482 :   var = make_ssa_name (cst_uchar_node, stmt);
    2728              : 
    2729          482 :   gimple_assign_set_lhs (stmt, var);
    2730          482 :   gimple_seq_add_stmt_without_update (stmts, stmt);
    2731              : 
    2732          482 :   return var;
    2733              : }
    2734              : 
    2735              : /* Fold a call to the str{n}{case}cmp builtin pointed by GSI iterator.  */
    2736              : 
    2737              : static bool
    2738      1251050 : gimple_fold_builtin_string_compare (gimple_stmt_iterator *gsi)
    2739              : {
    2740      1251050 :   gimple *stmt = gsi_stmt (*gsi);
    2741      1251050 :   tree callee = gimple_call_fndecl (stmt);
    2742      1251050 :   enum built_in_function fcode = DECL_FUNCTION_CODE (callee);
    2743              : 
    2744      1251050 :   tree type = integer_type_node;
    2745      1251050 :   tree str1 = gimple_call_arg (stmt, 0);
    2746      1251050 :   tree str2 = gimple_call_arg (stmt, 1);
    2747      1251050 :   tree lhs = gimple_call_lhs (stmt);
    2748              : 
    2749      1251050 :   tree bound_node = NULL_TREE;
    2750      1251050 :   unsigned HOST_WIDE_INT bound = HOST_WIDE_INT_M1U;
    2751              : 
    2752              :   /* Handle strncmp and strncasecmp functions.  */
    2753      1251050 :   if (gimple_call_num_args (stmt) == 3)
    2754              :     {
    2755        23293 :       bound_node = gimple_call_arg (stmt, 2);
    2756        23293 :       if (tree_fits_uhwi_p (bound_node))
    2757        17569 :         bound = tree_to_uhwi (bound_node);
    2758              :     }
    2759              : 
    2760              :   /* If the BOUND parameter is zero, return zero.  */
    2761        17569 :   if (bound == 0)
    2762              :     {
    2763            4 :       replace_call_with_value (gsi, integer_zero_node);
    2764            4 :       return true;
    2765              :     }
    2766              : 
    2767              :   /* If ARG1 and ARG2 are the same (and not volatile), return zero.  */
    2768      1251046 :   if (operand_equal_p (str1, str2, 0))
    2769              :     {
    2770           41 :       replace_call_with_value (gsi, integer_zero_node);
    2771           41 :       return true;
    2772              :     }
    2773              : 
    2774      2502010 :   if (!gimple_vuse (stmt) && gimple_in_ssa_p (cfun))
    2775              :     return false;
    2776              : 
    2777              :   /* Initially set to the number of characters, including the terminating
    2778              :      nul if each array has one.   LENx == strnlen (Sx, LENx) implies that
    2779              :      the array Sx is not terminated by a nul.
    2780              :      For nul-terminated strings then adjusted to their length so that
    2781              :      LENx == NULPOSx holds.  */
    2782      1251005 :   unsigned HOST_WIDE_INT len1 = HOST_WIDE_INT_MAX, len2 = len1;
    2783      1251005 :   const char *p1 = getbyterep (str1, &len1);
    2784      1251005 :   const char *p2 = getbyterep (str2, &len2);
    2785              : 
    2786              :   /* The position of the terminating nul character if one exists, otherwise
    2787              :      a value greater than LENx.  */
    2788      1251005 :   unsigned HOST_WIDE_INT nulpos1 = HOST_WIDE_INT_MAX, nulpos2 = nulpos1;
    2789              : 
    2790      1251005 :   if (p1)
    2791              :     {
    2792        42819 :       size_t n = strnlen (p1, len1);
    2793        42819 :       if (n < len1)
    2794        42712 :         len1 = nulpos1 = n;
    2795              :     }
    2796              : 
    2797      1251005 :   if (p2)
    2798              :     {
    2799      1219912 :       size_t n = strnlen (p2, len2);
    2800      1219912 :       if (n < len2)
    2801      1219853 :         len2 = nulpos2 = n;
    2802              :     }
    2803              : 
    2804              :   /* For known strings, return an immediate value.  */
    2805      1251005 :   if (p1 && p2)
    2806              :     {
    2807        39245 :       int r = 0;
    2808        39245 :       bool known_result = false;
    2809              : 
    2810        39245 :       switch (fcode)
    2811              :         {
    2812        38169 :         case BUILT_IN_STRCMP:
    2813        38169 :         case BUILT_IN_STRCMP_EQ:
    2814        38169 :           if (len1 != nulpos1 || len2 != nulpos2)
    2815              :             break;
    2816              : 
    2817        38144 :           r = strcmp (p1, p2);
    2818        38144 :           known_result = true;
    2819        38144 :           break;
    2820              : 
    2821         1002 :         case BUILT_IN_STRNCMP:
    2822         1002 :         case BUILT_IN_STRNCMP_EQ:
    2823         1002 :           {
    2824         1002 :             if (bound == HOST_WIDE_INT_M1U)
    2825              :               break;
    2826              : 
    2827              :             /* Reduce the bound to be no more than the length
    2828              :                of the shorter of the two strings, or the sizes
    2829              :                of the unterminated arrays.  */
    2830           38 :             unsigned HOST_WIDE_INT n = bound;
    2831              : 
    2832           38 :             if (len1 == nulpos1 && len1 < n)
    2833            4 :               n = len1 + 1;
    2834           38 :             if (len2 == nulpos2 && len2 < n)
    2835           11 :               n = len2 + 1;
    2836              : 
    2837           38 :             if (MIN (nulpos1, nulpos2) + 1 < n)
    2838              :               break;
    2839              : 
    2840           38 :             r = strncmp (p1, p2, n);
    2841           38 :             known_result = true;
    2842           38 :             break;
    2843              :           }
    2844              :         /* Only handleable situation is where the string are equal (result 0),
    2845              :            which is already handled by operand_equal_p case.  */
    2846              :         case BUILT_IN_STRCASECMP:
    2847              :           break;
    2848           37 :         case BUILT_IN_STRNCASECMP:
    2849           37 :           {
    2850           37 :             if (bound == HOST_WIDE_INT_M1U)
    2851              :               break;
    2852           37 :             r = strncmp (p1, p2, bound);
    2853           37 :             if (r == 0)
    2854              :               known_result = true;
    2855              :             break;
    2856              :           }
    2857            0 :         default:
    2858            0 :           gcc_unreachable ();
    2859              :         }
    2860              : 
    2861        38182 :       if (known_result)
    2862              :         {
    2863        38182 :           replace_call_with_value (gsi, build_cmp_result (type, r));
    2864        38182 :           return true;
    2865              :         }
    2866              :     }
    2867              : 
    2868      2425646 :   bool nonzero_bound = (bound >= 1 && bound < HOST_WIDE_INT_M1U)
    2869      1195398 :     || fcode == BUILT_IN_STRCMP
    2870      1195398 :     || fcode == BUILT_IN_STRCMP_EQ
    2871      1218737 :     || fcode == BUILT_IN_STRCASECMP;
    2872              : 
    2873      1212823 :   location_t loc = gimple_location (stmt);
    2874              : 
    2875              :   /* If the second arg is "", return *(const unsigned char*)arg1.  */
    2876      1212823 :   if (p2 && *p2 == '\0' && nonzero_bound)
    2877              :     {
    2878          150 :       gimple_seq stmts = NULL;
    2879          150 :       tree var = gimple_load_first_char (loc, str1, &stmts);
    2880          150 :       if (lhs)
    2881              :         {
    2882          150 :           stmt = gimple_build_assign (lhs, NOP_EXPR, var);
    2883          150 :           gimple_seq_add_stmt_without_update (&stmts, stmt);
    2884              :         }
    2885              : 
    2886          150 :       gsi_replace_with_seq_vops (gsi, stmts);
    2887          150 :       return true;
    2888              :     }
    2889              : 
    2890              :   /* If the first arg is "", return -*(const unsigned char*)arg2.  */
    2891      1212673 :   if (p1 && *p1 == '\0' && nonzero_bound)
    2892              :     {
    2893           99 :       gimple_seq stmts = NULL;
    2894           99 :       tree var = gimple_load_first_char (loc, str2, &stmts);
    2895              : 
    2896           99 :       if (lhs)
    2897              :         {
    2898           99 :           tree c = make_ssa_name (integer_type_node);
    2899           99 :           stmt = gimple_build_assign (c, NOP_EXPR, var);
    2900           99 :           gimple_seq_add_stmt_without_update (&stmts, stmt);
    2901              : 
    2902           99 :           stmt = gimple_build_assign (lhs, NEGATE_EXPR, c);
    2903           99 :           gimple_seq_add_stmt_without_update (&stmts, stmt);
    2904              :         }
    2905              : 
    2906           99 :       gsi_replace_with_seq_vops (gsi, stmts);
    2907           99 :       return true;
    2908              :     }
    2909              : 
    2910              :   /* If BOUND is one, return an expression corresponding to
    2911              :      (*(const unsigned char*)arg2 - *(const unsigned char*)arg1).  */
    2912      1212574 :   if (fcode == BUILT_IN_STRNCMP && bound == 1)
    2913              :     {
    2914          110 :       gimple_seq stmts = NULL;
    2915          110 :       tree temp1 = gimple_load_first_char (loc, str1, &stmts);
    2916          110 :       tree temp2 = gimple_load_first_char (loc, str2, &stmts);
    2917              : 
    2918          110 :       if (lhs)
    2919              :         {
    2920          107 :           tree c1 = make_ssa_name (integer_type_node);
    2921          107 :           gassign *convert1 = gimple_build_assign (c1, NOP_EXPR, temp1);
    2922          107 :           gimple_seq_add_stmt_without_update (&stmts, convert1);
    2923              : 
    2924          107 :           tree c2 = make_ssa_name (integer_type_node);
    2925          107 :           gassign *convert2 = gimple_build_assign (c2, NOP_EXPR, temp2);
    2926          107 :           gimple_seq_add_stmt_without_update (&stmts, convert2);
    2927              : 
    2928          107 :           stmt = gimple_build_assign (lhs, MINUS_EXPR, c1, c2);
    2929          107 :           gimple_seq_add_stmt_without_update (&stmts, stmt);
    2930              :         }
    2931              : 
    2932          110 :       gsi_replace_with_seq_vops (gsi, stmts);
    2933          110 :       return true;
    2934              :     }
    2935              : 
    2936              :   /* If BOUND is greater than the length of one constant string,
    2937              :      and the other argument is also a nul-terminated string, replace
    2938              :      strncmp with strcmp.  */
    2939      1212464 :   if (fcode == BUILT_IN_STRNCMP
    2940        17709 :       && bound > 0 && bound < HOST_WIDE_INT_M1U
    2941        12117 :       && ((p2 && len2 < bound && len2 == nulpos2)
    2942        11885 :           || (p1 && len1 < bound && len1 == nulpos1)))
    2943              :     {
    2944          310 :       tree fn = builtin_decl_implicit (BUILT_IN_STRCMP);
    2945          310 :       if (!fn)
    2946              :         return false;
    2947          310 :       gimple *repl = gimple_build_call (fn, 2, str1, str2);
    2948          310 :       replace_call_with_call_and_fold (gsi, repl);
    2949          310 :       return true;
    2950              :     }
    2951              : 
    2952              :   return false;
    2953              : }
    2954              : 
    2955              : /* Fold a call to the memchr pointed by GSI iterator.  */
    2956              : 
    2957              : static bool
    2958        30365 : gimple_fold_builtin_memchr (gimple_stmt_iterator *gsi)
    2959              : {
    2960        30365 :   gimple *stmt = gsi_stmt (*gsi);
    2961        30365 :   tree lhs = gimple_call_lhs (stmt);
    2962        30365 :   tree arg1 = gimple_call_arg (stmt, 0);
    2963        30365 :   tree arg2 = gimple_call_arg (stmt, 1);
    2964        30365 :   tree len = gimple_call_arg (stmt, 2);
    2965              : 
    2966              :   /* If the LEN parameter is zero, return zero.  */
    2967        30365 :   if (integer_zerop (len))
    2968              :     {
    2969            1 :       replace_call_with_value (gsi, build_int_cst (ptr_type_node, 0));
    2970            1 :       return true;
    2971              :     }
    2972              : 
    2973        30364 :   char c;
    2974        30364 :   if (TREE_CODE (arg2) != INTEGER_CST
    2975        16720 :       || !tree_fits_uhwi_p (len)
    2976        31092 :       || !target_char_cst_p (arg2, &c))
    2977              :     return false;
    2978              : 
    2979          728 :   unsigned HOST_WIDE_INT length = tree_to_uhwi (len);
    2980          728 :   unsigned HOST_WIDE_INT string_length;
    2981          728 :   const char *p1 = getbyterep (arg1, &string_length);
    2982              : 
    2983          728 :   if (p1)
    2984              :     {
    2985          110 :       const char *r = (const char *)memchr (p1, c, MIN (length, string_length));
    2986          110 :       if (r == NULL)
    2987              :         {
    2988           14 :           tree mem_size, offset_node;
    2989           14 :           byte_representation (arg1, &offset_node, &mem_size, NULL);
    2990           14 :           unsigned HOST_WIDE_INT offset = (offset_node == NULL_TREE)
    2991           14 :                                           ? 0 : tree_to_uhwi (offset_node);
    2992              :           /* MEM_SIZE is the size of the array the string literal
    2993              :              is stored in.  */
    2994           14 :           unsigned HOST_WIDE_INT string_size = tree_to_uhwi (mem_size) - offset;
    2995           14 :           gcc_checking_assert (string_length <= string_size);
    2996           14 :           if (length <= string_size)
    2997              :             {
    2998            4 :               replace_call_with_value (gsi, build_int_cst (ptr_type_node, 0));
    2999            4 :               return true;
    3000              :             }
    3001              :         }
    3002              :       else
    3003              :         {
    3004           96 :           unsigned HOST_WIDE_INT offset = r - p1;
    3005           96 :           gimple_seq stmts = NULL;
    3006           96 :           if (lhs != NULL_TREE)
    3007              :             {
    3008           94 :               tree offset_cst = build_int_cst (sizetype, offset);
    3009           94 :               gassign *stmt = gimple_build_assign (lhs, POINTER_PLUS_EXPR,
    3010              :                                                    arg1, offset_cst);
    3011           94 :               gimple_seq_add_stmt_without_update (&stmts, stmt);
    3012              :             }
    3013              :           else
    3014            2 :             gimple_seq_add_stmt_without_update (&stmts,
    3015              :                                                 gimple_build_nop ());
    3016              : 
    3017           96 :           gsi_replace_with_seq_vops (gsi, stmts);
    3018           96 :           return true;
    3019              :         }
    3020              :     }
    3021              : 
    3022              :   return false;
    3023              : }
    3024              : 
    3025              : /* Fold a call to the fputs builtin.  ARG0 and ARG1 are the arguments
    3026              :    to the call.  IGNORE is true if the value returned
    3027              :    by the builtin will be ignored.  UNLOCKED is true is true if this
    3028              :    actually a call to fputs_unlocked.  If LEN in non-NULL, it represents
    3029              :    the known length of the string.  Return NULL_TREE if no simplification
    3030              :    was possible.  */
    3031              : 
    3032              : static bool
    3033        20753 : gimple_fold_builtin_fputs (gimple_stmt_iterator *gsi,
    3034              :                            tree arg0, tree arg1,
    3035              :                            bool unlocked)
    3036              : {
    3037        20753 :   gimple *stmt = gsi_stmt (*gsi);
    3038              : 
    3039              :   /* If we're using an unlocked function, assume the other unlocked
    3040              :      functions exist explicitly.  */
    3041        20753 :   tree const fn_fputc = (unlocked
    3042        20753 :                          ? builtin_decl_explicit (BUILT_IN_FPUTC_UNLOCKED)
    3043        20753 :                          : builtin_decl_implicit (BUILT_IN_FPUTC));
    3044        20710 :   tree const fn_fwrite = (unlocked
    3045           43 :                           ? builtin_decl_explicit (BUILT_IN_FWRITE_UNLOCKED)
    3046        20753 :                           : builtin_decl_implicit (BUILT_IN_FWRITE));
    3047              : 
    3048              :   /* If the return value is used, don't do the transformation.  */
    3049        20753 :   if (gimple_call_lhs (stmt))
    3050              :     return false;
    3051              : 
    3052              :   /* Get the length of the string passed to fputs.  If the length
    3053              :      can't be determined, punt.  */
    3054        20682 :   tree len = get_maxval_strlen (arg0, SRK_STRLEN);
    3055        20682 :   if (!len || TREE_CODE (len) != INTEGER_CST)
    3056              :     return false;
    3057              : 
    3058        16246 :   switch (compare_tree_int (len, 1))
    3059              :     {
    3060           91 :     case -1: /* length is 0, delete the call entirely .  */
    3061           91 :       replace_call_with_value (gsi, integer_zero_node);
    3062           91 :       return true;
    3063              : 
    3064         1064 :     case 0: /* length is 1, call fputc.  */
    3065         1064 :       {
    3066         1064 :         const char *p = c_getstr (arg0);
    3067         1064 :         if (p != NULL)
    3068              :           {
    3069         2100 :             if (!fn_fputc || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3070              :               return false;
    3071              : 
    3072         1050 :             gimple *repl
    3073         1050 :               = gimple_build_call (fn_fputc, 2,
    3074         1050 :                                    build_int_cst (integer_type_node, p[0]),
    3075              :                                    arg1);
    3076         1050 :             replace_call_with_call_and_fold (gsi, repl);
    3077         1050 :             return true;
    3078              :           }
    3079              :       }
    3080              :       /* FALLTHROUGH */
    3081        15105 :     case 1: /* length is greater than 1, call fwrite.  */
    3082        15105 :       {
    3083              :         /* If optimizing for size keep fputs.  */
    3084        15105 :         if (optimize_function_for_size_p (cfun))
    3085              :           return false;
    3086              :         /* New argument list transforming fputs(string, stream) to
    3087              :            fwrite(string, 1, len, stream).  */
    3088        16524 :         if (!fn_fwrite || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3089              :           return false;
    3090              : 
    3091         8262 :         gimple *repl
    3092         8262 :           = gimple_build_call (fn_fwrite, 4, arg0, size_one_node,
    3093              :                                fold_convert (size_type_node, len), arg1);
    3094         8262 :         replace_call_with_call_and_fold (gsi, repl);
    3095         8262 :         return true;
    3096              :       }
    3097            0 :     default:
    3098            0 :       gcc_unreachable ();
    3099              :     }
    3100              : }
    3101              : 
    3102              : /* Fold a call to fwrite (PTR, SIZE, N, STREAM) at *GSI.  UNLOCKED says whether
    3103              :    the callee is fwrite_unlocked rather than fwrite.  A call that transfers a
    3104              :    single byte and whose result is nobody's business writes the same byte as
    3105              :    fputc (*PTR, STREAM), which reaches the stream without going through the
    3106              :    generic buffered-write path.  Return true if the call was folded.  */
    3107              : 
    3108              : static bool
    3109        41124 : gimple_fold_builtin_fwrite (gimple_stmt_iterator *gsi, bool unlocked)
    3110              : {
    3111        41124 :   gimple *stmt = gsi_stmt (*gsi);
    3112              : 
    3113              :   /* fwrite reports the number of items transferred and fputc the character
    3114              :      written, so only fold when nothing looks at the result.  */
    3115        41124 :   if (gimple_call_lhs (stmt))
    3116              :     return false;
    3117              : 
    3118              :   /* fwrite transfers SIZE * N bytes, so writing a single byte needs both
    3119              :      counts to be one: no other pair of non-negative values multiplies to
    3120              :      one.  */
    3121        39979 :   if (!integer_onep (gimple_call_arg (stmt, 1))
    3122        39979 :       || !integer_onep (gimple_call_arg (stmt, 2)))
    3123              :     return false;
    3124              : 
    3125              :   /* If we're using an unlocked function, assume the other unlocked
    3126              :      functions exist explicitly.  */
    3127           13 :   tree const fn_fputc = (unlocked
    3128           13 :                          ? builtin_decl_explicit (BUILT_IN_FPUTC_UNLOCKED)
    3129        41137 :                          : builtin_decl_implicit (BUILT_IN_FPUTC));
    3130           26 :   if (!fn_fputc || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3131              :     return false;
    3132              : 
    3133           13 :   location_t loc = gimple_location (stmt);
    3134           13 :   gimple_seq stmts = NULL;
    3135           13 :   tree byte = gimple_load_first_char (loc, gimple_call_arg (stmt, 0), &stmts);
    3136           13 :   tree c = gimple_convert (&stmts, integer_type_node, byte);
    3137           13 :   tree stream = gimple_call_arg (stmt, 3);
    3138           13 :   gcall *repl = gimple_build_call (fn_fputc, 2, c, stream);
    3139           13 :   gimple_seq_add_stmt_without_update (&stmts, repl);
    3140           13 :   gsi_replace_with_seq_vops (gsi, stmts);
    3141           13 :   return true;
    3142              : }
    3143              : 
    3144              : /* Fold a call to the __mem{cpy,pcpy,move,set}_chk builtin.
    3145              :    DEST, SRC, LEN, and SIZE are the arguments to the call.
    3146              :    IGNORE is true, if return value can be ignored.  FCODE is the BUILT_IN_*
    3147              :    code of the builtin.  If MAXLEN is not NULL, it is maximum length
    3148              :    passed as third argument.  */
    3149              : 
    3150              : static bool
    3151        25701 : gimple_fold_builtin_memory_chk (gimple_stmt_iterator *gsi,
    3152              :                                 tree dest, tree src, tree len, tree size,
    3153              :                                 enum built_in_function fcode)
    3154              : {
    3155        25701 :   gimple *stmt = gsi_stmt (*gsi);
    3156        25701 :   location_t loc = gimple_location (stmt);
    3157        25701 :   bool ignore = gimple_call_lhs (stmt) == NULL_TREE;
    3158        25701 :   tree fn;
    3159              : 
    3160              :   /* If SRC and DEST are the same (and not volatile), return DEST
    3161              :      (resp. DEST+LEN for __mempcpy_chk).  */
    3162        25701 :   if (fcode != BUILT_IN_MEMSET_CHK && operand_equal_p (src, dest, 0))
    3163              :     {
    3164           13 :       if (fcode != BUILT_IN_MEMPCPY_CHK)
    3165              :         {
    3166            7 :           replace_call_with_value (gsi, dest);
    3167            7 :           return true;
    3168              :         }
    3169              :       else
    3170              :         {
    3171            6 :           gimple_seq stmts = NULL;
    3172            6 :           len = gimple_convert_to_ptrofftype (&stmts, loc, len);
    3173            6 :           tree temp = gimple_build (&stmts, loc, POINTER_PLUS_EXPR,
    3174            6 :                                     TREE_TYPE (dest), dest, len);
    3175            6 :           gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    3176            6 :           replace_call_with_value (gsi, temp);
    3177            6 :           return true;
    3178              :         }
    3179              :     }
    3180              : 
    3181        51376 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    3182              :     return false;
    3183              : 
    3184        25688 :   tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
    3185        25688 :   if (! integer_all_onesp (size)
    3186        24689 :       && !known_lower (stmt, len, size)
    3187        42927 :       && !known_lower (stmt, maxlen, size))
    3188              :     {
    3189              :       /* MAXLEN and LEN both cannot be proved to be less than SIZE, at
    3190              :          least try to optimize (void) __mempcpy_chk () into
    3191              :          (void) __memcpy_chk () */
    3192        17162 :       if (fcode == BUILT_IN_MEMPCPY_CHK && ignore)
    3193              :         {
    3194           43 :           fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
    3195           43 :           if (!fn)
    3196              :             return false;
    3197              : 
    3198           43 :           gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
    3199           43 :           replace_call_with_call_and_fold (gsi, repl);
    3200           43 :           return true;
    3201              :         }
    3202              :       return false;
    3203              :     }
    3204              : 
    3205         8526 :   fn = NULL_TREE;
    3206              :   /* If __builtin_mem{cpy,pcpy,move,set}_chk is used, assume
    3207              :      mem{cpy,pcpy,move,set} is available.  */
    3208         8526 :   switch (fcode)
    3209              :     {
    3210         1781 :     case BUILT_IN_MEMCPY_CHK:
    3211         1781 :       fn = builtin_decl_explicit (BUILT_IN_MEMCPY);
    3212         1781 :       break;
    3213         1068 :     case BUILT_IN_MEMPCPY_CHK:
    3214         1068 :       fn = builtin_decl_explicit (BUILT_IN_MEMPCPY);
    3215         1068 :       break;
    3216         1652 :     case BUILT_IN_MEMMOVE_CHK:
    3217         1652 :       fn = builtin_decl_explicit (BUILT_IN_MEMMOVE);
    3218         1652 :       break;
    3219         4025 :     case BUILT_IN_MEMSET_CHK:
    3220         4025 :       fn = builtin_decl_explicit (BUILT_IN_MEMSET);
    3221         4025 :       break;
    3222              :     default:
    3223              :       break;
    3224              :     }
    3225              : 
    3226         8526 :   if (!fn)
    3227              :     return false;
    3228              : 
    3229         8526 :   gimple *repl = gimple_build_call (fn, 3, dest, src, len);
    3230         8526 :   replace_call_with_call_and_fold (gsi, repl);
    3231         8526 :   return true;
    3232              : }
    3233              : 
    3234              : /* Fold a call to the __st[rp]cpy_chk builtin.
    3235              :    DEST, SRC, and SIZE are the arguments to the call.
    3236              :    IGNORE is true if return value can be ignored.  FCODE is the BUILT_IN_*
    3237              :    code of the builtin.  If MAXLEN is not NULL, it is maximum length of
    3238              :    strings passed as second argument.  */
    3239              : 
    3240              : static bool
    3241         2566 : gimple_fold_builtin_stxcpy_chk (gimple_stmt_iterator *gsi,
    3242              :                                 tree dest,
    3243              :                                 tree src, tree size,
    3244              :                                 enum built_in_function fcode)
    3245              : {
    3246         2566 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3247         2566 :   location_t loc = gimple_location (stmt);
    3248         2566 :   bool ignore = gimple_call_lhs (stmt) == NULL_TREE;
    3249         2566 :   tree len, fn;
    3250              : 
    3251              :   /* If SRC and DEST are the same (and not volatile), return DEST.  */
    3252         2566 :   if (fcode == BUILT_IN_STRCPY_CHK && operand_equal_p (src, dest, 0))
    3253              :     {
    3254              :       /* Issue -Wrestrict unless the pointers are null (those do
    3255              :          not point to objects and so do not indicate an overlap;
    3256              :          such calls could be the result of sanitization and jump
    3257              :          threading).  */
    3258            0 :       if (!integer_zerop (dest)
    3259            0 :           && !warning_suppressed_p (stmt, OPT_Wrestrict))
    3260              :         {
    3261            0 :           tree func = gimple_call_fndecl (stmt);
    3262              : 
    3263            0 :           warning_at (loc, OPT_Wrestrict,
    3264              :                       "%qD source argument is the same as destination",
    3265              :                       func);
    3266              :         }
    3267              : 
    3268            0 :       replace_call_with_value (gsi, dest);
    3269            0 :       return true;
    3270              :     }
    3271              : 
    3272         5132 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    3273              :     return false;
    3274              : 
    3275         2566 :   tree maxlen = get_maxval_strlen (src, SRK_STRLENMAX);
    3276         2566 :   if (! integer_all_onesp (size))
    3277              :     {
    3278         2497 :       len = c_strlen (src, 1);
    3279         2497 :       if (!known_lower (stmt, len, size, true)
    3280         2497 :           && !known_lower (stmt, maxlen, size, true))
    3281              :         {
    3282         2185 :           if (fcode == BUILT_IN_STPCPY_CHK)
    3283              :             {
    3284         1077 :               if (! ignore)
    3285              :                 return false;
    3286              : 
    3287              :               /* If return value of __stpcpy_chk is ignored,
    3288              :                  optimize into __strcpy_chk.  */
    3289           35 :               fn = builtin_decl_explicit (BUILT_IN_STRCPY_CHK);
    3290           35 :               if (!fn)
    3291              :                 return false;
    3292              : 
    3293           35 :               gimple *repl = gimple_build_call (fn, 3, dest, src, size);
    3294           35 :               replace_call_with_call_and_fold (gsi, repl);
    3295           35 :               return true;
    3296              :             }
    3297              : 
    3298         1108 :           if (! len || TREE_SIDE_EFFECTS (len))
    3299              :             return false;
    3300              : 
    3301              :           /* If c_strlen returned something, but not provably less than size,
    3302              :              transform __strcpy_chk into __memcpy_chk.  */
    3303          106 :           fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
    3304          106 :           if (!fn)
    3305              :             return false;
    3306              : 
    3307          106 :           gimple_seq stmts = NULL;
    3308          106 :           len = force_gimple_operand (len, &stmts, true, NULL_TREE);
    3309          106 :           len = gimple_convert (&stmts, loc, size_type_node, len);
    3310          106 :           len = gimple_build (&stmts, loc, PLUS_EXPR, size_type_node, len,
    3311              :                               build_int_cst (size_type_node, 1));
    3312          106 :           gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    3313          106 :           gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
    3314          106 :           replace_call_with_call_and_fold (gsi, repl);
    3315          106 :           return true;
    3316              :         }
    3317              :     }
    3318              : 
    3319              :   /* If __builtin_st{r,p}cpy_chk is used, assume st{r,p}cpy is available.  */
    3320          618 :   fn = builtin_decl_explicit (fcode == BUILT_IN_STPCPY_CHK && !ignore
    3321              :                               ? BUILT_IN_STPCPY : BUILT_IN_STRCPY);
    3322          381 :   if (!fn)
    3323              :     return false;
    3324              : 
    3325          381 :   gcall *repl = gimple_build_call (fn, 2, dest, src);
    3326          381 :   replace_call_with_call_and_fold (gsi, repl);
    3327          381 :   return true;
    3328              : }
    3329              : 
    3330              : /* Fold a call to the __st{r,p}ncpy_chk builtin.  DEST, SRC, LEN, and SIZE
    3331              :    are the arguments to the call.  If MAXLEN is not NULL, it is maximum
    3332              :    length passed as third argument. IGNORE is true if return value can be
    3333              :    ignored. FCODE is the BUILT_IN_* code of the builtin. */
    3334              : 
    3335              : static bool
    3336         2721 : gimple_fold_builtin_stxncpy_chk (gimple_stmt_iterator *gsi,
    3337              :                                  tree dest, tree src,
    3338              :                                  tree len, tree size,
    3339              :                                  enum built_in_function fcode)
    3340              : {
    3341         2721 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3342         2721 :   bool ignore = gimple_call_lhs (stmt) == NULL_TREE;
    3343         2721 :   tree fn;
    3344              : 
    3345         2721 :   tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
    3346         2721 :   if (! integer_all_onesp (size)
    3347         2721 :       && !known_lower (stmt, len, size) && !known_lower (stmt, maxlen, size))
    3348              :     {
    3349         2264 :       if (fcode == BUILT_IN_STPNCPY_CHK && ignore)
    3350              :         {
    3351              :           /* If return value of __stpncpy_chk is ignored,
    3352              :              optimize into __strncpy_chk.  */
    3353           39 :           fn = builtin_decl_explicit (BUILT_IN_STRNCPY_CHK);
    3354           39 :           if (fn)
    3355              :             {
    3356           39 :               gimple *repl = gimple_build_call (fn, 4, dest, src, len, size);
    3357           39 :               replace_call_with_call_and_fold (gsi, repl);
    3358           39 :               return true;
    3359              :             }
    3360              :         }
    3361              :       return false;
    3362              :     }
    3363              : 
    3364              :   /* If __builtin_st{r,p}ncpy_chk is used, assume st{r,p}ncpy is available.  */
    3365          717 :   fn = builtin_decl_explicit (fcode == BUILT_IN_STPNCPY_CHK && !ignore
    3366              :                               ? BUILT_IN_STPNCPY : BUILT_IN_STRNCPY);
    3367          914 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3368              :     return false;
    3369              : 
    3370          457 :   gcall *repl = gimple_build_call (fn, 3, dest, src, len);
    3371          457 :   replace_call_with_call_and_fold (gsi, repl);
    3372          457 :   return true;
    3373              : }
    3374              : 
    3375              : /* Fold function call to builtin stpcpy with arguments DEST and SRC.
    3376              :    Return NULL_TREE if no simplification can be made.  */
    3377              : 
    3378              : static bool
    3379         3674 : gimple_fold_builtin_stpcpy (gimple_stmt_iterator *gsi)
    3380              : {
    3381         3674 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3382         3674 :   location_t loc = gimple_location (stmt);
    3383         3674 :   tree dest = gimple_call_arg (stmt, 0);
    3384         3674 :   tree src = gimple_call_arg (stmt, 1);
    3385         3674 :   tree fn, lenp1;
    3386              : 
    3387         7348 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    3388              :     return false;
    3389              : 
    3390              :   /* If the result is unused, replace stpcpy with strcpy.  */
    3391         3674 :   if (gimple_call_lhs (stmt) == NULL_TREE)
    3392              :     {
    3393           29 :       tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
    3394           29 :       if (!fn)
    3395              :         return false;
    3396           29 :       gimple_call_set_fndecl (stmt, fn);
    3397           29 :       fold_stmt (gsi);
    3398           29 :       return true;
    3399              :     }
    3400              : 
    3401              :   /* Set to non-null if ARG refers to an unterminated array.  */
    3402         3645 :   c_strlen_data data = { };
    3403              :   /* The size of the unterminated array if SRC refers to one.  */
    3404         3645 :   tree size;
    3405              :   /* True if the size is exact/constant, false if it's the lower bound
    3406              :      of a range.  */
    3407         3645 :   bool exact;
    3408         3645 :   tree len = c_strlen (src, 1, &data, 1);
    3409         3645 :   if (!len
    3410          703 :       || TREE_CODE (len) != INTEGER_CST)
    3411              :     {
    3412         3174 :       data.decl = unterminated_array (src, &size, &exact);
    3413         3174 :       if (!data.decl)
    3414              :         return false;
    3415              :     }
    3416              : 
    3417         1076 :   if (data.decl)
    3418              :     {
    3419              :       /* Avoid folding calls with unterminated arrays.  */
    3420          605 :       if (!warning_suppressed_p (stmt, OPT_Wstringop_overread))
    3421           75 :         warn_string_no_nul (loc, stmt, "stpcpy", src, data.decl, size,
    3422              :                             exact);
    3423          605 :       suppress_warning (stmt, OPT_Wstringop_overread);
    3424          605 :       return false;
    3425              :     }
    3426              : 
    3427          471 :   if (optimize_function_for_size_p (cfun)
    3428              :       /* If length is zero it's small enough.  */
    3429          471 :       && !integer_zerop (len))
    3430              :     return false;
    3431              : 
    3432              :   /* If the source has a known length replace stpcpy with memcpy.  */
    3433          287 :   fn = builtin_decl_implicit (BUILT_IN_MEMCPY);
    3434          287 :   if (!fn)
    3435              :     return false;
    3436              : 
    3437          287 :   gimple_seq stmts = NULL;
    3438          287 :   tree tem = gimple_convert (&stmts, loc, size_type_node, len);
    3439          287 :   lenp1 = gimple_build (&stmts, loc, PLUS_EXPR, size_type_node,
    3440              :                         tem, build_int_cst (size_type_node, 1));
    3441          287 :   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    3442          287 :   gcall *repl = gimple_build_call (fn, 3, dest, src, lenp1);
    3443          287 :   gimple_move_vops (repl, stmt);
    3444          287 :   gsi_insert_before (gsi, repl, GSI_SAME_STMT);
    3445              :   /* Replace the result with dest + len.  */
    3446          287 :   stmts = NULL;
    3447          287 :   tem = gimple_convert (&stmts, loc, sizetype, len);
    3448          287 :   gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    3449          287 :   gassign *ret = gimple_build_assign (gimple_call_lhs (stmt),
    3450              :                                       POINTER_PLUS_EXPR, dest, tem);
    3451          287 :   gsi_replace (gsi, ret, false);
    3452              :   /* Finally fold the memcpy call.  */
    3453          287 :   gimple_stmt_iterator gsi2 = *gsi;
    3454          287 :   gsi_prev (&gsi2);
    3455          287 :   fold_stmt (&gsi2);
    3456          287 :   return true;
    3457              : }
    3458              : 
    3459              : /* Simplify mempcpy call stmt at GSI, returning true if simplified.
    3460              :    Currently only handling mempcpy -> memcpy when the return value
    3461              :    is ignored.  */
    3462              : 
    3463              : static bool
    3464         9545 : gimple_fold_builtin_mempcpy (gimple_stmt_iterator *gsi)
    3465              : {
    3466         9545 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3467              : 
    3468         9545 :   if (gimple_call_lhs (stmt) != NULL_TREE)
    3469              :     return false;
    3470              : 
    3471          385 :   tree fn = builtin_decl_explicit (BUILT_IN_MEMCPY);
    3472          385 :   if (!fn)
    3473              :     return false;
    3474              : 
    3475          385 :   tree dest = gimple_call_arg (stmt, 0);
    3476          385 :   tree src = gimple_call_arg (stmt, 1);
    3477          385 :   tree n = gimple_call_arg (stmt, 2);
    3478              : 
    3479          385 :   gcall *repl = gimple_build_call (fn, 3, dest, src, n);
    3480          385 :   replace_call_with_call_and_fold (gsi, repl);
    3481              : 
    3482          385 :   return true;
    3483              : }
    3484              : 
    3485              : /* Fold a call EXP to {,v}snprintf having NARGS passed as ARGS.  Return
    3486              :    NULL_TREE if a normal call should be emitted rather than expanding
    3487              :    the function inline.  FCODE is either BUILT_IN_SNPRINTF_CHK or
    3488              :    BUILT_IN_VSNPRINTF_CHK.  If MAXLEN is not NULL, it is maximum length
    3489              :    passed as second argument.  */
    3490              : 
    3491              : static bool
    3492         2359 : gimple_fold_builtin_snprintf_chk (gimple_stmt_iterator *gsi,
    3493              :                                   enum built_in_function fcode)
    3494              : {
    3495         2359 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3496         2359 :   tree dest, size, len, fn, fmt, flag;
    3497         2359 :   const char *fmt_str;
    3498              : 
    3499              :   /* Verify the required arguments in the original call.  */
    3500         2359 :   if (gimple_call_num_args (stmt) < 5)
    3501              :     return false;
    3502              : 
    3503         2359 :   dest = gimple_call_arg (stmt, 0);
    3504         2359 :   len = gimple_call_arg (stmt, 1);
    3505         2359 :   flag = gimple_call_arg (stmt, 2);
    3506         2359 :   size = gimple_call_arg (stmt, 3);
    3507         2359 :   fmt = gimple_call_arg (stmt, 4);
    3508              : 
    3509         2359 :   tree maxlen = get_maxval_strlen (len, SRK_INT_VALUE);
    3510         2359 :   if (! integer_all_onesp (size)
    3511         2359 :       && !known_lower (stmt, len, size) && !known_lower (stmt, maxlen, size))
    3512              :     return false;
    3513              : 
    3514          308 :   if (!init_target_chars ())
    3515              :     return false;
    3516              : 
    3517              :   /* Only convert __{,v}snprintf_chk to {,v}snprintf if flag is 0
    3518              :      or if format doesn't contain % chars or is "%s".  */
    3519          308 :   if (! integer_zerop (flag))
    3520              :     {
    3521           52 :       fmt_str = c_getstr (fmt);
    3522           52 :       if (fmt_str == NULL)
    3523              :         return false;
    3524           52 :       if (strchr (fmt_str, target_percent) != NULL
    3525           51 :           && strcmp (fmt_str, target_percent_s))
    3526              :         return false;
    3527              :     }
    3528              : 
    3529              :   /* If __builtin_{,v}snprintf_chk is used, assume {,v}snprintf is
    3530              :      available.  */
    3531          415 :   fn = builtin_decl_explicit (fcode == BUILT_IN_VSNPRINTF_CHK
    3532              :                               ? BUILT_IN_VSNPRINTF : BUILT_IN_SNPRINTF);
    3533          518 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3534              :     return false;
    3535              : 
    3536              :   /* Replace the called function and the first 5 argument by 3 retaining
    3537              :      trailing varargs.  */
    3538          259 :   gimple_call_set_fndecl (stmt, fn);
    3539          259 :   gimple_call_set_fntype (stmt, TREE_TYPE (fn));
    3540          259 :   gimple_call_set_arg (stmt, 0, dest);
    3541          259 :   gimple_call_set_arg (stmt, 1, len);
    3542          259 :   gimple_call_set_arg (stmt, 2, fmt);
    3543          546 :   for (unsigned i = 3; i < gimple_call_num_args (stmt) - 2; ++i)
    3544          287 :     gimple_call_set_arg (stmt, i, gimple_call_arg (stmt, i + 2));
    3545          259 :   gimple_set_num_ops (stmt, gimple_num_ops (stmt) - 2);
    3546          259 :   fold_stmt (gsi);
    3547          259 :   return true;
    3548              : }
    3549              : 
    3550              : /* Fold a call EXP to __{,v}sprintf_chk having NARGS passed as ARGS.
    3551              :    Return NULL_TREE if a normal call should be emitted rather than
    3552              :    expanding the function inline.  FCODE is either BUILT_IN_SPRINTF_CHK
    3553              :    or BUILT_IN_VSPRINTF_CHK.  */
    3554              : 
    3555              : static bool
    3556         4471 : gimple_fold_builtin_sprintf_chk (gimple_stmt_iterator *gsi,
    3557              :                                  enum built_in_function fcode)
    3558              : {
    3559         4471 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3560         4471 :   tree dest, size, len, fn, fmt, flag;
    3561         4471 :   const char *fmt_str;
    3562         4471 :   unsigned nargs = gimple_call_num_args (stmt);
    3563              : 
    3564              :   /* Verify the required arguments in the original call.  */
    3565         4471 :   if (nargs < 4)
    3566              :     return false;
    3567         4471 :   dest = gimple_call_arg (stmt, 0);
    3568         4471 :   flag = gimple_call_arg (stmt, 1);
    3569         4471 :   size = gimple_call_arg (stmt, 2);
    3570         4471 :   fmt = gimple_call_arg (stmt, 3);
    3571              : 
    3572         4471 :   len = NULL_TREE;
    3573              : 
    3574         4471 :   if (!init_target_chars ())
    3575              :     return false;
    3576              : 
    3577              :   /* Check whether the format is a literal string constant.  */
    3578         4471 :   fmt_str = c_getstr (fmt);
    3579         4471 :   if (fmt_str != NULL)
    3580              :     {
    3581              :       /* If the format doesn't contain % args or %%, we know the size.  */
    3582         4078 :       if (strchr (fmt_str, target_percent) == 0)
    3583              :         {
    3584          251 :           if (fcode != BUILT_IN_SPRINTF_CHK || nargs == 4)
    3585          251 :             len = build_int_cstu (size_type_node, strlen (fmt_str));
    3586              :         }
    3587              :       /* If the format is "%s" and first ... argument is a string literal,
    3588              :          we know the size too.  */
    3589         3827 :       else if (fcode == BUILT_IN_SPRINTF_CHK
    3590         2971 :                && strcmp (fmt_str, target_percent_s) == 0)
    3591              :         {
    3592          395 :           tree arg;
    3593              : 
    3594          395 :           if (nargs == 5)
    3595              :             {
    3596          395 :               arg = gimple_call_arg (stmt, 4);
    3597          395 :               if (POINTER_TYPE_P (TREE_TYPE (arg)))
    3598          363 :                 len = c_strlen (arg, 1);
    3599              :             }
    3600              :         }
    3601              :     }
    3602              : 
    3603         4471 :   if (! integer_all_onesp (size) && !known_lower (stmt, len, size, true))
    3604              :     return false;
    3605              : 
    3606              :   /* Only convert __{,v}sprintf_chk to {,v}sprintf if flag is 0
    3607              :      or if format doesn't contain % chars or is "%s".  */
    3608          202 :   if (! integer_zerop (flag))
    3609              :     {
    3610            1 :       if (fmt_str == NULL)
    3611              :         return false;
    3612            1 :       if (strchr (fmt_str, target_percent) != NULL
    3613            0 :           && strcmp (fmt_str, target_percent_s))
    3614              :         return false;
    3615              :     }
    3616              : 
    3617              :   /* If __builtin_{,v}sprintf_chk is used, assume {,v}sprintf is available.  */
    3618          347 :   fn = builtin_decl_explicit (fcode == BUILT_IN_VSPRINTF_CHK
    3619              :                               ? BUILT_IN_VSPRINTF : BUILT_IN_SPRINTF);
    3620          404 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3621              :     return false;
    3622              : 
    3623              :   /* Replace the called function and the first 4 argument by 2 retaining
    3624              :      trailing varargs.  */
    3625          202 :   gimple_call_set_fndecl (stmt, fn);
    3626          202 :   gimple_call_set_fntype (stmt, TREE_TYPE (fn));
    3627          202 :   gimple_call_set_arg (stmt, 0, dest);
    3628          202 :   gimple_call_set_arg (stmt, 1, fmt);
    3629          400 :   for (unsigned i = 2; i < gimple_call_num_args (stmt) - 2; ++i)
    3630          198 :     gimple_call_set_arg (stmt, i, gimple_call_arg (stmt, i + 2));
    3631          202 :   gimple_set_num_ops (stmt, gimple_num_ops (stmt) - 2);
    3632          202 :   fold_stmt (gsi);
    3633          202 :   return true;
    3634              : }
    3635              : 
    3636              : /* Simplify a call to the sprintf builtin with arguments DEST, FMT, and ORIG.
    3637              :    ORIG may be null if this is a 2-argument call.  We don't attempt to
    3638              :    simplify calls with more than 3 arguments.
    3639              : 
    3640              :    Return true if simplification was possible, otherwise false.  */
    3641              : 
    3642              : bool
    3643         2344 : gimple_fold_builtin_sprintf (gimple_stmt_iterator *gsi)
    3644              : {
    3645         2344 :   gimple *stmt = gsi_stmt (*gsi);
    3646              : 
    3647              :   /* Verify the required arguments in the original call.  We deal with two
    3648              :      types of sprintf() calls: 'sprintf (str, fmt)' and
    3649              :      'sprintf (dest, "%s", orig)'.  */
    3650         2344 :   if (gimple_call_num_args (stmt) > 3)
    3651              :     return false;
    3652              : 
    3653         1943 :   tree orig = NULL_TREE;
    3654         1943 :   if (gimple_call_num_args (stmt) == 3)
    3655         1846 :     orig = gimple_call_arg (stmt, 2);
    3656              : 
    3657              :   /* Check whether the format is a literal string constant.  */
    3658         1943 :   tree fmt = gimple_call_arg (stmt, 1);
    3659         1943 :   const char *fmt_str = c_getstr (fmt);
    3660         1943 :   if (fmt_str == NULL)
    3661              :     return false;
    3662              : 
    3663         1943 :   tree dest = gimple_call_arg (stmt, 0);
    3664              : 
    3665         1943 :   if (!init_target_chars ())
    3666              :     return false;
    3667              : 
    3668         1943 :   tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
    3669         3886 :   if (!fn || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3670              :     return false;
    3671              : 
    3672              :   /* If the format doesn't contain % args or %%, use strcpy.  */
    3673         1943 :   if (strchr (fmt_str, target_percent) == NULL)
    3674              :     {
    3675              :       /* Don't optimize sprintf (buf, "abc", ptr++).  */
    3676          109 :       if (orig)
    3677              :         return false;
    3678              : 
    3679              :       /* Convert sprintf (str, fmt) into strcpy (str, fmt) when
    3680              :          'format' is known to contain no % formats.  */
    3681           96 :       gimple_seq stmts = NULL;
    3682           96 :       gimple *repl = gimple_build_call (fn, 2, dest, fmt);
    3683              : 
    3684              :       /* Propagate the NO_WARNING bit to avoid issuing the same
    3685              :          warning more than once.  */
    3686           96 :       copy_warning (repl, stmt);
    3687              : 
    3688           96 :       gimple_seq_add_stmt_without_update (&stmts, repl);
    3689           96 :       if (tree lhs = gimple_call_lhs (stmt))
    3690              :         {
    3691            0 :           repl = gimple_build_assign (lhs, build_int_cst (TREE_TYPE (lhs),
    3692            0 :                                                           strlen (fmt_str)));
    3693            0 :           gimple_seq_add_stmt_without_update (&stmts, repl);
    3694            0 :           gsi_replace_with_seq_vops (gsi, stmts);
    3695              :           /* gsi now points at the assignment to the lhs, get a
    3696              :              stmt iterator to the memcpy call.
    3697              :              ???  We can't use gsi_for_stmt as that doesn't work when the
    3698              :              CFG isn't built yet.  */
    3699            0 :           gimple_stmt_iterator gsi2 = *gsi;
    3700            0 :           gsi_prev (&gsi2);
    3701            0 :           fold_stmt (&gsi2);
    3702              :         }
    3703              :       else
    3704              :         {
    3705           96 :           gsi_replace_with_seq_vops (gsi, stmts);
    3706           96 :           fold_stmt (gsi);
    3707              :         }
    3708              :       return true;
    3709              :     }
    3710              : 
    3711              :   /* If the format is "%s", use strcpy if the result isn't used.  */
    3712         1834 :   else if (fmt_str && strcmp (fmt_str, target_percent_s) == 0)
    3713              :     {
    3714              :       /* Don't crash on sprintf (str1, "%s").  */
    3715          746 :       if (!orig)
    3716              :         return false;
    3717              : 
    3718              :       /* Don't fold calls with source arguments of invalid (nonpointer)
    3719              :          types.  */
    3720          745 :       if (!POINTER_TYPE_P (TREE_TYPE (orig)))
    3721              :         return false;
    3722              : 
    3723          739 :       tree orig_len = NULL_TREE;
    3724          739 :       if (gimple_call_lhs (stmt))
    3725              :         {
    3726           17 :           orig_len = get_maxval_strlen (orig, SRK_STRLEN);
    3727           17 :           if (!orig_len)
    3728              :             return false;
    3729              :         }
    3730              : 
    3731              :       /* Convert sprintf (str1, "%s", str2) into strcpy (str1, str2).  */
    3732          722 :       gimple_seq stmts = NULL;
    3733          722 :       gimple *repl = gimple_build_call (fn, 2, dest, orig);
    3734              : 
    3735              :       /* Propagate the NO_WARNING bit to avoid issuing the same
    3736              :          warning more than once.  */
    3737          722 :       copy_warning (repl, stmt);
    3738              : 
    3739          722 :       gimple_seq_add_stmt_without_update (&stmts, repl);
    3740          722 :       if (tree lhs = gimple_call_lhs (stmt))
    3741              :         {
    3742            0 :           if (!useless_type_conversion_p (TREE_TYPE (lhs),
    3743            0 :                                           TREE_TYPE (orig_len)))
    3744            0 :             orig_len = fold_convert (TREE_TYPE (lhs), orig_len);
    3745            0 :           repl = gimple_build_assign (lhs, orig_len);
    3746            0 :           gimple_seq_add_stmt_without_update (&stmts, repl);
    3747            0 :           gsi_replace_with_seq_vops (gsi, stmts);
    3748              :           /* gsi now points at the assignment to the lhs, get a
    3749              :              stmt iterator to the memcpy call.
    3750              :              ???  We can't use gsi_for_stmt as that doesn't work when the
    3751              :              CFG isn't built yet.  */
    3752            0 :           gimple_stmt_iterator gsi2 = *gsi;
    3753            0 :           gsi_prev (&gsi2);
    3754            0 :           fold_stmt (&gsi2);
    3755              :         }
    3756              :       else
    3757              :         {
    3758          722 :           gsi_replace_with_seq_vops (gsi, stmts);
    3759          722 :           fold_stmt (gsi);
    3760              :         }
    3761              :       return true;
    3762              :     }
    3763              :   return false;
    3764              : }
    3765              : 
    3766              : /* Simplify a call to the snprintf builtin with arguments DEST, DESTSIZE,
    3767              :    FMT, and ORIG.  ORIG may be null if this is a 3-argument call.  We don't
    3768              :    attempt to simplify calls with more than 4 arguments.
    3769              : 
    3770              :    Return true if simplification was possible, otherwise false.  */
    3771              : 
    3772              : bool
    3773         1878 : gimple_fold_builtin_snprintf (gimple_stmt_iterator *gsi)
    3774              : {
    3775         1878 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3776         1878 :   tree dest = gimple_call_arg (stmt, 0);
    3777         1878 :   tree destsize = gimple_call_arg (stmt, 1);
    3778         1878 :   tree fmt = gimple_call_arg (stmt, 2);
    3779         1878 :   tree orig = NULL_TREE;
    3780         1878 :   const char *fmt_str = NULL;
    3781              : 
    3782         1878 :   if (gimple_call_num_args (stmt) > 4
    3783         2784 :       || (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun)))
    3784              :     return false;
    3785              : 
    3786          903 :   if (gimple_call_num_args (stmt) == 4)
    3787          704 :     orig = gimple_call_arg (stmt, 3);
    3788              : 
    3789              :   /* Check whether the format is a literal string constant.  */
    3790          903 :   fmt_str = c_getstr (fmt);
    3791          903 :   if (fmt_str == NULL)
    3792              :     return false;
    3793              : 
    3794          903 :   if (!init_target_chars ())
    3795              :     return false;
    3796              : 
    3797              :   /* If the format doesn't contain % args or %%, use strcpy.  */
    3798          903 :   if (strchr (fmt_str, target_percent) == NULL)
    3799              :     {
    3800          396 :       tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
    3801          198 :       if (!fn)
    3802              :         return false;
    3803              : 
    3804              :       /* Don't optimize snprintf (buf, 4, "abc", ptr++).  */
    3805          198 :       if (orig)
    3806              :         return false;
    3807              : 
    3808          198 :       tree len = build_int_cstu (TREE_TYPE (destsize), strlen (fmt_str));
    3809              : 
    3810              :       /* We could expand this as
    3811              :          memcpy (str, fmt, cst - 1); str[cst - 1] = '\0';
    3812              :          or to
    3813              :          memcpy (str, fmt_with_nul_at_cstm1, cst);
    3814              :          but in the former case that might increase code size
    3815              :          and in the latter case grow .rodata section too much.
    3816              :          So punt for now.  */
    3817          198 :       if (!known_lower (stmt, len, destsize, true))
    3818              :         return false;
    3819              : 
    3820          168 :       gimple_seq stmts = NULL;
    3821          168 :       gimple *repl = gimple_build_call (fn, 2, dest, fmt);
    3822          168 :       gimple_seq_add_stmt_without_update (&stmts, repl);
    3823          168 :       if (tree lhs = gimple_call_lhs (stmt))
    3824              :         {
    3825            0 :           repl = gimple_build_assign (lhs,
    3826            0 :                                       fold_convert (TREE_TYPE (lhs), len));
    3827            0 :           gimple_seq_add_stmt_without_update (&stmts, repl);
    3828            0 :           gsi_replace_with_seq_vops (gsi, stmts);
    3829              :           /* gsi now points at the assignment to the lhs, get a
    3830              :              stmt iterator to the memcpy call.
    3831              :              ???  We can't use gsi_for_stmt as that doesn't work when the
    3832              :              CFG isn't built yet.  */
    3833            0 :           gimple_stmt_iterator gsi2 = *gsi;
    3834            0 :           gsi_prev (&gsi2);
    3835            0 :           fold_stmt (&gsi2);
    3836              :         }
    3837              :       else
    3838              :         {
    3839          168 :           gsi_replace_with_seq_vops (gsi, stmts);
    3840          168 :           fold_stmt (gsi);
    3841              :         }
    3842              :       return true;
    3843              :     }
    3844              : 
    3845              :   /* If the format is "%s", use strcpy if the result isn't used.  */
    3846          705 :   else if (fmt_str && strcmp (fmt_str, target_percent_s) == 0)
    3847              :     {
    3848          516 :       tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
    3849          258 :       if (!fn)
    3850              :         return false;
    3851              : 
    3852              :       /* Don't crash on snprintf (str1, cst, "%s").  */
    3853          258 :       if (!orig)
    3854              :         return false;
    3855              : 
    3856          258 :       tree orig_len = get_maxval_strlen (orig, SRK_STRLEN);
    3857              : 
    3858              :       /* We could expand this as
    3859              :          memcpy (str1, str2, cst - 1); str1[cst - 1] = '\0';
    3860              :          or to
    3861              :          memcpy (str1, str2_with_nul_at_cstm1, cst);
    3862              :          but in the former case that might increase code size
    3863              :          and in the latter case grow .rodata section too much.
    3864              :          So punt for now.  */
    3865          258 :       if (!known_lower (stmt, orig_len, destsize, true))
    3866              :         return false;
    3867              : 
    3868              :       /* Convert snprintf (str1, cst, "%s", str2) into
    3869              :          strcpy (str1, str2) if strlen (str2) < cst.  */
    3870           64 :       gimple_seq stmts = NULL;
    3871           64 :       gimple *repl = gimple_build_call (fn, 2, dest, orig);
    3872           64 :       gimple_seq_add_stmt_without_update (&stmts, repl);
    3873           64 :       if (tree lhs = gimple_call_lhs (stmt))
    3874              :         {
    3875            0 :           if (!useless_type_conversion_p (TREE_TYPE (lhs),
    3876            0 :                                           TREE_TYPE (orig_len)))
    3877            0 :             orig_len = fold_convert (TREE_TYPE (lhs), orig_len);
    3878            0 :           repl = gimple_build_assign (lhs, orig_len);
    3879            0 :           gimple_seq_add_stmt_without_update (&stmts, repl);
    3880            0 :           gsi_replace_with_seq_vops (gsi, stmts);
    3881              :           /* gsi now points at the assignment to the lhs, get a
    3882              :              stmt iterator to the memcpy call.
    3883              :              ???  We can't use gsi_for_stmt as that doesn't work when the
    3884              :              CFG isn't built yet.  */
    3885            0 :           gimple_stmt_iterator gsi2 = *gsi;
    3886            0 :           gsi_prev (&gsi2);
    3887            0 :           fold_stmt (&gsi2);
    3888              :         }
    3889              :       else
    3890              :         {
    3891           64 :           gsi_replace_with_seq_vops (gsi, stmts);
    3892           64 :           fold_stmt (gsi);
    3893              :         }
    3894              :       return true;
    3895              :     }
    3896              :   return false;
    3897              : }
    3898              : 
    3899              : /* Fold a call to the {,v}fprintf{,_unlocked} and __{,v}printf_chk builtins.
    3900              :    FP, FMT, and ARG are the arguments to the call.  We don't fold calls with
    3901              :    more than 3 arguments, and ARG may be null in the 2-argument case.
    3902              : 
    3903              :    Return NULL_TREE if no simplification was possible, otherwise return the
    3904              :    simplified form of the call as a tree.  FCODE is the BUILT_IN_*
    3905              :    code of the function to be simplified.  */
    3906              : 
    3907              : static bool
    3908        54660 : gimple_fold_builtin_fprintf (gimple_stmt_iterator *gsi,
    3909              :                              tree fp, tree fmt, tree arg,
    3910              :                              enum built_in_function fcode)
    3911              : {
    3912        54660 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    3913        54660 :   tree fn_fputc, fn_fputs;
    3914        54660 :   const char *fmt_str = NULL;
    3915              : 
    3916              :   /* If the return value is used, don't do the transformation.  */
    3917        54660 :   if (gimple_call_lhs (stmt) != NULL_TREE)
    3918              :     return false;
    3919              : 
    3920              :   /* Check whether the format is a literal string constant.  */
    3921        50483 :   fmt_str = c_getstr (fmt);
    3922        50483 :   if (fmt_str == NULL)
    3923              :     return false;
    3924              : 
    3925        50153 :   if (fcode == BUILT_IN_FPRINTF_UNLOCKED)
    3926              :     {
    3927              :       /* If we're using an unlocked function, assume the other
    3928              :          unlocked functions exist explicitly.  */
    3929           80 :       fn_fputc = builtin_decl_explicit (BUILT_IN_FPUTC_UNLOCKED);
    3930           80 :       fn_fputs = builtin_decl_explicit (BUILT_IN_FPUTS_UNLOCKED);
    3931              :     }
    3932              :   else
    3933              :     {
    3934        50073 :       fn_fputc = builtin_decl_implicit (BUILT_IN_FPUTC);
    3935        50073 :       fn_fputs = builtin_decl_implicit (BUILT_IN_FPUTS);
    3936              :     }
    3937              : 
    3938        50153 :   if (!init_target_chars ())
    3939              :     return false;
    3940              : 
    3941       100306 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    3942              :     return false;
    3943              : 
    3944              :   /* If the format doesn't contain % args or %%, use strcpy.  */
    3945        50153 :   if (strchr (fmt_str, target_percent) == NULL)
    3946              :     {
    3947         9627 :       if (fcode != BUILT_IN_VFPRINTF && fcode != BUILT_IN_VFPRINTF_CHK
    3948         9557 :           && arg)
    3949              :         return false;
    3950              : 
    3951              :       /* If the format specifier was "", fprintf does nothing.  */
    3952         9627 :       if (fmt_str[0] == '\0')
    3953              :         {
    3954           58 :           replace_call_with_value (gsi, NULL_TREE);
    3955           58 :           return true;
    3956              :         }
    3957              : 
    3958              :       /* When "string" doesn't contain %, replace all cases of
    3959              :          fprintf (fp, string) with fputs (string, fp).  The fputs
    3960              :          builtin will take care of special cases like length == 1.  */
    3961         9569 :       if (fn_fputs)
    3962              :         {
    3963         9569 :           gcall *repl = gimple_build_call (fn_fputs, 2, fmt, fp);
    3964         9569 :           replace_call_with_call_and_fold (gsi, repl);
    3965         9569 :           return true;
    3966              :         }
    3967              :     }
    3968              : 
    3969              :   /* The other optimizations can be done only on the non-va_list variants.  */
    3970        40526 :   else if (fcode == BUILT_IN_VFPRINTF || fcode == BUILT_IN_VFPRINTF_CHK)
    3971              :     return false;
    3972              : 
    3973              :   /* If the format specifier was "%s", call __builtin_fputs (arg, fp).  */
    3974        39477 :   else if (strcmp (fmt_str, target_percent_s) == 0)
    3975              :     {
    3976          643 :       if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
    3977              :         return false;
    3978          643 :       if (fn_fputs)
    3979              :         {
    3980          643 :           gcall *repl = gimple_build_call (fn_fputs, 2, arg, fp);
    3981          643 :           replace_call_with_call_and_fold (gsi, repl);
    3982          643 :           return true;
    3983              :         }
    3984              :     }
    3985              : 
    3986              :   /* If the format specifier was "%c", call __builtin_fputc (arg, fp).  */
    3987        38834 :   else if (strcmp (fmt_str, target_percent_c) == 0)
    3988              :     {
    3989           49 :       if (!arg
    3990           49 :           || ! useless_type_conversion_p (integer_type_node, TREE_TYPE (arg)))
    3991              :         return false;
    3992           49 :       if (fn_fputc)
    3993              :         {
    3994           49 :           gcall *repl = gimple_build_call (fn_fputc, 2, arg, fp);
    3995           49 :           replace_call_with_call_and_fold (gsi, repl);
    3996           49 :           return true;
    3997              :         }
    3998              :     }
    3999              : 
    4000              :   return false;
    4001              : }
    4002              : 
    4003              : /* Fold a call to the {,v}printf{,_unlocked} and __{,v}printf_chk builtins.
    4004              :    FMT and ARG are the arguments to the call; we don't fold cases with
    4005              :    more than 2 arguments, and ARG may be null if this is a 1-argument case.
    4006              : 
    4007              :    Return NULL_TREE if no simplification was possible, otherwise return the
    4008              :    simplified form of the call as a tree.  FCODE is the BUILT_IN_*
    4009              :    code of the function to be simplified.  */
    4010              : 
    4011              : static bool
    4012       123990 : gimple_fold_builtin_printf (gimple_stmt_iterator *gsi, tree fmt,
    4013              :                             tree arg, enum built_in_function fcode)
    4014              : {
    4015       123990 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    4016       123990 :   tree fn_putchar, fn_puts, newarg;
    4017       123990 :   const char *fmt_str = NULL;
    4018              : 
    4019              :   /* If the return value is used, don't do the transformation.  */
    4020       123990 :   if (gimple_call_lhs (stmt) != NULL_TREE)
    4021              :     return false;
    4022              : 
    4023       241464 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    4024              :     return false;
    4025              : 
    4026              :   /* Check whether the format is a literal string constant.  */
    4027       120732 :   fmt_str = c_getstr (fmt);
    4028       120732 :   if (fmt_str == NULL)
    4029              :     return false;
    4030              : 
    4031       117695 :   if (fcode == BUILT_IN_PRINTF_UNLOCKED)
    4032              :     {
    4033              :       /* If we're using an unlocked function, assume the other
    4034              :          unlocked functions exist explicitly.  */
    4035           80 :       fn_putchar = builtin_decl_explicit (BUILT_IN_PUTCHAR_UNLOCKED);
    4036           80 :       fn_puts = builtin_decl_explicit (BUILT_IN_PUTS_UNLOCKED);
    4037              :     }
    4038              :   else
    4039              :     {
    4040       117615 :       fn_putchar = builtin_decl_implicit (BUILT_IN_PUTCHAR);
    4041       117615 :       fn_puts = builtin_decl_implicit (BUILT_IN_PUTS);
    4042              :     }
    4043              : 
    4044       117695 :   if (!init_target_chars ())
    4045              :     return false;
    4046              : 
    4047       117695 :   if (strcmp (fmt_str, target_percent_s) == 0
    4048       111309 :       || strchr (fmt_str, target_percent) == NULL)
    4049              :     {
    4050        14218 :       const char *str;
    4051              : 
    4052        14218 :       if (strcmp (fmt_str, target_percent_s) == 0)
    4053              :         {
    4054         6386 :           if (fcode == BUILT_IN_VPRINTF || fcode == BUILT_IN_VPRINTF_CHK)
    4055              :             return false;
    4056              : 
    4057         6098 :           if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
    4058              :             return false;
    4059              : 
    4060         6093 :           str = c_getstr (arg);
    4061         6093 :           if (str == NULL)
    4062              :             return false;
    4063              :         }
    4064              :       else
    4065              :         {
    4066              :           /* The format specifier doesn't contain any '%' characters.  */
    4067         7832 :           if (fcode != BUILT_IN_VPRINTF && fcode != BUILT_IN_VPRINTF_CHK
    4068         7700 :               && arg)
    4069              :             return false;
    4070              :           str = fmt_str;
    4071              :         }
    4072              : 
    4073              :       /* If the string was "", printf does nothing.  */
    4074         5783 :       if (str[0] == '\0')
    4075              :         {
    4076          109 :           replace_call_with_value (gsi, NULL_TREE);
    4077          109 :           return true;
    4078              :         }
    4079              : 
    4080              :       /* If the string has length of 1, call putchar.  */
    4081         5674 :       if (str[1] == '\0')
    4082              :         {
    4083              :           /* Given printf("c"), (where c is any one character,)
    4084              :              convert "c"[0] to an int and pass that to the replacement
    4085              :              function.  */
    4086          559 :           newarg = build_int_cst (integer_type_node, str[0]);
    4087          559 :           if (fn_putchar)
    4088              :             {
    4089          559 :               gcall *repl = gimple_build_call (fn_putchar, 1, newarg);
    4090          559 :               replace_call_with_call_and_fold (gsi, repl);
    4091          559 :               return true;
    4092              :             }
    4093              :         }
    4094              :       else
    4095              :         {
    4096              :           /* If the string was "string\n", call puts("string").  */
    4097         5115 :           size_t len = strlen (str);
    4098         5115 :           if ((unsigned char)str[len - 1] == target_newline
    4099         4015 :               && (size_t) (int) len == len
    4100         4015 :               && (int) len > 0)
    4101              :             {
    4102         4015 :               char *newstr;
    4103              : 
    4104              :               /* Create a NUL-terminated string that's one char shorter
    4105              :                  than the original, stripping off the trailing '\n'.  */
    4106         4015 :               newstr = xstrdup (str);
    4107         4015 :               newstr[len - 1] = '\0';
    4108         4015 :               newarg = build_string_literal (len, newstr);
    4109         4015 :               free (newstr);
    4110         4015 :               if (fn_puts)
    4111              :                 {
    4112         4015 :                   gcall *repl = gimple_build_call (fn_puts, 1, newarg);
    4113         4015 :                   replace_call_with_call_and_fold (gsi, repl);
    4114         4015 :                   return true;
    4115              :                 }
    4116              :             }
    4117              :           else
    4118              :             /* We'd like to arrange to call fputs(string,stdout) here,
    4119              :                but we need stdout and don't have a way to get it yet.  */
    4120              :             return false;
    4121              :         }
    4122              :     }
    4123              : 
    4124              :   /* The other optimizations can be done only on the non-va_list variants.  */
    4125       103477 :   else if (fcode == BUILT_IN_VPRINTF || fcode == BUILT_IN_VPRINTF_CHK)
    4126              :     return false;
    4127              : 
    4128              :   /* If the format specifier was "%s\n", call __builtin_puts(arg).  */
    4129       103227 :   else if (strcmp (fmt_str, target_percent_s_newline) == 0)
    4130              :     {
    4131          177 :       if (!arg || ! POINTER_TYPE_P (TREE_TYPE (arg)))
    4132              :         return false;
    4133          177 :       if (fn_puts)
    4134              :         {
    4135          177 :           gcall *repl = gimple_build_call (fn_puts, 1, arg);
    4136          177 :           replace_call_with_call_and_fold (gsi, repl);
    4137          177 :           return true;
    4138              :         }
    4139              :     }
    4140              : 
    4141              :   /* If the format specifier was "%c", call __builtin_putchar(arg).  */
    4142       103050 :   else if (strcmp (fmt_str, target_percent_c) == 0)
    4143              :     {
    4144           94 :       if (!arg || ! useless_type_conversion_p (integer_type_node,
    4145           47 :                                                TREE_TYPE (arg)))
    4146              :         return false;
    4147           47 :       if (fn_putchar)
    4148              :         {
    4149           47 :           gcall *repl = gimple_build_call (fn_putchar, 1, arg);
    4150           47 :           replace_call_with_call_and_fold (gsi, repl);
    4151           47 :           return true;
    4152              :         }
    4153              :     }
    4154              : 
    4155              :   return false;
    4156              : }
    4157              : 
    4158              : 
    4159              : 
    4160              : /* Fold a call to __builtin_strlen with known length LEN.  */
    4161              : 
    4162              : static bool
    4163       140552 : gimple_fold_builtin_strlen (gimple_stmt_iterator *gsi)
    4164              : {
    4165       140552 :   gimple *stmt = gsi_stmt (*gsi);
    4166       140552 :   tree arg = gimple_call_arg (stmt, 0);
    4167              : 
    4168       140552 :   wide_int minlen;
    4169       140552 :   wide_int maxlen;
    4170              : 
    4171       140552 :   c_strlen_data lendata = { };
    4172       140552 :   if (get_range_strlen (arg, &lendata, /* eltsize = */ 1)
    4173        35209 :       && !lendata.decl
    4174        32246 :       && lendata.minlen && TREE_CODE (lendata.minlen) == INTEGER_CST
    4175       172693 :       && lendata.maxlen && TREE_CODE (lendata.maxlen) == INTEGER_CST)
    4176              :     {
    4177              :       /* The range of lengths refers to either a single constant
    4178              :          string or to the longest and shortest constant string
    4179              :          referenced by the argument of the strlen() call, or to
    4180              :          the strings that can possibly be stored in the arrays
    4181              :          the argument refers to.  */
    4182        32141 :       minlen = wi::to_wide (lendata.minlen);
    4183        32141 :       maxlen = wi::to_wide (lendata.maxlen);
    4184              :     }
    4185              :   else
    4186              :     {
    4187       108411 :       unsigned prec = TYPE_PRECISION (sizetype);
    4188              : 
    4189       108411 :       minlen = wi::shwi (0, prec);
    4190       108411 :       maxlen = wi::to_wide (max_object_size (), prec) - 2;
    4191              :     }
    4192              : 
    4193              :   /* For -fsanitize=address, don't optimize the upper bound of the
    4194              :      length to be able to diagnose UB on non-zero terminated arrays.  */
    4195       140552 :   if (sanitize_flags_p (SANITIZE_ADDRESS))
    4196          278 :     maxlen = wi::max_value (TYPE_PRECISION (sizetype), UNSIGNED);
    4197              : 
    4198       140552 :   if (minlen == maxlen)
    4199              :     {
    4200              :       /* Fold the strlen call to a constant.  */
    4201         1654 :       tree type = TREE_TYPE (lendata.minlen);
    4202         3308 :       tree len = force_gimple_operand_gsi (gsi,
    4203         1654 :                                            wide_int_to_tree (type, minlen),
    4204              :                                            true, NULL, true, GSI_SAME_STMT);
    4205         1654 :       replace_call_with_value (gsi, len);
    4206         1654 :       return true;
    4207              :     }
    4208              : 
    4209              :   /* Set the strlen() range to [0, MAXLEN].  */
    4210       138898 :   if (tree lhs = gimple_call_lhs (stmt))
    4211       138894 :     set_strlen_range (lhs, minlen, maxlen);
    4212              : 
    4213              :   return false;
    4214       140552 : }
    4215              : 
    4216              : static bool
    4217          262 : gimple_fold_builtin_omp_is_initial_device (gimple_stmt_iterator *gsi)
    4218              : {
    4219              : #if ACCEL_COMPILER
    4220              :   replace_call_with_value (gsi, integer_zero_node);
    4221              :   return true;
    4222              : #else
    4223          262 :   if (!ENABLE_OFFLOADING || symtab->state == EXPANSION)
    4224              :     {
    4225            0 :       replace_call_with_value (gsi, integer_one_node);
    4226          262 :       return true;
    4227              :     }
    4228              : #endif
    4229              :   return false;
    4230              : }
    4231              : 
    4232              : /* omp_get_initial_device was in OpenMP 5.0/5.1 explicitly and in
    4233              :    5.0 implicitly the same as omp_get_num_devices; since 6.0 it is
    4234              :    unspecified whether -1 or omp_get_num_devices() is returned.  For
    4235              :    better backward compatibility, use omp_get_num_devices() on the
    4236              :    host - and -1 on the device (where the result is unspecified).  */
    4237              : 
    4238              : static bool
    4239          103 : gimple_fold_builtin_omp_get_initial_device (gimple_stmt_iterator *gsi)
    4240              : {
    4241              : #if ACCEL_COMPILER
    4242              :   replace_call_with_value (gsi, build_int_cst (integer_type_node, -1));
    4243              : #else
    4244          103 :   if (!ENABLE_OFFLOADING)
    4245            0 :     replace_call_with_value (gsi, integer_zero_node);
    4246              :   else
    4247              :     {
    4248              :       tree fn = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_DEVICES);
    4249              :       gcall *repl = gimple_build_call (fn, 0);
    4250              :       replace_call_with_call_and_fold (gsi, repl);
    4251              :     }
    4252              : #endif
    4253          103 :   return true;
    4254              : }
    4255              : 
    4256              : static bool
    4257          340 : gimple_fold_builtin_omp_get_num_devices (gimple_stmt_iterator *gsi)
    4258              : {
    4259          340 :   if (!ENABLE_OFFLOADING)
    4260              :     {
    4261            0 :       replace_call_with_value (gsi, integer_zero_node);
    4262          340 :       return true;
    4263              :     }
    4264              :   return false;
    4265              : }
    4266              : 
    4267              : /* Fold a call to __builtin_acc_on_device.  */
    4268              : 
    4269              : static bool
    4270         2866 : gimple_fold_builtin_acc_on_device (gimple_stmt_iterator *gsi, tree arg0)
    4271              : {
    4272              :   /* Defer folding until we know which compiler we're in.  */
    4273         2866 :   if (symtab->state != EXPANSION)
    4274              :     return false;
    4275              : 
    4276          554 :   unsigned val_host = GOMP_DEVICE_HOST;
    4277          554 :   unsigned val_dev = GOMP_DEVICE_NONE;
    4278              : 
    4279              : #ifdef ACCEL_COMPILER
    4280              :   val_host = GOMP_DEVICE_NOT_HOST;
    4281              :   val_dev = ACCEL_COMPILER_acc_device;
    4282              : #endif
    4283              : 
    4284          554 :   location_t loc = gimple_location (gsi_stmt (*gsi));
    4285              : 
    4286          554 :   tree host_eq = make_ssa_name (boolean_type_node);
    4287          554 :   gimple *host_ass = gimple_build_assign
    4288          554 :     (host_eq, EQ_EXPR, arg0, build_int_cst (TREE_TYPE (arg0), val_host));
    4289          554 :   gimple_set_location (host_ass, loc);
    4290          554 :   gsi_insert_before (gsi, host_ass, GSI_SAME_STMT);
    4291              : 
    4292          554 :   tree dev_eq = make_ssa_name (boolean_type_node);
    4293          554 :   gimple *dev_ass = gimple_build_assign
    4294          554 :     (dev_eq, EQ_EXPR, arg0, build_int_cst (TREE_TYPE (arg0), val_dev));
    4295          554 :   gimple_set_location (dev_ass, loc);
    4296          554 :   gsi_insert_before (gsi, dev_ass, GSI_SAME_STMT);
    4297              : 
    4298          554 :   tree result = make_ssa_name (boolean_type_node);
    4299          554 :   gimple *result_ass = gimple_build_assign
    4300          554 :     (result, BIT_IOR_EXPR, host_eq, dev_eq);
    4301          554 :   gimple_set_location (result_ass, loc);
    4302          554 :   gsi_insert_before (gsi, result_ass, GSI_SAME_STMT);
    4303              : 
    4304          554 :   replace_call_with_value (gsi, result);
    4305              : 
    4306          554 :   return true;
    4307              : }
    4308              : 
    4309              : /* Fold realloc (0, n) -> malloc (n).  */
    4310              : 
    4311              : static bool
    4312        50579 : gimple_fold_builtin_realloc (gimple_stmt_iterator *gsi)
    4313              : {
    4314        50579 :   gimple *stmt = gsi_stmt (*gsi);
    4315        50579 :   tree arg = gimple_call_arg (stmt, 0);
    4316        50579 :   tree size = gimple_call_arg (stmt, 1);
    4317              : 
    4318       101158 :   if (!gimple_vdef (stmt) && gimple_in_ssa_p (cfun))
    4319              :     return false;
    4320              : 
    4321        50579 :   if (operand_equal_p (arg, null_pointer_node, 0))
    4322              :     {
    4323         1423 :       tree fn_malloc = builtin_decl_implicit (BUILT_IN_MALLOC);
    4324         1423 :       if (fn_malloc)
    4325              :         {
    4326         1423 :           gcall *repl = gimple_build_call (fn_malloc, 1, size);
    4327         1423 :           replace_call_with_call_and_fold (gsi, repl);
    4328         1423 :           return true;
    4329              :         }
    4330              :     }
    4331              :   return false;
    4332              : }
    4333              : 
    4334              : /* Number of bytes into which any type but aggregate, vector or
    4335              :    _BitInt types should fit.  */
    4336              : static constexpr size_t clear_padding_unit
    4337              :   = MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT;
    4338              : /* Buffer size on which __builtin_clear_padding folding code works.  */
    4339              : static const size_t clear_padding_buf_size = 32 * clear_padding_unit;
    4340              : 
    4341              : /* Data passed through __builtin_clear_padding folding.  */
    4342              : struct clear_padding_struct {
    4343              :   location_t loc;
    4344              :   /* 0 during __builtin_clear_padding folding, nonzero during
    4345              :      clear_type_padding_in_mask.  In that case, instead of clearing the
    4346              :      non-padding bits in union_ptr array clear the padding bits in there.  */
    4347              :   bool clear_in_mask;
    4348              :   tree base;
    4349              :   tree alias_type;
    4350              :   gimple_stmt_iterator *gsi;
    4351              :   /* Alignment of buf->base + 0.  */
    4352              :   unsigned align;
    4353              :   /* Offset from buf->base.  Should be always a multiple of UNITS_PER_WORD.  */
    4354              :   HOST_WIDE_INT off;
    4355              :   /* Number of padding bytes before buf->off that don't have padding clear
    4356              :      code emitted yet.  */
    4357              :   HOST_WIDE_INT padding_bytes;
    4358              :   /* The size of the whole object.  Never emit code to touch
    4359              :      buf->base + buf->sz or following bytes.  */
    4360              :   HOST_WIDE_INT sz;
    4361              :   /* Number of bytes recorded in buf->buf.  */
    4362              :   size_t size;
    4363              :   /* When inside union, instead of emitting code we and bits inside of
    4364              :      the union_ptr array.  */
    4365              :   unsigned char *union_ptr;
    4366              :   /* Set bits mean padding bits that need to be cleared by the builtin.  */
    4367              :   unsigned char buf[clear_padding_buf_size + clear_padding_unit];
    4368              : };
    4369              : 
    4370              : /* Emit code to clear padding requested in BUF->buf - set bits
    4371              :    in there stand for padding that should be cleared.  FULL is true
    4372              :    if everything from the buffer should be flushed, otherwise
    4373              :    it can leave up to 2 * clear_padding_unit bytes for further
    4374              :    processing.  */
    4375              : 
    4376              : static void
    4377        40090 : clear_padding_flush (clear_padding_struct *buf, bool full)
    4378              : {
    4379        40090 :   gcc_assert ((clear_padding_unit % UNITS_PER_WORD) == 0);
    4380        40090 :   if (!full && buf->size < 2 * clear_padding_unit)
    4381              :     return;
    4382        41135 :   gcc_assert ((buf->off % UNITS_PER_WORD) == 0);
    4383        40048 :   size_t end = buf->size;
    4384        40048 :   if (!full)
    4385           42 :     end = ((end - clear_padding_unit - 1) / clear_padding_unit
    4386              :            * clear_padding_unit);
    4387        40048 :   size_t padding_bytes = buf->padding_bytes;
    4388        40048 :   if (buf->union_ptr)
    4389              :     {
    4390        39278 :       if (buf->clear_in_mask)
    4391              :         {
    4392              :           /* During clear_type_padding_in_mask, clear the padding
    4393              :              bits set in buf->buf in the buf->union_ptr mask.  */
    4394       339406 :           for (size_t i = 0; i < end; i++)
    4395              :             {
    4396       300521 :               if (buf->buf[i] == (unsigned char) ~0)
    4397         9052 :                 padding_bytes++;
    4398              :               else
    4399              :                 {
    4400       291469 :                   memset (&buf->union_ptr[buf->off + i - padding_bytes],
    4401              :                           0, padding_bytes);
    4402       291469 :                   padding_bytes = 0;
    4403       291469 :                   buf->union_ptr[buf->off + i] &= ~buf->buf[i];
    4404              :                 }
    4405              :             }
    4406        38885 :           if (full)
    4407              :             {
    4408        38885 :               memset (&buf->union_ptr[buf->off + end - padding_bytes],
    4409              :                       0, padding_bytes);
    4410        38885 :               buf->off = 0;
    4411        38885 :               buf->size = 0;
    4412        38885 :               buf->padding_bytes = 0;
    4413              :             }
    4414              :           else
    4415              :             {
    4416            0 :               memmove (buf->buf, buf->buf + end, buf->size - end);
    4417            0 :               buf->off += end;
    4418            0 :               buf->size -= end;
    4419            0 :               buf->padding_bytes = padding_bytes;
    4420              :             }
    4421        38885 :           return;
    4422              :         }
    4423              :       /* Inside of a union, instead of emitting any code, instead
    4424              :          clear all bits in the union_ptr buffer that are clear
    4425              :          in buf.  Whole padding bytes don't clear anything.  */
    4426         3017 :       for (size_t i = 0; i < end; i++)
    4427              :         {
    4428         2624 :           if (buf->buf[i] == (unsigned char) ~0)
    4429         1424 :             padding_bytes++;
    4430              :           else
    4431              :             {
    4432         1200 :               padding_bytes = 0;
    4433         1200 :               buf->union_ptr[buf->off + i] &= buf->buf[i];
    4434              :             }
    4435              :         }
    4436          393 :       if (full)
    4437              :         {
    4438              :           buf->off = 0;
    4439              :           buf->size = 0;
    4440              :           buf->padding_bytes = 0;
    4441              :         }
    4442              :       else
    4443              :         {
    4444            0 :           memmove (buf->buf, buf->buf + end, buf->size - end);
    4445            0 :           buf->off += end;
    4446            0 :           buf->size -= end;
    4447            0 :           buf->padding_bytes = padding_bytes;
    4448              :         }
    4449          393 :       return;
    4450              :     }
    4451          770 :   size_t wordsize = UNITS_PER_WORD;
    4452        23525 :   for (size_t i = 0; i < end; i += wordsize)
    4453              :     {
    4454        22755 :       size_t nonzero_first = wordsize;
    4455        22755 :       size_t nonzero_last = 0;
    4456        22755 :       size_t zero_first = wordsize;
    4457        22755 :       size_t zero_last = 0;
    4458        22755 :       bool all_ones = true, bytes_only = true;
    4459        23046 :       if ((unsigned HOST_WIDE_INT) (buf->off + i + wordsize)
    4460        22755 :           > (unsigned HOST_WIDE_INT) buf->sz)
    4461              :         {
    4462          291 :           gcc_assert (wordsize > 1);
    4463          291 :           wordsize /= 2;
    4464          291 :           i -= wordsize;
    4465          291 :           continue;
    4466              :         }
    4467        22464 :       size_t endsize = end - i > wordsize ? wordsize : end - i;
    4468       200916 :       for (size_t j = i; j < i + endsize; j++)
    4469              :         {
    4470       178452 :           if (buf->buf[j])
    4471              :             {
    4472       168415 :               if (nonzero_first == wordsize)
    4473              :                 {
    4474        21522 :                   nonzero_first = j - i;
    4475        21522 :                   nonzero_last = j - i;
    4476              :                 }
    4477       168415 :               if (nonzero_last != j - i)
    4478          166 :                 all_ones = false;
    4479       168415 :               nonzero_last = j + 1 - i;
    4480              :             }
    4481              :           else
    4482              :             {
    4483        10037 :               if (zero_first == wordsize)
    4484         1948 :                 zero_first = j - i;
    4485        10037 :               zero_last = j + 1 - i;
    4486              :             }
    4487       178452 :           if (buf->buf[j] != 0 && buf->buf[j] != (unsigned char) ~0)
    4488              :             {
    4489          104 :               all_ones = false;
    4490          104 :               bytes_only = false;
    4491              :             }
    4492              :         }
    4493        22464 :       size_t padding_end = i;
    4494        22464 :       if (padding_bytes)
    4495              :         {
    4496        20849 :           if (nonzero_first == 0
    4497        20849 :               && nonzero_last == endsize
    4498        20400 :               && all_ones)
    4499              :             {
    4500              :               /* All bits are padding and we had some padding
    4501              :                  before too.  Just extend it.  */
    4502        20400 :               padding_bytes += endsize;
    4503        20400 :               continue;
    4504              :             }
    4505          449 :           if (all_ones && nonzero_first == 0)
    4506              :             {
    4507            4 :               padding_bytes += nonzero_last;
    4508            4 :               padding_end += nonzero_last;
    4509            4 :               nonzero_first = wordsize;
    4510            4 :               nonzero_last = 0;
    4511              :             }
    4512          445 :           else if (bytes_only && nonzero_first == 0)
    4513              :             {
    4514            0 :               gcc_assert (zero_first && zero_first != wordsize);
    4515            0 :               padding_bytes += zero_first;
    4516            0 :               padding_end += zero_first;
    4517              :             }
    4518          449 :           tree atype, src;
    4519          449 :           if (padding_bytes == 1)
    4520              :             {
    4521           33 :               atype = char_type_node;
    4522           33 :               src = build_zero_cst (char_type_node);
    4523              :             }
    4524              :           else
    4525              :             {
    4526          416 :               atype = build_array_type_nelts (char_type_node, padding_bytes);
    4527          416 :               src = build_constructor (atype, NULL);
    4528              :             }
    4529          449 :           tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base,
    4530              :                                  build_int_cst (buf->alias_type,
    4531          449 :                                                 buf->off + padding_end
    4532          449 :                                                 - padding_bytes));
    4533          449 :           gimple *g = gimple_build_assign (dst, src);
    4534          449 :           gimple_set_location (g, buf->loc);
    4535          449 :           gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4536          449 :           padding_bytes = 0;
    4537          449 :           buf->padding_bytes = 0;
    4538              :         }
    4539         2064 :       if (nonzero_first == wordsize)
    4540              :         /* All bits in a word are 0, there are no padding bits.  */
    4541          946 :         continue;
    4542         1118 :       if (all_ones && nonzero_last == endsize)
    4543              :         {
    4544              :           /* All bits between nonzero_first and end of word are padding
    4545              :              bits, start counting padding_bytes.  */
    4546          841 :           padding_bytes = nonzero_last - nonzero_first;
    4547          841 :           continue;
    4548              :         }
    4549          277 :       if (bytes_only)
    4550              :         {
    4551              :           /* If bitfields aren't involved in this word, prefer storing
    4552              :              individual bytes or groups of them over performing a RMW
    4553              :              operation on the whole word.  */
    4554          227 :           gcc_assert (i + zero_last <= end);
    4555         1117 :           for (size_t j = padding_end; j < i + zero_last; j++)
    4556              :             {
    4557          890 :               if (buf->buf[j])
    4558              :                 {
    4559              :                   size_t k;
    4560          606 :                   for (k = j; k < i + zero_last; k++)
    4561          606 :                     if (buf->buf[k] == 0)
    4562              :                       break;
    4563          259 :                   HOST_WIDE_INT off = buf->off + j;
    4564          259 :                   tree atype, src;
    4565          259 :                   if (k - j == 1)
    4566              :                     {
    4567          215 :                       atype = char_type_node;
    4568          215 :                       src = build_zero_cst (char_type_node);
    4569              :                     }
    4570              :                   else
    4571              :                     {
    4572           44 :                       atype = build_array_type_nelts (char_type_node, k - j);
    4573           44 :                       src = build_constructor (atype, NULL);
    4574              :                     }
    4575          259 :                   tree dst = build2_loc (buf->loc, MEM_REF, atype,
    4576              :                                          buf->base,
    4577          259 :                                          build_int_cst (buf->alias_type, off));
    4578          259 :                   gimple *g = gimple_build_assign (dst, src);
    4579          259 :                   gimple_set_location (g, buf->loc);
    4580          259 :                   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4581          259 :                   j = k;
    4582              :                 }
    4583              :             }
    4584          227 :           if (nonzero_last == endsize)
    4585           98 :             padding_bytes = nonzero_last - zero_last;
    4586          227 :           continue;
    4587          227 :         }
    4588          163 :       for (size_t eltsz = 1; eltsz <= wordsize; eltsz <<= 1)
    4589              :         {
    4590          163 :           if (nonzero_last - nonzero_first <= eltsz
    4591           50 :               && ((nonzero_first & ~(eltsz - 1))
    4592           50 :                   == ((nonzero_last - 1) & ~(eltsz - 1))))
    4593              :             {
    4594           50 :               tree type;
    4595           50 :               if (eltsz == 1)
    4596            3 :                 type = char_type_node;
    4597              :               else
    4598           47 :                 type = lang_hooks.types.type_for_size (eltsz * BITS_PER_UNIT,
    4599              :                                                        0);
    4600           50 :               size_t start = nonzero_first & ~(eltsz - 1);
    4601           50 :               HOST_WIDE_INT off = buf->off + i + start;
    4602           50 :               tree atype = type;
    4603           50 :               if (eltsz > 1 && buf->align < TYPE_ALIGN (type))
    4604            8 :                 atype = build_aligned_type (type, buf->align);
    4605           50 :               tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base,
    4606           50 :                                      build_int_cst (buf->alias_type, off));
    4607           50 :               tree src;
    4608           50 :               gimple *g;
    4609           50 :               if (all_ones
    4610           50 :                   && nonzero_first == start
    4611            0 :                   && nonzero_last == start + eltsz)
    4612            0 :                 src = build_zero_cst (type);
    4613              :               else
    4614              :                 {
    4615           50 :                   src = make_ssa_name (type);
    4616           50 :                   tree tmp_dst = unshare_expr (dst);
    4617              :                   /* The folding introduces a read from the tmp_dst, we should
    4618              :                      prevent uninitialized warning analysis from issuing warning
    4619              :                      for such fake read.  In order to suppress warning only for
    4620              :                      this expr, we should set the location of tmp_dst to
    4621              :                      UNKNOWN_LOCATION first, then suppress_warning will call
    4622              :                      set_no_warning_bit to set the no_warning flag only for
    4623              :                      tmp_dst.  */
    4624           50 :                   SET_EXPR_LOCATION (tmp_dst, UNKNOWN_LOCATION);
    4625           50 :                   suppress_warning (tmp_dst, OPT_Wuninitialized);
    4626           50 :                   g = gimple_build_assign (src, tmp_dst);
    4627           50 :                   gimple_set_location (g, buf->loc);
    4628           50 :                   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4629          100 :                   tree mask = native_interpret_expr (type,
    4630           50 :                                                      buf->buf + i + start,
    4631              :                                                      eltsz);
    4632           50 :                   gcc_assert (mask && TREE_CODE (mask) == INTEGER_CST);
    4633           50 :                   mask = fold_build1 (BIT_NOT_EXPR, type, mask);
    4634           50 :                   tree src_masked = make_ssa_name (type);
    4635           50 :                   g = gimple_build_assign (src_masked, BIT_AND_EXPR,
    4636              :                                            src, mask);
    4637           50 :                   gimple_set_location (g, buf->loc);
    4638           50 :                   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4639           50 :                   src = src_masked;
    4640              :                 }
    4641           50 :               g = gimple_build_assign (dst, src);
    4642           50 :               gimple_set_location (g, buf->loc);
    4643           50 :               gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4644           50 :               break;
    4645              :             }
    4646              :         }
    4647              :     }
    4648          770 :   if (full)
    4649              :     {
    4650          728 :       if (padding_bytes)
    4651              :         {
    4652          490 :           tree atype, src;
    4653          490 :           if (padding_bytes == 1)
    4654              :             {
    4655          110 :               atype = char_type_node;
    4656          110 :               src = build_zero_cst (char_type_node);
    4657              :             }
    4658              :           else
    4659              :             {
    4660          380 :               atype = build_array_type_nelts (char_type_node, padding_bytes);
    4661          380 :               src = build_constructor (atype, NULL);
    4662              :             }
    4663          490 :           tree dst = build2_loc (buf->loc, MEM_REF, atype, buf->base,
    4664              :                                  build_int_cst (buf->alias_type,
    4665          490 :                                                 buf->off + end
    4666          490 :                                                 - padding_bytes));
    4667          490 :           gimple *g = gimple_build_assign (dst, src);
    4668          490 :           gimple_set_location (g, buf->loc);
    4669          490 :           gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4670              :         }
    4671          728 :       size_t end_rem = end % UNITS_PER_WORD;
    4672          728 :       buf->off += end - end_rem;
    4673          728 :       buf->size = end_rem;
    4674          728 :       memset (buf->buf, 0, buf->size);
    4675          728 :       buf->padding_bytes = 0;
    4676              :     }
    4677              :   else
    4678              :     {
    4679           42 :       memmove (buf->buf, buf->buf + end, buf->size - end);
    4680           42 :       buf->off += end;
    4681           42 :       buf->size -= end;
    4682           42 :       buf->padding_bytes = padding_bytes;
    4683              :     }
    4684              : }
    4685              : 
    4686              : /* Append PADDING_BYTES padding bytes.  */
    4687              : 
    4688              : static void
    4689        26529 : clear_padding_add_padding (clear_padding_struct *buf,
    4690              :                            HOST_WIDE_INT padding_bytes)
    4691              : {
    4692        26529 :   if (padding_bytes == 0)
    4693              :     return;
    4694         1713 :   if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size
    4695              :       > (unsigned HOST_WIDE_INT) clear_padding_buf_size)
    4696           42 :     clear_padding_flush (buf, false);
    4697         1713 :   if ((unsigned HOST_WIDE_INT) padding_bytes + buf->size
    4698              :       > (unsigned HOST_WIDE_INT) clear_padding_buf_size)
    4699              :     {
    4700           42 :       memset (buf->buf + buf->size, ~0, clear_padding_buf_size - buf->size);
    4701           42 :       padding_bytes -= clear_padding_buf_size - buf->size;
    4702           42 :       buf->size = clear_padding_buf_size;
    4703           42 :       clear_padding_flush (buf, false);
    4704           42 :       gcc_assert (buf->padding_bytes);
    4705              :       /* At this point buf->buf[0] through buf->buf[buf->size - 1]
    4706              :          is guaranteed to be all ones.  */
    4707           42 :       padding_bytes += buf->size;
    4708           42 :       buf->size = padding_bytes % UNITS_PER_WORD;
    4709           42 :       memset (buf->buf, ~0, buf->size);
    4710           42 :       buf->off += padding_bytes - buf->size;
    4711           42 :       buf->padding_bytes += padding_bytes - buf->size;
    4712              :     }
    4713              :   else
    4714              :     {
    4715         1671 :       memset (buf->buf + buf->size, ~0, padding_bytes);
    4716         1671 :       buf->size += padding_bytes;
    4717              :     }
    4718              : }
    4719              : 
    4720              : static void clear_padding_type (clear_padding_struct *, tree,
    4721              :                                 HOST_WIDE_INT, bool);
    4722              : 
    4723              : /* Clear padding bits of union type TYPE.  */
    4724              : 
    4725              : static void
    4726          128 : clear_padding_union (clear_padding_struct *buf, tree type,
    4727              :                      HOST_WIDE_INT sz, bool for_auto_init)
    4728              : {
    4729          128 :   clear_padding_struct *union_buf;
    4730          128 :   HOST_WIDE_INT start_off = 0, next_off = 0;
    4731          128 :   size_t start_size = 0;
    4732          128 :   if (buf->union_ptr)
    4733              :     {
    4734           42 :       start_off = buf->off + buf->size;
    4735           42 :       next_off = start_off + sz;
    4736           42 :       start_size = start_off % UNITS_PER_WORD;
    4737           42 :       start_off -= start_size;
    4738           42 :       clear_padding_flush (buf, true);
    4739           42 :       union_buf = buf;
    4740              :     }
    4741              :   else
    4742              :     {
    4743           86 :       if (sz + buf->size > clear_padding_buf_size)
    4744            0 :         clear_padding_flush (buf, false);
    4745           86 :       union_buf = XALLOCA (clear_padding_struct);
    4746           86 :       union_buf->loc = buf->loc;
    4747           86 :       union_buf->clear_in_mask = buf->clear_in_mask;
    4748           86 :       union_buf->base = NULL_TREE;
    4749           86 :       union_buf->alias_type = NULL_TREE;
    4750           86 :       union_buf->gsi = NULL;
    4751           86 :       union_buf->align = 0;
    4752           86 :       union_buf->off = 0;
    4753           86 :       union_buf->padding_bytes = 0;
    4754           86 :       union_buf->sz = sz;
    4755           86 :       union_buf->size = 0;
    4756           86 :       if (sz + buf->size <= clear_padding_buf_size)
    4757           86 :         union_buf->union_ptr = buf->buf + buf->size;
    4758              :       else
    4759            0 :         union_buf->union_ptr = XNEWVEC (unsigned char, sz);
    4760           86 :       memset (union_buf->union_ptr, ~0, sz);
    4761              :     }
    4762              : 
    4763         1193 :   for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
    4764         1065 :     if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field))
    4765              :       {
    4766          359 :         if (DECL_SIZE_UNIT (field) == NULL_TREE)
    4767              :           {
    4768            8 :             if (TREE_TYPE (field) == error_mark_node)
    4769            0 :               continue;
    4770            8 :             gcc_assert (TREE_CODE (TREE_TYPE (field)) == ARRAY_TYPE
    4771              :                         && !COMPLETE_TYPE_P (TREE_TYPE (field)));
    4772            8 :             if (!buf->clear_in_mask && !for_auto_init)
    4773            8 :               error_at (buf->loc, "flexible array member %qD does not have "
    4774              :                                   "well defined padding bits for %qs",
    4775              :                         field, "__builtin_clear_padding");
    4776            8 :             continue;
    4777              :           }
    4778          351 :         HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field));
    4779          351 :         gcc_assert (union_buf->size == 0);
    4780          351 :         union_buf->off = start_off;
    4781          351 :         union_buf->size = start_size;
    4782          351 :         memset (union_buf->buf, ~0, start_size);
    4783          351 :         clear_padding_type (union_buf, TREE_TYPE (field), fldsz, for_auto_init);
    4784          351 :         clear_padding_add_padding (union_buf, sz - fldsz);
    4785          351 :         clear_padding_flush (union_buf, true);
    4786              :       }
    4787              : 
    4788          128 :   if (buf == union_buf)
    4789              :     {
    4790           42 :       buf->off = next_off;
    4791           42 :       buf->size = next_off % UNITS_PER_WORD;
    4792           42 :       buf->off -= buf->size;
    4793           42 :       memset (buf->buf, ~0, buf->size);
    4794              :     }
    4795           86 :   else if (sz + buf->size <= clear_padding_buf_size)
    4796           86 :     buf->size += sz;
    4797              :   else
    4798              :     {
    4799            0 :       unsigned char *union_ptr = union_buf->union_ptr;
    4800            0 :       while (sz)
    4801              :         {
    4802            0 :           clear_padding_flush (buf, false);
    4803            0 :           HOST_WIDE_INT this_sz
    4804            0 :             = MIN ((unsigned HOST_WIDE_INT) sz,
    4805              :                    clear_padding_buf_size - buf->size);
    4806            0 :           memcpy (buf->buf + buf->size, union_ptr, this_sz);
    4807            0 :           buf->size += this_sz;
    4808            0 :           union_ptr += this_sz;
    4809            0 :           sz -= this_sz;
    4810              :         }
    4811            0 :       XDELETE (union_buf->union_ptr);
    4812              :     }
    4813          128 : }
    4814              : 
    4815              : /* The only known floating point formats with padding bits are the
    4816              :    IEEE extended ones.  */
    4817              : 
    4818              : static bool
    4819        39362 : clear_padding_real_needs_padding_p (tree type)
    4820              : {
    4821        39362 :   const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
    4822        39362 :   return (fmt->b == 2
    4823        38751 :           && fmt->signbit_ro == fmt->signbit_rw
    4824        78113 :           && (fmt->signbit_ro == 79 || fmt->signbit_ro == 95));
    4825              : }
    4826              : 
    4827              : /* _BitInt has padding bits if it isn't extended in the ABI and has smaller
    4828              :    precision than bits in limb or corresponding number of limbs.  */
    4829              : 
    4830              : static bool
    4831           60 : clear_padding_bitint_needs_padding_p (tree type)
    4832              : {
    4833           60 :   struct bitint_info info;
    4834           60 :   bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
    4835           60 :   gcc_assert (ok);
    4836           60 :   if (info.extended)
    4837              :     return false;
    4838           60 :   scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.abi_limb_mode);
    4839           60 :   if (TYPE_PRECISION (type) < GET_MODE_PRECISION (limb_mode))
    4840              :     return true;
    4841            4 :   else if (TYPE_PRECISION (type) == GET_MODE_PRECISION (limb_mode))
    4842              :     return false;
    4843              :   else
    4844            4 :     return (((unsigned) TYPE_PRECISION (type))
    4845            4 :             % GET_MODE_PRECISION (limb_mode)) != 0;
    4846              : }
    4847              : 
    4848              : /* Return true if TYPE might contain any padding bits.  */
    4849              : 
    4850              : bool
    4851       953427 : clear_padding_type_may_have_padding_p (tree type)
    4852              : {
    4853      1093656 :   switch (TREE_CODE (type))
    4854              :     {
    4855              :     case RECORD_TYPE:
    4856              :     case UNION_TYPE:
    4857              :       return true;
    4858       140229 :     case ARRAY_TYPE:
    4859       140229 :     case COMPLEX_TYPE:
    4860       140229 :     case VECTOR_TYPE:
    4861       140229 :       return clear_padding_type_may_have_padding_p (TREE_TYPE (type));
    4862         1935 :     case REAL_TYPE:
    4863         1935 :       return clear_padding_real_needs_padding_p (type);
    4864           63 :     case ENUMERAL_TYPE:
    4865           63 :       if (BITINT_TYPE_P (type))
    4866            0 :         return clear_padding_bitint_needs_padding_p (type);
    4867              :       return false;
    4868           60 :     case BITINT_TYPE:
    4869           60 :       return clear_padding_bitint_needs_padding_p (type);
    4870              :     default:
    4871              :       return false;
    4872              :     }
    4873              : }
    4874              : 
    4875              : /* Return true if TYPE has padding bits aside from those in fields,
    4876              :    elements, etc.  */
    4877              : 
    4878              : bool
    4879      1205238 : type_has_padding_at_level_p (tree type)
    4880              : {
    4881      1205238 :   switch (TREE_CODE (type))
    4882              :     {
    4883      1062094 :     case RECORD_TYPE:
    4884      1062094 :       {
    4885      1062094 :         tree bitpos = size_zero_node;
    4886              :         /* Expect fields to be sorted by bit position.  */
    4887      8732402 :         for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
    4888      7676165 :           if (TREE_CODE (f) == FIELD_DECL)
    4889              :             {
    4890      2310396 :               if (DECL_PADDING_P (f))
    4891              :                 return true;
    4892      2310393 :               tree pos = bit_position (f);
    4893      2310393 :               if (simple_cst_equal (bitpos, pos) != 1)
    4894              :                 return true;
    4895      2304562 :               if (!DECL_SIZE (f))
    4896              :                 return true;
    4897      2304539 :               bitpos = int_const_binop (PLUS_EXPR, pos, DECL_SIZE (f));
    4898              :             }
    4899      1056237 :         if (simple_cst_equal (bitpos, TYPE_SIZE (type)) != 1)
    4900        12611 :           return true;
    4901              :         return false;
    4902              :       }
    4903            3 :     case UNION_TYPE:
    4904            3 :     case QUAL_UNION_TYPE:
    4905            3 :       bool any_fields;
    4906            3 :       any_fields = false;
    4907              :       /* If any of the fields is smaller than the whole, there is padding.  */
    4908            6 :       for (tree f = TYPE_FIELDS (type); f; f = DECL_CHAIN (f))
    4909            3 :         if (TREE_CODE (f) != FIELD_DECL || TREE_TYPE (f) == error_mark_node)
    4910            3 :           continue;
    4911            0 :         else if (simple_cst_equal (TYPE_SIZE (TREE_TYPE (f)),
    4912            0 :                                    TYPE_SIZE (type)) != 1)
    4913              :           return true;
    4914              :         else
    4915              :           any_fields = true;
    4916              :       /* If the union doesn't have any fields and still has non-zero size,
    4917              :          all of it is padding.  */
    4918            3 :       if (!any_fields && !integer_zerop (TYPE_SIZE (type)))
    4919            3 :         return true;
    4920              :       return false;
    4921              :     case ARRAY_TYPE:
    4922              :     case COMPLEX_TYPE:
    4923              :     case VECTOR_TYPE:
    4924              :       /* No recursing here, no padding at this level.  */
    4925              :       return false;
    4926            0 :     case REAL_TYPE:
    4927            0 :       return clear_padding_real_needs_padding_p (type);
    4928            0 :     case ENUMERAL_TYPE:
    4929            0 :       if (BITINT_TYPE_P (type))
    4930            0 :         return clear_padding_bitint_needs_padding_p (type);
    4931              :       return false;
    4932            0 :     case BITINT_TYPE:
    4933            0 :       return clear_padding_bitint_needs_padding_p (type);
    4934              :     default:
    4935              :       return false;
    4936              :     }
    4937              : }
    4938              : 
    4939              : /* Emit a runtime loop:
    4940              :    for (; buf.base != end; buf.base += sz)
    4941              :      __builtin_clear_padding (buf.base);  */
    4942              : 
    4943              : static void
    4944          114 : clear_padding_emit_loop (clear_padding_struct *buf, tree type,
    4945              :                          tree end, bool for_auto_init)
    4946              : {
    4947          114 :   tree l1 = create_artificial_label (buf->loc);
    4948          114 :   tree l2 = create_artificial_label (buf->loc);
    4949          114 :   tree l3 = create_artificial_label (buf->loc);
    4950          114 :   gimple *g = gimple_build_goto (l2);
    4951          114 :   gimple_set_location (g, buf->loc);
    4952          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4953          114 :   g = gimple_build_label (l1);
    4954          114 :   gimple_set_location (g, buf->loc);
    4955          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4956          114 :   clear_padding_type (buf, type, buf->sz, for_auto_init);
    4957          114 :   clear_padding_flush (buf, true);
    4958          114 :   g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR, buf->base,
    4959          114 :                            size_int (buf->sz));
    4960          114 :   gimple_set_location (g, buf->loc);
    4961          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4962          114 :   g = gimple_build_label (l2);
    4963          114 :   gimple_set_location (g, buf->loc);
    4964          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4965          114 :   g = gimple_build_cond (NE_EXPR, buf->base, end, l1, l3);
    4966          114 :   gimple_set_location (g, buf->loc);
    4967          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4968          114 :   g = gimple_build_label (l3);
    4969          114 :   gimple_set_location (g, buf->loc);
    4970          114 :   gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    4971          114 : }
    4972              : 
    4973              : /* Clear padding bits for TYPE.  Called recursively from
    4974              :    gimple_fold_builtin_clear_padding.  If FOR_AUTO_INIT is true,
    4975              :    the __builtin_clear_padding is not called by the end user,
    4976              :    instead, it's inserted by the compiler to initialize the
    4977              :    paddings of automatic variable.  Therefore, we should not
    4978              :    emit the error messages for flexible array members to confuse
    4979              :    the end user.  */
    4980              : 
    4981              : static void
    4982        74968 : clear_padding_type (clear_padding_struct *buf, tree type,
    4983              :                     HOST_WIDE_INT sz, bool for_auto_init)
    4984              : {
    4985        74968 :   switch (TREE_CODE (type))
    4986              :     {
    4987        10304 :     case RECORD_TYPE:
    4988        10304 :       HOST_WIDE_INT cur_pos;
    4989        10304 :       cur_pos = 0;
    4990       793667 :       for (tree field = TYPE_FIELDS (type); field; field = DECL_CHAIN (field))
    4991       783363 :         if (TREE_CODE (field) == FIELD_DECL && !DECL_PADDING_P (field))
    4992              :           {
    4993        24977 :             tree ftype = TREE_TYPE (field);
    4994        24977 :             if (DECL_BIT_FIELD (field))
    4995              :               {
    4996          292 :                 HOST_WIDE_INT fldsz = TYPE_PRECISION (ftype);
    4997          292 :                 if (fldsz == 0)
    4998            0 :                   continue;
    4999          292 :                 HOST_WIDE_INT pos = int_byte_position (field);
    5000          292 :                 if (pos >= sz)
    5001            0 :                   continue;
    5002          292 :                 HOST_WIDE_INT bpos
    5003          292 :                   = tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field));
    5004          292 :                 bpos %= BITS_PER_UNIT;
    5005          292 :                 HOST_WIDE_INT end
    5006          292 :                   = ROUND_UP (bpos + fldsz, BITS_PER_UNIT) / BITS_PER_UNIT;
    5007          292 :                 if (pos + end > cur_pos)
    5008              :                   {
    5009          215 :                     clear_padding_add_padding (buf, pos + end - cur_pos);
    5010          215 :                     cur_pos = pos + end;
    5011              :                   }
    5012          292 :                 gcc_assert (cur_pos > pos
    5013              :                             && ((unsigned HOST_WIDE_INT) buf->size
    5014              :                                 >= (unsigned HOST_WIDE_INT) cur_pos - pos));
    5015          292 :                 unsigned char *p = buf->buf + buf->size - (cur_pos - pos);
    5016          292 :                 if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
    5017              :                   sorry_at (buf->loc, "PDP11 bit-field handling unsupported"
    5018              :                                       " in %qs", "__builtin_clear_padding");
    5019          292 :                 else if (BYTES_BIG_ENDIAN)
    5020              :                   {
    5021              :                     /* Big endian.  */
    5022              :                     if (bpos + fldsz <= BITS_PER_UNIT)
    5023              :                       *p &= ~(((1 << fldsz) - 1)
    5024              :                               << (BITS_PER_UNIT - bpos - fldsz));
    5025              :                     else
    5026              :                       {
    5027              :                         if (bpos)
    5028              :                           {
    5029              :                             *p &= ~(((1U << BITS_PER_UNIT) - 1) >> bpos);
    5030              :                             p++;
    5031              :                             fldsz -= BITS_PER_UNIT - bpos;
    5032              :                           }
    5033              :                         memset (p, 0, fldsz / BITS_PER_UNIT);
    5034              :                         p += fldsz / BITS_PER_UNIT;
    5035              :                         fldsz %= BITS_PER_UNIT;
    5036              :                         if (fldsz)
    5037              :                           *p &= ((1U << BITS_PER_UNIT) - 1) >> fldsz;
    5038              :                       }
    5039              :                   }
    5040              :                 else
    5041              :                   {
    5042              :                     /* Little endian.  */
    5043          292 :                     if (bpos + fldsz <= BITS_PER_UNIT)
    5044          175 :                       *p &= ~(((1 << fldsz) - 1) << bpos);
    5045              :                     else
    5046              :                       {
    5047          117 :                         if (bpos)
    5048              :                           {
    5049           33 :                             *p &= ~(((1 << BITS_PER_UNIT) - 1) << bpos);
    5050           33 :                             p++;
    5051           33 :                             fldsz -= BITS_PER_UNIT - bpos;
    5052              :                           }
    5053          117 :                         memset (p, 0, fldsz / BITS_PER_UNIT);
    5054          117 :                         p += fldsz / BITS_PER_UNIT;
    5055          117 :                         fldsz %= BITS_PER_UNIT;
    5056          117 :                         if (fldsz)
    5057           72 :                           *p &= ~((1 << fldsz) - 1);
    5058              :                       }
    5059              :                   }
    5060              :               }
    5061        24685 :             else if (DECL_SIZE_UNIT (field) == NULL_TREE)
    5062              :               {
    5063           32 :                 if (ftype == error_mark_node)
    5064            0 :                   continue;
    5065           32 :                 gcc_assert (TREE_CODE (ftype) == ARRAY_TYPE
    5066              :                             && !COMPLETE_TYPE_P (ftype));
    5067           32 :                 if (!buf->clear_in_mask && !for_auto_init)
    5068           24 :                   error_at (buf->loc, "flexible array member %qD does not "
    5069              :                                       "have well defined padding bits for %qs",
    5070              :                             field, "__builtin_clear_padding");
    5071              :               }
    5072        24653 :             else if (is_empty_type (ftype))
    5073         8994 :               continue;
    5074              :             else
    5075              :               {
    5076        15659 :                 HOST_WIDE_INT pos = int_byte_position (field);
    5077        15659 :                 if (pos >= sz)
    5078            0 :                   continue;
    5079        15659 :                 HOST_WIDE_INT fldsz = tree_to_shwi (DECL_SIZE_UNIT (field));
    5080        15659 :                 gcc_assert (pos >= 0 && fldsz >= 0 && pos >= cur_pos);
    5081        15659 :                 clear_padding_add_padding (buf, pos - cur_pos);
    5082        15659 :                 cur_pos = pos;
    5083        15659 :                 if (tree asbase = lang_hooks.types.classtype_as_base (field))
    5084          300 :                   ftype = asbase;
    5085        15659 :                 clear_padding_type (buf, ftype, fldsz, for_auto_init);
    5086        15659 :                 cur_pos += fldsz;
    5087              :               }
    5088              :           }
    5089        10304 :       gcc_assert (sz >= cur_pos);
    5090        10304 :       clear_padding_add_padding (buf, sz - cur_pos);
    5091        10304 :       break;
    5092          326 :     case ARRAY_TYPE:
    5093          326 :       HOST_WIDE_INT nelts, fldsz;
    5094          326 :       fldsz = int_size_in_bytes (TREE_TYPE (type));
    5095          326 :       if (fldsz == 0)
    5096              :         break;
    5097          312 :       nelts = sz / fldsz;
    5098          312 :       if (nelts > 1
    5099          305 :           && sz > 8 * UNITS_PER_WORD
    5100           78 :           && buf->union_ptr == NULL
    5101          390 :           && clear_padding_type_may_have_padding_p (TREE_TYPE (type)))
    5102              :         {
    5103              :           /* For sufficiently large array of more than one elements,
    5104              :              emit a runtime loop to keep code size manageable.  */
    5105           66 :           tree base = buf->base;
    5106           66 :           unsigned int prev_align = buf->align;
    5107           66 :           HOST_WIDE_INT off = buf->off + buf->size;
    5108           66 :           HOST_WIDE_INT prev_sz = buf->sz;
    5109           66 :           clear_padding_flush (buf, true);
    5110           66 :           tree elttype = TREE_TYPE (type);
    5111           66 :           buf->base = create_tmp_var (build_pointer_type (elttype));
    5112           66 :           tree end = make_ssa_name (TREE_TYPE (buf->base));
    5113           66 :           gimple *g = gimple_build_assign (buf->base, POINTER_PLUS_EXPR,
    5114           66 :                                            base, size_int (off));
    5115           66 :           gimple_set_location (g, buf->loc);
    5116           66 :           gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    5117           66 :           g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf->base,
    5118           66 :                                    size_int (sz));
    5119           66 :           gimple_set_location (g, buf->loc);
    5120           66 :           gsi_insert_before (buf->gsi, g, GSI_SAME_STMT);
    5121           66 :           buf->sz = fldsz;
    5122           66 :           buf->align = TYPE_ALIGN (elttype);
    5123           66 :           buf->off = 0;
    5124           66 :           buf->size = 0;
    5125           66 :           clear_padding_emit_loop (buf, elttype, end, for_auto_init);
    5126           66 :           off += sz;
    5127           66 :           buf->base = base;
    5128           66 :           buf->sz = prev_sz;
    5129           66 :           buf->align = prev_align;
    5130           66 :           buf->size = off % UNITS_PER_WORD;
    5131           66 :           buf->off = off - buf->size;
    5132           66 :           memset (buf->buf, 0, buf->size);
    5133           66 :           break;
    5134              :         }
    5135         1180 :       for (HOST_WIDE_INT i = 0; i < nelts; i++)
    5136          934 :         clear_padding_type (buf, TREE_TYPE (type), fldsz, for_auto_init);
    5137              :       break;
    5138          128 :     case UNION_TYPE:
    5139          128 :       clear_padding_union (buf, type, sz, for_auto_init);
    5140          128 :       break;
    5141        37427 :     case REAL_TYPE:
    5142        37427 :       gcc_assert ((size_t) sz <= clear_padding_unit);
    5143        37427 :       if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
    5144            0 :         clear_padding_flush (buf, false);
    5145        37427 :       if (clear_padding_real_needs_padding_p (type))
    5146              :         {
    5147              :           /* Use native_interpret_real + native_encode_expr to figure out
    5148              :              which bits are padding.  */
    5149         1587 :           memset (buf->buf + buf->size, ~0, sz);
    5150         1587 :           tree cst = native_interpret_real (type, buf->buf + buf->size, sz);
    5151         1587 :           gcc_assert (cst && TREE_CODE (cst) == REAL_CST);
    5152         1587 :           int len = native_encode_expr (cst, buf->buf + buf->size, sz);
    5153         1587 :           gcc_assert (len > 0 && (size_t) len == (size_t) sz);
    5154        26979 :           for (size_t i = 0; i < (size_t) sz; i++)
    5155        25392 :             buf->buf[buf->size + i] ^= ~0;
    5156              :         }
    5157              :       else
    5158        35840 :         memset (buf->buf + buf->size, 0, sz);
    5159        37427 :       buf->size += sz;
    5160        37427 :       break;
    5161            0 :     case COMPLEX_TYPE:
    5162            0 :       fldsz = int_size_in_bytes (TREE_TYPE (type));
    5163            0 :       clear_padding_type (buf, TREE_TYPE (type), fldsz, for_auto_init);
    5164            0 :       clear_padding_type (buf, TREE_TYPE (type), fldsz, for_auto_init);
    5165            0 :       break;
    5166         5180 :     case VECTOR_TYPE:
    5167         5180 :       nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
    5168         5180 :       fldsz = int_size_in_bytes (TREE_TYPE (type));
    5169        28828 :       for (HOST_WIDE_INT i = 0; i < nelts; i++)
    5170        18468 :         clear_padding_type (buf, TREE_TYPE (type), fldsz, for_auto_init);
    5171              :       break;
    5172            7 :     case NULLPTR_TYPE:
    5173            7 :       gcc_assert ((size_t) sz <= clear_padding_unit);
    5174            7 :       if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
    5175            0 :         clear_padding_flush (buf, false);
    5176            7 :       memset (buf->buf + buf->size, ~0, sz);
    5177            7 :       buf->size += sz;
    5178            7 :       break;
    5179            7 :     case BITINT_TYPE:
    5180            7 :     do_bitint:
    5181            7 :       {
    5182            7 :         struct bitint_info info;
    5183            7 :         bool ok = targetm.c.bitint_type_info (TYPE_PRECISION (type), &info);
    5184            7 :         gcc_assert (ok);
    5185            7 :         scalar_int_mode limb_mode
    5186            7 :           = as_a <scalar_int_mode> (info.abi_limb_mode);
    5187            7 :         if (TYPE_PRECISION (type) <= GET_MODE_PRECISION (limb_mode))
    5188              :           {
    5189            5 :             gcc_assert ((size_t) sz <= clear_padding_unit);
    5190            5 :             if ((unsigned HOST_WIDE_INT) sz + buf->size
    5191              :                 > clear_padding_buf_size)
    5192            0 :               clear_padding_flush (buf, false);
    5193            5 :             if (!info.extended
    5194            5 :                 && TYPE_PRECISION (type) < GET_MODE_PRECISION (limb_mode))
    5195              :               {
    5196            5 :                 int tprec = GET_MODE_PRECISION (limb_mode);
    5197            5 :                 int prec = TYPE_PRECISION (type);
    5198            5 :                 tree t = build_nonstandard_integer_type (tprec, 1);
    5199            5 :                 tree cst = wide_int_to_tree (t, wi::mask (prec, true, tprec));
    5200            5 :                 int len = native_encode_expr (cst, buf->buf + buf->size, sz);
    5201            5 :                 gcc_assert (len > 0 && (size_t) len == (size_t) sz);
    5202              :               }
    5203              :             else
    5204            0 :               memset (buf->buf + buf->size, 0, sz);
    5205            5 :             buf->size += sz;
    5206            5 :             break;
    5207              :           }
    5208            2 :         tree limbtype
    5209            2 :           = build_nonstandard_integer_type (GET_MODE_PRECISION (limb_mode), 1);
    5210            2 :         fldsz = int_size_in_bytes (limbtype);
    5211            2 :         nelts = int_size_in_bytes (type) / fldsz;
    5212           13 :         for (HOST_WIDE_INT i = 0; i < nelts; i++)
    5213              :           {
    5214           11 :             if (!info.extended
    5215           11 :                 && i == (info.big_endian ? 0 : nelts - 1)
    5216           13 :                 && (((unsigned) TYPE_PRECISION (type))
    5217            2 :                     % TYPE_PRECISION (limbtype)) != 0)
    5218              :               {
    5219            2 :                 int tprec = GET_MODE_PRECISION (limb_mode);
    5220            2 :                 int prec = (((unsigned) TYPE_PRECISION (type)) % tprec);
    5221            2 :                 tree cst = wide_int_to_tree (limbtype,
    5222            2 :                                              wi::mask (prec, true, tprec));
    5223            2 :                 int len = native_encode_expr (cst, buf->buf + buf->size,
    5224              :                                               fldsz);
    5225            2 :                 gcc_assert (len > 0 && (size_t) len == (size_t) fldsz);
    5226            2 :                 buf->size += fldsz;
    5227              :               }
    5228              :             else
    5229            9 :               clear_padding_type (buf, limbtype, fldsz, for_auto_init);
    5230              :           }
    5231              :         break;
    5232              :       }
    5233            0 :     case ENUMERAL_TYPE:
    5234            0 :       if (BITINT_TYPE_P (type))
    5235            0 :         goto do_bitint;
    5236              :       /* FALLTHRU */
    5237        21589 :     default:
    5238        21589 :       gcc_assert ((size_t) sz <= clear_padding_unit);
    5239        21589 :       if ((unsigned HOST_WIDE_INT) sz + buf->size > clear_padding_buf_size)
    5240            0 :         clear_padding_flush (buf, false);
    5241        21589 :       memset (buf->buf + buf->size, 0, sz);
    5242        21589 :       buf->size += sz;
    5243        21589 :       break;
    5244              :     }
    5245        74968 : }
    5246              : 
    5247              : /* Clear padding bits of TYPE in MASK.  */
    5248              : 
    5249              : void
    5250        38885 : clear_type_padding_in_mask (tree type, unsigned char *mask)
    5251              : {
    5252        38885 :   clear_padding_struct buf;
    5253        38885 :   buf.loc = UNKNOWN_LOCATION;
    5254        38885 :   buf.clear_in_mask = true;
    5255        38885 :   buf.base = NULL_TREE;
    5256        38885 :   buf.alias_type = NULL_TREE;
    5257        38885 :   buf.gsi = NULL;
    5258        38885 :   buf.align = 0;
    5259        38885 :   buf.off = 0;
    5260        38885 :   buf.padding_bytes = 0;
    5261        38885 :   buf.sz = int_size_in_bytes (type);
    5262        38885 :   buf.size = 0;
    5263        38885 :   buf.union_ptr = mask;
    5264        38885 :   clear_padding_type (&buf, type, buf.sz, false);
    5265        38885 :   clear_padding_flush (&buf, true);
    5266        38885 : }
    5267              : 
    5268              : /* Fold __builtin_clear_padding builtin.  */
    5269              : 
    5270              : static bool
    5271          634 : gimple_fold_builtin_clear_padding (gimple_stmt_iterator *gsi)
    5272              : {
    5273          634 :   gimple *stmt = gsi_stmt (*gsi);
    5274          634 :   gcc_assert (gimple_call_num_args (stmt) == 2);
    5275          634 :   tree ptr = gimple_call_arg (stmt, 0);
    5276          634 :   tree typearg = gimple_call_arg (stmt, 1);
    5277              :   /* The 2nd argument of __builtin_clear_padding's value is used to
    5278              :      distinguish whether this call is made by the user or by the compiler
    5279              :      for automatic variable initialization.  */
    5280          634 :   bool for_auto_init = (bool) TREE_INT_CST_LOW (typearg);
    5281          634 :   tree type = TREE_TYPE (TREE_TYPE (typearg));
    5282          634 :   location_t loc = gimple_location (stmt);
    5283          634 :   clear_padding_struct buf;
    5284          634 :   gimple_stmt_iterator gsiprev = *gsi;
    5285              :   /* This should be folded during the lower pass.  */
    5286         1268 :   gcc_assert (!gimple_in_ssa_p (cfun) && cfun->cfg == NULL);
    5287          634 :   gcc_assert (COMPLETE_TYPE_P (type));
    5288          634 :   gsi_prev (&gsiprev);
    5289              : 
    5290          634 :   buf.loc = loc;
    5291          634 :   buf.clear_in_mask = false;
    5292          634 :   buf.base = ptr;
    5293          634 :   buf.alias_type = NULL_TREE;
    5294          634 :   buf.gsi = gsi;
    5295          634 :   buf.align = get_pointer_alignment (ptr);
    5296          634 :   unsigned int talign = min_align_of_type (type) * BITS_PER_UNIT;
    5297          634 :   buf.align = MAX (buf.align, talign);
    5298          634 :   buf.off = 0;
    5299          634 :   buf.padding_bytes = 0;
    5300          634 :   buf.size = 0;
    5301          634 :   buf.sz = int_size_in_bytes (type);
    5302          634 :   buf.union_ptr = NULL;
    5303          634 :   if (buf.sz < 0 && int_size_in_bytes (strip_array_types (type)) < 0)
    5304            1 :     sorry_at (loc, "%s not supported for variable length aggregates",
    5305              :               "__builtin_clear_padding");
    5306              :   /* The implementation currently assumes 8-bit host and target
    5307              :      chars which is the case for all currently supported targets
    5308              :      and hosts and is required e.g. for native_{encode,interpret}* APIs.  */
    5309          633 :   else if (CHAR_BIT != 8 || BITS_PER_UNIT != 8)
    5310              :     sorry_at (loc, "%s not supported on this target",
    5311              :               "__builtin_clear_padding");
    5312          633 :   else if (!clear_padding_type_may_have_padding_p (type))
    5313              :     ;
    5314          596 :   else if (TREE_CODE (type) == ARRAY_TYPE && buf.sz < 0)
    5315              :     {
    5316           48 :       tree sz = TYPE_SIZE_UNIT (type);
    5317           48 :       tree elttype = type;
    5318              :       /* Only supports C/C++ VLAs and flattens all the VLA levels.  */
    5319           48 :       while (TREE_CODE (elttype) == ARRAY_TYPE
    5320          144 :              && int_size_in_bytes (elttype) < 0)
    5321           96 :         elttype = TREE_TYPE (elttype);
    5322           48 :       HOST_WIDE_INT eltsz = int_size_in_bytes (elttype);
    5323           48 :       gcc_assert (eltsz >= 0);
    5324           48 :       if (eltsz)
    5325              :         {
    5326           48 :           buf.base = create_tmp_var (build_pointer_type (elttype));
    5327           48 :           tree end = make_ssa_name (TREE_TYPE (buf.base));
    5328           48 :           gimple *g = gimple_build_assign (buf.base, ptr);
    5329           48 :           gimple_set_location (g, loc);
    5330           48 :           gsi_insert_before (gsi, g, GSI_SAME_STMT);
    5331           48 :           g = gimple_build_assign (end, POINTER_PLUS_EXPR, buf.base, sz);
    5332           48 :           gimple_set_location (g, loc);
    5333           48 :           gsi_insert_before (gsi, g, GSI_SAME_STMT);
    5334           48 :           buf.sz = eltsz;
    5335           48 :           buf.align = TYPE_ALIGN (elttype);
    5336           48 :           buf.alias_type = build_pointer_type (elttype);
    5337           48 :           clear_padding_emit_loop (&buf, elttype, end, for_auto_init);
    5338              :         }
    5339              :     }
    5340              :   else
    5341              :     {
    5342          548 :       if (!is_gimple_mem_ref_addr (buf.base))
    5343              :         {
    5344           28 :           buf.base = make_ssa_name (TREE_TYPE (ptr));
    5345           28 :           gimple *g = gimple_build_assign (buf.base, ptr);
    5346           28 :           gimple_set_location (g, loc);
    5347           28 :           gsi_insert_before (gsi, g, GSI_SAME_STMT);
    5348              :         }
    5349          548 :       buf.alias_type = build_pointer_type (type);
    5350          548 :       clear_padding_type (&buf, type, buf.sz, for_auto_init);
    5351          548 :       clear_padding_flush (&buf, true);
    5352              :     }
    5353              : 
    5354          634 :   gimple_stmt_iterator gsiprev2 = *gsi;
    5355          634 :   gsi_prev (&gsiprev2);
    5356          634 :   if (gsi_stmt (gsiprev) == gsi_stmt (gsiprev2))
    5357          126 :     gsi_replace (gsi, gimple_build_nop (), true);
    5358              :   else
    5359              :     {
    5360          508 :       gsi_remove (gsi, true);
    5361          508 :       *gsi = gsiprev2;
    5362              :     }
    5363          634 :   return true;
    5364              : }
    5365              : 
    5366              : /* Fold __builtin_constant_p builtin.  */
    5367              : 
    5368              : static bool
    5369       171030 : gimple_fold_builtin_constant_p (gimple_stmt_iterator *gsi)
    5370              : {
    5371       171030 :   gcall *call = as_a<gcall*>(gsi_stmt (*gsi));
    5372              : 
    5373       171030 :   if (gimple_call_num_args (call) != 1)
    5374              :     return false;
    5375              : 
    5376       171025 :   tree arg = gimple_call_arg (call, 0);
    5377       171025 :   tree result = fold_builtin_constant_p (arg);
    5378              : 
    5379              :   /* Resolve __builtin_constant_p.  If it hasn't been
    5380              :      folded to integer_one_node by now, it's fairly
    5381              :      certain that the value simply isn't constant.  */
    5382       339979 :   if (!result && fold_before_rtl_expansion_p ())
    5383            3 :     result = integer_zero_node;
    5384              : 
    5385       171025 :   if (!result)
    5386              :     return false;
    5387              : 
    5388         2074 :   gimplify_and_update_call_from_tree (gsi, result);
    5389         2074 :   return true;
    5390              : }
    5391              : 
    5392              : /* If va_list type is a simple pointer and nothing special is needed,
    5393              :    optimize __builtin_va_start (&ap, 0) into ap = __builtin_next_arg (0),
    5394              :    __builtin_va_end (&ap) out as NOP and __builtin_va_copy into a simple
    5395              :    pointer assignment.  Returns true if a change happened.  */
    5396              : 
    5397              : static bool
    5398       115243 : gimple_fold_builtin_stdarg (gimple_stmt_iterator *gsi, gcall *call)
    5399              : {
    5400              :   /* These shouldn't be folded before pass_stdarg.  */
    5401       115243 :   if (!fold_before_rtl_expansion_p ())
    5402              :     return false;
    5403              : 
    5404        10831 :   tree callee, lhs, rhs, cfun_va_list;
    5405        10831 :   bool va_list_simple_ptr;
    5406        10831 :   location_t loc = gimple_location (call);
    5407        10831 :   gimple *nstmt0, *nstmt;
    5408        10831 :   tree tlhs, oldvdef, newvdef;
    5409              : 
    5410        10831 :   callee = gimple_call_fndecl (call);
    5411              : 
    5412        10831 :   cfun_va_list = targetm.fn_abi_va_list (callee);
    5413        21662 :   va_list_simple_ptr = POINTER_TYPE_P (cfun_va_list)
    5414        10831 :                        && (TREE_TYPE (cfun_va_list) == void_type_node
    5415          436 :                            || TREE_TYPE (cfun_va_list) == char_type_node);
    5416              : 
    5417        10831 :   switch (DECL_FUNCTION_CODE (callee))
    5418              :     {
    5419         7016 :     case BUILT_IN_VA_START:
    5420         7016 :       if (!va_list_simple_ptr
    5421          172 :           || targetm.expand_builtin_va_start != NULL
    5422         7172 :           || !builtin_decl_explicit_p (BUILT_IN_NEXT_ARG))
    5423              :         return false;
    5424              : 
    5425          156 :       if (gimple_call_num_args (call) != 2)
    5426              :         return false;
    5427              : 
    5428          156 :       lhs = gimple_call_arg (call, 0);
    5429          156 :       if (!POINTER_TYPE_P (TREE_TYPE (lhs))
    5430          156 :           || TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (lhs)))
    5431          156 :              != TYPE_MAIN_VARIANT (cfun_va_list))
    5432              :         return false;
    5433              :       /* Create `tlhs = __builtin_next_arg(0);`. */
    5434          156 :       tlhs = make_ssa_name (cfun_va_list);
    5435          156 :       nstmt0 = gimple_build_call (builtin_decl_explicit (BUILT_IN_NEXT_ARG), 1, integer_zero_node);
    5436          156 :       lhs = fold_build2 (MEM_REF, cfun_va_list, lhs, build_zero_cst (TREE_TYPE (lhs)));
    5437          156 :       gimple_call_set_lhs (nstmt0, tlhs);
    5438          156 :       gimple_set_location (nstmt0, loc);
    5439          156 :       gimple_move_vops (nstmt0, call);
    5440          156 :       gsi_replace (gsi, nstmt0, false);
    5441          156 :       oldvdef = gimple_vdef (nstmt0);
    5442          156 :       newvdef = make_ssa_name (gimple_vop (cfun), nstmt0);
    5443          156 :       gimple_set_vdef (nstmt0, newvdef);
    5444              : 
    5445              :       /* Create `*lhs = tlhs;`.  */
    5446          156 :       nstmt = gimple_build_assign (lhs, tlhs);
    5447          156 :       gimple_set_location (nstmt, loc);
    5448          156 :       gimple_set_vuse (nstmt, newvdef);
    5449          156 :       gimple_set_vdef (nstmt, oldvdef);
    5450          156 :       SSA_NAME_DEF_STMT (oldvdef) = nstmt;
    5451          156 :       gsi_insert_after (gsi, nstmt, GSI_NEW_STMT);
    5452              : 
    5453          156 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5454              :         {
    5455            0 :           fprintf (dump_file, "Simplified\n  ");
    5456            0 :           print_gimple_stmt (dump_file, call, 0, dump_flags);
    5457            0 :           fprintf (dump_file, "into\n  ");
    5458            0 :           print_gimple_stmt (dump_file, nstmt0, 0, dump_flags);
    5459            0 :           fprintf (dump_file, "  ");
    5460            0 :           print_gimple_stmt (dump_file, nstmt, 0, dump_flags);
    5461              :         }
    5462              :       return true;
    5463              : 
    5464          244 :     case BUILT_IN_VA_COPY:
    5465          244 :       if (!va_list_simple_ptr)
    5466              :         return false;
    5467              : 
    5468           47 :       if (gimple_call_num_args (call) != 2)
    5469              :         return false;
    5470              : 
    5471           47 :       lhs = gimple_call_arg (call, 0);
    5472           47 :       if (!POINTER_TYPE_P (TREE_TYPE (lhs))
    5473           47 :           || TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (lhs)))
    5474           47 :              != TYPE_MAIN_VARIANT (cfun_va_list))
    5475              :         return false;
    5476           47 :       rhs = gimple_call_arg (call, 1);
    5477           47 :       if (TYPE_MAIN_VARIANT (TREE_TYPE (rhs))
    5478           47 :           != TYPE_MAIN_VARIANT (cfun_va_list))
    5479              :         return false;
    5480              : 
    5481           47 :       lhs = fold_build2 (MEM_REF, cfun_va_list, lhs, build_zero_cst (TREE_TYPE (lhs)));
    5482           47 :       nstmt = gimple_build_assign (lhs, rhs);
    5483           47 :       gimple_set_location (nstmt, loc);
    5484           47 :       gimple_move_vops (nstmt, call);
    5485           47 :       gsi_replace (gsi, nstmt, false);
    5486              : 
    5487           47 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5488              :         {
    5489            0 :           fprintf (dump_file, "Simplified\n  ");
    5490            0 :           print_gimple_stmt (dump_file, call, 0, dump_flags);
    5491            0 :           fprintf (dump_file, "into\n  ");
    5492            0 :           print_gimple_stmt (dump_file, nstmt, 0, dump_flags);
    5493              :         }
    5494              :       return true;
    5495              : 
    5496         3571 :     case BUILT_IN_VA_END:
    5497              :       /* No effect, so the statement will be deleted.  */
    5498         3571 :       if (dump_file && (dump_flags & TDF_DETAILS))
    5499              :         {
    5500            0 :           fprintf (dump_file, "Removed\n  ");
    5501            0 :           print_gimple_stmt (dump_file, call, 0, dump_flags);
    5502              :         }
    5503         3571 :       unlink_stmt_vdef (call);
    5504         3571 :       release_defs (call);
    5505         3571 :       gsi_replace (gsi, gimple_build_nop (), false);
    5506         3571 :       return true;
    5507              : 
    5508            0 :     default:
    5509            0 :       gcc_unreachable ();
    5510              :     }
    5511              : }
    5512              : 
    5513              : /* Fold __builtin_call_{code_address,static_chain} builtins.  This handles
    5514              :    only the trivial left-over cases not processed in tree-nested.cc.  */
    5515              : 
    5516              : static bool
    5517            2 : gimple_fold_builtin_call_info (gimple_stmt_iterator *gsi,
    5518              :                                enum built_in_function fcode)
    5519              : {
    5520            2 :   gcall *stmt = as_a <gcall *>(gsi_stmt (*gsi));
    5521            2 :   tree arg = gimple_call_arg (stmt, 0);
    5522              : 
    5523              :   /* The error will be emitted in builtins.cc.  */
    5524            2 :   if (TREE_CODE (arg) != ADDR_EXPR
    5525            2 :       || FUNCTION_DECL != TREE_CODE (TREE_OPERAND (arg, 0)))
    5526              :     return false;
    5527              : 
    5528              :   /* The case with static chain is handled in tree-nested.cc.  */
    5529            2 :   gcc_assert (!DECL_STATIC_CHAIN (TREE_OPERAND (arg, 0)));
    5530              : 
    5531            2 :   if (fcode == BUILT_IN_CALL_STATIC_CHAIN)
    5532            1 :     replace_call_with_value (gsi, null_pointer_node);
    5533              :   else
    5534            1 :     replace_call_with_value (gsi, arg);
    5535              : 
    5536              :   return true;
    5537              : }
    5538              : 
    5539              : 
    5540              : /* Fold the non-target builtin at *GSI and return whether any simplification
    5541              :    was made.  */
    5542              : 
    5543              : static bool
    5544      9666288 : gimple_fold_builtin (gimple_stmt_iterator *gsi)
    5545              : {
    5546      9666288 :   gcall *stmt = as_a <gcall *>(gsi_stmt (*gsi));
    5547      9666288 :   tree callee = gimple_call_fndecl (stmt);
    5548              : 
    5549              :   /* Give up for always_inline inline builtins until they are
    5550              :      inlined.  */
    5551      9666288 :   if (avoid_folding_inline_builtin (callee))
    5552              :     return false;
    5553              : 
    5554      9665111 :   unsigned n = gimple_call_num_args (stmt);
    5555      9665111 :   enum built_in_function fcode = DECL_FUNCTION_CODE (callee);
    5556      9665111 :   switch (fcode)
    5557              :     {
    5558       115243 :     case BUILT_IN_VA_START:
    5559       115243 :     case BUILT_IN_VA_END:
    5560       115243 :     case BUILT_IN_VA_COPY:
    5561       115243 :       return gimple_fold_builtin_stdarg (gsi, stmt);
    5562          149 :     case BUILT_IN_BCMP:
    5563          149 :       return gimple_fold_builtin_bcmp (gsi);
    5564          367 :     case BUILT_IN_BCOPY:
    5565          367 :       return gimple_fold_builtin_bcopy (gsi);
    5566          250 :     case BUILT_IN_BZERO:
    5567          250 :       return gimple_fold_builtin_bzero (gsi);
    5568              : 
    5569       318809 :     case BUILT_IN_MEMSET:
    5570       318809 :       return gimple_fold_builtin_memset (gsi,
    5571              :                                          gimple_call_arg (stmt, 1),
    5572       318809 :                                          gimple_call_arg (stmt, 2));
    5573        10287 :     case BUILT_IN_MEMPCPY:
    5574        10287 :       if (gimple_fold_builtin_memory_op (gsi, gimple_call_arg (stmt, 0),
    5575              :                                             gimple_call_arg (stmt, 1), fcode))
    5576              :         return true;
    5577         9545 :       return gimple_fold_builtin_mempcpy (gsi);
    5578      1020556 :     case BUILT_IN_MEMCPY:
    5579      1020556 :     case BUILT_IN_MEMMOVE:
    5580      1020556 :       return gimple_fold_builtin_memory_op (gsi, gimple_call_arg (stmt, 0),
    5581      1020556 :                                             gimple_call_arg (stmt, 1), fcode);
    5582         4471 :     case BUILT_IN_SPRINTF_CHK:
    5583         4471 :     case BUILT_IN_VSPRINTF_CHK:
    5584         4471 :       return gimple_fold_builtin_sprintf_chk (gsi, fcode);
    5585         1493 :     case BUILT_IN_STRCAT_CHK:
    5586         1493 :       return gimple_fold_builtin_strcat_chk (gsi);
    5587         1116 :     case BUILT_IN_STRNCAT_CHK:
    5588         1116 :       return gimple_fold_builtin_strncat_chk (gsi);
    5589       140552 :     case BUILT_IN_STRLEN:
    5590       140552 :       return gimple_fold_builtin_strlen (gsi);
    5591        26067 :     case BUILT_IN_STRCPY:
    5592        26067 :       return gimple_fold_builtin_strcpy (gsi,
    5593              :                                          gimple_call_arg (stmt, 0),
    5594        26067 :                                          gimple_call_arg (stmt, 1));
    5595        17276 :     case BUILT_IN_STRNCPY:
    5596        17276 :       return gimple_fold_builtin_strncpy (gsi,
    5597              :                                           gimple_call_arg (stmt, 0),
    5598              :                                           gimple_call_arg (stmt, 1),
    5599        17276 :                                           gimple_call_arg (stmt, 2));
    5600         7330 :     case BUILT_IN_STRCAT:
    5601         7330 :       return gimple_fold_builtin_strcat (gsi, gimple_call_arg (stmt, 0),
    5602         7330 :                                          gimple_call_arg (stmt, 1));
    5603         6786 :     case BUILT_IN_STRNCAT:
    5604         6786 :       return gimple_fold_builtin_strncat (gsi);
    5605         4743 :     case BUILT_IN_INDEX:
    5606         4743 :     case BUILT_IN_STRCHR:
    5607         4743 :       return gimple_fold_builtin_strchr (gsi, false);
    5608          729 :     case BUILT_IN_RINDEX:
    5609          729 :     case BUILT_IN_STRRCHR:
    5610          729 :       return gimple_fold_builtin_strchr (gsi, true);
    5611         4250 :     case BUILT_IN_STRSTR:
    5612         4250 :       return gimple_fold_builtin_strstr (gsi);
    5613      1251050 :     case BUILT_IN_STRCMP:
    5614      1251050 :     case BUILT_IN_STRCMP_EQ:
    5615      1251050 :     case BUILT_IN_STRCASECMP:
    5616      1251050 :     case BUILT_IN_STRNCMP:
    5617      1251050 :     case BUILT_IN_STRNCMP_EQ:
    5618      1251050 :     case BUILT_IN_STRNCASECMP:
    5619      1251050 :       return gimple_fold_builtin_string_compare (gsi);
    5620        30365 :     case BUILT_IN_MEMCHR:
    5621        30365 :       return gimple_fold_builtin_memchr (gsi);
    5622        20710 :     case BUILT_IN_FPUTS:
    5623        20710 :       return gimple_fold_builtin_fputs (gsi, gimple_call_arg (stmt, 0),
    5624        20710 :                                         gimple_call_arg (stmt, 1), false);
    5625           43 :     case BUILT_IN_FPUTS_UNLOCKED:
    5626           43 :       return gimple_fold_builtin_fputs (gsi, gimple_call_arg (stmt, 0),
    5627           43 :                                         gimple_call_arg (stmt, 1), true);
    5628        40857 :     case BUILT_IN_FWRITE:
    5629        40857 :       return gimple_fold_builtin_fwrite (gsi, false);
    5630          267 :     case BUILT_IN_FWRITE_UNLOCKED:
    5631          267 :       return gimple_fold_builtin_fwrite (gsi, true);
    5632        25701 :     case BUILT_IN_MEMCPY_CHK:
    5633        25701 :     case BUILT_IN_MEMPCPY_CHK:
    5634        25701 :     case BUILT_IN_MEMMOVE_CHK:
    5635        25701 :     case BUILT_IN_MEMSET_CHK:
    5636        25701 :       return gimple_fold_builtin_memory_chk (gsi,
    5637              :                                              gimple_call_arg (stmt, 0),
    5638              :                                              gimple_call_arg (stmt, 1),
    5639              :                                              gimple_call_arg (stmt, 2),
    5640              :                                              gimple_call_arg (stmt, 3),
    5641        25701 :                                              fcode);
    5642         3674 :     case BUILT_IN_STPCPY:
    5643         3674 :       return gimple_fold_builtin_stpcpy (gsi);
    5644         2566 :     case BUILT_IN_STRCPY_CHK:
    5645         2566 :     case BUILT_IN_STPCPY_CHK:
    5646         2566 :       return gimple_fold_builtin_stxcpy_chk (gsi,
    5647              :                                              gimple_call_arg (stmt, 0),
    5648              :                                              gimple_call_arg (stmt, 1),
    5649              :                                              gimple_call_arg (stmt, 2),
    5650         2566 :                                              fcode);
    5651         2721 :     case BUILT_IN_STRNCPY_CHK:
    5652         2721 :     case BUILT_IN_STPNCPY_CHK:
    5653         2721 :       return gimple_fold_builtin_stxncpy_chk (gsi,
    5654              :                                               gimple_call_arg (stmt, 0),
    5655              :                                               gimple_call_arg (stmt, 1),
    5656              :                                               gimple_call_arg (stmt, 2),
    5657              :                                               gimple_call_arg (stmt, 3),
    5658         2721 :                                               fcode);
    5659         2359 :     case BUILT_IN_SNPRINTF_CHK:
    5660         2359 :     case BUILT_IN_VSNPRINTF_CHK:
    5661         2359 :       return gimple_fold_builtin_snprintf_chk (gsi, fcode);
    5662              : 
    5663       867081 :     case BUILT_IN_FPRINTF:
    5664       867081 :     case BUILT_IN_FPRINTF_UNLOCKED:
    5665       867081 :     case BUILT_IN_VFPRINTF:
    5666       867081 :       if (n == 2 || n == 3)
    5667        95246 :         return gimple_fold_builtin_fprintf (gsi,
    5668              :                                             gimple_call_arg (stmt, 0),
    5669              :                                             gimple_call_arg (stmt, 1),
    5670              :                                             n == 3
    5671        42656 :                                             ? gimple_call_arg (stmt, 2)
    5672              :                                             : NULL_TREE,
    5673        52590 :                                             fcode);
    5674              :       break;
    5675         2229 :     case BUILT_IN_FPRINTF_CHK:
    5676         2229 :     case BUILT_IN_VFPRINTF_CHK:
    5677         2229 :       if (n == 3 || n == 4)
    5678         3814 :         return gimple_fold_builtin_fprintf (gsi,
    5679              :                                             gimple_call_arg (stmt, 0),
    5680              :                                             gimple_call_arg (stmt, 2),
    5681              :                                             n == 4
    5682         1744 :                                             ? gimple_call_arg (stmt, 3)
    5683              :                                             : NULL_TREE,
    5684         2070 :                                             fcode);
    5685              :       break;
    5686       199424 :     case BUILT_IN_PRINTF:
    5687       199424 :     case BUILT_IN_PRINTF_UNLOCKED:
    5688       199424 :     case BUILT_IN_VPRINTF:
    5689       199424 :       if (n == 1 || n == 2)
    5690       234627 :         return gimple_fold_builtin_printf (gsi, gimple_call_arg (stmt, 0),
    5691              :                                            n == 2
    5692       112759 :                                            ? gimple_call_arg (stmt, 1)
    5693       121868 :                                            : NULL_TREE, fcode);
    5694              :       break;
    5695         2273 :     case BUILT_IN_PRINTF_CHK:
    5696         2273 :     case BUILT_IN_VPRINTF_CHK:
    5697         2273 :       if (n == 2 || n == 3)
    5698         3893 :         return gimple_fold_builtin_printf (gsi, gimple_call_arg (stmt, 1),
    5699              :                                            n == 3
    5700         1771 :                                            ? gimple_call_arg (stmt, 2)
    5701         2122 :                                            : NULL_TREE, fcode);
    5702              :       break;
    5703         2866 :     case BUILT_IN_ACC_ON_DEVICE:
    5704         2866 :       return gimple_fold_builtin_acc_on_device (gsi,
    5705         2866 :                                                 gimple_call_arg (stmt, 0));
    5706          262 :     case BUILT_IN_OMP_IS_INITIAL_DEVICE:
    5707          262 :       return gimple_fold_builtin_omp_is_initial_device (gsi);
    5708              : 
    5709          103 :     case BUILT_IN_OMP_GET_INITIAL_DEVICE:
    5710          103 :       return gimple_fold_builtin_omp_get_initial_device (gsi);
    5711              : 
    5712          340 :     case BUILT_IN_OMP_GET_NUM_DEVICES:
    5713          340 :       return gimple_fold_builtin_omp_get_num_devices (gsi);
    5714              : 
    5715        50579 :     case BUILT_IN_REALLOC:
    5716        50579 :       return gimple_fold_builtin_realloc (gsi);
    5717              : 
    5718          634 :     case BUILT_IN_CLEAR_PADDING:
    5719          634 :       return gimple_fold_builtin_clear_padding (gsi);
    5720              : 
    5721       171030 :     case BUILT_IN_CONSTANT_P:
    5722       171030 :       return gimple_fold_builtin_constant_p (gsi);
    5723              : 
    5724            2 :     case BUILT_IN_CALL_CODE_ADDRESS:
    5725            2 :     case BUILT_IN_CALL_STATIC_CHAIN:
    5726            2 :       return gimple_fold_builtin_call_info (gsi, fcode);
    5727              : 
    5728      6199858 :     default:;
    5729              :     }
    5730              : 
    5731              :   /* Try the generic builtin folder.  */
    5732      6199858 :   bool ignore = (gimple_call_lhs (stmt) == NULL);
    5733      6199858 :   tree result = fold_call_stmt (stmt, ignore);
    5734      6199858 :   if (result)
    5735              :     {
    5736         5131 :       if (ignore)
    5737         1261 :         STRIP_NOPS (result);
    5738              :       else
    5739         3870 :         result = fold_convert (gimple_call_return_type (stmt), result);
    5740         5131 :       gimplify_and_update_call_from_tree (gsi, result);
    5741         5131 :       return true;
    5742              :     }
    5743              : 
    5744              :   return false;
    5745              : }
    5746              : 
    5747              : /* Transform IFN_GOACC_DIM_SIZE and IFN_GOACC_DIM_POS internal
    5748              :    function calls to constants, where possible.  */
    5749              : 
    5750              : static tree
    5751        20589 : fold_internal_goacc_dim (const gimple *call)
    5752              : {
    5753        20589 :   int axis = oacc_get_ifn_dim_arg (call);
    5754        20589 :   int size = oacc_get_fn_dim_size (current_function_decl, axis);
    5755        20589 :   tree result = NULL_TREE;
    5756        20589 :   tree type = TREE_TYPE (gimple_call_lhs (call));
    5757              : 
    5758        20589 :   switch (gimple_call_internal_fn (call))
    5759              :     {
    5760         8915 :     case IFN_GOACC_DIM_POS:
    5761              :       /* If the size is 1, we know the answer.  */
    5762         8915 :       if (size == 1)
    5763         8915 :         result = build_int_cst (type, 0);
    5764              :       break;
    5765        11674 :     case IFN_GOACC_DIM_SIZE:
    5766              :       /* If the size is not dynamic, we know the answer.  */
    5767        11674 :       if (size)
    5768        11674 :         result = build_int_cst (type, size);
    5769              :       break;
    5770              :     default:
    5771              :       break;
    5772              :     }
    5773              : 
    5774        20589 :   return result;
    5775              : }
    5776              : 
    5777              : /* Return true if stmt is __atomic_compare_exchange_N call which is suitable
    5778              :    for conversion into ATOMIC_COMPARE_EXCHANGE if the second argument is
    5779              :    &var where var is only addressable because of such calls.  */
    5780              : 
    5781              : bool
    5782     61148270 : optimize_atomic_compare_exchange_p (gimple *stmt)
    5783              : {
    5784     61148270 :   if (gimple_call_num_args (stmt) != 6
    5785      1685655 :       || !flag_inline_atomics
    5786      1685655 :       || !optimize
    5787      1685655 :       || sanitize_flags_p (SANITIZE_THREAD | SANITIZE_ADDRESS)
    5788      1685574 :       || !gimple_call_builtin_p (stmt, BUILT_IN_NORMAL)
    5789      1134464 :       || !gimple_vdef (stmt)
    5790     62160407 :       || !gimple_vuse (stmt))
    5791              :     return false;
    5792              : 
    5793      1012137 :   tree fndecl = gimple_call_fndecl (stmt);
    5794      1012137 :   switch (DECL_FUNCTION_CODE (fndecl))
    5795              :     {
    5796        52030 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
    5797        52030 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
    5798        52030 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
    5799        52030 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
    5800        52030 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
    5801        52030 :       break;
    5802              :     default:
    5803              :       return false;
    5804              :     }
    5805              : 
    5806        52030 :   tree expected = gimple_call_arg (stmt, 1);
    5807        52030 :   if (TREE_CODE (expected) != ADDR_EXPR
    5808        52030 :       || !SSA_VAR_P (TREE_OPERAND (expected, 0)))
    5809              :     return false;
    5810              : 
    5811        49642 :   tree etype = TREE_TYPE (TREE_OPERAND (expected, 0));
    5812        49642 :   if (!is_gimple_reg_type (etype)
    5813        49144 :       || !auto_var_in_fn_p (TREE_OPERAND (expected, 0), current_function_decl)
    5814        46746 :       || TREE_THIS_VOLATILE (etype)
    5815        46746 :       || VECTOR_TYPE_P (etype)
    5816              :       || TREE_CODE (etype) == COMPLEX_TYPE
    5817              :       /* Don't optimize floating point expected vars, VIEW_CONVERT_EXPRs
    5818              :          might not preserve all the bits.  See PR71716.  */
    5819              :       || SCALAR_FLOAT_TYPE_P (etype)
    5820        67726 :       || maybe_ne (TYPE_PRECISION (etype),
    5821        36168 :                    GET_MODE_BITSIZE (TYPE_MODE (etype))))
    5822              :     return false;
    5823              : 
    5824        11660 :   tree weak = gimple_call_arg (stmt, 3);
    5825        11660 :   if (!integer_zerop (weak) && !integer_onep (weak))
    5826              :     return false;
    5827              : 
    5828        11660 :   tree parmt = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
    5829        11660 :   tree itype = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (parmt)));
    5830        11660 :   machine_mode mode = TYPE_MODE (itype);
    5831              : 
    5832        11660 :   if (direct_optab_handler (atomic_compare_and_swap_optab, mode)
    5833              :       == CODE_FOR_nothing
    5834        11660 :       && optab_handler (sync_compare_and_swap_optab, mode) == CODE_FOR_nothing)
    5835              :     return false;
    5836              : 
    5837        23320 :   if (maybe_ne (int_size_in_bytes (etype), GET_MODE_SIZE (mode)))
    5838            0 :     return false;
    5839              : 
    5840              :   return true;
    5841              : }
    5842              : 
    5843              : /* Fold
    5844              :      r = __atomic_compare_exchange_N (p, &e, d, w, s, f);
    5845              :    into
    5846              :      _Complex uintN_t t = ATOMIC_COMPARE_EXCHANGE (p, e, d, w * 256 + N, s, f);
    5847              :      i = IMAGPART_EXPR <t>;
    5848              :      r = (_Bool) i;
    5849              :      e = REALPART_EXPR <t>;  */
    5850              : 
    5851              : void
    5852         5787 : fold_builtin_atomic_compare_exchange (gimple_stmt_iterator *gsi)
    5853              : {
    5854         5787 :   gimple *stmt = gsi_stmt (*gsi);
    5855         5787 :   tree fndecl = gimple_call_fndecl (stmt);
    5856         5787 :   tree parmt = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
    5857         5787 :   tree itype = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (parmt)));
    5858         5787 :   tree ctype = build_complex_type (itype);
    5859         5787 :   tree expected = TREE_OPERAND (gimple_call_arg (stmt, 1), 0);
    5860         5787 :   bool throws = false;
    5861         5787 :   edge e = NULL;
    5862         5787 :   gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (expected)),
    5863              :                                    expected);
    5864         5787 :   gsi_insert_before (gsi, g, GSI_SAME_STMT);
    5865         5787 :   gimple_stmt_iterator gsiret = gsi_for_stmt (g);
    5866         5787 :   if (!useless_type_conversion_p (itype, TREE_TYPE (expected)))
    5867              :     {
    5868         2651 :       g = gimple_build_assign (make_ssa_name (itype), VIEW_CONVERT_EXPR,
    5869              :                                build1 (VIEW_CONVERT_EXPR, itype,
    5870              :                                        gimple_assign_lhs (g)));
    5871         2651 :       gsi_insert_before (gsi, g, GSI_SAME_STMT);
    5872              :     }
    5873         5787 :   int flag = (integer_onep (gimple_call_arg (stmt, 3)) ? 256 : 0)
    5874        11119 :              + int_size_in_bytes (itype);
    5875         5787 :   g = gimple_build_call_internal (IFN_ATOMIC_COMPARE_EXCHANGE, 6,
    5876              :                                   gimple_call_arg (stmt, 0),
    5877              :                                   gimple_assign_lhs (g),
    5878              :                                   gimple_call_arg (stmt, 2),
    5879         5787 :                                   build_int_cst (integer_type_node, flag),
    5880              :                                   gimple_call_arg (stmt, 4),
    5881              :                                   gimple_call_arg (stmt, 5));
    5882         5787 :   tree lhs = make_ssa_name (ctype);
    5883         5787 :   gimple_call_set_lhs (g, lhs);
    5884         5787 :   gimple_move_vops (g, stmt);
    5885         5787 :   tree oldlhs = gimple_call_lhs (stmt);
    5886         5787 :   if (stmt_can_throw_internal (cfun, stmt))
    5887              :     {
    5888          203 :       throws = true;
    5889          203 :       e = find_fallthru_edge (gsi_bb (*gsi)->succs);
    5890              :     }
    5891         5787 :   gimple_call_set_nothrow (as_a <gcall *> (g),
    5892         5787 :                            gimple_call_nothrow_p (as_a <gcall *> (stmt)));
    5893         5787 :   gimple_call_set_lhs (stmt, NULL_TREE);
    5894         5787 :   gsi_replace (gsi, g, true);
    5895         5787 :   if (oldlhs)
    5896              :     {
    5897         5740 :       g = gimple_build_assign (make_ssa_name (itype), IMAGPART_EXPR,
    5898              :                                build1 (IMAGPART_EXPR, itype, lhs));
    5899         5740 :       if (throws)
    5900              :         {
    5901          197 :           gsi_insert_on_edge_immediate (e, g);
    5902          197 :           *gsi = gsi_for_stmt (g);
    5903              :         }
    5904              :       else
    5905         5543 :         gsi_insert_after (gsi, g, GSI_NEW_STMT);
    5906         5740 :       g = gimple_build_assign (oldlhs, NOP_EXPR, gimple_assign_lhs (g));
    5907         5740 :       gsi_insert_after (gsi, g, GSI_NEW_STMT);
    5908              :     }
    5909         5787 :   g = gimple_build_assign (make_ssa_name (itype), REALPART_EXPR,
    5910              :                            build1 (REALPART_EXPR, itype, lhs));
    5911         5787 :   if (throws && oldlhs == NULL_TREE)
    5912              :     {
    5913            6 :       gsi_insert_on_edge_immediate (e, g);
    5914            6 :       *gsi = gsi_for_stmt (g);
    5915              :     }
    5916              :   else
    5917         5781 :     gsi_insert_after (gsi, g, GSI_NEW_STMT);
    5918         5787 :   if (!useless_type_conversion_p (TREE_TYPE (expected), itype))
    5919              :     {
    5920         5302 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (expected)),
    5921              :                                VIEW_CONVERT_EXPR,
    5922         2651 :                                build1 (VIEW_CONVERT_EXPR, TREE_TYPE (expected),
    5923              :                                        gimple_assign_lhs (g)));
    5924         2651 :       gsi_insert_after (gsi, g, GSI_NEW_STMT);
    5925              :     }
    5926         5787 :   g = gimple_build_assign (expected, SSA_NAME, gimple_assign_lhs (g));
    5927         5787 :   gsi_insert_after (gsi, g, GSI_NEW_STMT);
    5928         5787 :   *gsi = gsiret;
    5929         5787 : }
    5930              : 
    5931              : /* Return true if ARG0 CODE ARG1 in infinite signed precision operation
    5932              :    doesn't fit into TYPE.  The test for overflow should be regardless of
    5933              :    -fwrapv, and even for unsigned types.  */
    5934              : 
    5935              : bool
    5936       451942 : arith_overflowed_p (enum tree_code code, const_tree type,
    5937              :                     const_tree arg0, const_tree arg1)
    5938              : {
    5939       451942 :   widest2_int warg0 = widest2_int_cst (arg0);
    5940       451942 :   widest2_int warg1 = widest2_int_cst (arg1);
    5941       451942 :   widest2_int wres;
    5942       451942 :   switch (code)
    5943              :     {
    5944        97911 :     case PLUS_EXPR: wres = wi::add (warg0, warg1); break;
    5945       115917 :     case MINUS_EXPR: wres = wi::sub (warg0, warg1); break;
    5946       239565 :     case MULT_EXPR: wres = wi::mul (warg0, warg1); break;
    5947            0 :     default: gcc_unreachable ();
    5948              :     }
    5949       451942 :   signop sign = TYPE_SIGN (type);
    5950       451942 :   if (sign == UNSIGNED && wi::neg_p (wres))
    5951              :     return true;
    5952       380004 :   return wi::min_precision (wres, sign) > TYPE_PRECISION (type);
    5953       452082 : }
    5954              : 
    5955              : /* Mask state for partial load/store operations (mask and length).  */
    5956              : enum mask_load_store_state {
    5957              :   MASK_ALL_INACTIVE,  /* All lanes/elements are inactive (can be elided).  */
    5958              :   MASK_ALL_ACTIVE,    /* All lanes/elements are active (unconditional).  */
    5959              :   MASK_UNKNOWN
    5960              : };
    5961              : 
    5962              : /* Check the mask/length state of IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL.
    5963              :    Returns whether all elements are active, all inactive, or mixed.
    5964              :    VECTYPE is the vector type of the operation.  */
    5965              : 
    5966              : static enum mask_load_store_state
    5967         8059 : partial_load_store_mask_state (gcall *call, tree vectype)
    5968              : {
    5969         8059 :   internal_fn ifn = gimple_call_internal_fn (call);
    5970         8059 :   int mask_index = internal_fn_mask_index (ifn);
    5971         8059 :   int len_index = internal_fn_len_index (ifn);
    5972              : 
    5973              :   /* Extract length and mask arguments up front.  */
    5974         8059 :   tree len = len_index != -1 ? gimple_call_arg (call, len_index) : NULL_TREE;
    5975            0 :   tree bias = len ? gimple_call_arg (call, len_index + 1) : NULL_TREE;
    5976         8059 :   tree mask = mask_index != -1 ? gimple_call_arg (call, mask_index) : NULL_TREE;
    5977              : 
    5978        16118 :   poly_int64 nelts = GET_MODE_NUNITS (TYPE_MODE (vectype));
    5979              : 
    5980         8059 :   poly_widest_int wlen = -1;
    5981         8059 :   bool full_length_p = !len;  /* No length means full length.  */
    5982              : 
    5983              :   /* Compute effective length.  */
    5984         8059 :   if (len && poly_int_tree_p (len))
    5985              :     {
    5986            0 :       gcc_assert (TREE_CODE (bias) == INTEGER_CST);
    5987            0 :       wlen = wi::to_poly_widest (len) - wi::to_widest (bias);
    5988              : 
    5989            0 :       if (known_eq (wlen, 0))
    5990              :         return MASK_ALL_INACTIVE;
    5991              : 
    5992            0 :       if (known_eq (wlen, nelts))
    5993              :         full_length_p = true;
    5994              :       else
    5995         8059 :         full_length_p = false;
    5996              :     }
    5997              : 
    5998              :   /* Check mask for early return cases.  */
    5999         8059 :   if (mask)
    6000              :     {
    6001         8059 :       if (integer_zerop (mask))
    6002              :         return MASK_ALL_INACTIVE;
    6003              : 
    6004         8044 :       if (full_length_p && integer_all_onesp (mask))
    6005              :         return MASK_ALL_ACTIVE;
    6006              :     }
    6007            0 :   else if (full_length_p)
    6008              :     /* No mask and full length means all active.  */
    6009              :     return MASK_ALL_ACTIVE;
    6010              : 
    6011              :   /* For VLA vectors, we can't do much more.  */
    6012         7894 :   if (!nelts.is_constant ())
    6013              :     return MASK_UNKNOWN;
    6014              : 
    6015              :   /* Same for VLS vectors with non-constant mask.  */
    6016         7894 :   if (mask && TREE_CODE (mask) != VECTOR_CST)
    6017              :     return MASK_UNKNOWN;
    6018              : 
    6019              :   /* Check VLS vector elements.  */
    6020          504 :   gcc_assert (wlen.is_constant ());
    6021              : 
    6022          504 :   HOST_WIDE_INT active_len = wlen.to_constant ().to_shwi ();
    6023          504 :   if (active_len == -1)
    6024          504 :     active_len = nelts.to_constant ();
    6025              : 
    6026              :   /* Check if all elements in the active range match the mask.  */
    6027         1038 :   for (HOST_WIDE_INT i = 0; i < active_len; i++)
    6028              :     {
    6029         1038 :       bool elt_active = !mask || !integer_zerop (vector_cst_elt (mask, i));
    6030              :       if (!elt_active)
    6031              :         {
    6032              :           /* Found an inactive element.  Check if all are inactive.  */
    6033         1002 :           for (HOST_WIDE_INT j = 0; j < active_len; j++)
    6034         1002 :             if (!mask || !integer_zerop (vector_cst_elt (mask, j)))
    6035              :               return MASK_UNKNOWN;  /* Mixed state.  */
    6036              :           return MASK_ALL_INACTIVE;
    6037              :         }
    6038              :     }
    6039              : 
    6040              :   /* All elements in active range are active.  */
    6041            0 :   return full_length_p ? MASK_ALL_ACTIVE : MASK_UNKNOWN;
    6042         8059 : }
    6043              : 
    6044              : 
    6045              : /* If IFN_{MASK,LEN,MASK_LEN}_LOAD/STORE call CALL is unconditional
    6046              :    (all lanes active), return a MEM_REF for the memory it references.
    6047              :    Otherwise return NULL_TREE.  VECTYPE is the type of the memory vector.  */
    6048              : 
    6049              : static tree
    6050         4022 : gimple_fold_partial_load_store_mem_ref (gcall *call, tree vectype)
    6051              : {
    6052              :   /* Only fold if all lanes are active (unconditional).  */
    6053         4022 :   if (partial_load_store_mask_state (call, vectype) != MASK_ALL_ACTIVE)
    6054              :     return NULL_TREE;
    6055              : 
    6056           75 :   tree ptr = gimple_call_arg (call, 0);
    6057           75 :   tree alias_align = gimple_call_arg (call, 1);
    6058           75 :   if (!tree_fits_uhwi_p (alias_align))
    6059              :     return NULL_TREE;
    6060              : 
    6061           75 :   unsigned HOST_WIDE_INT align = tree_to_uhwi (alias_align);
    6062           75 :   if (TYPE_ALIGN (vectype) != align)
    6063           14 :     vectype = build_aligned_type (vectype, align);
    6064           75 :   tree offset = build_zero_cst (TREE_TYPE (alias_align));
    6065           75 :   return fold_build2 (MEM_REF, vectype, ptr, offset);
    6066              : }
    6067              : 
    6068              : /* Try to fold IFN_{MASK,LEN}_LOAD/STORE call CALL.  Return true on success.  */
    6069              : 
    6070              : static bool
    6071         4037 : gimple_fold_partial_load_store (gimple_stmt_iterator *gsi, gcall *call)
    6072              : {
    6073         4037 :   internal_fn ifn = gimple_call_internal_fn (call);
    6074         4037 :   tree lhs = gimple_call_lhs (call);
    6075         4037 :   bool is_load = (lhs != NULL_TREE);
    6076         4037 :   tree vectype;
    6077              : 
    6078         4037 :   if (is_load)
    6079         2003 :     vectype = TREE_TYPE (lhs);
    6080              :   else
    6081              :     {
    6082         2034 :       tree rhs = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
    6083         2034 :       vectype = TREE_TYPE (rhs);
    6084              :     }
    6085              : 
    6086         4037 :   enum mask_load_store_state state
    6087         4037 :     = partial_load_store_mask_state (call, vectype);
    6088              : 
    6089              :   /* Handle all-inactive case.  */
    6090         4037 :   if (state == MASK_ALL_INACTIVE)
    6091              :     {
    6092           15 :       if (is_load)
    6093              :         {
    6094              :           /* Replace load with else value.  */
    6095           15 :           int else_index = internal_fn_else_index (ifn);
    6096           15 :           tree else_value = gimple_call_arg (call, else_index);
    6097           15 :           if (!is_gimple_reg (lhs))
    6098              :             {
    6099            0 :               if (!zerop (else_value))
    6100              :                 return false;
    6101            0 :               else_value = build_constructor (TREE_TYPE (lhs), NULL);
    6102              :             }
    6103           15 :           gassign *new_stmt = gimple_build_assign (lhs, else_value);
    6104           15 :           gimple_set_location (new_stmt, gimple_location (call));
    6105              :           /* When the lhs is an array for LANES version, then there is still
    6106              :              a store, move the vops from the old stmt to the new one.  */
    6107           15 :           if (!is_gimple_reg (lhs))
    6108            0 :             gimple_move_vops (new_stmt, call);
    6109           15 :           gsi_replace (gsi, new_stmt, false);
    6110           15 :           return true;
    6111              :         }
    6112              :       else
    6113              :         {
    6114              :           /* Remove inactive store altogether.  */
    6115            0 :           unlink_stmt_vdef (call);
    6116            0 :           release_defs (call);
    6117            0 :           gsi_replace (gsi, gimple_build_nop (), true);
    6118            0 :           return true;
    6119              :         }
    6120              :     }
    6121              : 
    6122              :   /* We cannot simplify a gather/scatter or load/store lanes further.  */
    6123         4022 :   if (internal_gather_scatter_fn_p (ifn)
    6124         4022 :       || TREE_CODE (vectype) == ARRAY_TYPE)
    6125              :     return false;
    6126              : 
    6127              :   /* Handle all-active case by folding to regular memory operation.  */
    6128         4022 :   if (tree mem_ref = gimple_fold_partial_load_store_mem_ref (call, vectype))
    6129              :     {
    6130           75 :       gassign *new_stmt;
    6131           75 :       if (is_load)
    6132           21 :         new_stmt = gimple_build_assign (lhs, mem_ref);
    6133              :       else
    6134              :         {
    6135           54 :           tree rhs
    6136           54 :             = gimple_call_arg (call, internal_fn_stored_value_index (ifn));
    6137           54 :           new_stmt = gimple_build_assign (mem_ref, rhs);
    6138              :         }
    6139              : 
    6140           75 :       gimple_set_location (new_stmt, gimple_location (call));
    6141           75 :       gimple_move_vops (new_stmt, call);
    6142           75 :       gsi_replace (gsi, new_stmt, false);
    6143           75 :       return true;
    6144              :     }
    6145              :   return false;
    6146              : }
    6147              : 
    6148              : /* Attempt to fold a call statement referenced by the statement iterator GSI.
    6149              :    The statement may be replaced by another statement, e.g., if the call
    6150              :    simplifies to a constant value. Return true if any changes were made.
    6151              :    It is assumed that the operands have been previously folded.  */
    6152              : 
    6153              : static bool
    6154     58242506 : gimple_fold_call (gimple_stmt_iterator *gsi, bool inplace)
    6155              : {
    6156     58242506 :   gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
    6157     58242506 :   tree callee;
    6158     58242506 :   bool changed = false;
    6159              : 
    6160              :   /* Check for virtual calls that became direct calls.  */
    6161     58242506 :   callee = gimple_call_fn (stmt);
    6162     58242506 :   if (callee && TREE_CODE (callee) == OBJ_TYPE_REF)
    6163              :     {
    6164       455993 :       if (gimple_call_addr_fndecl (OBJ_TYPE_REF_EXPR (callee)) != NULL_TREE)
    6165              :         {
    6166            6 :           if (dump_file && virtual_method_call_p (callee)
    6167          408 :               && !possible_polymorphic_call_target_p
    6168            6 :                     (callee, stmt, cgraph_node::get (gimple_call_addr_fndecl
    6169            6 :                                                      (OBJ_TYPE_REF_EXPR (callee)))))
    6170              :             {
    6171            0 :               fprintf (dump_file,
    6172              :                        "Type inheritance inconsistent devirtualization of ");
    6173            0 :               print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
    6174            0 :               fprintf (dump_file, " to ");
    6175            0 :               print_generic_expr (dump_file, callee, TDF_SLIM);
    6176            0 :               fprintf (dump_file, "\n");
    6177              :             }
    6178              : 
    6179          402 :           gimple_call_set_fn (stmt, OBJ_TYPE_REF_EXPR (callee));
    6180          402 :           changed = true;
    6181              :         }
    6182       455591 :       else if (flag_devirtualize && !inplace && virtual_method_call_p (callee))
    6183              :         {
    6184       450514 :           bool final;
    6185       450514 :           vec <cgraph_node *>targets
    6186       450514 :             = possible_polymorphic_call_targets (callee, stmt, &final);
    6187       453179 :           if (final && targets.length () <= 1 && dbg_cnt (devirt))
    6188              :             {
    6189         2029 :               tree lhs = gimple_call_lhs (stmt);
    6190         2029 :               if (dump_enabled_p ())
    6191              :                 {
    6192           34 :                   dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, stmt,
    6193              :                                    "folding virtual function call to %s\n",
    6194           34 :                                    targets.length () == 1
    6195           17 :                                    ? targets[0]->name ()
    6196              :                                    : "__builtin_unreachable");
    6197              :                 }
    6198         2029 :               if (targets.length () == 1)
    6199              :                 {
    6200         1990 :                   tree fndecl = targets[0]->decl;
    6201         1990 :                   gimple_call_set_fndecl (stmt, fndecl);
    6202         1990 :                   changed = true;
    6203              :                   /* If changing the call to __cxa_pure_virtual
    6204              :                      or similar noreturn function, adjust gimple_call_fntype
    6205              :                      too.  */
    6206         1990 :                   if (gimple_call_noreturn_p (stmt)
    6207           19 :                       && VOID_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl)))
    6208           13 :                       && TYPE_ARG_TYPES (TREE_TYPE (fndecl))
    6209         2003 :                       && (TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
    6210           13 :                           == void_type_node))
    6211           13 :                     gimple_call_set_fntype (stmt, TREE_TYPE (fndecl));
    6212              :                   /* If the call becomes noreturn, remove the lhs.  */
    6213         1990 :                   if (lhs
    6214         1669 :                       && gimple_call_noreturn_p (stmt)
    6215         2005 :                       && (VOID_TYPE_P (TREE_TYPE (gimple_call_fntype (stmt)))
    6216            6 :                           || should_remove_lhs_p (lhs)))
    6217              :                     {
    6218           12 :                       if (TREE_CODE (lhs) == SSA_NAME)
    6219              :                         {
    6220            0 :                           tree var = create_tmp_var (TREE_TYPE (lhs));
    6221            0 :                           tree def = get_or_create_ssa_default_def (cfun, var);
    6222            0 :                           gimple *new_stmt = gimple_build_assign (lhs, def);
    6223            0 :                           gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    6224              :                         }
    6225           12 :                       gimple_call_set_lhs (stmt, NULL_TREE);
    6226              :                     }
    6227         1990 :                   maybe_remove_unused_call_args (cfun, stmt);
    6228              :                 }
    6229              :               else
    6230              :                 {
    6231           39 :                   location_t loc = gimple_location (stmt);
    6232           39 :                   gimple *new_stmt = gimple_build_builtin_unreachable (loc);
    6233           39 :                   gimple_call_set_ctrl_altering (new_stmt, false);
    6234              :                   /* If the call had a SSA name as lhs morph that into
    6235              :                      an uninitialized value.  */
    6236           39 :                   if (lhs && TREE_CODE (lhs) == SSA_NAME)
    6237              :                     {
    6238           12 :                       tree var = create_tmp_var (TREE_TYPE (lhs));
    6239           12 :                       SET_SSA_NAME_VAR_OR_IDENTIFIER (lhs, var);
    6240           12 :                       SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
    6241           12 :                       set_ssa_default_def (cfun, var, lhs);
    6242              :                     }
    6243           39 :                   gimple_move_vops (new_stmt, stmt);
    6244           39 :                   gsi_replace (gsi, new_stmt, false);
    6245           39 :                   return true;
    6246              :                 }
    6247              :             }
    6248              :         }
    6249              :     }
    6250              : 
    6251              :   /* Check for indirect calls that became direct calls, and then
    6252              :      no longer require a static chain.  */
    6253     58242467 :   if (gimple_call_chain (stmt))
    6254              :     {
    6255       246800 :       tree fn = gimple_call_fndecl (stmt);
    6256       296027 :       if (fn && !DECL_STATIC_CHAIN (fn))
    6257              :         {
    6258         2054 :           gimple_call_set_chain (stmt, NULL);
    6259         2054 :           changed = true;
    6260              :         }
    6261              :     }
    6262              : 
    6263     58242467 :   if (inplace)
    6264              :     return changed;
    6265              : 
    6266              :   /* Don't constant fold functions which can change the control. */
    6267     58239353 :   if (gimple_call_ctrl_altering_p (stmt))
    6268              :     return changed;
    6269              : 
    6270              :   /* Check for builtins that CCP can handle using information not
    6271              :      available in the generic fold routines.  */
    6272     50364490 :   if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
    6273              :     {
    6274      9666288 :       if (gimple_fold_builtin (gsi))
    6275       212985 :         changed = true;
    6276              :     }
    6277     40698202 :   else if (gimple_call_builtin_p (stmt, BUILT_IN_MD))
    6278              :     {
    6279      1160820 :         changed |= targetm.gimple_fold_builtin (gsi);
    6280              :     }
    6281     39537382 :   else if (gimple_call_internal_p (stmt))
    6282              :     {
    6283      1966712 :       enum tree_code subcode = ERROR_MARK;
    6284      1966712 :       tree result = NULL_TREE;
    6285      1966712 :       bool cplx_result = false;
    6286      1966712 :       bool uaddc_usubc = false;
    6287      1966712 :       tree overflow = NULL_TREE;
    6288      1966712 :       switch (gimple_call_internal_fn (stmt))
    6289              :         {
    6290         1292 :         case IFN_ASSUME:
    6291              :           /* Remove .ASSUME calls during the last fold since it is no
    6292              :              longer needed.  */
    6293         1292 :           if (fold_before_rtl_expansion_p ())
    6294          168 :             replace_call_with_value (gsi, NULL_TREE);
    6295              :           break;
    6296       169879 :         case IFN_BUILTIN_EXPECT:
    6297       169879 :           result = fold_builtin_expect (gimple_location (stmt),
    6298              :                                         gimple_call_arg (stmt, 0),
    6299              :                                         gimple_call_arg (stmt, 1),
    6300              :                                         gimple_call_arg (stmt, 2),
    6301              :                                         NULL_TREE);
    6302       169879 :           break;
    6303         8881 :         case IFN_UBSAN_OBJECT_SIZE:
    6304         8881 :           {
    6305         8881 :             tree offset = gimple_call_arg (stmt, 1);
    6306         8881 :             tree objsize = gimple_call_arg (stmt, 2);
    6307         8881 :             if (integer_all_onesp (objsize)
    6308         8881 :                 || (TREE_CODE (offset) == INTEGER_CST
    6309         4828 :                     && TREE_CODE (objsize) == INTEGER_CST
    6310         1126 :                     && tree_int_cst_le (offset, objsize)))
    6311              :               {
    6312         1539 :                 replace_call_with_value (gsi, NULL_TREE);
    6313         1539 :                 return true;
    6314              :               }
    6315              :           }
    6316              :           break;
    6317        11660 :         case IFN_UBSAN_PTR:
    6318        11660 :           if (integer_zerop (gimple_call_arg (stmt, 1)))
    6319              :             {
    6320           30 :               replace_call_with_value (gsi, NULL_TREE);
    6321           30 :               return true;
    6322              :             }
    6323              :           break;
    6324         8493 :         case IFN_UBSAN_BOUNDS:
    6325         8493 :           {
    6326         8493 :             tree index = gimple_call_arg (stmt, 1);
    6327         8493 :             tree bound = gimple_call_arg (stmt, 2);
    6328         8493 :             if (TREE_CODE (index) == INTEGER_CST
    6329         5471 :                 && TREE_CODE (bound) == INTEGER_CST)
    6330              :               {
    6331         4340 :                 index = fold_convert (TREE_TYPE (bound), index);
    6332         4340 :                 if (TREE_CODE (index) == INTEGER_CST
    6333         4340 :                     && tree_int_cst_lt (index, bound))
    6334              :                   {
    6335          286 :                     replace_call_with_value (gsi, NULL_TREE);
    6336          286 :                     return true;
    6337              :                   }
    6338              :               }
    6339              :           }
    6340              :           break;
    6341        20589 :         case IFN_GOACC_DIM_SIZE:
    6342        20589 :         case IFN_GOACC_DIM_POS:
    6343        20589 :           result = fold_internal_goacc_dim (stmt);
    6344        20589 :           break;
    6345              :         case IFN_UBSAN_CHECK_ADD:
    6346              :           subcode = PLUS_EXPR;
    6347              :           break;
    6348              :         case IFN_UBSAN_CHECK_SUB:
    6349              :           subcode = MINUS_EXPR;
    6350              :           break;
    6351              :         case IFN_UBSAN_CHECK_MUL:
    6352              :           subcode = MULT_EXPR;
    6353              :           break;
    6354              :         case IFN_ADD_OVERFLOW:
    6355              :           subcode = PLUS_EXPR;
    6356              :           cplx_result = true;
    6357              :           break;
    6358              :         case IFN_SUB_OVERFLOW:
    6359              :           subcode = MINUS_EXPR;
    6360              :           cplx_result = true;
    6361              :           break;
    6362              :         case IFN_MUL_OVERFLOW:
    6363              :           subcode = MULT_EXPR;
    6364              :           cplx_result = true;
    6365              :           break;
    6366              :         case IFN_UADDC:
    6367              :           subcode = PLUS_EXPR;
    6368              :           cplx_result = true;
    6369              :           uaddc_usubc = true;
    6370              :           break;
    6371              :         case IFN_USUBC:
    6372              :           subcode = MINUS_EXPR;
    6373              :           cplx_result = true;
    6374              :           uaddc_usubc = true;
    6375              :           break;
    6376         4037 :         case IFN_LEN_LOAD:
    6377         4037 :         case IFN_MASK_LOAD:
    6378         4037 :         case IFN_MASK_LEN_LOAD:
    6379         4037 :         case IFN_MASK_GATHER_LOAD:
    6380         4037 :         case IFN_MASK_LEN_GATHER_LOAD:
    6381         4037 :         case IFN_MASK_LOAD_LANES:
    6382         4037 :         case IFN_MASK_LEN_LOAD_LANES:
    6383         4037 :         case IFN_LEN_STORE:
    6384         4037 :         case IFN_MASK_STORE:
    6385         4037 :         case IFN_MASK_LEN_STORE:
    6386         4037 :         case IFN_MASK_SCATTER_STORE:
    6387         4037 :         case IFN_MASK_LEN_SCATTER_STORE:
    6388         4037 :         case IFN_MASK_STORE_LANES:
    6389         4037 :         case IFN_MASK_LEN_STORE_LANES:
    6390         4037 :           changed |= gimple_fold_partial_load_store (gsi, stmt);
    6391         4037 :           break;
    6392              :         default:
    6393              :           break;
    6394              :         }
    6395       194673 :       if (subcode != ERROR_MARK)
    6396              :         {
    6397       500502 :           tree arg0 = gimple_call_arg (stmt, 0);
    6398       500502 :           tree arg1 = gimple_call_arg (stmt, 1);
    6399       500502 :           tree arg2 = NULL_TREE;
    6400       500502 :           tree type = TREE_TYPE (arg0);
    6401       500502 :           if (cplx_result)
    6402              :             {
    6403       481131 :               tree lhs = gimple_call_lhs (stmt);
    6404       481131 :               if (lhs == NULL_TREE)
    6405              :                 type = NULL_TREE;
    6406              :               else
    6407       481131 :                 type = TREE_TYPE (TREE_TYPE (lhs));
    6408       481131 :               if (uaddc_usubc)
    6409        33559 :                 arg2 = gimple_call_arg (stmt, 2);
    6410              :             }
    6411       500502 :           if (type == NULL_TREE)
    6412              :             ;
    6413       500502 :           else if (uaddc_usubc)
    6414              :             {
    6415        33559 :               if (!integer_zerop (arg2))
    6416              :                 ;
    6417              :               /* x = y + 0 + 0; x = y - 0 - 0; */
    6418         5303 :               else if (integer_zerop (arg1))
    6419              :                 result = arg0;
    6420              :               /* x = 0 + y + 0; */
    6421         4665 :               else if (subcode != MINUS_EXPR && integer_zerop (arg0))
    6422              :                 result = arg1;
    6423              :               /* x = y - y - 0; */
    6424         4665 :               else if (subcode == MINUS_EXPR
    6425         4665 :                        && operand_equal_p (arg0, arg1, 0))
    6426            0 :                 result = integer_zero_node;
    6427              :             }
    6428              :           /* x = y + 0; x = y - 0; x = y * 0; */
    6429       466943 :           else if (integer_zerop (arg1))
    6430        10103 :             result = subcode == MULT_EXPR ? integer_zero_node : arg0;
    6431              :           /* x = 0 + y; x = 0 * y; */
    6432       456840 :           else if (subcode != MINUS_EXPR && integer_zerop (arg0))
    6433            0 :             result = subcode == MULT_EXPR ? integer_zero_node : arg1;
    6434              :           /* x = y - y; */
    6435       456840 :           else if (subcode == MINUS_EXPR && operand_equal_p (arg0, arg1, 0))
    6436           13 :             result = integer_zero_node;
    6437              :           /* x = y * 1; x = 1 * y; */
    6438       456827 :           else if (subcode == MULT_EXPR && integer_onep (arg1))
    6439              :             result = arg0;
    6440       451632 :           else if (subcode == MULT_EXPR && integer_onep (arg0))
    6441              :             result = arg1;
    6442       500502 :           if (result)
    6443              :             {
    6444        15949 :               if (result == integer_zero_node)
    6445         2148 :                 result = build_zero_cst (type);
    6446        13801 :               else if (cplx_result && TREE_TYPE (result) != type)
    6447              :                 {
    6448         9708 :                   if (TREE_CODE (result) == INTEGER_CST)
    6449              :                     {
    6450            0 :                       if (arith_overflowed_p (PLUS_EXPR, type, result,
    6451              :                                               integer_zero_node))
    6452            0 :                         overflow = build_one_cst (type);
    6453              :                     }
    6454         9708 :                   else if ((!TYPE_UNSIGNED (TREE_TYPE (result))
    6455         6988 :                             && TYPE_UNSIGNED (type))
    6456         9890 :                            || (TYPE_PRECISION (type)
    6457         2902 :                                < (TYPE_PRECISION (TREE_TYPE (result))
    6458         2902 :                                   + (TYPE_UNSIGNED (TREE_TYPE (result))
    6459         3297 :                                      && !TYPE_UNSIGNED (type)))))
    6460              :                     result = NULL_TREE;
    6461           64 :                   if (result)
    6462           64 :                     result = fold_convert (type, result);
    6463              :                 }
    6464              :             }
    6465              :         }
    6466              : 
    6467      1470660 :       if (result)
    6468              :         {
    6469        28876 :           if (TREE_CODE (result) == INTEGER_CST && TREE_OVERFLOW (result))
    6470            0 :             result = drop_tree_overflow (result);
    6471        28876 :           if (cplx_result)
    6472              :             {
    6473         6294 :               if (overflow == NULL_TREE)
    6474         6294 :                 overflow = build_zero_cst (TREE_TYPE (result));
    6475         6294 :               tree ctype = build_complex_type (TREE_TYPE (result));
    6476         6294 :               if (TREE_CODE (result) == INTEGER_CST
    6477         2148 :                   && TREE_CODE (overflow) == INTEGER_CST)
    6478         2148 :                 result = build_complex (ctype, result, overflow);
    6479              :               else
    6480         4146 :                 result = build2_loc (gimple_location (stmt), COMPLEX_EXPR,
    6481              :                                      ctype, result, overflow);
    6482              :             }
    6483        28876 :           gimplify_and_update_call_from_tree (gsi, result);
    6484        28876 :           changed = true;
    6485              :         }
    6486              :     }
    6487              : 
    6488              :   return changed;
    6489              : }
    6490              : 
    6491              : 
    6492              : /* Return true whether NAME has a use on STMT.  Note this can return
    6493              :    false even though there's a use on STMT if SSA operands are not
    6494              :    up-to-date.  */
    6495              : 
    6496              : static bool
    6497         1787 : has_use_on_stmt (tree name, gimple *stmt)
    6498              : {
    6499         1787 :   ssa_op_iter iter;
    6500         1787 :   tree op;
    6501         3608 :   FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
    6502         1867 :     if (op == name)
    6503              :       return true;
    6504              :   return false;
    6505              : }
    6506              : 
    6507              : /* Add the lhs of each statement of SEQ to DCE_WORKLIST. */
    6508              : 
    6509              : void
    6510      4922279 : mark_lhs_in_seq_for_dce (bitmap dce_worklist, gimple_seq seq)
    6511              : {
    6512      4922279 :   if (!dce_worklist)
    6513              :     return;
    6514              : 
    6515      1746515 :   for (gimple_stmt_iterator i = gsi_start (seq);
    6516      2047006 :        !gsi_end_p (i); gsi_next (&i))
    6517              :     {
    6518       300491 :       gimple *stmt = gsi_stmt (i);
    6519       300491 :       tree name = gimple_get_lhs (stmt);
    6520       300491 :       if (name && TREE_CODE (name) == SSA_NAME)
    6521       300491 :         bitmap_set_bit (dce_worklist, SSA_NAME_VERSION (name));
    6522              :     }
    6523              : }
    6524              : 
    6525              : /* Worker for fold_stmt_1 dispatch to pattern based folding with
    6526              :    gimple_simplify.
    6527              : 
    6528              :    Replaces *GSI with the simplification result in RCODE and OPS
    6529              :    and the associated statements in *SEQ.  Does the replacement
    6530              :    according to INPLACE and returns true if the operation succeeded.  */
    6531              : 
    6532              : static bool
    6533      9462931 : replace_stmt_with_simplification (gimple_stmt_iterator *gsi,
    6534              :                                   gimple_match_op *res_op,
    6535              :                                   gimple_seq *seq, bool inplace,
    6536              :                                   bitmap dce_worklist)
    6537              : {
    6538      9462931 :   gimple *stmt = gsi_stmt (*gsi);
    6539      9462931 :   tree *ops = res_op->ops;
    6540      9462931 :   unsigned int num_ops = res_op->num_ops;
    6541              : 
    6542              :   /* Play safe and do not allow abnormals to be mentioned in
    6543              :      newly created statements.  See also maybe_push_res_to_seq.
    6544              :      As an exception allow such uses if there was a use of the
    6545              :      same SSA name on the old stmt.  */
    6546     20960152 :   for (unsigned int i = 0; i < num_ops; ++i)
    6547     11498962 :     if (TREE_CODE (ops[i]) == SSA_NAME
    6548      6929814 :         && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (ops[i])
    6549     11500749 :         && !has_use_on_stmt (ops[i], stmt))
    6550              :       return false;
    6551              : 
    6552      9461190 :   if (num_ops > 0 && COMPARISON_CLASS_P (ops[0]))
    6553            0 :     for (unsigned int i = 0; i < 2; ++i)
    6554            0 :       if (TREE_CODE (TREE_OPERAND (ops[0], i)) == SSA_NAME
    6555            0 :           && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (TREE_OPERAND (ops[0], i))
    6556            0 :           && !has_use_on_stmt (TREE_OPERAND (ops[0], i), stmt))
    6557              :         return false;
    6558              : 
    6559              :   /* Don't insert new statements when INPLACE is true, even if we could
    6560              :      reuse STMT for the final statement.  */
    6561      9461190 :   if (inplace && !gimple_seq_empty_p (*seq))
    6562              :     return false;
    6563              : 
    6564      9461190 :   if (gcond *cond_stmt = dyn_cast <gcond *> (stmt))
    6565              :     {
    6566      7073715 :       gcc_assert (res_op->code.is_tree_code ());
    6567      7073715 :       auto code = tree_code (res_op->code);
    6568      7073715 :       if (TREE_CODE_CLASS (code) == tcc_comparison
    6569              :           /* GIMPLE_CONDs condition may not throw.  */
    6570      7073715 :           && ((cfun
    6571      1165508 :                && (!flag_exceptions
    6572       784112 :                    || !cfun->can_throw_non_call_exceptions))
    6573       569069 :               || !operation_could_trap_p (code,
    6574       569069 :                                           FLOAT_TYPE_P (TREE_TYPE (ops[0])),
    6575              :                                           false, NULL_TREE)))
    6576      1154060 :         gimple_cond_set_condition (cond_stmt, code, ops[0], ops[1]);
    6577      5919655 :       else if (code == SSA_NAME)
    6578              :         {
    6579              :           /* If setting the gimple cond to the same thing,
    6580              :              return false as nothing changed.  */
    6581      4064261 :           if (gimple_cond_code (cond_stmt) == NE_EXPR
    6582      4039285 :               && operand_equal_p (gimple_cond_lhs (cond_stmt), ops[0])
    6583      8100799 :               && integer_zerop (gimple_cond_rhs (cond_stmt)))
    6584              :             return false;
    6585        27723 :           gimple_cond_set_condition (cond_stmt, NE_EXPR, ops[0],
    6586        27723 :                                      build_zero_cst (TREE_TYPE (ops[0])));
    6587              :         }
    6588      1855394 :       else if (code == INTEGER_CST)
    6589              :         {
    6590              :           /* Make into the canonical form `1 != 0` and `0 != 0`.
    6591              :              If already in the canonical form return false
    6592              :              saying nothing has been done.  */
    6593      1307432 :           if (integer_zerop (ops[0]))
    6594              :             {
    6595       903946 :               if (gimple_cond_false_canonical_p (cond_stmt))
    6596              :                 return false;
    6597       519283 :               gimple_cond_make_false (cond_stmt);
    6598              :             }
    6599              :           else
    6600              :             {
    6601       403486 :               if (gimple_cond_true_canonical_p (cond_stmt))
    6602              :                 return false;
    6603       211230 :               gimple_cond_make_true (cond_stmt);
    6604              :             }
    6605              :         }
    6606       547962 :       else if (!inplace)
    6607              :         {
    6608              :           /* For throwing comparisons, see if the GIMPLE_COND is the same as
    6609              :              the comparison would be.
    6610              :              This can happen due to the match pattern for
    6611              :              `(ne (cmp @0 @1) integer_zerop)` which creates a new expression
    6612              :              for the comparison.  */
    6613       547962 :           if (TREE_CODE_CLASS (code) == tcc_comparison
    6614        11448 :               && (!cfun
    6615        11448 :                   || (flag_exceptions
    6616        11448 :                       && cfun->can_throw_non_call_exceptions))
    6617       559410 :               && operation_could_trap_p (code,
    6618        11448 :                                          FLOAT_TYPE_P (TREE_TYPE (ops[0])),
    6619              :                                          false, NULL_TREE))
    6620              :             {
    6621        11448 :               tree lhs = gimple_cond_lhs (cond_stmt);
    6622        11448 :               if (gimple_cond_code (cond_stmt) == NE_EXPR
    6623        11448 :                   && TREE_CODE (lhs) == SSA_NAME
    6624        11448 :                   && INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    6625        22896 :                   && integer_zerop (gimple_cond_rhs (cond_stmt)))
    6626              :                 {
    6627        11448 :                   gimple *s = SSA_NAME_DEF_STMT (lhs);
    6628        11448 :                   if (is_gimple_assign (s)
    6629        11448 :                       && gimple_assign_rhs_code (s) == code
    6630        11448 :                       && operand_equal_p (gimple_assign_rhs1 (s), ops[0])
    6631        22896 :                       && operand_equal_p (gimple_assign_rhs2 (s), ops[1]))
    6632              :                     return false;
    6633              :                 }
    6634              :             }
    6635       536514 :           tree res = maybe_push_res_to_seq (res_op, seq);
    6636       536514 :           if (!res)
    6637              :             return false;
    6638       536514 :           gimple_cond_set_condition (cond_stmt, NE_EXPR, res,
    6639       536514 :                                      build_zero_cst (TREE_TYPE (res)));
    6640              :         }
    6641              :       else
    6642              :         return false;
    6643      2448810 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6644              :         {
    6645          865 :           fprintf (dump_file, "gimple_simplified to ");
    6646          865 :           if (!gimple_seq_empty_p (*seq))
    6647            0 :             print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
    6648          865 :           print_gimple_stmt (dump_file, gsi_stmt (*gsi),
    6649              :                              0, TDF_SLIM);
    6650              :         }
    6651              :       // Mark the lhs of the new statements maybe for dce
    6652      2448810 :       mark_lhs_in_seq_for_dce (dce_worklist, *seq);
    6653      2448810 :       gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
    6654      2448810 :       return true;
    6655              :     }
    6656      2387475 :   else if (is_gimple_assign (stmt)
    6657      2387475 :            && res_op->code.is_tree_code ())
    6658              :     {
    6659      2310875 :       auto code = tree_code (res_op->code);
    6660      2310875 :       if (!inplace
    6661      2310875 :           || gimple_num_ops (stmt) > get_gimple_rhs_num_ops (code))
    6662              :         {
    6663      2310875 :           maybe_build_generic_op (res_op);
    6664      5449488 :           gimple_assign_set_rhs_with_ops (gsi, code,
    6665              :                                           res_op->op_or_null (0),
    6666              :                                           res_op->op_or_null (1),
    6667              :                                           res_op->op_or_null (2));
    6668      2310875 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6669              :             {
    6670        17180 :               fprintf (dump_file, "gimple_simplified to ");
    6671        17180 :               if (!gimple_seq_empty_p (*seq))
    6672          330 :                 print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
    6673        17180 :               print_gimple_stmt (dump_file, gsi_stmt (*gsi),
    6674              :                                  0, TDF_SLIM);
    6675              :             }
    6676              :           // Mark the lhs of the new statements maybe for dce
    6677      2310875 :           mark_lhs_in_seq_for_dce (dce_worklist, *seq);
    6678      2310875 :           gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
    6679      2310875 :           return true;
    6680              :         }
    6681              :     }
    6682        76600 :   else if (res_op->code.is_fn_code ()
    6683        76600 :            && gimple_call_combined_fn (stmt) == combined_fn (res_op->code))
    6684              :     {
    6685         8294 :       gcc_assert (num_ops == gimple_call_num_args (stmt));
    6686        24385 :       for (unsigned int i = 0; i < num_ops; ++i)
    6687        16091 :         gimple_call_set_arg (stmt, i, ops[i]);
    6688         8294 :       if (dump_file && (dump_flags & TDF_DETAILS))
    6689              :         {
    6690            0 :           fprintf (dump_file, "gimple_simplified to ");
    6691            0 :           if (!gimple_seq_empty_p (*seq))
    6692            0 :             print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
    6693            0 :           print_gimple_stmt (dump_file, gsi_stmt (*gsi), 0, TDF_SLIM);
    6694              :         }
    6695              :       // Mark the lhs of the new statements maybe for dce
    6696         8294 :       mark_lhs_in_seq_for_dce (dce_worklist, *seq);
    6697         8294 :       gsi_insert_seq_before (gsi, *seq, GSI_SAME_STMT);
    6698         8294 :       return true;
    6699              :     }
    6700        68306 :   else if (!inplace)
    6701              :     {
    6702       135733 :       if (gimple_has_lhs (stmt))
    6703              :         {
    6704        68306 :           tree lhs = gimple_get_lhs (stmt);
    6705        68306 :           if (!maybe_push_res_to_seq (res_op, seq, lhs))
    6706              :             return false;
    6707        67317 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6708              :             {
    6709           10 :               fprintf (dump_file, "gimple_simplified to ");
    6710           10 :               print_gimple_seq (dump_file, *seq, 0, TDF_SLIM);
    6711              :             }
    6712              :           // Mark the lhs of the new statements maybe for dce
    6713        67317 :           mark_lhs_in_seq_for_dce (dce_worklist, *seq);
    6714        67317 :           gsi_replace_with_seq_vops (gsi, *seq);
    6715        67317 :           return true;
    6716              :         }
    6717              :       else
    6718            0 :         gcc_unreachable ();
    6719              :     }
    6720              : 
    6721              :   return false;
    6722              : }
    6723              : 
    6724              : /* Canonicalize MEM_REFs invariant address operand after propagation.  */
    6725              : 
    6726              : static bool
    6727    200104228 : maybe_canonicalize_mem_ref_addr (tree *t, bool is_debug = false)
    6728              : {
    6729    200104228 :   bool res = false;
    6730    200104228 :   tree *orig_t = t;
    6731              : 
    6732    200104228 :   if (TREE_CODE (*t) == ADDR_EXPR)
    6733     68114517 :     t = &TREE_OPERAND (*t, 0);
    6734              : 
    6735              :   /* The C and C++ frontends use an ARRAY_REF for indexing with their
    6736              :      generic vector extension.  The actual vector referenced is
    6737              :      view-converted to an array type for this purpose.  If the index
    6738              :      is constant the canonical representation in the middle-end is a
    6739              :      BIT_FIELD_REF so re-write the former to the latter here.  */
    6740    200104228 :   if (TREE_CODE (*t) == ARRAY_REF
    6741     11593045 :       && TREE_CODE (TREE_OPERAND (*t, 0)) == VIEW_CONVERT_EXPR
    6742       190525 :       && TREE_CODE (TREE_OPERAND (*t, 1)) == INTEGER_CST
    6743    200159835 :       && VECTOR_TYPE_P (TREE_TYPE (TREE_OPERAND (TREE_OPERAND (*t, 0), 0))))
    6744              :     {
    6745        26562 :       tree vtype = TREE_TYPE (TREE_OPERAND (TREE_OPERAND (*t, 0), 0));
    6746        26562 :       if (VECTOR_TYPE_P (vtype))
    6747              :         {
    6748        26562 :           tree low = array_ref_low_bound (*t);
    6749        26562 :           if (TREE_CODE (low) == INTEGER_CST)
    6750              :             {
    6751        26562 :               if (tree_int_cst_le (low, TREE_OPERAND (*t, 1)))
    6752              :                 {
    6753        53064 :                   widest_int idx = wi::sub (wi::to_widest (TREE_OPERAND (*t, 1)),
    6754        53064 :                                             wi::to_widest (low));
    6755        26532 :                   idx = wi::mul (idx, wi::to_widest
    6756        53064 :                                          (TYPE_SIZE (TREE_TYPE (*t))));
    6757        26532 :                   widest_int ext
    6758        26532 :                     = wi::add (idx, wi::to_widest (TYPE_SIZE (TREE_TYPE (*t))));
    6759        26532 :                   if (maybe_le (ext, wi::to_poly_widest (TYPE_SIZE (vtype))))
    6760              :                     {
    6761        51912 :                       *t = build3_loc (EXPR_LOCATION (*t), BIT_FIELD_REF,
    6762        25956 :                                        TREE_TYPE (*t),
    6763        25956 :                                        TREE_OPERAND (TREE_OPERAND (*t, 0), 0),
    6764        25956 :                                        TYPE_SIZE (TREE_TYPE (*t)),
    6765        25956 :                                        wide_int_to_tree (bitsizetype, idx));
    6766        25956 :                       res = true;
    6767              :                     }
    6768        26532 :                 }
    6769              :             }
    6770              :         }
    6771              :     }
    6772              : 
    6773    377576519 :   while (handled_component_p (*t))
    6774    177472291 :     t = &TREE_OPERAND (*t, 0);
    6775              : 
    6776              :   /* Canonicalize MEM [&foo.bar, 0] which appears after propagating
    6777              :      of invariant addresses into a SSA name MEM_REF address.  */
    6778    200104228 :   if (TREE_CODE (*t) == MEM_REF
    6779    200104228 :       || TREE_CODE (*t) == TARGET_MEM_REF)
    6780              :     {
    6781    105322729 :       tree addr = TREE_OPERAND (*t, 0);
    6782    105322729 :       if (TREE_CODE (addr) == ADDR_EXPR
    6783    105322729 :           && (TREE_CODE (TREE_OPERAND (addr, 0)) == MEM_REF
    6784     31978751 :               || handled_component_p (TREE_OPERAND (addr, 0))))
    6785              :         {
    6786       553579 :           tree base;
    6787       553579 :           poly_int64 coffset;
    6788       553579 :           base = get_addr_base_and_unit_offset (TREE_OPERAND (addr, 0),
    6789              :                                                 &coffset);
    6790       553579 :           if (!base)
    6791              :             {
    6792           18 :               if (is_debug)
    6793           18 :                 return false;
    6794            0 :               gcc_unreachable ();
    6795              :             }
    6796              : 
    6797       553561 :           TREE_OPERAND (*t, 0) = build_fold_addr_expr (base);
    6798       553561 :           TREE_OPERAND (*t, 1) = int_const_binop (PLUS_EXPR,
    6799       553561 :                                                   TREE_OPERAND (*t, 1),
    6800       553561 :                                                   size_int (coffset));
    6801       553561 :           res = true;
    6802              :         }
    6803    105322711 :       gcc_checking_assert (TREE_CODE (TREE_OPERAND (*t, 0)) == DEBUG_EXPR_DECL
    6804              :                            || is_gimple_mem_ref_addr (TREE_OPERAND (*t, 0)));
    6805              :     }
    6806              : 
    6807              :   /* Canonicalize back MEM_REFs to plain reference trees if the object
    6808              :      accessed is a decl that has the same access semantics as the MEM_REF.  */
    6809    200104210 :   if (TREE_CODE (*t) == MEM_REF
    6810    103342270 :       && TREE_CODE (TREE_OPERAND (*t, 0)) == ADDR_EXPR
    6811     31685477 :       && integer_zerop (TREE_OPERAND (*t, 1))
    6812    217405942 :       && MR_DEPENDENCE_CLIQUE (*t) == 0)
    6813              :     {
    6814     12219247 :       tree decl = TREE_OPERAND (TREE_OPERAND (*t, 0), 0);
    6815     12219247 :       tree alias_type = TREE_TYPE (TREE_OPERAND (*t, 1));
    6816     12219247 :       if (/* Same volatile qualification.  */
    6817     12219247 :           TREE_THIS_VOLATILE (*t) == TREE_THIS_VOLATILE (decl)
    6818              :           /* Same TBAA behavior with -fstrict-aliasing.  */
    6819     12216231 :           && !TYPE_REF_CAN_ALIAS_ALL (alias_type)
    6820     11860051 :           && (TYPE_MAIN_VARIANT (TREE_TYPE (decl))
    6821     11860051 :               == TYPE_MAIN_VARIANT (TREE_TYPE (alias_type)))
    6822              :           /* Same alignment.  */
    6823      4883859 :           && TYPE_ALIGN (TREE_TYPE (decl)) == TYPE_ALIGN (TREE_TYPE (*t))
    6824              :           /* We have to look out here to not drop a required conversion
    6825              :              from the rhs to the lhs if *t appears on the lhs or vice-versa
    6826              :              if it appears on the rhs.  Thus require strict type
    6827              :              compatibility.  */
    6828     16817119 :           && types_compatible_p (TREE_TYPE (*t), TREE_TYPE (decl)))
    6829              :         {
    6830      2535079 :           *t = TREE_OPERAND (TREE_OPERAND (*t, 0), 0);
    6831      2535079 :           res = true;
    6832              :         }
    6833              :     }
    6834              : 
    6835    187884963 :   else if (TREE_CODE (*orig_t) == ADDR_EXPR
    6836     65261179 :            && TREE_CODE (*t) == MEM_REF
    6837    210552508 :            && TREE_CODE (TREE_OPERAND (*t, 0)) == INTEGER_CST)
    6838              :     {
    6839          864 :       tree base;
    6840          864 :       poly_int64 coffset;
    6841          864 :       base = get_addr_base_and_unit_offset (TREE_OPERAND (*orig_t, 0),
    6842              :                                             &coffset);
    6843          864 :       if (base)
    6844              :         {
    6845          760 :           gcc_assert (TREE_CODE (base) == MEM_REF);
    6846          760 :           poly_int64 moffset;
    6847          760 :           if (mem_ref_offset (base).to_shwi (&moffset))
    6848              :             {
    6849          760 :               coffset += moffset;
    6850          760 :               if (wi::to_poly_wide (TREE_OPERAND (base, 0)).to_shwi (&moffset))
    6851              :                 {
    6852          760 :                   coffset += moffset;
    6853          760 :                   *orig_t = build_int_cst (TREE_TYPE (*orig_t), coffset);
    6854          760 :                   return true;
    6855              :                 }
    6856              :             }
    6857              :         }
    6858              :     }
    6859              : 
    6860              :   /* Canonicalize TARGET_MEM_REF in particular with respect to
    6861              :      the indexes becoming constant.  */
    6862    187884099 :   else if (TREE_CODE (*t) == TARGET_MEM_REF)
    6863              :     {
    6864      1980441 :       tree tem = maybe_fold_tmr (*t);
    6865      1980441 :       if (tem)
    6866              :         {
    6867         4169 :           *t = tem;
    6868         4169 :           if (TREE_CODE (*orig_t) == ADDR_EXPR)
    6869            0 :             recompute_tree_invariant_for_addr_expr (*orig_t);
    6870              :           res = true;
    6871              :         }
    6872              :     }
    6873              : 
    6874              :   return res;
    6875              : }
    6876              : 
    6877              : /* Worker for both fold_stmt and fold_stmt_inplace.  The INPLACE argument
    6878              :    distinguishes both cases.  */
    6879              : 
    6880              : static bool
    6881    809177515 : fold_stmt_1 (gimple_stmt_iterator *gsi, bool inplace, tree (*valueize) (tree),
    6882              :              bitmap dce_worklist = nullptr)
    6883              : {
    6884    809177515 :   bool changed = false;
    6885    809177515 :   gimple *stmt = gsi_stmt (*gsi);
    6886    809177515 :   unsigned i;
    6887              : 
    6888              :   /* First do required canonicalization of [TARGET_]MEM_REF addresses
    6889              :      after propagation.
    6890              :      ???  This shouldn't be done in generic folding but in the
    6891              :      propagation helpers which also know whether an address was
    6892              :      propagated.
    6893              :      Also canonicalize operand order.  */
    6894    809177515 :   switch (gimple_code (stmt))
    6895              :     {
    6896    264342181 :     case GIMPLE_ASSIGN:
    6897    264342181 :       if (gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
    6898              :         {
    6899    174555201 :           tree *rhs = gimple_assign_rhs1_ptr (stmt);
    6900    174555201 :           if ((REFERENCE_CLASS_P (*rhs)
    6901    110602979 :                || TREE_CODE (*rhs) == ADDR_EXPR)
    6902    189930946 :               && maybe_canonicalize_mem_ref_addr (rhs))
    6903              :             changed = true;
    6904    174555201 :           tree *lhs = gimple_assign_lhs_ptr (stmt);
    6905    174555201 :           if (REFERENCE_CLASS_P (*lhs)
    6906    174555201 :               && maybe_canonicalize_mem_ref_addr (lhs))
    6907              :             changed = true;
    6908              :           /* Canonicalize &MEM[ssa_n, CST] to ssa_n p+ CST.
    6909              :              This cannot be done in maybe_canonicalize_mem_ref_addr
    6910              :              as the gimple now has two operands rather than one.
    6911              :              The same reason why this can't be done in
    6912              :              maybe_canonicalize_mem_ref_addr is the same reason why
    6913              :              this can't be done inplace.  */
    6914    174555201 :           if (!inplace && TREE_CODE (*rhs) == ADDR_EXPR)
    6915              :             {
    6916     15172806 :               tree inner = TREE_OPERAND (*rhs, 0);
    6917     15172806 :               if (TREE_CODE (inner) == MEM_REF
    6918      1086468 :                   && TREE_CODE (TREE_OPERAND (inner, 0)) == SSA_NAME
    6919     15238474 :                   && TREE_CODE (TREE_OPERAND (inner, 1)) == INTEGER_CST)
    6920              :                 {
    6921        65668 :                   tree ptr = TREE_OPERAND (inner, 0);
    6922        65668 :                   tree addon = TREE_OPERAND (inner, 1);
    6923        65668 :                   addon = fold_convert (sizetype, addon);
    6924        65668 :                   gimple_assign_set_rhs_with_ops (gsi, POINTER_PLUS_EXPR,
    6925              :                                                   ptr, addon);
    6926        65668 :                   changed = true;
    6927        65668 :                   stmt = gsi_stmt (*gsi);
    6928              :                 }
    6929              :             }
    6930              :         }
    6931              :       else
    6932              :         {
    6933              :           /* Canonicalize operand order.  */
    6934     89786980 :           enum tree_code code = gimple_assign_rhs_code (stmt);
    6935     89786980 :           if (TREE_CODE_CLASS (code) == tcc_comparison
    6936     83502726 :               || commutative_tree_code (code)
    6937    135016364 :               || commutative_ternary_tree_code (code))
    6938              :             {
    6939     44558604 :               tree rhs1 = gimple_assign_rhs1 (stmt);
    6940     44558604 :               tree rhs2 = gimple_assign_rhs2 (stmt);
    6941     44558604 :               if (tree_swap_operands_p (rhs1, rhs2))
    6942              :                 {
    6943      2657719 :                   gimple_assign_set_rhs1 (stmt, rhs2);
    6944      2657719 :                   gimple_assign_set_rhs2 (stmt, rhs1);
    6945      2657719 :                   if (TREE_CODE_CLASS (code) == tcc_comparison)
    6946       322027 :                     gimple_assign_set_rhs_code (stmt,
    6947              :                                                 swap_tree_comparison (code));
    6948              :                   changed = true;
    6949              :                 }
    6950              :             }
    6951              :         }
    6952              :       break;
    6953     58299001 :     case GIMPLE_CALL:
    6954     58299001 :       {
    6955     58299001 :         gcall *call = as_a<gcall *> (stmt);
    6956    233335352 :         for (i = 0; i < gimple_call_num_args (call); ++i)
    6957              :           {
    6958    116737350 :             tree *arg = gimple_call_arg_ptr (call, i);
    6959    116737350 :             if (REFERENCE_CLASS_P (*arg)
    6960    116737350 :                 && maybe_canonicalize_mem_ref_addr (arg))
    6961              :               changed = true;
    6962              :           }
    6963     58299001 :         tree *lhs = gimple_call_lhs_ptr (call);
    6964     58299001 :         if (*lhs
    6965     23652816 :             && REFERENCE_CLASS_P (*lhs)
    6966     58424140 :             && maybe_canonicalize_mem_ref_addr (lhs))
    6967              :           changed = true;
    6968     58299001 :         if (*lhs)
    6969              :           {
    6970     23652816 :             combined_fn cfn = gimple_call_combined_fn (call);
    6971     23652816 :             internal_fn ifn = associated_internal_fn (cfn, TREE_TYPE (*lhs));
    6972     23652816 :             int opno = first_commutative_argument (ifn);
    6973     23652816 :             if (opno >= 0)
    6974              :               {
    6975       363285 :                 tree arg1 = gimple_call_arg (call, opno);
    6976       363285 :                 tree arg2 = gimple_call_arg (call, opno + 1);
    6977       363285 :                 if (tree_swap_operands_p (arg1, arg2))
    6978              :                   {
    6979        22783 :                     gimple_call_set_arg (call, opno, arg2);
    6980        22783 :                     gimple_call_set_arg (call, opno + 1, arg1);
    6981        22783 :                     changed = true;
    6982              :                   }
    6983              :               }
    6984              :           }
    6985              :         break;
    6986              :       }
    6987       548202 :     case GIMPLE_ASM:
    6988       548202 :       {
    6989       548202 :         gasm *asm_stmt = as_a <gasm *> (stmt);
    6990      1774348 :         for (i = 0; i < gimple_asm_noutputs (asm_stmt); ++i)
    6991              :           {
    6992       677944 :             tree link = gimple_asm_output_op (asm_stmt, i);
    6993       677944 :             tree op = TREE_VALUE (link);
    6994       677944 :             if (REFERENCE_CLASS_P (op)
    6995       677944 :                 && maybe_canonicalize_mem_ref_addr (&TREE_VALUE (link)))
    6996              :               changed = true;
    6997              :           }
    6998       974524 :         for (i = 0; i < gimple_asm_ninputs (asm_stmt); ++i)
    6999              :           {
    7000       426322 :             tree link = gimple_asm_input_op (asm_stmt, i);
    7001       426322 :             tree op = TREE_VALUE (link);
    7002       426322 :             if ((REFERENCE_CLASS_P (op)
    7003       403911 :                  || TREE_CODE (op) == ADDR_EXPR)
    7004       458232 :                 && maybe_canonicalize_mem_ref_addr (&TREE_VALUE (link)))
    7005              :               changed = true;
    7006              :           }
    7007              :       }
    7008              :       break;
    7009    418769317 :     case GIMPLE_DEBUG:
    7010    418769317 :       if (gimple_debug_bind_p (stmt))
    7011              :         {
    7012    324017498 :           tree *val = gimple_debug_bind_get_value_ptr (stmt);
    7013    324017498 :           if (*val
    7014    185442980 :               && (REFERENCE_CLASS_P (*val)
    7015    182753422 :                   || TREE_CODE (*val) == ADDR_EXPR)
    7016    379413918 :               && maybe_canonicalize_mem_ref_addr (val, true))
    7017              :             changed = true;
    7018              :         }
    7019              :       break;
    7020     46118141 :     case GIMPLE_COND:
    7021     46118141 :       {
    7022              :         /* Canonicalize operand order.  */
    7023     46118141 :         tree lhs = gimple_cond_lhs (stmt);
    7024     46118141 :         tree rhs = gimple_cond_rhs (stmt);
    7025     46118141 :         if (tree_swap_operands_p (lhs, rhs))
    7026              :           {
    7027      1452718 :             gcond *gc = as_a <gcond *> (stmt);
    7028      1452718 :             gimple_cond_set_lhs (gc, rhs);
    7029      1452718 :             gimple_cond_set_rhs (gc, lhs);
    7030      1452718 :             gimple_cond_set_code (gc,
    7031              :                                   swap_tree_comparison (gimple_cond_code (gc)));
    7032      1452718 :             changed = true;
    7033              :           }
    7034              :       }
    7035    807636242 :     default:;
    7036              :     }
    7037              : 
    7038              :   /* Dispatch to pattern-based folding.  */
    7039    807636242 :   if (!inplace
    7040      3261565 :       || is_gimple_assign (stmt)
    7041    808562464 :       || gimple_code (stmt) == GIMPLE_COND)
    7042              :     {
    7043    808251293 :       gimple_seq seq = NULL;
    7044    808251293 :       gimple_match_op res_op;
    7045   1614167243 :       if (gimple_simplify (stmt, &res_op, inplace ? NULL : &seq,
    7046              :                            valueize, valueize)
    7047    808251293 :           && replace_stmt_with_simplification (gsi, &res_op, &seq, inplace,
    7048              :                                                dce_worklist))
    7049              :         changed = true;
    7050              :       else
    7051    803415997 :         gimple_seq_discard (seq);
    7052              :     }
    7053              : 
    7054    809177515 :   stmt = gsi_stmt (*gsi);
    7055              : 
    7056              :   /* Fold the main computation performed by the statement.  */
    7057    809177515 :   switch (gimple_code (stmt))
    7058              :     {
    7059    264398676 :     case GIMPLE_ASSIGN:
    7060    264398676 :       {
    7061              :         /* Try to canonicalize for boolean-typed X the comparisons
    7062              :            X == 0, X == 1, X != 0, and X != 1.  */
    7063    264398676 :         if (gimple_assign_rhs_code (stmt) == EQ_EXPR
    7064    264398676 :             || gimple_assign_rhs_code (stmt) == NE_EXPR)
    7065              :           {
    7066      3519682 :             tree lhs = gimple_assign_lhs (stmt);
    7067      3519682 :             tree op1 = gimple_assign_rhs1 (stmt);
    7068      3519682 :             tree op2 = gimple_assign_rhs2 (stmt);
    7069      3519682 :             tree type = TREE_TYPE (op1);
    7070              : 
    7071              :             /* Check whether the comparison operands are of the same boolean
    7072              :                type as the result type is.
    7073              :                Check that second operand is an integer-constant with value
    7074              :                one or zero.  */
    7075      3519682 :             if (TREE_CODE (op2) == INTEGER_CST
    7076      2419129 :                 && (integer_zerop (op2) || integer_onep (op2))
    7077      5311810 :                 && useless_type_conversion_p (TREE_TYPE (lhs), type))
    7078              :               {
    7079         4716 :                 enum tree_code cmp_code = gimple_assign_rhs_code (stmt);
    7080         4716 :                 bool is_logical_not = false;
    7081              : 
    7082              :                 /* X == 0 and X != 1 is a logical-not.of X
    7083              :                    X == 1 and X != 0 is X  */
    7084         3896 :                 if ((cmp_code == EQ_EXPR && integer_zerop (op2))
    7085         4716 :                     || (cmp_code == NE_EXPR && integer_onep (op2)))
    7086         4659 :                   is_logical_not = true;
    7087              : 
    7088         4716 :                 if (is_logical_not == false)
    7089           57 :                   gimple_assign_set_rhs_with_ops (gsi, TREE_CODE (op1), op1);
    7090              :                 /* Only for one-bit precision typed X the transformation
    7091              :                    !X -> ~X is valid.  */
    7092         4659 :                 else if (TYPE_PRECISION (type) == 1)
    7093         4659 :                   gimple_assign_set_rhs_with_ops (gsi, BIT_NOT_EXPR, op1);
    7094              :                 /* Otherwise we use !X -> X ^ 1.  */
    7095              :                 else
    7096            0 :                   gimple_assign_set_rhs_with_ops (gsi, BIT_XOR_EXPR, op1,
    7097              :                                                   build_int_cst (type, 1));
    7098              :                 changed = true;
    7099              :                 break;
    7100              :               }
    7101              :           }
    7102              : 
    7103    264393960 :         unsigned old_num_ops = gimple_num_ops (stmt);
    7104    264393960 :         tree lhs = gimple_assign_lhs (stmt);
    7105    264393960 :         tree new_rhs = fold_gimple_assign (gsi);
    7106    264393960 :         if (new_rhs
    7107    264513204 :             && !useless_type_conversion_p (TREE_TYPE (lhs),
    7108       119244 :                                            TREE_TYPE (new_rhs)))
    7109            0 :           new_rhs = fold_convert (TREE_TYPE (lhs), new_rhs);
    7110    264393960 :         if (new_rhs
    7111    264393960 :             && (!inplace
    7112         1082 :                 || get_gimple_rhs_num_ops (TREE_CODE (new_rhs)) < old_num_ops))
    7113              :           {
    7114       119244 :             gimple_assign_set_rhs_from_tree (gsi, new_rhs);
    7115       119244 :             changed = true;
    7116              :           }
    7117              :         break;
    7118              :       }
    7119              : 
    7120     58242506 :     case GIMPLE_CALL:
    7121     58242506 :       changed |= gimple_fold_call (gsi, inplace);
    7122     58242506 :       break;
    7123              : 
    7124    418769317 :     case GIMPLE_DEBUG:
    7125    418769317 :       if (gimple_debug_bind_p (stmt))
    7126              :         {
    7127    324017498 :           tree val = gimple_debug_bind_get_value (stmt);
    7128    324017498 :           if (val && REFERENCE_CLASS_P (val))
    7129              :             {
    7130      2687583 :               tree tem = maybe_fold_reference (val);
    7131      2687583 :               if (tem)
    7132              :                 {
    7133         4420 :                   gimple_debug_bind_set_value (stmt, tem);
    7134         4420 :                   changed = true;
    7135              :                 }
    7136              :             }
    7137              :         }
    7138              :       break;
    7139              : 
    7140     11013487 :     case GIMPLE_RETURN:
    7141     11013487 :       {
    7142     11013487 :         greturn *ret_stmt = as_a<greturn *> (stmt);
    7143     11013487 :         tree ret = gimple_return_retval(ret_stmt);
    7144              : 
    7145     11013487 :         if (ret && TREE_CODE (ret) == SSA_NAME && valueize)
    7146              :           {
    7147      4613516 :             tree val = valueize (ret);
    7148      4613516 :             if (val && val != ret
    7149      4613516 :                 && may_propagate_copy (ret, val))
    7150              :               {
    7151            0 :                 gimple_return_set_retval (ret_stmt, val);
    7152            0 :                 changed = true;
    7153              :               }
    7154              :           }
    7155              :       }
    7156              :       break;
    7157              : 
    7158    809177515 :     default:;
    7159              :     }
    7160              : 
    7161    809177515 :   return changed;
    7162              : }
    7163              : 
    7164              : /* Valueziation callback that ends up not following SSA edges.  */
    7165              : 
    7166              : tree
    7167   6375319174 : no_follow_ssa_edges (tree)
    7168              : {
    7169   6375319174 :   return NULL_TREE;
    7170              : }
    7171              : 
    7172              : /* Valueization callback that ends up following single-use SSA edges only.  */
    7173              : 
    7174              : tree
    7175    979611589 : follow_single_use_edges (tree val)
    7176              : {
    7177    979611589 :   if (TREE_CODE (val) == SSA_NAME
    7178    979611589 :       && !has_single_use (val))
    7179    501713857 :     return NULL_TREE;
    7180              :   return val;
    7181              : }
    7182              : 
    7183              : /* Valueization callback that follows all SSA edges.  */
    7184              : 
    7185              : tree
    7186    197928501 : follow_all_ssa_edges (tree val)
    7187              : {
    7188    197928501 :   return val;
    7189              : }
    7190              : 
    7191              : /* Fold the statement pointed to by GSI.  In some cases, this function may
    7192              :    replace the whole statement with a new one.  Returns true iff folding
    7193              :    makes any changes.
    7194              :    The statement pointed to by GSI should be in valid gimple form but may
    7195              :    be in unfolded state as resulting from for example constant propagation
    7196              :    which can produce *&x = 0.  */
    7197              : 
    7198              : bool
    7199    161915672 : fold_stmt (gimple_stmt_iterator *gsi, bitmap dce_bitmap)
    7200              : {
    7201    161915672 :   return fold_stmt_1 (gsi, false, no_follow_ssa_edges, dce_bitmap);
    7202              : }
    7203              : 
    7204              : bool
    7205    644000278 : fold_stmt (gimple_stmt_iterator *gsi, tree (*valueize) (tree), bitmap dce_bitmap)
    7206              : {
    7207    644000278 :   return fold_stmt_1 (gsi, false, valueize, dce_bitmap);
    7208              : }
    7209              : 
    7210              : /* Perform the minimal folding on statement *GSI.  Only operations like
    7211              :    *&x created by constant propagation are handled.  The statement cannot
    7212              :    be replaced with a new one.  Return true if the statement was
    7213              :    changed, false otherwise.
    7214              :    The statement *GSI should be in valid gimple form but may
    7215              :    be in unfolded state as resulting from for example constant propagation
    7216              :    which can produce *&x = 0.  */
    7217              : 
    7218              : bool
    7219      3261565 : fold_stmt_inplace (gimple_stmt_iterator *gsi, tree (*valueize) (tree))
    7220              : {
    7221      3261565 :   gimple *stmt = gsi_stmt (*gsi);
    7222      3261565 :   bool changed = fold_stmt_1 (gsi, true, valueize);
    7223      3261565 :   gcc_assert (gsi_stmt (*gsi) == stmt);
    7224      3261565 :   return changed;
    7225              : }
    7226              : 
    7227              : /* Canonicalize and possibly invert the boolean EXPR; return NULL_TREE
    7228              :    if EXPR is null or we don't know how.
    7229              :    If non-null, the result always has boolean type.  */
    7230              : 
    7231              : static tree
    7232       212713 : canonicalize_bool (tree expr, bool invert)
    7233              : {
    7234       212713 :   if (!expr)
    7235              :     return NULL_TREE;
    7236           25 :   else if (invert)
    7237              :     {
    7238           17 :       if (integer_nonzerop (expr))
    7239            0 :         return boolean_false_node;
    7240           17 :       else if (integer_zerop (expr))
    7241            0 :         return boolean_true_node;
    7242           17 :       else if (TREE_CODE (expr) == SSA_NAME)
    7243            0 :         return fold_build2 (EQ_EXPR, boolean_type_node, expr,
    7244              :                             build_int_cst (TREE_TYPE (expr), 0));
    7245           17 :       else if (COMPARISON_CLASS_P (expr))
    7246              :         {
    7247           17 :           tree_code code
    7248           17 :             = invert_tree_comparison (TREE_CODE (expr),
    7249           17 :                                       HONOR_NANS (TREE_OPERAND (expr, 0)));
    7250           17 :           if (code == ERROR_MARK)
    7251              :             return NULL_TREE;
    7252           16 :           return fold_build2 (code, boolean_type_node,
    7253              :                               TREE_OPERAND (expr, 0),
    7254              :                               TREE_OPERAND (expr, 1));
    7255              :         }
    7256              :       else
    7257              :         return NULL_TREE;
    7258              :     }
    7259              :   else
    7260              :     {
    7261            8 :       if (TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE)
    7262              :         return expr;
    7263            0 :       if (integer_nonzerop (expr))
    7264            0 :         return boolean_true_node;
    7265            0 :       else if (integer_zerop (expr))
    7266            0 :         return boolean_false_node;
    7267            0 :       else if (TREE_CODE (expr) == SSA_NAME)
    7268            0 :         return fold_build2 (NE_EXPR, boolean_type_node, expr,
    7269              :                             build_int_cst (TREE_TYPE (expr), 0));
    7270            0 :       else if (COMPARISON_CLASS_P (expr))
    7271            0 :         return fold_build2 (TREE_CODE (expr),
    7272              :                             boolean_type_node,
    7273              :                             TREE_OPERAND (expr, 0),
    7274              :                             TREE_OPERAND (expr, 1));
    7275              :       else
    7276              :         return NULL_TREE;
    7277              :     }
    7278              : }
    7279              : 
    7280              : /* Check to see if a boolean expression EXPR is logically equivalent to the
    7281              :    comparison (OP1 CODE OP2).  Check for various identities involving
    7282              :    SSA_NAMEs.  */
    7283              : 
    7284              : static bool
    7285          902 : same_bool_comparison_p (const_tree expr, enum tree_code code,
    7286              :                         const_tree op1, const_tree op2)
    7287              : {
    7288          902 :   gimple *s;
    7289              : 
    7290              :   /* The obvious case.  */
    7291          902 :   if (TREE_CODE (expr) == code
    7292            7 :       && operand_equal_p (TREE_OPERAND (expr, 0), op1, 0)
    7293          909 :       && operand_equal_p (TREE_OPERAND (expr, 1), op2, 0))
    7294              :     return true;
    7295              : 
    7296              :   /* Check for comparing (name, name != 0) and the case where expr
    7297              :      is an SSA_NAME with a definition matching the comparison.  */
    7298          895 :   if (TREE_CODE (expr) == SSA_NAME
    7299          895 :       && TREE_CODE (TREE_TYPE (expr)) == BOOLEAN_TYPE)
    7300              :     {
    7301            0 :       if (operand_equal_p (expr, op1, 0))
    7302            0 :         return ((code == NE_EXPR && integer_zerop (op2))
    7303            0 :                 || (code == EQ_EXPR && integer_nonzerop (op2)));
    7304            0 :       s = SSA_NAME_DEF_STMT (expr);
    7305            0 :       if (is_gimple_assign (s)
    7306            0 :           && gimple_assign_rhs_code (s) == code
    7307            0 :           && operand_equal_p (gimple_assign_rhs1 (s), op1, 0)
    7308            0 :           && operand_equal_p (gimple_assign_rhs2 (s), op2, 0))
    7309              :         return true;
    7310              :     }
    7311              : 
    7312              :   /* If op1 is of the form (name != 0) or (name == 0), and the definition
    7313              :      of name is a comparison, recurse.  */
    7314          895 :   if (TREE_CODE (op1) == SSA_NAME
    7315          895 :       && TREE_CODE (TREE_TYPE (op1)) == BOOLEAN_TYPE)
    7316              :     {
    7317          207 :       s = SSA_NAME_DEF_STMT (op1);
    7318          207 :       if (is_gimple_assign (s)
    7319          207 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison)
    7320              :         {
    7321            1 :           enum tree_code c = gimple_assign_rhs_code (s);
    7322            1 :           tree rhs1 = gimple_assign_rhs1 (s);
    7323            1 :           if ((code == EQ_EXPR || code == NE_EXPR)
    7324            0 :               && TREE_CODE (op2) == INTEGER_CST)
    7325              :             {
    7326              :               /* OP1 == 0 and OP1 != true test the inverse.  */
    7327            0 :               if ((code == EQ_EXPR) == integer_zerop (op2))
    7328            0 :                 c = invert_tree_comparison (c, HONOR_NANS (rhs1));
    7329            0 :               return (c != ERROR_MARK
    7330            0 :                       && same_bool_comparison_p (expr, c, rhs1,
    7331            0 :                                                  gimple_assign_rhs2 (s)));
    7332              :             }
    7333              :         }
    7334              :     }
    7335              :   return false;
    7336              : }
    7337              : 
    7338              : /* Check to see if two boolean expressions OP1 and OP2 are logically
    7339              :    equivalent.  */
    7340              : 
    7341              : static bool
    7342            1 : same_bool_result_p (const_tree op1, const_tree op2)
    7343              : {
    7344              :   /* Simple cases first.  */
    7345            1 :   if (operand_equal_p (op1, op2, 0))
    7346              :     return true;
    7347              : 
    7348              :   /* Check the cases where at least one of the operands is a comparison.
    7349              :      These are a bit smarter than operand_equal_p in that they apply some
    7350              :      identifies on SSA_NAMEs.  */
    7351            1 :   if (COMPARISON_CLASS_P (op2)
    7352            2 :       && same_bool_comparison_p (op1, TREE_CODE (op2),
    7353            1 :                                  TREE_OPERAND (op2, 0),
    7354            1 :                                  TREE_OPERAND (op2, 1)))
    7355              :     return true;
    7356            1 :   if (COMPARISON_CLASS_P (op1)
    7357            2 :       && same_bool_comparison_p (op2, TREE_CODE (op1),
    7358            1 :                                  TREE_OPERAND (op1, 0),
    7359            1 :                                  TREE_OPERAND (op1, 1)))
    7360              :     return true;
    7361              : 
    7362              :   /* Default case.  */
    7363              :   return false;
    7364              : }
    7365              : 
    7366              : /* Forward declarations for some mutually recursive functions.  */
    7367              : 
    7368              : static tree
    7369              : and_comparisons_1 (tree type, enum tree_code code,
    7370              :                    enum tree_code code1, tree op1a, tree op1b,
    7371              :                    enum tree_code code2, tree op2a, tree op2b, basic_block);
    7372              : static tree
    7373              : and_var_with_comparison (tree type, enum tree_code code, tree var,
    7374              :                          bool invert, enum tree_code code2, tree op2a,
    7375              :                          tree op2b, basic_block);
    7376              : static tree
    7377              : and_var_with_comparison_1 (tree type, enum tree_code code, gimple *stmt,
    7378              :                            enum tree_code code2, tree op2a, tree op2b,
    7379              :                            basic_block);
    7380              : static tree
    7381              : or_comparisons_1 (tree, enum tree_code code,
    7382              :                   enum tree_code code1, tree op1a, tree op1b,
    7383              :                   enum tree_code code2, tree op2a, tree op2b,
    7384              :                   basic_block);
    7385              : static tree
    7386              : or_var_with_comparison (tree, enum tree_code code, tree var,
    7387              :                         bool invert, enum tree_code code2, tree op2a,
    7388              :                         tree op2b, basic_block);
    7389              : static tree
    7390              : or_var_with_comparison_1 (tree, enum tree_code code, gimple *stmt,
    7391              :                           enum tree_code code2, tree op2a, tree op2b,
    7392              :                           basic_block);
    7393              : 
    7394              : /* Return the OR code that evaluates its operands as the AND code CODE
    7395              :    does, or the AND code for the OR code CODE.  */
    7396              : 
    7397              : static enum tree_code
    7398        86406 : truth_andor_compl (enum tree_code code)
    7399              : {
    7400        86406 :   switch (code)
    7401              :     {
    7402              :     case TRUTH_AND_EXPR: return TRUTH_OR_EXPR;
    7403              :     case TRUTH_ANDIF_EXPR: return TRUTH_ORIF_EXPR;
    7404              :     case TRUTH_OR_EXPR: return TRUTH_AND_EXPR;
    7405              :     case TRUTH_ORIF_EXPR: return TRUTH_ANDIF_EXPR;
    7406            0 :     default: gcc_unreachable ();
    7407              :     }
    7408              : }
    7409              : 
    7410              : /* Helper function for and_comparisons_1:  try to simplify the AND of the
    7411              :    ssa variable VAR with the comparison specified by (OP2A CODE2 OP2B).
    7412              :    If INVERT is true, invert the value of the VAR before doing the AND.
    7413              :    Return NULL_EXPR if we can't simplify this to a single expression.  */
    7414              : 
    7415              : static tree
    7416       182186 : and_var_with_comparison (tree type, enum tree_code code, tree var,
    7417              :                          bool invert, enum tree_code code2, tree op2a,
    7418              :                          tree op2b, basic_block outer_cond_bb)
    7419              : {
    7420       182186 :   tree t;
    7421       182186 :   gimple *stmt = SSA_NAME_DEF_STMT (var);
    7422              : 
    7423              :   /* We can only deal with variables whose definitions are assignments.  */
    7424       182186 :   if (!is_gimple_assign (stmt))
    7425              :     return NULL_TREE;
    7426              : 
    7427              :   /* If we have an inverted comparison, apply DeMorgan's law and rewrite
    7428              :      !var AND (op2a code2 op2b) => !(var OR !(op2a code2 op2b))
    7429              :      Then we only have to consider the simpler non-inverted cases.  */
    7430       181851 :   if (invert)
    7431              :     {
    7432        64575 :       code2 = invert_tree_comparison (code2, HONOR_NANS (op2a));
    7433        64575 :       if (code2 == ERROR_MARK)
    7434              :         return NULL_TREE;
    7435        64575 :       t = or_var_with_comparison_1 (type, truth_andor_compl (code), stmt,
    7436              :                                     code2, op2a, op2b, outer_cond_bb);
    7437              :     }
    7438              :   else
    7439       117276 :     t = and_var_with_comparison_1 (type, code, stmt, code2, op2a, op2b,
    7440              :                                    outer_cond_bb);
    7441       181851 :   return canonicalize_bool (t, invert);
    7442              : }
    7443              : 
    7444              : /* Try to simplify the AND of the ssa variable defined by the assignment
    7445              :    STMT with the comparison specified by (OP2A CODE2 OP2B).
    7446              :    Return NULL_EXPR if we can't simplify this to a single expression.  */
    7447              : 
    7448              : static tree
    7449       139107 : and_var_with_comparison_1 (tree type, enum tree_code code, gimple *stmt,
    7450              :                            enum tree_code code2, tree op2a, tree op2b,
    7451              :                            basic_block outer_cond_bb)
    7452              : {
    7453       139107 :   tree var = gimple_assign_lhs (stmt);
    7454       139107 :   tree true_test_var = NULL_TREE;
    7455       139107 :   tree false_test_var = NULL_TREE;
    7456       139107 :   enum tree_code innercode = gimple_assign_rhs_code (stmt);
    7457              : 
    7458              :   /* Check for identities like (var AND (var == 0)) => false.  */
    7459       139107 :   if (TREE_CODE (op2a) == SSA_NAME
    7460       139107 :       && TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE)
    7461              :     {
    7462         6181 :       if ((code2 == NE_EXPR && integer_zerop (op2b))
    7463        14546 :           || (code2 == EQ_EXPR && integer_nonzerop (op2b)))
    7464              :         {
    7465         6702 :           true_test_var = op2a;
    7466         6702 :           if (var == true_test_var)
    7467              :             return var;
    7468              :         }
    7469         3135 :       else if ((code2 == EQ_EXPR && integer_zerop (op2b))
    7470         9325 :                || (code2 == NE_EXPR && integer_nonzerop (op2b)))
    7471              :         {
    7472         1304 :           false_test_var = op2a;
    7473         1304 :           if (var == false_test_var)
    7474            0 :             return boolean_false_node;
    7475              :         }
    7476              :     }
    7477              : 
    7478              :   /* If the definition is a comparison, recurse on it.  */
    7479       139107 :   if (TREE_CODE_CLASS (innercode) == tcc_comparison)
    7480              :     {
    7481          250 :       tree t = and_comparisons_1 (type, code, innercode,
    7482              :                                   gimple_assign_rhs1 (stmt),
    7483              :                                   gimple_assign_rhs2 (stmt),
    7484              :                                   code2,
    7485              :                                   op2a,
    7486              :                                   op2b, outer_cond_bb);
    7487          250 :       if (t)
    7488              :         return t;
    7489              :     }
    7490              : 
    7491              :   /* If the definition is an AND or OR expression, we may be able to
    7492              :      simplify by reassociating.  */
    7493       139105 :   if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
    7494       139105 :       && (innercode == BIT_AND_EXPR || innercode == BIT_IOR_EXPR))
    7495              :     {
    7496         9272 :       tree inner1 = gimple_assign_rhs1 (stmt);
    7497         9272 :       tree inner2 = gimple_assign_rhs2 (stmt);
    7498         9272 :       gimple *s;
    7499         9272 :       tree t;
    7500         9272 :       tree partial = NULL_TREE;
    7501         9272 :       bool is_and = (innercode == BIT_AND_EXPR);
    7502              : 
    7503              :       /* Check for boolean identities that don't require recursive examination
    7504              :          of inner1/inner2:
    7505              :          inner1 AND (inner1 AND inner2) => inner1 AND inner2 => var
    7506              :          inner1 AND (inner1 OR inner2) => inner1
    7507              :          !inner1 AND (inner1 AND inner2) => false
    7508              :          !inner1 AND (inner1 OR inner2) => !inner1 AND inner2
    7509              :          Likewise for similar cases involving inner2.  */
    7510         9272 :       if (inner1 == true_test_var)
    7511            0 :         return (is_and ? var : inner1);
    7512         9272 :       else if (inner2 == true_test_var)
    7513            0 :         return (is_and ? var : inner2);
    7514         9272 :       else if (inner1 == false_test_var)
    7515            0 :         return (is_and
    7516            0 :                 ? boolean_false_node
    7517            0 :                 : and_var_with_comparison (type, code, inner2, false, code2,
    7518            0 :                                            op2a, op2b, outer_cond_bb));
    7519         9272 :       else if (inner2 == false_test_var)
    7520            0 :         return (is_and
    7521            0 :                 ? boolean_false_node
    7522            0 :                 : and_var_with_comparison (type, code, inner1, false, code2,
    7523            0 :                                            op2a, op2b, outer_cond_bb));
    7524              : 
    7525              :       /* Next, redistribute/reassociate the AND across the inner tests.
    7526              :          Compute the first partial result, (inner1 AND (op2a code op2b))  */
    7527         9272 :       if (TREE_CODE (inner1) == SSA_NAME
    7528         9272 :           && is_gimple_assign (s = SSA_NAME_DEF_STMT (inner1))
    7529         8829 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
    7530        16975 :           && (t = maybe_fold_and_comparisons (type, code,
    7531              :                                               gimple_assign_rhs_code (s),
    7532              :                                               gimple_assign_rhs1 (s),
    7533              :                                               gimple_assign_rhs2 (s),
    7534              :                                               code2, op2a, op2b,
    7535              :                                               outer_cond_bb)))
    7536              :         {
    7537              :           /* Handle the AND case, where we are reassociating:
    7538              :              (inner1 AND inner2) AND (op2a code2 op2b)
    7539              :              => (t AND inner2)
    7540              :              If the partial result t is a constant, we win.  Otherwise
    7541              :              continue on to try reassociating with the other inner test.  */
    7542           14 :           if (is_and)
    7543              :             {
    7544            0 :               if (integer_onep (t))
    7545              :                 return inner2;
    7546            0 :               else if (integer_zerop (t))
    7547            0 :                 return boolean_false_node;
    7548              :             }
    7549              : 
    7550              :           /* Handle the OR case, where we are redistributing:
    7551              :              (inner1 OR inner2) AND (op2a code2 op2b)
    7552              :              => (t OR (inner2 AND (op2a code2 op2b)))  */
    7553           14 :           else if (integer_onep (t))
    7554            0 :             return boolean_true_node;
    7555              : 
    7556              :           /* Save partial result for later.  */
    7557              :           partial = t;
    7558              :         }
    7559              : 
    7560              :       /* Compute the second partial result, (inner2 AND (op2a code op2b)) */
    7561         9272 :       if (TREE_CODE (inner2) == SSA_NAME
    7562         9272 :           && is_gimple_assign (s = SSA_NAME_DEF_STMT (inner2))
    7563         9003 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
    7564        17150 :           && (t = maybe_fold_and_comparisons (type, code,
    7565              :                                               gimple_assign_rhs_code (s),
    7566              :                                               gimple_assign_rhs1 (s),
    7567              :                                               gimple_assign_rhs2 (s),
    7568              :                                               code2, op2a, op2b,
    7569              :                                               outer_cond_bb)))
    7570              :         {
    7571              :           /* Handle the AND case, where we are reassociating:
    7572              :              (inner1 AND inner2) AND (op2a code2 op2b)
    7573              :              => (inner1 AND t)  */
    7574           37 :           if (is_and)
    7575              :             {
    7576           15 :               if (integer_onep (t))
    7577              :                 return inner1;
    7578           15 :               else if (integer_zerop (t))
    7579            1 :                 return boolean_false_node;
    7580              :               /* If both are the same, we can apply the identity
    7581              :                  (x AND x) == x.  */
    7582           14 :               else if (partial && same_bool_result_p (t, partial))
    7583            0 :                 return t;
    7584              :             }
    7585              : 
    7586              :           /* Handle the OR case. where we are redistributing:
    7587              :              (inner1 OR inner2) AND (op2a code2 op2b)
    7588              :              => (t OR (inner1 AND (op2a code2 op2b)))
    7589              :              => (t OR partial)  */
    7590              :           else
    7591              :             {
    7592           22 :               if (integer_onep (t))
    7593            0 :                 return boolean_true_node;
    7594           22 :               else if (partial)
    7595              :                 {
    7596              :                   /* We already got a simplification for the other
    7597              :                      operand to the redistributed OR expression.  The
    7598              :                      interesting case is when at least one is false.
    7599              :                      Or, if both are the same, we can apply the identity
    7600              :                      (x OR x) == x.  */
    7601            7 :                   if (integer_zerop (partial))
    7602              :                     return t;
    7603            2 :                   else if (integer_zerop (t))
    7604              :                     return partial;
    7605            0 :                   else if (same_bool_result_p (t, partial))
    7606            0 :                     return t;
    7607              :                 }
    7608              :             }
    7609              :         }
    7610              :     }
    7611              :   return NULL_TREE;
    7612              : }
    7613              : 
    7614              : /* Try to simplify the AND of two comparisons defined by
    7615              :    (OP1A CODE1 OP1B) and (OP2A CODE2 OP2B), respectively.
    7616              :    If this can be done without constructing an intermediate value,
    7617              :    return the resulting tree; otherwise NULL_TREE is returned.
    7618              :    This function is deliberately asymmetric as it recurses on SSA_DEFs
    7619              :    in the first comparison but not the second.  CODE is as for
    7620              :    maybe_fold_and_comparisons.  */
    7621              : 
    7622              : static tree
    7623       609107 : and_comparisons_1 (tree type, enum tree_code code,
    7624              :                    enum tree_code code1, tree op1a, tree op1b,
    7625              :                    enum tree_code code2, tree op2a, tree op2b,
    7626              :                    basic_block outer_cond_bb)
    7627              : {
    7628       609107 :   tree truth_type = truth_type_for (TREE_TYPE (op1a));
    7629              : 
    7630              :   /* First check for ((x CODE1 y) AND (x CODE2 y)).  */
    7631       609107 :   if (operand_equal_p (op1a, op2a, 0)
    7632       609107 :       && operand_equal_p (op1b, op2b, 0))
    7633              :     {
    7634              :       /* Result will be either NULL_TREE, or a combined comparison.  */
    7635         3166 :       tree t = combine_comparisons (UNKNOWN_LOCATION,
    7636              :                                     code, code1, code2,
    7637              :                                     truth_type, truth_type, op1a, op1b);
    7638         3166 :       if (t)
    7639              :         return t;
    7640              :     }
    7641              : 
    7642              :   /* Likewise the swapped case of the above.  */
    7643       607541 :   if (operand_equal_p (op1a, op2b, 0)
    7644       607541 :       && operand_equal_p (op1b, op2a, 0))
    7645              :     {
    7646              :       /* Result will be either NULL_TREE, or a combined comparison.  */
    7647            5 :       tree t = combine_comparisons (UNKNOWN_LOCATION,
    7648              :                                     code, code1,
    7649              :                                     swap_tree_comparison (code2),
    7650              :                                     truth_type, truth_type, op1a, op1b);
    7651            5 :       if (t)
    7652              :         return t;
    7653              :     }
    7654              : 
    7655              :   /* Perhaps the first comparison is (NAME != 0) or (NAME == 1) where
    7656              :      NAME's definition is a truth value.  See if there are any simplifications
    7657              :      that can be done against the NAME's definition.  */
    7658              :   /* ???  Do the same for op2a, but as we are recursing we have to track
    7659              :      whether we swap the order of the comparsions since that's important
    7660              :      with respect to exceptions.  */
    7661       607540 :   if (TREE_CODE (op1a) == SSA_NAME
    7662       607528 :       && (code1 == NE_EXPR || code1 == EQ_EXPR)
    7663      1089004 :       && (integer_zerop (op1b) || integer_onep (op1b)))
    7664              :     {
    7665        93762 :       bool invert = ((code1 == EQ_EXPR && integer_zerop (op1b))
    7666       236081 :                      || (code1 == NE_EXPR && integer_onep (op1b)));
    7667       216233 :       gimple *stmt = SSA_NAME_DEF_STMT (op1a);
    7668       216233 :       switch (gimple_code (stmt))
    7669              :         {
    7670       181186 :         case GIMPLE_ASSIGN:
    7671              :           /* Try to simplify by copy-propagating the definition.  */
    7672       181186 :           return and_var_with_comparison (type, code, op1a, invert, code2,
    7673       181186 :                                           op2a, op2b, outer_cond_bb);
    7674              : 
    7675        14914 :         case GIMPLE_PHI:
    7676              :           /* If every argument to the PHI produces the same result when
    7677              :              ANDed with the second comparison, we win.
    7678              :              Do not do this unless the type is bool since we need a bool
    7679              :              result here anyway.  */
    7680        14914 :           if (TREE_CODE (TREE_TYPE (op1a)) == BOOLEAN_TYPE)
    7681              :             {
    7682              :               tree result = NULL_TREE;
    7683              :               unsigned i;
    7684         6509 :               for (i = 0; i < gimple_phi_num_args (stmt); i++)
    7685              :                 {
    7686         6509 :                   tree arg = gimple_phi_arg_def (stmt, i);
    7687              : 
    7688              :                   /* If this PHI has itself as an argument, ignore it.
    7689              :                      If all the other args produce the same result,
    7690              :                      we're still OK.  */
    7691         6509 :                   if (arg == gimple_phi_result (stmt))
    7692            0 :                     continue;
    7693         6509 :                   else if (TREE_CODE (arg) == INTEGER_CST)
    7694              :                     {
    7695         5127 :                       if (invert ? integer_nonzerop (arg) : integer_zerop (arg))
    7696              :                         {
    7697         3281 :                           if (!result)
    7698         1152 :                             result = boolean_false_node;
    7699         2129 :                           else if (!integer_zerop (result))
    7700              :                             return NULL_TREE;
    7701              :                         }
    7702         1846 :                       else if (!result)
    7703          986 :                         result = fold_build2 (code2, boolean_type_node,
    7704              :                                               op2a, op2b);
    7705          860 :                       else if (!same_bool_comparison_p (result,
    7706              :                                                         code2, op2a, op2b))
    7707              :                         return NULL_TREE;
    7708              :                     }
    7709         1382 :                   else if (TREE_CODE (arg) == SSA_NAME
    7710         1382 :                            && !SSA_NAME_IS_DEFAULT_DEF (arg))
    7711              :                     {
    7712         1379 :                       tree temp;
    7713         1379 :                       gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
    7714              :                       /* In simple cases we can look through PHI nodes,
    7715              :                          but we have to be careful with loops.
    7716              :                          See PR49073.  */
    7717         1379 :                       if (! dom_info_available_p (CDI_DOMINATORS)
    7718         1379 :                           || gimple_bb (def_stmt) == gimple_bb (stmt)
    7719         2758 :                           || dominated_by_p (CDI_DOMINATORS,
    7720         1379 :                                              gimple_bb (def_stmt),
    7721         1379 :                                              gimple_bb (stmt)))
    7722              :                         return NULL_TREE;
    7723         1000 :                       temp = and_var_with_comparison (type, code, arg, invert,
    7724              :                                                       code2, op2a, op2b,
    7725              :                                                       outer_cond_bb);
    7726         1000 :                       if (!temp)
    7727              :                         return NULL_TREE;
    7728            0 :                       else if (!result)
    7729              :                         result = temp;
    7730            0 :                       else if (!same_bool_result_p (result, temp))
    7731              :                         return NULL_TREE;
    7732              :                     }
    7733              :                   else
    7734              :                     return NULL_TREE;
    7735              :                 }
    7736              :               return result;
    7737              :             }
    7738              : 
    7739              :         default:
    7740              :           break;
    7741              :         }
    7742              :     }
    7743              :   return NULL_TREE;
    7744              : }
    7745              : 
    7746              : static basic_block fosa_bb;
    7747              : static vec<std::pair<tree, flow_sensitive_info_storage> > *fosa_unwind;
    7748              : static tree
    7749     53093863 : follow_outer_ssa_edges (tree val)
    7750              : {
    7751     53093863 :   if (TREE_CODE (val) == SSA_NAME
    7752     53093863 :       && !SSA_NAME_IS_DEFAULT_DEF (val))
    7753              :     {
    7754     52363971 :       basic_block def_bb = gimple_bb (SSA_NAME_DEF_STMT (val));
    7755     52363971 :       if (!def_bb
    7756     18431056 :           || def_bb == fosa_bb
    7757     62012892 :           || (dom_info_available_p (CDI_DOMINATORS)
    7758      9648921 :               && (def_bb == fosa_bb
    7759      9648921 :                   || dominated_by_p (CDI_DOMINATORS, fosa_bb, def_bb))))
    7760              :         return val;
    7761      6920039 :       flow_sensitive_info_storage storage;
    7762      6920039 :       storage.save_and_clear (val);
    7763              :       /* If the definition does not dominate fosa_bb temporarily reset
    7764              :          flow-sensitive info.  */
    7765      6920039 :       fosa_unwind->safe_push (std::make_pair (val, storage));
    7766              :       /* We cannot temporarily rewrite stmts with undefined overflow
    7767              :          behavior, so avoid expanding them. But still save off the
    7768              :          flow-sensitive info as we might be using the ssa name as the leaf.  */
    7769     13803745 :       if ((ANY_INTEGRAL_TYPE_P (TREE_TYPE (val))
    7770       831996 :            || POINTER_TYPE_P (TREE_TYPE (val)))
    7771     13388723 :           && !TYPE_OVERFLOW_WRAPS (TREE_TYPE (val)))
    7772      3975264 :         return NULL_TREE;
    7773              :       return val;
    7774              :     }
    7775              :   return val;
    7776              : }
    7777              : 
    7778              : /* Helper function for maybe_fold_and_comparisons and maybe_fold_or_comparisons
    7779              :    : try to simplify the AND/OR of the ssa variable VAR with the comparison
    7780              :    specified by (OP2A CODE2 OP2B) from match.pd.  Return NULL_EXPR if we can't
    7781              :    simplify this to a single expression.  As we are going to lower the cost
    7782              :    of building SSA names / gimple stmts significantly, we need to allocate
    7783              :    them ont the stack.  This will cause the code to be a bit ugly.  */
    7784              : 
    7785              : static tree
    7786       841553 : maybe_fold_comparisons_from_match_pd (tree type, enum tree_code code,
    7787              :                                       enum tree_code code1,
    7788              :                                       tree op1a, tree op1b,
    7789              :                                       enum tree_code code2, tree op2a,
    7790              :                                       tree op2b,
    7791              :                                       basic_block outer_cond_bb)
    7792              : {
    7793              :   /* Allocate gimple stmt1 on the stack.  */
    7794       841553 :   gassign *stmt1
    7795       841553 :     = (gassign *) XALLOCAVEC (char, gimple_size (GIMPLE_ASSIGN, 3));
    7796       841553 :   gimple_init (stmt1, GIMPLE_ASSIGN, 3);
    7797       841553 :   gimple_assign_set_rhs_code (stmt1, code1);
    7798       841553 :   gimple_assign_set_rhs1 (stmt1, op1a);
    7799       841553 :   gimple_assign_set_rhs2 (stmt1, op1b);
    7800       841553 :   gimple_set_bb (stmt1, NULL);
    7801              : 
    7802              :   /* Allocate gimple stmt2 on the stack.  */
    7803       841553 :   gassign *stmt2
    7804       841553 :     = (gassign *) XALLOCAVEC (char, gimple_size (GIMPLE_ASSIGN, 3));
    7805       841553 :   gimple_init (stmt2, GIMPLE_ASSIGN, 3);
    7806       841553 :   gimple_assign_set_rhs_code (stmt2, code2);
    7807       841553 :   gimple_assign_set_rhs1 (stmt2, op2a);
    7808       841553 :   gimple_assign_set_rhs2 (stmt2, op2b);
    7809       841553 :   gimple_set_bb (stmt2, NULL);
    7810              : 
    7811              :   /* Allocate SSA names(lhs1) on the stack.  */
    7812       841553 :   alignas (tree_node) unsigned char lhs1buf[sizeof (tree_ssa_name)];
    7813       841553 :   tree lhs1 = (tree) &lhs1buf[0];
    7814       841553 :   memset (lhs1, 0, sizeof (tree_ssa_name));
    7815       841553 :   TREE_SET_CODE (lhs1, SSA_NAME);
    7816       841553 :   TREE_TYPE (lhs1) = type;
    7817       841553 :   init_ssa_name_imm_use (lhs1);
    7818              : 
    7819              :   /* Allocate SSA names(lhs2) on the stack.  */
    7820       841553 :   alignas (tree_node) unsigned char lhs2buf[sizeof (tree_ssa_name)];
    7821       841553 :   tree lhs2 = (tree) &lhs2buf[0];
    7822       841553 :   memset (lhs2, 0, sizeof (tree_ssa_name));
    7823       841553 :   TREE_SET_CODE (lhs2, SSA_NAME);
    7824       841553 :   TREE_TYPE (lhs2) = type;
    7825       841553 :   init_ssa_name_imm_use (lhs2);
    7826              : 
    7827       841553 :   gimple_assign_set_lhs (stmt1, lhs1);
    7828       841553 :   gimple_assign_set_lhs (stmt2, lhs2);
    7829              : 
    7830       841553 :   gimple_match_op op (gimple_match_cond::UNCOND, code,
    7831              :                       type, gimple_assign_lhs (stmt1),
    7832       841553 :                       gimple_assign_lhs (stmt2));
    7833       841553 :   fosa_bb = outer_cond_bb;
    7834       841553 :   auto_vec<std::pair<tree, flow_sensitive_info_storage>, 8> unwind_stack;
    7835       841553 :   fosa_unwind = &unwind_stack;
    7836      1337087 :   if (op.resimplify (NULL, (!outer_cond_bb
    7837              :                             ? follow_all_ssa_edges : follow_outer_ssa_edges)))
    7838              :     {
    7839         1984 :       fosa_unwind = NULL;
    7840         7231 :       for (auto p : unwind_stack)
    7841         1279 :         p.second.restore (p.first);
    7842         1984 :       if (gimple_simplified_result_is_gimple_val (&op))
    7843              :         {
    7844         1109 :           tree res = op.ops[0];
    7845         1109 :           if (res == lhs1)
    7846           52 :             return build2 (code1, type, op1a, op1b);
    7847         1057 :           else if (res == lhs2)
    7848         1035 :             return build2 (code2, type, op2a, op2b);
    7849              :           else
    7850              :             return res;
    7851              :         }
    7852          875 :       else if (op.code.is_tree_code ()
    7853          875 :                && TREE_CODE_CLASS ((tree_code)op.code) == tcc_comparison)
    7854              :         {
    7855          875 :           tree op0 = op.ops[0];
    7856          875 :           tree op1 = op.ops[1];
    7857          875 :           if (op0 == lhs1 || op0 == lhs2 || op1 == lhs1 || op1 == lhs2)
    7858              :             return NULL_TREE;  /* not simple */
    7859              : 
    7860          875 :           return build2 ((enum tree_code)op.code, op.type, op0, op1);
    7861              :         }
    7862              :     }
    7863       839569 :   fosa_unwind = NULL;
    7864      9437467 :   for (auto p : unwind_stack)
    7865      6918760 :     p.second.restore (p.first);
    7866              : 
    7867       839569 :   return NULL_TREE;
    7868       841553 : }
    7869              : 
    7870              : /* Return TRUE and set op[0] if T, following all SSA edges, is a type
    7871              :    conversion.  Reject loads if LOAD is NULL, otherwise set *LOAD if a
    7872              :    converting load is found.  */
    7873              : 
    7874              : static bool
    7875      3349000 : gimple_convert_def_p (tree t, tree op[1], gimple **load = NULL)
    7876              : {
    7877      3349000 :   bool ret = false;
    7878              : 
    7879      3349000 :   if (TREE_CODE (t) == SSA_NAME
    7880      3349000 :       && !SSA_NAME_IS_DEFAULT_DEF (t))
    7881      1861764 :     if (gassign *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (t)))
    7882              :       {
    7883      1768312 :         bool load_p = gimple_assign_load_p (def);
    7884      1768312 :         if (load_p && !load)
    7885              :           return false;
    7886      1755655 :         switch (gimple_assign_rhs_code (def))
    7887              :           {
    7888        14741 :           CASE_CONVERT:
    7889        14741 :             op[0] = gimple_assign_rhs1 (def);
    7890        14741 :             ret = true;
    7891        14741 :             break;
    7892              : 
    7893         4093 :           case VIEW_CONVERT_EXPR:
    7894         4093 :             op[0] = TREE_OPERAND (gimple_assign_rhs1 (def), 0);
    7895         4093 :             ret = true;
    7896         4093 :             break;
    7897              : 
    7898              :           default:
    7899              :             break;
    7900              :           }
    7901              : 
    7902        18834 :         if (ret && load_p)
    7903            0 :           *load = def;
    7904              :       }
    7905              : 
    7906              :   return ret;
    7907              : }
    7908              : 
    7909              : /* Return TRUE and set op[*] if T, following all SSA edges, resolves to a
    7910              :    binary expression with code CODE.  */
    7911              : 
    7912              : static bool
    7913      3385281 : gimple_binop_def_p (enum tree_code code, tree t, tree op[2])
    7914              : {
    7915      3385281 :   if (TREE_CODE (t) == SSA_NAME
    7916      3385281 :       && !SSA_NAME_IS_DEFAULT_DEF (t))
    7917      1895502 :     if (gimple *def = dyn_cast <gassign *> (SSA_NAME_DEF_STMT (t)))
    7918      1799426 :       if (gimple_assign_rhs_code (def) == code)
    7919              :         {
    7920        59026 :           op[0] = gimple_assign_rhs1 (def);
    7921        59026 :           op[1] = gimple_assign_rhs2 (def);
    7922        59026 :           return true;
    7923              :         }
    7924              :   return false;
    7925              : }
    7926              : /* Subroutine for fold_truth_andor_1: decode a field reference.
    7927              : 
    7928              :    If *PEXP is a comparison reference, we return the innermost reference.
    7929              : 
    7930              :    *PBITSIZE is set to the number of bits in the reference, *PBITPOS is
    7931              :    set to the starting bit number.
    7932              : 
    7933              :    *PVOLATILEP is set to 1 if the any expression encountered is volatile;
    7934              :    otherwise it is not changed.
    7935              : 
    7936              :    *PUNSIGNEDP is set to the signedness of the field.
    7937              : 
    7938              :    *PREVERSEP is set to the storage order of the field.
    7939              : 
    7940              :    *PAND_MASK is set to the mask found in a BIT_AND_EXPR, if any.  If
    7941              :    *PAND_MASK is initially set to a mask with nonzero precision, that mask is
    7942              :    combined with the found mask, or adjusted in precision to match.
    7943              : 
    7944              :    *PSIGNBIT is set to 1 if, before clipping to *PBITSIZE, the mask
    7945              :    encompassed bits that corresponded to extensions of the sign bit, or to -1
    7946              :    if the mask encompassed bits that would be extensions of the sign bit
    7947              :    and zeroed them out.  Otherwise, it is set to 0.
    7948              : 
    7949              :    *PXORP is to be FALSE if EXP might be a XOR used in a compare, in which
    7950              :    case, if PXOR_CMP_OP is a zero constant, it will be overridden with *PEXP,
    7951              :    *PXORP will be set to TRUE, *PXOR_AND_MASK will be copied from *PAND_MASK,
    7952              :    and the left-hand operand of the XOR will be decoded.  If *PXORP is TRUE,
    7953              :    PXOR_CMP_OP and PXOR_AND_MASK are supposed to be NULL, and then the
    7954              :    right-hand operand of the XOR will be decoded.
    7955              : 
    7956              :    *LOAD is set to the load stmt of the innermost reference, if any,
    7957              :    *and NULL otherwise.
    7958              : 
    7959              :    LOC[0..3] are filled in as conversion, masking, shifting and loading
    7960              :    operations are located.
    7961              : 
    7962              :    Return 0 if this is not a component reference or is one that we can't
    7963              :    do anything with.  */
    7964              : 
    7965              : static tree
    7966      1198917 : decode_field_reference (tree *pexp, HOST_WIDE_INT *pbitsize,
    7967              :                         HOST_WIDE_INT *pbitpos,
    7968              :                         bool *punsignedp, bool *preversep, bool *pvolatilep,
    7969              :                         wide_int *pand_mask, int *psignbit,
    7970              :                         bool *pxorp, tree *pxor_cmp_op, wide_int *pxor_and_mask,
    7971              :                         gimple **pload, location_t loc[4])
    7972              : {
    7973      1198917 :   tree exp = *pexp;
    7974      1198917 :   tree outer_type = 0;
    7975      1198917 :   wide_int and_mask;
    7976      1198917 :   tree inner, offset;
    7977      1198917 :   int shiftrt = 0;
    7978      1198917 :   tree res_ops[2];
    7979      1198917 :   machine_mode mode;
    7980      1198917 :   bool convert_before_shift = false;
    7981      1198917 :   int signbit = 0;
    7982      1198917 :   bool xorp = false;
    7983      1198917 :   tree xor_cmp_op;
    7984      1198917 :   wide_int xor_and_mask;
    7985      1198917 :   gimple *load = NULL;
    7986              : 
    7987              :   /* All the optimizations using this function assume integer fields.
    7988              :      There are problems with FP fields since the type_for_size call
    7989              :      below can fail for, e.g., XFmode.  */
    7990      1198917 :   if (! INTEGRAL_TYPE_P (TREE_TYPE (exp)))
    7991              :     return NULL_TREE;
    7992              : 
    7993              :   /* Drop casts, saving only the outermost type, effectively used in
    7994              :      the compare.  We can deal with at most one conversion, and it may
    7995              :      appear at various points in the chain of recognized preparation
    7996              :      statements.  Earlier optimizers will often have already dropped
    7997              :      unneeded extensions, but they may survive, as in PR118046.  ???
    7998              :      Can we do better and allow multiple conversions, perhaps taking
    7999              :      note of the narrowest intermediate type, sign extensions and
    8000              :      whatnot?  */
    8001      1128445 :   if (!outer_type && gimple_convert_def_p (exp, res_ops))
    8002              :     {
    8003        17413 :       outer_type = TREE_TYPE (exp);
    8004        17413 :       loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
    8005        17413 :       exp = res_ops[0];
    8006              :     }
    8007              : 
    8008              :   /* Recognize and save a masking operation.  Combine it with an
    8009              :      incoming mask.  */
    8010      1128445 :   if (gimple_binop_def_p (BIT_AND_EXPR, exp, res_ops)
    8011      1128445 :       && TREE_CODE (res_ops[1]) == INTEGER_CST)
    8012              :     {
    8013        34419 :       loc[1] = gimple_location (SSA_NAME_DEF_STMT (exp));
    8014        34419 :       exp = res_ops[0];
    8015        34419 :       and_mask = wi::to_wide (res_ops[1]);
    8016        34419 :       unsigned prec_in = pand_mask->get_precision ();
    8017        34419 :       if (prec_in)
    8018              :         {
    8019           14 :           unsigned prec_op = and_mask.get_precision ();
    8020           14 :           if (prec_in >= prec_op)
    8021              :             {
    8022           14 :               if (prec_in > prec_op)
    8023           28 :                 and_mask = wide_int::from (and_mask, prec_in,
    8024           28 :                                            TYPE_SIGN (TREE_TYPE (res_ops[1])));
    8025           14 :               and_mask &= *pand_mask;
    8026              :             }
    8027              :           else
    8028            0 :             and_mask &= wide_int::from (*pand_mask, prec_op, UNSIGNED);
    8029              :         }
    8030              :       /* If we have already dropped a narrowing conversion, it applies to
    8031              :          the result of the masking, so mask bits at or above the precision
    8032              :          of OUTER_TYPE are discarded by the conversion.  Drop them from the
    8033              :          mask, so that they're not mistaken for tests of extensions of the
    8034              :          sign bit below.  Bits below that precision but above the field
    8035              :          width, e.g. after a right shift, still test such extensions.  */
    8036        34419 :       if (outer_type
    8037        34419 :           && TYPE_PRECISION (outer_type) < and_mask.get_precision ())
    8038           76 :         and_mask = wide_int::from (and_mask, TYPE_PRECISION (outer_type),
    8039           38 :                                    UNSIGNED);
    8040              :     }
    8041              :   else
    8042      1094026 :     and_mask = *pand_mask;
    8043              : 
    8044              :   /* Turn (a ^ b) [!]= 0 into a [!]= b.  */
    8045      1128445 :   if (pxorp && gimple_binop_def_p (BIT_XOR_EXPR, exp, res_ops))
    8046              :     {
    8047              :       /* No location recorded for this one, it's entirely subsumed by the
    8048              :          compare.  */
    8049        11142 :       if (*pxorp)
    8050              :         {
    8051         5543 :           exp = res_ops[1];
    8052         5543 :           gcc_checking_assert (!pxor_cmp_op && !pxor_and_mask);
    8053         5543 :           if (outer_type && TREE_CODE (exp) == INTEGER_CST)
    8054            2 :             exp = fold_convert (outer_type, exp);
    8055              :         }
    8056         5599 :       else if (!pxor_cmp_op)
    8057              :         /* Not much we can do when xor appears in the right-hand compare
    8058              :            operand.  */
    8059              :         return NULL_TREE;
    8060         5545 :       else if (integer_zerop (*pxor_cmp_op))
    8061              :         {
    8062         5543 :           xorp = true;
    8063         5543 :           exp = res_ops[0];
    8064         5543 :           xor_cmp_op = *pexp;
    8065         5543 :           xor_and_mask = *pand_mask;
    8066              :         }
    8067              :     }
    8068              : 
    8069              :   /* Another chance to drop conversions.  */
    8070      1128391 :   if (!outer_type && gimple_convert_def_p (exp, res_ops))
    8071              :     {
    8072         1401 :       outer_type = TREE_TYPE (exp);
    8073         1401 :       loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
    8074         1401 :       exp = res_ops[0];
    8075              :     }
    8076              : 
    8077              :   /* Take note of shifts.  */
    8078      1128391 :   if (gimple_binop_def_p (RSHIFT_EXPR, exp, res_ops)
    8079      1128391 :       && TREE_CODE (res_ops[1]) == INTEGER_CST)
    8080              :     {
    8081          658 :       loc[2] = gimple_location (SSA_NAME_DEF_STMT (exp));
    8082          658 :       exp = res_ops[0];
    8083          658 :       if (!tree_fits_shwi_p (res_ops[1]))
    8084              :         return NULL_TREE;
    8085          658 :       shiftrt = tree_to_shwi (res_ops[1]);
    8086          658 :       if (shiftrt <= 0)
    8087              :         return NULL_TREE;
    8088              :     }
    8089              : 
    8090              :   /* Yet another chance to drop conversions.  This one is allowed to
    8091              :      match a converting load, subsuming the load identification block
    8092              :      below.  */
    8093      1128391 :   if (!outer_type && gimple_convert_def_p (exp, res_ops, &load))
    8094              :     {
    8095           20 :       outer_type = TREE_TYPE (exp);
    8096           20 :       loc[0] = gimple_location (SSA_NAME_DEF_STMT (exp));
    8097           20 :       if (load)
    8098            0 :         loc[3] = gimple_location (load);
    8099           20 :       exp = res_ops[0];
    8100              :       /* This looks backwards, but we're going back the def chain, so if we
    8101              :          find the conversion here, after finding a shift, that's because the
    8102              :          convert appears before the shift, and we should thus adjust the bit
    8103              :          pos and size because of the shift after adjusting it due to type
    8104              :          conversion.  */
    8105           20 :       convert_before_shift = true;
    8106              :     }
    8107              : 
    8108              :   /* Identify the load, if there is one.  */
    8109      1128391 :   if (!load && TREE_CODE (exp) == SSA_NAME && !SSA_NAME_IS_DEFAULT_DEF (exp))
    8110              :     {
    8111       630945 :       gimple *def = SSA_NAME_DEF_STMT (exp);
    8112       630945 :       if (gimple_assign_load_p (def))
    8113              :         {
    8114       405519 :           loc[3] = gimple_location (def);
    8115       405519 :           load = def;
    8116       405519 :           exp = gimple_assign_rhs1 (def);
    8117              :         }
    8118              :     }
    8119              : 
    8120              :   /* Identify the relevant bits.  */
    8121      1128391 :   poly_int64 poly_bitsize, poly_bitpos;
    8122      1128391 :   int unsignedp, reversep = *preversep, volatilep = *pvolatilep;
    8123      1128391 :   inner = get_inner_reference (exp, &poly_bitsize, &poly_bitpos, &offset,
    8124              :                                &mode, &unsignedp, &reversep, &volatilep);
    8125              : 
    8126      1128391 :   HOST_WIDE_INT bs, bp;
    8127      1128391 :   if (!poly_bitsize.is_constant (&bs)
    8128      1128391 :       || !poly_bitpos.is_constant (&bp)
    8129      1128391 :       || bs <= shiftrt
    8130      1128391 :       || offset != 0
    8131      1127138 :       || TREE_CODE (inner) == PLACEHOLDER_EXPR
    8132              :       /* Reject out-of-bound accesses (PR79731, PR118514).  */
    8133      1177718 :       || !access_in_bounds_of_type_p (TREE_TYPE (inner), bs, bp)
    8134      1127104 :       || (INTEGRAL_TYPE_P (TREE_TYPE (inner))
    8135       765942 :           && !type_has_mode_precision_p (TREE_TYPE (inner))))
    8136              :     return NULL_TREE;
    8137              : 
    8138              :   /* Adjust shifts...  */
    8139      1077777 :   if (convert_before_shift
    8140      1077777 :       && outer_type && bs > TYPE_PRECISION (outer_type))
    8141              :     {
    8142            2 :       HOST_WIDE_INT excess = bs - TYPE_PRECISION (outer_type);
    8143            2 :       if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8144            0 :         bp += excess;
    8145              :       bs -= excess;
    8146              :     }
    8147              : 
    8148      1077777 :   if (shiftrt)
    8149              :     {
    8150              :       /* Punt if we're shifting by more than the loaded bitfield (after
    8151              :          adjustment), or if there's a shift after a change of signedness, punt.
    8152              :          When comparing this field with a constant, we'll check that the
    8153              :          constant is a proper sign- or zero-extension (depending on signedness)
    8154              :          of a value that would fit in the selected portion of the bitfield.  A
    8155              :          shift after a change of signedness would make the extension
    8156              :          non-uniform, and we can't deal with that (yet ???).  See
    8157              :          gcc.dg/field-merge-22.c for a test that would go wrong.  */
    8158          658 :       if (bs <= shiftrt
    8159          658 :           || (convert_before_shift
    8160           20 :               && outer_type && unsignedp != TYPE_UNSIGNED (outer_type)))
    8161              :         return NULL_TREE;
    8162          639 :       if (!reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8163          639 :         bp += shiftrt;
    8164          639 :       bs -= shiftrt;
    8165              :     }
    8166              : 
    8167              :   /* ... and bit position.  */
    8168      1077758 :   if (!convert_before_shift
    8169      1077758 :       && outer_type && bs > TYPE_PRECISION (outer_type))
    8170              :     {
    8171         8094 :       HOST_WIDE_INT excess = bs - TYPE_PRECISION (outer_type);
    8172         8094 :       if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8173            0 :         bp += excess;
    8174              :       bs -= excess;
    8175              :     }
    8176              : 
    8177              :   /* If the number of bits in the reference is the same as the bitsize of
    8178              :      the outer type, then the outer type gives the signedness. Otherwise
    8179              :      (in case of a small bitfield) the signedness is unchanged.  */
    8180      1077758 :   if (outer_type && bs == TYPE_PRECISION (outer_type))
    8181        13852 :     unsignedp = TYPE_UNSIGNED (outer_type);
    8182              : 
    8183              :   /* Make the mask the expected width.  */
    8184      1077758 :   if (and_mask.get_precision () != 0)
    8185              :     {
    8186              :       /* If the AND_MASK encompasses bits that would be extensions of
    8187              :          the sign bit, set SIGNBIT to 1 if any such bit is set, or -1 if
    8188              :          they are all clear.  */
    8189        39721 :       if (!unsignedp
    8190        39721 :           && and_mask.get_precision () > bs)
    8191              :         {
    8192           74 :           if ((and_mask & wi::mask (bs, true, and_mask.get_precision ())) != 0)
    8193              :             signbit = 1;
    8194              :           else
    8195           28 :             signbit = -1;
    8196              :         }
    8197        39721 :       and_mask = wide_int::from (and_mask, bs, UNSIGNED);
    8198              :     }
    8199              : 
    8200      1077758 :   *pexp = exp;
    8201      1077758 :   *pload = load;
    8202      1077758 :   *pbitsize = bs;
    8203      1077758 :   *pbitpos = bp;
    8204      1077758 :   *punsignedp = unsignedp;
    8205      1077758 :   *preversep = reversep;
    8206      1077758 :   *pvolatilep = volatilep;
    8207      1077758 :   *psignbit = signbit;
    8208      1077758 :   *pand_mask = and_mask;
    8209      1077758 :   if (xorp)
    8210              :     {
    8211         5543 :       *pxorp = xorp;
    8212         5543 :       *pxor_cmp_op = xor_cmp_op;
    8213         5543 :       *pxor_and_mask = xor_and_mask;
    8214              :     }
    8215              : 
    8216              :   return inner;
    8217      1198917 : }
    8218              : 
    8219              : /* Return the one bitpos within bit extents L or R that is at an
    8220              :    ALIGN-bit alignment boundary, or -1 if there is more than one such
    8221              :    boundary, if there isn't any, or if there is any such boundary
    8222              :    between the extents.  L and R are given by bitpos and bitsize.  If
    8223              :    it doesn't return -1, there are two consecutive ALIGN-bit words
    8224              :    that contain both extents, and at least one of the extents
    8225              :    straddles across the returned alignment boundary.  */
    8226              : 
    8227              : static inline HOST_WIDE_INT
    8228        50403 : compute_split_boundary_from_align (HOST_WIDE_INT align,
    8229              :                                    HOST_WIDE_INT l_bitpos,
    8230              :                                    HOST_WIDE_INT l_bitsize,
    8231              :                                    HOST_WIDE_INT r_bitpos,
    8232              :                                    HOST_WIDE_INT r_bitsize)
    8233              : {
    8234        50403 :   HOST_WIDE_INT amask = ~(align - 1);
    8235              : 
    8236        50403 :   HOST_WIDE_INT first_bit = MIN (l_bitpos, r_bitpos);
    8237        50403 :   HOST_WIDE_INT end_bit = MAX (l_bitpos + l_bitsize, r_bitpos + r_bitsize);
    8238              : 
    8239        50403 :   HOST_WIDE_INT boundary = (end_bit - 1) & amask;
    8240              : 
    8241              :   /* Make sure we're crossing no more than one alignment boundary.
    8242              : 
    8243              :      ??? We don't have logic to recombine loads of two adjacent
    8244              :      fields that each crosses a different alignment boundary, so
    8245              :      as to load the middle word only once, if other words can't be
    8246              :      otherwise recombined.  */
    8247        50403 :   if (boundary - first_bit > align)
    8248              :     return -1;
    8249              : 
    8250        19411 :   HOST_WIDE_INT l_start_word = l_bitpos & amask;
    8251        19411 :   HOST_WIDE_INT l_end_word = (l_bitpos + l_bitsize - 1) & amask;
    8252              : 
    8253        19411 :   HOST_WIDE_INT r_start_word = r_bitpos & amask;
    8254        19411 :   HOST_WIDE_INT r_end_word = (r_bitpos + r_bitsize - 1) & amask;
    8255              : 
    8256              :   /* If neither field straddles across an alignment boundary, it's no
    8257              :      use to even try to merge them.  */
    8258        19411 :   if (l_start_word == l_end_word && r_start_word == r_end_word)
    8259        19104 :     return -1;
    8260              : 
    8261              :   return boundary;
    8262              : }
    8263              : 
    8264              : /* Make a bit_field_ref.  If POINT is NULL, return the BIT_FIELD_REF.
    8265              :    Otherwise, build and insert a load stmt before POINT, and return
    8266              :    the SSA_NAME.  ???  Rewrite LOAD in terms of the bitfield?  */
    8267              : 
    8268              : static tree
    8269         4867 : make_bit_field_load (location_t loc, tree inner, tree orig_inner, tree type,
    8270              :                      HOST_WIDE_INT bitsize, poly_int64 bitpos,
    8271              :                      bool unsignedp, bool reversep, gimple *point)
    8272              : {
    8273         4867 :   if (point && loc == UNKNOWN_LOCATION)
    8274           22 :     loc = gimple_location (point);
    8275              : 
    8276         4867 :   tree ref = make_bit_field_ref (loc, unshare_expr (inner),
    8277              :                                  unshare_expr (orig_inner),
    8278              :                                  type, bitsize, bitpos,
    8279              :                                  unsignedp, reversep);
    8280         4867 :   if (!point)
    8281              :     return ref;
    8282              : 
    8283              :   /* If we're remaking the same load, reuse the SSA NAME it is already loaded
    8284              :      into.  */
    8285         4662 :   if (gimple_assign_load_p (point)
    8286         4662 :       && operand_equal_p (ref, gimple_assign_rhs1 (point)))
    8287              :     {
    8288         1692 :       gcc_checking_assert (TREE_CODE (gimple_assign_lhs (point)) == SSA_NAME);
    8289              :       return gimple_assign_lhs (point);
    8290              :     }
    8291              : 
    8292         2970 :   gimple_seq stmts = NULL;
    8293         2970 :   tree ret = force_gimple_operand (ref, &stmts, true, NULL_TREE);
    8294              : 
    8295              :   /* We know the vuse is supposed to end up being the same as that at the
    8296              :      original load at the insertion point, but if we don't set it, it will be a
    8297              :      generic placeholder that only the global SSA update at the end of the pass
    8298              :      would make equal, too late for us to use in further combinations.  So go
    8299              :      ahead and copy the vuse.  */
    8300              : 
    8301         2970 :   tree reaching_vuse = gimple_vuse (point);
    8302         2970 :   for (gimple_stmt_iterator i = gsi_start (stmts);
    8303         6594 :        !gsi_end_p (i); gsi_next (&i))
    8304              :     {
    8305         3624 :       gimple *new_stmt = gsi_stmt (i);
    8306         7248 :       if (gimple_has_mem_ops (new_stmt))
    8307         3624 :         gimple_set_vuse (new_stmt, reaching_vuse);
    8308              :     }
    8309              : 
    8310         2970 :   gimple_stmt_iterator gsi = gsi_for_stmt (point);
    8311         2970 :   gsi_insert_seq_before (&gsi, stmts, GSI_SAME_STMT);
    8312         2970 :   return ret;
    8313              : }
    8314              : 
    8315              : /* Initialize ln_arg[0] and ln_arg[1] to a pair of newly-created (at
    8316              :    LOC) loads from INNER (from ORIG_INNER), of modes MODE and MODE2,
    8317              :    respectively, starting at BIT_POS, using reversed endianness if
    8318              :    REVERSEP.  Also initialize BITPOS (the starting position of each
    8319              :    part into INNER), BITSIZ (the bit count starting at BITPOS),
    8320              :    TOSHIFT[1] (the amount by which the part and its mask are to be
    8321              :    shifted right to bring its least-significant bit to bit zero) and
    8322              :    SHIFTED (the amount by which the part, by separate loading, has
    8323              :    already been shifted right, but that the mask needs shifting to
    8324              :    match).  */
    8325              : 
    8326              : static inline void
    8327          307 : build_split_load (tree /* out */ ln_arg[2],
    8328              :                   HOST_WIDE_INT /* out */ bitpos[2],
    8329              :                   HOST_WIDE_INT /* out */ bitsiz[2],
    8330              :                   HOST_WIDE_INT /* in[0] out[0..1] */ toshift[2],
    8331              :                   HOST_WIDE_INT /* out */ shifted[2],
    8332              :                   location_t loc, tree inner, tree orig_inner,
    8333              :                   scalar_int_mode mode, scalar_int_mode mode2,
    8334              :                   HOST_WIDE_INT bit_pos, bool reversep,
    8335              :                   gimple *point[2])
    8336              : {
    8337          307 :   scalar_int_mode modes[2] = { mode, mode2 };
    8338          307 :   bitsiz[0] = GET_MODE_BITSIZE (mode);
    8339          307 :   bitsiz[1] = GET_MODE_BITSIZE (mode2);
    8340              : 
    8341          921 :   for (int i = 0; i < 2; i++)
    8342              :     {
    8343          614 :       tree type = lang_hooks.types.type_for_mode (modes[i], 1);
    8344          614 :       if (!type)
    8345              :         {
    8346            0 :           type = build_nonstandard_integer_type (bitsiz[0], 1);
    8347            0 :           gcc_assert (type);
    8348              :         }
    8349          614 :       bitpos[i] = bit_pos;
    8350         1228 :       ln_arg[i] = make_bit_field_load (loc, inner, orig_inner,
    8351          614 :                                        type, bitsiz[i],
    8352          614 :                                        bit_pos, 1, reversep, point[i]);
    8353          614 :       bit_pos += bitsiz[i];
    8354              :     }
    8355              : 
    8356          307 :   toshift[1] = toshift[0];
    8357          307 :   if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8358              :     {
    8359            3 :       shifted[0] = bitsiz[1];
    8360            3 :       shifted[1] = 0;
    8361            3 :       toshift[0] = 0;
    8362              :     }
    8363              :   else
    8364              :     {
    8365          304 :       shifted[1] = bitsiz[0];
    8366          304 :       shifted[0] = 0;
    8367          304 :       toshift[1] = 0;
    8368              :     }
    8369          307 : }
    8370              : 
    8371              : /* Make arrangements to split at bit BOUNDARY a single loaded word
    8372              :    (with REVERSEP bit order) LN_ARG[0], to be shifted right by
    8373              :    TOSHIFT[0] to bring the field of interest to the least-significant
    8374              :    bit.  The expectation is that the same loaded word will be
    8375              :    propagated from part 0 to part 1, with just different shifting and
    8376              :    masking to extract both parts.  MASK is not expected to do more
    8377              :    than masking out the bits that belong to the other part.  See
    8378              :    build_split_load for more information on the other fields.  */
    8379              : 
    8380              : static inline void
    8381           51 : reuse_split_load (tree /* in[0] out[1] */ ln_arg[2],
    8382              :                   HOST_WIDE_INT /* in[0] out[1] */ bitpos[2],
    8383              :                   HOST_WIDE_INT /* in[0] out[1] */ bitsiz[2],
    8384              :                   HOST_WIDE_INT /* in[0] out[0..1] */ toshift[2],
    8385              :                   HOST_WIDE_INT /* out */ shifted[2],
    8386              :                   wide_int /* out */ mask[2],
    8387              :                   HOST_WIDE_INT boundary, bool reversep)
    8388              : {
    8389           51 :   unsigned prec = TYPE_PRECISION (TREE_TYPE (ln_arg[0]));
    8390              : 
    8391           51 :   ln_arg[1] = ln_arg[0];
    8392           51 :   bitpos[1] = bitpos[0];
    8393           51 :   bitsiz[1] = bitsiz[0];
    8394           51 :   shifted[1] = shifted[0] = 0;
    8395              : 
    8396           51 :   if (reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8397              :     {
    8398            3 :       toshift[1] = toshift[0];
    8399            3 :       toshift[0] = bitpos[0] + bitsiz[0] - boundary;
    8400            3 :       mask[0] = wi::mask (toshift[0], true, prec);
    8401            3 :       mask[1] = wi::mask (toshift[0], false, prec);
    8402              :     }
    8403              :   else
    8404              :     {
    8405           48 :       toshift[1] = boundary - bitpos[1];
    8406           48 :       mask[1] = wi::mask (toshift[1], true, prec);
    8407           48 :       mask[0] = wi::mask (toshift[1], false, prec);
    8408              :     }
    8409           51 : }
    8410              : 
    8411              : /* Find ways of folding logical expressions of LHS and RHS:
    8412              : 
    8413              :    Try to merge two comparisons to nearby fields.
    8414              : 
    8415              :    For example, if we have p->a == 2 && p->b == 4 and we can load both A and B
    8416              :    at once, we can do this with a comparison against the object ANDed with the
    8417              :    a mask.
    8418              : 
    8419              :    If we have p->a == q->a && p->b == q->b, we may be able to use bit masking
    8420              :    operations to do this with one comparison, loading both fields from P at
    8421              :    once, and likewise from Q.
    8422              : 
    8423              :    Herein, loading at once means loading from within the same alignment
    8424              :    boundary for the enclosing object.  If (packed) fields cross such alignment
    8425              :    boundaries, we may still recombine the compares, so that loads do not cross
    8426              :    the boundaries.
    8427              : 
    8428              :    CODE is the logical operation being done.  It can be TRUTH_ANDIF_EXPR,
    8429              :    TRUTH_AND_EXPR, TRUTH_ORIF_EXPR, or TRUTH_OR_EXPR.
    8430              : 
    8431              :    TRUTH_TYPE is the type of the logical operand.
    8432              : 
    8433              :    LHS is denoted as LL_ARG LCODE LR_ARG.
    8434              : 
    8435              :    RHS is denoted as RL_ARG RCODE RR_ARG.
    8436              : 
    8437              :    LHS is assumed to dominate RHS.
    8438              : 
    8439              :    Combined loads are inserted next to preexisting loads, once we determine
    8440              :    that the combination is viable, and the combined condition references new
    8441              :    SSA_NAMEs that hold the loaded values.  Since the original loads are
    8442              :    verified to have the same gimple_vuse, the insertion point doesn't matter
    8443              :    for correctness.  ??? The loads may be a lot earlier than the compares, and
    8444              :    it's conceivable that one or two loads for RHS appear before those for LHS.
    8445              :    It could be advantageous to try to place the loads optimally, taking
    8446              :    advantage of knowing whether RHS is accessed before LHS, or that both are
    8447              :    accessed before both compares, but we don't do that (yet?).
    8448              : 
    8449              :    SEPARATEP should be NULL if the combined condition must be returned as a
    8450              :    single expression, even if it is a compound condition.  This must only be
    8451              :    done if LHS and RHS are adjacent, without intervening conditions, and the
    8452              :    combined condition is to replace RHS, while LHS is dropped altogether.
    8453              : 
    8454              :    Otherwise, SEPARATEP must be a non-NULL pointer to a NULL_TREE, that may be
    8455              :    replaced by a part of the compound condition that could replace RHS, while
    8456              :    the returned expression replaces LHS.  This works whether or not LHS and RHS
    8457              :    are adjacent, as long as there aren't VDEFs or other side effects between
    8458              :    them.
    8459              : 
    8460              :    If the "words" accessed by RHS are already accessed by LHS, this won't
    8461              :    matter, but if RHS accesses "words" that LHS doesn't, then *SEPARATEP will
    8462              :    be set to the compares that should take RHS's place.  By "words" we mean
    8463              :    contiguous bits that do not cross a an TYPE_ALIGN boundary of the accessed
    8464              :    object's type.
    8465              : 
    8466              :    We return the simplified tree or 0 if no optimization is possible.  */
    8467              : 
    8468              : tree
    8469       440944 : fold_truth_andor_for_ifcombine (enum tree_code code, tree truth_type,
    8470              :                                 location_t lloc, enum tree_code lcode,
    8471              :                                 tree ll_arg, tree lr_arg,
    8472              :                                 location_t rloc, enum tree_code rcode,
    8473              :                                 tree rl_arg, tree rr_arg,
    8474              :                                 tree *separatep)
    8475              : {
    8476              :   /* If this is the "or" of two comparisons, we can do something if
    8477              :      the comparisons are NE_EXPR.  If this is the "and", we can do something
    8478              :      if the comparisons are EQ_EXPR.  I.e.,
    8479              :         (a->b == 2 && a->c == 4) can become (a->new == NEW).
    8480              : 
    8481              :      WANTED_CODE is this operation code.  For single bit fields, we can
    8482              :      convert EQ_EXPR to NE_EXPR so we need not reject the "wrong"
    8483              :      comparison for one-bit fields.  */
    8484              : 
    8485       440944 :   enum tree_code orig_code = code;
    8486       440944 :   enum tree_code wanted_code;
    8487       440944 :   tree ll_inner, lr_inner, rl_inner, rr_inner;
    8488       440944 :   gimple *ll_load, *lr_load, *rl_load, *rr_load;
    8489       440944 :   HOST_WIDE_INT ll_bitsize, ll_bitpos, lr_bitsize, lr_bitpos;
    8490       440944 :   HOST_WIDE_INT rl_bitsize, rl_bitpos, rr_bitsize, rr_bitpos;
    8491       440944 :   HOST_WIDE_INT xll_bitpos, xlr_bitpos, xrl_bitpos, xrr_bitpos;
    8492       440944 :   HOST_WIDE_INT lnbitsize, lnbitpos, lnprec;
    8493       440944 :   HOST_WIDE_INT rnbitsize, rnbitpos, rnprec;
    8494       440944 :   bool ll_unsignedp, lr_unsignedp, rl_unsignedp, rr_unsignedp;
    8495       440944 :   bool ll_reversep, lr_reversep, rl_reversep, rr_reversep;
    8496       440944 :   int ll_signbit, lr_signbit, rl_signbit, rr_signbit;
    8497       440944 :   scalar_int_mode lnmode, lnmode2, rnmode;
    8498       440944 :   wide_int ll_and_mask, lr_and_mask, rl_and_mask, rr_and_mask;
    8499       440944 :   wide_int l_const, r_const;
    8500       440944 :   tree lntype, rntype, result;
    8501       440944 :   HOST_WIDE_INT first_bit, end_bit;
    8502       440944 :   bool volatilep;
    8503       440944 :   bool l_split_load;
    8504              : 
    8505              :   /* These are indexed by: conv, mask, shft, load.  */
    8506       440944 :   location_t ll_loc[4] = { lloc, lloc, lloc, UNKNOWN_LOCATION };
    8507       440944 :   location_t lr_loc[4] = { lloc, lloc, lloc, UNKNOWN_LOCATION };
    8508       440944 :   location_t rl_loc[4] = { rloc, rloc, rloc, UNKNOWN_LOCATION };
    8509       440944 :   location_t rr_loc[4] = { rloc, rloc, rloc, UNKNOWN_LOCATION };
    8510              : 
    8511       440944 :   gcc_checking_assert (!separatep || !*separatep);
    8512              : 
    8513              :   /* Start by getting the comparison codes.  Fail if anything is volatile.
    8514              :      If one operand is a BIT_AND_EXPR with the constant one, treat it as if
    8515              :      it were surrounded with a NE_EXPR.  */
    8516              : 
    8517       440944 :   if (TREE_CODE_CLASS (lcode) != tcc_comparison
    8518       440944 :       || TREE_CODE_CLASS (rcode) != tcc_comparison)
    8519              :     return 0;
    8520              : 
    8521              :   /* We don't normally find TRUTH_*IF_EXPR in gimple, but these codes may be
    8522              :      given by our caller to denote conditions from different blocks.  */
    8523       440944 :   switch (code)
    8524              :     {
    8525              :     case TRUTH_AND_EXPR:
    8526              :     case TRUTH_ANDIF_EXPR:
    8527              :       code = TRUTH_AND_EXPR;
    8528              :       break;
    8529              : 
    8530            0 :     case TRUTH_OR_EXPR:
    8531            0 :     case TRUTH_ORIF_EXPR:
    8532            0 :       code = TRUTH_OR_EXPR;
    8533            0 :       break;
    8534              : 
    8535              :     default:
    8536              :       return 0;
    8537              :     }
    8538              : 
    8539              :   /* Prepare to turn compares of signed quantities with zero into sign-bit
    8540              :      tests.  We need not worry about *_reversep here for these compare
    8541              :      rewrites: loads will have already been reversed before compares.  Save the
    8542              :      precision, because [lr]l_arg may change and we won't be able to tell how
    8543              :      wide it was originally.  */
    8544       440944 :   unsigned lsignbit = 0, rsignbit = 0;
    8545       440944 :   if ((lcode == LT_EXPR || lcode == GE_EXPR)
    8546        15132 :       && integer_zerop (lr_arg)
    8547         3745 :       && INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
    8548       444689 :       && !TYPE_UNSIGNED (TREE_TYPE (ll_arg)))
    8549              :     {
    8550         3745 :       lsignbit = TYPE_PRECISION (TREE_TYPE (ll_arg));
    8551         3745 :       lcode = (lcode == LT_EXPR ? NE_EXPR : EQ_EXPR);
    8552              :     }
    8553              :   /* Turn compares of unsigned quantities with powers of two into
    8554              :      equality tests of masks.  */
    8555       437199 :   else if ((lcode == LT_EXPR || lcode == GE_EXPR)
    8556        11387 :            && INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
    8557        10623 :            && TYPE_UNSIGNED (TREE_TYPE (ll_arg))
    8558         8339 :            && TREE_CODE (lr_arg) == INTEGER_CST
    8559       437199 :            && wi::popcount (wi::to_wide (lr_arg)) == 1)
    8560              :     {
    8561            0 :       ll_and_mask = ~(wi::to_wide (lr_arg) - 1);
    8562            0 :       lcode = (lcode == GE_EXPR ? NE_EXPR : EQ_EXPR);
    8563            0 :       lr_arg = wide_int_to_tree (TREE_TYPE (ll_arg), ll_and_mask * 0);
    8564              :     }
    8565              :   /* Turn compares of unsigned quantities with powers of two minus one
    8566              :      into equality tests of masks.  */
    8567       874398 :   else if ((lcode == LE_EXPR || lcode == GT_EXPR)
    8568        34932 :            && INTEGRAL_TYPE_P (TREE_TYPE (ll_arg))
    8569        34702 :            && TYPE_UNSIGNED (TREE_TYPE (ll_arg))
    8570        28233 :            && TREE_CODE (lr_arg) == INTEGER_CST
    8571       497407 :            && wi::popcount (wi::to_wide (lr_arg) + 1) == 1)
    8572              :     {
    8573         4215 :       ll_and_mask = ~wi::to_wide (lr_arg);
    8574         4215 :       lcode = (lcode == GT_EXPR ? NE_EXPR : EQ_EXPR);
    8575         4215 :       lr_arg = wide_int_to_tree (TREE_TYPE (ll_arg), ll_and_mask * 0);
    8576              :     }
    8577              :   /* Likewise for the second compare.  */
    8578       440944 :   if ((rcode == LT_EXPR || rcode == GE_EXPR)
    8579        16638 :       && integer_zerop (rr_arg)
    8580         2748 :       && INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
    8581       443692 :       && !TYPE_UNSIGNED (TREE_TYPE (rl_arg)))
    8582              :     {
    8583         2748 :       rsignbit = TYPE_PRECISION (TREE_TYPE (rl_arg));
    8584         2748 :       rcode = (rcode == LT_EXPR ? NE_EXPR : EQ_EXPR);
    8585              :     }
    8586       438196 :   else if ((rcode == LT_EXPR || rcode == GE_EXPR)
    8587        13890 :            && INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
    8588        12759 :            && TYPE_UNSIGNED (TREE_TYPE (rl_arg))
    8589         4692 :            && TREE_CODE (rr_arg) == INTEGER_CST
    8590       438196 :            && wi::popcount (wi::to_wide (rr_arg)) == 1)
    8591              :     {
    8592            0 :       rl_and_mask = ~(wi::to_wide (rr_arg) - 1);
    8593            0 :       rcode = (rcode == GE_EXPR ? NE_EXPR : EQ_EXPR);
    8594            0 :       rr_arg = wide_int_to_tree (TREE_TYPE (rl_arg), rl_and_mask * 0);
    8595              :     }
    8596       876392 :   else if ((rcode == LE_EXPR || rcode == GT_EXPR)
    8597        42297 :            && INTEGRAL_TYPE_P (TREE_TYPE (rl_arg))
    8598        42055 :            && TYPE_UNSIGNED (TREE_TYPE (rl_arg))
    8599        32698 :            && TREE_CODE (rr_arg) == INTEGER_CST
    8600       503458 :            && wi::popcount (wi::to_wide (rr_arg) + 1) == 1)
    8601              :     {
    8602         3112 :       rl_and_mask = ~wi::to_wide (rr_arg);
    8603         3112 :       rcode = (rcode == GT_EXPR ? NE_EXPR : EQ_EXPR);
    8604         3112 :       rr_arg = wide_int_to_tree (TREE_TYPE (rl_arg), rl_and_mask * 0);
    8605              :     }
    8606              : 
    8607              :   /* See if the comparisons can be merged.  Then get all the parameters for
    8608              :      each side.  */
    8609              : 
    8610       440944 :   if ((lcode != EQ_EXPR && lcode != NE_EXPR)
    8611       397692 :       || (rcode != EQ_EXPR && rcode != NE_EXPR))
    8612              :     return 0;
    8613              : 
    8614       377315 :   ll_reversep = lr_reversep = rl_reversep = rr_reversep = 0;
    8615       377315 :   volatilep = 0;
    8616       377315 :   bool l_xor = false, r_xor = false;
    8617       377315 :   ll_inner = decode_field_reference (&ll_arg, &ll_bitsize, &ll_bitpos,
    8618              :                                      &ll_unsignedp, &ll_reversep, &volatilep,
    8619              :                                      &ll_and_mask, &ll_signbit,
    8620              :                                      &l_xor, &lr_arg, &lr_and_mask,
    8621              :                                      &ll_load, ll_loc);
    8622       377315 :   if (!ll_inner)
    8623              :     return 0;
    8624       284069 :   lr_inner = decode_field_reference (&lr_arg, &lr_bitsize, &lr_bitpos,
    8625              :                                      &lr_unsignedp, &lr_reversep, &volatilep,
    8626              :                                      &lr_and_mask, &lr_signbit, &l_xor, 0, 0,
    8627              :                                      &lr_load, lr_loc);
    8628       284069 :   if (!lr_inner)
    8629              :     return 0;
    8630       278780 :   rl_inner = decode_field_reference (&rl_arg, &rl_bitsize, &rl_bitpos,
    8631              :                                      &rl_unsignedp, &rl_reversep, &volatilep,
    8632              :                                      &rl_and_mask, &rl_signbit,
    8633              :                                      &r_xor, &rr_arg, &rr_and_mask,
    8634              :                                      &rl_load, rl_loc);
    8635       278780 :   if (!rl_inner)
    8636              :     return 0;
    8637       258753 :   rr_inner = decode_field_reference (&rr_arg, &rr_bitsize, &rr_bitpos,
    8638              :                                      &rr_unsignedp, &rr_reversep, &volatilep,
    8639              :                                      &rr_and_mask, &rr_signbit, &r_xor, 0, 0,
    8640              :                                      &rr_load, rr_loc);
    8641       258753 :   if (!rr_inner)
    8642              :     return 0;
    8643              : 
    8644              :   /* It must be true that the inner operation on the lhs of each
    8645              :      comparison must be the same if we are to be able to do anything.
    8646              :      Then see if we have constants.  If not, the same must be true for
    8647              :      the rhs's.  If one is a load and the other isn't, we have to be
    8648              :      conservative and avoid the optimization, otherwise we could get
    8649              :      SRAed fields wrong.  */
    8650       256156 :   if (volatilep)
    8651              :     return 0;
    8652              : 
    8653       256147 :   if (ll_reversep != rl_reversep
    8654       256147 :       || ! operand_equal_p (ll_inner, rl_inner, 0))
    8655              :     {
    8656              :       /* Try swapping the operands.  */
    8657       195733 :       if (ll_reversep != rr_reversep || rsignbit
    8658       390702 :           || !operand_equal_p (ll_inner, rr_inner, 0))
    8659              :         return 0;
    8660              : 
    8661         1638 :       rcode = swap_tree_comparison (rcode);
    8662         1638 :       std::swap (rl_arg, rr_arg);
    8663         1638 :       std::swap (rl_inner, rr_inner);
    8664         1638 :       std::swap (rl_bitsize, rr_bitsize);
    8665         1638 :       std::swap (rl_bitpos, rr_bitpos);
    8666         1638 :       std::swap (rl_unsignedp, rr_unsignedp);
    8667         1638 :       std::swap (rl_reversep, rr_reversep);
    8668         1638 :       std::swap (rl_and_mask, rr_and_mask);
    8669         1638 :       std::swap (rl_signbit, rr_signbit);
    8670         1638 :       std::swap (rl_load, rr_load);
    8671         1638 :       std::swap (rl_loc, rr_loc);
    8672              :     }
    8673              : 
    8674       121612 :   if ((ll_load && rl_load)
    8675       240934 :       ? gimple_vuse (ll_load) != gimple_vuse (rl_load)
    8676         2290 :       : (!ll_load != !rl_load))
    8677              :     return 0;
    8678              : 
    8679              :   /* ??? Can we do anything with these?  */
    8680        60967 :   if (lr_signbit > 0 || rr_signbit > 0)
    8681              :     return 0;
    8682              : 
    8683              :   /* If the mask encompassed extensions of the sign bit before
    8684              :      clipping, try to include the sign bit in the test.  If we're not
    8685              :      comparing with zero, don't even try to deal with it (for now?).
    8686              :      If we've already committed to a sign test, the extended (before
    8687              :      clipping) mask could already be messing with it.  */
    8688        60967 :   if (ll_signbit > 0)
    8689              :     {
    8690           28 :       if (!integer_zerop (lr_arg) || lsignbit)
    8691            0 :         return 0;
    8692           28 :       wide_int sign = wi::mask (ll_bitsize - 1, true, ll_bitsize);
    8693           28 :       if (!ll_and_mask.get_precision ())
    8694            0 :         ll_and_mask = sign;
    8695              :       else
    8696           28 :         ll_and_mask |= sign;
    8697           28 :     }
    8698              : 
    8699        60967 :   if (rl_signbit > 0)
    8700              :     {
    8701            4 :       if (!integer_zerop (rr_arg) || rsignbit)
    8702            1 :         return 0;
    8703            3 :       wide_int sign = wi::mask (rl_bitsize - 1, true, rl_bitsize);
    8704            3 :       if (!rl_and_mask.get_precision ())
    8705            0 :         rl_and_mask = sign;
    8706              :       else
    8707            3 :         rl_and_mask |= sign;
    8708            3 :     }
    8709              : 
    8710        60966 :   if (TREE_CODE (lr_arg) == INTEGER_CST
    8711        47712 :       && TREE_CODE (rr_arg) == INTEGER_CST)
    8712              :     {
    8713        47087 :       l_const = wi::to_wide (lr_arg);
    8714              :       /* We don't expect masks on constants, but if there are any, apply
    8715              :          them now.  */
    8716        47087 :       if (lr_and_mask.get_precision ())
    8717            4 :         l_const &= wide_int::from (lr_and_mask,
    8718            2 :                                    l_const.get_precision (), UNSIGNED);
    8719        47087 :       r_const = wi::to_wide (rr_arg);
    8720        47087 :       if (rr_and_mask.get_precision ())
    8721            0 :         r_const &= wide_int::from (rr_and_mask,
    8722            0 :                                    r_const.get_precision (), UNSIGNED);
    8723        47087 :       lr_reversep = ll_reversep;
    8724              :     }
    8725        13879 :   else if (lr_reversep != rr_reversep
    8726        13871 :            || ! operand_equal_p (lr_inner, rr_inner, 0)
    8727        24196 :            || ((lr_load && rr_load)
    8728        30753 :                ? gimple_vuse (lr_load) != gimple_vuse (rr_load)
    8729           66 :                : (!lr_load != !rr_load)))
    8730              :     return 0;
    8731              : 
    8732              :   /* If we found sign tests, finish turning them into bit tests.  */
    8733              : 
    8734        57101 :   if (lsignbit)
    8735              :     {
    8736           70 :       wide_int sign = wi::mask (ll_bitsize - 1, true, ll_bitsize);
    8737              :       /* If ll_arg is zero-extended and we're testing the sign bit, we know
    8738              :          what the result should be.  Shifting the sign bit out of sign will get
    8739              :          us to mask the entire field out, yielding zero, i.e., the sign bit of
    8740              :          the zero-extended value.  We know the masked value is being compared
    8741              :          with zero, so the compare will get us the result we're looking
    8742              :          for: TRUE if EQ_EXPR, FALSE if NE_EXPR.  Likewise if ll_arg is
    8743              :          masked so that all the extension bits are clear (ll_signbit < 0):
    8744              :          it can't be negative then, even if the field is signed.  */
    8745           70 :       if (lsignbit > ll_bitsize && (ll_unsignedp || ll_signbit < 0))
    8746            5 :         sign <<= 1;
    8747           70 :       if (!ll_and_mask.get_precision ())
    8748           66 :         ll_and_mask = sign;
    8749              :       else
    8750            4 :         ll_and_mask &= sign;
    8751           70 :       if (l_xor)
    8752              :         {
    8753            3 :           if (ll_bitsize != lr_bitsize)
    8754            1 :             return 0;
    8755            2 :           if (!lr_and_mask.get_precision ())
    8756            2 :             lr_and_mask = sign;
    8757              :           else
    8758            0 :             lr_and_mask &= sign;
    8759            2 :           if (l_const.get_precision ())
    8760            0 :             l_const &= wide_int::from (lr_and_mask,
    8761            0 :                                        l_const.get_precision (), UNSIGNED);
    8762              :         }
    8763           70 :     }
    8764              : 
    8765        57100 :   if (rsignbit)
    8766              :     {
    8767          198 :       wide_int sign = wi::mask (rl_bitsize - 1, true, rl_bitsize);
    8768          198 :       if (rsignbit > rl_bitsize && (rl_unsignedp || rl_signbit < 0))
    8769            1 :         sign <<= 1;
    8770          198 :       if (!rl_and_mask.get_precision ())
    8771          182 :         rl_and_mask = sign;
    8772              :       else
    8773           16 :         rl_and_mask &= sign;
    8774          198 :       if (r_xor)
    8775              :         {
    8776           17 :           if (rl_bitsize != rr_bitsize)
    8777            0 :             return 0;
    8778           17 :           if (!rr_and_mask.get_precision ())
    8779           17 :             rr_and_mask = sign;
    8780              :           else
    8781            0 :             rr_and_mask &= sign;
    8782           17 :           if (r_const.get_precision ())
    8783           24 :             r_const &= wide_int::from (rr_and_mask,
    8784           12 :                                        r_const.get_precision (), UNSIGNED);
    8785              :         }
    8786          198 :     }
    8787              : 
    8788              :   /* If either comparison code is not correct for our logical operation,
    8789              :      fail.  However, we can convert a one-bit comparison against zero into
    8790              :      the opposite comparison against that bit being set in the field.  */
    8791              : 
    8792        57100 :   wanted_code = (code == TRUTH_AND_EXPR ? EQ_EXPR : NE_EXPR);
    8793        57100 :   if (lcode != wanted_code)
    8794              :     {
    8795         5064 :       if (l_const.get_precision ()
    8796         4886 :           && l_const == 0
    8797         1660 :           && ll_and_mask.get_precision ()
    8798         5452 :           && wi::popcount (ll_and_mask) == 1)
    8799              :         {
    8800              :           /* Make the left operand unsigned, since we are only interested
    8801              :              in the value of one bit.  Otherwise we are doing the wrong
    8802              :              thing below.  */
    8803          228 :           ll_unsignedp = 1;
    8804          228 :           l_const = ll_and_mask;
    8805              :         }
    8806              :       else
    8807         4836 :         return 0;
    8808              :     }
    8809              : 
    8810              :   /* This is analogous to the code for l_const above.  */
    8811        52264 :   if (rcode != wanted_code)
    8812              :     {
    8813         1010 :       if (r_const.get_precision ()
    8814         1010 :           && r_const == 0
    8815          963 :           && rl_and_mask.get_precision ()
    8816         1903 :           && wi::popcount (rl_and_mask) == 1)
    8817              :         {
    8818          548 :           rl_unsignedp = 1;
    8819          548 :           r_const = rl_and_mask;
    8820              :         }
    8821              :       else
    8822          462 :         return 0;
    8823              :     }
    8824              : 
    8825              :   /* This will be bumped to 2 if any of the field pairs crosses an
    8826              :      alignment boundary, so the merged compare has to be done in two
    8827              :      parts.  */
    8828        51802 :   int parts = 1;
    8829              :   /* Set to true if the second combined compare should come first,
    8830              :      e.g., because the second original compare accesses a word that
    8831              :      the first one doesn't, and the combined compares access those in
    8832              :      cmp[0].  */
    8833        51802 :   bool first1 = false;
    8834              :   /* Set to true if the first original compare is not the one being
    8835              :      split.  */
    8836        51802 :   bool maybe_separate = false;
    8837              : 
    8838              :   /* The following 2-dimensional arrays use the first index to
    8839              :      identify left(0)- vs right(1)-hand compare operands, and the
    8840              :      second one to identify merged compare parts.  */
    8841              :   /* The memory loads or constants to be compared.  */
    8842        51802 :   tree ld_arg[2][2];
    8843              :   /* The first bit of the corresponding inner object that the
    8844              :      corresponding LD_ARG covers.  */
    8845        51802 :   HOST_WIDE_INT bitpos[2][2];
    8846              :   /* The bit count starting at BITPOS that the corresponding LD_ARG
    8847              :      covers.  */
    8848        51802 :   HOST_WIDE_INT bitsiz[2][2];
    8849              :   /* The number of bits by which LD_ARG has already been shifted
    8850              :      right, WRT mask.  */
    8851        51802 :   HOST_WIDE_INT shifted[2][2];
    8852              :   /* The number of bits by which both LD_ARG and MASK need shifting to
    8853              :      bring its least-significant bit to bit zero.  */
    8854        51802 :   HOST_WIDE_INT toshift[2][2];
    8855              :   /* An additional mask to be applied to LD_ARG, to remove any bits
    8856              :      that may have been loaded for use in another compare, but that
    8857              :      don't belong in the corresponding compare.  */
    8858       621624 :   wide_int xmask[2][2] = {};
    8859              : 
    8860              :   /* The combined compare or compares.  */
    8861        51802 :   tree cmp[2];
    8862              : 
    8863              :   /* Consider we're comparing two non-contiguous fields of packed
    8864              :      structs, both aligned at 32-bit boundaries:
    8865              : 
    8866              :      ll_arg: an 8-bit field at offset 0
    8867              :      lr_arg: a 16-bit field at offset 2
    8868              : 
    8869              :      rl_arg: an 8-bit field at offset 1
    8870              :      rr_arg: a 16-bit field at offset 3
    8871              : 
    8872              :      We'll have r_split_load, because rr_arg straddles across an
    8873              :      alignment boundary.
    8874              : 
    8875              :      We'll want to have:
    8876              : 
    8877              :      bitpos  = { {  0,  0 }, {  0, 32 } }
    8878              :      bitsiz  = { { 32, 32 }, { 32,  8 } }
    8879              : 
    8880              :      And, for little-endian:
    8881              : 
    8882              :      shifted = { {  0,  0 }, {  0, 32 } }
    8883              :      toshift = { {  0, 24 }, {  0,  0 } }
    8884              : 
    8885              :      Or, for big-endian:
    8886              : 
    8887              :      shifted = { {  0,  0 }, {  8,  0 } }
    8888              :      toshift = { {  8,  0 }, {  0,  0 } }
    8889              :   */
    8890              : 
    8891              :   /* See if we can find a mode that contains both fields being compared on
    8892              :      the left.  If we can't, fail.  Otherwise, update all constants and masks
    8893              :      to be relative to a field of that size.  */
    8894        51802 :   first_bit = MIN (ll_bitpos, rl_bitpos);
    8895        51802 :   end_bit = MAX (ll_bitpos + ll_bitsize, rl_bitpos + rl_bitsize);
    8896        51802 :   HOST_WIDE_INT ll_align = TYPE_ALIGN (TREE_TYPE (ll_inner));
    8897        51802 :   poly_uint64 ll_end_region = 0;
    8898        51802 :   if (TYPE_SIZE (TREE_TYPE (ll_inner))
    8899        51802 :       && tree_fits_poly_uint64_p (TYPE_SIZE (TREE_TYPE (ll_inner))))
    8900        51802 :     ll_end_region = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (ll_inner)));
    8901        51802 :   if (get_best_mode (end_bit - first_bit, first_bit, 0, ll_end_region,
    8902        51802 :                      ll_align, BITS_PER_WORD, volatilep, &lnmode))
    8903              :     l_split_load = false;
    8904              :   /* ??? If ll and rl share the same load, reuse that?
    8905              :      See PR 118206 -> gcc.dg/field-merge-18.c  */
    8906              :   else
    8907              :     {
    8908              :       /* Consider the possibility of recombining loads if any of the
    8909              :          fields straddles across an alignment boundary, so that either
    8910              :          part can be loaded along with the other field.  Since we
    8911              :          limit access modes to BITS_PER_WORD, don't exceed that,
    8912              :          otherwise on a 32-bit host and a 64-bit-aligned data
    8913              :          structure, we'll fail the above for a field that straddles
    8914              :          across two words, and would fail here for not even trying to
    8915              :          split it at between 32-bit words.  */
    8916        48266 :       HOST_WIDE_INT boundary = compute_split_boundary_from_align
    8917        51488 :         (MIN (ll_align, BITS_PER_WORD),
    8918              :          ll_bitpos, ll_bitsize, rl_bitpos, rl_bitsize);
    8919              : 
    8920        48266 :       if (boundary < 0
    8921          219 :           || !get_best_mode (boundary - first_bit, first_bit, 0, ll_end_region,
    8922              :                              ll_align, BITS_PER_WORD, volatilep, &lnmode)
    8923        48443 :           || !get_best_mode (end_bit - boundary, boundary, 0, ll_end_region,
    8924          177 :                              ll_align, BITS_PER_WORD, volatilep, &lnmode2))
    8925              :         {
    8926        51269 :           if (ll_align <= BITS_PER_WORD)
    8927              :             return 0;
    8928              : 
    8929              :           /* As a last resort, try double-word access modes.  This
    8930              :              enables us to deal with misaligned double-word fields
    8931              :              that straddle across 3 separate words.  */
    8932         1999 :           boundary = compute_split_boundary_from_align
    8933         2101 :             (MIN (ll_align, 2 * BITS_PER_WORD),
    8934              :              ll_bitpos, ll_bitsize, rl_bitpos, rl_bitsize);
    8935         1999 :           if (boundary < 0
    8936            0 :               || !get_best_mode (boundary - first_bit, first_bit,
    8937              :                                  0, ll_end_region, ll_align, 2 * BITS_PER_WORD,
    8938              :                                  volatilep, &lnmode)
    8939         1999 :               || !get_best_mode (end_bit - boundary, boundary,
    8940            0 :                                  0, ll_end_region, ll_align, 2 * BITS_PER_WORD,
    8941              :                                  volatilep, &lnmode2))
    8942              :             return 0;
    8943              :         }
    8944              : 
    8945              :       /* If we can't have a single load, but can with two, figure out whether
    8946              :          the two compares can be separated, i.e., whether the entirety of the
    8947              :          first original compare is encompassed by the entirety of the first
    8948              :          combined compare.  If the first original compare is past the alignment
    8949              :          boundary, arrange to compare that range first, by setting first1
    8950              :          (meaning make cmp[1] first, instead of cmp[0]).  */
    8951          177 :       l_split_load = true;
    8952          177 :       parts = 2;
    8953          177 :       if (ll_bitpos >= boundary)
    8954              :         maybe_separate = first1 = true;
    8955          132 :       else if (ll_bitpos + ll_bitsize <= boundary)
    8956           32 :         maybe_separate = true;
    8957              :     }
    8958              : 
    8959         3713 :   lnbitsize = GET_MODE_BITSIZE (lnmode);
    8960         3713 :   lnbitpos = first_bit & ~ (lnbitsize - 1);
    8961              :   /* Avoid situations that the code below can't handle.  */
    8962         3713 :   if (lnbitpos < 0)
    8963              :     return 0;
    8964              : 
    8965              :   /* Choose the type for the combined compare.  Even if we're splitting loads,
    8966              :      make it wide enough to hold both.  */
    8967         3713 :   if (l_split_load)
    8968          354 :     lnbitsize += GET_MODE_BITSIZE (lnmode2);
    8969         3713 :   lntype = build_nonstandard_integer_type (lnbitsize, 1);
    8970         3713 :   if (!lntype)
    8971              :     return NULL_TREE;
    8972         3713 :   lnprec = TYPE_PRECISION (lntype);
    8973         3713 :   xll_bitpos = ll_bitpos - lnbitpos, xrl_bitpos = rl_bitpos - lnbitpos;
    8974              : 
    8975              :   /* Adjust bit ranges for reverse endianness.  */
    8976         3713 :   if (ll_reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    8977              :     {
    8978            6 :       xll_bitpos = lnbitsize - xll_bitpos - ll_bitsize;
    8979            6 :       xrl_bitpos = lnbitsize - xrl_bitpos - rl_bitsize;
    8980              :     }
    8981              : 
    8982              :   /* Adjust masks to match the positions in the combined lntype.  */
    8983         7426 :   wide_int ll_mask, rl_mask, r_mask;
    8984         3713 :   if (ll_and_mask.get_precision ())
    8985         4448 :     ll_mask = wi::lshift (wide_int::from (ll_and_mask, lnprec, UNSIGNED),
    8986         2224 :                           xll_bitpos);
    8987              :   else
    8988         1489 :     ll_mask = wi::shifted_mask (xll_bitpos, ll_bitsize, false, lnprec);
    8989         3713 :   if (rl_and_mask.get_precision ())
    8990         4076 :     rl_mask = wi::lshift (wide_int::from (rl_and_mask, lnprec, UNSIGNED),
    8991         2038 :                           xrl_bitpos);
    8992              :   else
    8993         1675 :     rl_mask = wi::shifted_mask (xrl_bitpos, rl_bitsize, false, lnprec);
    8994              : 
    8995              :   /* When we set l_const, we also set r_const.  */
    8996         3713 :   gcc_checking_assert (!l_const.get_precision () == !r_const.get_precision ());
    8997              : 
    8998              :   /* Adjust right-hand constants in both original comparisons to match width
    8999              :      and bit position.  */
    9000         3713 :   if (l_const.get_precision ())
    9001              :     {
    9002              :       /* Before clipping upper bits of the right-hand operand of the compare,
    9003              :          check that they're sign or zero extensions, depending on how the
    9004              :          left-hand operand would be extended.  If it is unsigned, or if there's
    9005              :          a mask that zeroes out extension bits, whether because we've checked
    9006              :          for upper bits in the mask and found them all clear (ll_signbit < 0),
    9007              :          or because the sign bit itself is masked out, check that the
    9008              :          right-hand operand is zero-extended.  */
    9009         2195 :       bool l_non_ext_bits = false;
    9010         2195 :       if (ll_bitsize < lr_bitsize)
    9011              :         {
    9012           81 :           wide_int zext = wi::zext (l_const, ll_bitsize);
    9013          162 :           if ((ll_unsignedp
    9014           69 :                || ll_signbit < 0
    9015           57 :                || (ll_and_mask.get_precision ()
    9016          139 :                    && ((ll_and_mask & wi::mask (ll_bitsize - 1, true, ll_bitsize))
    9017           58 :                        == 0))
    9018          312 :                ? zext : wi::sext (l_const, ll_bitsize)) == l_const)
    9019           80 :             l_const = zext;
    9020              :           else
    9021              :             l_non_ext_bits = true;
    9022           81 :         }
    9023              :       /* We're doing bitwise equality tests, so don't bother with sign
    9024              :          extensions.  */
    9025         2195 :       l_const = wide_int::from (l_const, lnprec, UNSIGNED);
    9026         2195 :       if (ll_and_mask.get_precision ())
    9027         1221 :         l_const &= wide_int::from (ll_and_mask, lnprec, UNSIGNED);
    9028         2195 :       l_const <<= xll_bitpos;
    9029         6583 :       if (l_non_ext_bits || (l_const & ~ll_mask) != 0)
    9030              :         {
    9031            1 :           warning_at (lloc, OPT_Wtautological_compare,
    9032              :                       "comparison is always %d", wanted_code == NE_EXPR);
    9033              : 
    9034            1 :           return constant_boolean_node (wanted_code == NE_EXPR, truth_type);
    9035              :         }
    9036              : 
    9037              :       /* Before clipping upper bits of the right-hand operand of the compare,
    9038              :          check that they're sign or zero extensions, depending on how the
    9039              :          left-hand operand would be extended.  */
    9040         2194 :       bool r_non_ext_bits = false;
    9041         2194 :       if (rl_bitsize < rr_bitsize)
    9042              :         {
    9043           11 :           wide_int zext = wi::zext (r_const, rl_bitsize);
    9044           22 :           if ((rl_unsignedp
    9045            7 :                || rl_signbit < 0
    9046            5 :                || (rl_and_mask.get_precision ()
    9047           19 :                    && ((rl_and_mask & wi::mask (rl_bitsize - 1, true, rl_bitsize))
    9048            8 :                        == 0))
    9049           40 :                ? zext : wi::sext (r_const, rl_bitsize)) == r_const)
    9050           11 :             r_const = zext;
    9051              :           else
    9052              :             r_non_ext_bits = true;
    9053           11 :         }
    9054         2194 :       r_const = wide_int::from (r_const, lnprec, UNSIGNED);
    9055         2194 :       if (rl_and_mask.get_precision ())
    9056         1083 :         r_const &= wide_int::from (rl_and_mask, lnprec, UNSIGNED);
    9057         2194 :       r_const <<= xrl_bitpos;
    9058         6582 :       if (r_non_ext_bits || (r_const & ~rl_mask) != 0)
    9059              :         {
    9060            0 :           warning_at (rloc, OPT_Wtautological_compare,
    9061              :                       "comparison is always %d", wanted_code == NE_EXPR);
    9062              : 
    9063            0 :           return constant_boolean_node (wanted_code == NE_EXPR, truth_type);
    9064              :         }
    9065              : 
    9066              :       /* If there is something in common between the masks, those bits of the
    9067              :          constants must be the same.  If not, the combined condition cannot be
    9068              :          met, and the result is known.  Test for this to avoid generating
    9069              :          incorrect code below.  */
    9070         2194 :       wide_int mask = ll_mask & rl_mask;
    9071         2194 :       if (mask != 0
    9072         2287 :           && (l_const & mask) != (r_const & mask))
    9073              :         {
    9074            0 :           if (wanted_code == NE_EXPR)
    9075            0 :             return constant_boolean_node (true, truth_type);
    9076              :           else
    9077            0 :             return constant_boolean_node (false, truth_type);
    9078              :         }
    9079              : 
    9080              :       /* The constants are combined so as to line up with the loaded field, so
    9081              :          tentatively use the same parameters for the second combined
    9082              :          compare.  */
    9083         2194 :       ld_arg[1][0] = wide_int_to_tree (lntype, l_const | r_const);
    9084         2194 :       toshift[1][0] = MIN (xll_bitpos, xrl_bitpos);
    9085         2194 :       shifted[1][0] = 0;
    9086         2194 :       bitpos[1][0] = lnbitpos;
    9087         2194 :       bitsiz[1][0] = lnbitsize;
    9088              : 
    9089         2194 :       if (parts > 1)
    9090           49 :         reuse_split_load (ld_arg[1], bitpos[1], bitsiz[1], toshift[1],
    9091              :                           shifted[1], xmask[1],
    9092           49 :                           lnbitpos + GET_MODE_BITSIZE (lnmode),
    9093              :                           lr_reversep);
    9094              : 
    9095              :       /* No masking needed, we know the full constants.  */
    9096         2194 :       r_mask = wi::mask (0, true, lnprec);
    9097              : 
    9098              :       /* If the compiler thinks this is used uninitialized below, it's
    9099              :          because it can't realize that parts can only be 2 when
    9100              :          comparing with constants if l_split_load is also true.  This
    9101              :          just silences the warning.  */
    9102         2194 :       rnbitpos = 0;
    9103         2194 :     }
    9104              : 
    9105              :   /* Likewise, if the right sides are not constant, align them for the combined
    9106              :      compare.  Also, disallow this optimization if a size, signedness or
    9107              :      storage order mismatch occurs between the left and right sides.
    9108              :      A signed field whose mask clears all the extension bits
    9109              :      (signbit < 0) is effectively zero-extended, so it must not be
    9110              :      compared with a sign-extended field: clipping both to the field
    9111              :      width would drop the test that the sign bit of the latter is
    9112              :      clear.  */
    9113              :   else
    9114              :     {
    9115         1518 :       if (ll_bitsize != lr_bitsize || rl_bitsize != rr_bitsize
    9116         2864 :           || ((ll_unsignedp || ll_signbit < 0)
    9117         1432 :               != (lr_unsignedp || lr_signbit < 0))
    9118         2858 :           || ((rl_unsignedp || rl_signbit < 0)
    9119         1429 :               != (rr_unsignedp || rr_signbit < 0))
    9120         1428 :           || ll_reversep != lr_reversep
    9121              :           /* Make sure the two fields on the right
    9122              :              correspond to the left without being swapped.  */
    9123         1428 :           || ll_bitpos - rl_bitpos != lr_bitpos - rr_bitpos)
    9124          335 :         return 0;
    9125              : 
    9126              :       /* Each compare tests, for every bit in its masks, either that the bits
    9127              :          on both sides are equal (bit in both masks), or that the bit on one
    9128              :          side is zero (bit in one mask only).  The combined compare can only
    9129              :          express one such test per bit, so reject combinations in which both
    9130              :          compares test the same bit in different ways.  Check this in the
    9131              :          left-hand frame, since the right-hand fields are laid out alike.  */
    9132         1196 :       wide_int lr_m, rr_m;
    9133         1196 :       if (lr_and_mask.get_precision ())
    9134         2002 :         lr_m = wi::lshift (wide_int::from (lr_and_mask, lnprec, UNSIGNED),
    9135         1001 :                            xll_bitpos);
    9136              :       else
    9137          195 :         lr_m = wi::shifted_mask (xll_bitpos, lr_bitsize, false, lnprec);
    9138         1196 :       if (rr_and_mask.get_precision ())
    9139         1892 :         rr_m = wi::lshift (wide_int::from (rr_and_mask, lnprec, UNSIGNED),
    9140          946 :                            xrl_bitpos);
    9141              :       else
    9142          250 :         rr_m = wi::shifted_mask (xrl_bitpos, rr_bitsize, false, lnprec);
    9143         2392 :       if (((ll_mask | lr_m) & (rl_mask | rr_m)
    9144         3588 :            & ((ll_mask ^ rl_mask) | (lr_m ^ rr_m))) != 0)
    9145              :         return 0;
    9146              : 
    9147         1191 :       bool r_split_load;
    9148         1191 :       scalar_int_mode rnmode2;
    9149              : 
    9150              :       /* Figure out how to load the bits for the right-hand size of the
    9151              :          combined compare.  As in the left-hand size, we may have to split it,
    9152              :          and then we use two separate compares.  */
    9153         1191 :       first_bit = MIN (lr_bitpos, rr_bitpos);
    9154         1191 :       end_bit = MAX (lr_bitpos + lr_bitsize, rr_bitpos + rr_bitsize);
    9155         1191 :       HOST_WIDE_INT lr_align = TYPE_ALIGN (TREE_TYPE (lr_inner));
    9156         1191 :       poly_uint64 lr_end_region = 0;
    9157         1191 :       if (TYPE_SIZE (TREE_TYPE (lr_inner))
    9158         1191 :           && tree_fits_poly_uint64_p (TYPE_SIZE (TREE_TYPE (lr_inner))))
    9159         1191 :         lr_end_region = tree_to_poly_uint64 (TYPE_SIZE (TREE_TYPE (lr_inner)));
    9160         1191 :       if (!get_best_mode (end_bit - first_bit, first_bit, 0, lr_end_region,
    9161         1191 :                           lr_align, BITS_PER_WORD, volatilep, &rnmode))
    9162              :         {
    9163              :           /* Consider the possibility of recombining loads if any of the
    9164              :              fields straddles across an alignment boundary, so that either
    9165              :              part can be loaded along with the other field.  */
    9166          138 :           HOST_WIDE_INT boundary = compute_split_boundary_from_align
    9167          138 :             (lr_align, lr_bitpos, lr_bitsize, rr_bitpos, rr_bitsize);
    9168              : 
    9169          138 :           if (boundary < 0
    9170              :               /* If we're to split both, make sure the split point is
    9171              :                  the same.  */
    9172          130 :               || (l_split_load
    9173          128 :                   && (boundary - lr_bitpos
    9174          128 :                       != (lnbitpos + GET_MODE_BITSIZE (lnmode)) - ll_bitpos))
    9175          130 :               || !get_best_mode (boundary - first_bit, first_bit,
    9176              :                                  0, lr_end_region,
    9177          130 :                                  lr_align, BITS_PER_WORD, volatilep, &rnmode)
    9178          268 :               || !get_best_mode (end_bit - boundary, boundary, 0, lr_end_region,
    9179          130 :                                  lr_align, BITS_PER_WORD, volatilep, &rnmode2))
    9180              :             return 0;
    9181              : 
    9182          130 :           r_split_load = true;
    9183          130 :           parts = 2;
    9184          130 :           if (lr_bitpos >= boundary)
    9185              :             maybe_separate = first1 = true;
    9186           88 :           else if (lr_bitpos + lr_bitsize <= boundary)
    9187           29 :             maybe_separate = true;
    9188              :         }
    9189              :       else
    9190              :         r_split_load = false;
    9191              : 
    9192              :       /* Find a type that can hold the entire right-hand operand.  */
    9193         1183 :       rnbitsize = GET_MODE_BITSIZE (rnmode);
    9194         1183 :       rnbitpos = first_bit & ~ (rnbitsize - 1);
    9195         1183 :       if (r_split_load)
    9196          260 :         rnbitsize += GET_MODE_BITSIZE (rnmode2);
    9197         1183 :       rntype = build_nonstandard_integer_type (rnbitsize, 1);
    9198         1183 :       if (!rntype)
    9199              :         return  0;
    9200         1183 :       rnprec = TYPE_PRECISION (rntype);
    9201         1183 :       xlr_bitpos = lr_bitpos - rnbitpos, xrr_bitpos = rr_bitpos - rnbitpos;
    9202              : 
    9203              :       /* Adjust for reversed endianness.  */
    9204         1183 :       if (lr_reversep ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    9205              :         {
    9206            0 :           xlr_bitpos = rnbitsize - xlr_bitpos - lr_bitsize;
    9207            0 :           xrr_bitpos = rnbitsize - xrr_bitpos - rr_bitsize;
    9208              :         }
    9209              : 
    9210              :       /* Adjust the masks to match the combined type, and combine them.  */
    9211         1183 :       wide_int lr_mask, rr_mask;
    9212         1183 :       if (lr_and_mask.get_precision ())
    9213         1994 :         lr_mask = wi::lshift (wide_int::from (lr_and_mask, rnprec, UNSIGNED),
    9214          997 :                               xlr_bitpos);
    9215              :       else
    9216          186 :         lr_mask = wi::shifted_mask (xlr_bitpos, lr_bitsize, false, rnprec);
    9217         1183 :       if (rr_and_mask.get_precision ())
    9218         1890 :         rr_mask = wi::lshift (wide_int::from (rr_and_mask, rnprec, UNSIGNED),
    9219          945 :                               xrr_bitpos);
    9220              :       else
    9221          238 :         rr_mask = wi::shifted_mask (xrr_bitpos, rr_bitsize, false, rnprec);
    9222         1183 :       r_mask = lr_mask | rr_mask;
    9223              : 
    9224              :       /* Load the right-hand operand of the combined compare.  */
    9225         1183 :       toshift[1][0] = MIN (xlr_bitpos, xrr_bitpos);
    9226         1183 :       shifted[1][0] = 0;
    9227              : 
    9228         1183 :       if (!r_split_load)
    9229              :         {
    9230         1053 :           bitpos[1][0] = rnbitpos;
    9231         1053 :           bitsiz[1][0] = rnbitsize;
    9232         2106 :           ld_arg[1][0] = make_bit_field_load (ll_loc[3], lr_inner, lr_arg,
    9233         1053 :                                               rntype, rnbitsize, rnbitpos,
    9234              :                                               lr_unsignedp || rr_unsignedp,
    9235              :                                               lr_reversep, lr_load);
    9236              :         }
    9237              : 
    9238              :       /* ... and the second part of the right-hand operand if needed.  */
    9239         1183 :       if (parts > 1)
    9240              :         {
    9241          130 :           if (r_split_load)
    9242              :             {
    9243          130 :               gimple *point[2];
    9244          130 :               point[0] = lr_load;
    9245          130 :               point[1] = rr_load;
    9246          130 :               build_split_load (ld_arg[1], bitpos[1], bitsiz[1], toshift[1],
    9247              :                                 shifted[1], rl_loc[3], lr_inner, lr_arg,
    9248              :                                 rnmode, rnmode2, rnbitpos, lr_reversep, point);
    9249              :             }
    9250              :           else
    9251            0 :             reuse_split_load (ld_arg[1], bitpos[1], bitsiz[1], toshift[1],
    9252              :                               shifted[1], xmask[1],
    9253            0 :                               lnbitpos + GET_MODE_BITSIZE (lnmode)
    9254            0 :                               - ll_bitpos + lr_bitpos, lr_reversep);
    9255              :         }
    9256         1196 :     }
    9257              : 
    9258              :   /* Now issue the loads for the left-hand combined operand/s.  */
    9259         6754 :   wide_int l_mask = ll_mask | rl_mask;
    9260         3377 :   toshift[0][0] = MIN (xll_bitpos, xrl_bitpos);
    9261         3377 :   shifted[0][0] = 0;
    9262              : 
    9263         3377 :   if (!l_split_load)
    9264              :     {
    9265         3200 :       bitpos[0][0] = lnbitpos;
    9266         3200 :       bitsiz[0][0] = lnbitsize;
    9267         6400 :       ld_arg[0][0] = make_bit_field_load (ll_loc[3], ll_inner, ll_arg,
    9268         3200 :                                           lntype, lnbitsize, lnbitpos,
    9269              :                                           ll_unsignedp || rl_unsignedp,
    9270              :                                           ll_reversep, ll_load);
    9271              :     }
    9272              : 
    9273         3377 :   if (parts > 1)
    9274              :     {
    9275          179 :       if (l_split_load)
    9276              :             {
    9277          177 :               gimple *point[2];
    9278          177 :               point[0] = ll_load;
    9279          177 :               point[1] = rl_load;
    9280          177 :               build_split_load (ld_arg[0], bitpos[0], bitsiz[0], toshift[0],
    9281              :                                 shifted[0], rl_loc[3], ll_inner, ll_arg,
    9282              :                                 lnmode, lnmode2, lnbitpos, ll_reversep, point);
    9283              :             }
    9284              :       else
    9285            2 :         reuse_split_load (ld_arg[0], bitpos[0], bitsiz[0], toshift[0],
    9286              :                           shifted[0], xmask[0],
    9287            2 :                           rnbitpos + GET_MODE_BITSIZE (rnmode)
    9288            2 :                           - lr_bitpos + ll_bitpos, ll_reversep);
    9289              :     }
    9290              : 
    9291              :   /* Compute the compares.  */
    9292         6933 :   for (int i = 0; i < parts; i++)
    9293              :     {
    9294         3556 :       tree op[2] = { ld_arg[0][i], ld_arg[1][i] };
    9295        10668 :       wide_int mask[2] = { l_mask, r_mask };
    9296         3556 :       location_t *locs[2] = { i ? rl_loc : ll_loc, i ? rr_loc : lr_loc };
    9297              : 
    9298              :       /* Figure out the masks, and unshare the original operands.  */
    9299        10668 :       for (int j = 0; j < 2; j++)
    9300              :         {
    9301         7112 :           unsigned prec = TYPE_PRECISION (TREE_TYPE (op[j]));
    9302         7112 :           op[j] = unshare_expr (op[j]);
    9303              : 
    9304              :           /* Mask out the bits belonging to the other part.  */
    9305         7112 :           if (xmask[j][i].get_precision ())
    9306          102 :             mask[j] &= xmask[j][i];
    9307              : 
    9308         7112 :           if (shifted[j][i])
    9309              :             {
    9310          307 :               wide_int shift = wide_int::from (shifted[j][i], prec, UNSIGNED);
    9311          307 :               mask[j] = wi::lrshift (mask[j], shift);
    9312          307 :             }
    9313         7112 :           mask[j] = wide_int::from (mask[j], prec, UNSIGNED);
    9314              :         }
    9315              : 
    9316              :       /* Line up the operands for a compare.  */
    9317         3556 :       HOST_WIDE_INT shift = (toshift[0][i] - toshift[1][i]);
    9318              : 
    9319         3556 :       if (shift)
    9320              :         {
    9321           54 :           int j;
    9322           54 :           if (shift > 0)
    9323              :             j = 0;
    9324              :           else
    9325              :             {
    9326           52 :               j = 1;
    9327           52 :               shift = -shift;
    9328              :             }
    9329              : 
    9330           54 :           tree shiftsz = bitsize_int (shift);
    9331           54 :           op[j] = fold_build2_loc (locs[j][1], RSHIFT_EXPR, TREE_TYPE (op[j]),
    9332              :                                    op[j], shiftsz);
    9333           54 :           mask[j] = wi::lrshift (mask[j], shift);
    9334              :         }
    9335              : 
    9336              :       /* Convert to the smaller type before masking out unwanted
    9337              :          bits.  */
    9338         3556 :       tree type = TREE_TYPE (op[0]);
    9339         3556 :       if (type != TREE_TYPE (op[1]))
    9340              :         {
    9341          190 :           int j = (TYPE_PRECISION (type)
    9342          190 :                    < TYPE_PRECISION (TREE_TYPE (op[1])));
    9343          190 :           if (!j)
    9344           91 :             type = TREE_TYPE (op[1]);
    9345          190 :           op[j] = fold_convert_loc (locs[j][0], type, op[j]);
    9346          190 :           mask[j] = wide_int::from (mask[j], TYPE_PRECISION (type), UNSIGNED);
    9347              :         }
    9348              : 
    9349              :       /* Apply masks.  */
    9350        10668 :       for (int j = 0; j < 2; j++)
    9351         7112 :         if (mask[j] != wi::mask (0, true, mask[j].get_precision ()))
    9352         2758 :           op[j] = fold_build2_loc (locs[j][2], BIT_AND_EXPR, type,
    9353         5516 :                                    op[j], wide_int_to_tree (type, mask[j]));
    9354              : 
    9355         6933 :       cmp[i] = fold_build2_loc (i ? rloc : lloc, wanted_code, truth_type,
    9356              :                                 op[0], op[1]);
    9357        10668 :     }
    9358              : 
    9359              :   /* Reorder the compares if needed.  */
    9360         3377 :   if (first1)
    9361           45 :     std::swap (cmp[0], cmp[1]);
    9362              : 
    9363              :   /* Prepare to return the resulting compares.  Combine two parts if
    9364              :      needed.  */
    9365         3377 :   if (parts == 1)
    9366         3198 :     result = cmp[0];
    9367          179 :   else if (!separatep || !maybe_separate)
    9368              :     {
    9369              :       /* Only fold if any of the cmp is known, otherwise we may lose the
    9370              :          sequence point, and that may prevent further optimizations.  */
    9371          173 :       if (TREE_CODE (cmp[0]) == INTEGER_CST
    9372          137 :           || TREE_CODE (cmp[1]) == INTEGER_CST)
    9373           37 :         result = fold_build2_loc (rloc, orig_code, truth_type, cmp[0], cmp[1]);
    9374              :       else
    9375          136 :         result = build2_loc (rloc, orig_code, truth_type, cmp[0], cmp[1]);
    9376              :     }
    9377              :   else
    9378              :     {
    9379            6 :       result = cmp[0];
    9380            6 :       *separatep = cmp[1];
    9381              :     }
    9382              : 
    9383         3377 :   return result;
    9384       440944 : }
    9385              : 
    9386              : /* Try to simplify the AND of two comparisons, specified by
    9387              :    (OP1A CODE1 OP1B) and (OP2B CODE2 OP2B), respectively.
    9388              :    If this can be simplified to a single expression (without requiring
    9389              :    introducing more SSA variables to hold intermediate values),
    9390              :    return the resulting tree.  Otherwise return NULL_TREE.
    9391              :    If the result expression is non-null, it has boolean type.
    9392              :    CODE is TRUTH_AND_EXPR if both comparisons are evaluated and
    9393              :    TRUTH_ANDIF_EXPR if the second one is only evaluated when the first
    9394              :    one is true; with -ftrapping-math the result raises an exception
    9395              :    for a NaN exactly when the evaluated comparisons do.  */
    9396              : 
    9397              : tree
    9398       608857 : maybe_fold_and_comparisons (tree type, enum tree_code code,
    9399              :                             enum tree_code code1, tree op1a, tree op1b,
    9400              :                             enum tree_code code2, tree op2a, tree op2b,
    9401              :                             basic_block outer_cond_bb)
    9402              : {
    9403       608857 :   if (tree t = and_comparisons_1 (type, code, code1, op1a, op1b,
    9404              :                                   code2, op2a, op2b, outer_cond_bb))
    9405              :     return t;
    9406              : 
    9407       607270 :   if (tree t = maybe_fold_comparisons_from_match_pd (type, BIT_AND_EXPR, code1,
    9408              :                                                      op1a, op1b, code2, op2a,
    9409              :                                                      op2b, outer_cond_bb))
    9410              :     return t;
    9411              : 
    9412              :   return NULL_TREE;
    9413              : }
    9414              : 
    9415              : /* Helper function for or_comparisons_1:  try to simplify the OR of the
    9416              :    ssa variable VAR with the comparison specified by (OP2A CODE2 OP2B).
    9417              :    If INVERT is true, invert the value of VAR before doing the OR.
    9418              :    Return NULL_EXPR if we can't simplify this to a single expression.  */
    9419              : 
    9420              : static tree
    9421        30923 : or_var_with_comparison (tree type, enum tree_code code, tree var,
    9422              :                         bool invert, enum tree_code code2, tree op2a,
    9423              :                         tree op2b, basic_block outer_cond_bb)
    9424              : {
    9425        30923 :   tree t;
    9426        30923 :   gimple *stmt = SSA_NAME_DEF_STMT (var);
    9427              : 
    9428              :   /* We can only deal with variables whose definitions are assignments.  */
    9429        30923 :   if (!is_gimple_assign (stmt))
    9430              :     return NULL_TREE;
    9431              : 
    9432              :   /* If we have an inverted comparison, apply DeMorgan's law and rewrite
    9433              :      !var OR (op2a code2 op2b) => !(var AND !(op2a code2 op2b))
    9434              :      Then we only have to consider the simpler non-inverted cases.  */
    9435        30862 :   if (invert)
    9436              :     {
    9437        21831 :       code2 = invert_tree_comparison (code2, HONOR_NANS (op2a));
    9438        21831 :       if (code2 == ERROR_MARK)
    9439              :         return NULL_TREE;
    9440        21831 :       t = and_var_with_comparison_1 (type, truth_andor_compl (code), stmt,
    9441              :                                      code2, op2a, op2b, outer_cond_bb);
    9442              :     }
    9443              :   else
    9444         9031 :     t = or_var_with_comparison_1 (type, code, stmt, code2, op2a, op2b,
    9445              :                                   outer_cond_bb);
    9446        30862 :   return canonicalize_bool (t, invert);
    9447              : }
    9448              : 
    9449              : /* Try to simplify the OR of the ssa variable defined by the assignment
    9450              :    STMT with the comparison specified by (OP2A CODE2 OP2B).
    9451              :    Return NULL_EXPR if we can't simplify this to a single expression.  */
    9452              : 
    9453              : static tree
    9454        73606 : or_var_with_comparison_1 (tree type, enum tree_code code, gimple *stmt,
    9455              :                           enum tree_code code2, tree op2a, tree op2b,
    9456              :                           basic_block outer_cond_bb)
    9457              : {
    9458        73606 :   tree var = gimple_assign_lhs (stmt);
    9459        73606 :   tree true_test_var = NULL_TREE;
    9460        73606 :   tree false_test_var = NULL_TREE;
    9461        73606 :   enum tree_code innercode = gimple_assign_rhs_code (stmt);
    9462              : 
    9463              :   /* Check for identities like (var OR (var != 0)) => true .  */
    9464        73606 :   if (TREE_CODE (op2a) == SSA_NAME
    9465        73606 :       && TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE)
    9466              :     {
    9467        14836 :       if ((code2 == NE_EXPR && integer_zerop (op2b))
    9468        31736 :           || (code2 == EQ_EXPR && integer_nonzerop (op2b)))
    9469              :         {
    9470        12766 :           true_test_var = op2a;
    9471        12766 :           if (var == true_test_var)
    9472              :             return var;
    9473              :         }
    9474         2485 :       else if ((code2 == EQ_EXPR && integer_zerop (op2b))
    9475        15401 :                || (code2 == NE_EXPR && integer_nonzerop (op2b)))
    9476              :         {
    9477         4611 :           false_test_var = op2a;
    9478         4611 :           if (var == false_test_var)
    9479            0 :             return boolean_true_node;
    9480              :         }
    9481              :     }
    9482              : 
    9483              :   /* If the definition is a comparison, recurse on it.  */
    9484        73606 :   if (TREE_CODE_CLASS (innercode) == tcc_comparison)
    9485              :     {
    9486          326 :       tree t = or_comparisons_1 (type, code, innercode,
    9487              :                                  gimple_assign_rhs1 (stmt),
    9488              :                                  gimple_assign_rhs2 (stmt),
    9489              :                                  code2, op2a, op2b, outer_cond_bb);
    9490          326 :       if (t)
    9491              :         return t;
    9492              :     }
    9493              : 
    9494              :   /* If the definition is an AND or OR expression, we may be able to
    9495              :      simplify by reassociating.  */
    9496        73604 :   if (TREE_CODE (TREE_TYPE (var)) == BOOLEAN_TYPE
    9497        73604 :       && (innercode == BIT_AND_EXPR || innercode == BIT_IOR_EXPR))
    9498              :     {
    9499        23876 :       tree inner1 = gimple_assign_rhs1 (stmt);
    9500        23876 :       tree inner2 = gimple_assign_rhs2 (stmt);
    9501        23876 :       gimple *s;
    9502        23876 :       tree t;
    9503        23876 :       tree partial = NULL_TREE;
    9504        23876 :       bool is_or = (innercode == BIT_IOR_EXPR);
    9505              : 
    9506              :       /* Check for boolean identities that don't require recursive examination
    9507              :          of inner1/inner2:
    9508              :          inner1 OR (inner1 OR inner2) => inner1 OR inner2 => var
    9509              :          inner1 OR (inner1 AND inner2) => inner1
    9510              :          !inner1 OR (inner1 OR inner2) => true
    9511              :          !inner1 OR (inner1 AND inner2) => !inner1 OR inner2
    9512              :       */
    9513        23876 :       if (inner1 == true_test_var)
    9514            0 :         return (is_or ? var : inner1);
    9515        23876 :       else if (inner2 == true_test_var)
    9516            0 :         return (is_or ? var : inner2);
    9517        23876 :       else if (inner1 == false_test_var)
    9518            0 :         return (is_or
    9519            0 :                 ? boolean_true_node
    9520            0 :                 : or_var_with_comparison (type, code, inner2, false, code2,
    9521            0 :                                           op2a, op2b, outer_cond_bb));
    9522        23876 :       else if (inner2 == false_test_var)
    9523            0 :         return (is_or
    9524            0 :                 ? boolean_true_node
    9525            0 :                 : or_var_with_comparison (type, code, inner1, false, code2,
    9526            0 :                                           op2a, op2b, outer_cond_bb));
    9527              : 
    9528              :       /* Next, redistribute/reassociate the OR across the inner tests.
    9529              :          Compute the first partial result, (inner1 OR (op2a code op2b))  */
    9530        23876 :       if (TREE_CODE (inner1) == SSA_NAME
    9531        23876 :           && is_gimple_assign (s = SSA_NAME_DEF_STMT (inner1))
    9532        23246 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
    9533        39722 :           && (t = maybe_fold_or_comparisons (type, code,
    9534              :                                              gimple_assign_rhs_code (s),
    9535              :                                              gimple_assign_rhs1 (s),
    9536              :                                              gimple_assign_rhs2 (s),
    9537              :                                              code2, op2a, op2b,
    9538              :                                              outer_cond_bb)))
    9539              :         {
    9540              :           /* Handle the OR case, where we are reassociating:
    9541              :              (inner1 OR inner2) OR (op2a code2 op2b)
    9542              :              => (t OR inner2)
    9543              :              If the partial result t is a constant, we win.  Otherwise
    9544              :              continue on to try reassociating with the other inner test.  */
    9545           45 :           if (is_or)
    9546              :             {
    9547           17 :               if (integer_onep (t))
    9548            0 :                 return boolean_true_node;
    9549           17 :               else if (integer_zerop (t))
    9550              :                 return inner2;
    9551              :             }
    9552              : 
    9553              :           /* Handle the AND case, where we are redistributing:
    9554              :              (inner1 AND inner2) OR (op2a code2 op2b)
    9555              :              => (t AND (inner2 OR (op2a code op2b)))  */
    9556           28 :           else if (integer_zerop (t))
    9557            0 :             return boolean_false_node;
    9558              : 
    9559              :           /* Save partial result for later.  */
    9560              :           partial = t;
    9561              :         }
    9562              : 
    9563              :       /* Compute the second partial result, (inner2 OR (op2a code op2b)) */
    9564        23876 :       if (TREE_CODE (inner2) == SSA_NAME
    9565        23876 :           && is_gimple_assign (s = SSA_NAME_DEF_STMT (inner2))
    9566        23423 :           && TREE_CODE_CLASS (gimple_assign_rhs_code (s)) == tcc_comparison
    9567        45568 :           && (t = maybe_fold_or_comparisons (type, code,
    9568              :                                              gimple_assign_rhs_code (s),
    9569              :                                              gimple_assign_rhs1 (s),
    9570              :                                              gimple_assign_rhs2 (s),
    9571              :                                              code2, op2a, op2b,
    9572              :                                              outer_cond_bb)))
    9573              :         {
    9574              :           /* Handle the OR case, where we are reassociating:
    9575              :              (inner1 OR inner2) OR (op2a code2 op2b)
    9576              :              => (inner1 OR t)
    9577              :              => (t OR partial)  */
    9578           60 :           if (is_or)
    9579              :             {
    9580           23 :               if (integer_zerop (t))
    9581              :                 return inner1;
    9582           23 :               else if (integer_onep (t))
    9583            0 :                 return boolean_true_node;
    9584              :               /* If both are the same, we can apply the identity
    9585              :                  (x OR x) == x.  */
    9586           23 :               else if (partial && same_bool_result_p (t, partial))
    9587            0 :                 return t;
    9588              :             }
    9589              : 
    9590              :           /* Handle the AND case, where we are redistributing:
    9591              :              (inner1 AND inner2) OR (op2a code2 op2b)
    9592              :              => (t AND (inner1 OR (op2a code2 op2b)))
    9593              :              => (t AND partial)  */
    9594              :           else
    9595              :             {
    9596           37 :               if (integer_zerop (t))
    9597            0 :                 return boolean_false_node;
    9598           37 :               else if (partial)
    9599              :                 {
    9600              :                   /* We already got a simplification for the other
    9601              :                      operand to the redistributed AND expression.  The
    9602              :                      interesting case is when at least one is true.
    9603              :                      Or, if both are the same, we can apply the identity
    9604              :                      (x AND x) == x.  */
    9605           14 :                   if (integer_onep (partial))
    9606              :                     return t;
    9607           13 :                   else if (integer_onep (t))
    9608              :                     return partial;
    9609            1 :                   else if (same_bool_result_p (t, partial))
    9610            0 :                     return t;
    9611              :                 }
    9612              :             }
    9613              :         }
    9614              :     }
    9615              :   return NULL_TREE;
    9616              : }
    9617              : 
    9618              : /* Try to simplify the OR of two comparisons defined by
    9619              :    (OP1A CODE1 OP1B) and (OP2A CODE2 OP2B), respectively.
    9620              :    If this can be done without constructing an intermediate value,
    9621              :    return the resulting tree; otherwise NULL_TREE is returned.
    9622              :    This function is deliberately asymmetric as it recurses on SSA_DEFs
    9623              :    in the first comparison but not the second.  CODE is as for
    9624              :    maybe_fold_or_comparisons.  */
    9625              : 
    9626              : static tree
    9627       433755 : or_comparisons_1 (tree type, enum tree_code code,
    9628              :                   enum tree_code code1, tree op1a, tree op1b,
    9629              :                   enum tree_code code2, tree op2a, tree op2b,
    9630              :                   basic_block outer_cond_bb)
    9631              : {
    9632       433755 :   tree truth_type = truth_type_for (TREE_TYPE (op1a));
    9633              : 
    9634              :   /* First check for ((x CODE1 y) OR (x CODE2 y)).  */
    9635       433755 :   if (operand_equal_p (op1a, op2a, 0)
    9636       433755 :       && operand_equal_p (op1b, op2b, 0))
    9637              :     {
    9638              :       /* Result will be either NULL_TREE, or a combined comparison.  */
    9639         2450 :       tree t = combine_comparisons (UNKNOWN_LOCATION,
    9640              :                                     code, code1, code2,
    9641              :                                     truth_type, truth_type, op1a, op1b);
    9642         2450 :       if (t)
    9643              :         return t;
    9644              :     }
    9645              : 
    9646              :   /* Likewise the swapped case of the above.  */
    9647       431419 :   if (operand_equal_p (op1a, op2b, 0)
    9648       431419 :       && operand_equal_p (op1b, op2a, 0))
    9649              :     {
    9650              :       /* Result will be either NULL_TREE, or a combined comparison.  */
    9651            0 :       tree t = combine_comparisons (UNKNOWN_LOCATION,
    9652              :                                     code, code1,
    9653              :                                     swap_tree_comparison (code2),
    9654              :                                     truth_type, truth_type, op1a, op1b);
    9655            0 :       if (t)
    9656              :         return t;
    9657              :     }
    9658              : 
    9659              :   /* Perhaps the first comparison is (NAME != 0) or (NAME == 1) where
    9660              :      NAME's definition is a truth value.  See if there are any simplifications
    9661              :      that can be done against the NAME's definition.  */
    9662       431419 :   if (TREE_CODE (op1a) == SSA_NAME
    9663       431344 :       && (code1 == NE_EXPR || code1 == EQ_EXPR)
    9664       730810 :       && (integer_zerop (op1b) || integer_onep (op1b)))
    9665              :     {
    9666        36677 :       bool invert = ((code1 == EQ_EXPR && integer_zerop (op1b))
    9667        60145 :                      || (code1 == NE_EXPR && integer_onep (op1b)));
    9668        56167 :       gimple *stmt = SSA_NAME_DEF_STMT (op1a);
    9669        56167 :       switch (gimple_code (stmt))
    9670              :         {
    9671        30862 :         case GIMPLE_ASSIGN:
    9672              :           /* Try to simplify by copy-propagating the definition.  */
    9673        30862 :           return or_var_with_comparison (type, code, op1a, invert, code2,
    9674        30862 :                                          op2a, op2b, outer_cond_bb);
    9675              : 
    9676        12888 :         case GIMPLE_PHI:
    9677              :           /* If every argument to the PHI produces the same result when
    9678              :              ORed with the second comparison, we win.
    9679              :              Do not do this unless the type is bool since we need a bool
    9680              :              result here anyway.  */
    9681        12888 :           if (TREE_CODE (TREE_TYPE (op1a)) == BOOLEAN_TYPE)
    9682              :             {
    9683              :               tree result = NULL_TREE;
    9684              :               unsigned i;
    9685          388 :               for (i = 0; i < gimple_phi_num_args (stmt); i++)
    9686              :                 {
    9687          388 :                   tree arg = gimple_phi_arg_def (stmt, i);
    9688              : 
    9689              :                   /* If this PHI has itself as an argument, ignore it.
    9690              :                      If all the other args produce the same result,
    9691              :                      we're still OK.  */
    9692          388 :                   if (arg == gimple_phi_result (stmt))
    9693            0 :                     continue;
    9694          388 :                   else if (TREE_CODE (arg) == INTEGER_CST)
    9695              :                     {
    9696          322 :                       if (invert ? integer_zerop (arg) : integer_nonzerop (arg))
    9697              :                         {
    9698          163 :                           if (!result)
    9699           45 :                             result = boolean_true_node;
    9700          118 :                           else if (!integer_onep (result))
    9701              :                             return NULL_TREE;
    9702              :                         }
    9703          159 :                       else if (!result)
    9704          119 :                         result = fold_build2 (code2, boolean_type_node,
    9705              :                                               op2a, op2b);
    9706           40 :                       else if (!same_bool_comparison_p (result,
    9707              :                                                         code2, op2a, op2b))
    9708              :                         return NULL_TREE;
    9709              :                     }
    9710           66 :                   else if (TREE_CODE (arg) == SSA_NAME
    9711           66 :                            && !SSA_NAME_IS_DEFAULT_DEF (arg))
    9712              :                     {
    9713           66 :                       tree temp;
    9714           66 :                       gimple *def_stmt = SSA_NAME_DEF_STMT (arg);
    9715              :                       /* In simple cases we can look through PHI nodes,
    9716              :                          but we have to be careful with loops.
    9717              :                          See PR49073.  */
    9718           66 :                       if (! dom_info_available_p (CDI_DOMINATORS)
    9719           66 :                           || gimple_bb (def_stmt) == gimple_bb (stmt)
    9720          132 :                           || dominated_by_p (CDI_DOMINATORS,
    9721           66 :                                              gimple_bb (def_stmt),
    9722           66 :                                              gimple_bb (stmt)))
    9723              :                         return NULL_TREE;
    9724           61 :                       temp = or_var_with_comparison (type, code, arg, invert,
    9725              :                                                      code2, op2a, op2b,
    9726              :                                                      outer_cond_bb);
    9727           61 :                       if (!temp)
    9728              :                         return NULL_TREE;
    9729            0 :                       else if (!result)
    9730              :                         result = temp;
    9731            0 :                       else if (!same_bool_result_p (result, temp))
    9732              :                         return NULL_TREE;
    9733              :                     }
    9734              :                   else
    9735              :                     return NULL_TREE;
    9736              :                 }
    9737              :               return result;
    9738              :             }
    9739              : 
    9740              :         default:
    9741              :           break;
    9742              :         }
    9743              :     }
    9744              :   return NULL_TREE;
    9745              : }
    9746              : 
    9747              : /* Try to simplify the OR of two comparisons, specified by
    9748              :    (OP1A CODE1 OP1B) and (OP2B CODE2 OP2B), respectively.
    9749              :    If this can be simplified to a single expression (without requiring
    9750              :    introducing more SSA variables to hold intermediate values),
    9751              :    return the resulting tree.  Otherwise return NULL_TREE.
    9752              :    If the result expression is non-null, it has boolean type.
    9753              :    CODE is TRUTH_OR_EXPR if both comparisons are evaluated and
    9754              :    TRUTH_ORIF_EXPR if the second one is only evaluated when the first
    9755              :    one is false, as for maybe_fold_and_comparisons.  */
    9756              : 
    9757              : tree
    9758       236619 : maybe_fold_or_comparisons (tree type, enum tree_code code,
    9759              :                            enum tree_code code1, tree op1a, tree op1b,
    9760              :                            enum tree_code code2, tree op2a, tree op2b,
    9761              :                            basic_block outer_cond_bb)
    9762              : {
    9763       236619 :   if (tree t = or_comparisons_1 (type, code, code1, op1a, op1b,
    9764              :                                  code2, op2a, op2b, outer_cond_bb))
    9765              :     return t;
    9766              : 
    9767              :   /* Also try the other order, unless the second comparison is only
    9768              :      evaluated when the first one is false.  */
    9769       234284 :   if (code == TRUTH_OR_EXPR)
    9770       196810 :     if (tree t = or_comparisons_1 (type, code, code2, op2a, op2b,
    9771              :                                    code1, op1a, op1b, outer_cond_bb))
    9772              :       return t;
    9773              : 
    9774       234283 :   if (tree t = maybe_fold_comparisons_from_match_pd (type, BIT_IOR_EXPR, code1,
    9775              :                                                      op1a, op1b, code2, op2a,
    9776              :                                                      op2b, outer_cond_bb))
    9777              :     return t;
    9778              : 
    9779              :   return NULL_TREE;
    9780              : }
    9781              : 
    9782              : /* Fold STMT to a constant using VALUEIZE to valueize SSA names.
    9783              : 
    9784              :    Either NULL_TREE, a simplified but non-constant or a constant
    9785              :    is returned.
    9786              : 
    9787              :    ???  This should go into a gimple-fold-inline.h file to be eventually
    9788              :    privatized with the single valueize function used in the various TUs
    9789              :    to avoid the indirect function call overhead.  */
    9790              : 
    9791              : tree
    9792    434632445 : gimple_fold_stmt_to_constant_1 (gimple *stmt, tree (*valueize) (tree),
    9793              :                                 tree (*gvalueize) (tree))
    9794              : {
    9795    434632445 :   gimple_match_op res_op;
    9796              :   /* ???  The SSA propagators do not correctly deal with following SSA use-def
    9797              :      edges if there are intermediate VARYING defs.  For this reason
    9798              :      do not follow SSA edges here even though SCCVN can technically
    9799              :      just deal fine with that.  */
    9800    434632445 :   if (gimple_simplify (stmt, &res_op, NULL, gvalueize, valueize))
    9801              :     {
    9802     57301596 :       tree res = NULL_TREE;
    9803     57301596 :       if (gimple_simplified_result_is_gimple_val (&res_op))
    9804     35038185 :         res = res_op.ops[0];
    9805     22263411 :       else if (mprts_hook)
    9806      7806538 :         res = mprts_hook (&res_op);
    9807     42844723 :       if (res)
    9808              :         {
    9809     36954041 :           if (dump_file && dump_flags & TDF_DETAILS)
    9810              :             {
    9811         9605 :               fprintf (dump_file, "Match-and-simplified ");
    9812         9605 :               print_gimple_expr (dump_file, stmt, 0, TDF_SLIM);
    9813         9605 :               fprintf (dump_file, " to ");
    9814         9605 :               print_generic_expr (dump_file, res);
    9815         9605 :               fprintf (dump_file, "\n");
    9816              :             }
    9817              :           return res;
    9818              :         }
    9819              :     }
    9820              : 
    9821    397678404 :   location_t loc = gimple_location (stmt);
    9822    397678404 :   switch (gimple_code (stmt))
    9823              :     {
    9824    345015417 :     case GIMPLE_ASSIGN:
    9825    345015417 :       {
    9826    345015417 :         enum tree_code subcode = gimple_assign_rhs_code (stmt);
    9827              : 
    9828    345015417 :         switch (get_gimple_rhs_class (subcode))
    9829              :           {
    9830    118112434 :           case GIMPLE_SINGLE_RHS:
    9831    118112434 :             {
    9832    118112434 :               tree rhs = gimple_assign_rhs1 (stmt);
    9833    118112434 :               enum tree_code_class kind = TREE_CODE_CLASS (subcode);
    9834              : 
    9835    118112434 :               if (TREE_CODE (rhs) == SSA_NAME)
    9836              :                 {
    9837              :                   /* If the RHS is an SSA_NAME, return its known constant value,
    9838              :                      if any.  */
    9839     11770202 :                   return (*valueize) (rhs);
    9840              :                 }
    9841              :               /* Handle propagating invariant addresses into address
    9842              :                  operations.  */
    9843    106342232 :               else if (TREE_CODE (rhs) == ADDR_EXPR
    9844    106342232 :                        && !is_gimple_min_invariant (rhs))
    9845              :                 {
    9846      6330122 :                   poly_int64 offset = 0;
    9847      6330122 :                   tree base;
    9848      6330122 :                   base = get_addr_base_and_unit_offset_1 (TREE_OPERAND (rhs, 0),
    9849              :                                                           &offset,
    9850              :                                                           valueize);
    9851      6330122 :                   if (base
    9852      6330122 :                       && (CONSTANT_CLASS_P (base)
    9853      5658580 :                           || decl_address_invariant_p (base)))
    9854       202166 :                     return build_invariant_address (TREE_TYPE (rhs),
    9855       202166 :                                                     base, offset);
    9856              :                 }
    9857    100012110 :               else if (TREE_CODE (rhs) == CONSTRUCTOR
    9858      1137080 :                        && TREE_CODE (TREE_TYPE (rhs)) == VECTOR_TYPE
    9859    101611160 :                        && known_eq (CONSTRUCTOR_NELTS (rhs),
    9860              :                                     TYPE_VECTOR_SUBPARTS (TREE_TYPE (rhs))))
    9861              :                 {
    9862       438931 :                   unsigned i, nelts;
    9863       438931 :                   tree val;
    9864              : 
    9865       438931 :                   nelts = CONSTRUCTOR_NELTS (rhs);
    9866       438931 :                   tree_vector_builder vec (TREE_TYPE (rhs), nelts, 1);
    9867       983800 :                   FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (rhs), i, val)
    9868              :                     {
    9869       532792 :                       val = (*valueize) (val);
    9870       532792 :                       if (TREE_CODE (val) == INTEGER_CST
    9871       448678 :                           || TREE_CODE (val) == REAL_CST
    9872       426854 :                           || TREE_CODE (val) == FIXED_CST)
    9873       105938 :                         vec.quick_push (val);
    9874              :                       else
    9875              :                         return NULL_TREE;
    9876              :                     }
    9877              : 
    9878        12077 :                   return vec.build ();
    9879       438931 :                 }
    9880    105701135 :               if (subcode == OBJ_TYPE_REF)
    9881              :                 {
    9882       393080 :                   tree val = (*valueize) (OBJ_TYPE_REF_EXPR (rhs));
    9883              :                   /* If callee is constant, we can fold away the wrapper.  */
    9884       393080 :                   if (is_gimple_min_invariant (val))
    9885              :                     return val;
    9886              :                 }
    9887              : 
    9888    105700843 :               if (kind == tcc_reference)
    9889              :                 {
    9890     70127337 :                   if ((TREE_CODE (rhs) == VIEW_CONVERT_EXPR
    9891     67852093 :                        || TREE_CODE (rhs) == REALPART_EXPR
    9892     66828728 :                        || TREE_CODE (rhs) == IMAGPART_EXPR)
    9893     72332616 :                       && TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
    9894              :                     {
    9895      3548800 :                       tree val = (*valueize) (TREE_OPERAND (rhs, 0));
    9896      3548800 :                       return fold_unary_loc (EXPR_LOCATION (rhs),
    9897      3548800 :                                              TREE_CODE (rhs),
    9898      7097600 :                                              TREE_TYPE (rhs), val);
    9899              :                     }
    9900     66578537 :                   else if (TREE_CODE (rhs) == BIT_FIELD_REF
    9901     66578537 :                            && TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
    9902              :                     {
    9903       671603 :                       tree val = (*valueize) (TREE_OPERAND (rhs, 0));
    9904       671603 :                       return fold_ternary_loc (EXPR_LOCATION (rhs),
    9905       671603 :                                                TREE_CODE (rhs),
    9906       671603 :                                                TREE_TYPE (rhs), val,
    9907       671603 :                                                TREE_OPERAND (rhs, 1),
    9908      1343206 :                                                TREE_OPERAND (rhs, 2));
    9909              :                     }
    9910     65906934 :                   else if (TREE_CODE (rhs) == MEM_REF
    9911     65906934 :                            && TREE_CODE (TREE_OPERAND (rhs, 0)) == SSA_NAME)
    9912              :                     {
    9913     13585562 :                       tree val = (*valueize) (TREE_OPERAND (rhs, 0));
    9914     13585562 :                       if (TREE_CODE (val) == ADDR_EXPR
    9915     13585562 :                           && is_gimple_min_invariant (val))
    9916              :                         {
    9917       979333 :                           tree tem = fold_build2 (MEM_REF, TREE_TYPE (rhs),
    9918              :                                                   unshare_expr (val),
    9919              :                                                   TREE_OPERAND (rhs, 1));
    9920       979333 :                           if (tem)
    9921     65906934 :                             rhs = tem;
    9922              :                         }
    9923              :                     }
    9924     65906934 :                   return fold_const_aggregate_ref_1 (rhs, valueize);
    9925              :                 }
    9926     35573506 :               else if (kind == tcc_declaration)
    9927      7947137 :                 return get_symbol_constant_value (rhs);
    9928              :               return rhs;
    9929              :             }
    9930              : 
    9931              :           case GIMPLE_UNARY_RHS:
    9932              :             return NULL_TREE;
    9933              : 
    9934    170167212 :           case GIMPLE_BINARY_RHS:
    9935              :             /* Translate &x + CST into an invariant form suitable for
    9936              :                further propagation.  */
    9937    170167212 :             if (subcode == POINTER_PLUS_EXPR)
    9938              :               {
    9939     25541150 :                 tree op0 = (*valueize) (gimple_assign_rhs1 (stmt));
    9940     25541150 :                 tree op1 = (*valueize) (gimple_assign_rhs2 (stmt));
    9941     25541150 :                 if (TREE_CODE (op0) == ADDR_EXPR
    9942      5824218 :                     && TREE_CODE (op1) == INTEGER_CST)
    9943              :                   {
    9944       587637 :                     tree off = fold_convert (ptr_type_node, op1);
    9945       587637 :                     return build1_loc
    9946      1175274 :                         (loc, ADDR_EXPR, TREE_TYPE (op0),
    9947       587637 :                          fold_build2 (MEM_REF,
    9948              :                                       TREE_TYPE (TREE_TYPE (op0)),
    9949       587637 :                                       unshare_expr (op0), off));
    9950              :                   }
    9951              :               }
    9952              :             /* Canonicalize bool != 0 and bool == 0 appearing after
    9953              :                valueization.  While gimple_simplify handles this
    9954              :                it can get confused by the ~X == 1 -> X == 0 transform
    9955              :                which we cant reduce to a SSA name or a constant
    9956              :                (and we have no way to tell gimple_simplify to not
    9957              :                consider those transforms in the first place).  */
    9958    144626062 :             else if (subcode == EQ_EXPR
    9959    144626062 :                      || subcode == NE_EXPR)
    9960              :               {
    9961      3422271 :                 tree lhs = gimple_assign_lhs (stmt);
    9962      3422271 :                 tree op0 = gimple_assign_rhs1 (stmt);
    9963      3422271 :                 if (useless_type_conversion_p (TREE_TYPE (lhs),
    9964      3422271 :                                                TREE_TYPE (op0)))
    9965              :                   {
    9966        24505 :                     tree op1 = (*valueize) (gimple_assign_rhs2 (stmt));
    9967        24505 :                     op0 = (*valueize) (op0);
    9968        24505 :                     if (TREE_CODE (op0) == INTEGER_CST)
    9969          518 :                       std::swap (op0, op1);
    9970        24505 :                     if (TREE_CODE (op1) == INTEGER_CST
    9971        24505 :                         && ((subcode == NE_EXPR && integer_zerop (op1))
    9972         2078 :                             || (subcode == EQ_EXPR && integer_onep (op1))))
    9973          276 :                       return op0;
    9974              :                   }
    9975              :               }
    9976              :             return NULL_TREE;
    9977              : 
    9978       770072 :           case GIMPLE_TERNARY_RHS:
    9979       770072 :             {
    9980              :               /* Handle ternary operators that can appear in GIMPLE form.  */
    9981       770072 :               tree op0 = (*valueize) (gimple_assign_rhs1 (stmt));
    9982       770072 :               tree op1 = (*valueize) (gimple_assign_rhs2 (stmt));
    9983       770072 :               tree op2 = (*valueize) (gimple_assign_rhs3 (stmt));
    9984       770072 :               return fold_ternary_loc (loc, subcode,
    9985       770072 :                                        TREE_TYPE (gimple_assign_lhs (stmt)),
    9986       770072 :                                        op0, op1, op2);
    9987              :             }
    9988              : 
    9989            0 :           default:
    9990            0 :             gcc_unreachable ();
    9991              :           }
    9992              :       }
    9993              : 
    9994     14916941 :     case GIMPLE_CALL:
    9995     14916941 :       {
    9996     14916941 :         tree fn;
    9997     14916941 :         gcall *call_stmt = as_a <gcall *> (stmt);
    9998              : 
    9999     14916941 :         if (gimple_call_internal_p (stmt))
   10000              :           {
   10001      1355406 :             enum tree_code subcode = ERROR_MARK;
   10002      1355406 :             switch (gimple_call_internal_fn (stmt))
   10003              :               {
   10004              :               case IFN_UBSAN_CHECK_ADD:
   10005              :                 subcode = PLUS_EXPR;
   10006              :                 break;
   10007         8239 :               case IFN_UBSAN_CHECK_SUB:
   10008         8239 :                 subcode = MINUS_EXPR;
   10009         8239 :                 break;
   10010         6847 :               case IFN_UBSAN_CHECK_MUL:
   10011         6847 :                 subcode = MULT_EXPR;
   10012         6847 :                 break;
   10013       147864 :               case IFN_BUILTIN_EXPECT:
   10014       147864 :                   {
   10015       147864 :                     tree arg0 = gimple_call_arg (stmt, 0);
   10016       147864 :                     tree op0 = (*valueize) (arg0);
   10017       147864 :                     if (TREE_CODE (op0) == INTEGER_CST)
   10018              :                       return op0;
   10019       115838 :                     return NULL_TREE;
   10020              :                   }
   10021              :               default:
   10022              :                 return NULL_TREE;
   10023              :               }
   10024        23269 :             tree arg0 = gimple_call_arg (stmt, 0);
   10025        23269 :             tree arg1 = gimple_call_arg (stmt, 1);
   10026        23269 :             tree op0 = (*valueize) (arg0);
   10027        23269 :             tree op1 = (*valueize) (arg1);
   10028              : 
   10029        23269 :             if (TREE_CODE (op0) != INTEGER_CST
   10030         2561 :                 || TREE_CODE (op1) != INTEGER_CST)
   10031              :               {
   10032        22747 :                 switch (subcode)
   10033              :                   {
   10034         6747 :                   case MULT_EXPR:
   10035              :                     /* x * 0 = 0 * x = 0 without overflow.  */
   10036         6747 :                     if (integer_zerop (op0) || integer_zerop (op1))
   10037           20 :                       return build_zero_cst (TREE_TYPE (arg0));
   10038              :                     break;
   10039         7897 :                   case MINUS_EXPR:
   10040              :                     /* y - y = 0 without overflow.  */
   10041         7897 :                     if (operand_equal_p (op0, op1, 0))
   10042            0 :                       return build_zero_cst (TREE_TYPE (arg0));
   10043              :                     break;
   10044              :                   default:
   10045              :                     break;
   10046              :                   }
   10047              :               }
   10048        23249 :             tree res
   10049        23249 :               = fold_binary_loc (loc, subcode, TREE_TYPE (arg0), op0, op1);
   10050        23249 :             if (res
   10051         3078 :                 && TREE_CODE (res) == INTEGER_CST
   10052        23771 :                 && !TREE_OVERFLOW (res))
   10053            0 :               return res;
   10054              :             return NULL_TREE;
   10055              :           }
   10056              : 
   10057     13561535 :         fn = (*valueize) (gimple_call_fn (stmt));
   10058     13561535 :         if (TREE_CODE (fn) == ADDR_EXPR
   10059     12920553 :             && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL
   10060     12920489 :             && fndecl_built_in_p (TREE_OPERAND (fn, 0))
   10061     19824116 :             && gimple_builtin_call_types_compatible_p (stmt,
   10062      6262581 :                                                        TREE_OPERAND (fn, 0)))
   10063              :           {
   10064      6157975 :             tree *args = XALLOCAVEC (tree, gimple_call_num_args (stmt));
   10065      6157975 :             tree retval;
   10066      6157975 :             unsigned i;
   10067     19016920 :             for (i = 0; i < gimple_call_num_args (stmt); ++i)
   10068     12858945 :               args[i] = (*valueize) (gimple_call_arg (stmt, i));
   10069      6157975 :             retval = fold_builtin_call_array (loc,
   10070              :                                          gimple_call_return_type (call_stmt),
   10071              :                                          fn, gimple_call_num_args (stmt), args);
   10072      6157975 :             if (retval)
   10073              :               {
   10074              :                 /* fold_call_expr wraps the result inside a NOP_EXPR.  */
   10075        61921 :                 STRIP_NOPS (retval);
   10076        61921 :                 retval = fold_convert (gimple_call_return_type (call_stmt),
   10077              :                                        retval);
   10078              :               }
   10079              :             return retval;
   10080              :           }
   10081              :         return NULL_TREE;
   10082              :       }
   10083              : 
   10084              :     default:
   10085              :       return NULL_TREE;
   10086              :     }
   10087              : }
   10088              : 
   10089              : /* Fold STMT to a constant using VALUEIZE to valueize SSA names.
   10090              :    Returns NULL_TREE if folding to a constant is not possible, otherwise
   10091              :    returns a constant according to is_gimple_min_invariant.  */
   10092              : 
   10093              : tree
   10094         4384 : gimple_fold_stmt_to_constant (gimple *stmt, tree (*valueize) (tree))
   10095              : {
   10096         4384 :   tree res = gimple_fold_stmt_to_constant_1 (stmt, valueize);
   10097         4384 :   if (res && is_gimple_min_invariant (res))
   10098         2369 :     return res;
   10099              :   return NULL_TREE;
   10100              : }
   10101              : 
   10102              : 
   10103              : /* The following set of functions are supposed to fold references using
   10104              :    their constant initializers.  */
   10105              : 
   10106              : /* See if we can find constructor defining value of BASE.
   10107              :    When we know the constructor with constant offset (such as
   10108              :    base is array[40] and we do know constructor of array), then
   10109              :    BIT_OFFSET is adjusted accordingly.
   10110              : 
   10111              :    As a special case, return error_mark_node when constructor
   10112              :    is not explicitly available, but it is known to be zero
   10113              :    such as 'static const int a;'.  */
   10114              : static tree
   10115    125558960 : get_base_constructor (tree base, poly_int64 *bit_offset,
   10116              :                       tree (*valueize)(tree))
   10117              : {
   10118    125663299 :   poly_int64 bit_offset2, size, max_size;
   10119    125663299 :   bool reverse;
   10120              : 
   10121    125663299 :   if (TREE_CODE (base) == MEM_REF)
   10122              :     {
   10123    133791876 :       poly_offset_int boff = *bit_offset + mem_ref_offset (base) * BITS_PER_UNIT;
   10124     66895938 :       if (!boff.to_shwi (bit_offset))
   10125     66525114 :         return NULL_TREE;
   10126              : 
   10127     66895586 :       if (valueize
   10128     66895586 :           && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME)
   10129     35043251 :         base = valueize (TREE_OPERAND (base, 0));
   10130     66895586 :       if (!base || TREE_CODE (base) != ADDR_EXPR)
   10131              :         return NULL_TREE;
   10132       370824 :       base = TREE_OPERAND (base, 0);
   10133              :     }
   10134     58767361 :   else if (valueize
   10135     30510837 :            && TREE_CODE (base) == SSA_NAME)
   10136            0 :     base = valueize (base);
   10137              : 
   10138              :   /* Get a CONSTRUCTOR.  If BASE is a VAR_DECL, get its
   10139              :      DECL_INITIAL.  If BASE is a nested reference into another
   10140              :      ARRAY_REF or COMPONENT_REF, make a recursive call to resolve
   10141              :      the inner reference.  */
   10142     59138185 :   switch (TREE_CODE (base))
   10143              :     {
   10144     51502038 :     case VAR_DECL:
   10145     51502038 :     case CONST_DECL:
   10146     51502038 :       {
   10147     51502038 :         tree init = ctor_for_folding (base);
   10148              : 
   10149              :         /* Our semantic is exact opposite of ctor_for_folding;
   10150              :            NULL means unknown, while error_mark_node is 0.  */
   10151     51502038 :         if (init == error_mark_node)
   10152              :           return NULL_TREE;
   10153      1300150 :         if (!init)
   10154         1180 :           return error_mark_node;
   10155              :         return init;
   10156              :       }
   10157              : 
   10158       104339 :     case VIEW_CONVERT_EXPR:
   10159       104339 :       return get_base_constructor (TREE_OPERAND (base, 0),
   10160       104339 :                                    bit_offset, valueize);
   10161              : 
   10162       361231 :     case ARRAY_REF:
   10163       361231 :     case COMPONENT_REF:
   10164       361231 :       base = get_ref_base_and_extent (base, &bit_offset2, &size, &max_size,
   10165              :                                       &reverse);
   10166       361231 :       if (!known_size_p (max_size) || maybe_ne (size, max_size))
   10167              :         return NULL_TREE;
   10168       300041 :       *bit_offset +=  bit_offset2;
   10169       300041 :       return get_base_constructor (base, bit_offset, valueize);
   10170              : 
   10171              :     case CONSTRUCTOR:
   10172              :       return base;
   10173              : 
   10174      7170577 :     default:
   10175      7170577 :       if (CONSTANT_CLASS_P (base))
   10176        94658 :         return base;
   10177              : 
   10178              :       return NULL_TREE;
   10179              :     }
   10180              : }
   10181              : 
   10182              : /* CTOR is a CONSTRUCTOR of an array or vector type.  Fold a reference of SIZE
   10183              :    bits to the memory at bit OFFSET.  If non-null, TYPE is the expected type of
   10184              :    the reference; otherwise the type of the referenced element is used instead.
   10185              :    When SIZE is zero, attempt to fold a reference to the entire element OFFSET
   10186              :    refers to.  Increment *SUBOFF by the bit offset of the accessed element.  */
   10187              : 
   10188              : static tree
   10189       693550 : fold_array_ctor_reference (tree type, tree ctor,
   10190              :                            unsigned HOST_WIDE_INT offset,
   10191              :                            unsigned HOST_WIDE_INT size,
   10192              :                            tree from_decl,
   10193              :                            unsigned HOST_WIDE_INT *suboff)
   10194              : {
   10195       693550 :   offset_int low_bound;
   10196       693550 :   offset_int elt_size;
   10197       693550 :   offset_int access_index;
   10198       693550 :   tree domain_type = NULL_TREE;
   10199       693550 :   HOST_WIDE_INT inner_offset;
   10200              : 
   10201              :   /* Compute low bound and elt size.  */
   10202       693550 :   if (TREE_CODE (TREE_TYPE (ctor)) == ARRAY_TYPE)
   10203       693550 :     domain_type = TYPE_DOMAIN (TREE_TYPE (ctor));
   10204       693550 :   if (domain_type && TYPE_MIN_VALUE (domain_type))
   10205              :     {
   10206              :       /* Static constructors for variably sized objects make no sense.  */
   10207       693550 :       if (TREE_CODE (TYPE_MIN_VALUE (domain_type)) != INTEGER_CST)
   10208              :         return NULL_TREE;
   10209       693550 :       low_bound = wi::to_offset (TYPE_MIN_VALUE (domain_type));
   10210              :     }
   10211              :   else
   10212            0 :     low_bound = 0;
   10213              :   /* Static constructors for variably sized objects make no sense.  */
   10214       693550 :   if (TREE_CODE (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ctor)))) != INTEGER_CST)
   10215              :     return NULL_TREE;
   10216       693550 :   elt_size = wi::to_offset (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ctor))));
   10217              : 
   10218              :   /* When TYPE is non-null, verify that it specifies a constant-sized
   10219              :      access of a multiple of the array element size.  Avoid division
   10220              :      by zero below when ELT_SIZE is zero, such as with the result of
   10221              :      an initializer for a zero-length array or an empty struct.  */
   10222       693586 :   if (elt_size == 0
   10223       693550 :       || (type
   10224       693514 :           && (!TYPE_SIZE_UNIT (type)
   10225       693514 :               || TREE_CODE (TYPE_SIZE_UNIT (type)) != INTEGER_CST)))
   10226              :     return NULL_TREE;
   10227              : 
   10228              :   /* Compute the array index we look for.  */
   10229       693514 :   access_index = wi::udiv_trunc (offset_int (offset / BITS_PER_UNIT),
   10230              :                                  elt_size);
   10231       693514 :   access_index += low_bound;
   10232              : 
   10233              :   /* And offset within the access.  */
   10234       693514 :   inner_offset = offset % (elt_size.to_uhwi () * BITS_PER_UNIT);
   10235              : 
   10236       693514 :   unsigned HOST_WIDE_INT elt_sz = elt_size.to_uhwi ();
   10237       693514 :   if (size > elt_sz * BITS_PER_UNIT)
   10238              :     {
   10239              :       /* native_encode_expr constraints.  */
   10240        21656 :       if (size > MAX_BITSIZE_MODE_ANY_MODE
   10241        17683 :           || size % BITS_PER_UNIT != 0
   10242        17683 :           || inner_offset % BITS_PER_UNIT != 0
   10243        17683 :           || elt_sz > MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT)
   10244              :         return NULL_TREE;
   10245              : 
   10246        17683 :       unsigned ctor_idx;
   10247        17683 :       tree val = get_array_ctor_element_at_index (ctor, access_index,
   10248              :                                                   &ctor_idx);
   10249        17707 :       if (!val && ctor_idx >= CONSTRUCTOR_NELTS (ctor))
   10250           23 :         return build_zero_cst (type);
   10251              : 
   10252              :       /* native-encode adjacent ctor elements.  */
   10253        17660 :       unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
   10254        17660 :       unsigned bufoff = 0;
   10255        17660 :       offset_int index = 0;
   10256        17660 :       offset_int max_index = access_index;
   10257        17660 :       constructor_elt *elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
   10258        17660 :       if (!val)
   10259            1 :         val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
   10260        17659 :       else if (!CONSTANT_CLASS_P (val))
   10261              :         return NULL_TREE;
   10262        17504 :       if (!elt->index)
   10263              :         ;
   10264        17369 :       else if (TREE_CODE (elt->index) == RANGE_EXPR)
   10265              :         {
   10266           20 :           index = wi::to_offset (TREE_OPERAND (elt->index, 0));
   10267           20 :           max_index = wi::to_offset (TREE_OPERAND (elt->index, 1));
   10268              :         }
   10269              :       else
   10270        17349 :         index = max_index = wi::to_offset (elt->index);
   10271        17504 :       index = wi::umax (index, access_index);
   10272       113558 :       do
   10273              :         {
   10274       113558 :           if (bufoff + elt_sz > sizeof (buf))
   10275            0 :             elt_sz = sizeof (buf) - bufoff;
   10276       113558 :           int len;
   10277       113558 :           if (TREE_CODE (val) == RAW_DATA_CST)
   10278              :             {
   10279           20 :               gcc_assert (inner_offset == 0);
   10280           20 :               if (!elt->index || TREE_CODE (elt->index) != INTEGER_CST)
   10281              :                 return NULL_TREE;
   10282           40 :               inner_offset = (access_index
   10283           20 :                               - wi::to_offset (elt->index)).to_uhwi ();
   10284           20 :               len = MIN (sizeof (buf) - bufoff,
   10285              :                          (unsigned) (RAW_DATA_LENGTH (val) - inner_offset));
   10286           20 :               memcpy (buf + bufoff, RAW_DATA_POINTER (val) + inner_offset,
   10287              :                       len);
   10288           20 :               access_index += len - 1;
   10289              :             }
   10290              :           else
   10291              :             {
   10292       227076 :               len = native_encode_expr (val, buf + bufoff, elt_sz,
   10293       113538 :                                         inner_offset / BITS_PER_UNIT);
   10294       113538 :               if (len != (int) elt_sz - inner_offset / BITS_PER_UNIT)
   10295              :                 return NULL_TREE;
   10296              :             }
   10297       113558 :           inner_offset = 0;
   10298       113558 :           bufoff += len;
   10299              : 
   10300       113558 :           access_index += 1;
   10301       113558 :           if (wi::cmpu (access_index, index) == 0)
   10302            2 :             val = elt->value;
   10303       113556 :           else if (wi::cmpu (access_index, max_index) > 0)
   10304              :             {
   10305       113396 :               ctor_idx++;
   10306       113396 :               if (ctor_idx >= CONSTRUCTOR_NELTS (ctor))
   10307              :                 {
   10308        15572 :                   val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
   10309        15572 :                   ++max_index;
   10310              :                 }
   10311              :               else
   10312              :                 {
   10313        97824 :                   elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
   10314        97824 :                   index = 0;
   10315        97824 :                   max_index = access_index;
   10316        97824 :                   if (!elt->index)
   10317              :                     ;
   10318        97016 :                   else if (TREE_CODE (elt->index) == RANGE_EXPR)
   10319              :                     {
   10320            0 :                       index = wi::to_offset (TREE_OPERAND (elt->index, 0));
   10321            0 :                       max_index = wi::to_offset (TREE_OPERAND (elt->index, 1));
   10322              :                     }
   10323              :                   else
   10324        97016 :                     index = max_index = wi::to_offset (elt->index);
   10325        97824 :                   index = wi::umax (index, access_index);
   10326        97824 :                   if (wi::cmpu (access_index, index) == 0)
   10327        97819 :                     val = elt->value;
   10328              :                   else
   10329            5 :                     val = build_zero_cst (TREE_TYPE (TREE_TYPE (ctor)));
   10330              :                 }
   10331              :             }
   10332              :         }
   10333       113558 :       while (bufoff < size / BITS_PER_UNIT);
   10334        17504 :       *suboff += size;
   10335        17504 :       return native_interpret_expr (type, buf, size / BITS_PER_UNIT);
   10336              :     }
   10337              : 
   10338       671858 :   unsigned ctor_idx;
   10339       671858 :   if (tree val = get_array_ctor_element_at_index (ctor, access_index,
   10340              :                                                   &ctor_idx))
   10341              :     {
   10342       670703 :       if (TREE_CODE (val) == RAW_DATA_CST)
   10343              :         {
   10344         2590 :           if (size != BITS_PER_UNIT || elt_sz != 1 || inner_offset != 0)
   10345              :             return NULL_TREE;
   10346         2582 :           constructor_elt *elt = CONSTRUCTOR_ELT (ctor, ctor_idx);
   10347         2582 :           if (elt->index == NULL_TREE || TREE_CODE (elt->index) != INTEGER_CST)
   10348              :             return NULL_TREE;
   10349         2582 :           unsigned o = (access_index - wi::to_offset (elt->index)).to_uhwi ();
   10350         2582 :           val = build_int_cst (TREE_TYPE (val), RAW_DATA_UCHAR_ELT (val, o));
   10351              :         }
   10352       670695 :       if (!size && TREE_CODE (val) != CONSTRUCTOR)
   10353              :         {
   10354              :           /* For the final reference to the entire accessed element
   10355              :              (SIZE is zero), reset INNER_OFFSET, disegard TYPE (which
   10356              :              may be null) in favor of the type of the element, and set
   10357              :              SIZE to the size of the accessed element.  */
   10358        23857 :           inner_offset = 0;
   10359        23857 :           type = TREE_TYPE (val);
   10360        23857 :           size = elt_sz * BITS_PER_UNIT;
   10361              :         }
   10362      1721312 :       else if (size && access_index < CONSTRUCTOR_NELTS (ctor) - 1
   10363       468511 :                && TREE_CODE (val) == CONSTRUCTOR
   10364        18116 :                && (elt_sz * BITS_PER_UNIT - inner_offset) < size)
   10365              :         /* If this isn't the last element in the CTOR and a CTOR itself
   10366              :            and it does not cover the whole object we are requesting give up
   10367              :            since we're not set up for combining from multiple CTORs.  */
   10368           26 :         return NULL_TREE;
   10369              : 
   10370       670669 :       *suboff += access_index.to_uhwi () * elt_sz * BITS_PER_UNIT;
   10371       670669 :       return fold_ctor_reference (type, val, inner_offset, size, from_decl,
   10372              :                                   suboff);
   10373              :     }
   10374              : 
   10375              :   /* Memory not explicitly mentioned in constructor is 0 (or
   10376              :      the reference is out of range).  */
   10377         1155 :   return type ? build_zero_cst (type) : NULL_TREE;
   10378              : }
   10379              : 
   10380              : /* CTOR is a CONSTRUCTOR of a record or union type.  Fold a reference of SIZE
   10381              :    bits to the memory at bit OFFSET.  If non-null, TYPE is the expected type of
   10382              :    the reference; otherwise the type of the referenced member is used instead.
   10383              :    When SIZE is zero, attempt to fold a reference to the entire member OFFSET
   10384              :    refers to.  Increment *SUBOFF by the bit offset of the accessed member.  */
   10385              : 
   10386              : static tree
   10387        85752 : fold_nonarray_ctor_reference (tree type, tree ctor,
   10388              :                               unsigned HOST_WIDE_INT offset,
   10389              :                               unsigned HOST_WIDE_INT size,
   10390              :                               tree from_decl,
   10391              :                               unsigned HOST_WIDE_INT *suboff)
   10392              : {
   10393        85752 :   unsigned HOST_WIDE_INT cnt;
   10394        85752 :   tree cfield, cval;
   10395              : 
   10396       129037 :   FOR_EACH_CONSTRUCTOR_ELT (CONSTRUCTOR_ELTS (ctor), cnt, cfield, cval)
   10397              :     {
   10398       120080 :       tree byte_offset = DECL_FIELD_OFFSET (cfield);
   10399       120080 :       tree field_offset = DECL_FIELD_BIT_OFFSET (cfield);
   10400       120080 :       tree field_size = DECL_SIZE (cfield);
   10401              : 
   10402       120080 :       if (!field_size)
   10403              :         {
   10404              :           /* Determine the size of the flexible array member from
   10405              :              the size of the initializer provided for it.  */
   10406          847 :           field_size = TYPE_SIZE (TREE_TYPE (cval));
   10407              :         }
   10408              : 
   10409              :       /* Variable sized objects in static constructors makes no sense,
   10410              :          but field_size can be NULL for flexible array members.  */
   10411       120080 :       gcc_assert (TREE_CODE (field_offset) == INTEGER_CST
   10412              :                   && TREE_CODE (byte_offset) == INTEGER_CST
   10413              :                   && (field_size != NULL_TREE
   10414              :                       ? TREE_CODE (field_size) == INTEGER_CST
   10415              :                       : TREE_CODE (TREE_TYPE (cfield)) == ARRAY_TYPE));
   10416              : 
   10417              :       /* Compute bit offset of the field.  */
   10418       120080 :       offset_int bitoffset
   10419       120080 :         = (wi::to_offset (field_offset)
   10420       120080 :            + (wi::to_offset (byte_offset) << LOG2_BITS_PER_UNIT));
   10421              :       /* Compute bit offset where the field ends.  */
   10422       120080 :       offset_int bitoffset_end;
   10423       120080 :       if (field_size != NULL_TREE)
   10424       120080 :         bitoffset_end = bitoffset + wi::to_offset (field_size);
   10425              :       else
   10426            0 :         bitoffset_end = 0;
   10427              : 
   10428              :       /* Compute the bit offset of the end of the desired access.
   10429              :          As a special case, if the size of the desired access is
   10430              :          zero, assume the access is to the entire field (and let
   10431              :          the caller make any necessary adjustments by storing
   10432              :          the actual bounds of the field in FIELDBOUNDS).  */
   10433       120080 :       offset_int access_end = offset_int (offset);
   10434       120080 :       if (size)
   10435        73455 :         access_end += size;
   10436              :       else
   10437        46625 :         access_end = bitoffset_end;
   10438              : 
   10439              :       /* Is there any overlap between the desired access at
   10440              :          [OFFSET, OFFSET+SIZE) and the offset of the field within
   10441              :          the object at [BITOFFSET, BITOFFSET_END)?  */
   10442       120080 :       if (wi::cmps (access_end, bitoffset) > 0
   10443       120080 :           && (field_size == NULL_TREE
   10444       117375 :               || wi::lts_p (offset, bitoffset_end)))
   10445              :         {
   10446        76795 :           *suboff += bitoffset.to_uhwi ();
   10447              : 
   10448        76795 :           if (!size && TREE_CODE (cval) != CONSTRUCTOR)
   10449              :             {
   10450              :               /* For the final reference to the entire accessed member
   10451              :                  (SIZE is zero), reset OFFSET, disegard TYPE (which may
   10452              :                  be null) in favor of the type of the member, and set
   10453              :                  SIZE to the size of the accessed member.  */
   10454        21470 :               offset = bitoffset.to_uhwi ();
   10455        21470 :               type = TREE_TYPE (cval);
   10456        21470 :               size = (bitoffset_end - bitoffset).to_uhwi ();
   10457              :             }
   10458              : 
   10459              :           /* We do have overlap.  Now see if the field is large enough
   10460              :              to cover the access.  Give up for accesses that extend
   10461              :              beyond the end of the object or that span multiple fields.  */
   10462        76795 :           if (wi::cmps (access_end, bitoffset_end) > 0)
   10463              :             return NULL_TREE;
   10464        76168 :           if (offset < bitoffset)
   10465              :             return NULL_TREE;
   10466              : 
   10467        76168 :           offset_int inner_offset = offset_int (offset) - bitoffset;
   10468              : 
   10469              :           /* Integral bit-fields are left-justified on big-endian targets, so
   10470              :              we must arrange for native_encode_int to start at their MSB.  */
   10471        76168 :           if (DECL_BIT_FIELD (cfield) && INTEGRAL_TYPE_P (TREE_TYPE (cfield)))
   10472              :             {
   10473        76168 :               if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
   10474              :                 return NULL_TREE;
   10475        76168 :               if (BYTES_BIG_ENDIAN)
   10476              :                 {
   10477              :                   tree ctype = TREE_TYPE (cfield);
   10478              :                   unsigned int encoding_size;
   10479              :                   if (TYPE_MODE (ctype) != BLKmode)
   10480              :                     encoding_size
   10481              :                       = GET_MODE_BITSIZE (SCALAR_INT_TYPE_MODE (ctype));
   10482              :                   else
   10483              :                     encoding_size = TREE_INT_CST_LOW (TYPE_SIZE (ctype));
   10484              :                   inner_offset += encoding_size - wi::to_offset (field_size);
   10485              :                 }
   10486              :             }
   10487              : 
   10488        76168 :           return fold_ctor_reference (type, cval,
   10489        76168 :                                       inner_offset.to_uhwi (), size,
   10490              :                                       from_decl, suboff);
   10491              :         }
   10492              :     }
   10493              : 
   10494         8957 :   if (!type)
   10495              :     return NULL_TREE;
   10496              : 
   10497         8957 :   return build_zero_cst (type);
   10498              : }
   10499              : 
   10500              : /* CTOR is a value initializing memory.  Fold a reference of TYPE and
   10501              :    bit size POLY_SIZE to the memory at bit POLY_OFFSET.  When POLY_SIZE
   10502              :    is zero, attempt to fold a reference to the entire subobject
   10503              :    which OFFSET refers to.  This is used when folding accesses to
   10504              :    string members of aggregates.  When non-null, set *SUBOFF to
   10505              :    the bit offset of the accessed subobject.  */
   10506              : 
   10507              : tree
   10508      1612960 : fold_ctor_reference (tree type, tree ctor, const poly_uint64 &poly_offset,
   10509              :                      const poly_uint64 &poly_size, tree from_decl,
   10510              :                      unsigned HOST_WIDE_INT *suboff /* = NULL */)
   10511              : {
   10512      1612960 :   tree ret;
   10513              : 
   10514              :   /* We found the field with exact match.  */
   10515      1612960 :   if (type
   10516      1612960 :       && useless_type_conversion_p (type, TREE_TYPE (ctor))
   10517      2306600 :       && known_eq (poly_offset, 0U))
   10518       692374 :     return canonicalize_constructor_val (unshare_expr (ctor), from_decl);
   10519              : 
   10520              :   /* The remaining optimizations need a constant size and offset.  */
   10521       920586 :   unsigned HOST_WIDE_INT size, offset;
   10522       920586 :   if (!poly_size.is_constant (&size) || !poly_offset.is_constant (&offset))
   10523              :     return NULL_TREE;
   10524              : 
   10525              :   /* We are at the end of walk, see if we can view convert the
   10526              :      result.  */
   10527       920586 :   if (!AGGREGATE_TYPE_P (TREE_TYPE (ctor)) && !offset
   10528              :       /* VIEW_CONVERT_EXPR is defined only for matching sizes.  */
   10529        23376 :       && known_eq (wi::to_poly_widest (TYPE_SIZE (type)), size)
   10530        23269 :       && known_eq (wi::to_poly_widest (TYPE_SIZE (TREE_TYPE (ctor))), size))
   10531              :     {
   10532        14474 :       ret = canonicalize_constructor_val (unshare_expr (ctor), from_decl);
   10533        14474 :       if (ret)
   10534              :         {
   10535        14474 :           ret = fold_unary (VIEW_CONVERT_EXPR, type, ret);
   10536        14474 :           if (ret)
   10537        14454 :             STRIP_USELESS_TYPE_CONVERSION (ret);
   10538              :         }
   10539              :       return ret;
   10540              :     }
   10541              : 
   10542              :   /* For constants and byte-aligned/sized reads, try to go through
   10543              :      native_encode/interpret.  */
   10544       906112 :   if (CONSTANT_CLASS_P (ctor)
   10545              :       && BITS_PER_UNIT == 8
   10546       118542 :       && offset % BITS_PER_UNIT == 0
   10547       118538 :       && offset / BITS_PER_UNIT <= INT_MAX
   10548       118506 :       && size % BITS_PER_UNIT == 0
   10549       118497 :       && size <= MAX_BITSIZE_MODE_ANY_MODE
   10550      1024311 :       && can_native_interpret_type_p (type))
   10551              :     {
   10552       102826 :       unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
   10553       205652 :       int len = native_encode_expr (ctor, buf, size / BITS_PER_UNIT,
   10554       102826 :                                     offset / BITS_PER_UNIT);
   10555       102826 :       if (len > 0)
   10556       102110 :         return native_interpret_expr (type, buf, len);
   10557              :     }
   10558              : 
   10559              :   /* For constructors, try first a recursive local processing, but in any case
   10560              :      this requires the native storage order.  */
   10561       804002 :   if (TREE_CODE (ctor) == CONSTRUCTOR
   10562       804002 :       && !(AGGREGATE_TYPE_P (TREE_TYPE (ctor))
   10563       779532 :            && TYPE_REVERSE_STORAGE_ORDER (TREE_TYPE (ctor))))
   10564              :     {
   10565       779302 :       unsigned HOST_WIDE_INT dummy = 0;
   10566       779302 :       if (!suboff)
   10567       641696 :         suboff = &dummy;
   10568              : 
   10569       779302 :       tree ret;
   10570       779302 :       if (TREE_CODE (TREE_TYPE (ctor)) == ARRAY_TYPE
   10571       779302 :           || TREE_CODE (TREE_TYPE (ctor)) == VECTOR_TYPE)
   10572       693550 :         ret = fold_array_ctor_reference (type, ctor, offset, size,
   10573              :                                          from_decl, suboff);
   10574              :       else
   10575        85752 :         ret = fold_nonarray_ctor_reference (type, ctor, offset, size,
   10576              :                                             from_decl, suboff);
   10577              : 
   10578              :       /* Otherwise fall back to native_encode_initializer.  This may be done
   10579              :          only from the outermost fold_ctor_reference call (because it itself
   10580              :          recurses into CONSTRUCTORs and doesn't update suboff).  */
   10581       779302 :       if (ret == NULL_TREE
   10582       206762 :           && suboff == &dummy
   10583              :           && BITS_PER_UNIT == 8
   10584       198403 :           && offset % BITS_PER_UNIT == 0
   10585       198401 :           && offset / BITS_PER_UNIT <= INT_MAX
   10586       198401 :           && size % BITS_PER_UNIT == 0
   10587       198392 :           && size <= MAX_BITSIZE_MODE_ANY_MODE
   10588       973721 :           && can_native_interpret_type_p (type))
   10589              :         {
   10590       180722 :           unsigned char buf[MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT];
   10591       361444 :           int len = native_encode_initializer (ctor, buf, size / BITS_PER_UNIT,
   10592       180722 :                                                offset / BITS_PER_UNIT);
   10593       180722 :           if (len > 0)
   10594         1302 :             return native_interpret_expr (type, buf, len);
   10595              :         }
   10596              : 
   10597              :       return ret;
   10598              :     }
   10599              : 
   10600              :   return NULL_TREE;
   10601              : }
   10602              : 
   10603              : /* Return the tree representing the element referenced by T if T is an
   10604              :    ARRAY_REF or COMPONENT_REF into constant aggregates valuezing SSA
   10605              :    names using VALUEIZE.  Return NULL_TREE otherwise.  */
   10606              : 
   10607              : tree
   10608    130838405 : fold_const_aggregate_ref_1 (tree t, tree (*valueize) (tree))
   10609              : {
   10610    130838405 :   tree ctor, idx, base;
   10611    130838405 :   poly_int64 offset, size, max_size;
   10612    130838405 :   tree tem;
   10613    130838405 :   bool reverse;
   10614              : 
   10615    130838405 :   if (TREE_THIS_VOLATILE (t))
   10616              :     return NULL_TREE;
   10617              : 
   10618    130517360 :   if (DECL_P (t))
   10619       301274 :     return get_symbol_constant_value (t);
   10620              : 
   10621    130216086 :   tem = fold_read_from_constant_string (t);
   10622    130216086 :   if (tem)
   10623              :     return tem;
   10624              : 
   10625    130214244 :   switch (TREE_CODE (t))
   10626              :     {
   10627     10838760 :     case ARRAY_REF:
   10628     10838760 :     case ARRAY_RANGE_REF:
   10629              :       /* Constant indexes are handled well by get_base_constructor.
   10630              :          Only special case variable offsets.
   10631              :          FIXME: This code can't handle nested references with variable indexes
   10632              :          (they will be handled only by iteration of ccp).  Perhaps we can bring
   10633              :          get_ref_base_and_extent here and make it use a valueize callback.  */
   10634     10838760 :       if (TREE_CODE (TREE_OPERAND (t, 1)) == SSA_NAME
   10635      6362492 :           && valueize
   10636      4278128 :           && (idx = (*valueize) (TREE_OPERAND (t, 1)))
   10637     15116888 :           && poly_int_tree_p (idx))
   10638              :         {
   10639      1705139 :           tree low_bound, unit_size;
   10640              : 
   10641              :           /* If the resulting bit-offset is constant, track it.  */
   10642      1705139 :           if ((low_bound = array_ref_low_bound (t),
   10643      1705139 :                poly_int_tree_p (low_bound))
   10644      1705139 :               && (unit_size = array_ref_element_size (t),
   10645      1705139 :                   tree_fits_uhwi_p (unit_size)))
   10646              :             {
   10647      1705139 :               poly_offset_int woffset
   10648      1705139 :                 = wi::sext (wi::to_poly_offset (idx)
   10649      3410278 :                             - wi::to_poly_offset (low_bound),
   10650      1705139 :                             TYPE_PRECISION (sizetype));
   10651      1705139 :               woffset *= tree_to_uhwi (unit_size);
   10652      1705139 :               woffset *= BITS_PER_UNIT;
   10653      1705139 :               if (woffset.to_shwi (&offset))
   10654              :                 {
   10655      1705020 :                   base = TREE_OPERAND (t, 0);
   10656      1705020 :                   ctor = get_base_constructor (base, &offset, valueize);
   10657              :                   /* Empty constructor.  Always fold to 0.  */
   10658      1705020 :                   if (ctor == error_mark_node)
   10659      1705020 :                     return build_zero_cst (TREE_TYPE (t));
   10660              :                   /* Out of bound array access.  Value is undefined,
   10661              :                      but don't fold.  */
   10662      1704943 :                   if (maybe_lt (offset, 0))
   10663              :                     return NULL_TREE;
   10664              :                   /* We cannot determine ctor.  */
   10665      1704524 :                   if (!ctor)
   10666              :                     return NULL_TREE;
   10667       180402 :                   return fold_ctor_reference (TREE_TYPE (t), ctor, offset,
   10668       180402 :                                               tree_to_uhwi (unit_size)
   10669       360804 :                                               * BITS_PER_UNIT,
   10670              :                                               base);
   10671              :                 }
   10672              :             }
   10673              :         }
   10674              :       /* Fallthru.  */
   10675              : 
   10676    123553899 :     case COMPONENT_REF:
   10677    123553899 :     case BIT_FIELD_REF:
   10678    123553899 :     case TARGET_MEM_REF:
   10679    123553899 :     case MEM_REF:
   10680    123553899 :       base = get_ref_base_and_extent (t, &offset, &size, &max_size, &reverse);
   10681    123553899 :       ctor = get_base_constructor (base, &offset, valueize);
   10682              : 
   10683              :       /* We cannot determine ctor.  */
   10684    123553899 :       if (!ctor)
   10685              :         return NULL_TREE;
   10686              :       /* Empty constructor.  Always fold to 0.  */
   10687      1214309 :       if (ctor == error_mark_node)
   10688         1103 :         return build_zero_cst (TREE_TYPE (t));
   10689              :       /* We do not know precise access.  */
   10690      1213206 :       if (!known_size_p (max_size) || maybe_ne (max_size, size))
   10691              :         return NULL_TREE;
   10692              :       /* Out of bound array access.  Value is undefined, but don't fold.  */
   10693       498643 :       if (maybe_lt (offset, 0))
   10694              :         return NULL_TREE;
   10695              :       /* Access with reverse storage order.  */
   10696       498262 :       if (reverse)
   10697              :         return NULL_TREE;
   10698              : 
   10699       498262 :       tem = fold_ctor_reference (TREE_TYPE (t), ctor, offset, size, base);
   10700       498262 :       if (tem)
   10701              :         return tem;
   10702              : 
   10703              :       /* For bit field reads try to read the representative and
   10704              :          adjust.  */
   10705       171361 :       if (TREE_CODE (t) == COMPONENT_REF
   10706         5149 :           && DECL_BIT_FIELD (TREE_OPERAND (t, 1))
   10707       171445 :           && DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1)))
   10708              :         {
   10709           84 :           HOST_WIDE_INT csize, coffset;
   10710           84 :           tree field = TREE_OPERAND (t, 1);
   10711           84 :           tree repr = DECL_BIT_FIELD_REPRESENTATIVE (field);
   10712          168 :           if (INTEGRAL_TYPE_P (TREE_TYPE (repr))
   10713           83 :               && size.is_constant (&csize)
   10714           83 :               && offset.is_constant (&coffset)
   10715           83 :               && (coffset % BITS_PER_UNIT != 0
   10716           81 :                   || csize % BITS_PER_UNIT != 0)
   10717           84 :               && BYTES_BIG_ENDIAN == WORDS_BIG_ENDIAN)
   10718              :             {
   10719           10 :               poly_int64 bitoffset;
   10720           10 :               poly_uint64 field_offset, repr_offset;
   10721           10 :               if (poly_int_tree_p (DECL_FIELD_OFFSET (field), &field_offset)
   10722           20 :                   && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
   10723           10 :                 bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
   10724              :               else
   10725              :                 bitoffset = 0;
   10726           10 :               bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
   10727           10 :                             - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
   10728           10 :               HOST_WIDE_INT bitoff;
   10729           10 :               int diff = (TYPE_PRECISION (TREE_TYPE (repr))
   10730           10 :                           - TYPE_PRECISION (TREE_TYPE (field)));
   10731           10 :               if (bitoffset.is_constant (&bitoff)
   10732           10 :                   && bitoff >= 0
   10733           10 :                   && bitoff <= diff)
   10734              :                 {
   10735           10 :                   offset -= bitoff;
   10736           10 :                   size = tree_to_uhwi (DECL_SIZE (repr));
   10737              : 
   10738           20 :                   tem = fold_ctor_reference (TREE_TYPE (repr), ctor, offset,
   10739           10 :                                              size, base);
   10740           10 :                   if (tem && TREE_CODE (tem) == INTEGER_CST)
   10741              :                     {
   10742           10 :                       if (!BYTES_BIG_ENDIAN)
   10743           10 :                         tem = wide_int_to_tree (TREE_TYPE (field),
   10744           10 :                                                 wi::lrshift (wi::to_wide (tem),
   10745              :                                                              bitoff));
   10746              :                       else
   10747              :                         tem = wide_int_to_tree (TREE_TYPE (field),
   10748              :                                                 wi::lrshift (wi::to_wide (tem),
   10749              :                                                              diff - bitoff));
   10750           10 :                       return tem;
   10751              :                     }
   10752              :                 }
   10753              :             }
   10754              :         }
   10755              :       break;
   10756              : 
   10757      1823471 :     case REALPART_EXPR:
   10758      1823471 :     case IMAGPART_EXPR:
   10759      1823471 :       {
   10760      1823471 :         tree c = fold_const_aggregate_ref_1 (TREE_OPERAND (t, 0), valueize);
   10761      1823471 :         if (c && TREE_CODE (c) == COMPLEX_CST)
   10762         2786 :           return fold_build1_loc (EXPR_LOCATION (t),
   10763         5572 :                                   TREE_CODE (t), TREE_TYPE (t), c);
   10764              :         break;
   10765              :       }
   10766              : 
   10767              :     default:
   10768              :       break;
   10769              :     }
   10770              : 
   10771              :   return NULL_TREE;
   10772              : }
   10773              : 
   10774              : tree
   10775     63108000 : fold_const_aggregate_ref (tree t)
   10776              : {
   10777     63108000 :   return fold_const_aggregate_ref_1 (t, NULL);
   10778              : }
   10779              : 
   10780              : /* Lookup virtual method with index TOKEN in a virtual table V
   10781              :    at OFFSET.
   10782              :    Set CAN_REFER if non-NULL to false if method
   10783              :    is not referable or if the virtual table is ill-formed (such as rewritten
   10784              :    by non-C++ produced symbol). Otherwise just return NULL in that calse.  */
   10785              : 
   10786              : tree
   10787       284772 : gimple_get_virt_method_for_vtable (HOST_WIDE_INT token,
   10788              :                                    tree v,
   10789              :                                    unsigned HOST_WIDE_INT offset,
   10790              :                                    bool *can_refer)
   10791              : {
   10792       284772 :   tree vtable = v, init, fn;
   10793       284772 :   unsigned HOST_WIDE_INT size;
   10794       284772 :   unsigned HOST_WIDE_INT elt_size, access_index;
   10795       284772 :   tree domain_type;
   10796              : 
   10797       284772 :   if (can_refer)
   10798       284772 :     *can_refer = true;
   10799              : 
   10800              :   /* First of all double check we have virtual table.  */
   10801       284772 :   if (!VAR_P (v) || !DECL_VIRTUAL_P (v))
   10802              :     {
   10803              :       /* Pass down that we lost track of the target.  */
   10804            0 :       if (can_refer)
   10805            0 :         *can_refer = false;
   10806              :       return NULL_TREE;
   10807              :     }
   10808              : 
   10809       284772 :   init = ctor_for_folding (v);
   10810              : 
   10811              :   /* The virtual tables should always be born with constructors
   10812              :      and we always should assume that they are available for
   10813              :      folding.  At the moment we do not stream them in all cases,
   10814              :      but it should never happen that ctor seem unreachable.  */
   10815       284772 :   gcc_assert (init);
   10816       284772 :   if (init == error_mark_node)
   10817              :     {
   10818              :       /* Pass down that we lost track of the target.  */
   10819          213 :       if (can_refer)
   10820          213 :         *can_refer = false;
   10821              :       return NULL_TREE;
   10822              :     }
   10823       284559 :   gcc_checking_assert (TREE_CODE (TREE_TYPE (v)) == ARRAY_TYPE);
   10824       284559 :   size = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (TREE_TYPE (v))));
   10825       284559 :   offset *= BITS_PER_UNIT;
   10826       284559 :   offset += token * size;
   10827              : 
   10828              :   /* Lookup the value in the constructor that is assumed to be array.
   10829              :      This is equivalent to
   10830              :      fn = fold_ctor_reference (TREE_TYPE (TREE_TYPE (v)), init,
   10831              :                                offset, size, NULL);
   10832              :      but in a constant time.  We expect that frontend produced a simple
   10833              :      array without indexed initializers.  */
   10834              : 
   10835       284559 :   gcc_checking_assert (TREE_CODE (TREE_TYPE (init)) == ARRAY_TYPE);
   10836       284559 :   domain_type = TYPE_DOMAIN (TREE_TYPE (init));
   10837       284559 :   gcc_checking_assert (integer_zerop (TYPE_MIN_VALUE (domain_type)));
   10838       284559 :   elt_size = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (init))));
   10839              : 
   10840       284559 :   access_index = offset / BITS_PER_UNIT / elt_size;
   10841       284559 :   gcc_checking_assert (offset % (elt_size * BITS_PER_UNIT) == 0);
   10842              : 
   10843              :   /* This code makes an assumption that there are no
   10844              :      indexed fields produced by C++ FE, so we can directly index the array.  */
   10845       284559 :   if (access_index < CONSTRUCTOR_NELTS (init))
   10846              :     {
   10847       284558 :       fn = CONSTRUCTOR_ELT (init, access_index)->value;
   10848       284558 :       gcc_checking_assert (!CONSTRUCTOR_ELT (init, access_index)->index);
   10849       284558 :       STRIP_NOPS (fn);
   10850              :     }
   10851              :   else
   10852              :     fn = NULL;
   10853              : 
   10854              :   /* For type inconsistent program we may end up looking up virtual method
   10855              :      in virtual table that does not contain TOKEN entries.  We may overrun
   10856              :      the virtual table and pick up a constant or RTTI info pointer.
   10857              :      In any case the call is undefined.  */
   10858       284558 :   if (!fn
   10859       284558 :       || (TREE_CODE (fn) != ADDR_EXPR && TREE_CODE (fn) != FDESC_EXPR)
   10860       563312 :       || TREE_CODE (TREE_OPERAND (fn, 0)) != FUNCTION_DECL)
   10861         5805 :     fn = builtin_decl_unreachable ();
   10862              :   else
   10863              :     {
   10864       278754 :       fn = TREE_OPERAND (fn, 0);
   10865              : 
   10866              :       /* When cgraph node is missing and function is not public, we cannot
   10867              :          devirtualize.  This can happen in WHOPR when the actual method
   10868              :          ends up in other partition, because we found devirtualization
   10869              :          possibility too late.  */
   10870       278754 :       if (!can_refer_decl_in_current_unit_p (fn, vtable))
   10871              :         {
   10872        37254 :           if (can_refer)
   10873              :             {
   10874        37254 :               *can_refer = false;
   10875        37254 :               return fn;
   10876              :             }
   10877              :           return NULL_TREE;
   10878              :         }
   10879              :     }
   10880              : 
   10881              :   /* Make sure we create a cgraph node for functions we'll reference.
   10882              :      They can be non-existent if the reference comes from an entry
   10883              :      of an external vtable for example.  */
   10884       247305 :   cgraph_node::get_create (fn);
   10885              : 
   10886       247305 :   return fn;
   10887              : }
   10888              : 
   10889              : /* Return a declaration of a function which an OBJ_TYPE_REF references. TOKEN
   10890              :    is integer form of OBJ_TYPE_REF_TOKEN of the reference expression.
   10891              :    KNOWN_BINFO carries the binfo describing the true type of
   10892              :    OBJ_TYPE_REF_OBJECT(REF).
   10893              :    Set CAN_REFER if non-NULL to false if method
   10894              :    is not referable or if the virtual table is ill-formed (such as rewritten
   10895              :    by non-C++ produced symbol). Otherwise just return NULL in that calse.  */
   10896              : 
   10897              : tree
   10898       275739 : gimple_get_virt_method_for_binfo (HOST_WIDE_INT token, tree known_binfo,
   10899              :                                   bool *can_refer)
   10900              : {
   10901       275739 :   unsigned HOST_WIDE_INT offset;
   10902       275739 :   tree v;
   10903              : 
   10904       275739 :   v = BINFO_VTABLE (known_binfo);
   10905              :   /* If there is no virtual methods table, leave the OBJ_TYPE_REF alone.  */
   10906       275739 :   if (!v)
   10907              :     return NULL_TREE;
   10908              : 
   10909       275739 :   if (!vtable_pointer_value_to_vtable (v, &v, &offset))
   10910              :     {
   10911            0 :       if (can_refer)
   10912            0 :         *can_refer = false;
   10913              :       return NULL_TREE;
   10914              :     }
   10915       275739 :   return gimple_get_virt_method_for_vtable (token, v, offset, can_refer);
   10916              : }
   10917              : 
   10918              : /* Given a pointer value T, return a simplified version of an
   10919              :    indirection through T, or NULL_TREE if no simplification is
   10920              :    possible.  Note that the resulting type may be different from
   10921              :    the type pointed to in the sense that it is still compatible
   10922              :    from the langhooks point of view. */
   10923              : 
   10924              : tree
   10925      2528439 : gimple_fold_indirect_ref (tree t)
   10926              : {
   10927      2528439 :   tree ptype = TREE_TYPE (t), type = TREE_TYPE (ptype);
   10928      2528439 :   tree sub = t;
   10929      2528439 :   tree subtype;
   10930              : 
   10931      2528439 :   STRIP_NOPS (sub);
   10932      2528439 :   subtype = TREE_TYPE (sub);
   10933      2528439 :   if (!POINTER_TYPE_P (subtype)
   10934      2528439 :       || TYPE_REF_CAN_ALIAS_ALL (ptype))
   10935              :     return NULL_TREE;
   10936              : 
   10937      2526730 :   if (TREE_CODE (sub) == ADDR_EXPR)
   10938              :     {
   10939        97610 :       tree op = TREE_OPERAND (sub, 0);
   10940        97610 :       tree optype = TREE_TYPE (op);
   10941              :       /* *&p => p */
   10942        97610 :       if (useless_type_conversion_p (type, optype))
   10943              :         return op;
   10944              : 
   10945              :       /* *(foo *)&fooarray => fooarray[0] */
   10946         1021 :       if (TREE_CODE (optype) == ARRAY_TYPE
   10947          335 :           && TREE_CODE (TYPE_SIZE (TREE_TYPE (optype))) == INTEGER_CST
   10948         1356 :           && useless_type_conversion_p (type, TREE_TYPE (optype)))
   10949              :        {
   10950           54 :          tree type_domain = TYPE_DOMAIN (optype);
   10951           54 :          tree min_val = size_zero_node;
   10952           54 :          if (type_domain && TYPE_MIN_VALUE (type_domain))
   10953           54 :            min_val = TYPE_MIN_VALUE (type_domain);
   10954           54 :          if (TREE_CODE (min_val) == INTEGER_CST)
   10955           54 :            return build4 (ARRAY_REF, type, op, min_val, NULL_TREE, NULL_TREE);
   10956              :        }
   10957              :       /* *(foo *)&complexfoo => __real__ complexfoo */
   10958          967 :       else if (TREE_CODE (optype) == COMPLEX_TYPE
   10959          967 :                && useless_type_conversion_p (type, TREE_TYPE (optype)))
   10960            4 :         return fold_build1 (REALPART_EXPR, type, op);
   10961              :       /* *(foo *)&vectorfoo => BIT_FIELD_REF<vectorfoo,...> */
   10962          963 :       else if (TREE_CODE (optype) == VECTOR_TYPE
   10963          963 :                && useless_type_conversion_p (type, TREE_TYPE (optype)))
   10964              :         {
   10965           26 :           tree part_width = TYPE_SIZE (type);
   10966           26 :           tree index = bitsize_int (0);
   10967           26 :           return fold_build3 (BIT_FIELD_REF, type, op, part_width, index);
   10968              :         }
   10969              :     }
   10970              : 
   10971              :   /* *(p + CST) -> ...  */
   10972      2430057 :   if (TREE_CODE (sub) == POINTER_PLUS_EXPR
   10973      2430057 :       && TREE_CODE (TREE_OPERAND (sub, 1)) == INTEGER_CST)
   10974              :     {
   10975        33935 :       tree addr = TREE_OPERAND (sub, 0);
   10976        33935 :       tree off = TREE_OPERAND (sub, 1);
   10977        33935 :       tree addrtype;
   10978              : 
   10979        33935 :       STRIP_NOPS (addr);
   10980        33935 :       addrtype = TREE_TYPE (addr);
   10981              : 
   10982              :       /* ((foo*)&vectorfoo)[1] -> BIT_FIELD_REF<vectorfoo,...> */
   10983        33935 :       if (TREE_CODE (addr) == ADDR_EXPR
   10984           92 :           && TREE_CODE (TREE_TYPE (addrtype)) == VECTOR_TYPE
   10985           39 :           && useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (addrtype)))
   10986        33963 :           && tree_fits_uhwi_p (off))
   10987              :         {
   10988           28 :           unsigned HOST_WIDE_INT offset = tree_to_uhwi (off);
   10989           28 :           tree part_width = TYPE_SIZE (type);
   10990           28 :           unsigned HOST_WIDE_INT part_widthi
   10991           28 :             = tree_to_shwi (part_width) / BITS_PER_UNIT;
   10992           28 :           unsigned HOST_WIDE_INT indexi = offset * BITS_PER_UNIT;
   10993           28 :           tree index = bitsize_int (indexi);
   10994           28 :           if (known_lt (offset / part_widthi,
   10995              :                         TYPE_VECTOR_SUBPARTS (TREE_TYPE (addrtype))))
   10996           28 :             return fold_build3 (BIT_FIELD_REF, type, TREE_OPERAND (addr, 0),
   10997              :                                 part_width, index);
   10998              :         }
   10999              : 
   11000              :       /* ((foo*)&complexfoo)[1] -> __imag__ complexfoo */
   11001        33907 :       if (TREE_CODE (addr) == ADDR_EXPR
   11002           64 :           && TREE_CODE (TREE_TYPE (addrtype)) == COMPLEX_TYPE
   11003        33908 :           && useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (addrtype))))
   11004              :         {
   11005            1 :           tree size = TYPE_SIZE_UNIT (type);
   11006            1 :           if (tree_int_cst_equal (size, off))
   11007            1 :             return fold_build1 (IMAGPART_EXPR, type, TREE_OPERAND (addr, 0));
   11008              :         }
   11009              : 
   11010              :       /* *(p + CST) -> MEM_REF <p, CST>.  */
   11011        33906 :       if (TREE_CODE (addr) != ADDR_EXPR
   11012        33906 :           || DECL_P (TREE_OPERAND (addr, 0)))
   11013        33888 :         return fold_build2 (MEM_REF, type,
   11014              :                             addr,
   11015              :                             wide_int_to_tree (ptype, wi::to_wide (off)));
   11016              :     }
   11017              : 
   11018              :   /* *(foo *)fooarrptr => (*fooarrptr)[0] */
   11019      2396140 :   if (TREE_CODE (TREE_TYPE (subtype)) == ARRAY_TYPE
   11020         2546 :       && TREE_CODE (TYPE_SIZE (TREE_TYPE (TREE_TYPE (subtype)))) == INTEGER_CST
   11021      2398656 :       && useless_type_conversion_p (type, TREE_TYPE (TREE_TYPE (subtype))))
   11022              :     {
   11023            1 :       tree type_domain;
   11024            1 :       tree min_val = size_zero_node;
   11025            1 :       tree osub = sub;
   11026            1 :       sub = gimple_fold_indirect_ref (sub);
   11027            1 :       if (! sub)
   11028            1 :         sub = build1 (INDIRECT_REF, TREE_TYPE (subtype), osub);
   11029            1 :       type_domain = TYPE_DOMAIN (TREE_TYPE (sub));
   11030            1 :       if (type_domain && TYPE_MIN_VALUE (type_domain))
   11031            1 :         min_val = TYPE_MIN_VALUE (type_domain);
   11032            1 :       if (TREE_CODE (min_val) == INTEGER_CST)
   11033            1 :         return build4 (ARRAY_REF, type, sub, min_val, NULL_TREE, NULL_TREE);
   11034              :     }
   11035              : 
   11036              :   return NULL_TREE;
   11037              : }
   11038              : 
   11039              : /* Return true if CODE is an operation that when operating on signed
   11040              :    integer types involves undefined behavior on overflow and the
   11041              :    operation can be expressed with unsigned arithmetic.  */
   11042              : 
   11043              : bool
   11044       619802 : arith_code_with_undefined_signed_overflow (tree_code code)
   11045              : {
   11046       619802 :   switch (code)
   11047              :     {
   11048              :     case ABS_EXPR:
   11049              :     case PLUS_EXPR:
   11050              :     case MINUS_EXPR:
   11051              :     case MULT_EXPR:
   11052              :     case NEGATE_EXPR:
   11053              :     case POINTER_PLUS_EXPR:
   11054              :       return true;
   11055       275604 :     default:
   11056       275604 :       return false;
   11057              :     }
   11058              : }
   11059              : 
   11060              : /* Return true if STMT has an operation that operates on a signed
   11061              :    integer types involves undefined behavior on overflow and the
   11062              :    operation can be expressed with unsigned arithmetic.
   11063              :    Also returns true if STMT is a VCE that needs to be rewritten
   11064              :    if moved to be executed unconditionally.
   11065              :    SHIFT_COUNT additionally asks for a shift or rotate whose count may be
   11066              :    out of range; rewrite_to_defined_unconditional reduces such a count.  */
   11067              : 
   11068              : bool
   11069      1625598 : gimple_needing_rewrite_undefined (gimple *stmt, bool shift_count)
   11070              : {
   11071      1625598 :   if (!is_gimple_assign (stmt))
   11072              :     return false;
   11073      1386965 :   tree lhs = gimple_assign_lhs (stmt);
   11074      1386965 :   if (!lhs)
   11075              :     return false;
   11076      1386965 :   tree lhs_type = TREE_TYPE (lhs);
   11077      1386965 :   if (!INTEGRAL_TYPE_P (lhs_type)
   11078       159151 :       && !POINTER_TYPE_P (lhs_type))
   11079              :     return false;
   11080      1339172 :   tree rhs = gimple_assign_rhs1 (stmt);
   11081              :   /* Boolean loads need special handling as they are treated as a full MODE load
   11082              :      and don't mask off the bits for the precision.  */
   11083      1339172 :   if (gimple_assign_load_p (stmt)
   11084              :       /* Booleans are the integral type which has this non-masking issue. */
   11085       166886 :       && TREE_CODE (lhs_type) == BOOLEAN_TYPE
   11086              :       /* Only non mode precision booleans are need the masking.  */
   11087          587 :       && !type_has_mode_precision_p (lhs_type)
   11088              :       /* BFR should be the correct thing and just grab the precision.  */
   11089          587 :       && TREE_CODE (rhs) != BIT_FIELD_REF
   11090              :       /* Bit-fields loads don't need a rewrite as the masking
   11091              :          happens for them.  */
   11092      1339759 :       && (TREE_CODE (rhs) != COMPONENT_REF
   11093          199 :           || !DECL_BIT_FIELD (TREE_OPERAND (rhs, 1))))
   11094              :     return true;
   11095              :   /* VCE from integral types to a integral types but with
   11096              :      a smaller precision need to be changed into casts
   11097              :      to be well defined. */
   11098      1338585 :   if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR
   11099          202 :       && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (rhs, 0)))
   11100          173 :       && is_gimple_val (TREE_OPERAND (rhs, 0))
   11101      1338758 :       && TYPE_PRECISION (lhs_type)
   11102          173 :           < TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (rhs, 0))))
   11103              :     return true;
   11104              :   /* A shift or rotate by a count that may be out of range; the count can
   11105              :      be reduced into range.  */
   11106      1338418 :   enum tree_code code = gimple_assign_rhs_code (stmt);
   11107      1338418 :   if (shift_count
   11108       799376 :       && (code == LSHIFT_EXPR
   11109              :           || code == RSHIFT_EXPR
   11110              :           || code == LROTATE_EXPR
   11111       799376 :           || code == RROTATE_EXPR)
   11112        59916 :       && INTEGRAL_TYPE_P (lhs_type)
   11113      1398334 :       && (TREE_CODE (gimple_assign_rhs2 (stmt)) != INTEGER_CST
   11114        59916 :           || wi::geu_p (wi::to_wide (gimple_assign_rhs2 (stmt)),
   11115        16544 :                         TYPE_PRECISION (lhs_type))))
   11116              :     return true;
   11117      1295042 :   if (!TYPE_OVERFLOW_UNDEFINED (lhs_type))
   11118              :     return false;
   11119       495254 :   if (!arith_code_with_undefined_signed_overflow (code))
   11120              :     return false;
   11121              :   return true;
   11122              : }
   11123              : 
   11124              : /* Rewrite STMT, an assignment with a signed integer or pointer arithmetic
   11125              :    operation that can be transformed to unsigned arithmetic by converting
   11126              :    its operand, carrying out the operation in the corresponding unsigned
   11127              :    type and converting the result back to the original type.
   11128              : 
   11129              :    If IN_PLACE is true, *GSI points to STMT, adjust the stmt in place and
   11130              :    return NULL.
   11131              :    Otherwise returns a sequence of statements that replace STMT and also
   11132              :    contain a modified form of STMT itself.  */
   11133              : 
   11134              : static gimple_seq
   11135        69683 : rewrite_to_defined_unconditional (gimple_stmt_iterator *gsi, gimple *stmt,
   11136              :                                   bool in_place)
   11137              : {
   11138        69683 :   gcc_assert (gimple_needing_rewrite_undefined (stmt, true));
   11139        69683 :   if (dump_file && (dump_flags & TDF_DETAILS))
   11140              :     {
   11141           59 :       fprintf (dump_file, "rewriting stmt for being unconditional defined");
   11142           59 :       print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
   11143              :     }
   11144        69683 :   gimple_seq stmts = NULL;
   11145        69683 :   tree lhs = gimple_assign_lhs (stmt);
   11146              : 
   11147              :   /* Boolean loads need to be rewritten to be a load from the same mode
   11148              :      and then a cast to the other type so the other bits are masked off
   11149              :      correctly since the load was done conditionally. It is similar to the VCE
   11150              :      case below.  */
   11151        69683 :   if (gimple_assign_load_p (stmt)
   11152        69683 :       && TREE_CODE (TREE_TYPE (lhs)) == BOOLEAN_TYPE)
   11153              :     {
   11154          135 :       tree rhs = gimple_assign_rhs1 (stmt);
   11155              : 
   11156              :       /* Double check that gimple_needing_rewrite_undefined was called.  */
   11157              :       /* Bit-fields loads will do the masking so don't need the rewriting.  */
   11158          135 :       gcc_assert (TREE_CODE (rhs) != COMPONENT_REF
   11159              :                   || !DECL_BIT_FIELD (TREE_OPERAND (rhs, 1)));
   11160              :       /* BFR is like a bit field load and will do the correct thing.  */
   11161          135 :       gcc_assert (TREE_CODE (lhs) != BIT_FIELD_REF);
   11162              :       /* Complex boolean types are not valid so REAL/IMAG part will
   11163              :          never show up. */
   11164          135 :       gcc_assert (TREE_CODE (rhs) != REALPART_EXPR
   11165              :                   && TREE_CODE (lhs) != IMAGPART_EXPR);
   11166              : 
   11167          135 :       auto bits = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (rhs)));
   11168          135 :       tree new_type = build_nonstandard_integer_type (bits, true);
   11169          135 :       location_t loc = gimple_location (stmt);
   11170          135 :       tree mem_ref = fold_build1_loc (loc, VIEW_CONVERT_EXPR, new_type, rhs);
   11171              :       /* Replace the original load with a new load and a new lhs. */
   11172          135 :       tree new_lhs = make_ssa_name (new_type);
   11173          135 :       gimple_assign_set_rhs1 (stmt, mem_ref);
   11174          135 :       gimple_assign_set_lhs (stmt, new_lhs);
   11175              : 
   11176          135 :       if (in_place)
   11177           49 :           update_stmt (stmt);
   11178              :       else
   11179              :         {
   11180           86 :           gimple_set_modified (stmt, true);
   11181           86 :           gimple_seq_add_stmt (&stmts, stmt);
   11182              :         }
   11183              : 
   11184              :       /* Build the conversion statement.  */
   11185          135 :       gimple *cvt = gimple_build_assign (lhs, NOP_EXPR, new_lhs);
   11186          135 :       if (in_place)
   11187              :         {
   11188           49 :           gsi_insert_after (gsi, cvt, GSI_SAME_STMT);
   11189           49 :           update_stmt (stmt);
   11190              :         }
   11191              :       else
   11192           86 :         gimple_seq_add_stmt (&stmts, cvt);
   11193          135 :       return stmts;
   11194              :     }
   11195              : 
   11196              :   /* VCE from integral types to another integral types but with
   11197              :      smaller precisions need to be changed into casts
   11198              :      to be well defined. */
   11199        69548 :   if (gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
   11200              :     {
   11201           60 :       tree rhs = gimple_assign_rhs1 (stmt);
   11202           60 :       tree new_rhs = TREE_OPERAND (rhs, 0);
   11203           60 :       gcc_assert (TYPE_PRECISION (TREE_TYPE (rhs))
   11204              :                   < TYPE_PRECISION (TREE_TYPE (new_rhs)));
   11205           60 :       gcc_assert (is_gimple_val (new_rhs));
   11206           60 :       gimple_assign_set_rhs_code (stmt, NOP_EXPR);
   11207           60 :       gimple_assign_set_rhs1 (stmt, new_rhs);
   11208           60 :       if (in_place)
   11209           51 :           update_stmt (stmt);
   11210              :       else
   11211              :         {
   11212            9 :           gimple_set_modified (stmt, true);
   11213            9 :           gimple_seq_add_stmt (&stmts, stmt);
   11214              :         }
   11215           60 :       return stmts;
   11216              :     }
   11217              :   /* Reduce a shift or rotate count into range.  The mask keeps the count's
   11218              :      own type where that holds PREC - 1, so that a SHIFT_COUNT_TRUNCATED
   11219              :      target folds it away; MIN needs an unsigned count so a negative one
   11220              :      clamps.  */
   11221        69488 :   enum tree_code code = gimple_assign_rhs_code (stmt);
   11222        69488 :   if (code == LSHIFT_EXPR
   11223              :       || code == RSHIFT_EXPR
   11224              :       || code == LROTATE_EXPR
   11225        69488 :       || code == RROTATE_EXPR)
   11226              :     {
   11227          141 :       unsigned prec = TYPE_PRECISION (TREE_TYPE (lhs));
   11228          141 :       bool pow2 = pow2p_hwi (prec);
   11229          141 :       tree count = gimple_assign_rhs2 (stmt);
   11230          141 :       tree type = TREE_TYPE (count);
   11231          281 :       if (!pow2 || wi::ltu_p (wi::max_value (type), prec - 1))
   11232            1 :         type = unsigned_type_node;
   11233          141 :       count = gimple_convert (&stmts, type, count);
   11234          142 :       count = gimple_build (&stmts, pow2 ? BIT_AND_EXPR : MIN_EXPR, type,
   11235          141 :                             count, build_int_cst (type, prec - 1));
   11236          141 :       gimple_assign_set_rhs2 (stmt, count);
   11237          141 :       gimple_set_modified (stmt, true);
   11238          141 :       if (in_place)
   11239              :         {
   11240            0 :           gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
   11241            0 :           update_stmt (stmt);
   11242            0 :           return NULL;
   11243              :         }
   11244          141 :       gimple_seq_add_stmt (&stmts, stmt);
   11245          141 :       return stmts;
   11246              :     }
   11247        69347 :   tree type = unsigned_type_for (TREE_TYPE (lhs));
   11248        69347 :   if (gimple_assign_rhs_code (stmt) == ABS_EXPR)
   11249           21 :     gimple_assign_set_rhs_code (stmt, ABSU_EXPR);
   11250              :   else
   11251       207448 :     for (unsigned i = 1; i < gimple_num_ops (stmt); ++i)
   11252              :       {
   11253       138122 :         tree op = gimple_op (stmt, i);
   11254       138122 :         op = gimple_convert (&stmts, type, op);
   11255       138122 :         gimple_set_op (stmt, i, op);
   11256              :       }
   11257        69347 :   gimple_assign_set_lhs (stmt, make_ssa_name (type, stmt));
   11258        69347 :   if (gimple_assign_rhs_code (stmt) == POINTER_PLUS_EXPR)
   11259         9525 :     gimple_assign_set_rhs_code (stmt, PLUS_EXPR);
   11260        69347 :   gimple_set_modified (stmt, true);
   11261        69347 :   if (in_place)
   11262              :     {
   11263        49140 :       if (stmts)
   11264        48878 :         gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
   11265        49140 :       stmts = NULL;
   11266              :     }
   11267              :   else
   11268        20207 :     gimple_seq_add_stmt (&stmts, stmt);
   11269        69347 :   gimple *cvt = gimple_build_assign (lhs, NOP_EXPR, gimple_assign_lhs (stmt));
   11270        69347 :   if (in_place)
   11271              :     {
   11272        49140 :       gsi_insert_after (gsi, cvt, GSI_SAME_STMT);
   11273        49140 :       update_stmt (stmt);
   11274              :     }
   11275              :   else
   11276        20207 :     gimple_seq_add_stmt (&stmts, cvt);
   11277              : 
   11278        69347 :   return stmts;
   11279              : }
   11280              : 
   11281              : void
   11282        49240 : rewrite_to_defined_unconditional (gimple_stmt_iterator *gsi)
   11283              : {
   11284        49240 :   rewrite_to_defined_unconditional (gsi, gsi_stmt (*gsi), true);
   11285        49240 : }
   11286              : 
   11287              : gimple_seq
   11288        20443 : rewrite_to_defined_unconditional (gimple *stmt)
   11289              : {
   11290        20443 :   return rewrite_to_defined_unconditional (nullptr, stmt, false);
   11291              : }
   11292              : 
   11293              : /* The valueization hook we use for the gimple_build API simplification.
   11294              :    This makes us match fold_buildN behavior by only combining with
   11295              :    statements in the sequence(s) we are currently building.  */
   11296              : 
   11297              : static tree
   11298     22403452 : gimple_build_valueize (tree op)
   11299              : {
   11300     22403452 :   if (gimple_bb (SSA_NAME_DEF_STMT (op)) == NULL)
   11301      4986129 :     return op;
   11302              :   return NULL_TREE;
   11303              : }
   11304              : 
   11305              : /* Helper for gimple_build to perform the final insertion of stmts on SEQ.  */
   11306              : 
   11307              : static inline void
   11308      1409306 : gimple_build_insert_seq (gimple_stmt_iterator *gsi,
   11309              :                          bool before, gsi_iterator_update update,
   11310              :                          gimple_seq seq)
   11311              : {
   11312      1409306 :   if (before)
   11313              :     {
   11314        97791 :       if (gsi->bb)
   11315        97791 :         gsi_insert_seq_before (gsi, seq, update);
   11316              :       else
   11317            0 :         gsi_insert_seq_before_without_update (gsi, seq, update);
   11318              :     }
   11319              :   else
   11320              :     {
   11321      1311515 :       if (gsi->bb)
   11322          199 :         gsi_insert_seq_after (gsi, seq, update);
   11323              :       else
   11324      1311316 :         gsi_insert_seq_after_without_update (gsi, seq, update);
   11325              :     }
   11326      1409306 : }
   11327              : 
   11328              : /* Build the expression CODE OP0 of type TYPE with location LOC,
   11329              :    simplifying it first if possible.  Returns the built
   11330              :    expression value and inserts statements possibly defining it
   11331              :    before GSI if BEFORE is true or after GSI if false and advance
   11332              :    the iterator accordingly.
   11333              :    If gsi refers to a basic block simplifying is allowed to look
   11334              :    at all SSA defs while when it does not it is restricted to
   11335              :    SSA defs that are not associated with a basic block yet,
   11336              :    indicating they belong to the currently building sequence.  */
   11337              : 
   11338              : tree
   11339       366111 : gimple_build (gimple_stmt_iterator *gsi,
   11340              :               bool before, gsi_iterator_update update,
   11341              :               location_t loc, enum tree_code code, tree type, tree op0)
   11342              : {
   11343       366111 :   gimple_seq seq = NULL;
   11344       366111 :   tree res
   11345       366111 :     = gimple_simplify (code, type, op0, &seq,
   11346       366111 :                        gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
   11347       366111 :   if (!res)
   11348              :     {
   11349       313974 :       res = make_ssa_name (type);
   11350       313974 :       gimple *stmt;
   11351       313974 :       if (code == REALPART_EXPR
   11352              :           || code == IMAGPART_EXPR
   11353       313974 :           || code == VIEW_CONVERT_EXPR)
   11354        11875 :         stmt = gimple_build_assign (res, code, build1 (code, type, op0));
   11355              :       else
   11356       302099 :         stmt = gimple_build_assign (res, code, op0);
   11357       313974 :       gimple_set_location (stmt, loc);
   11358       313974 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11359              :     }
   11360       366111 :   gimple_build_insert_seq (gsi, before, update, seq);
   11361       366111 :   return res;
   11362              : }
   11363              : 
   11364              : /* Build the expression OP0 CODE OP1 of type TYPE with location LOC,
   11365              :    simplifying it first if possible.  Returns the built
   11366              :    expression value inserting any new statements at GSI honoring BEFORE
   11367              :    and UPDATE.  */
   11368              : 
   11369              : tree
   11370       829183 : gimple_build (gimple_stmt_iterator *gsi,
   11371              :               bool before, gsi_iterator_update update,
   11372              :               location_t loc, enum tree_code code, tree type,
   11373              :               tree op0, tree op1)
   11374              : {
   11375       829183 :   gimple_seq seq = NULL;
   11376       829183 :   tree res
   11377       829183 :     = gimple_simplify (code, type, op0, op1, &seq,
   11378       829183 :                        gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
   11379       829183 :   if (!res)
   11380              :     {
   11381       732375 :       res = make_ssa_name (type);
   11382       732375 :       gimple *stmt = gimple_build_assign (res, code, op0, op1);
   11383       732375 :       gimple_set_location (stmt, loc);
   11384       732375 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11385              :     }
   11386       829183 :   gimple_build_insert_seq (gsi, before, update, seq);
   11387       829183 :   return res;
   11388              : }
   11389              : 
   11390              : /* Build the expression (CODE OP0 OP1 OP2) of type TYPE with location LOC,
   11391              :    simplifying it first if possible.  Returns the built
   11392              :    expression value inserting any new statements at GSI honoring BEFORE
   11393              :    and UPDATE.  */
   11394              : 
   11395              : tree
   11396        48106 : gimple_build (gimple_stmt_iterator *gsi,
   11397              :               bool before, gsi_iterator_update update,
   11398              :               location_t loc, enum tree_code code, tree type,
   11399              :               tree op0, tree op1, tree op2)
   11400              : {
   11401              : 
   11402        48106 :   gimple_seq seq = NULL;
   11403        48106 :   tree res
   11404        48106 :     = gimple_simplify (code, type, op0, op1, op2, &seq,
   11405        48106 :                        gsi->bb ? follow_all_ssa_edges : gimple_build_valueize);
   11406        48106 :   if (!res)
   11407              :     {
   11408        34326 :       res = make_ssa_name (type);
   11409        34326 :       gimple *stmt;
   11410        34326 :       if (code == BIT_FIELD_REF)
   11411        26874 :         stmt = gimple_build_assign (res, code,
   11412              :                                     build3 (code, type, op0, op1, op2));
   11413              :       else
   11414         7452 :         stmt = gimple_build_assign (res, code, op0, op1, op2);
   11415        34326 :       gimple_set_location (stmt, loc);
   11416        34326 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11417              :     }
   11418        48106 :   gimple_build_insert_seq (gsi, before, update, seq);
   11419        48106 :   return res;
   11420              : }
   11421              : 
   11422              : /* Build the call FN () with a result of type TYPE (or no result if TYPE is
   11423              :    void) with a location LOC.  Returns the built expression value (or NULL_TREE
   11424              :    if TYPE is void) inserting any new statements at GSI honoring BEFORE
   11425              :    and UPDATE.  */
   11426              : 
   11427              : tree
   11428            0 : gimple_build (gimple_stmt_iterator *gsi,
   11429              :               bool before, gsi_iterator_update update,
   11430              :               location_t loc, combined_fn fn, tree type)
   11431              : {
   11432            0 :   tree res = NULL_TREE;
   11433            0 :   gimple_seq seq = NULL;
   11434            0 :   gcall *stmt;
   11435            0 :   if (internal_fn_p (fn))
   11436            0 :     stmt = gimple_build_call_internal (as_internal_fn (fn), 0);
   11437              :   else
   11438              :     {
   11439            0 :       tree decl = builtin_decl_implicit (as_builtin_fn (fn));
   11440            0 :       stmt = gimple_build_call (decl, 0);
   11441              :     }
   11442            0 :   if (!VOID_TYPE_P (type))
   11443              :     {
   11444            0 :       res = make_ssa_name (type);
   11445            0 :       gimple_call_set_lhs (stmt, res);
   11446              :     }
   11447            0 :   gimple_set_location (stmt, loc);
   11448            0 :   gimple_seq_add_stmt_without_update (&seq, stmt);
   11449            0 :   gimple_build_insert_seq (gsi, before, update, seq);
   11450            0 :   return res;
   11451              : }
   11452              : 
   11453              : /* Build the call FN (ARG0) with a result of type TYPE
   11454              :    (or no result if TYPE is void) with location LOC,
   11455              :    simplifying it first if possible.  Returns the built
   11456              :    expression value (or NULL_TREE if TYPE is void) inserting any new
   11457              :    statements at GSI honoring BEFORE and UPDATE.  */
   11458              : 
   11459              : tree
   11460        21812 : gimple_build (gimple_stmt_iterator *gsi,
   11461              :               bool before, gsi_iterator_update update,
   11462              :               location_t loc, combined_fn fn,
   11463              :               tree type, tree arg0)
   11464              : {
   11465        21812 :   gimple_seq seq = NULL;
   11466        21812 :   tree res = gimple_simplify (fn, type, arg0, &seq, gimple_build_valueize);
   11467        21812 :   if (!res)
   11468              :     {
   11469        21812 :       gcall *stmt;
   11470        21812 :       if (internal_fn_p (fn))
   11471        21440 :         stmt = gimple_build_call_internal (as_internal_fn (fn), 1, arg0);
   11472              :       else
   11473              :         {
   11474          372 :           tree decl = builtin_decl_implicit (as_builtin_fn (fn));
   11475          372 :           stmt = gimple_build_call (decl, 1, arg0);
   11476              :         }
   11477        21812 :       if (!VOID_TYPE_P (type))
   11478              :         {
   11479        21440 :           res = make_ssa_name (type);
   11480        21440 :           gimple_call_set_lhs (stmt, res);
   11481              :         }
   11482        21812 :       gimple_set_location (stmt, loc);
   11483        21812 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11484              :     }
   11485        21812 :   gimple_build_insert_seq  (gsi, before, update, seq);
   11486        21812 :   return res;
   11487              : }
   11488              : 
   11489              : /* Build the call FN (ARG0, ARG1) with a result of type TYPE
   11490              :    (or no result if TYPE is void) with location LOC,
   11491              :    simplifying it first if possible.  Returns the built
   11492              :    expression value (or NULL_TREE if TYPE is void) inserting any new
   11493              :    statements at GSI honoring BEFORE and UPDATE.  */
   11494              : 
   11495              : tree
   11496          138 : gimple_build (gimple_stmt_iterator *gsi,
   11497              :               bool before, gsi_iterator_update update,
   11498              :               location_t loc, combined_fn fn,
   11499              :               tree type, tree arg0, tree arg1)
   11500              : {
   11501          138 :   gimple_seq seq = NULL;
   11502          138 :   tree res = gimple_simplify (fn, type, arg0, arg1, &seq,
   11503              :                               gimple_build_valueize);
   11504          138 :   if (!res)
   11505              :     {
   11506          138 :       gcall *stmt;
   11507          138 :       if (internal_fn_p (fn))
   11508          138 :         stmt = gimple_build_call_internal (as_internal_fn (fn), 2, arg0, arg1);
   11509              :       else
   11510              :         {
   11511            0 :           tree decl = builtin_decl_implicit (as_builtin_fn (fn));
   11512            0 :           stmt = gimple_build_call (decl, 2, arg0, arg1);
   11513              :         }
   11514          138 :       if (!VOID_TYPE_P (type))
   11515              :         {
   11516          138 :           res = make_ssa_name (type);
   11517          138 :           gimple_call_set_lhs (stmt, res);
   11518              :         }
   11519          138 :       gimple_set_location (stmt, loc);
   11520          138 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11521              :     }
   11522          138 :   gimple_build_insert_seq (gsi, before, update, seq);
   11523          138 :   return res;
   11524              : }
   11525              : 
   11526              : /* Build the call FN (ARG0, ARG1, ARG2) with a result of type TYPE
   11527              :    (or no result if TYPE is void) with location LOC,
   11528              :    simplifying it first if possible.  Returns the built
   11529              :    expression value (or NULL_TREE if TYPE is void) inserting any new
   11530              :    statements at GSI honoring BEFORE and UPDATE.  */
   11531              : 
   11532              : tree
   11533            0 : gimple_build (gimple_stmt_iterator *gsi,
   11534              :               bool before, gsi_iterator_update update,
   11535              :               location_t loc, combined_fn fn,
   11536              :               tree type, tree arg0, tree arg1, tree arg2)
   11537              : {
   11538            0 :   gimple_seq seq = NULL;
   11539            0 :   tree res = gimple_simplify (fn, type, arg0, arg1, arg2,
   11540              :                               &seq, gimple_build_valueize);
   11541            0 :   if (!res)
   11542              :     {
   11543            0 :       gcall *stmt;
   11544            0 :       if (internal_fn_p (fn))
   11545            0 :         stmt = gimple_build_call_internal (as_internal_fn (fn),
   11546              :                                            3, arg0, arg1, arg2);
   11547              :       else
   11548              :         {
   11549            0 :           tree decl = builtin_decl_implicit (as_builtin_fn (fn));
   11550            0 :           stmt = gimple_build_call (decl, 3, arg0, arg1, arg2);
   11551              :         }
   11552            0 :       if (!VOID_TYPE_P (type))
   11553              :         {
   11554            0 :           res = make_ssa_name (type);
   11555            0 :           gimple_call_set_lhs (stmt, res);
   11556              :         }
   11557            0 :       gimple_set_location (stmt, loc);
   11558            0 :       gimple_seq_add_stmt_without_update (&seq, stmt);
   11559              :     }
   11560            0 :   gimple_build_insert_seq (gsi, before, update, seq);
   11561            0 :   return res;
   11562              : }
   11563              : 
   11564              : /* Build CODE (OP0) with a result of type TYPE (or no result if TYPE is
   11565              :    void) with location LOC, simplifying it first if possible.  Returns the
   11566              :    built expression value (or NULL_TREE if TYPE is void) inserting any new
   11567              :    statements at GSI honoring BEFORE and UPDATE.  */
   11568              : 
   11569              : tree
   11570           21 : gimple_build (gimple_stmt_iterator *gsi,
   11571              :               bool before, gsi_iterator_update update,
   11572              :               location_t loc, code_helper code, tree type, tree op0)
   11573              : {
   11574           21 :   if (code.is_tree_code ())
   11575            0 :     return gimple_build (gsi, before, update, loc, tree_code (code), type, op0);
   11576           21 :   return gimple_build (gsi, before, update, loc, combined_fn (code), type, op0);
   11577              : }
   11578              : 
   11579              : /* Build CODE (OP0, OP1) with a result of type TYPE (or no result if TYPE is
   11580              :    void) with location LOC, simplifying it first if possible.  Returns the
   11581              :    built expression value (or NULL_TREE if TYPE is void) inserting any new
   11582              :    statements at GSI honoring BEFORE and UPDATE.  */
   11583              : 
   11584              : tree
   11585        23795 : gimple_build (gimple_stmt_iterator *gsi,
   11586              :               bool before, gsi_iterator_update update,
   11587              :               location_t loc, code_helper code, tree type, tree op0, tree op1)
   11588              : {
   11589        23795 :   if (code.is_tree_code ())
   11590        23657 :     return gimple_build (gsi, before, update,
   11591        23657 :                          loc, tree_code (code), type, op0, op1);
   11592          138 :   return gimple_build (gsi, before, update,
   11593          138 :                        loc, combined_fn (code), type, op0, op1);
   11594              : }
   11595              : 
   11596              : /* Build CODE (OP0, OP1, OP2) with a result of type TYPE (or no result if TYPE
   11597              :    is void) with location LOC, simplifying it first if possible.  Returns the
   11598              :    built expression value (or NULL_TREE if TYPE is void) inserting any new
   11599              :    statements at GSI honoring BEFORE and UPDATE.  */
   11600              : 
   11601              : tree
   11602            0 : gimple_build (gimple_stmt_iterator *gsi,
   11603              :               bool before, gsi_iterator_update update,
   11604              :               location_t loc, code_helper code,
   11605              :               tree type, tree op0, tree op1, tree op2)
   11606              : {
   11607            0 :   if (code.is_tree_code ())
   11608            0 :     return gimple_build (gsi, before, update,
   11609            0 :                          loc, tree_code (code), type, op0, op1, op2);
   11610            0 :   return gimple_build (gsi, before, update,
   11611            0 :                        loc, combined_fn (code), type, op0, op1, op2);
   11612              : }
   11613              : 
   11614              : /* Build the conversion (TYPE) OP with a result of type TYPE
   11615              :    with location LOC if such conversion is necessary in GIMPLE,
   11616              :    simplifying it first.
   11617              :    Returns the built expression inserting any new statements
   11618              :    at GSI honoring BEFORE and UPDATE.  */
   11619              : 
   11620              : tree
   11621      2077035 : gimple_convert (gimple_stmt_iterator *gsi,
   11622              :                 bool before, gsi_iterator_update update,
   11623              :                 location_t loc, tree type, tree op)
   11624              : {
   11625      2077035 :   if (useless_type_conversion_p (type, TREE_TYPE (op)))
   11626              :     return op;
   11627       186253 :   return gimple_build (gsi, before, update, loc, NOP_EXPR, type, op);
   11628              : }
   11629              : 
   11630              : /* Build the conversion (ptrofftype) OP with a result of a type
   11631              :    compatible with ptrofftype with location LOC if such conversion
   11632              :    is necessary in GIMPLE, simplifying it first.
   11633              :    Returns the built expression value inserting any new statements
   11634              :    at GSI honoring BEFORE and UPDATE.  */
   11635              : 
   11636              : tree
   11637          210 : gimple_convert_to_ptrofftype (gimple_stmt_iterator *gsi,
   11638              :                               bool before, gsi_iterator_update update,
   11639              :                               location_t loc, tree op)
   11640              : {
   11641          210 :   if (ptrofftype_p (TREE_TYPE (op)))
   11642              :     return op;
   11643            0 :   return gimple_convert (gsi, before, update, loc, sizetype, op);
   11644              : }
   11645              : 
   11646              : /* Build a vector of type TYPE in which each element has the value OP.
   11647              :    Return a gimple value for the result, inserting any new statements
   11648              :    at GSI honoring BEFORE and UPDATE.  */
   11649              : 
   11650              : tree
   11651       329612 : gimple_build_vector_from_val (gimple_stmt_iterator *gsi,
   11652              :                               bool before, gsi_iterator_update update,
   11653              :                               location_t loc, tree type, tree op)
   11654              : {
   11655       329612 :   if (!TYPE_VECTOR_SUBPARTS (type).is_constant ()
   11656              :       && !CONSTANT_CLASS_P (op))
   11657              :     return gimple_build (gsi, before, update,
   11658              :                          loc, VEC_DUPLICATE_EXPR, type, op);
   11659              : 
   11660       329612 :   tree res, vec = build_vector_from_val (type, op);
   11661       329612 :   if (is_gimple_val (vec))
   11662              :     return vec;
   11663        28145 :   if (gimple_in_ssa_p (cfun))
   11664        28145 :     res = make_ssa_name (type);
   11665              :   else
   11666            0 :     res = create_tmp_reg (type);
   11667        28145 :   gimple_seq seq = NULL;
   11668        28145 :   gimple *stmt = gimple_build_assign (res, vec);
   11669        28145 :   gimple_set_location (stmt, loc);
   11670        28145 :   gimple_seq_add_stmt_without_update (&seq, stmt);
   11671        28145 :   gimple_build_insert_seq (gsi, before, update, seq);
   11672        28145 :   return res;
   11673              : }
   11674              : 
   11675              : /* Build a vector from BUILDER, handling the case in which some elements
   11676              :    are non-constant.  Return a gimple value for the result, inserting
   11677              :    any new instructions to GSI honoring BEFORE and UPDATE.
   11678              : 
   11679              :    BUILDER must not have a stepped encoding on entry.  This is because
   11680              :    the function is not geared up to handle the arithmetic that would
   11681              :    be needed in the variable case, and any code building a vector that
   11682              :    is known to be constant should use BUILDER->build () directly.  */
   11683              : 
   11684              : tree
   11685       379555 : gimple_build_vector (gimple_stmt_iterator *gsi,
   11686              :                      bool before, gsi_iterator_update update,
   11687              :                      location_t loc, tree_vector_builder *builder)
   11688              : {
   11689       379555 :   gcc_assert (builder->nelts_per_pattern () <= 2);
   11690       379555 :   unsigned int encoded_nelts = builder->encoded_nelts ();
   11691      1337915 :   for (unsigned int i = 0; i < encoded_nelts; ++i)
   11692      1074171 :     if (!CONSTANT_CLASS_P ((*builder)[i]))
   11693              :       {
   11694       115811 :         gimple_seq seq = NULL;
   11695       115811 :         tree type = builder->type ();
   11696       115811 :         unsigned int nelts = TYPE_VECTOR_SUBPARTS (type).to_constant ();
   11697       115811 :         vec<constructor_elt, va_gc> *v;
   11698       115811 :         vec_alloc (v, nelts);
   11699       485400 :         for (i = 0; i < nelts; ++i)
   11700       253778 :           CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, builder->elt (i));
   11701              : 
   11702       115811 :         tree res;
   11703       115811 :         if (gimple_in_ssa_p (cfun))
   11704       115811 :           res = make_ssa_name (type);
   11705              :         else
   11706            0 :           res = create_tmp_reg (type);
   11707       115811 :         gimple *stmt = gimple_build_assign (res, build_constructor (type, v));
   11708       115811 :         gimple_set_location (stmt, loc);
   11709       115811 :         gimple_seq_add_stmt_without_update (&seq, stmt);
   11710       115811 :         gimple_build_insert_seq (gsi, before, update, seq);
   11711       115811 :         return res;
   11712              :       }
   11713       263744 :   return builder->build ();
   11714              : }
   11715              : 
   11716              : /* Emit gimple statements into &stmts that take a value given in OLD_SIZE
   11717              :    and generate a value guaranteed to be rounded upwards to ALIGN.
   11718              : 
   11719              :    Return the tree node representing this size, it is of TREE_TYPE TYPE.  */
   11720              : 
   11721              : tree
   11722            0 : gimple_build_round_up (gimple_stmt_iterator *gsi,
   11723              :                        bool before, gsi_iterator_update update,
   11724              :                        location_t loc, tree type,
   11725              :                        tree old_size, unsigned HOST_WIDE_INT align)
   11726              : {
   11727            0 :   unsigned HOST_WIDE_INT tg_mask = align - 1;
   11728              :   /* tree new_size = (old_size + tg_mask) & ~tg_mask;  */
   11729            0 :   gcc_assert (INTEGRAL_TYPE_P (type));
   11730            0 :   tree tree_mask = build_int_cst (type, tg_mask);
   11731            0 :   tree oversize = gimple_build (gsi, before, update,
   11732              :                                 loc, PLUS_EXPR, type, old_size, tree_mask);
   11733              : 
   11734            0 :   tree mask = build_int_cst (type, -align);
   11735            0 :   return gimple_build (gsi, before, update,
   11736            0 :                        loc, BIT_AND_EXPR, type, oversize, mask);
   11737              : }
   11738              : 
   11739              : /* Return true if the result of assignment STMT is known to be non-negative.
   11740              :    DEPTH is the current nesting depth of the query.  */
   11741              : 
   11742              : static bool
   11743      6134718 : gimple_assign_nonnegative_p (gimple *stmt, int depth)
   11744              : {
   11745      6134718 :   enum tree_code code = gimple_assign_rhs_code (stmt);
   11746      6134718 :   tree type = TREE_TYPE (gimple_assign_lhs (stmt));
   11747      6134718 :   switch (get_gimple_rhs_class (code))
   11748              :     {
   11749       468595 :     case GIMPLE_UNARY_RHS:
   11750       468595 :       return tree_unary_nonnegative_p (gimple_assign_rhs_code (stmt),
   11751              :                                        type,
   11752              :                                        gimple_assign_rhs1 (stmt),
   11753       468595 :                                        depth);
   11754      3402675 :     case GIMPLE_BINARY_RHS:
   11755      3402675 :       return tree_binary_nonnegative_p (gimple_assign_rhs_code (stmt),
   11756              :                                         type,
   11757              :                                         gimple_assign_rhs1 (stmt),
   11758              :                                         gimple_assign_rhs2 (stmt),
   11759      3402675 :                                         depth);
   11760              :     case GIMPLE_TERNARY_RHS:
   11761              :       return false;
   11762      2250238 :     case GIMPLE_SINGLE_RHS:
   11763      2250238 :       return tree_single_nonnegative_p (gimple_assign_rhs1 (stmt),
   11764      2250238 :                                         depth);
   11765              :     case GIMPLE_INVALID_RHS:
   11766              :       break;
   11767              :     }
   11768            0 :   gcc_unreachable ();
   11769              : }
   11770              : 
   11771              : /* Return true if return value of call STMT is known to be non-negative.
   11772              :    DEPTH is the current nesting depth of the query.  */
   11773              : 
   11774              : static bool
   11775     13598735 : gimple_call_nonnegative_p (gimple *stmt, int depth)
   11776              : {
   11777     13598735 :   tree arg0
   11778     13598735 :     = gimple_call_num_args (stmt) > 0 ? gimple_call_arg (stmt, 0) : NULL_TREE;
   11779     13598735 :   tree arg1
   11780     13598735 :     = gimple_call_num_args (stmt) > 1 ? gimple_call_arg (stmt, 1) : NULL_TREE;
   11781     13598735 :   tree lhs = gimple_call_lhs (stmt);
   11782     13598735 :   return (lhs
   11783     13598735 :           && tree_call_nonnegative_p (TREE_TYPE (lhs),
   11784              :                                       gimple_call_combined_fn (stmt),
   11785     13598735 :                                       arg0, arg1, depth));
   11786              : }
   11787              : 
   11788              : /* Return true if return value of call STMT is known to be non-negative.
   11789              :    DEPTH is the current nesting depth of the query.  */
   11790              : 
   11791              : static bool
   11792       972957 : gimple_phi_nonnegative_p (gimple *stmt, int depth)
   11793              : {
   11794      1331998 :   for (unsigned i = 0; i < gimple_phi_num_args (stmt); ++i)
   11795              :     {
   11796      1326962 :       tree arg = gimple_phi_arg_def (stmt, i);
   11797      1326962 :       if (!tree_single_nonnegative_p (arg, depth + 1))
   11798              :         return false;
   11799              :     }
   11800              :   return true;
   11801              : }
   11802              : 
   11803              : /* Return true if STMT is known to compute a non-negative value.
   11804              :    DEPTH is the current nesting depth of the query.  */
   11805              : 
   11806              : bool
   11807     21394569 : gimple_stmt_nonnegative_p (gimple *stmt, int depth)
   11808              : {
   11809     21394569 :   tree type = gimple_range_type (stmt);
   11810     21394569 :   if (type && frange::supports_p (type))
   11811              :     {
   11812      1459055 :       frange r;
   11813      1459055 :       bool sign;
   11814      1459055 :       if (get_global_range_query ()->range_of_stmt (r, stmt)
   11815      1459055 :           && r.signbit_p (sign))
   11816        25658 :         return !sign;
   11817      1459055 :     }
   11818     21368911 :   switch (gimple_code (stmt))
   11819              :     {
   11820      6134718 :     case GIMPLE_ASSIGN:
   11821      6134718 :       return gimple_assign_nonnegative_p (stmt, depth);
   11822     13598735 :     case GIMPLE_CALL:
   11823     13598735 :       return gimple_call_nonnegative_p (stmt, depth);
   11824       972957 :     case GIMPLE_PHI:
   11825       972957 :       return gimple_phi_nonnegative_p (stmt, depth);
   11826              :     default:
   11827              :       return false;
   11828              :     }
   11829              : }
   11830              : 
   11831              : /* Return true if the floating-point value computed by assignment STMT
   11832              :    is known to have an integer value.  We also allow +Inf, -Inf and NaN
   11833              :    to be considered integer values. Return false for signaling NaN.
   11834              : 
   11835              :    DEPTH is the current nesting depth of the query.  */
   11836              : 
   11837              : static bool
   11838        60469 : gimple_assign_integer_valued_real_p (gimple *stmt, int depth)
   11839              : {
   11840        60469 :   enum tree_code code = gimple_assign_rhs_code (stmt);
   11841        60469 :   switch (get_gimple_rhs_class (code))
   11842              :     {
   11843        15211 :     case GIMPLE_UNARY_RHS:
   11844        15211 :       return integer_valued_real_unary_p (gimple_assign_rhs_code (stmt),
   11845        15211 :                                           gimple_assign_rhs1 (stmt), depth);
   11846        13863 :     case GIMPLE_BINARY_RHS:
   11847        13863 :       return integer_valued_real_binary_p (gimple_assign_rhs_code (stmt),
   11848              :                                            gimple_assign_rhs1 (stmt),
   11849        13863 :                                            gimple_assign_rhs2 (stmt), depth);
   11850              :     case GIMPLE_TERNARY_RHS:
   11851              :       return false;
   11852        30211 :     case GIMPLE_SINGLE_RHS:
   11853        30211 :       return integer_valued_real_single_p (gimple_assign_rhs1 (stmt), depth);
   11854              :     case GIMPLE_INVALID_RHS:
   11855              :       break;
   11856              :     }
   11857            0 :   gcc_unreachable ();
   11858              : }
   11859              : 
   11860              : /* Return true if the floating-point value computed by call STMT is known
   11861              :    to have an integer value.  We also allow +Inf, -Inf and NaN to be
   11862              :    considered integer values. Return false for signaling NaN.
   11863              : 
   11864              :    DEPTH is the current nesting depth of the query.  */
   11865              : 
   11866              : static bool
   11867         1125 : gimple_call_integer_valued_real_p (gimple *stmt, int depth)
   11868              : {
   11869         1125 :   tree arg0 = (gimple_call_num_args (stmt) > 0
   11870         1125 :                ? gimple_call_arg (stmt, 0)
   11871              :                : NULL_TREE);
   11872         1125 :   tree arg1 = (gimple_call_num_args (stmt) > 1
   11873         1125 :                ? gimple_call_arg (stmt, 1)
   11874              :                : NULL_TREE);
   11875         1125 :   return integer_valued_real_call_p (gimple_call_combined_fn (stmt),
   11876         1125 :                                      arg0, arg1, depth);
   11877              : }
   11878              : 
   11879              : /* Return true if the floating-point result of phi STMT is known to have
   11880              :    an integer value.  We also allow +Inf, -Inf and NaN to be considered
   11881              :    integer values. Return false for signaling NaN.
   11882              : 
   11883              :    DEPTH is the current nesting depth of the query.  */
   11884              : 
   11885              : static bool
   11886         1569 : gimple_phi_integer_valued_real_p (gimple *stmt, int depth)
   11887              : {
   11888         1736 :   for (unsigned i = 0; i < gimple_phi_num_args (stmt); ++i)
   11889              :     {
   11890         1727 :       tree arg = gimple_phi_arg_def (stmt, i);
   11891         1727 :       if (!integer_valued_real_single_p (arg, depth + 1))
   11892              :         return false;
   11893              :     }
   11894              :   return true;
   11895              : }
   11896              : 
   11897              : /* Return true if the floating-point value computed by STMT is known
   11898              :    to have an integer value.  We also allow +Inf, -Inf and NaN to be
   11899              :    considered integer values. Return false for signaling NaN.
   11900              : 
   11901              :    DEPTH is the current nesting depth of the query.  */
   11902              : 
   11903              : bool
   11904        90577 : gimple_stmt_integer_valued_real_p (gimple *stmt, int depth)
   11905              : {
   11906        90577 :   switch (gimple_code (stmt))
   11907              :     {
   11908        60469 :     case GIMPLE_ASSIGN:
   11909        60469 :       return gimple_assign_integer_valued_real_p (stmt, depth);
   11910         1125 :     case GIMPLE_CALL:
   11911         1125 :       return gimple_call_integer_valued_real_p (stmt, depth);
   11912         1569 :     case GIMPLE_PHI:
   11913         1569 :       return gimple_phi_integer_valued_real_p (stmt, depth);
   11914              :     default:
   11915              :       return false;
   11916              :     }
   11917              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.