LCOV - code coverage report
Current view: top level - gcc - cfgexpand.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 88.5 % 3529 3124
Test Date: 2026-11-14 16:39:50 Functions: 94.3 % 106 100
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* A pass for lowering trees to RTL.
       2              :    Copyright (C) 2004-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify
       7              : it under the terms of the GNU General Public License as published by
       8              : the Free Software Foundation; either version 3, or (at your option)
       9              : any later version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful,
      12              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      13              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      14              : GNU General Public License for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : #include "config.h"
      21              : #include "system.h"
      22              : #include "coretypes.h"
      23              : #include "backend.h"
      24              : #include "target.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "gimple.h"
      28              : #include "cfghooks.h"
      29              : #include "tree-pass.h"
      30              : #include "memmodel.h"
      31              : #include "tm_p.h"
      32              : #include "ssa.h"
      33              : #include "optabs.h"
      34              : #include "regs.h" /* For reg_renumber.  */
      35              : #include "emit-rtl.h"
      36              : #include "recog.h"
      37              : #include "cgraph.h"
      38              : #include "diagnostic.h"
      39              : #include "fold-const.h"
      40              : #include "varasm.h"
      41              : #include "stor-layout.h"
      42              : #include "stmt.h"
      43              : #include "print-tree.h"
      44              : #include "cfgrtl.h"
      45              : #include "cfganal.h"
      46              : #include "cfgbuild.h"
      47              : #include "cfgcleanup.h"
      48              : #include "dojump.h"
      49              : #include "explow.h"
      50              : #include "calls.h"
      51              : #include "expr.h"
      52              : #include "internal-fn.h"
      53              : #include "tree-eh.h"
      54              : #include "gimple-iterator.h"
      55              : #include "gimple-expr.h"
      56              : #include "gimple-walk.h"
      57              : #include "tree-cfg.h"
      58              : #include "tree-dfa.h"
      59              : #include "tree-ssa.h"
      60              : #include "except.h"
      61              : #include "gimple-pretty-print.h"
      62              : #include "toplev.h"
      63              : #include "debug.h"
      64              : #include "tree-inline.h"
      65              : #include "value-prof.h"
      66              : #include "tree-ssa-live.h"
      67              : #include "tree-outof-ssa.h"
      68              : #include "cfgloop.h"
      69              : #include "insn-attr.h" /* For INSN_SCHEDULING.  */
      70              : #include "stringpool.h"
      71              : #include "attribs.h"
      72              : #include "asan.h"
      73              : #include "tree-ssa-address.h"
      74              : #include "output.h"
      75              : #include "builtins.h"
      76              : #include "opts.h"
      77              : #include "gimple-range.h"
      78              : #include "rtl-iter.h"
      79              : 
      80              : /* Some systems use __main in a way incompatible with its use in gcc, in these
      81              :    cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
      82              :    give the same symbol without quotes for an alternative entry point.  You
      83              :    must define both, or neither.  */
      84              : #ifndef NAME__MAIN
      85              : #define NAME__MAIN "__main"
      86              : #endif
      87              : 
      88              : /* This variable holds information helping the rewriting of SSA trees
      89              :    into RTL.  */
      90              : struct ssaexpand SA;
      91              : 
      92              : /* This variable holds the currently expanded gimple statement for purposes
      93              :    of communicating the profile info to the builtin expanders.  */
      94              : gimple *currently_expanding_gimple_stmt;
      95              : 
      96              : static rtx expand_debug_expr (tree);
      97              : 
      98              : static bool defer_stack_allocation (tree, bool);
      99              : 
     100              : static void record_alignment_for_reg_var (unsigned int);
     101              : 
     102              : /* Return an expression tree corresponding to the RHS of GIMPLE
     103              :    statement STMT.  */
     104              : 
     105              : tree
     106      3510805 : gimple_assign_rhs_to_tree (gimple *stmt)
     107              : {
     108      3510805 :   tree t;
     109      3510805 :   switch (gimple_assign_rhs_class (stmt))
     110              :     {
     111         3839 :     case GIMPLE_TERNARY_RHS:
     112         3839 :       t = build3 (gimple_assign_rhs_code (stmt),
     113         3839 :                   TREE_TYPE (gimple_assign_lhs (stmt)),
     114              :                   gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
     115              :                   gimple_assign_rhs3 (stmt));
     116         3839 :       break;
     117      1357574 :     case GIMPLE_BINARY_RHS:
     118      1357574 :       t = build2 (gimple_assign_rhs_code (stmt),
     119      1357574 :                   TREE_TYPE (gimple_assign_lhs (stmt)),
     120              :                   gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
     121      1357574 :       break;
     122       370472 :     case GIMPLE_UNARY_RHS:
     123       370472 :       t = build1 (gimple_assign_rhs_code (stmt),
     124       370472 :                   TREE_TYPE (gimple_assign_lhs (stmt)),
     125              :                   gimple_assign_rhs1 (stmt));
     126       370472 :       break;
     127      1778920 :     case GIMPLE_SINGLE_RHS:
     128      1778920 :       {
     129      1778920 :         t = gimple_assign_rhs1 (stmt);
     130              :         /* Avoid modifying this tree in place below.  */
     131      3472038 :         if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
     132      1661588 :              && gimple_location (stmt) != EXPR_LOCATION (t))
     133      1969814 :             || (gimple_block (stmt) && currently_expanding_to_rtl
     134        37274 :                 && EXPR_P (t)))
     135      1531969 :           t = copy_node (t);
     136              :         break;
     137              :       }
     138            0 :     default:
     139            0 :       gcc_unreachable ();
     140              :     }
     141              : 
     142      3510805 :   if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
     143      3264782 :     SET_EXPR_LOCATION (t, gimple_location (stmt));
     144              : 
     145      3510805 :   return t;
     146              : }
     147              : 
     148              : 
     149              : #ifndef STACK_ALIGNMENT_NEEDED
     150              : #define STACK_ALIGNMENT_NEEDED 1
     151              : #endif
     152              : 
     153              : #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
     154              : 
     155              : /* Choose either CUR or NEXT as the leader DECL for a partition.
     156              :    Prefer ignored decls, to simplify debug dumps and reduce ambiguity
     157              :    out of the same user variable being in multiple partitions (this is
     158              :    less likely for compiler-introduced temps).  Also used by out-of-SSA
     159              :    to work out which variable a partition will be given.  */
     160              : 
     161              : tree
     162     80012754 : expand_leader_merge (tree cur, tree next)
     163              : {
     164     80012754 :   if (cur == NULL || cur == next)
     165              :     return next;
     166              : 
     167      3867619 :   if (DECL_P (cur) && DECL_IGNORED_P (cur))
     168              :     return cur;
     169              : 
     170      2806314 :   if (DECL_P (next) && DECL_IGNORED_P (next))
     171       319354 :     return next;
     172              : 
     173              :   return cur;
     174              : }
     175              : 
     176              : /* Associate declaration T with storage space X.  If T is no
     177              :    SSA name this is exactly SET_DECL_RTL, otherwise make the
     178              :    partition of T associated with X.  */
     179              : static inline void
     180     72695814 : set_rtl (tree t, rtx x)
     181              : {
     182     72695814 :   gcc_checking_assert (!x
     183              :                        || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
     184              :                        || (use_register_for_decl (t)
     185              :                            ? (REG_P (x)
     186              :                               || (GET_CODE (x) == CONCAT
     187              :                                   && (REG_P (XEXP (x, 0))
     188              :                                       || SUBREG_P (XEXP (x, 0)))
     189              :                                   && (REG_P (XEXP (x, 1))
     190              :                                       || SUBREG_P (XEXP (x, 1))))
     191              :                               /* We need to accept PARALLELs for RESUT_DECLs
     192              :                                  because of vector types with BLKmode returned
     193              :                                  in multiple registers, but they are supposed
     194              :                                  to be uncoalesced.  */
     195              :                               || (GET_CODE (x) == PARALLEL
     196              :                                   && SSAVAR (t)
     197              :                                   && TREE_CODE (SSAVAR (t)) == RESULT_DECL
     198              :                                   && (GET_MODE (x) == BLKmode
     199              :                                       || !flag_tree_coalesce_vars)))
     200              :                            : (MEM_P (x) || x == pc_rtx
     201              :                               || (GET_CODE (x) == CONCAT
     202              :                                   && MEM_P (XEXP (x, 0))
     203              :                                   && MEM_P (XEXP (x, 1))))));
     204              :   /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
     205              :      RESULT_DECLs has the expected mode.  For memory, we accept
     206              :      unpromoted modes, since that's what we're likely to get.  For
     207              :      PARM_DECLs and RESULT_DECLs, we'll have been called by
     208              :      set_parm_rtl, which will give us the default def, so we don't
     209              :      have to compute it ourselves.  For RESULT_DECLs, we accept mode
     210              :      mismatches too, as long as we have BLKmode or are not coalescing
     211              :      across variables, so that we don't reject BLKmode PARALLELs or
     212              :      unpromoted REGs.  */
     213     67353365 :   gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
     214              :                        || (SSAVAR (t)
     215              :                            && TREE_CODE (SSAVAR (t)) == RESULT_DECL
     216              :                            && (promote_ssa_mode (t, NULL) == BLKmode
     217              :                                || !flag_tree_coalesce_vars))
     218              :                        || !use_register_for_decl (t)
     219              :                        || GET_MODE (x) == promote_ssa_mode (t, NULL));
     220              : 
     221     67353365 :   if (x)
     222              :     {
     223     67789354 :       bool skip = false;
     224     67789354 :       tree cur = NULL_TREE;
     225              :       rtx xm = x;
     226              : 
     227       435989 :     retry:
     228     67789354 :       if (MEM_P (xm))
     229      4865580 :         cur = MEM_EXPR (xm);
     230              :       else if (REG_P (xm))
     231     56635833 :         cur = REG_EXPR (xm);
     232              :       else if (SUBREG_P (xm))
     233              :         {
     234            0 :           gcc_assert (subreg_lowpart_p (xm));
     235            0 :           xm = SUBREG_REG (xm);
     236            0 :           goto retry;
     237              :         }
     238              :       else if (GET_CODE (xm) == CONCAT)
     239              :         {
     240       432151 :           xm = XEXP (xm, 0);
     241       432151 :           goto retry;
     242              :         }
     243              :       else if (GET_CODE (xm) == PARALLEL)
     244              :         {
     245         3838 :           xm = XVECEXP (xm, 0, 0);
     246         3838 :           gcc_assert (GET_CODE (xm) == EXPR_LIST);
     247         3838 :           xm = XEXP (xm, 0);
     248         3838 :           goto retry;
     249              :         }
     250      5851952 :       else if (xm == pc_rtx)
     251              :         skip = true;
     252              :       else
     253            0 :         gcc_unreachable ();
     254              : 
     255     35280120 :       tree next
     256     61501413 :         = skip ? cur : expand_leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
     257              : 
     258     67353365 :       if (cur != next)
     259              :         {
     260     28426298 :           if (MEM_P (x))
     261      4089916 :             set_mem_attributes (x,
     262      2044958 :                                 next && TREE_CODE (next) == SSA_NAME
     263        12556 :                                 ? TREE_TYPE (next)
     264              :                                 : next, true);
     265              :           else
     266     26381340 :             set_reg_attrs_for_decl_rtl (next, x);
     267              :         }
     268              :     }
     269              : 
     270     72695814 :   if (TREE_CODE (t) == SSA_NAME)
     271              :     {
     272     58962413 :       int part = var_to_partition (SA.map, t);
     273     58962413 :       if (part != NO_PARTITION)
     274              :         {
     275     58962413 :           if (SA.partition_to_pseudo[part])
     276     32247359 :             gcc_assert (SA.partition_to_pseudo[part] == x);
     277     26715054 :           else if (x != pc_rtx)
     278     26711816 :             SA.partition_to_pseudo[part] = x;
     279              :         }
     280              :       /* For the benefit of debug information at -O0 (where
     281              :          vartracking doesn't run) record the place also in the base
     282              :          DECL.  For PARMs and RESULTs, do so only when setting the
     283              :          default def.  */
     284    117921588 :       if (x && x != pc_rtx && SSA_NAME_VAR (t)
     285     77315838 :           && (VAR_P (SSA_NAME_VAR (t))
     286      7554118 :               || SSA_NAME_IS_DEFAULT_DEF (t)))
     287              :         {
     288     18173665 :           tree var = SSA_NAME_VAR (t);
     289              :           /* If we don't yet have something recorded, just record it now.  */
     290     18173665 :           if (!DECL_RTL_SET_P (var))
     291      7649064 :             SET_DECL_RTL (var, x);
     292              :           /* If we have it set already to "multiple places" don't
     293              :              change this.  */
     294     10524601 :           else if (DECL_RTL (var) == pc_rtx)
     295              :             ;
     296              :           /* If we have something recorded and it's not the same place
     297              :              as we want to record now, we have multiple partitions for the
     298              :              same base variable, with different places.  We can't just
     299              :              randomly chose one, hence we have to say that we don't know.
     300              :              This only happens with optimization, and there var-tracking
     301              :              will figure out the right thing.  */
     302      9358248 :           else if (DECL_RTL (var) != x)
     303       248538 :             SET_DECL_RTL (var, pc_rtx);
     304              :         }
     305              :     }
     306              :   else
     307     13733401 :     SET_DECL_RTL (t, x);
     308     72695814 : }
     309              : 
     310              : /* This structure holds data relevant to one variable that will be
     311              :    placed in a stack slot.  */
     312              : class stack_var
     313              : {
     314              : public:
     315              :   /* The Variable.  */
     316              :   tree decl;
     317              : 
     318              :   /* Initially, the size of the variable.  Later, the size of the partition,
     319              :      if this variable becomes it's partition's representative.  */
     320              :   poly_uint64 size;
     321              : 
     322              :   /* The *byte* alignment required for this variable.  Or as, with the
     323              :      size, the alignment for this partition.  */
     324              :   unsigned int alignb;
     325              : 
     326              :   /* The partition representative.  */
     327              :   unsigned representative;
     328              : 
     329              :   /* The next stack variable in the partition, or EOC.  */
     330              :   unsigned next;
     331              : 
     332              :   /* The numbers of conflicting stack variables.  */
     333              :   bitmap conflicts;
     334              : };
     335              : 
     336              : #define EOC  ((unsigned)-1)
     337              : 
     338              : /* We have an array of such objects while deciding allocation.  */
     339              : static class stack_var *stack_vars;
     340              : static unsigned stack_vars_alloc;
     341              : static unsigned stack_vars_num;
     342              : static hash_map<tree, unsigned> *decl_to_stack_part;
     343              : 
     344              : #define INVALID_STACK_INDEX ((unsigned)-1)
     345              : 
     346              : /* Conflict bitmaps go on this obstack.  This allows us to destroy
     347              :    all of them in one big sweep.  */
     348              : static bitmap_obstack stack_var_bitmap_obstack;
     349              : 
     350              : /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
     351              :    is non-decreasing.  */
     352              : static unsigned *stack_vars_sorted;
     353              : 
     354              : /* The phase of the stack frame.  This is the known misalignment of
     355              :    virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY.  That is,
     356              :    (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0.  */
     357              : static int frame_phase;
     358              : 
     359              : /* Used during expand_used_vars to remember if we saw any decls for
     360              :    which we'd like to enable stack smashing protection.  */
     361              : static bool has_protected_decls;
     362              : 
     363              : /* Used during expand_used_vars.  Remember if we say a character buffer
     364              :    smaller than our cutoff threshold.  Used for -Wstack-protector.  */
     365              : static bool has_short_buffer;
     366              : 
     367              : /* Compute the byte alignment to use for DECL.  Ignore alignment
     368              :    we can't do with expected alignment of the stack boundary.  */
     369              : 
     370              : static unsigned int
     371     13464934 : align_local_variable (tree decl, bool really_expand)
     372              : {
     373     13464934 :   unsigned int align;
     374              : 
     375     13464934 :   if (TREE_CODE (decl) == SSA_NAME)
     376              :     {
     377       918907 :       tree type = TREE_TYPE (decl);
     378       918907 :       machine_mode mode = TYPE_MODE (type);
     379              : 
     380       918907 :       align = TYPE_ALIGN (type);
     381       918907 :       if (mode != BLKmode
     382       918907 :           && align < GET_MODE_ALIGNMENT (mode))
     383          924 :         align = GET_MODE_ALIGNMENT (mode);
     384              :     }
     385              :   else
     386     12546027 :     align = LOCAL_DECL_ALIGNMENT (decl);
     387              : 
     388     13464934 :   if (hwassist_sanitize_stack_p ())
     389          568 :     align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
     390              : 
     391     13464934 :   if (TREE_CODE (decl) != SSA_NAME && really_expand)
     392              :     /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
     393              :        That is done before IPA and could bump alignment based on host
     394              :        backend even for offloaded code which wants different
     395              :        LOCAL_DECL_ALIGNMENT.  */
     396      1576798 :     SET_DECL_ALIGN (decl, align);
     397              : 
     398     13464934 :   return align / BITS_PER_UNIT;
     399              : }
     400              : 
     401              : /* Align given offset BASE with ALIGN.  Truncate up if ALIGN_UP is true,
     402              :    down otherwise.  Return truncated BASE value.  */
     403              : 
     404              : static inline unsigned HOST_WIDE_INT
     405              : align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
     406              : {
     407              :   return align_up ? (base + align - 1) & -align : base & -align;
     408              : }
     409              : 
     410              : /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
     411              :    Return the frame offset.  */
     412              : 
     413              : static poly_int64
     414      1766024 : alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
     415              : {
     416      1766024 :   poly_int64 offset, new_frame_offset;
     417              : 
     418      1766024 :   if (FRAME_GROWS_DOWNWARD)
     419              :     {
     420      1766024 :       new_frame_offset
     421      1766024 :         = aligned_lower_bound (frame_offset - frame_phase - size,
     422      1766024 :                                align) + frame_phase;
     423      1766024 :       offset = new_frame_offset;
     424              :     }
     425              :   else
     426              :     {
     427              :       new_frame_offset
     428              :         = aligned_upper_bound (frame_offset - frame_phase,
     429              :                                align) + frame_phase;
     430              :       offset = new_frame_offset;
     431              :       new_frame_offset += size;
     432              :     }
     433      1766024 :   frame_offset = new_frame_offset;
     434              : 
     435      1766024 :   if (frame_offset_overflow (frame_offset, cfun->decl))
     436            0 :     frame_offset = offset = 0;
     437              : 
     438      1766024 :   return offset;
     439              : }
     440              : 
     441              : /* Ensure that the stack is aligned to ALIGN bytes.
     442              :    Return the new frame offset.  */
     443              : static poly_int64
     444         1910 : align_frame_offset (unsigned HOST_WIDE_INT align)
     445              : {
     446            0 :   return alloc_stack_frame_space (0, align);
     447              : }
     448              : 
     449              : /* Accumulate DECL into STACK_VARS.  */
     450              : 
     451              : static void
     452      5851952 : add_stack_var (tree decl, bool really_expand)
     453              : {
     454      5851952 :   class stack_var *v;
     455              : 
     456      5851952 :   if (stack_vars_num >= stack_vars_alloc)
     457              :     {
     458      1362096 :       if (stack_vars_alloc)
     459        34871 :         stack_vars_alloc = stack_vars_alloc * 3 / 2;
     460              :       else
     461              :         stack_vars_alloc = 32;
     462      1362096 :       stack_vars
     463      1362096 :         = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
     464              :     }
     465      5851952 :   if (!decl_to_stack_part)
     466            0 :     decl_to_stack_part = new hash_map<tree, unsigned>;
     467              : 
     468      5851952 :   v = &stack_vars[stack_vars_num];
     469      5851952 :   decl_to_stack_part->put (decl, stack_vars_num);
     470              : 
     471      5851952 :   v->decl = decl;
     472      5851952 :   tree size = TREE_CODE (decl) == SSA_NAME
     473      5851952 :     ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
     474      5851952 :     : DECL_SIZE_UNIT (decl);
     475      5851952 :   v->size = tree_to_poly_uint64 (size);
     476              :   /* Ensure that all variables have size, so that &a != &b for any two
     477              :      variables that are simultaneously live.  */
     478      5851952 :   if (known_eq (v->size, 0U))
     479        24515 :     v->size = 1;
     480      5851952 :   v->alignb = align_local_variable (decl, really_expand);
     481              :   /* An alignment of zero can mightily confuse us later.  */
     482      5851952 :   gcc_assert (v->alignb != 0);
     483              : 
     484              :   /* All variables are initially in their own partition.  */
     485      5851952 :   v->representative = stack_vars_num;
     486      5851952 :   v->next = EOC;
     487              : 
     488              :   /* All variables initially conflict with no other.  */
     489      5851952 :   v->conflicts = NULL;
     490              : 
     491              :   /* Ensure that this decl doesn't get put onto the list twice.  */
     492      5851952 :   set_rtl (decl, pc_rtx);
     493              : 
     494      5851952 :   stack_vars_num++;
     495      5851952 : }
     496              : 
     497              : /* Make the decls associated with luid's X and Y conflict.  */
     498              : 
     499              : static void
     500     11710625 : add_stack_var_conflict (unsigned x, unsigned y)
     501              : {
     502     11710625 :   class stack_var *a = &stack_vars[x];
     503     11710625 :   class stack_var *b = &stack_vars[y];
     504     11710625 :   if (x == y)
     505              :     return;
     506     10621131 :   if (!a->conflicts)
     507       673063 :     a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     508     10621131 :   if (!b->conflicts)
     509       121301 :     b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     510     10621131 :   bitmap_set_bit (a->conflicts, y);
     511     10621131 :   bitmap_set_bit (b->conflicts, x);
     512              : }
     513              : 
     514              : /* Check whether the decls associated with luid's X and Y conflict.  */
     515              : 
     516              : static bool
     517     11558681 : stack_var_conflict_p (unsigned x, unsigned y)
     518              : {
     519     11558681 :   class stack_var *a = &stack_vars[x];
     520     11558681 :   class stack_var *b = &stack_vars[y];
     521     11558681 :   if (x == y)
     522              :     return false;
     523              :   /* Partitions containing an SSA name result from gimple registers
     524              :      with things like unsupported modes.  They are top-level and
     525              :      hence conflict with everything else.  */
     526     11558681 :   if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
     527              :     return true;
     528              : 
     529     11523343 :   if (!a->conflicts || !b->conflicts)
     530              :     return false;
     531     11472267 :   return bitmap_bit_p (a->conflicts, y);
     532              : }
     533              : 
     534              : /* Returns the DECL's index into the stack_vars array.
     535              :    If the DECL does not exist return INVALID_STACK_INDEX.  */
     536              : static unsigned
     537     47612605 : decl_stack_index (tree decl)
     538              : {
     539     47612605 :   if (!decl)
     540              :     return INVALID_STACK_INDEX;
     541     47612605 :   if (!DECL_P (decl))
     542              :     return INVALID_STACK_INDEX;
     543     36557704 :   if (DECL_RTL_IF_SET (decl) != pc_rtx)
     544              :     return INVALID_STACK_INDEX;
     545     24599383 :   unsigned *v = decl_to_stack_part->get (decl);
     546     24599383 :   if (!v)
     547              :     return INVALID_STACK_INDEX;
     548              : 
     549     24599383 :   unsigned indx = *v;
     550     24599383 :   gcc_checking_assert (indx != INVALID_STACK_INDEX);
     551     24599383 :   gcc_checking_assert (indx < stack_vars_num);
     552              :   return indx;
     553              : }
     554              : 
     555              : /* Callback for walk_stmt_ops.  If OP is a decl touched by add_stack_var
     556              :    enter its partition number into bitmap DATA.  */
     557              : 
     558              : static bool
     559     30103076 : visit_op (gimple *, tree op, tree, void *data)
     560              : {
     561     30103076 :   bitmap active = (bitmap)data;
     562     30103076 :   op = get_base_address (op);
     563     30103076 :   unsigned idx = decl_stack_index (op);
     564     30103076 :   if (idx != INVALID_STACK_INDEX)
     565     14856550 :     bitmap_set_bit (active, idx);
     566     30103076 :   return false;
     567              : }
     568              : 
     569              : /* Callback for walk_stmt_ops.  If OP is a decl touched by add_stack_var
     570              :    record conflicts between it and all currently active other partitions
     571              :    from bitmap DATA.  */
     572              : 
     573              : static bool
     574     13944119 : visit_conflict (gimple *, tree op, tree, void *data)
     575              : {
     576     13944119 :   bitmap active = (bitmap)data;
     577     13944119 :   op = get_base_address (op);
     578     13944119 :   unsigned num = decl_stack_index (op);
     579     13944119 :   if (num != INVALID_STACK_INDEX
     580     13944119 :       && bitmap_set_bit (active, num))
     581              :     {
     582      1009003 :       bitmap_iterator bi;
     583      1009003 :       unsigned i;
     584      1009003 :       gcc_assert (num < stack_vars_num);
     585     10929059 :       EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
     586      9920056 :         add_stack_var_conflict (num, i);
     587              :     }
     588     13944119 :   return false;
     589              : }
     590              : 
     591              : /* A cache for ssa name to address of stack variables.
     592              :    When taking into account if a ssa name refers to an
     593              :    address of a stack variable, we need to walk the
     594              :    expressions backwards to find the addresses. This
     595              :    cache is there so we don't need to walk the expressions
     596              :    all the time.  */
     597              : struct vars_ssa_cache
     598              : {
     599              : private:
     600              :   /* Currently an entry is a bitmap of all of the known stack variables
     601              :      addresses that are referenced by the ssa name.
     602              :      When the bitmap is the nullptr, then there is no cache.
     603              :      Currently only empty bitmaps are shared.
     604              :      The reason for why empty cache is not just a null is so we know the
     605              :      cache for an entry is filled in.  */
     606              :   struct entry
     607              :   {
     608              :     bitmap bmap = nullptr;
     609              :   };
     610              :   entry *vars_ssa_caches;
     611              : public:
     612              : 
     613              :   vars_ssa_cache();
     614              :   ~vars_ssa_cache();
     615              :   const_bitmap operator() (tree name);
     616              :   void dump (FILE *file);
     617              : 
     618              : private:
     619              :   /* Can't copy. */
     620              :   vars_ssa_cache(const vars_ssa_cache&) = delete;
     621              :   vars_ssa_cache(vars_ssa_cache&&) = delete;
     622              : 
     623              :   /* The shared empty bitmap.  */
     624              :   bitmap empty;
     625              : 
     626              :   /* Unshare the index, currently only need
     627              :      to unshare if the entry was empty. */
     628      1107831 :   void unshare(int indx)
     629              :   {
     630      1107831 :     if (vars_ssa_caches[indx].bmap == empty)
     631       475796 :         vars_ssa_caches[indx].bmap = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     632      1107831 :   }
     633              :   void create (tree);
     634              :   bool exists (tree use);
     635              :   void add_one (tree old_name, unsigned);
     636              :   bool update (tree old_name, tree use);
     637              : };
     638              : 
     639              : /* Constructor of the cache, create the cache array. */
     640       149743 : vars_ssa_cache::vars_ssa_cache ()
     641              : {
     642     33249107 :   vars_ssa_caches = new entry[num_ssa_names]{};
     643              : 
     644              :   /* Create the shared empty bitmap too. */
     645       149743 :   empty = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     646       149743 : }
     647              : 
     648              : /* Delete the array. The bitmaps will be freed
     649              :    when stack_var_bitmap_obstack is freed.  */
     650       149743 : vars_ssa_cache::~vars_ssa_cache ()
     651              : {
     652       149743 :   delete []vars_ssa_caches;
     653       149743 : }
     654              : 
     655              : /* Create an empty entry for the USE ssa name.  */
     656              : void
     657     10990337 : vars_ssa_cache::create (tree use)
     658              : {
     659     10990337 :   int num = SSA_NAME_VERSION (use);
     660     10990337 :   if (vars_ssa_caches[num].bmap)
     661              :     return;
     662     10990337 :   vars_ssa_caches[num].bmap = empty;
     663              : }
     664              : 
     665              : /* Returns true if the cache for USE exists.  */
     666              : bool
     667     71898247 : vars_ssa_cache::exists (tree use)
     668              : {
     669     71898247 :   int num = SSA_NAME_VERSION (use);
     670     71898247 :   return vars_ssa_caches[num].bmap != nullptr;
     671              : }
     672              : 
     673              : /* Add to USE's bitmap for stack variable IDX.  */
     674              : void
     675       164410 : vars_ssa_cache::add_one (tree use, unsigned idx)
     676              : {
     677       164410 :   gcc_assert (idx != INVALID_STACK_INDEX);
     678       164410 :   int num = SSA_NAME_VERSION (use);
     679       164410 :   gcc_assert (vars_ssa_caches[num].bmap);
     680       164410 :   unshare (num);
     681       164410 :   bitmap_set_bit (vars_ssa_caches[num].bmap, idx);
     682       164410 : }
     683              : 
     684              : /* Update cache of OLD_NAME from the USE's cache. */
     685              : bool
     686     25352441 : vars_ssa_cache::update (tree old_name, tree use)
     687              : {
     688     25352441 :   if (old_name == use)
     689              :     return false;
     690     16194167 :   int num = SSA_NAME_VERSION (use);
     691     16194167 :   int old_num = SSA_NAME_VERSION (old_name);
     692              : 
     693              :   /* If the old name was empty, then there is nothing to be updated. */
     694     16194167 :   if (vars_ssa_caches[num].bmap == empty)
     695              :     return false;
     696       943421 :   unshare (old_num);
     697       943421 :   return bitmap_ior_into (vars_ssa_caches[old_num].bmap, vars_ssa_caches[num].bmap);
     698              : }
     699              : 
     700              : /* Dump out the cache. Note empty and non-filled
     701              :    in ssa names are not printed out. */
     702              : void
     703            0 : vars_ssa_cache::dump (FILE *file)
     704              : {
     705            0 :   fprintf (file, "var ssa address cache\n");
     706            0 :   for (unsigned num = 0; num < num_ssa_names; num++)
     707              :     {
     708            0 :       if (!vars_ssa_caches[num].bmap
     709            0 :           || vars_ssa_caches[num].bmap == empty)
     710            0 :         continue;
     711            0 :       fprintf (file, "_%d refers to:\n", num);
     712            0 :       bitmap_iterator bi;
     713            0 :       unsigned i;
     714            0 :       EXECUTE_IF_SET_IN_BITMAP (vars_ssa_caches[num].bmap, 0, i, bi)
     715              :         {
     716            0 :           fputc ('\t', file);
     717            0 :           print_generic_expr (file, stack_vars[i].decl, dump_flags);
     718              :         }
     719            0 :       fputc ('\n', file);
     720              :   }
     721            0 :   fputc ('\n', file);
     722            0 : }
     723              : 
     724              : /* Returns the filled in cache for NAME.
     725              :    This will fill in the cache if it does not exist already.
     726              :    Returns an empty for ssa names that can't contain pointers
     727              :    (only integral types and pointer types will contain pointers).  */
     728              : 
     729              : const_bitmap
     730     56356229 : vars_ssa_cache::operator() (tree name)
     731              : {
     732     56356229 :   gcc_assert (TREE_CODE (name) == SSA_NAME);
     733              : 
     734     92728590 :   if (!POINTER_TYPE_P (TREE_TYPE (name))
     735     91558559 :       && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (name)))
     736      2502103 :     return empty;
     737              : 
     738     53854126 :   if (exists (name))
     739     44749098 :     return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
     740              : 
     741      9105028 :   auto_vec<std::pair<tree,tree>, 4> work_list;
     742      9105028 :   auto_vec<std::pair<tree,tree>, 4> update_cache_list;
     743              : 
     744      9105028 :   work_list.safe_push (std::make_pair (name, name));
     745              : 
     746     33542002 :   while (!work_list.is_empty ())
     747              :     {
     748     24436974 :       auto item = work_list.pop();
     749     24436974 :       tree use = item.first;
     750     24436974 :       tree old_name = item.second;
     751     24436974 :       if (TREE_CODE (use) == ADDR_EXPR)
     752              :         {
     753       417031 :           tree op = TREE_OPERAND (use, 0);
     754       417031 :           op = get_base_address (op);
     755       417031 :           unsigned idx = decl_stack_index (op);
     756       417031 :           if (idx != INVALID_STACK_INDEX)
     757       164410 :             add_one (old_name, idx);
     758     13446637 :           continue;
     759       417031 :         }
     760              : 
     761     24019943 :       if (TREE_CODE (use) != SSA_NAME)
     762      5897334 :         continue;
     763              : 
     764     31601204 :       if (!POINTER_TYPE_P (TREE_TYPE (use))
     765     31467383 :           && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (use)))
     766        78488 :         continue;
     767              : 
     768              :       /* Mark the old ssa name needs to be update from the use. */
     769     18044121 :       update_cache_list.safe_push (item);
     770              : 
     771              :       /* If the cache exists for the use, don't try to recreate it. */
     772     18044121 :       if (exists (use))
     773              :         {
     774              :           /* Update the cache here, this can reduce the number of
     775              :              times through the update loop below.  */
     776      7053784 :           update (old_name, use);
     777      7053784 :           continue;
     778              :         }
     779              : 
     780              :       /* Create the cache bitmap for the use and also
     781              :          so we don't go into an infinite loop for some phi nodes with loops.  */
     782     10990337 :       create (use);
     783              : 
     784     10990337 :       gimple *g = SSA_NAME_DEF_STMT (use);
     785              :  
     786              :       /* CONSTRUCTOR here is always a vector initialization,
     787              :          walk each element too. */
     788     10990337 :       if (gimple_assign_single_p (g)
     789     10990337 :           && TREE_CODE (gimple_assign_rhs1 (g)) == CONSTRUCTOR)
     790              :         {
     791              :           tree ctr = gimple_assign_rhs1 (g);
     792              :           unsigned i;
     793              :           tree elm;
     794       314745 :           FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctr), i, elm)
     795       221191 :             work_list.safe_push (std::make_pair (elm, use));
     796              :         }
     797              :       /* For assignments, walk each operand for possible addresses.
     798              :          For PHI nodes, walk each argument. */
     799     10896783 :       else if (gassign *a = dyn_cast <gassign *> (g))
     800              :         {
     801              :           /* operand 0 is the lhs. */
     802     19443359 :           for (unsigned i = 1; i < gimple_num_ops (g); i++)
     803     11229319 :             work_list.safe_push (std::make_pair (gimple_op (a, i), use));
     804              :         }
     805     12564532 :       else if (gphi *p = dyn_cast <gphi *> (g))
     806      5455631 :         for (unsigned i = 0; i < gimple_phi_num_args (p); ++i)
     807      3881436 :           work_list.safe_push (std::make_pair (gimple_phi_arg_def (p, i), use));
     808              :     }
     809              : 
     810              :   /* Update the cache. Note a loop is needed as phi nodes could
     811              :      cause a loop to form. The number of times through this loop
     812              :      will be small though.  */
     813      9158218 :   bool changed;
     814     18316436 :   do {
     815      9158218 :     changed = false;
     816      9158218 :     unsigned int i;
     817      9158218 :     std::pair<tree,tree> *e;
     818     45720121 :     FOR_EACH_VEC_ELT_REVERSE (update_cache_list, i, e)
     819              :       {
     820     18298657 :         if (update (e->second, e->first))
     821        59685 :           changed = true;
     822              :       }
     823              :   } while (changed);
     824              : 
     825      9105028 :   return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
     826      9105028 : }
     827              : 
     828              : /* Helper function for add_scope_conflicts_1.  For USE on
     829              :    a stmt, if it is a SSA_NAME and in its defining statement
     830              :    is known to be based on some ADDR_EXPR, invoke VISIT
     831              :    on that ADDR_EXPR.  */
     832              : 
     833              : static inline void
     834     56356229 : add_scope_conflicts_2 (vars_ssa_cache &cache, tree name,
     835              :                        bitmap work, walk_stmt_load_store_addr_fn visit)
     836              : {
     837     56356229 :   gcc_assert (TREE_CODE (name) == SSA_NAME);
     838              : 
     839              :   /* Query the cache for the mapping of addresses that are referenced by
     840              :      ssa name NAME.  Querying it will fill in it.  */
     841     56356229 :   bitmap_iterator bi;
     842     56356229 :   unsigned i;
     843     56356229 :   const_bitmap bmap = cache (name);
     844              :   /* Visit each stack variable that is referenced.  */
     845     58767850 :   EXECUTE_IF_SET_IN_BITMAP (bmap, 0, i, bi)
     846      2411621 :     visit (nullptr, stack_vars[i].decl, nullptr, work);
     847     56356229 : }
     848              : 
     849              : /* Helper routine for add_scope_conflicts, calculating the active partitions
     850              :    at the end of BB, leaving the result in WORK.  We're called to generate
     851              :    conflicts when FOR_CONFLICT is true, otherwise we're just tracking
     852              :    liveness.  */
     853              : 
     854              : static void
     855     16463613 : add_scope_conflicts_1 (vars_ssa_cache &cache, basic_block bb, bitmap work, bool for_conflict)
     856              : {
     857     16463613 :   edge e;
     858     16463613 :   edge_iterator ei;
     859     16463613 :   gimple_stmt_iterator gsi;
     860     16463613 :   walk_stmt_load_store_addr_fn visit;
     861     16463613 :   use_operand_p use_p;
     862     16463613 :   ssa_op_iter iter;
     863     16463613 :   bool had_non_clobbers = false;
     864              : 
     865     16463613 :   bitmap_clear (work);
     866              :   /* The copy what was alive out going from the edges. */
     867     41602893 :   FOR_EACH_EDGE (e, ei, bb->preds)
     868     25139280 :     bitmap_ior_into (work, (bitmap)e->src->aux);
     869              : 
     870     16463613 :   visit = for_conflict ? visit_conflict : visit_op;
     871              : 
     872              :   /* Addresses coming into the bb via phis are alive at the entry point. */
     873     21591835 :   for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
     874      5128222 :     add_scope_conflicts_2 (cache, gimple_phi_result (gsi_stmt (gsi)), work, visit);
     875              : 
     876    189120182 :   for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
     877              :     {
     878    172656569 :       gimple *stmt = gsi_stmt (gsi);
     879              : 
     880              :       /* Debug statements are not considered for liveness. */
     881    172656569 :       if (is_gimple_debug (stmt))
     882    108730146 :         continue;
     883              : 
     884              :       /* If we had `var = {CLOBBER}`, then var is no longer
     885              :          considered alive after this point but might become
     886              :          alive later on. */
     887     63926423 :       if (gimple_clobber_p (stmt))
     888              :         {
     889      3227961 :           tree lhs = gimple_assign_lhs (stmt);
     890              :           /* Handle only plain var clobbers, not partial ones.
     891              :              Nested functions lowering and C++ front-end inserts clobbers
     892              :              which are partial clobbers.  */
     893      3227961 :           if (!VAR_P (lhs))
     894        79582 :             continue;
     895      3148379 :           unsigned indx = decl_stack_index (lhs);
     896      3148379 :           if (indx != INVALID_STACK_INDEX)
     897      2590029 :             bitmap_clear_bit (work, indx);
     898              :         }
     899              :       else
     900              :         {
     901     60698462 :           if (for_conflict && !had_non_clobbers)
     902              :             {
     903              :               /* When we are inheriting live variables from our predecessors
     904              :                  through a CFG merge we might not see an actual mention of
     905              :                  the variables to record the appropriate conflict as defs/uses
     906              :                  might be through indirect stores/loads.  For this reason
     907              :                  we have to make sure each live variable conflicts with
     908              :                  each other.  When there's just a single predecessor the
     909              :                  set of conflicts is already up-to-date.
     910              :                  We perform this delayed at the first real instruction to
     911              :                  allow clobbers starting this block to remove variables from
     912              :                  the set of live variables.  */
     913      4969938 :               bitmap_iterator bi;
     914      4969938 :               unsigned i;
     915      4969938 :               if (EDGE_COUNT (bb->preds) > 1)
     916     12605495 :                 EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
     917              :                   {
     918     11157942 :                     class stack_var *a = &stack_vars[i];
     919     11157942 :                     if (!a->conflicts)
     920       254049 :                       a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     921     11157942 :                     bitmap_ior_into (a->conflicts, work);
     922              :                   }
     923      4969938 :               had_non_clobbers = true;
     924              :             }
     925     60698462 :           walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
     926    111926469 :           FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
     927     51228007 :             add_scope_conflicts_2 (cache, USE_FROM_PTR (use_p), work, visit);
     928              :         }
     929              :     }
     930              : 
     931              :   /* When there was no real instruction but there's a CFG merge we need
     932              :      to add the conflicts now.  */
     933     16463613 :   if (for_conflict && !had_non_clobbers && EDGE_COUNT (bb->preds) > 1)
     934              :     {
     935       129824 :       bitmap_iterator bi;
     936       129824 :       unsigned i;
     937       943343 :       EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
     938              :         {
     939       813519 :           class stack_var *a = &stack_vars[i];
     940       813519 :           if (!a->conflicts)
     941        22506 :             a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     942       813519 :           bitmap_ior_into (a->conflicts, work);
     943              :         }
     944              :     }
     945     16463613 : }
     946              : 
     947              : /* Generate stack partition conflicts between all partitions that are
     948              :    simultaneously live.  */
     949              : 
     950              : static void
     951       236836 : add_scope_conflicts (void)
     952              : {
     953              :   /* If there is only one variable, there is nothing to be done as
     954              :      there is only possible partition.  */
     955       236836 :   if (stack_vars_num == 1)
     956        87093 :     return;
     957              : 
     958       149743 :   basic_block bb;
     959       149743 :   bool changed;
     960       149743 :   bitmap work = BITMAP_ALLOC (NULL);
     961       149743 :   int *rpo;
     962       149743 :   int n_bbs;
     963              : 
     964       149743 :   vars_ssa_cache cache;
     965              : 
     966              :   /* We approximate the live range of a stack variable by taking the first
     967              :      mention of its name as starting point(s), and by the end-of-scope
     968              :      death clobber added by gimplify as ending point(s) of the range.
     969              :      This overapproximates in the case we for instance moved an address-taken
     970              :      operation upward, without also moving a dereference to it upwards.
     971              :      But it's conservatively correct as a variable never can hold values
     972              :      before its name is mentioned at least once.
     973              : 
     974              :      We then do a mostly classical bitmap liveness algorithm.  */
     975              : 
     976      5624003 :   FOR_ALL_BB_FN (bb, cfun)
     977      5474260 :     bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
     978              : 
     979       149743 :   rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
     980       149743 :   n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
     981              : 
     982       149743 :   changed = true;
     983       605295 :   while (changed)
     984              :     {
     985              :       int i;
     986              :       changed = false;
     987     11594648 :       for (i = 0; i < n_bbs; i++)
     988              :         {
     989     11288839 :           bitmap active;
     990     11288839 :           bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
     991     11288839 :           active = (bitmap)bb->aux;
     992     11288839 :           add_scope_conflicts_1 (cache, bb, work, false);
     993     11288839 :           if (bitmap_ior_into (active, work))
     994      4647572 :             changed = true;
     995              :         }
     996              :     }
     997              : 
     998      5324517 :   FOR_EACH_BB_FN (bb, cfun)
     999      5174774 :     add_scope_conflicts_1 (cache, bb, work, true);
    1000              : 
    1001       149743 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1002            0 :     cache.dump (dump_file);
    1003              : 
    1004       149743 :   free (rpo);
    1005       149743 :   BITMAP_FREE (work);
    1006      5624003 :   FOR_ALL_BB_FN (bb, cfun)
    1007      5474260 :     BITMAP_FREE (bb->aux);
    1008       149743 : }
    1009              : 
    1010              : /* A subroutine of partition_stack_vars.  A comparison function for qsort,
    1011              :    sorting an array of indices by the properties of the object.  */
    1012              : 
    1013              : static int
    1014     20406680 : stack_var_cmp (const void *a, const void *b)
    1015              : {
    1016     20406680 :   unsigned ia = *(const unsigned *)a;
    1017     20406680 :   unsigned ib = *(const unsigned *)b;
    1018     20406680 :   unsigned int aligna = stack_vars[ia].alignb;
    1019     20406680 :   unsigned int alignb = stack_vars[ib].alignb;
    1020     20406680 :   poly_int64 sizea = stack_vars[ia].size;
    1021     20406680 :   poly_int64 sizeb = stack_vars[ib].size;
    1022     20406680 :   tree decla = stack_vars[ia].decl;
    1023     20406680 :   tree declb = stack_vars[ib].decl;
    1024     20406680 :   bool largea, largeb;
    1025     20406680 :   unsigned int uida, uidb;
    1026              : 
    1027              :   /* Primary compare on "large" alignment.  Large comes first.  */
    1028     20406680 :   largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
    1029     20406680 :   largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
    1030     20406680 :   if (largea != largeb)
    1031            0 :     return (int)largeb - (int)largea;
    1032              : 
    1033              :   /* Secondary compare on size, decreasing  */
    1034     36752293 :   int diff = compare_sizes_for_sort (sizeb, sizea);
    1035      4715073 :   if (diff != 0)
    1036              :     return diff;
    1037              : 
    1038              :   /* Tertiary compare on true alignment, decreasing.  */
    1039     11630540 :   if (aligna < alignb)
    1040              :     return -1;
    1041     11325039 :   if (aligna > alignb)
    1042              :     return 1;
    1043              : 
    1044              :   /* Final compare on ID for sort stability, increasing.
    1045              :      Two SSA names are compared by their version, SSA names come before
    1046              :      non-SSA names, and two normal decls are compared by their DECL_UID.  */
    1047     11075940 :   if (TREE_CODE (decla) == SSA_NAME)
    1048              :     {
    1049        37938 :       if (TREE_CODE (declb) == SSA_NAME)
    1050        33214 :         uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
    1051              :       else
    1052              :         return -1;
    1053              :     }
    1054     11038002 :   else if (TREE_CODE (declb) == SSA_NAME)
    1055              :     return 1;
    1056              :   else
    1057     11032265 :     uida = DECL_UID (decla), uidb = DECL_UID (declb);
    1058     11065479 :   if (uida < uidb)
    1059              :     return 1;
    1060      5507928 :   if (uida > uidb)
    1061      5507928 :     return -1;
    1062              :   return 0;
    1063              : }
    1064              : 
    1065              : struct part_traits : unbounded_int_hashmap_traits <unsigned , bitmap> {};
    1066              : typedef hash_map<unsigned, bitmap, part_traits> part_hashmap;
    1067              : 
    1068              : /* If the points-to solution *PI points to variables that are in a partition
    1069              :    together with other variables add all partition members to the pointed-to
    1070              :    variables bitmap.  */
    1071              : 
    1072              : static void
    1073      1626949 : add_partitioned_vars_to_ptset (const pt_solution *pt,
    1074              :                                part_hashmap *decls_to_partitions,
    1075              :                                hash_set<bitmap> *visited, bitmap temp)
    1076              : {
    1077      1626949 :   bitmap_iterator bi;
    1078      1626949 :   unsigned i;
    1079      1626949 :   bitmap *part;
    1080              : 
    1081      1626949 :   if (pt->anything
    1082      1582549 :       || pt->vars == NULL
    1083              :       /* The pointed-to vars bitmap is shared, it is enough to
    1084              :          visit it once.  */
    1085      3209498 :       || visited->add (pt->vars))
    1086      1314696 :     return;
    1087              : 
    1088       312253 :   bitmap_clear (temp);
    1089              : 
    1090              :   /* By using a temporary bitmap to store all members of the partitions
    1091              :      we have to add we make sure to visit each of the partitions only
    1092              :      once.  */
    1093      1343529 :   EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
    1094      1031276 :     if ((!temp
    1095      1031276 :          || !bitmap_bit_p (temp, i))
    1096      1671903 :         && (part = decls_to_partitions->get (i)))
    1097       176670 :       bitmap_ior_into (temp, *part);
    1098       312253 :   if (!bitmap_empty_p (temp))
    1099       149536 :     bitmap_ior_into (pt->vars, temp);
    1100              : }
    1101              : 
    1102              : /* Update points-to sets based on partition info, so we can use them on RTL.
    1103              :    The bitmaps representing stack partitions will be saved until expand,
    1104              :    where partitioned decls used as bases in memory expressions will be
    1105              :    rewritten.
    1106              : 
    1107              :    It is not necessary to update TBAA info on accesses to the coalesced
    1108              :    storage since our memory model doesn't allow TBAA to be used for
    1109              :    WAW or WAR dependences.  For RAW when the write is to an old object
    1110              :    the new object would not have been initialized at the point of the
    1111              :    read, invoking undefined behavior.  */
    1112              : 
    1113              : static void
    1114       149743 : update_alias_info_with_stack_vars (void)
    1115              : {
    1116       149743 :   part_hashmap *decls_to_partitions = NULL;
    1117       149743 :   unsigned i, j;
    1118       149743 :   tree var = NULL_TREE;
    1119              : 
    1120      1280937 :   for (i = 0; i < stack_vars_num; i++)
    1121              :     {
    1122      1131194 :       bitmap part = NULL;
    1123      1131194 :       tree name;
    1124              : 
    1125              :       /* Not interested in partitions with single variable.  */
    1126      1131194 :       if (stack_vars[i].representative != i
    1127       856955 :           || stack_vars[i].next == EOC)
    1128      1058814 :         continue;
    1129              : 
    1130        72380 :       if (!decls_to_partitions)
    1131              :         {
    1132        43169 :           decls_to_partitions = new part_hashmap;
    1133        43169 :           cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
    1134              :         }
    1135              : 
    1136              :       /* Create an SSA_NAME that points to the partition for use
    1137              :          as base during alias-oracle queries on RTL for bases that
    1138              :          have been partitioned.  */
    1139        72380 :       if (var == NULL_TREE)
    1140        43169 :         var = create_tmp_var (ptr_type_node);
    1141        72380 :       name = make_ssa_name (var);
    1142              : 
    1143              :       /* Create bitmaps representing partitions.  They will be used for
    1144              :          points-to sets later, so use GGC alloc.  */
    1145        72380 :       part = BITMAP_GGC_ALLOC ();
    1146       418999 :       for (j = i; j != EOC; j = stack_vars[j].next)
    1147              :         {
    1148       346619 :           tree decl = stack_vars[j].decl;
    1149       346619 :           unsigned int uid = DECL_PT_UID (decl);
    1150       346619 :           bitmap_set_bit (part, uid);
    1151       346619 :           decls_to_partitions->put (uid, part);
    1152       346619 :           cfun->gimple_df->decls_to_pointers->put (decl, name);
    1153       346619 :           if (TREE_ADDRESSABLE (decl))
    1154       316884 :             TREE_ADDRESSABLE (name) = 1;
    1155              :         }
    1156              : 
    1157              :       /* Make the SSA name point to all partition members.  */
    1158        72380 :       pt_solution pt;
    1159        72380 :       pt_solution_set (&pt, part, false);
    1160        72380 :       set_ssa_name_pt_solution (name, pt);
    1161              :     }
    1162              : 
    1163              :   /* Make all points-to sets that contain one member of a partition
    1164              :      contain all members of the partition.  */
    1165       149743 :   if (decls_to_partitions)
    1166              :     {
    1167        43169 :       unsigned i;
    1168        43169 :       tree name;
    1169        43169 :       hash_set<bitmap> visited;
    1170        43169 :       bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
    1171              : 
    1172     17079070 :       FOR_EACH_SSA_NAME (i, name, cfun)
    1173              :         {
    1174     27516528 :           const pt_solution *pt = ssa_name_pt_solution (name);
    1175              : 
    1176      1549678 :           if (pt)
    1177      1540611 :             add_partitioned_vars_to_ptset (pt, decls_to_partitions,
    1178              :                                            &visited, temp);
    1179              :         }
    1180              : 
    1181        43169 :       add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
    1182              :                                      decls_to_partitions, &visited, temp);
    1183        43169 :       add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped_return,
    1184              :                                      decls_to_partitions, &visited, temp);
    1185        43169 :       delete decls_to_partitions;
    1186        43169 :       BITMAP_FREE (temp);
    1187        43169 :     }
    1188       149743 : }
    1189              : 
    1190              : /* A subroutine of partition_stack_vars.  The UNION portion of a UNION/FIND
    1191              :    partitioning algorithm.  Partitions A and B are known to be non-conflicting.
    1192              :    Merge them into a single partition A.  */
    1193              : 
    1194              : static void
    1195       274239 : union_stack_vars (unsigned a, unsigned b)
    1196              : {
    1197       274239 :   class stack_var *vb = &stack_vars[b];
    1198       274239 :   bitmap_iterator bi;
    1199       274239 :   unsigned u;
    1200              : 
    1201       274239 :   gcc_assert (stack_vars[b].next == EOC);
    1202              :    /* Add B to A's partition.  */
    1203       274239 :   stack_vars[b].next = stack_vars[a].next;
    1204       274239 :   stack_vars[b].representative = a;
    1205       274239 :   stack_vars[a].next = b;
    1206              : 
    1207              :   /* Make sure A is big enough to hold B.  */
    1208       274239 :   stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
    1209              : 
    1210              :   /* Update the required alignment of partition A to account for B.  */
    1211       274239 :   if (stack_vars[a].alignb < stack_vars[b].alignb)
    1212         1813 :     stack_vars[a].alignb = stack_vars[b].alignb;
    1213              : 
    1214              :   /* Update the interference graph and merge the conflicts.  */
    1215       274239 :   if (vb->conflicts)
    1216              :     {
    1217      2015232 :       EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
    1218      1790240 :         add_stack_var_conflict (a, stack_vars[u].representative);
    1219       224992 :       BITMAP_FREE (vb->conflicts);
    1220              :     }
    1221       274239 : }
    1222              : 
    1223              : /* A subroutine of expand_used_vars.  Binpack the variables into
    1224              :    partitions constrained by the interference graph.  The overall
    1225              :    algorithm used is as follows:
    1226              : 
    1227              :         Sort the objects by size in descending order.
    1228              :         For each object A {
    1229              :           S = size(A)
    1230              :           O = 0
    1231              :           loop {
    1232              :             Look for the largest non-conflicting object B with size <= S.
    1233              :             UNION (A, B)
    1234              :           }
    1235              :         }
    1236              : */
    1237              : 
    1238              : static void
    1239       236836 : partition_stack_vars (void)
    1240              : {
    1241       236836 :   unsigned si, sj, n = stack_vars_num;
    1242              : 
    1243       236836 :   stack_vars_sorted = XNEWVEC (unsigned, stack_vars_num);
    1244      1455123 :   for (si = 0; si < n; ++si)
    1245      1218287 :     stack_vars_sorted[si] = si;
    1246              : 
    1247       236836 :   if (n == 1)
    1248              :     return;
    1249              : 
    1250       149743 :   qsort (stack_vars_sorted, n, sizeof (unsigned), stack_var_cmp);
    1251              : 
    1252      1430680 :   for (si = 0; si < n; ++si)
    1253              :     {
    1254      1131194 :       unsigned i = stack_vars_sorted[si];
    1255      1131194 :       unsigned int ialign = stack_vars[i].alignb;
    1256      1131194 :       poly_int64 isize = stack_vars[i].size;
    1257              : 
    1258              :       /* Ignore objects that aren't partition representatives. If we
    1259              :          see a var that is not a partition representative, it must
    1260              :          have been merged earlier.  */
    1261      1131194 :       if (stack_vars[i].representative != i)
    1262       274239 :         continue;
    1263              : 
    1264     13169656 :       for (sj = si + 1; sj < n; ++sj)
    1265              :         {
    1266     12313948 :           unsigned j = stack_vars_sorted[sj];
    1267     12313948 :           unsigned int jalign = stack_vars[j].alignb;
    1268     12313948 :           poly_int64 jsize = stack_vars[j].size;
    1269              : 
    1270              :           /* Ignore objects that aren't partition representatives.  */
    1271     12313948 :           if (stack_vars[j].representative != j)
    1272     12312701 :             continue;
    1273              : 
    1274              :           /* Do not mix objects of "small" (supported) alignment
    1275              :              and "large" (unsupported) alignment.  */
    1276     11559928 :           if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
    1277     11559928 :               != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
    1278              :             break;
    1279              : 
    1280              :           /* For Address Sanitizer do not mix objects with different
    1281              :              sizes, as the shorter vars wouldn't be adequately protected.
    1282              :              Don't do that for "large" (unsupported) alignment objects,
    1283              :              those aren't protected anyway.  */
    1284     11559928 :           if (asan_sanitize_stack_p ()
    1285         6753 :               && maybe_ne (isize, jsize)
    1286     11561175 :               && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
    1287              :             break;
    1288              : 
    1289              :           /* Ignore conflicting objects.  */
    1290     11558681 :           if (stack_var_conflict_p (i, j))
    1291     11284442 :             continue;
    1292              : 
    1293              :           /* UNION the objects, placing J at OFFSET.  */
    1294       274239 :           union_stack_vars (i, j);
    1295              :         }
    1296              :     }
    1297              : 
    1298       149743 :   update_alias_info_with_stack_vars ();
    1299              : }
    1300              : 
    1301              : /* A debugging aid for expand_used_vars.  Dump the generated partitions.  */
    1302              : 
    1303              : static void
    1304           41 : dump_stack_var_partition (void)
    1305              : {
    1306           41 :   unsigned si, i, j, n = stack_vars_num;
    1307              : 
    1308           95 :   for (si = 0; si < n; ++si)
    1309              :     {
    1310           54 :       i = stack_vars_sorted[si];
    1311              : 
    1312              :       /* Skip variables that aren't partition representatives, for now.  */
    1313           54 :       if (stack_vars[i].representative != i)
    1314            3 :         continue;
    1315              : 
    1316           51 :       fprintf (dump_file, "Partition %u: size ", i);
    1317           51 :       print_dec (stack_vars[i].size, dump_file);
    1318           51 :       fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
    1319              : 
    1320          156 :       for (j = i; j != EOC; j = stack_vars[j].next)
    1321              :         {
    1322           54 :           fputc ('\t', dump_file);
    1323           54 :           print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
    1324              :         }
    1325           51 :       fputc ('\n', dump_file);
    1326              :     }
    1327           41 : }
    1328              : 
    1329              : /* Assign rtl to DECL at BASE + OFFSET.  */
    1330              : 
    1331              : static void
    1332      2036615 : expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
    1333              :                          poly_int64 offset)
    1334              : {
    1335      2036615 :   unsigned align;
    1336      2036615 :   rtx x;
    1337              : 
    1338              :   /* If this fails, we've overflowed the stack frame.  Error nicely?  */
    1339      2312415 :   gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
    1340              : 
    1341      2036615 :   if (hwassist_sanitize_stack_p ())
    1342           86 :     x = targetm.memtag.add_tag (base, offset,
    1343              :                                 hwasan_current_frame_tag ());
    1344              :   else
    1345      2312329 :     x = plus_constant (Pmode, base, offset);
    1346              : 
    1347      4073230 :   x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
    1348       459817 :                    ? TYPE_MODE (TREE_TYPE (decl))
    1349      1576798 :                    : DECL_MODE (decl), x);
    1350              : 
    1351              :   /* Set alignment we actually gave this decl if it isn't an SSA name.
    1352              :      If it is we generate stack slots only accidentally so it isn't as
    1353              :      important, we'll simply set the alignment directly on the MEM.  */
    1354              : 
    1355      2036615 :   if (stack_vars_base_reg_p (base))
    1356      2033223 :     offset -= frame_phase;
    1357      2036615 :   align = known_alignment (offset);
    1358      2036615 :   align *= BITS_PER_UNIT;
    1359      2036615 :   if (align == 0 || align > base_align)
    1360       827457 :     align = base_align;
    1361              : 
    1362      2036615 :   if (TREE_CODE (decl) != SSA_NAME)
    1363              :     {
    1364              :       /* One would think that we could assert that we're not decreasing
    1365              :          alignment here, but (at least) the i386 port does exactly this
    1366              :          via the MINIMUM_ALIGNMENT hook.  */
    1367              : 
    1368      1576798 :       SET_DECL_ALIGN (decl, align);
    1369      1576798 :       DECL_USER_ALIGN (decl) = 0;
    1370              :     }
    1371              : 
    1372      2036615 :   set_rtl (decl, x);
    1373              : 
    1374      2036615 :   set_mem_align (x, align);
    1375      2036615 : }
    1376              : 
    1377       236836 : class stack_vars_data
    1378              : {
    1379              : public:
    1380              :   /* Vector of offset pairs, always end of some padding followed
    1381              :      by start of the padding that needs Address Sanitizer protection.
    1382              :      The vector is in reversed, highest offset pairs come first.  */
    1383              :   auto_vec<HOST_WIDE_INT> asan_vec;
    1384              : 
    1385              :   /* Vector of partition representative decls in between the paddings.  */
    1386              :   auto_vec<tree> asan_decl_vec;
    1387              : 
    1388              :   /* Base pseudo register for Address Sanitizer protected automatic vars.  */
    1389              :   rtx asan_base;
    1390              : 
    1391              :   /* Alignment needed for the Address Sanitizer protected automatic vars.  */
    1392              :   unsigned int asan_alignb;
    1393              : };
    1394              : 
    1395              : /* A subroutine of expand_used_vars.  Give each partition representative
    1396              :    a unique location within the stack frame.  Update each partition member
    1397              :    with that location.  */
    1398              : static void
    1399       239043 : expand_stack_vars (bool (*pred) (unsigned), class stack_vars_data *data)
    1400              : {
    1401       239043 :   unsigned si, i, j, n = stack_vars_num;
    1402       239043 :   poly_uint64 large_size = 0, large_alloc = 0;
    1403       239043 :   rtx large_base = NULL;
    1404       239043 :   rtx large_untagged_base = NULL;
    1405       239043 :   unsigned large_align = 0;
    1406       239043 :   bool large_allocation_done = false;
    1407       239043 :   tree decl;
    1408              : 
    1409              :   /* Determine if there are any variables requiring "large" alignment.
    1410              :      Since these are dynamically allocated, we only process these if
    1411              :      no predicate involved.  */
    1412       239043 :   large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
    1413       239043 :   if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1414              :     {
    1415              :       /* Find the total size of these variables.  */
    1416            0 :       for (si = 0; si < n; ++si)
    1417              :         {
    1418            0 :           unsigned alignb;
    1419              : 
    1420            0 :           i = stack_vars_sorted[si];
    1421            0 :           alignb = stack_vars[i].alignb;
    1422              : 
    1423              :           /* All "large" alignment decls come before all "small" alignment
    1424              :              decls, but "large" alignment decls are not sorted based on
    1425              :              their alignment.  Increase large_align to track the largest
    1426              :              required alignment.  */
    1427            0 :           if ((alignb * BITS_PER_UNIT) > large_align)
    1428              :             large_align = alignb * BITS_PER_UNIT;
    1429              : 
    1430              :           /* Stop when we get to the first decl with "small" alignment.  */
    1431            0 :           if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
    1432              :             break;
    1433              : 
    1434              :           /* Skip variables that aren't partition representatives.  */
    1435            0 :           if (stack_vars[i].representative != i)
    1436            0 :             continue;
    1437              : 
    1438              :           /* Skip variables that have already had rtl assigned.  See also
    1439              :              add_stack_var where we perpetrate this pc_rtx hack.  */
    1440            0 :           decl = stack_vars[i].decl;
    1441            0 :           if (TREE_CODE (decl) == SSA_NAME
    1442            0 :               ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
    1443            0 :               : DECL_RTL (decl) != pc_rtx)
    1444            0 :             continue;
    1445              : 
    1446            0 :           large_size = aligned_upper_bound (large_size, alignb);
    1447            0 :           large_size += stack_vars[i].size;
    1448              :         }
    1449              :     }
    1450              : 
    1451      1462214 :   for (si = 0; si < n; ++si)
    1452              :     {
    1453      1223171 :       rtx base;
    1454      1223171 :       unsigned base_align, alignb;
    1455      1223171 :       poly_int64 offset = 0;
    1456              : 
    1457      1223171 :       i = stack_vars_sorted[si];
    1458              : 
    1459              :       /* Skip variables that aren't partition representatives, for now.  */
    1460      1223171 :       if (stack_vars[i].representative != i)
    1461       274250 :         continue;
    1462              : 
    1463              :       /* Skip variables that have already had rtl assigned.  See also
    1464              :          add_stack_var where we perpetrate this pc_rtx hack.  */
    1465       948921 :       decl = stack_vars[i].decl;
    1466       948921 :       if (TREE_CODE (decl) == SSA_NAME
    1467       948921 :           ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
    1468       944959 :           : DECL_RTL (decl) != pc_rtx)
    1469         3696 :         continue;
    1470              : 
    1471              :       /* Check the predicate to see whether this variable should be
    1472              :          allocated in this pass.  */
    1473       945225 :       if (pred && !pred (i))
    1474         1177 :         continue;
    1475              : 
    1476       944048 :       base = (hwassist_sanitize_stack_p ()
    1477       944048 :               ? hwasan_frame_base ()
    1478       943992 :               : virtual_stack_vars_rtx);
    1479       944048 :       alignb = stack_vars[i].alignb;
    1480       944048 :       if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
    1481              :         {
    1482       944048 :           poly_int64 hwasan_orig_offset;
    1483       944048 :           if (hwassist_sanitize_stack_p ())
    1484              :             {
    1485              :               /* There must be no tag granule "shared" between different
    1486              :                  objects.  This means that no HWASAN_TAG_GRANULE_SIZE byte
    1487              :                  chunk can have more than one object in it.
    1488              : 
    1489              :                  We ensure this by forcing the end of the last bit of data to
    1490              :                  be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
    1491              :                  the start of each variable to be aligned to
    1492              :                  HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
    1493              : 
    1494              :                  We can't align just one of the start or end, since there are
    1495              :                  untagged things stored on the stack which we do not align to
    1496              :                  HWASAN_TAG_GRANULE_SIZE bytes.  If we only aligned the start
    1497              :                  or the end of tagged objects then untagged objects could end
    1498              :                  up sharing the first granule of a tagged object or sharing the
    1499              :                  last granule of a tagged object respectively.  */
    1500           56 :               hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
    1501           56 :               gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
    1502              :             }
    1503              :           /* ASAN description strings don't yet have a syntax for expressing
    1504              :              polynomial offsets.  */
    1505       944048 :           HOST_WIDE_INT prev_offset;
    1506       944048 :           if (asan_sanitize_stack_p ()
    1507         4263 :               && pred
    1508         3387 :               && frame_offset.is_constant (&prev_offset)
    1509       944048 :               && stack_vars[i].size.is_constant ())
    1510              :             {
    1511         3387 :               if (data->asan_vec.is_empty ())
    1512              :                 {
    1513         1738 :                   align_frame_offset (ASAN_RED_ZONE_SIZE);
    1514         1738 :                   prev_offset = frame_offset.to_constant ();
    1515              :                 }
    1516         3387 :               prev_offset = align_base (prev_offset,
    1517              :                                         ASAN_MIN_RED_ZONE_SIZE,
    1518              :                                         !FRAME_GROWS_DOWNWARD);
    1519         3387 :               tree repr_decl = NULL_TREE;
    1520         3387 :               unsigned HOST_WIDE_INT size
    1521         3387 :                 = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
    1522         3387 :               if (data->asan_vec.is_empty ())
    1523         1738 :                 size = MAX (size, ASAN_RED_ZONE_SIZE);
    1524              : 
    1525         3387 :               unsigned HOST_WIDE_INT alignment = MAX (alignb,
    1526              :                                                       ASAN_MIN_RED_ZONE_SIZE);
    1527         3387 :               offset = alloc_stack_frame_space (size, alignment);
    1528              : 
    1529         3387 :               data->asan_vec.safe_push (prev_offset);
    1530              :               /* Allocating a constant amount of space from a constant
    1531              :                  starting offset must give a constant result.  */
    1532         3387 :               data->asan_vec.safe_push ((offset + stack_vars[i].size)
    1533         3387 :                                         .to_constant ());
    1534              :               /* Find best representative of the partition.
    1535              :                  Prefer those with DECL_NAME, even better
    1536              :                  satisfying asan_protect_stack_decl predicate.  */
    1537         3949 :               for (j = i; j != EOC; j = stack_vars[j].next)
    1538         3391 :                 if (asan_protect_stack_decl (stack_vars[j].decl)
    1539         3391 :                     && DECL_NAME (stack_vars[j].decl))
    1540              :                   {
    1541         2829 :                     repr_decl = stack_vars[j].decl;
    1542         2829 :                     break;
    1543              :                   }
    1544          562 :                 else if (repr_decl == NULL_TREE
    1545          562 :                          && DECL_P (stack_vars[j].decl)
    1546         1124 :                          && DECL_NAME (stack_vars[j].decl))
    1547            0 :                   repr_decl = stack_vars[j].decl;
    1548         3387 :               if (repr_decl == NULL_TREE)
    1549          558 :                 repr_decl = stack_vars[i].decl;
    1550         3387 :               data->asan_decl_vec.safe_push (repr_decl);
    1551              : 
    1552              :               /* Make sure a representative is unpoison if another
    1553              :                  variable in the partition is handled by
    1554              :                  use-after-scope sanitization.  */
    1555         3387 :               if (asan_handled_variables != NULL
    1556         3387 :                   && !asan_handled_variables->contains (repr_decl))
    1557              :                 {
    1558          440 :                   for (j = i; j != EOC; j = stack_vars[j].next)
    1559          222 :                     if (asan_handled_variables->contains (stack_vars[j].decl))
    1560              :                       break;
    1561          220 :                   if (j != EOC)
    1562            2 :                     asan_handled_variables->add (repr_decl);
    1563              :                 }
    1564              : 
    1565         3387 :               data->asan_alignb = MAX (data->asan_alignb, alignb);
    1566         3387 :               if (data->asan_base == NULL)
    1567         1738 :                 data->asan_base = gen_reg_rtx (Pmode);
    1568         3387 :               base = data->asan_base;
    1569              : 
    1570         3387 :               if (!STRICT_ALIGNMENT)
    1571         3387 :                 base_align = crtl->max_used_stack_slot_alignment;
    1572              :               else
    1573              :                 base_align = MAX (crtl->max_used_stack_slot_alignment,
    1574              :                                   GET_MODE_ALIGNMENT (SImode)
    1575              :                                   << ASAN_SHADOW_SHIFT);
    1576              :             }
    1577              :           else
    1578              :             {
    1579       940661 :               offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
    1580       940661 :               base_align = crtl->max_used_stack_slot_alignment;
    1581              : 
    1582       940661 :               if (hwassist_sanitize_stack_p ())
    1583              :                 {
    1584              :                   /* Align again since the point of this alignment is to handle
    1585              :                      the "end" of the object (i.e. smallest address after the
    1586              :                      stack object).  For FRAME_GROWS_DOWNWARD that requires
    1587              :                      aligning the stack before allocating, but for a frame that
    1588              :                      grows upwards that requires aligning the stack after
    1589              :                      allocation.
    1590              : 
    1591              :                      Use `frame_offset` to record the offset value rather than
    1592              :                      `offset` since the `frame_offset` describes the extent
    1593              :                      allocated for this particular variable while `offset`
    1594              :                      describes the address that this variable starts at.  */
    1595           56 :                   align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
    1596           56 :                   hwasan_record_stack_var (virtual_stack_vars_rtx, base,
    1597              :                                            hwasan_orig_offset, frame_offset);
    1598              :                 }
    1599              :             }
    1600              :         }
    1601              :       else
    1602              :         {
    1603              :           /* Large alignment is only processed in the last pass.  */
    1604            0 :           if (pred)
    1605            0 :             continue;
    1606              : 
    1607              :           /* If there were any variables requiring "large" alignment, allocate
    1608              :              space.  */
    1609            0 :           if (maybe_ne (large_size, 0U) && ! large_allocation_done)
    1610              :             {
    1611            0 :               poly_int64 loffset;
    1612            0 :               rtx large_allocsize;
    1613              : 
    1614            0 :               large_allocsize = gen_int_mode (large_size, Pmode);
    1615            0 :               get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
    1616            0 :               loffset = alloc_stack_frame_space
    1617            0 :                 (rtx_to_poly_int64 (large_allocsize),
    1618            0 :                  PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
    1619            0 :               large_base = get_dynamic_stack_base (loffset, large_align, base);
    1620            0 :               large_allocation_done = true;
    1621              :             }
    1622              : 
    1623            0 :           gcc_assert (large_base != NULL);
    1624            0 :           large_alloc = aligned_upper_bound (large_alloc, alignb);
    1625            0 :           offset = large_alloc;
    1626            0 :           large_alloc += stack_vars[i].size;
    1627            0 :           if (hwassist_sanitize_stack_p ())
    1628              :             {
    1629              :               /* An object with a large alignment requirement means that the
    1630              :                  alignment requirement is greater than the required alignment
    1631              :                  for tags.  */
    1632            0 :               if (!large_untagged_base)
    1633            0 :                 large_untagged_base
    1634            0 :                   = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
    1635              :               /* Ensure the end of the variable is also aligned correctly.  */
    1636            0 :               poly_int64 align_again
    1637            0 :                 = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
    1638              :               /* For large allocations we always allocate a chunk of space
    1639              :                  (which is addressed by large_untagged_base/large_base) and
    1640              :                  then use positive offsets from that.  Hence the farthest
    1641              :                  offset is `align_again` and the nearest offset from the base
    1642              :                  is `offset`.  */
    1643            0 :               hwasan_record_stack_var (large_untagged_base, large_base,
    1644              :                                        offset, align_again);
    1645              :             }
    1646              : 
    1647              :           base = large_base;
    1648              :           base_align = large_align;
    1649              :         }
    1650              : 
    1651              :       /* Create rtl for each variable based on their location within the
    1652              :          partition.  */
    1653      2162335 :       for (j = i; j != EOC; j = stack_vars[j].next)
    1654              :         {
    1655      1218287 :           expand_one_stack_var_at (stack_vars[j].decl,
    1656              :                                    base, base_align, offset);
    1657              :         }
    1658       944048 :       if (hwassist_sanitize_stack_p ())
    1659           56 :         hwasan_increment_frame_tag ();
    1660              :     }
    1661              : 
    1662       239043 :   gcc_assert (known_eq (large_alloc, large_size));
    1663       239043 : }
    1664              : 
    1665              : /* Take into account all sizes of partitions and reset DECL_RTLs.  */
    1666              : static poly_uint64
    1667      1090389 : account_stack_vars (void)
    1668              : {
    1669      1090389 :   unsigned si, j, i, n = stack_vars_num;
    1670      1090389 :   poly_uint64 size = 0;
    1671              : 
    1672      5724054 :   for (si = 0; si < n; ++si)
    1673              :     {
    1674      4633665 :       i = stack_vars_sorted[si];
    1675              : 
    1676              :       /* Skip variables that aren't partition representatives, for now.  */
    1677      4633665 :       if (stack_vars[i].representative != i)
    1678            0 :         continue;
    1679              : 
    1680      9267330 :       size += stack_vars[i].size;
    1681      9267330 :       for (j = i; j != EOC; j = stack_vars[j].next)
    1682      4633665 :         set_rtl (stack_vars[j].decl, NULL);
    1683              :     }
    1684      1090389 :   return size;
    1685              : }
    1686              : 
    1687              : /* Record the RTL assignment X for the default def of PARM.  */
    1688              : 
    1689              : extern void
    1690      4774793 : set_parm_rtl (tree parm, rtx x)
    1691              : {
    1692      4774793 :   gcc_assert (TREE_CODE (parm) == PARM_DECL
    1693              :               || TREE_CODE (parm) == RESULT_DECL);
    1694              : 
    1695      4774793 :   if (x && !MEM_P (x))
    1696              :     {
    1697      3081114 :       unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
    1698              :                                               TYPE_MODE (TREE_TYPE (parm)),
    1699              :                                               TYPE_ALIGN (TREE_TYPE (parm)));
    1700              : 
    1701              :       /* If the variable alignment is very large we'll dynamically
    1702              :          allocate it, which means that in-frame portion is just a
    1703              :          pointer.  ??? We've got a pseudo for sure here, do we
    1704              :          actually dynamically allocate its spilling area if needed?
    1705              :          ??? Isn't it a problem when Pmode alignment also exceeds
    1706              :          MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32?  */
    1707      3081114 :       if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1708            0 :         align = GET_MODE_ALIGNMENT (Pmode);
    1709              : 
    1710      3081114 :       record_alignment_for_reg_var (align);
    1711              :     }
    1712              : 
    1713      4774793 :   tree ssa = ssa_default_def (cfun, parm);
    1714      4774793 :   if (!ssa)
    1715      1087614 :     return set_rtl (parm, x);
    1716              : 
    1717      3687179 :   int part = var_to_partition (SA.map, ssa);
    1718      3687179 :   gcc_assert (part != NO_PARTITION);
    1719              : 
    1720      3687179 :   bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
    1721      3687179 :   gcc_assert (changed);
    1722              : 
    1723      3687179 :   set_rtl (ssa, x);
    1724      3687179 :   gcc_assert (DECL_RTL (parm) == x);
    1725              : }
    1726              : 
    1727              : /* A subroutine of expand_one_var.  Called to immediately assign rtl
    1728              :    to a variable to be allocated in the stack frame.  */
    1729              : 
    1730              : static void
    1731       818328 : expand_one_stack_var_1 (tree var)
    1732              : {
    1733       818328 :   poly_uint64 size;
    1734       818328 :   poly_int64 offset;
    1735       818328 :   unsigned byte_align;
    1736              : 
    1737       818328 :   if (TREE_CODE (var) == SSA_NAME)
    1738              :     {
    1739       456579 :       tree type = TREE_TYPE (var);
    1740       456579 :       size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
    1741              :     }
    1742              :   else
    1743       361749 :     size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
    1744              : 
    1745       818328 :   byte_align = align_local_variable (var, true);
    1746              : 
    1747              :   /* We handle highly aligned variables in expand_stack_vars.  */
    1748       818328 :   gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
    1749              : 
    1750       818328 :   rtx base;
    1751       818328 :   if (hwassist_sanitize_stack_p ())
    1752              :     {
    1753              :       /* Allocate zero bytes to align the stack.  */
    1754           30 :       poly_int64 hwasan_orig_offset
    1755           30 :         = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
    1756           30 :       offset = alloc_stack_frame_space (size, byte_align);
    1757           30 :       align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
    1758           30 :       base = hwasan_frame_base ();
    1759              :       /* Use `frame_offset` to automatically account for machines where the
    1760              :          frame grows upwards.
    1761              : 
    1762              :          `offset` will always point to the "start" of the stack object, which
    1763              :          will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
    1764              :          the "furthest" offset from the base delimiting the current stack
    1765              :          object.  `frame_offset` will always delimit the extent that the frame.
    1766              :          */
    1767           30 :       hwasan_record_stack_var (virtual_stack_vars_rtx, base,
    1768              :                                hwasan_orig_offset, frame_offset);
    1769              :     }
    1770              :   else
    1771              :     {
    1772       818298 :       offset = alloc_stack_frame_space (size, byte_align);
    1773       818298 :       base = virtual_stack_vars_rtx;
    1774              :     }
    1775              : 
    1776       818328 :   expand_one_stack_var_at (var, base,
    1777              :                            crtl->max_used_stack_slot_alignment, offset);
    1778              : 
    1779       818328 :   if (hwassist_sanitize_stack_p ())
    1780           30 :     hwasan_increment_frame_tag ();
    1781       818328 : }
    1782              : 
    1783              : /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
    1784              :    already assigned some MEM.  */
    1785              : 
    1786              : static void
    1787       361749 : expand_one_stack_var (tree var)
    1788              : {
    1789       361749 :   if (TREE_CODE (var) == SSA_NAME)
    1790              :     {
    1791            0 :       int part = var_to_partition (SA.map, var);
    1792            0 :       if (part != NO_PARTITION)
    1793              :         {
    1794            0 :           rtx x = SA.partition_to_pseudo[part];
    1795            0 :           gcc_assert (x);
    1796            0 :           gcc_assert (MEM_P (x));
    1797              :           return;
    1798              :         }
    1799              :     }
    1800              : 
    1801       361749 :   return expand_one_stack_var_1 (var);
    1802              : }
    1803              : 
    1804              : /* A subroutine of expand_one_var.  Called to assign rtl to a VAR_DECL
    1805              :    that will reside in a hard register.  */
    1806              : 
    1807              : static void
    1808         1110 : expand_one_hard_reg_var (tree var)
    1809              : {
    1810            0 :   rest_of_decl_compilation (var, 0, 0);
    1811            0 : }
    1812              : 
    1813              : /* Record the alignment requirements of some variable assigned to a
    1814              :    pseudo.  */
    1815              : 
    1816              : static void
    1817     41806985 : record_alignment_for_reg_var (unsigned int align)
    1818              : {
    1819     41806985 :   if (SUPPORTS_STACK_ALIGNMENT
    1820     41806985 :       && crtl->stack_alignment_estimated < align)
    1821              :     {
    1822              :       /* stack_alignment_estimated shouldn't change after stack
    1823              :          realign decision made */
    1824      1749830 :       gcc_assert (!crtl->stack_realign_processed);
    1825      1749830 :       crtl->stack_alignment_estimated = align;
    1826              :     }
    1827              : 
    1828              :   /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
    1829              :      So here we only make sure stack_alignment_needed >= align.  */
    1830     41806985 :   if (crtl->stack_alignment_needed < align)
    1831       565665 :     crtl->stack_alignment_needed = align;
    1832     41806985 :   if (crtl->max_used_stack_slot_alignment < align)
    1833       565665 :     crtl->max_used_stack_slot_alignment = align;
    1834     41806985 : }
    1835              : 
    1836              : /* Create RTL for an SSA partition.  */
    1837              : 
    1838              : static void
    1839     23024637 : expand_one_ssa_partition (tree var)
    1840              : {
    1841     23024637 :   int part = var_to_partition (SA.map, var);
    1842     23024637 :   gcc_assert (part != NO_PARTITION);
    1843              : 
    1844     23024637 :   if (SA.partition_to_pseudo[part])
    1845              :     return;
    1846              : 
    1847     23024637 :   unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
    1848              :                                           TYPE_MODE (TREE_TYPE (var)),
    1849              :                                           TYPE_ALIGN (TREE_TYPE (var)));
    1850              : 
    1851              :   /* If the variable alignment is very large we'll dynamically allocate
    1852              :      it, which means that in-frame portion is just a pointer.  */
    1853     23024637 :   if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1854            0 :     align = GET_MODE_ALIGNMENT (Pmode);
    1855              : 
    1856     23024637 :   record_alignment_for_reg_var (align);
    1857              : 
    1858     23024637 :   if (!use_register_for_decl (var))
    1859              :     {
    1860       459817 :       if (defer_stack_allocation (var, true))
    1861         3238 :         add_stack_var (var, true);
    1862              :       else
    1863       456579 :         expand_one_stack_var_1 (var);
    1864              :       return;
    1865              :     }
    1866              : 
    1867     22564820 :   machine_mode reg_mode = promote_ssa_mode (var, NULL);
    1868     22564820 :   rtx x = gen_reg_rtx (reg_mode);
    1869              : 
    1870     22564820 :   set_rtl (var, x);
    1871              : 
    1872              :   /* For a promoted variable, X will not be used directly but wrapped in a
    1873              :      SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
    1874              :      will assume that its upper bits can be inferred from its lower bits.
    1875              :      Therefore, if X isn't initialized on every path from the entry, then
    1876              :      we must do it manually in order to fulfill the above assumption.  */
    1877     22564820 :   if (reg_mode != TYPE_MODE (TREE_TYPE (var))
    1878     22564820 :       && bitmap_bit_p (SA.partitions_for_undefined_values, part))
    1879            0 :     emit_move_insn (x, CONST0_RTX (reg_mode));
    1880              : }
    1881              : 
    1882              : /* Record the association between the RTL generated for partition PART
    1883              :    and the underlying variable of the SSA_NAME VAR.  */
    1884              : 
    1885              : static void
    1886     50203415 : adjust_one_expanded_partition_var (tree var)
    1887              : {
    1888     50203415 :   if (!var)
    1889              :     return;
    1890              : 
    1891     50203415 :   tree decl = SSA_NAME_VAR (var);
    1892              : 
    1893     50203415 :   int part = var_to_partition (SA.map, var);
    1894     50203415 :   if (part == NO_PARTITION)
    1895              :     return;
    1896              : 
    1897     32247359 :   rtx x = SA.partition_to_pseudo[part];
    1898              : 
    1899     32247359 :   gcc_assert (x);
    1900              : 
    1901     32247359 :   set_rtl (var, x);
    1902              : 
    1903     32247359 :   if (!REG_P (x))
    1904              :     return;
    1905              : 
    1906              :   /* Note if the object is a user variable.  */
    1907     30185590 :   if (decl && !DECL_ARTIFICIAL (decl))
    1908      4209148 :     mark_user_reg (x);
    1909              : 
    1910     30185590 :   if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
    1911      7714390 :     mark_reg_pointer (x, get_pointer_alignment (var));
    1912              : }
    1913              : 
    1914              : /* A subroutine of expand_one_var.  Called to assign rtl to a VAR_DECL
    1915              :    that will reside in a pseudo register.  */
    1916              : 
    1917              : static void
    1918       586610 : expand_one_register_var (tree var)
    1919              : {
    1920       586610 :   if (TREE_CODE (var) == SSA_NAME)
    1921              :     {
    1922            0 :       int part = var_to_partition (SA.map, var);
    1923            0 :       if (part != NO_PARTITION)
    1924              :         {
    1925            0 :           rtx x = SA.partition_to_pseudo[part];
    1926            0 :           gcc_assert (x);
    1927            0 :           gcc_assert (REG_P (x));
    1928              :           return;
    1929              :         }
    1930            0 :       gcc_unreachable ();
    1931              :     }
    1932              : 
    1933       586610 :   tree decl = var;
    1934       586610 :   tree type = TREE_TYPE (decl);
    1935       586610 :   machine_mode reg_mode = promote_decl_mode (decl, NULL);
    1936       586610 :   rtx x = gen_reg_rtx (reg_mode);
    1937              : 
    1938       586610 :   set_rtl (var, x);
    1939              : 
    1940              :   /* Note if the object is a user variable.  */
    1941       586610 :   if (!DECL_ARTIFICIAL (decl))
    1942        94577 :     mark_user_reg (x);
    1943              : 
    1944       586610 :   if (POINTER_TYPE_P (type))
    1945          100 :     mark_reg_pointer (x, get_pointer_alignment (var));
    1946              : }
    1947              : 
    1948              : /* A subroutine of expand_one_var.  Called to assign rtl to a VAR_DECL that
    1949              :    has some associated error, e.g. its type is error-mark.  We just need
    1950              :    to pick something that won't crash the rest of the compiler.  */
    1951              : 
    1952              : static void
    1953           33 : expand_one_error_var (tree var)
    1954              : {
    1955           33 :   machine_mode mode = DECL_MODE (var);
    1956           33 :   rtx x;
    1957              : 
    1958           33 :   if (mode == BLKmode)
    1959            1 :     x = gen_rtx_MEM (BLKmode, const0_rtx);
    1960           32 :   else if (mode == VOIDmode)
    1961            0 :     x = const0_rtx;
    1962              :   else
    1963           32 :     x = gen_reg_rtx (mode);
    1964              : 
    1965           33 :   SET_DECL_RTL (var, x);
    1966           33 : }
    1967              : 
    1968              : /* A subroutine of expand_one_var.  VAR is a variable that will be
    1969              :    allocated to the local stack frame.  Return true if we wish to
    1970              :    add VAR to STACK_VARS so that it will be coalesced with other
    1971              :    variables.  Return false to allocate VAR immediately.
    1972              : 
    1973              :    This function is used to reduce the number of variables considered
    1974              :    for coalescing, which reduces the size of the quadratic problem.  */
    1975              : 
    1976              : static bool
    1977      6809219 : defer_stack_allocation (tree var, bool toplevel)
    1978              : {
    1979      6809219 :   tree size_unit = TREE_CODE (var) == SSA_NAME
    1980      6809219 :     ? TYPE_SIZE_UNIT (TREE_TYPE (var))
    1981      6809219 :     : DECL_SIZE_UNIT (var);
    1982      6809219 :   poly_uint64 size;
    1983              : 
    1984              :   /* Whether the variable is small enough for immediate allocation not to be
    1985              :      a problem with regard to the frame size.  */
    1986      6809219 :   bool smallish
    1987      6809219 :     = (poly_int_tree_p (size_unit, &size)
    1988      6809219 :        && (estimated_poly_value (size)
    1989      6809219 :            < param_min_size_for_stack_sharing));
    1990              : 
    1991              :   /* If stack protection is enabled, *all* stack variables must be deferred,
    1992              :      so that we can re-order the strings to the top of the frame.
    1993              :      Similarly for Address Sanitizer.  */
    1994      6809219 :   if (flag_stack_protect || asan_sanitize_stack_p ())
    1995              :     return true;
    1996              : 
    1997              :   /* Use the same effective alignment that expand_one_stack_var_1 would use.
    1998              :      If that alignment exceeds MAX_SUPPORTED_STACK_ALIGNMENT, defer
    1999              :      the variable so that expand_stack_vars handles its large alignment,
    2000              :      rather than calling expand_one_stack_var_1 and failing its assertion.  */
    2001      6794654 :   unsigned int align = align_local_variable (var, false) * BITS_PER_UNIT;
    2002              : 
    2003              :   /* We handle "large" alignment via dynamic allocation.  We want to handle
    2004              :      this extra complication in only one place, so defer them.  */
    2005      6794654 :   if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
    2006              :     return true;
    2007              : 
    2008      6794654 :   bool ignored = TREE_CODE (var) == SSA_NAME
    2009      6801124 :     ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
    2010      6335564 :     : DECL_IGNORED_P (var);
    2011              : 
    2012              :   /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
    2013              :      might be detached from their block and appear at toplevel when we reach
    2014              :      here.  We want to coalesce them with variables from other blocks when
    2015              :      the immediate contribution to the frame size would be noticeable.  */
    2016      6794654 :   if (toplevel && optimize > 0 && ignored && !smallish)
    2017              :     return true;
    2018              : 
    2019              :   /* Variables declared in the outermost scope automatically conflict
    2020              :      with every other variable.  The only reason to want to defer them
    2021              :      at all is that, after sorting, we can more efficiently pack
    2022              :      small variables in the stack frame.  Continue to defer at -O2.  */
    2023      5255871 :   if (toplevel && optimize < 2)
    2024              :     return false;
    2025              : 
    2026              :   /* Without optimization, *most* variables are allocated from the
    2027              :      stack, which makes the quadratic problem large exactly when we
    2028              :      want compilation to proceed as quickly as possible.  On the
    2029              :      other hand, we don't want the function's stack frame size to
    2030              :      get completely out of hand.  So we avoid adding scalars and
    2031              :      "small" aggregates to the list at all.  */
    2032      4597808 :   if (optimize == 0 && smallish)
    2033        49566 :     return false;
    2034              : 
    2035              :   return true;
    2036              : }
    2037              : 
    2038              : /* A subroutine of expand_used_vars.  Expand one variable according to
    2039              :    its flavor.  Variables to be placed on the stack are not actually
    2040              :    expanded yet, merely recorded.
    2041              :    When REALLY_EXPAND is false, only add stack values to be allocated.
    2042              :    Return stack usage this variable is supposed to take.
    2043              : */
    2044              : 
    2045              : static poly_uint64
    2046     16007763 : expand_one_var (tree var, bool toplevel, bool really_expand,
    2047              :                 bitmap forced_stack_var = NULL)
    2048              : {
    2049     16007763 :   unsigned int align = BITS_PER_UNIT;
    2050     16007763 :   tree origvar = var;
    2051              : 
    2052     16007763 :   var = SSAVAR (var);
    2053              : 
    2054     16007763 :   if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
    2055              :     {
    2056     16007763 :       if (is_global_var (var))
    2057       306529 :         return 0;
    2058              : 
    2059              :       /* Because we don't know if VAR will be in register or on stack,
    2060              :          we conservatively assume it will be on stack even if VAR is
    2061              :          eventually put into register after RA pass.  For non-automatic
    2062              :          variables, which won't be on stack, we collect alignment of
    2063              :          type and ignore user specified alignment.  Similarly for
    2064              :          SSA_NAMEs for which use_register_for_decl returns true.  */
    2065     15701234 :       if (TREE_STATIC (var)
    2066     15701234 :           || DECL_EXTERNAL (var)
    2067     31402468 :           || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
    2068            0 :         align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
    2069              :                                    TYPE_MODE (TREE_TYPE (var)),
    2070              :                                    TYPE_ALIGN (TREE_TYPE (var)));
    2071     15701234 :       else if (DECL_HAS_VALUE_EXPR_P (var)
    2072     15701234 :                || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
    2073              :         /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
    2074              :            or variables which were assigned a stack slot already by
    2075              :            expand_one_stack_var_at - in the latter case DECL_ALIGN has been
    2076              :            changed from the offset chosen to it.  */
    2077        17565 :         align = crtl->stack_alignment_estimated;
    2078              :       else
    2079     15683669 :         align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
    2080              : 
    2081              :       /* If the variable alignment is very large we'll dynamically allocate
    2082              :          it, which means that in-frame portion is just a pointer.  */
    2083     15701234 :       if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
    2084            0 :         align = GET_MODE_ALIGNMENT (Pmode);
    2085              :     }
    2086              : 
    2087     15701234 :   record_alignment_for_reg_var (align);
    2088              : 
    2089     15701234 :   poly_uint64 size;
    2090     15701234 :   if (TREE_CODE (origvar) == SSA_NAME)
    2091              :     {
    2092            0 :       gcc_assert (!VAR_P (var)
    2093              :                   || (!DECL_EXTERNAL (var)
    2094              :                       && !DECL_HAS_VALUE_EXPR_P (var)
    2095              :                       && !TREE_STATIC (var)
    2096              :                       && TREE_TYPE (var) != error_mark_node
    2097              :                       && !DECL_HARD_REGISTER (var)
    2098              :                       && really_expand));
    2099              :     }
    2100     15701234 :   if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
    2101              :     ;
    2102     15701234 :   else if (DECL_EXTERNAL (var))
    2103              :     ;
    2104     15701234 :   else if (DECL_HAS_VALUE_EXPR_P (var))
    2105              :     ;
    2106     15683674 :   else if (TREE_STATIC (var))
    2107              :     ;
    2108     15683674 :   else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
    2109              :     ;
    2110     15683669 :   else if (TREE_TYPE (var) == error_mark_node)
    2111              :     {
    2112            0 :       if (really_expand)
    2113            0 :         expand_one_error_var (var);
    2114              :     }
    2115     15683669 :   else if (VAR_P (var) && DECL_HARD_REGISTER (var))
    2116              :     {
    2117         4158 :       if (really_expand)
    2118              :         {
    2119         1110 :           expand_one_hard_reg_var (var);
    2120         1110 :           if (!DECL_HARD_REGISTER (var))
    2121              :             /* Invalid register specification.  */
    2122           33 :             expand_one_error_var (var);
    2123              :         }
    2124              :     }
    2125     15679511 :   else if (use_register_for_decl (var)
    2126     15679511 :            && (!forced_stack_var
    2127       595494 :                || !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
    2128              :     {
    2129      9330109 :       if (really_expand)
    2130       586610 :         expand_one_register_var (origvar);
    2131              :     }
    2132      6349402 :   else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
    2133      6349402 :            || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
    2134              :     {
    2135              :       /* Reject variables which cover more than half of the address-space.  */
    2136            0 :       if (really_expand)
    2137              :         {
    2138            0 :           if (DECL_NONLOCAL_FRAME (var))
    2139            0 :             error_at (DECL_SOURCE_LOCATION (current_function_decl),
    2140              :                       "total size of local objects is too large");
    2141              :           else
    2142            0 :             error_at (DECL_SOURCE_LOCATION (var),
    2143              :                       "size of variable %q+D is too large", var);
    2144            0 :           expand_one_error_var (var);
    2145              :         }
    2146              :     }
    2147      6349402 :   else if (defer_stack_allocation (var, toplevel))
    2148      5848714 :     add_stack_var (origvar, really_expand);
    2149              :   else
    2150              :     {
    2151       500688 :       if (really_expand)
    2152              :         {
    2153       361453 :           if (lookup_attribute ("naked",
    2154       361453 :                                 DECL_ATTRIBUTES (current_function_decl)))
    2155            0 :             error ("cannot allocate stack for variable %q+D, naked function",
    2156              :                    var);
    2157              : 
    2158       361453 :           expand_one_stack_var (origvar);
    2159              :         }
    2160       500688 :       return size;
    2161              :     }
    2162     15200546 :   return 0;
    2163              : }
    2164              : 
    2165              : /* A subroutine of expand_used_vars.  Walk down through the BLOCK tree
    2166              :    expanding variables.  Those variables that can be put into registers
    2167              :    are allocated pseudos; those that can't are put on the stack.
    2168              : 
    2169              :    TOPLEVEL is true if this is the outermost BLOCK.  */
    2170              : 
    2171              : static void
    2172     17239902 : expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
    2173              : {
    2174     17239902 :   tree t;
    2175              : 
    2176              :   /* Expand all variables at this level.  */
    2177     37563527 :   for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
    2178     20323625 :     if (TREE_USED (t)
    2179     20323625 :         && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
    2180       779333 :             || !DECL_NONSHAREABLE (t)))
    2181       779333 :       expand_one_var (t, toplevel, true, forced_stack_vars);
    2182              : 
    2183              :   /* Expand all variables at containing levels.  */
    2184     32961896 :   for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
    2185     15721994 :     expand_used_vars_for_block (t, false, forced_stack_vars);
    2186     17239902 : }
    2187              : 
    2188              : /* A subroutine of expand_used_vars.  Walk down through the BLOCK tree
    2189              :    and clear TREE_USED on all local variables.  */
    2190              : 
    2191              : static void
    2192     17239902 : clear_tree_used (tree block)
    2193              : {
    2194     17239902 :   tree t;
    2195              : 
    2196     37563527 :   for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
    2197              :     /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
    2198      1122202 :     if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
    2199     20323625 :         || !DECL_NONSHAREABLE (t))
    2200     20323625 :       TREE_USED (t) = 0;
    2201              : 
    2202     32961896 :   for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
    2203     15721994 :     clear_tree_used (t);
    2204     17239902 : }
    2205              : 
    2206              : /* Examine TYPE and determine a bit mask of the following features.  */
    2207              : 
    2208              : #define SPCT_HAS_LARGE_CHAR_ARRAY       1
    2209              : #define SPCT_HAS_SMALL_CHAR_ARRAY       2
    2210              : #define SPCT_HAS_ARRAY                  4
    2211              : #define SPCT_HAS_AGGREGATE              8
    2212              : 
    2213              : static unsigned int
    2214         2744 : stack_protect_classify_type (tree type)
    2215              : {
    2216         2744 :   unsigned int ret = 0;
    2217         2744 :   tree t;
    2218              : 
    2219         2744 :   switch (TREE_CODE (type))
    2220              :     {
    2221          708 :     case ARRAY_TYPE:
    2222          708 :       t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
    2223          708 :       if (t == char_type_node
    2224          178 :           || t == signed_char_type_node
    2225          178 :           || t == unsigned_char_type_node)
    2226              :         {
    2227          530 :           unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
    2228          530 :           unsigned HOST_WIDE_INT len;
    2229              : 
    2230          530 :           if (!TYPE_SIZE_UNIT (type)
    2231          530 :               || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
    2232              :             len = max;
    2233              :           else
    2234          530 :             len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
    2235              : 
    2236          530 :           if (len < max)
    2237              :             ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
    2238              :           else
    2239          498 :             ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
    2240              :         }
    2241              :       else
    2242              :         ret = SPCT_HAS_ARRAY;
    2243              :       break;
    2244              : 
    2245         1021 :     case UNION_TYPE:
    2246         1021 :     case QUAL_UNION_TYPE:
    2247         1021 :     case RECORD_TYPE:
    2248         1021 :       ret = SPCT_HAS_AGGREGATE;
    2249        10915 :       for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
    2250         9894 :         if (TREE_CODE (t) == FIELD_DECL)
    2251         1702 :           ret |= stack_protect_classify_type (TREE_TYPE (t));
    2252              :       break;
    2253              : 
    2254              :     default:
    2255              :       break;
    2256              :     }
    2257              : 
    2258         2744 :   return ret;
    2259              : }
    2260              : 
    2261              : /* Return nonzero if DECL should be segregated into the "vulnerable" upper
    2262              :    part of the local stack frame.  Remember if we ever return nonzero for
    2263              :    any variable in this function.  The return value is the phase number in
    2264              :    which the variable should be allocated.  */
    2265              : 
    2266              : static int
    2267         1042 : stack_protect_decl_phase (tree decl)
    2268              : {
    2269         1042 :   unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
    2270         1042 :   int ret = 0;
    2271              : 
    2272         1042 :   if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
    2273           28 :     has_short_buffer = true;
    2274              : 
    2275         1042 :   tree attribs = DECL_ATTRIBUTES (current_function_decl);
    2276         1042 :   if (!lookup_attribute ("no_stack_protector", attribs)
    2277         1042 :       && (flag_stack_protect == SPCT_FLAG_ALL
    2278         1042 :           || flag_stack_protect == SPCT_FLAG_STRONG
    2279          215 :           || (flag_stack_protect == SPCT_FLAG_EXPLICIT
    2280            5 :               && lookup_attribute ("stack_protect", attribs))))
    2281              :     {
    2282          832 :       if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
    2283          368 :           && !(bits & SPCT_HAS_AGGREGATE))
    2284              :         ret = 1;
    2285          752 :       else if (bits & SPCT_HAS_ARRAY)
    2286              :         ret = 2;
    2287              :     }
    2288              :   else
    2289          210 :     ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
    2290              : 
    2291          210 :   if (ret)
    2292          690 :     has_protected_decls = true;
    2293              : 
    2294         1042 :   return ret;
    2295              : }
    2296              : 
    2297              : /* Two helper routines that check for phase 1 and phase 2.  These are used
    2298              :    as callbacks for expand_stack_vars.  */
    2299              : 
    2300              : static bool
    2301          353 : stack_protect_decl_phase_1 (unsigned i)
    2302              : {
    2303          353 :   return stack_protect_decl_phase (stack_vars[i].decl) == 1;
    2304              : }
    2305              : 
    2306              : static bool
    2307          217 : stack_protect_decl_phase_2 (unsigned i)
    2308              : {
    2309          217 :   return stack_protect_decl_phase (stack_vars[i].decl) == 2;
    2310              : }
    2311              : 
    2312              : /* And helper function that checks for asan phase (with stack protector
    2313              :    it is phase 3).  This is used as callback for expand_stack_vars.
    2314              :    Returns true if any of the vars in the partition need to be protected.  */
    2315              : 
    2316              : static bool
    2317         4178 : asan_decl_phase_3 (unsigned i)
    2318              : {
    2319         5054 :   while (i != EOC)
    2320              :     {
    2321         4178 :       if (asan_protect_stack_decl (stack_vars[i].decl))
    2322              :         return true;
    2323          876 :       i = stack_vars[i].next;
    2324              :     }
    2325              :   return false;
    2326              : }
    2327              : 
    2328              : /* Ensure that variables in different stack protection phases conflict
    2329              :    so that they are not merged and share the same stack slot.
    2330              :    Return true if there are any address taken variables.  */
    2331              : 
    2332              : static bool
    2333          261 : add_stack_protection_conflicts (void)
    2334              : {
    2335          261 :   unsigned i, j, n = stack_vars_num;
    2336          261 :   unsigned char *phase;
    2337          261 :   bool ret = false;
    2338              : 
    2339          261 :   phase = XNEWVEC (unsigned char, n);
    2340          994 :   for (i = 0; i < n; ++i)
    2341              :     {
    2342          472 :       phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
    2343          472 :       if (TREE_ADDRESSABLE (stack_vars[i].decl))
    2344          395 :         ret = true;
    2345              :     }
    2346              : 
    2347          733 :   for (i = 0; i < n; ++i)
    2348              :     {
    2349          472 :       unsigned char ph_i = phase[i];
    2350         1188 :       for (j = i + 1; j < n; ++j)
    2351          716 :         if (ph_i != phase[j])
    2352          329 :           add_stack_var_conflict (i, j);
    2353              :     }
    2354              : 
    2355          261 :   XDELETEVEC (phase);
    2356          261 :   return ret;
    2357              : }
    2358              : 
    2359              : /* Create a decl for the guard at the top of the stack frame.  */
    2360              : 
    2361              : static void
    2362          296 : create_stack_guard (void)
    2363              : {
    2364          296 :   tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
    2365              :                            VAR_DECL, NULL, ptr_type_node);
    2366          296 :   TREE_THIS_VOLATILE (guard) = 1;
    2367          296 :   TREE_USED (guard) = 1;
    2368          296 :   expand_one_stack_var (guard);
    2369          296 :   crtl->stack_protect_guard = guard;
    2370          296 : }
    2371              : 
    2372              : /* Prepare for expanding variables.  */
    2373              : static void
    2374      8035674 : init_vars_expansion (void)
    2375              : {
    2376              :   /* Conflict bitmaps, and a few related temporary bitmaps, go here.  */
    2377      8035674 :   bitmap_obstack_initialize (&stack_var_bitmap_obstack);
    2378              : 
    2379              :   /* A map from decl to stack partition.  */
    2380      8035674 :   decl_to_stack_part = new hash_map<tree, unsigned>;
    2381              : 
    2382              :   /* Initialize local stack smashing state.  */
    2383      8035674 :   has_protected_decls = false;
    2384      8035674 :   has_short_buffer = false;
    2385      8035674 :   if (hwassist_sanitize_stack_p ())
    2386         1112 :     hwasan_record_frame_init ();
    2387      8035674 : }
    2388              : 
    2389              : /* Free up stack variable graph data.  */
    2390              : static void
    2391      8035674 : fini_vars_expansion (void)
    2392              : {
    2393      8035674 :   bitmap_obstack_release (&stack_var_bitmap_obstack);
    2394      8035674 :   if (stack_vars)
    2395      1327225 :     XDELETEVEC (stack_vars);
    2396      8035674 :   if (stack_vars_sorted)
    2397      1327225 :     XDELETEVEC (stack_vars_sorted);
    2398      8035674 :   stack_vars = NULL;
    2399      8035674 :   stack_vars_sorted = NULL;
    2400      8035674 :   stack_vars_alloc = stack_vars_num = 0;
    2401     16071348 :   delete decl_to_stack_part;
    2402      8035674 :   decl_to_stack_part = NULL;
    2403      8035674 : }
    2404              : 
    2405              : /* Make a fair guess for the size of the stack frame of the function
    2406              :    in NODE.  This doesn't have to be exact, the result is only used in
    2407              :    the inline heuristics.  So we don't want to run the full stack var
    2408              :    packing algorithm (which is quadratic in the number of stack vars).
    2409              :    Instead, we calculate the total size of all stack vars.  This turns
    2410              :    out to be a pretty fair estimate -- packing of stack vars doesn't
    2411              :    happen very often.  */
    2412              : 
    2413              : HOST_WIDE_INT
    2414      6517766 : estimated_stack_frame_size (struct cgraph_node *node)
    2415              : {
    2416      6517766 :   poly_int64 size = 0;
    2417      6517766 :   unsigned i;
    2418      6517766 :   tree var;
    2419      6517766 :   struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
    2420              : 
    2421      6517766 :   push_cfun (fn);
    2422              : 
    2423      6517766 :   init_vars_expansion ();
    2424              : 
    2425     25985493 :   FOR_EACH_LOCAL_DECL (fn, i, var)
    2426     14119442 :     if (auto_var_in_fn_p (var, fn->decl))
    2427     13519448 :       size += expand_one_var (var, true, false);
    2428              : 
    2429      6517766 :   if (stack_vars_num > 0)
    2430              :     {
    2431              :       /* Fake sorting the stack vars for account_stack_vars ().  */
    2432      1090389 :       stack_vars_sorted = XNEWVEC (unsigned , stack_vars_num);
    2433      5724054 :       for (i = 0; i < stack_vars_num; ++i)
    2434      4633665 :         stack_vars_sorted[i] = i;
    2435      1090389 :       size += account_stack_vars ();
    2436              :     }
    2437              : 
    2438      6517766 :   fini_vars_expansion ();
    2439      6517766 :   pop_cfun ();
    2440      6517766 :   return estimated_poly_value (size);
    2441              : }
    2442              : 
    2443              : /* Check if the current function has calls that use a return slot.  */
    2444              : 
    2445              : static bool
    2446          418 : stack_protect_return_slot_p ()
    2447              : {
    2448          418 :   basic_block bb;
    2449              : 
    2450         1876 :   FOR_ALL_BB_FN (bb, cfun)
    2451         2974 :     for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    2452         4028 :          !gsi_end_p (gsi); gsi_next (&gsi))
    2453              :       {
    2454         2570 :         gimple *stmt = gsi_stmt (gsi);
    2455              :         /* This assumes that calls to internal-only functions never
    2456              :            use a return slot.  */
    2457         2570 :         if (is_gimple_call (stmt)
    2458          542 :             && !gimple_call_internal_p (stmt)
    2459         3095 :             && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
    2460          525 :                                   gimple_call_fndecl (stmt)))
    2461          418 :           return true;
    2462              :       }
    2463              :   return false;
    2464              : }
    2465              : 
    2466              : /* Verify that partitions which claim to be the same object really are at the
    2467              :    same address.  MEM_EXPR-based disambiguation identifies a location by a
    2468              :    MEM_EXPR base and an offset from it, so two stack slots carrying one
    2469              :    MEM_EXPR read as a single object, which lets an access to one be redirected
    2470              :    to the other.  out-of-SSA keeps them apart, see the comment above
    2471              :    split_overlapping_partition_decls.
    2472              : 
    2473              :    Call this once the RTL of every partition is final.  The partition of a
    2474              :    PARM_DECL or RESULT_DECL default definition is given the MEM_EXPR of any
    2475              :    other variable in it while its names are walked, and only gets the decl
    2476              :    back when its RTL is restored in pass_expand::execute.  */
    2477              : 
    2478              : static void
    2479      1517888 : verify_partition_mem_exprs (void)
    2480              : {
    2481      1517888 :   hash_map<tree, rtx> slots;
    2482     29747533 :   for (unsigned i = 0; i < num_var_partitions (SA.map); i++)
    2483              :     {
    2484     26711757 :       rtx x = SA.partition_to_pseudo[i];
    2485     27959741 :       if (!x || !MEM_P (x) || !MEM_EXPR (x))
    2486     25469973 :         continue;
    2487      1241784 :       bool existed;
    2488      1241784 :       rtx &known = slots.get_or_insert (MEM_EXPR (x), &existed);
    2489      1241784 :       if (!existed)
    2490      1241784 :         known = x;
    2491              :       else
    2492            0 :         gcc_assert (rtx_equal_p (XEXP (known, 0), XEXP (x, 0)));
    2493              :     }
    2494      1517888 : }
    2495              : 
    2496              : /* Expand all variables used in the function.  */
    2497              : 
    2498              : static rtx_insn *
    2499      1517908 : expand_used_vars (bitmap forced_stack_vars)
    2500              : {
    2501      1517908 :   tree var, outer_block = DECL_INITIAL (current_function_decl);
    2502      1517908 :   auto_vec<tree> maybe_local_decls;
    2503      1517908 :   rtx_insn *var_end_seq = NULL;
    2504      1517908 :   unsigned i;
    2505      1517908 :   unsigned len;
    2506      1517908 :   bool gen_stack_protect_signal = false;
    2507              : 
    2508              :   /* Compute the phase of the stack frame for this function.  */
    2509      1517908 :   {
    2510      1517908 :     int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
    2511      1517908 :     int off = targetm.starting_frame_offset () % align;
    2512      1517908 :     frame_phase = off ? align - off : 0;
    2513              :   }
    2514              : 
    2515              :   /* Set TREE_USED on all variables in the local_decls.  */
    2516     12081913 :   FOR_EACH_LOCAL_DECL (cfun, i, var)
    2517      8019405 :     TREE_USED (var) = 1;
    2518              :   /* Clear TREE_USED on all variables associated with a block scope.  */
    2519      1517908 :   clear_tree_used (DECL_INITIAL (current_function_decl));
    2520              : 
    2521      1517908 :   init_vars_expansion ();
    2522              : 
    2523      1517908 :   if (targetm.use_pseudo_pic_reg ())
    2524        81059 :     pic_offset_table_rtx = gen_reg_rtx (Pmode);
    2525              : 
    2526     28229724 :   for (i = 0; i < SA.map->num_partitions; i++)
    2527              :     {
    2528     26711816 :       if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
    2529      3687179 :         continue;
    2530              : 
    2531     23024637 :       tree var = partition_to_var (SA.map, i);
    2532              : 
    2533     46049274 :       gcc_assert (!virtual_operand_p (var));
    2534              : 
    2535     23024637 :       expand_one_ssa_partition (var);
    2536              :     }
    2537              : 
    2538      1517908 :   if (flag_stack_protect == SPCT_FLAG_STRONG)
    2539          418 :     gen_stack_protect_signal = stack_protect_return_slot_p ();
    2540              : 
    2541              :   /* At this point all variables on the local_decls with TREE_USED
    2542              :      set are not associated with any block scope.  Lay them out.  */
    2543              : 
    2544      1517908 :   len = vec_safe_length (cfun->local_decls);
    2545     10809613 :   FOR_EACH_LOCAL_DECL (cfun, i, var)
    2546              :     {
    2547      8019405 :       bool expand_now = false;
    2548              : 
    2549              :       /* Expanded above already.  */
    2550      8019405 :       if (is_gimple_reg (var))
    2551              :         {
    2552      5618872 :           TREE_USED (var) = 0;
    2553      5618872 :           goto next;
    2554              :         }
    2555              :       /* We didn't set a block for static or extern because it's hard
    2556              :          to tell the difference between a global variable (re)declared
    2557              :          in a local scope, and one that's really declared there to
    2558              :          begin with.  And it doesn't really matter much, since we're
    2559              :          not giving them stack space.  Expand them now.  */
    2560      2400533 :       else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
    2561              :         expand_now = true;
    2562              : 
    2563              :       /* Expand variables not associated with any block now.  Those created by
    2564              :          the optimizers could be live anywhere in the function.  Those that
    2565              :          could possibly have been scoped originally and detached from their
    2566              :          block will have their allocation deferred so we coalesce them with
    2567              :          others when optimization is enabled.  */
    2568      2181786 :       else if (TREE_USED (var))
    2569      1708982 :         expand_now = true;
    2570              : 
    2571              :       /* Finally, mark all variables on the list as used.  We'll use
    2572              :          this in a moment when we expand those associated with scopes.  */
    2573      2400533 :       TREE_USED (var) = 1;
    2574              : 
    2575      2400533 :       if (expand_now)
    2576      1708982 :         expand_one_var (var, true, true, forced_stack_vars);
    2577              : 
    2578       691551 :     next:
    2579      8019405 :       if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
    2580              :         {
    2581       445045 :           rtx rtl = DECL_RTL_IF_SET (var);
    2582              : 
    2583              :           /* Keep artificial non-ignored vars in cfun->local_decls
    2584              :              chain until instantiate_decls.  */
    2585       336660 :           if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
    2586        23280 :             add_local_decl (cfun, var);
    2587       108385 :           else if (rtl == NULL_RTX)
    2588              :             /* If rtl isn't set yet, which can happen e.g. with
    2589              :                -fstack-protector, retry before returning from this
    2590              :                function.  */
    2591       108385 :             maybe_local_decls.safe_push (var);
    2592              :         }
    2593              :     }
    2594              : 
    2595              :   /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
    2596              : 
    2597              :      +-----------------+-----------------+
    2598              :      | ...processed... | ...duplicates...|
    2599              :      +-----------------+-----------------+
    2600              :                        ^
    2601              :                        +-- LEN points here.
    2602              : 
    2603              :      We just want the duplicates, as those are the artificial
    2604              :      non-ignored vars that we want to keep until instantiate_decls.
    2605              :      Move them down and truncate the array.  */
    2606      1517908 :   if (!vec_safe_is_empty (cfun->local_decls))
    2607       857825 :     cfun->local_decls->block_remove (0, len);
    2608              : 
    2609              :   /* At this point, all variables within the block tree with TREE_USED
    2610              :      set are actually used by the optimized function.  Lay them out.  */
    2611      1517908 :   expand_used_vars_for_block (outer_block, true, forced_stack_vars);
    2612              : 
    2613      1517908 :   tree attribs = DECL_ATTRIBUTES (current_function_decl);
    2614      1517908 :   if (stack_vars_num > 0)
    2615              :     {
    2616       236836 :       bool has_addressable_vars = false;
    2617              : 
    2618       236836 :       add_scope_conflicts ();
    2619              : 
    2620              :       /* If stack protection is enabled, we don't share space between
    2621              :          vulnerable data and non-vulnerable data.  */
    2622       236836 :       if (flag_stack_protect != 0
    2623          261 :           && !lookup_attribute ("no_stack_protector", attribs)
    2624       237097 :           && (flag_stack_protect != SPCT_FLAG_EXPLICIT
    2625              :               || (flag_stack_protect == SPCT_FLAG_EXPLICIT
    2626            1 :                   && lookup_attribute ("stack_protect", attribs))))
    2627          261 :         has_addressable_vars = add_stack_protection_conflicts ();
    2628              : 
    2629       236836 :       if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
    2630       236836 :         gen_stack_protect_signal = true;
    2631              : 
    2632              :       /* Now that we have collected all stack variables, and have computed a
    2633              :          minimal interference graph, attempt to save some stack space.  */
    2634       236836 :       partition_stack_vars ();
    2635       236836 :       if (dump_file)
    2636           41 :         dump_stack_var_partition ();
    2637              :     }
    2638              : 
    2639              : 
    2640      1517908 :   if (!lookup_attribute ("no_stack_protector", attribs))
    2641      1517899 :     switch (flag_stack_protect)
    2642              :       {
    2643           51 :       case SPCT_FLAG_ALL:
    2644           51 :         create_stack_guard ();
    2645           51 :         break;
    2646              : 
    2647          418 :       case SPCT_FLAG_STRONG:
    2648          418 :         if (gen_stack_protect_signal
    2649          274 :             || cfun->calls_alloca
    2650          272 :             || has_protected_decls
    2651          681 :             || lookup_attribute ("stack_protect", attribs))
    2652          155 :           create_stack_guard ();
    2653              :         break;
    2654              : 
    2655          169 :       case SPCT_FLAG_DEFAULT:
    2656          169 :         if (cfun->calls_alloca
    2657          166 :             || has_protected_decls
    2658          258 :             || lookup_attribute ("stack_protect", attribs))
    2659           80 :           create_stack_guard ();
    2660              :         break;
    2661              : 
    2662           14 :       case SPCT_FLAG_EXPLICIT:
    2663           14 :         if (lookup_attribute ("stack_protect", attribs))
    2664           10 :           create_stack_guard ();
    2665              :         break;
    2666              : 
    2667              :       default:
    2668              :         break;
    2669              :       }
    2670              : 
    2671              :   /* Assign rtl to each variable based on these partitions.  */
    2672      1517908 :   if (stack_vars_num > 0)
    2673              :     {
    2674       236836 :       class stack_vars_data data;
    2675              : 
    2676       236836 :       data.asan_base = NULL_RTX;
    2677       236836 :       data.asan_alignb = 0;
    2678              : 
    2679              :       /* Reorder decls to be protected by iterating over the variables
    2680              :          array multiple times, and allocating out of each phase in turn.  */
    2681              :       /* ??? We could probably integrate this into the qsort we did
    2682              :          earlier, such that we naturally see these variables first,
    2683              :          and thus naturally allocate things in the right order.  */
    2684       236836 :       if (has_protected_decls)
    2685              :         {
    2686              :           /* Phase 1 contains only character arrays.  */
    2687          166 :           expand_stack_vars (stack_protect_decl_phase_1, &data);
    2688              : 
    2689              :           /* Phase 2 contains other kinds of arrays.  */
    2690          166 :           if (!lookup_attribute ("no_stack_protector", attribs)
    2691          166 :               && (flag_stack_protect == SPCT_FLAG_ALL
    2692          166 :                   || flag_stack_protect == SPCT_FLAG_STRONG
    2693           78 :                   || (flag_stack_protect == SPCT_FLAG_EXPLICIT
    2694            1 :                       && lookup_attribute ("stack_protect", attribs))))
    2695           89 :             expand_stack_vars (stack_protect_decl_phase_2, &data);
    2696              :         }
    2697              : 
    2698       236836 :       if (asan_sanitize_stack_p ())
    2699              :         /* Phase 3, any partitions that need asan protection
    2700              :            in addition to phase 1 and 2.  */
    2701         1952 :         expand_stack_vars (asan_decl_phase_3, &data);
    2702              : 
    2703              :       /* ASAN description strings don't yet have a syntax for expressing
    2704              :          polynomial offsets.  */
    2705       236836 :       HOST_WIDE_INT prev_offset;
    2706       238574 :       if (!data.asan_vec.is_empty ()
    2707         1738 :           && frame_offset.is_constant (&prev_offset))
    2708              :         {
    2709         1738 :           HOST_WIDE_INT offset, sz, redzonesz;
    2710         1738 :           redzonesz = ASAN_RED_ZONE_SIZE;
    2711         1738 :           sz = data.asan_vec[0] - prev_offset;
    2712         1738 :           data.asan_alignb = MAX (data.asan_alignb,
    2713              :                                   crtl->stack_alignment_needed
    2714              :                                   / BITS_PER_UNIT);
    2715         1738 :           if (data.asan_alignb > ASAN_RED_ZONE_SIZE
    2716         1738 :               && data.asan_alignb <= 4096
    2717           54 :               && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
    2718           54 :             redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
    2719           54 :                          & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
    2720              :           /* Allocating a constant amount of space from a constant
    2721              :              starting offset must give a constant result.  */
    2722         1738 :           offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
    2723         1738 :                     .to_constant ());
    2724         1738 :           data.asan_vec.safe_push (prev_offset);
    2725         1738 :           data.asan_vec.safe_push (offset);
    2726              :           /* Leave space for alignment if STRICT_ALIGNMENT.  */
    2727         1738 :           if (STRICT_ALIGNMENT)
    2728              :             alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
    2729              :                                       << ASAN_SHADOW_SHIFT)
    2730              :                                      / BITS_PER_UNIT, 1);
    2731              : 
    2732         1738 :           var_end_seq
    2733         3476 :             = asan_emit_stack_protection (virtual_stack_vars_rtx,
    2734              :                                           data.asan_base,
    2735              :                                           data.asan_alignb,
    2736              :                                           data.asan_vec.address (),
    2737              :                                           data.asan_decl_vec.address (),
    2738         1738 :                                           data.asan_vec.length ());
    2739              :         }
    2740              : 
    2741       236836 :       expand_stack_vars (NULL, &data);
    2742       236836 :     }
    2743              : 
    2744      1517908 :   if (hwassist_sanitize_stack_p ())
    2745          368 :     hwasan_emit_prologue ();
    2746      1517908 :   if (asan_sanitize_allocas_p () && cfun->calls_alloca)
    2747          178 :     var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
    2748              :                                               virtual_stack_vars_rtx,
    2749              :                                               var_end_seq);
    2750      3035092 :   else if ((hwasan_sanitize_allocas_p () || memtag_sanitize_p ())
    2751      1517730 :            && cfun->calls_alloca)
    2752              :     /* When using out-of-line instrumentation we only want to emit one function
    2753              :        call for clearing the tags in a region of shadow stack.  When there are
    2754              :        alloca calls in this frame we want to emit a call using the
    2755              :        virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
    2756              :        rtx we created in expand_stack_vars.  */
    2757            0 :     var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
    2758              :                                            virtual_stack_vars_rtx);
    2759      1517730 :   else if (hwassist_sanitize_stack_p ())
    2760              :     /* If no variables were stored on the stack, `hwasan_get_frame_extent`
    2761              :        will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
    2762              :        NULL (i.e. an empty sequence).  */
    2763          368 :     var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
    2764              :                                            virtual_stack_vars_rtx);
    2765              : 
    2766      1517908 :   fini_vars_expansion ();
    2767              : 
    2768              :   /* If there were any artificial non-ignored vars without rtl
    2769              :      found earlier, see if deferred stack allocation hasn't assigned
    2770              :      rtl to them.  */
    2771      1715571 :   FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
    2772              :     {
    2773       108385 :       rtx rtl = DECL_RTL_IF_SET (var);
    2774              : 
    2775              :       /* Keep artificial non-ignored vars in cfun->local_decls
    2776              :          chain until instantiate_decls.  */
    2777         5145 :       if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
    2778         4888 :         add_local_decl (cfun, var);
    2779              :     }
    2780              : 
    2781              :   /* If the target requires that FRAME_OFFSET be aligned, do it.  */
    2782      1517908 :   if (STACK_ALIGNMENT_NEEDED)
    2783              :     {
    2784      1517908 :       HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
    2785      1517908 :       if (FRAME_GROWS_DOWNWARD)
    2786      1517908 :         frame_offset = aligned_lower_bound (frame_offset, align);
    2787              :       else
    2788              :         frame_offset = aligned_upper_bound (frame_offset, align);
    2789              :     }
    2790              : 
    2791      1517908 :   return var_end_seq;
    2792      1517908 : }
    2793              : 
    2794              : 
    2795              : /* If we need to produce a detailed dump, print the tree representation
    2796              :    for STMT to the dump file.  SINCE is the last RTX after which the RTL
    2797              :    generated for STMT should have been appended.  */
    2798              : 
    2799              : static void
    2800     43076344 : maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
    2801              : {
    2802     43076344 :   if (dump_file && (dump_flags & TDF_DETAILS))
    2803              :     {
    2804          407 :       fprintf (dump_file, "\n;; ");
    2805          407 :       print_gimple_stmt (dump_file, stmt, 0,
    2806              :                          TDF_SLIM | (dump_flags & TDF_LINENO));
    2807          407 :       fprintf (dump_file, "\n");
    2808              : 
    2809          407 :       print_rtl (dump_file, since ? NEXT_INSN (since) : since);
    2810              :     }
    2811     43076344 : }
    2812              : 
    2813              : /* Temporary storage for BB_HEAD and BB_END of bbs until they are converted
    2814              :    to BB_RTL.  */
    2815              : static vec<std::pair <rtx_insn *, rtx_insn *>> head_end_for_bb;
    2816              : 
    2817              : /* Maps the blocks that do not contain tree labels to rtx labels.  */
    2818              : 
    2819              : static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
    2820              : 
    2821              : /* Returns the label_rtx expression for a label starting basic block BB.  */
    2822              : 
    2823              : static rtx_code_label *
    2824      7846191 : label_rtx_for_bb (basic_block bb)
    2825              : {
    2826      7846191 :   if (bb->flags & BB_RTL)
    2827            0 :     return block_label (bb);
    2828              : 
    2829      7846191 :   if ((unsigned) bb->index < head_end_for_bb.length ()
    2830      7846191 :       && head_end_for_bb[bb->index].first)
    2831              :     {
    2832      1598741 :       if (!LABEL_P (head_end_for_bb[bb->index].first))
    2833              :         {
    2834       972157 :           head_end_for_bb[bb->index].first
    2835      1944314 :             = emit_label_before (gen_label_rtx (),
    2836       972157 :                                  head_end_for_bb[bb->index].first);
    2837              :         }
    2838      1598741 :       return as_a <rtx_code_label *> (head_end_for_bb[bb->index].first);
    2839              :     }
    2840              : 
    2841      6247450 :   rtx_code_label **elt = lab_rtx_for_bb->get (bb);
    2842      6247450 :   if (elt)
    2843      1303808 :     return *elt;
    2844              : 
    2845              :   /* Find the tree label if it is present.  */
    2846      4943642 :   gimple_stmt_iterator gsi = gsi_start_bb (bb);
    2847      4943642 :   glabel *lab_stmt;
    2848      4943642 :   if (!gsi_end_p (gsi)
    2849      4796700 :       && (lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)))
    2850      5180973 :       && !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
    2851       237331 :     return jump_target_rtx (gimple_label_label (lab_stmt));
    2852              : 
    2853      4706311 :   rtx_code_label *l = gen_label_rtx ();
    2854      4706311 :   lab_rtx_for_bb->put (bb, l);
    2855      4706311 :   return l;
    2856              : }
    2857              : 
    2858              : 
    2859              : /* Wrapper around remove_edge during expansion.  */
    2860              : 
    2861              : void
    2862       128613 : expand_remove_edge (edge e)
    2863              : {
    2864       128613 :   if (current_ir_type () != IR_GIMPLE
    2865       128613 :       && (e->dest->flags & BB_RTL) == 0
    2866       189321 :       && !gimple_seq_empty_p (phi_nodes (e->dest)))
    2867         9169 :     remove_phi_args (e);
    2868       128613 :   remove_edge (e);
    2869       128613 : }
    2870              : 
    2871              : /* Split edge E during expansion and instead of creating a new
    2872              :    bb on that edge, add there BB.  FLAGS should be flags on the
    2873              :    new edge from BB to former E->dest.  */
    2874              : 
    2875              : static void
    2876      1920278 : expand_split_edge (edge e, basic_block bb, int flags)
    2877              : {
    2878      1920278 :   unsigned int dest_idx = e->dest_idx;
    2879      1920278 :   basic_block dest = e->dest;
    2880      1920278 :   redirect_edge_succ (e, bb);
    2881      1920278 :   e = make_single_succ_edge (bb, dest, flags);
    2882      1920278 :   if ((dest->flags & BB_RTL) == 0
    2883      1920278 :       && phi_nodes (dest)
    2884      2059514 :       && e->dest_idx != dest_idx)
    2885              :     {
    2886              :       /* If there are any PHI nodes on dest, swap the new succ edge
    2887              :          with the one moved into false_edge's former position, so that
    2888              :          PHI arguments don't need adjustment.  */
    2889        86305 :       edge e2 = EDGE_PRED (dest, dest_idx);
    2890        86305 :       std::swap (e->dest_idx, e2->dest_idx);
    2891        86305 :       std::swap (EDGE_PRED (dest, e->dest_idx),
    2892              :                  EDGE_PRED (dest, e2->dest_idx));
    2893              :     }
    2894      1920278 : }
    2895              : 
    2896              : 
    2897              : /* A subroutine of expand_gimple_cond.  Given E, a fallthrough edge
    2898              :    of a basic block where we just expanded the conditional at the end,
    2899              :    possibly clean up the CFG and instruction sequence.  LAST is the
    2900              :    last instruction before the just emitted jump sequence.  */
    2901              : 
    2902              : static void
    2903      5378338 : maybe_cleanup_end_of_block (edge e, rtx_insn *last)
    2904              : {
    2905              :   /* Special case: when jumpif decides that the condition is
    2906              :      trivial it emits an unconditional jump (and the necessary
    2907              :      barrier).  But we still have two edges, the fallthru one is
    2908              :      wrong.  purge_dead_edges would clean this up later.  Unfortunately
    2909              :      we have to insert insns (and split edges) before
    2910              :      find_many_sub_basic_blocks and hence before purge_dead_edges.
    2911              :      But splitting edges might create new blocks which depend on the
    2912              :      fact that if there are two edges there's no barrier.  So the
    2913              :      barrier would get lost and verify_flow_info would ICE.  Instead
    2914              :      of auditing all edge splitters to care for the barrier (which
    2915              :      normally isn't there in a cleaned CFG), fix it here.  */
    2916      5378338 :   if (BARRIER_P (get_last_insn ()))
    2917              :     {
    2918          115 :       rtx_insn *insn;
    2919          115 :       expand_remove_edge (e);
    2920              :       /* Now, we have a single successor block, if we have insns to
    2921              :          insert on the remaining edge we potentially will insert
    2922              :          it at the end of this block (if the dest block isn't feasible)
    2923              :          in order to avoid splitting the edge.  This insertion will take
    2924              :          place in front of the last jump.  But we might have emitted
    2925              :          multiple jumps (conditional and one unconditional) to the
    2926              :          same destination.  Inserting in front of the last one then
    2927              :          is a problem.  See PR 40021.  We fix this by deleting all
    2928              :          jumps except the last unconditional one.  */
    2929          115 :       insn = PREV_INSN (get_last_insn ());
    2930              :       /* Make sure we have an unconditional jump.  Otherwise we're
    2931              :          confused.  */
    2932          115 :       gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
    2933          230 :       for (insn = PREV_INSN (insn); insn != last;)
    2934              :         {
    2935            0 :           insn = PREV_INSN (insn);
    2936            0 :           if (JUMP_P (NEXT_INSN (insn)))
    2937              :             {
    2938            0 :               if (!any_condjump_p (NEXT_INSN (insn)))
    2939              :                 {
    2940            0 :                   gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
    2941            0 :                   delete_insn (NEXT_INSN (NEXT_INSN (insn)));
    2942              :                 }
    2943            0 :               delete_insn (NEXT_INSN (insn));
    2944              :             }
    2945              :         }
    2946              :     }
    2947      5378338 : }
    2948              : 
    2949              : /* A subroutine of expand_gimple_basic_block.  Expand one GIMPLE_COND.
    2950              :    Returns a new basic block if we've terminated the current basic
    2951              :    block and created a new one.  */
    2952              : 
    2953              : static basic_block
    2954      5780708 : expand_gimple_cond (basic_block bb, gcond *stmt)
    2955              : {
    2956      5780708 :   basic_block new_bb, dest;
    2957      5780708 :   edge true_edge;
    2958      5780708 :   edge false_edge;
    2959      5780708 :   rtx_insn *last2, *last;
    2960      5780708 :   enum tree_code code;
    2961      5780708 :   tree op0, op1;
    2962              : 
    2963      5780708 :   code = gimple_cond_code (stmt);
    2964      5780708 :   op0 = gimple_cond_lhs (stmt);
    2965      5780708 :   op1 = gimple_cond_rhs (stmt);
    2966              :   /* We're sometimes presented with such code:
    2967              :        D.123_1 = x < y;
    2968              :        if (D.123_1 != 0)
    2969              :          ...
    2970              :      This would expand to two comparisons which then later might
    2971              :      be cleaned up by combine.  But some pattern matchers like if-conversion
    2972              :      work better when there's only one compare, so make up for this
    2973              :      here as special exception if TER would have made the same change.  */
    2974      5780708 :   if (SA.values
    2975      4233799 :       && TREE_CODE (op0) == SSA_NAME
    2976      4233093 :       && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
    2977       399836 :       && TREE_CODE (op1) == INTEGER_CST
    2978       389063 :       && ((gimple_cond_code (stmt) == NE_EXPR
    2979       389061 :            && integer_zerop (op1))
    2980            2 :           || (gimple_cond_code (stmt) == EQ_EXPR
    2981            2 :               && integer_onep (op1)))
    2982      6169771 :       && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
    2983              :     {
    2984       212452 :       gimple *second = SSA_NAME_DEF_STMT (op0);
    2985       212452 :       if (gimple_code (second) == GIMPLE_ASSIGN)
    2986              :         {
    2987       212433 :           enum tree_code code2 = gimple_assign_rhs_code (second);
    2988       212433 :           if (TREE_CODE_CLASS (code2) == tcc_comparison)
    2989              :             {
    2990         1220 :               code = code2;
    2991         1220 :               op0 = gimple_assign_rhs1 (second);
    2992         1220 :               op1 = gimple_assign_rhs2 (second);
    2993              :             }
    2994              :           /* If jumps are cheap and the target does not support conditional
    2995              :              compare, turn some more codes into jumpy sequences.  */
    2996       211213 :           else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
    2997       211213 :                    && !targetm.have_ccmp ())
    2998              :             {
    2999       211210 :               if ((code2 == BIT_AND_EXPR
    3000        69766 :                    && TYPE_PRECISION (TREE_TYPE (op0)) == 1
    3001        69766 :                    && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
    3002       211210 :                   || code2 == TRUTH_AND_EXPR)
    3003              :                 {
    3004        69766 :                   code = TRUTH_ANDIF_EXPR;
    3005        69766 :                   op0 = gimple_assign_rhs1 (second);
    3006        69766 :                   op1 = gimple_assign_rhs2 (second);
    3007              :                 }
    3008       141444 :               else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
    3009              :                 {
    3010        62491 :                   code = TRUTH_ORIF_EXPR;
    3011        62491 :                   op0 = gimple_assign_rhs1 (second);
    3012        62491 :                   op1 = gimple_assign_rhs2 (second);
    3013              :                 }
    3014              :             }
    3015              :         }
    3016              :     }
    3017              : 
    3018              :   /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
    3019              :      into (x - C2) * C3 < C4.  */
    3020      5780708 :   if ((code == EQ_EXPR || code == NE_EXPR)
    3021      4529881 :       && TREE_CODE (op0) == SSA_NAME
    3022      4529122 :       && TREE_CODE (op1) == INTEGER_CST)
    3023      3136410 :     code = maybe_optimize_mod_cmp (code, &op0, &op1);
    3024              : 
    3025              :   /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow.  */
    3026      5780708 :   if (!TYPE_UNSIGNED (TREE_TYPE (op0))
    3027      2329183 :       && (code == LT_EXPR || code == LE_EXPR
    3028      2329183 :           || code == GT_EXPR || code == GE_EXPR)
    3029       695830 :       && integer_zerop (op1)
    3030      5985776 :       && TREE_CODE (op0) == SSA_NAME)
    3031       205068 :     maybe_optimize_sub_cmp_0 (code, &op0, &op1);
    3032              : 
    3033      5780708 :   last2 = last = get_last_insn ();
    3034              : 
    3035      5780708 :   extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
    3036      5780708 :   set_curr_insn_location (gimple_location (stmt));
    3037              : 
    3038              :   /* We can either have a pure conditional jump with one fallthru edge or
    3039              :      two-way jump that needs to be decomposed into two basic blocks.  */
    3040      5780708 :   if (false_edge->dest == bb->next_bb)
    3041              :     {
    3042      2311682 :       jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
    3043              :                 true_edge->probability);
    3044      2311682 :       maybe_dump_rtl_for_gimple_stmt (stmt, last);
    3045      2311682 :       if (true_edge->goto_locus != UNKNOWN_LOCATION)
    3046       341511 :         set_curr_insn_location (true_edge->goto_locus);
    3047      2311682 :       false_edge->flags |= EDGE_FALLTHRU;
    3048      2311682 :       maybe_cleanup_end_of_block (false_edge, last);
    3049      2311682 :       return NULL;
    3050              :     }
    3051      3469026 :   if (true_edge->dest == bb->next_bb)
    3052              :     {
    3053      3066656 :       jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
    3054              :                    false_edge->probability);
    3055      3066656 :       maybe_dump_rtl_for_gimple_stmt (stmt, last);
    3056      3066656 :       if (false_edge->goto_locus != UNKNOWN_LOCATION)
    3057        24956 :         set_curr_insn_location (false_edge->goto_locus);
    3058      3066656 :       true_edge->flags |= EDGE_FALLTHRU;
    3059      3066656 :       maybe_cleanup_end_of_block (true_edge, last);
    3060      3066656 :       return NULL;
    3061              :     }
    3062              : 
    3063       402370 :   jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
    3064              :             true_edge->probability);
    3065       402370 :   last = get_last_insn ();
    3066       402370 :   if (false_edge->goto_locus != UNKNOWN_LOCATION)
    3067         1337 :     set_curr_insn_location (false_edge->goto_locus);
    3068       402370 :   emit_jump (label_rtx_for_bb (false_edge->dest));
    3069              : 
    3070       402370 :   head_end_for_bb[bb->index].second = last;
    3071       402370 :   if (BARRIER_P (head_end_for_bb[bb->index].second))
    3072          102 :     head_end_for_bb[bb->index].second
    3073           51 :       = PREV_INSN (head_end_for_bb[bb->index].second);
    3074       804740 :   update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
    3075       402370 :                             head_end_for_bb[bb->index].second, bb);
    3076              : 
    3077       402370 :   new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
    3078       402370 :   dest = false_edge->dest;
    3079       402370 :   expand_split_edge (false_edge, new_bb, 0);
    3080       402370 :   false_edge->flags |= EDGE_FALLTHRU;
    3081       402370 :   new_bb->count = false_edge->count ();
    3082       402370 :   loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
    3083       402370 :   add_bb_to_loop (new_bb, loop);
    3084       402370 :   if (loop->latch == bb
    3085        28144 :       && loop->header == dest)
    3086        28135 :     loop->latch = new_bb;
    3087       402370 :   if (BARRIER_P (BB_END (new_bb)))
    3088       402370 :     BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
    3089       402370 :   update_bb_for_insn (new_bb);
    3090              : 
    3091       402370 :   maybe_dump_rtl_for_gimple_stmt (stmt, last2);
    3092              : 
    3093       402370 :   if (true_edge->goto_locus != UNKNOWN_LOCATION)
    3094              :     {
    3095        45671 :       set_curr_insn_location (true_edge->goto_locus);
    3096        45671 :       true_edge->goto_locus = curr_insn_location ();
    3097              :     }
    3098              : 
    3099              :   return new_bb;
    3100              : }
    3101              : 
    3102              : /* Mark all calls that can have a transaction restart.  */
    3103              : 
    3104              : static void
    3105      6840444 : mark_transaction_restart_calls (gimple *stmt)
    3106              : {
    3107      6840444 :   struct tm_restart_node dummy;
    3108      6840444 :   tm_restart_node **slot;
    3109              : 
    3110      6840444 :   if (!cfun->gimple_df->tm_restart)
    3111      6837460 :     return;
    3112              : 
    3113         2984 :   dummy.stmt = stmt;
    3114         2984 :   slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
    3115         2984 :   if (slot)
    3116              :     {
    3117            0 :       struct tm_restart_node *n = *slot;
    3118            0 :       tree list = n->label_or_list;
    3119            0 :       rtx_insn *insn;
    3120              : 
    3121            0 :       for (insn = next_real_insn (get_last_insn ());
    3122            0 :            !CALL_P (insn);
    3123            0 :            insn = next_real_insn (insn))
    3124            0 :         continue;
    3125              : 
    3126            0 :       if (TREE_CODE (list) == LABEL_DECL)
    3127            0 :         add_reg_note (insn, REG_TM, label_rtx (list));
    3128              :       else
    3129            0 :         for (; list ; list = TREE_CHAIN (list))
    3130            0 :           add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
    3131            0 :     }
    3132              : }
    3133              : 
    3134              : /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
    3135              :    statement STMT.  */
    3136              : 
    3137              : static void
    3138      7072400 : expand_call_stmt (gcall *stmt)
    3139              : {
    3140      7072400 :   tree exp, decl, lhs;
    3141      7072400 :   bool builtin_p;
    3142      7072400 :   size_t i;
    3143              : 
    3144      7072400 :   if (gimple_call_internal_p (stmt))
    3145              :     {
    3146       198367 :       expand_internal_call (stmt);
    3147       198367 :       return;
    3148              :     }
    3149              : 
    3150              :   /* If this is a call to a built-in function and it has no effect other
    3151              :      than setting the lhs, try to implement it using an internal function
    3152              :      instead.  */
    3153      6874033 :   decl = gimple_call_fndecl (stmt);
    3154      6874033 :   if (gimple_call_lhs (stmt)
    3155      2420565 :       && !gimple_has_side_effects (stmt)
    3156      7585733 :       && (optimize || (decl && called_as_built_in (decl))))
    3157              :     {
    3158       690973 :       internal_fn ifn = replacement_internal_fn (stmt);
    3159       690973 :       if (ifn != IFN_LAST)
    3160              :         {
    3161        33587 :           expand_internal_call (ifn, stmt);
    3162        33587 :           return;
    3163              :         }
    3164              :     }
    3165              : 
    3166      6840446 :   exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
    3167              : 
    3168      6840446 :   CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
    3169      6840446 :   builtin_p = decl && fndecl_built_in_p (decl);
    3170              : 
    3171              :   /* If this is not a builtin function, the function type through which the
    3172              :      call is made may be different from the type of the function.  */
    3173      4810930 :   if (!builtin_p)
    3174      9621860 :     CALL_EXPR_FN (exp)
    3175     14432790 :       = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
    3176              :                       CALL_EXPR_FN (exp));
    3177              : 
    3178      6840446 :   TREE_TYPE (exp) = gimple_call_return_type (stmt);
    3179      6840446 :   CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
    3180              : 
    3181     20612895 :   for (i = 0; i < gimple_call_num_args (stmt); i++)
    3182              :     {
    3183     13772449 :       tree arg = gimple_call_arg (stmt, i);
    3184     13772449 :       gimple *def;
    3185              :       /* TER addresses into arguments of builtin functions so we have a
    3186              :          chance to infer more correct alignment information.  See PR39954.  */
    3187     13772449 :       if (builtin_p
    3188      4216415 :           && TREE_CODE (arg) == SSA_NAME
    3189      1439033 :           && (def = get_gimple_for_ssa_name (arg))
    3190       354609 :           && is_gimple_assign (def)
    3191     14126897 :           && gimple_assign_rhs_code (def) == ADDR_EXPR)
    3192        14882 :         arg = gimple_assign_rhs1 (def);
    3193     13772449 :       CALL_EXPR_ARG (exp, i) = arg;
    3194              :     }
    3195              : 
    3196      6840446 :   if (gimple_has_side_effects (stmt)
    3197              :       /* ???  Downstream in expand_expr_real_1 we assume that expressions
    3198              :          w/o side-effects do not throw so work around this here.  */
    3199      6840446 :       || stmt_could_throw_p (cfun, stmt))
    3200      6164306 :     TREE_SIDE_EFFECTS (exp) = 1;
    3201              : 
    3202      6840446 :   if (gimple_call_nothrow_p (stmt))
    3203      3062653 :     TREE_NOTHROW (exp) = 1;
    3204              : 
    3205      6840446 :   CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
    3206      6840446 :   CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
    3207      6840446 :   CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
    3208      6840446 :   if (decl
    3209      6653440 :       && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
    3210      8692282 :       && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
    3211        27680 :     CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
    3212              :   else
    3213      6812766 :     CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
    3214      6840446 :   CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
    3215      6840446 :   CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
    3216      6840446 :   SET_EXPR_LOCATION (exp, gimple_location (stmt));
    3217              : 
    3218              :   /* Must come after copying location.  */
    3219      6840446 :   copy_warning (exp, stmt);
    3220              : 
    3221              :   /* Ensure RTL is created for debug args.  */
    3222     13493886 :   if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
    3223              :     {
    3224        68666 :       vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
    3225        68666 :       unsigned int ix;
    3226        68666 :       tree dtemp;
    3227              : 
    3228        68666 :       if (debug_args)
    3229       150761 :         for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
    3230              :           {
    3231        82095 :             gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
    3232        82095 :             expand_debug_expr (dtemp);
    3233              :           }
    3234              :     }
    3235              : 
    3236      6840446 :   rtx_insn *before_call = get_last_insn ();
    3237      6840446 :   lhs = gimple_call_lhs (stmt);
    3238      6840446 :   if (lhs)
    3239      2386978 :     expand_assignment (lhs, exp, false);
    3240              :   else
    3241      4453468 :     expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
    3242              : 
    3243              :   /* If the gimple call is an indirect call and has 'nocf_check'
    3244              :      attribute find a generated CALL insn to mark it as no
    3245              :      control-flow verification is needed.  */
    3246      6840444 :   if (gimple_call_nocf_check_p (stmt)
    3247      6840465 :       && !gimple_call_fndecl (stmt))
    3248              :     {
    3249           20 :       rtx_insn *last = get_last_insn ();
    3250           20 :       while (!CALL_P (last)
    3251           34 :              && last != before_call)
    3252           14 :         last = PREV_INSN (last);
    3253              : 
    3254           20 :       if (last != before_call)
    3255           20 :         add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
    3256              :     }
    3257              : 
    3258      6840444 :   mark_transaction_restart_calls (stmt);
    3259              : }
    3260              : 
    3261              : 
    3262              : /* Generate RTL for an asm statement (explicit assembler code).
    3263              :    STRING is a STRING_CST node containing the assembler code text,
    3264              :    or an ADDR_EXPR containing a STRING_CST.  VOL nonzero means the
    3265              :    insn is volatile; don't optimize it.  */
    3266              : 
    3267              : static void
    3268         2876 : expand_asm_loc (tree string, int vol, location_t locus)
    3269              : {
    3270         2876 :   rtx body;
    3271              : 
    3272         2876 :   body = gen_rtx_ASM_INPUT_loc (VOIDmode,
    3273              :                                 ggc_strdup (TREE_STRING_POINTER (string)),
    3274              :                                 locus);
    3275              : 
    3276         2876 :   MEM_VOLATILE_P (body) = vol;
    3277              : 
    3278              :   /* Non-empty basic ASM implicitly clobbers memory.  */
    3279         2876 :   if (TREE_STRING_LENGTH (string) != 0)
    3280              :     {
    3281          858 :       rtx asm_op, clob;
    3282          858 :       unsigned i, nclobbers;
    3283          858 :       auto_vec<rtx> input_rvec, output_rvec;
    3284          858 :       auto_vec<machine_mode> input_mode;
    3285          858 :       auto_vec<const char *> constraints;
    3286          858 :       auto_vec<rtx> use_rvec;
    3287          858 :       auto_vec<rtx> clobber_rvec;
    3288          858 :       HARD_REG_SET clobbered_regs;
    3289          858 :       CLEAR_HARD_REG_SET (clobbered_regs);
    3290              : 
    3291          858 :       clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
    3292          858 :       clobber_rvec.safe_push (clob);
    3293              : 
    3294          858 :       if (targetm.md_asm_adjust)
    3295          858 :         targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
    3296              :                                constraints, use_rvec, clobber_rvec,
    3297              :                                clobbered_regs, locus);
    3298              : 
    3299          858 :       asm_op = body;
    3300          858 :       nclobbers = clobber_rvec.length ();
    3301          858 :       auto nuses = use_rvec.length ();
    3302          858 :       body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nuses + nclobbers));
    3303              : 
    3304          858 :       i = 0;
    3305          858 :       XVECEXP (body, 0, i++) = asm_op;
    3306          858 :       for (rtx use : use_rvec)
    3307            0 :         XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
    3308         4290 :       for (rtx clobber : clobber_rvec)
    3309         1716 :         XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobber);
    3310          858 :     }
    3311              : 
    3312         2876 :   emit_insn (body);
    3313         2876 : }
    3314              : 
    3315              : /* Check for overlap between registers marked in CLOBBERED_REGS and
    3316              :    anything inappropriate in T.  Emit error and return the register
    3317              :    variable definition for error, NULL_TREE for ok.  */
    3318              : 
    3319              : static bool
    3320       138513 : tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
    3321              :                                 location_t loc)
    3322              : {
    3323              :   /* Conflicts between asm-declared register variables and the clobber
    3324              :      list are not allowed.  */
    3325       138513 :   tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
    3326              : 
    3327       138513 :   if (overlap)
    3328              :     {
    3329           21 :       error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
    3330           21 :                 "%<asm%> clobber list", DECL_NAME (overlap));
    3331              : 
    3332              :       /* Reset registerness to stop multiple errors emitted for a single
    3333              :          variable.  */
    3334           21 :       DECL_REGISTER (overlap) = 0;
    3335           21 :       return true;
    3336              :     }
    3337              : 
    3338              :   return false;
    3339              : }
    3340              : 
    3341              : /* Check that the given REGNO spanning NREGS is a valid
    3342              :    asm clobber operand.  Some HW registers cannot be
    3343              :    saved/restored, hence they should not be clobbered by
    3344              :    asm statements.  */
    3345              : static bool
    3346        41660 : asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
    3347              : {
    3348        41660 :   bool is_valid = true;
    3349        41660 :   HARD_REG_SET regset;
    3350              : 
    3351        41660 :   CLEAR_HARD_REG_SET (regset);
    3352              : 
    3353        41660 :   add_range_to_hard_reg_set (&regset, regno, nregs);
    3354              : 
    3355              :   /* Clobbering the PIC register is an error.  */
    3356        41660 :   if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
    3357            0 :       && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
    3358              :     {
    3359              :       /* ??? Diagnose during gimplification?  */
    3360            0 :       error ("PIC register clobbered by %qs in %<asm%>", regname);
    3361            0 :       is_valid = false;
    3362              :     }
    3363        41660 :   else if (!in_hard_reg_set_p
    3364        41660 :            (accessible_reg_set, reg_raw_mode[regno], regno))
    3365              :     {
    3366              :       /* ??? Diagnose during gimplification?  */
    3367            0 :       error ("the register %qs cannot be clobbered in %<asm%>"
    3368              :              " for the current target", regname);
    3369            0 :       is_valid = false;
    3370              :     }
    3371              : 
    3372              :   /* Clobbering the stack pointer register is deprecated.  GCC expects
    3373              :      the value of the stack pointer after an asm statement to be the same
    3374              :      as it was before, so no asm can validly clobber the stack pointer in
    3375              :      the usual sense.  Adding the stack pointer to the clobber list has
    3376              :      traditionally had some undocumented and somewhat obscure side-effects.  */
    3377        41680 :   if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
    3378              :     {
    3379            1 :       crtl->sp_is_clobbered_by_asm = true;
    3380            1 :       if (warning (OPT_Wdeprecated, "listing the stack pointer register"
    3381              :                    " %qs in a clobber list is deprecated", regname))
    3382            1 :         inform (input_location, "the value of the stack pointer after"
    3383              :                 " an %<asm%> statement must be the same as it was before"
    3384              :                 " the statement");
    3385              :     }
    3386              : 
    3387        41660 :   return is_valid;
    3388              : }
    3389              : 
    3390              : /* Generate RTL for an asm statement with arguments.
    3391              :    STRING is the instruction template.
    3392              :    OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
    3393              :    Each output or input has an expression in the TREE_VALUE and
    3394              :    a tree list in TREE_PURPOSE which in turn contains a constraint
    3395              :    name in TREE_VALUE (or NULL_TREE) and a constraint string
    3396              :    in TREE_PURPOSE.
    3397              :    CLOBBERS is a list of STRING_CST nodes each naming a hard register
    3398              :    that is clobbered by this insn.
    3399              : 
    3400              :    LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
    3401              :    should be the fallthru basic block of the asm goto.
    3402              : 
    3403              :    Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
    3404              :    Some elements of OUTPUTS may be replaced with trees representing temporary
    3405              :    values.  The caller should copy those temporary values to the originally
    3406              :    specified lvalues.
    3407              : 
    3408              :    VOL nonzero means the insn is volatile; don't optimize it.  */
    3409              : 
    3410              : static void
    3411        96681 : expand_asm_stmt (gasm *stmt)
    3412              : {
    3413        96681 :   class save_input_location
    3414              :   {
    3415              :     location_t old;
    3416              : 
    3417              :   public:
    3418        93805 :     explicit save_input_location(location_t where)
    3419              :     {
    3420        93805 :       old = input_location;
    3421        93805 :       input_location = where;
    3422              :     }
    3423              : 
    3424        93805 :     ~save_input_location()
    3425              :     {
    3426        93805 :       input_location = old;
    3427            5 :     }
    3428              :   };
    3429              : 
    3430        96681 :   location_t locus = gimple_location (stmt);
    3431              : 
    3432        96681 :   if (gimple_asm_basic_p (stmt))
    3433              :     {
    3434         2876 :       const char *s = gimple_asm_string (stmt);
    3435         2876 :       tree string = build_string (strlen (s), s);
    3436         2876 :       expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
    3437         5757 :       return;
    3438              :     }
    3439              : 
    3440              :   /* There are some legacy diagnostics in here.  */
    3441        93805 :   save_input_location s_i_l(locus);
    3442              : 
    3443        93805 :   unsigned noutputs = gimple_asm_noutputs (stmt);
    3444        93805 :   unsigned ninputs = gimple_asm_ninputs (stmt);
    3445        93805 :   unsigned nlabels = gimple_asm_nlabels (stmt);
    3446        93805 :   unsigned i;
    3447        93805 :   bool error_seen = false;
    3448              : 
    3449              :   /* ??? Diagnose during gimplification?  */
    3450        93805 :   if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
    3451              :     {
    3452            5 :       error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
    3453            5 :       return;
    3454              :     }
    3455              : 
    3456        93800 :   auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
    3457        93800 :   auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
    3458        93800 :   auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
    3459              : 
    3460              :   /* Copy the gimple vectors into new vectors that we can manipulate.  */
    3461              : 
    3462        93800 :   output_tvec.safe_grow (noutputs, true);
    3463        93800 :   input_tvec.safe_grow (ninputs, true);
    3464        93800 :   constraints.safe_grow (noutputs + ninputs, true);
    3465              : 
    3466       267438 :   for (i = 0; i < noutputs; ++i)
    3467              :     {
    3468        79838 :       tree t = gimple_asm_output_op (stmt, i);
    3469        79838 :       output_tvec[i] = TREE_VALUE (t);
    3470        79838 :       constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
    3471              :     }
    3472       152479 :   for (i = 0; i < ninputs; i++)
    3473              :     {
    3474        58679 :       tree t = gimple_asm_input_op (stmt, i);
    3475        58679 :       input_tvec[i] = TREE_VALUE (t);
    3476        58679 :       constraints[i + noutputs]
    3477        58679 :         = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
    3478              :     }
    3479              : 
    3480              :   /* Count the number of meaningful clobbered registers, ignoring what
    3481              :      we would ignore later.  */
    3482        93800 :   auto_vec<rtx> clobber_rvec;
    3483        93800 :   HARD_REG_SET clobbered_regs;
    3484        93800 :   CLEAR_HARD_REG_SET (clobbered_regs);
    3485              : 
    3486        93800 :   if (unsigned n = gimple_asm_nclobbers (stmt))
    3487              :     {
    3488        54574 :       clobber_rvec.reserve (n);
    3489       191475 :       for (i = 0; i < n; i++)
    3490              :         {
    3491        82327 :           tree t = gimple_asm_clobber_op (stmt, i);
    3492        82327 :           const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
    3493        82327 :           int nregs, j;
    3494              : 
    3495        82327 :           j = decode_reg_name_and_count (regname, &nregs);
    3496        82327 :           if (j < 0)
    3497              :             {
    3498        40667 :               if (j == -2)
    3499              :                 {
    3500              :                   /* ??? Diagnose during gimplification?  */
    3501           15 :                   error_at (locus, "unknown register name %qs in %<asm%>",
    3502              :                             regname);
    3503           15 :                   error_seen = true;
    3504              :                 }
    3505        40652 :               else if (j == -4)
    3506              :                 {
    3507        40273 :                   rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
    3508        40273 :                   clobber_rvec.safe_push (x);
    3509              :                 }
    3510          379 :               else if (j == -5)
    3511              :                 {
    3512            2 :                   if (targetm.redzone_clobber)
    3513            2 :                     if (rtx x = targetm.redzone_clobber ())
    3514            2 :                       clobber_rvec.safe_push (x);
    3515              :                 }
    3516              :               else
    3517              :                 {
    3518              :                   /* Otherwise we should have -1 == empty string
    3519              :                      or -3 == cc, which is not a register.  */
    3520          377 :                   gcc_assert (j == -1 || j == -3);
    3521              :                 }
    3522              :             }
    3523              :           else
    3524        83320 :             for (int reg = j; reg < j + nregs; reg++)
    3525              :               {
    3526        41660 :                 if (!asm_clobber_reg_is_valid (reg, nregs, regname))
    3527            0 :                   return;
    3528              : 
    3529        41660 :                 SET_HARD_REG_BIT (clobbered_regs, reg);
    3530        41660 :                 rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
    3531        41660 :                 clobber_rvec.safe_push (x);
    3532              :               }
    3533              :         }
    3534              :     }
    3535              : 
    3536              :   /* First pass over inputs and outputs checks validity and sets
    3537              :      mark_addressable if needed.  */
    3538              :   /* ??? Diagnose during gimplification?  */
    3539              : 
    3540       173638 :   for (i = 0; i < noutputs; ++i)
    3541              :     {
    3542        79838 :       tree val = output_tvec[i];
    3543        79838 :       tree type = TREE_TYPE (val);
    3544        79838 :       const char *constraint;
    3545        79838 :       bool is_inout;
    3546        79838 :       bool allows_reg;
    3547        79838 :       bool allows_mem;
    3548              : 
    3549              :       /* Try to parse the output constraint.  If that fails, there's
    3550              :          no point in going further.  */
    3551        79838 :       constraint = constraints[i];
    3552        79838 :       if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
    3553              :                                     &allows_mem, &allows_reg, &is_inout,
    3554              :                                     nullptr))
    3555            0 :         return;
    3556              : 
    3557              :       /* If the output is a hard register, verify it doesn't conflict with
    3558              :          any other operand's possible hard register use.  */
    3559        79838 :       if (DECL_P (val)
    3560         7883 :           && REG_P (DECL_RTL (val))
    3561        80800 :           && HARD_REGISTER_P (DECL_RTL (val)))
    3562              :         {
    3563          942 :           unsigned j, output_hregno = REGNO (DECL_RTL (val));
    3564          942 :           bool early_clobber_p = strchr (constraints[i], '&') != NULL;
    3565          942 :           unsigned long match;
    3566              : 
    3567              :           /* Verify the other outputs do not use the same hard register.  */
    3568         1102 :           for (j = i + 1; j < noutputs; ++j)
    3569          160 :             if (DECL_P (output_tvec[j])
    3570          159 :                 && REG_P (DECL_RTL (output_tvec[j]))
    3571          159 :                 && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
    3572          319 :                 && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
    3573              :               {
    3574            1 :                 error_at (locus, "invalid hard register usage between output "
    3575              :                           "operands");
    3576            1 :                 error_seen = true;
    3577              :               }
    3578              : 
    3579              :           /* Verify matching constraint operands use the same hard register
    3580              :              and that the non-matching constraint operands do not use the same
    3581              :              hard register if the output is an early clobber operand.  */
    3582         1857 :           for (j = 0; j < ninputs; ++j)
    3583          915 :             if (DECL_P (input_tvec[j])
    3584          776 :                 && REG_P (DECL_RTL (input_tvec[j]))
    3585         1691 :                 && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
    3586              :               {
    3587          776 :                 unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
    3588          776 :                 switch (*constraints[j + noutputs])
    3589              :                   {
    3590          765 :                   case '0':  case '1':  case '2':  case '3':  case '4':
    3591          765 :                   case '5':  case '6':  case '7':  case '8':  case '9':
    3592          765 :                     match = strtoul (constraints[j + noutputs], NULL, 10);
    3593          765 :                     break;
    3594              :                   default:
    3595              :                     match = ULONG_MAX;
    3596              :                     break;
    3597              :                   }
    3598          765 :                 if (i == match
    3599          665 :                     && output_hregno != input_hregno)
    3600              :                   {
    3601            1 :                     error_at (locus, "invalid hard register usage between "
    3602              :                               "output operand and matching constraint operand");
    3603            1 :                     error_seen = true;
    3604              :                   }
    3605          775 :                 else if (early_clobber_p
    3606            4 :                          && i != match
    3607            2 :                          && output_hregno == input_hregno)
    3608              :                   {
    3609            0 :                     error_at (locus, "invalid hard register usage between "
    3610              :                               "earlyclobber operand and input operand");
    3611            0 :                     error_seen = true;
    3612              :                   }
    3613              :               }
    3614              :         }
    3615              : 
    3616        79838 :       if (! allows_reg
    3617        79838 :           && (allows_mem
    3618            0 :               || is_inout
    3619            0 :               || (DECL_P (val)
    3620            0 :                   && REG_P (DECL_RTL (val))
    3621            0 :                   && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
    3622         8347 :         mark_addressable (val);
    3623              :     }
    3624              : 
    3625       152479 :   for (i = 0; i < ninputs; ++i)
    3626              :     {
    3627        58679 :       bool allows_reg, allows_mem;
    3628        58679 :       const char *constraint;
    3629              : 
    3630        58679 :       constraint = constraints[i + noutputs];
    3631        58679 :       if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
    3632        58679 :                                     constraints.address (), &allows_mem,
    3633              :                                     &allows_reg, nullptr))
    3634            0 :         return;
    3635              : 
    3636        58679 :       if (! allows_reg && allows_mem)
    3637         8994 :         mark_addressable (input_tvec[i]);
    3638              :     }
    3639              : 
    3640              :   /* Second pass evaluates arguments.  */
    3641              : 
    3642              :   /* Make sure stack is consistent for asm goto.  */
    3643        93800 :   if (nlabels > 0)
    3644          633 :     do_pending_stack_adjust ();
    3645        93800 :   int old_generating_concat_p = generating_concat_p;
    3646              : 
    3647              :   /* Vector of RTX's of evaluated output operands.  */
    3648       187600 :   auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
    3649        93800 :   auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
    3650        93800 :   rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
    3651              : 
    3652        93800 :   output_rvec.safe_grow (noutputs, true);
    3653              : 
    3654       267438 :   for (i = 0; i < noutputs; ++i)
    3655              :     {
    3656        79838 :       tree val = output_tvec[i];
    3657        79838 :       tree type = TREE_TYPE (val);
    3658        79838 :       bool is_inout, allows_reg, allows_mem, ok;
    3659        79838 :       rtx op;
    3660              : 
    3661        79838 :       ok = parse_output_constraint (&constraints[i], i, ninputs,
    3662              :                                     noutputs, &allows_mem, &allows_reg,
    3663              :                                     &is_inout, nullptr);
    3664        79838 :       gcc_assert (ok);
    3665              : 
    3666              :       /* If an output operand is not a decl or indirect ref and our constraint
    3667              :          allows a register, make a temporary to act as an intermediate.
    3668              :          Make the asm insn write into that, then we will copy it to
    3669              :          the real output operand.  Likewise for promoted variables.  */
    3670              : 
    3671        79838 :       generating_concat_p = 0;
    3672              : 
    3673        79838 :       gcc_assert (TREE_CODE (val) != INDIRECT_REF);
    3674        79838 :       if (((TREE_CODE (val) == MEM_REF
    3675         3990 :             && TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
    3676         3873 :            && allows_mem)
    3677        77701 :           || (DECL_P (val)
    3678         7883 :               && (allows_mem || REG_P (DECL_RTL (val)))
    3679         8131 :               && ! (REG_P (DECL_RTL (val))
    3680          962 :                     && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
    3681        70532 :           || ! allows_reg
    3682        70321 :           || is_inout
    3683        70321 :           || TREE_ADDRESSABLE (type)
    3684       150159 :           || (!tree_fits_poly_int64_p (TYPE_SIZE (type))
    3685            0 :               && !known_size_p (max_int_size_in_bytes (type))))
    3686              :         {
    3687         9517 :           op = expand_expr (val, NULL_RTX, VOIDmode,
    3688              :                             !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
    3689         9517 :           if (MEM_P (op))
    3690         8555 :             op = validize_mem (op);
    3691              : 
    3692         9517 :           if (! allows_reg && !MEM_P (op))
    3693              :             {
    3694            0 :               error_at (locus, "output number %d not directly addressable", i);
    3695            0 :               error_seen = true;
    3696              :             }
    3697         9517 :           if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
    3698         9517 :               || GET_CODE (op) == CONCAT)
    3699              :             {
    3700            0 :               rtx old_op = op;
    3701            0 :               op = gen_reg_rtx (GET_MODE (op));
    3702              : 
    3703            0 :               generating_concat_p = old_generating_concat_p;
    3704              : 
    3705            0 :               if (is_inout)
    3706            0 :                 emit_move_insn (op, old_op);
    3707              : 
    3708            0 :               push_to_sequence2 (after_rtl_seq, after_rtl_end);
    3709            0 :               emit_move_insn (old_op, op);
    3710            0 :               after_rtl_seq = get_insns ();
    3711            0 :               after_rtl_end = get_last_insn ();
    3712            0 :               end_sequence ();
    3713              :             }
    3714              :         }
    3715              :       else
    3716              :         {
    3717        70321 :           op = assign_temp (type, 0, 1);
    3718        70321 :           op = validize_mem (op);
    3719        70321 :           if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
    3720        67122 :             set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
    3721              : 
    3722        70321 :           generating_concat_p = old_generating_concat_p;
    3723              : 
    3724        70321 :           push_to_sequence2 (after_rtl_seq, after_rtl_end);
    3725        70321 :           expand_assignment (val, make_tree (type, op), false);
    3726        70321 :           after_rtl_seq = get_insns ();
    3727        70321 :           after_rtl_end = get_last_insn ();
    3728        70321 :           end_sequence ();
    3729              :         }
    3730        79838 :       output_rvec[i] = op;
    3731              : 
    3732        79838 :       if (is_inout)
    3733            0 :         inout_opnum.safe_push (i);
    3734              :     }
    3735              : 
    3736        93800 :   const char *str = gimple_asm_string (stmt);
    3737        93800 :   if (error_seen)
    3738              :     {
    3739           17 :       ninputs = 0;
    3740           17 :       noutputs = 0;
    3741           17 :       inout_opnum.truncate (0);
    3742           17 :       output_rvec.truncate (0);
    3743           17 :       clobber_rvec.truncate (0);
    3744           17 :       constraints.truncate (0);
    3745           17 :       CLEAR_HARD_REG_SET (clobbered_regs);
    3746              :       str = "";
    3747              :     }
    3748              : 
    3749       187600 :   auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
    3750        93800 :   auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
    3751              : 
    3752        93800 :   input_rvec.safe_grow (ninputs, true);
    3753        93800 :   input_mode.safe_grow (ninputs, true);
    3754              : 
    3755        93800 :   generating_concat_p = 0;
    3756              : 
    3757       152478 :   for (i = 0; i < ninputs; ++i)
    3758              :     {
    3759        58678 :       tree val = input_tvec[i];
    3760        58678 :       tree type = TREE_TYPE (val);
    3761        58678 :       bool allows_reg, allows_mem, ok;
    3762        58678 :       const char *constraint;
    3763        58678 :       rtx op;
    3764              : 
    3765        58678 :       constraint = constraints[i + noutputs];
    3766        58678 :       ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
    3767        58678 :                                    constraints.address (),
    3768              :                                    &allows_mem, &allows_reg, nullptr);
    3769        58678 :       gcc_assert (ok);
    3770              : 
    3771              :       /* EXPAND_INITIALIZER will not generate code for valid initializer
    3772              :          constants, but will still generate code for other types of operand.
    3773              :          This is the behavior we want for constant constraints.  */
    3774        58678 :       op = expand_expr (val, NULL_RTX, VOIDmode,
    3775              :                         allows_reg ? EXPAND_NORMAL
    3776        10707 :                         : allows_mem ? EXPAND_MEMORY
    3777              :                         : EXPAND_INITIALIZER);
    3778              : 
    3779              :       /* Never pass a CONCAT to an ASM.  */
    3780        58678 :       if (GET_CODE (op) == CONCAT)
    3781            2 :         op = force_reg (GET_MODE (op), op);
    3782        58676 :       else if (MEM_P (op))
    3783              :         {
    3784        13991 :           op = validize_mem (op);
    3785        13991 :           MEM_NOTRAP_P (op) = !tree_could_trap_p (val);
    3786              :         }
    3787              : 
    3788        58678 :       if (asm_operand_ok (op, constraint, NULL) <= 0)
    3789              :         {
    3790        26059 :           if (allows_reg && TYPE_MODE (type) != BLKmode)
    3791        26038 :             op = force_reg (TYPE_MODE (type), op);
    3792           21 :           else if (!allows_mem)
    3793           20 :             warning_at (locus, 0, "%<asm%> operand %d probably does not match "
    3794              :                         "constraints", i + noutputs);
    3795            1 :           else if (MEM_P (op))
    3796              :             {
    3797              :               /* We won't recognize either volatile memory or memory
    3798              :                  with a queued address as available a memory_operand
    3799              :                  at this point.  Ignore it: clearly this *is* a memory.  */
    3800              :             }
    3801              :           else
    3802            0 :             gcc_unreachable ();
    3803              :         }
    3804        58678 :       input_rvec[i] = op;
    3805        58678 :       input_mode[i] = TYPE_MODE (type);
    3806              :     }
    3807              : 
    3808              :   /* For in-out operands, copy output rtx to input rtx.  */
    3809        93800 :   unsigned ninout = inout_opnum.length ();
    3810        93800 :   for (i = 0; i < ninout; i++)
    3811              :     {
    3812            0 :       int j = inout_opnum[i];
    3813            0 :       rtx o = output_rvec[j];
    3814              : 
    3815            0 :       input_rvec.safe_push (o);
    3816            0 :       input_mode.safe_push (GET_MODE (o));
    3817              : 
    3818            0 :       char buffer[16];
    3819            0 :       sprintf (buffer, "%d", j);
    3820            0 :       constraints.safe_push (ggc_strdup (buffer));
    3821              :     }
    3822        93800 :   ninputs += ninout;
    3823              : 
    3824              :   /* Sometimes we wish to automatically clobber registers across an asm.
    3825              :      Case in point is when the i386 backend moved from cc0 to a hard reg --
    3826              :      maintaining source-level compatibility means automatically clobbering
    3827              :      the flags register.  */
    3828       187600 :   auto_vec<rtx> use_rvec;
    3829        93800 :   if (targetm.md_asm_adjust)
    3830              :     {
    3831        93800 :       rtx_insn *after_md_seq
    3832        93800 :         = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
    3833              :                                  constraints, use_rvec, clobber_rvec,
    3834              :                                  clobbered_regs, locus);
    3835        93800 :       if (after_md_seq)
    3836              :         {
    3837           72 :           push_to_sequence (after_md_seq);
    3838           72 :           emit_insn (after_rtl_seq);
    3839           72 :           after_rtl_seq = get_insns ();
    3840           72 :           after_rtl_end = get_last_insn ();
    3841           72 :           end_sequence ();
    3842              :         }
    3843              :     }
    3844              : 
    3845              :   /* Do not allow the hook to change the output and input count,
    3846              :      lest it mess up the operand numbering.  */
    3847       187600 :   gcc_assert (output_rvec.length() == noutputs);
    3848       187600 :   gcc_assert (input_rvec.length() == ninputs);
    3849       187600 :   gcc_assert (constraints.length() == noutputs + ninputs);
    3850              : 
    3851              :   /* But it certainly can adjust the uses and clobbers.  */
    3852        93800 :   unsigned nuses = use_rvec.length ();
    3853        93800 :   unsigned nclobbers = clobber_rvec.length ();
    3854              : 
    3855              :   /* Third pass checks for easy conflicts.  */
    3856              :   /* ??? Why are we doing this on trees instead of rtx.  */
    3857              : 
    3858        93800 :   bool clobber_conflict_found = 0;
    3859       173635 :   for (i = 0; i < noutputs; ++i)
    3860        79835 :     if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs, locus))
    3861           10 :         clobber_conflict_found = 1;
    3862       152478 :   for (i = 0; i < ninputs - ninout; ++i)
    3863        58678 :     if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs, locus))
    3864           11 :         clobber_conflict_found = 1;
    3865              : 
    3866              :   /* Make vectors for the expression-rtx, constraint strings,
    3867              :      and named operands.  */
    3868              : 
    3869        93800 :   rtvec argvec = rtvec_alloc (ninputs);
    3870        93800 :   rtvec constraintvec = rtvec_alloc (ninputs);
    3871        93800 :   rtvec labelvec = rtvec_alloc (nlabels);
    3872              : 
    3873       130072 :   rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
    3874              :                                     : GET_MODE (output_rvec[0])),
    3875              :                                    ggc_strdup (str),
    3876              :                                    "", 0, argvec, constraintvec,
    3877              :                                    labelvec, locus);
    3878        93800 :   MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
    3879              : 
    3880       152478 :   for (i = 0; i < ninputs; ++i)
    3881              :     {
    3882        58678 :       ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
    3883       117356 :       ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
    3884        58678 :         = gen_rtx_ASM_INPUT_loc (input_mode[i],
    3885              :                                  constraints[i + noutputs],
    3886              :                                  locus);
    3887              :     }
    3888              : 
    3889              :   /* Copy labels to the vector.  */
    3890        93800 :   rtx_code_label *fallthru_label = NULL;
    3891        93800 :   if (nlabels > 0)
    3892              :     {
    3893          633 :       basic_block fallthru_bb = NULL;
    3894          633 :       edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
    3895          633 :       if (fallthru)
    3896          633 :         fallthru_bb = fallthru->dest;
    3897              : 
    3898         1525 :       for (i = 0; i < nlabels; ++i)
    3899              :         {
    3900          892 :           tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
    3901          892 :           rtx_insn *r;
    3902              :           /* If asm goto has any labels in the fallthru basic block, use
    3903              :              a label that we emit immediately after the asm goto.  Expansion
    3904              :              may insert further instructions into the same basic block after
    3905              :              asm goto and if we don't do this, insertion of instructions on
    3906              :              the fallthru edge might misbehave.  See PR58670.  */
    3907          892 :           if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
    3908              :             {
    3909          294 :               if (fallthru_label == NULL_RTX)
    3910          289 :                 fallthru_label = gen_label_rtx ();
    3911              :               r = fallthru_label;
    3912              :             }
    3913              :           else
    3914          598 :             r = label_rtx (label);
    3915          892 :           ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
    3916              :         }
    3917              :     }
    3918              : 
    3919              :   /* Now, for each output, construct an rtx
    3920              :      (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
    3921              :                                ARGVEC CONSTRAINTS OPNAMES))
    3922              :      If there is more than one, put them inside a PARALLEL.  */
    3923              : 
    3924        93800 :   if (noutputs == 0 && nuses == 0 && nclobbers == 0)
    3925              :     {
    3926              :       /* No output operands: put in a raw ASM_OPERANDS rtx.  */
    3927            0 :       if (nlabels > 0)
    3928            0 :         emit_jump_insn (body);
    3929              :       else
    3930            0 :         emit_insn (body);
    3931              :     }
    3932        93800 :   else if (noutputs == 1 && nuses == 0 && nclobbers == 0)
    3933              :     {
    3934           62 :       ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
    3935           62 :       if (nlabels > 0)
    3936            0 :         emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
    3937              :       else
    3938           62 :         emit_insn (gen_rtx_SET (output_rvec[0], body));
    3939              :     }
    3940              :   else
    3941              :     {
    3942        93738 :       rtx obody = body;
    3943        93738 :       int num = noutputs;
    3944              : 
    3945        93738 :       if (num == 0)
    3946        57528 :         num = 1;
    3947              : 
    3948        93738 :       body = gen_rtx_PARALLEL (VOIDmode,
    3949              :                                rtvec_alloc (num + nuses + nclobbers));
    3950              : 
    3951              :       /* For each output operand, store a SET.  */
    3952       173511 :       for (i = 0; i < noutputs; ++i)
    3953              :         {
    3954        79773 :           rtx src, o = output_rvec[i];
    3955        79773 :           if (i == 0)
    3956              :             {
    3957        36210 :               ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
    3958        36210 :               src = obody;
    3959              :             }
    3960              :           else
    3961              :             {
    3962        43563 :               src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
    3963              :                                           ASM_OPERANDS_TEMPLATE (obody),
    3964              :                                           constraints[i], i, argvec,
    3965              :                                           constraintvec, labelvec, locus);
    3966        43563 :               MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
    3967              :             }
    3968        79773 :           XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
    3969              :         }
    3970              : 
    3971              :       /* If there are no outputs (but there are some clobbers)
    3972              :          store the bare ASM_OPERANDS into the PARALLEL.  */
    3973        93738 :       if (i == 0)
    3974        57528 :         XVECEXP (body, 0, i++) = obody;
    3975              : 
    3976              :       /* Add the uses specified by the target hook.  No checking should
    3977              :          be needed since this doesn't come directly from user code.  */
    3978        93738 :       for (rtx use : use_rvec)
    3979            0 :         XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
    3980              : 
    3981              :       /* Store (clobber REG) for each clobbered register specified.  */
    3982       269398 :       for (unsigned j = 0; j < nclobbers; ++j)
    3983              :         {
    3984       175660 :           rtx clobbered_reg = clobber_rvec[j];
    3985              : 
    3986              :           /* Do sanity check for overlap between clobbers and respectively
    3987              :              input and outputs that hasn't been handled.  Such overlap
    3988              :              should have been detected and reported above.  */
    3989       175660 :           if (!clobber_conflict_found && REG_P (clobbered_reg))
    3990              :             {
    3991              :               /* We test the old body (obody) contents to avoid
    3992              :                  tripping over the under-construction body.  */
    3993       216187 :               for (unsigned k = 0; k < noutputs; ++k)
    3994        80832 :                 if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
    3995            0 :                   internal_error ("%<asm%> clobber conflict with "
    3996              :                                   "output operand");
    3997              : 
    3998       197262 :               for (unsigned k = 0; k < ninputs - ninout; ++k)
    3999        61907 :                 if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
    4000            0 :                   internal_error ("%<asm%> clobber conflict with "
    4001              :                                   "input operand");
    4002              :             }
    4003              : 
    4004       175660 :           XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
    4005              :         }
    4006              : 
    4007        93738 :       if (nlabels > 0)
    4008          633 :         emit_jump_insn (body);
    4009              :       else
    4010        93105 :         emit_insn (body);
    4011              :     }
    4012              : 
    4013        93800 :   generating_concat_p = old_generating_concat_p;
    4014              : 
    4015        93800 :   if (fallthru_label)
    4016          289 :     emit_label (fallthru_label);
    4017              : 
    4018        93800 :   if (after_rtl_seq)
    4019              :     {
    4020        29211 :       if (nlabels == 0)
    4021        29076 :         emit_insn (after_rtl_seq);
    4022              :       else
    4023              :         {
    4024          135 :           edge e;
    4025          135 :           edge_iterator ei;
    4026          135 :           unsigned int cnt = EDGE_COUNT (gimple_bb (stmt)->succs);
    4027              : 
    4028          480 :           FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
    4029              :             {
    4030          345 :               rtx_insn *copy;
    4031          345 :               if (--cnt == 0)
    4032              :                 copy = after_rtl_seq;
    4033              :               else
    4034              :                 {
    4035          210 :                   start_sequence ();
    4036          210 :                   duplicate_insn_chain (after_rtl_seq, after_rtl_end,
    4037              :                                         NULL, NULL);
    4038          210 :                   copy = end_sequence ();
    4039              :                 }
    4040          345 :               prepend_insn_to_edge (copy, e);
    4041              :             }
    4042              :         }
    4043              :     }
    4044              : 
    4045        93800 :   free_temp_slots ();
    4046        93800 :   crtl->has_asm_statement = 1;
    4047        93805 : }
    4048              : 
    4049              : /* Emit code to jump to the address
    4050              :    specified by the pointer expression EXP.  */
    4051              : 
    4052              : static void
    4053          422 : expand_computed_goto (tree exp)
    4054              : {
    4055          422 :   rtx x = expand_normal (exp);
    4056              : 
    4057          422 :   do_pending_stack_adjust ();
    4058          422 :   emit_indirect_jump (x);
    4059          422 : }
    4060              : 
    4061              : /* Generate RTL code for a `goto' statement with target label LABEL.
    4062              :    LABEL should be a LABEL_DECL tree node that was or will later be
    4063              :    defined with `expand_label'.  */
    4064              : 
    4065              : static void
    4066            0 : expand_goto (tree label)
    4067              : {
    4068            0 :   if (flag_checking)
    4069              :     {
    4070              :       /* Check for a nonlocal goto to a containing function.  Should have
    4071              :          gotten translated to __builtin_nonlocal_goto.  */
    4072            0 :       tree context = decl_function_context (label);
    4073            0 :       gcc_assert (!context || context == current_function_decl);
    4074              :     }
    4075              : 
    4076            0 :   emit_jump (jump_target_rtx (label));
    4077            0 : }
    4078              : 
    4079              : /* Output a return with no value.  */
    4080              : 
    4081              : static void
    4082       846661 : expand_null_return_1 (void)
    4083              : {
    4084       846661 :   clear_pending_stack_adjust ();
    4085       846661 :   do_pending_stack_adjust ();
    4086       846661 :   emit_jump (return_label);
    4087       846661 : }
    4088              : 
    4089              : /* Generate RTL to return from the current function, with no value.
    4090              :    (That is, we do not do anything about returning any value.)  */
    4091              : 
    4092              : void
    4093        70993 : expand_null_return (void)
    4094              : {
    4095              :   /* If this function was declared to return a value, but we
    4096              :      didn't, clobber the return registers so that they are not
    4097              :      propagated live to the rest of the function.  */
    4098        70993 :   clobber_return_register ();
    4099              : 
    4100        70993 :   expand_null_return_1 ();
    4101        70993 : }
    4102              : 
    4103              : /* Generate RTL to return from the current function, with value VAL.  */
    4104              : 
    4105              : static void
    4106       775668 : expand_value_return (rtx val)
    4107              : {
    4108              :   /* Copy the value to the return location unless it's already there.  */
    4109              : 
    4110       775668 :   tree decl = DECL_RESULT (current_function_decl);
    4111       775668 :   rtx return_reg = DECL_RTL (decl);
    4112       775668 :   if (return_reg != val)
    4113              :     {
    4114       450168 :       tree funtype = TREE_TYPE (current_function_decl);
    4115       450168 :       tree type = TREE_TYPE (decl);
    4116       450168 :       int unsignedp = TYPE_UNSIGNED (type);
    4117       450168 :       machine_mode old_mode = DECL_MODE (decl);
    4118       450168 :       machine_mode mode;
    4119       450168 :       if (DECL_BY_REFERENCE (decl))
    4120            0 :         mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
    4121              :       else
    4122       450168 :         mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
    4123              : 
    4124       450168 :       if (mode != old_mode)
    4125              :         {
    4126              :           /* Some ABIs require scalar floating point modes to be returned
    4127              :              in a wider scalar integer mode.  We need to explicitly
    4128              :              reinterpret to an integer mode of the correct precision
    4129              :              before extending to the desired result.  */
    4130            8 :           if (SCALAR_INT_MODE_P (mode)
    4131            8 :               && SCALAR_FLOAT_MODE_P (old_mode)
    4132            8 :               && known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
    4133            0 :             val = convert_float_to_wider_int (mode, old_mode, val);
    4134              :           else
    4135            8 :             val = convert_modes (mode, old_mode, val, unsignedp);
    4136              :         }
    4137              : 
    4138       450168 :       if (GET_CODE (return_reg) == PARALLEL)
    4139         3309 :         emit_group_load (return_reg, val, type, int_size_in_bytes (type));
    4140              :       else
    4141       446859 :         emit_move_insn (return_reg, val);
    4142              :     }
    4143              : 
    4144       775668 :   expand_null_return_1 ();
    4145       775668 : }
    4146              : 
    4147              : /* Generate RTL to evaluate the expression RETVAL and return it
    4148              :    from the current function.  */
    4149              : 
    4150              : static void
    4151       778209 : expand_return (tree retval)
    4152              : {
    4153       778209 :   rtx result_rtl;
    4154       778209 :   rtx val = 0;
    4155       778209 :   tree retval_rhs;
    4156              : 
    4157              :   /* If function wants no value, give it none.  */
    4158       778209 :   if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
    4159              :     {
    4160            0 :       expand_normal (retval);
    4161            0 :       expand_null_return ();
    4162            0 :       return;
    4163              :     }
    4164              : 
    4165       778209 :   if (retval == error_mark_node)
    4166              :     {
    4167              :       /* Treat this like a return of no value from a function that
    4168              :          returns a value.  */
    4169            0 :       expand_null_return ();
    4170            0 :       return;
    4171              :     }
    4172       778209 :   else if ((TREE_CODE (retval) == MODIFY_EXPR
    4173       778209 :             || TREE_CODE (retval) == INIT_EXPR)
    4174       778209 :            && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
    4175       714917 :     retval_rhs = TREE_OPERAND (retval, 1);
    4176              :   else
    4177              :     retval_rhs = retval;
    4178              : 
    4179       778209 :   result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
    4180              : 
    4181              :   /* If we are returning the RESULT_DECL, then the value has already
    4182              :      been stored into it, so we don't have to do anything special.  */
    4183       778209 :   if (TREE_CODE (retval_rhs) == RESULT_DECL)
    4184        63292 :     expand_value_return (result_rtl);
    4185              : 
    4186              :   /* If the result is an aggregate that is being returned in one (or more)
    4187              :      registers, load the registers here.  */
    4188              : 
    4189       714917 :   else if (retval_rhs != 0
    4190       714917 :            && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
    4191         5259 :            && REG_P (result_rtl))
    4192              :     {
    4193         3765 :       val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
    4194         3765 :       if (val)
    4195              :         {
    4196              :           /* Use the mode of the result value on the return register.  */
    4197         1224 :           PUT_MODE (result_rtl, GET_MODE (val));
    4198         1224 :           expand_value_return (val);
    4199              :         }
    4200              :       else
    4201         2541 :         expand_null_return ();
    4202              :     }
    4203       711152 :   else if (retval_rhs != 0
    4204       711152 :            && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
    4205       711152 :            && (REG_P (result_rtl)
    4206         6451 :                || (GET_CODE (result_rtl) == PARALLEL)))
    4207              :     {
    4208              :       /* Compute the return value into a temporary (usually a pseudo reg).  */
    4209       708010 :       val
    4210       708010 :         = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
    4211       708010 :       val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
    4212       708010 :       val = force_not_mem (val);
    4213       708010 :       expand_value_return (val);
    4214              :     }
    4215              :   else
    4216              :     {
    4217              :       /* No hard reg used; calculate value into hard return reg.  */
    4218         3142 :       expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
    4219         3142 :       expand_value_return (result_rtl);
    4220              :     }
    4221              : }
    4222              : 
    4223              : /* Expand a clobber of LHS.  If LHS is stored it in a multi-part
    4224              :    register, tell the rtl optimizers that its value is no longer
    4225              :    needed.  */
    4226              : 
    4227              : static void
    4228      1412366 : expand_clobber (tree lhs)
    4229              : {
    4230      1412366 :   if (DECL_P (lhs))
    4231              :     {
    4232      1383964 :       rtx decl_rtl = DECL_RTL_IF_SET (lhs);
    4233      1383873 :       if (decl_rtl && REG_P (decl_rtl))
    4234              :         {
    4235       189499 :           machine_mode decl_mode = GET_MODE (decl_rtl);
    4236       378998 :           if (maybe_gt (GET_MODE_SIZE (decl_mode),
    4237              :                         REGMODE_NATURAL_SIZE (decl_mode)))
    4238       146929 :             emit_clobber (decl_rtl);
    4239              :         }
    4240              :     }
    4241      1412366 : }
    4242              : 
    4243              : /* A subroutine of expand_gimple_stmt, expanding one gimple statement
    4244              :    STMT that doesn't require special handling for outgoing edges.  That
    4245              :    is no tailcalls and no GIMPLE_COND.  */
    4246              : 
    4247              : static void
    4248     32589326 : expand_gimple_stmt_1 (gimple *stmt)
    4249              : {
    4250     32589326 :   tree op0;
    4251              : 
    4252     32589326 :   set_curr_insn_location (gimple_location (stmt));
    4253              : 
    4254     32589326 :   switch (gimple_code (stmt))
    4255              :     {
    4256          422 :     case GIMPLE_GOTO:
    4257          422 :       op0 = gimple_goto_dest (stmt);
    4258          422 :       if (TREE_CODE (op0) == LABEL_DECL)
    4259            0 :         expand_goto (op0);
    4260              :       else
    4261          422 :         expand_computed_goto (op0);
    4262              :       break;
    4263       666412 :     case GIMPLE_LABEL:
    4264       666412 :       expand_label (gimple_label_label (as_a <glabel *> (stmt)));
    4265       666412 :       break;
    4266              :     case GIMPLE_NOP:
    4267              :     case GIMPLE_PREDICT:
    4268              :       break;
    4269         7659 :     case GIMPLE_SWITCH:
    4270         7659 :       {
    4271         7659 :         gswitch *swtch = as_a <gswitch *> (stmt);
    4272         7659 :         if (gimple_switch_num_labels (swtch) == 1)
    4273            0 :           expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
    4274              :         else
    4275         7659 :           expand_case (swtch);
    4276              :       }
    4277              :       break;
    4278        96681 :     case GIMPLE_ASM:
    4279        96681 :       expand_asm_stmt (as_a <gasm *> (stmt));
    4280        96681 :       break;
    4281      7072400 :     case GIMPLE_CALL:
    4282      7072400 :       expand_call_stmt (as_a <gcall *> (stmt));
    4283      7072400 :       break;
    4284              : 
    4285       846661 :     case GIMPLE_RETURN:
    4286       846661 :       {
    4287       846661 :         op0 = gimple_return_retval (as_a <greturn *> (stmt));
    4288              : 
    4289              :         /* If a return doesn't have a location, it very likely represents
    4290              :            multiple user returns so we cannot let it inherit the location
    4291              :            of the last statement of the previous basic block in RTL.  */
    4292       846661 :         if (!gimple_has_location (stmt))
    4293       218724 :           set_curr_insn_location (cfun->function_end_locus);
    4294              : 
    4295       846661 :         if (op0 && op0 != error_mark_node)
    4296              :           {
    4297       778209 :             tree result = DECL_RESULT (current_function_decl);
    4298              : 
    4299              :             /* If we are not returning the current function's RESULT_DECL,
    4300              :                build an assignment to it.  */
    4301       778209 :             if (op0 != result)
    4302              :               {
    4303              :                 /* I believe that a function's RESULT_DECL is unique.  */
    4304       714917 :                 gcc_assert (TREE_CODE (op0) != RESULT_DECL);
    4305              : 
    4306              :                 /* ??? We'd like to use simply expand_assignment here,
    4307              :                    but this fails if the value is of BLKmode but the return
    4308              :                    decl is a register.  expand_return has special handling
    4309              :                    for this combination, which eventually should move
    4310              :                    to common code.  See comments there.  Until then, let's
    4311              :                    build a modify expression :-/  */
    4312       714917 :                 op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
    4313              :                               result, op0);
    4314              :               }
    4315              :           }
    4316              : 
    4317       778209 :         if (!op0)
    4318        68452 :           expand_null_return ();
    4319              :         else
    4320       778209 :           expand_return (op0);
    4321              :       }
    4322              :       break;
    4323              : 
    4324     23849744 :     case GIMPLE_ASSIGN:
    4325     23849744 :       {
    4326     23849744 :         gassign *assign_stmt = as_a <gassign *> (stmt);
    4327     23849744 :         tree lhs = gimple_assign_lhs (assign_stmt);
    4328              : 
    4329              :         /* Tree expand used to fiddle with |= and &= of two bitfield
    4330              :            COMPONENT_REFs here.  This can't happen with gimple, the LHS
    4331              :            of binary assigns must be a gimple reg.  */
    4332              : 
    4333     23849744 :         if (TREE_CODE (lhs) != SSA_NAME
    4334     37906155 :             || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
    4335              :           {
    4336     17752918 :             tree rhs = gimple_assign_rhs1 (assign_stmt);
    4337     17752918 :             gcc_assert (gimple_assign_rhs_class (assign_stmt)
    4338              :                         == GIMPLE_SINGLE_RHS);
    4339     33873807 :             if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
    4340              :                 /* Do not put locations on possibly shared trees.  */
    4341     23860878 :                 && !is_gimple_min_invariant (rhs))
    4342      4964554 :               SET_EXPR_LOCATION (rhs, gimple_location (stmt));
    4343     17752918 :             if (TREE_CLOBBER_P (rhs))
    4344              :               /* This is a clobber to mark the going out of scope for
    4345              :                  this LHS.  */
    4346      1412366 :               expand_clobber (lhs);
    4347              :             else
    4348     16340552 :               expand_assignment (lhs, rhs,
    4349              :                                  gimple_assign_nontemporal_move_p (
    4350              :                                    assign_stmt));
    4351              :           }
    4352              :         else
    4353              :           {
    4354      6096826 :             rtx target, temp;
    4355      6096826 :             gcc_assert (!gimple_assign_nontemporal_move_p (assign_stmt));
    4356      6096826 :             bool promoted = false;
    4357              : 
    4358      6096826 :             target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    4359      6096826 :             if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
    4360              :               promoted = true;
    4361              : 
    4362              :            /* If we store into a promoted register, don't directly
    4363              :               expand to target.  */
    4364              :             temp = promoted ? NULL_RTX : target;
    4365     12193652 :             temp = expand_expr_real_gassign (assign_stmt, temp,
    4366      6096826 :                                              GET_MODE (target), EXPAND_NORMAL);
    4367              : 
    4368      6096826 :             if (temp == target)
    4369              :               ;
    4370       850827 :             else if (promoted)
    4371              :               {
    4372            2 :                 int unsignedp = SUBREG_PROMOTED_SIGN (target);
    4373              :                 /* If TEMP is a VOIDmode constant, use convert_modes to make
    4374              :                    sure that we properly convert it.  */
    4375            2 :                 if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
    4376              :                   {
    4377            0 :                     temp = convert_modes (GET_MODE (target),
    4378            0 :                                           TYPE_MODE (TREE_TYPE (lhs)),
    4379              :                                           temp, unsignedp);
    4380            0 :                     temp = convert_modes (GET_MODE (SUBREG_REG (target)),
    4381            0 :                                           GET_MODE (target), temp, unsignedp);
    4382              :                   }
    4383              : 
    4384            2 :                 convert_move (SUBREG_REG (target), temp, unsignedp);
    4385              :               }
    4386              :             else
    4387              :               {
    4388       850825 :                 temp = force_operand (temp, target);
    4389       850825 :                 if (temp == target)
    4390              :                   ;
    4391       850825 :                 else if (GET_MODE (target) != BLKmode)
    4392       850824 :                   emit_move_insn (target, temp);
    4393              :                 else
    4394            1 :                   emit_block_move (target, temp, expr_size (lhs),
    4395              :                                    BLOCK_OP_NORMAL);
    4396              :               }
    4397              :           }
    4398              :       }
    4399              :       break;
    4400              : 
    4401            0 :     default:
    4402            0 :       gcc_unreachable ();
    4403              :     }
    4404     32589324 : }
    4405              : 
    4406              : /* Expand one gimple statement STMT and return the last RTL instruction
    4407              :    before any of the newly generated ones.
    4408              : 
    4409              :    In addition to generating the necessary RTL instructions this also
    4410              :    sets REG_EH_REGION notes if necessary and sets the current source
    4411              :    location for diagnostics.  */
    4412              : 
    4413              : static rtx_insn *
    4414     32589326 : expand_gimple_stmt (gimple *stmt)
    4415              : {
    4416     32589326 :   location_t saved_location = input_location;
    4417     32589326 :   rtx_insn *last = get_last_insn ();
    4418     32589326 :   int lp_nr;
    4419              : 
    4420     32589326 :   gcc_assert (cfun);
    4421              : 
    4422              :   /* We need to save and restore the current source location so that errors
    4423              :      discovered during expansion are emitted with the right location.  But
    4424              :      it would be better if the diagnostic routines used the source location
    4425              :      embedded in the tree nodes rather than globals.  */
    4426     32589326 :   if (gimple_has_location (stmt))
    4427     28580372 :     input_location = gimple_location (stmt);
    4428              : 
    4429     32589326 :   expand_gimple_stmt_1 (stmt);
    4430              : 
    4431              :   /* Free any temporaries used to evaluate this statement.  */
    4432     32589324 :   free_temp_slots ();
    4433              : 
    4434     32589324 :   input_location = saved_location;
    4435              : 
    4436              :   /* Mark all insns that may trap.  */
    4437     32589324 :   lp_nr = lookup_stmt_eh_lp (stmt);
    4438     32589324 :   if (lp_nr)
    4439              :     {
    4440       745802 :       rtx_insn *insn;
    4441      4271369 :       for (insn = next_real_insn (last); insn;
    4442      3525567 :            insn = next_real_insn (insn))
    4443              :         {
    4444      3525567 :           if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
    4445              :               /* If we want exceptions for non-call insns, any
    4446              :                  may_trap_p instruction may throw.  */
    4447      3525473 :               && GET_CODE (PATTERN (insn)) != CLOBBER
    4448      3525403 :               && GET_CODE (PATTERN (insn)) != USE
    4449      7050970 :               && insn_could_throw_p (insn))
    4450       803085 :             make_reg_eh_region_note (insn, 0, lp_nr);
    4451              :         }
    4452              :     }
    4453              : 
    4454     32589324 :   return last;
    4455              : }
    4456              : 
    4457              : /* A subroutine of expand_gimple_basic_block.  Expand one GIMPLE_CALL
    4458              :    that has CALL_EXPR_TAILCALL set.  Returns non-null if we actually
    4459              :    generated a tail call (something that might be denied by the ABI
    4460              :    rules governing the call; see calls.cc).
    4461              : 
    4462              :    Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
    4463              :    can still reach the rest of BB.  The case here is __builtin_sqrt,
    4464              :    where the NaN result goes through the external function (with a
    4465              :    tailcall) and the normal result happens via a sqrt instruction.  */
    4466              : 
    4467              : static basic_block
    4468       178659 : expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru,
    4469              :                         rtx_insn *asan_epilog_seq)
    4470              : {
    4471       178659 :   rtx_insn *last2, *last, *first = get_last_insn ();
    4472       178659 :   edge e;
    4473       178659 :   edge_iterator ei;
    4474       178659 :   profile_probability probability;
    4475              : 
    4476       178659 :   last2 = last = expand_gimple_stmt (stmt);
    4477              : 
    4478       975745 :   for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
    4479       746861 :     if (CALL_P (last) && SIBLING_CALL_P (last))
    4480       128434 :       goto found;
    4481              : 
    4482        50225 :   maybe_dump_rtl_for_gimple_stmt (stmt, last2);
    4483              : 
    4484        50225 :   *can_fallthru = true;
    4485        50225 :   return NULL;
    4486              : 
    4487       128434 :  found:
    4488              : 
    4489       128434 :   if (asan_epilog_seq)
    4490              :     {
    4491              :       /* We need to emit a copy of the asan_epilog_seq before
    4492              :          the insns emitted by expand_gimple_stmt above.  The sequence
    4493              :          can contain labels, which need to be remapped.  */
    4494          117 :       hash_map<rtx, rtx_code_label *> label_map;
    4495          117 :       start_sequence ();
    4496          117 :       emit_note (NOTE_INSN_DELETED);
    4497         1767 :       for (rtx_insn *insn = asan_epilog_seq; insn; insn = NEXT_INSN (insn))
    4498         1533 :         switch (GET_CODE (insn))
    4499              :           {
    4500         1182 :           case INSN:
    4501         1182 :           case CALL_INSN:
    4502         1182 :           case JUMP_INSN:
    4503         1182 :             emit_copy_of_insn_after (insn, get_last_insn ());
    4504         1182 :             break;
    4505          234 :           case CODE_LABEL:
    4506          234 :             label_map.put ((rtx) insn, emit_label (gen_label_rtx ()));
    4507          234 :             break;
    4508          117 :           case BARRIER:
    4509          117 :             emit_barrier ();
    4510          117 :             break;
    4511            0 :           default:
    4512            0 :             gcc_unreachable ();
    4513              :           }
    4514         1767 :       for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn))
    4515         1650 :         if (JUMP_P (insn))
    4516              :           {
    4517          234 :             subrtx_ptr_iterator::array_type array;
    4518         1521 :             FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), ALL)
    4519              :               {
    4520         1287 :                 rtx *loc = *iter;
    4521         1287 :                 if (LABEL_REF_P (*loc))
    4522              :                   {
    4523          234 :                     rtx_code_label **lab
    4524          234 :                       = label_map.get ((rtx) label_ref_label (*loc));
    4525          234 :                     gcc_assert (lab);
    4526          234 :                     set_label_ref_label (*loc, *lab);
    4527              :                   }
    4528              :               }
    4529          234 :             if (JUMP_LABEL (insn))
    4530              :               {
    4531          234 :                 rtx_code_label **lab = label_map.get (JUMP_LABEL (insn));
    4532          234 :                 gcc_assert (lab);
    4533          234 :                 JUMP_LABEL (insn) = *lab;
    4534              :               }
    4535          234 :           }
    4536          117 :       asan_epilog_seq = NEXT_INSN (get_insns ());
    4537          117 :       end_sequence ();
    4538          117 :       emit_insn_before (asan_epilog_seq, NEXT_INSN (first));
    4539          117 :     }
    4540              : 
    4541              :   /* ??? Wouldn't it be better to just reset any pending stack adjust?
    4542              :      Any instructions emitted here are about to be deleted.  */
    4543       128434 :   do_pending_stack_adjust ();
    4544              : 
    4545              :   /* Remove any non-eh, non-abnormal edges that don't go to exit.  */
    4546              :   /* ??? I.e. the fallthrough edge.  HOWEVER!  If there were to be
    4547              :      EH or abnormal edges, we shouldn't have created a tail call in
    4548              :      the first place.  So it seems to me we should just be removing
    4549              :      all edges here, or redirecting the existing fallthru edge to
    4550              :      the exit block.  */
    4551              : 
    4552       128434 :   probability = profile_probability::never ();
    4553              : 
    4554       256862 :   for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
    4555              :     {
    4556       128428 :       if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
    4557              :         {
    4558       128422 :           if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
    4559        60517 :             e->dest->count -= e->count ();
    4560       128422 :           probability += e->probability;
    4561       128422 :           expand_remove_edge (e);
    4562              :         }
    4563              :       else
    4564            6 :         ei_next (&ei);
    4565              :     }
    4566              : 
    4567              :   /* This is somewhat ugly: the call_expr expander often emits instructions
    4568              :      after the sibcall (to perform the function return).  These confuse the
    4569              :      find_many_sub_basic_blocks code, so we need to get rid of these.  */
    4570       128434 :   last = NEXT_INSN (last);
    4571       128434 :   gcc_assert (BARRIER_P (last));
    4572              : 
    4573       128434 :   *can_fallthru = false;
    4574       158874 :   while (NEXT_INSN (last))
    4575              :     {
    4576              :       /* For instance an sqrt builtin expander expands if with
    4577              :          sibcall in the then and label for `else`.  */
    4578        30440 :       if (LABEL_P (NEXT_INSN (last)))
    4579              :         {
    4580            0 :           *can_fallthru = true;
    4581            0 :           break;
    4582              :         }
    4583        30440 :       delete_insn (NEXT_INSN (last));
    4584              :     }
    4585              : 
    4586       128434 :   e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
    4587              :                  | EDGE_SIBCALL);
    4588       128434 :   e->probability = probability;
    4589       128434 :   head_end_for_bb[bb->index].second = last;
    4590       256868 :   update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
    4591       128434 :                             head_end_for_bb[bb->index].second, bb);
    4592              : 
    4593       128434 :   if (NEXT_INSN (last))
    4594              :     {
    4595            0 :       bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
    4596              : 
    4597            0 :       last = BB_END (bb);
    4598            0 :       if (BARRIER_P (last))
    4599            0 :         BB_END (bb) = PREV_INSN (last);
    4600              :     }
    4601              : 
    4602       128434 :   maybe_dump_rtl_for_gimple_stmt (stmt, last2);
    4603              : 
    4604       128434 :   return bb;
    4605              : }
    4606              : 
    4607              : /* Return the difference between the floor and the truncated result of
    4608              :    a signed division by OP1 with remainder MOD.  */
    4609              : static rtx
    4610            4 : floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
    4611              : {
    4612              :   /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
    4613            4 :   return gen_rtx_IF_THEN_ELSE
    4614              :     (mode, gen_rtx_NE (BImode, mod, const0_rtx),
    4615              :      gen_rtx_IF_THEN_ELSE
    4616              :      (mode, gen_rtx_LT (BImode,
    4617              :                         gen_rtx_DIV (mode, op1, mod),
    4618              :                         const0_rtx),
    4619              :       constm1_rtx, const0_rtx),
    4620              :      const0_rtx);
    4621              : }
    4622              : 
    4623              : /* Return the difference between the ceil and the truncated result of
    4624              :    a signed division by OP1 with remainder MOD.  */
    4625              : static rtx
    4626            3 : ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
    4627              : {
    4628              :   /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
    4629            3 :   return gen_rtx_IF_THEN_ELSE
    4630              :     (mode, gen_rtx_NE (BImode, mod, const0_rtx),
    4631              :      gen_rtx_IF_THEN_ELSE
    4632              :      (mode, gen_rtx_GT (BImode,
    4633              :                         gen_rtx_DIV (mode, op1, mod),
    4634              :                         const0_rtx),
    4635              :       const1_rtx, const0_rtx),
    4636              :      const0_rtx);
    4637              : }
    4638              : 
    4639              : /* Return the difference between the ceil and the truncated result of
    4640              :    an unsigned division by OP1 with remainder MOD.  */
    4641              : static rtx
    4642            0 : ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
    4643              : {
    4644              :   /* (mod != 0 ? 1 : 0) */
    4645            0 :   return gen_rtx_IF_THEN_ELSE
    4646              :     (mode, gen_rtx_NE (BImode, mod, const0_rtx),
    4647              :      const1_rtx, const0_rtx);
    4648              : }
    4649              : 
    4650              : /* Return the difference between the rounded and the truncated result
    4651              :    of a signed division by OP1 with remainder MOD.  Halfway cases are
    4652              :    rounded away from zero, rather than to the nearest even number.  */
    4653              : static rtx
    4654            0 : round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
    4655              : {
    4656              :   /* (abs (mod) >= abs (op1) - abs (mod)
    4657              :       ? (op1 / mod > 0 ? 1 : -1)
    4658              :       : 0) */
    4659            0 :   return gen_rtx_IF_THEN_ELSE
    4660              :     (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
    4661              :                        gen_rtx_MINUS (mode,
    4662              :                                       gen_rtx_ABS (mode, op1),
    4663              :                                       gen_rtx_ABS (mode, mod))),
    4664              :      gen_rtx_IF_THEN_ELSE
    4665              :      (mode, gen_rtx_GT (BImode,
    4666              :                         gen_rtx_DIV (mode, op1, mod),
    4667              :                         const0_rtx),
    4668              :       const1_rtx, constm1_rtx),
    4669              :      const0_rtx);
    4670              : }
    4671              : 
    4672              : /* Return the difference between the rounded and the truncated result
    4673              :    of a unsigned division by OP1 with remainder MOD.  Halfway cases
    4674              :    are rounded away from zero, rather than to the nearest even
    4675              :    number.  */
    4676              : static rtx
    4677            0 : round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
    4678              : {
    4679              :   /* (mod >= op1 - mod ? 1 : 0) */
    4680            0 :   return gen_rtx_IF_THEN_ELSE
    4681              :     (mode, gen_rtx_GE (BImode, mod,
    4682              :                        gen_rtx_MINUS (mode, op1, mod)),
    4683              :      const1_rtx, const0_rtx);
    4684              : }
    4685              : 
    4686              : /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
    4687              :    any rtl.  */
    4688              : 
    4689              : static rtx
    4690      7510597 : convert_debug_memory_address (scalar_int_mode mode, rtx x,
    4691              :                               addr_space_t as)
    4692              : {
    4693              : #ifndef POINTERS_EXTEND_UNSIGNED
    4694              :   gcc_assert (mode == Pmode
    4695              :               || mode == targetm.addr_space.address_mode (as));
    4696              :   gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
    4697              : #else
    4698      7510597 :   rtx temp;
    4699              : 
    4700      7510597 :   gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
    4701              : 
    4702      7510597 :   if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
    4703              :     return x;
    4704              : 
    4705              :   /* X must have some form of address mode already.  */
    4706            0 :   scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
    4707            0 :   if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
    4708            0 :     x = lowpart_subreg (mode, x, xmode);
    4709            0 :   else if (POINTERS_EXTEND_UNSIGNED > 0)
    4710            0 :     x = gen_rtx_ZERO_EXTEND (mode, x);
    4711              :   else if (!POINTERS_EXTEND_UNSIGNED)
    4712              :     x = gen_rtx_SIGN_EXTEND (mode, x);
    4713              :   else
    4714              :     {
    4715              :       switch (GET_CODE (x))
    4716              :         {
    4717              :         case SUBREG:
    4718              :           if ((SUBREG_PROMOTED_VAR_P (x)
    4719              :                || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
    4720              :                || (GET_CODE (SUBREG_REG (x)) == PLUS
    4721              :                    && REG_P (XEXP (SUBREG_REG (x), 0))
    4722              :                    && REG_POINTER (XEXP (SUBREG_REG (x), 0))
    4723              :                    && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
    4724              :               && GET_MODE (SUBREG_REG (x)) == mode)
    4725              :             return SUBREG_REG (x);
    4726              :           break;
    4727              :         case LABEL_REF:
    4728              :           temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
    4729              :           LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
    4730              :           return temp;
    4731              :         case SYMBOL_REF:
    4732              :           temp = shallow_copy_rtx (x);
    4733              :           PUT_MODE (temp, mode);
    4734              :           return temp;
    4735              :         case CONST:
    4736              :           temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
    4737              :           if (temp)
    4738              :             temp = gen_rtx_CONST (mode, temp);
    4739              :           return temp;
    4740              :         case PLUS:
    4741              :         case MINUS:
    4742              :           if (CONST_INT_P (XEXP (x, 1)))
    4743              :             {
    4744              :               temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
    4745              :               if (temp)
    4746              :                 return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
    4747              :             }
    4748              :           break;
    4749              :         default:
    4750              :           break;
    4751              :         }
    4752              :       /* Don't know how to express ptr_extend as operation in debug info.  */
    4753              :       return NULL;
    4754              :     }
    4755              : #endif /* POINTERS_EXTEND_UNSIGNED */
    4756              : 
    4757              :   return x;
    4758              : }
    4759              : 
    4760              : /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
    4761              :    by avoid_deep_ter_for_debug.  */
    4762              : 
    4763              : static hash_map<tree, tree> *deep_ter_debug_map;
    4764              : 
    4765              : /* Split too deep TER chains for debug stmts using debug temporaries.  */
    4766              : 
    4767              : static void
    4768     42870112 : avoid_deep_ter_for_debug (gimple *stmt, int depth)
    4769              : {
    4770     42870112 :   use_operand_p use_p;
    4771     42870112 :   ssa_op_iter iter;
    4772     53147358 :   FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
    4773              :     {
    4774     10277246 :       tree use = USE_FROM_PTR (use_p);
    4775     10277246 :       if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
    4776      8533279 :         continue;
    4777      8904324 :       gimple *g = get_gimple_for_ssa_name (use);
    4778      8904324 :       if (g == NULL)
    4779      7156901 :         continue;
    4780      1747423 :       if ((depth > 6 || !is_gimple_assign (g)) && !stmt_ends_bb_p (g))
    4781              :         {
    4782         8867 :           if (deep_ter_debug_map == NULL)
    4783          729 :             deep_ter_debug_map = new hash_map<tree, tree>;
    4784              : 
    4785         8867 :           tree &vexpr = deep_ter_debug_map->get_or_insert (use);
    4786         8867 :           if (vexpr != NULL)
    4787         3456 :             continue;
    4788         5411 :           vexpr = build_debug_expr_decl (TREE_TYPE (use));
    4789         5411 :           gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
    4790         5411 :           gimple_stmt_iterator gsi = gsi_for_stmt (g);
    4791         5411 :           gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
    4792         5411 :           avoid_deep_ter_for_debug (def_temp, 0);
    4793              :         }
    4794              :       else
    4795      1738556 :         avoid_deep_ter_for_debug (g, depth + 1);
    4796              :     }
    4797     42870112 : }
    4798              : 
    4799              : /* Return an RTX equivalent to the value of the parameter DECL.  */
    4800              : 
    4801              : static rtx
    4802       189605 : expand_debug_parm_decl (tree decl)
    4803              : {
    4804       189605 :   rtx incoming = DECL_INCOMING_RTL (decl);
    4805              : 
    4806       189605 :   if (incoming
    4807        19661 :       && GET_MODE (incoming) != BLKmode
    4808       209262 :       && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
    4809          962 :           || (MEM_P (incoming)
    4810          958 :               && REG_P (XEXP (incoming, 0))
    4811          343 :               && HARD_REGISTER_P (XEXP (incoming, 0)))))
    4812              :     {
    4813        18722 :       rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
    4814              : 
    4815              : #ifdef HAVE_window_save
    4816              :       /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
    4817              :          If the target machine has an explicit window save instruction, the
    4818              :          actual entry value is the corresponding OUTGOING_REGNO instead.  */
    4819              :       if (REG_P (incoming)
    4820              :           && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
    4821              :         incoming
    4822              :           = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
    4823              :                                 OUTGOING_REGNO (REGNO (incoming)), 0);
    4824              :       else if (MEM_P (incoming))
    4825              :         {
    4826              :           rtx reg = XEXP (incoming, 0);
    4827              :           if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
    4828              :             {
    4829              :               reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
    4830              :               incoming = replace_equiv_address_nv (incoming, reg);
    4831              :             }
    4832              :           else
    4833              :             incoming = copy_rtx (incoming);
    4834              :         }
    4835              : #endif
    4836              : 
    4837        18722 :       ENTRY_VALUE_EXP (rtl) = incoming;
    4838        18722 :       return rtl;
    4839              :     }
    4840              : 
    4841       170883 :   if (incoming
    4842          939 :       && GET_MODE (incoming) != BLKmode
    4843          935 :       && !TREE_ADDRESSABLE (decl)
    4844          935 :       && MEM_P (incoming)
    4845          931 :       && (XEXP (incoming, 0) == virtual_incoming_args_rtx
    4846          615 :           || (GET_CODE (XEXP (incoming, 0)) == PLUS
    4847          615 :               && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
    4848          599 :               && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
    4849          915 :     return copy_rtx (incoming);
    4850              : 
    4851              :   return NULL_RTX;
    4852              : }
    4853              : 
    4854              : /* Return an RTX equivalent to the value of the tree expression EXP.  */
    4855              : 
    4856              : static rtx
    4857     43621833 : expand_debug_expr (tree exp)
    4858              : {
    4859     43621833 :   rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
    4860     43621833 :   machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
    4861     43621833 :   machine_mode inner_mode = VOIDmode;
    4862     43621833 :   int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
    4863     43621833 :   addr_space_t as;
    4864     43621833 :   scalar_int_mode op0_mode, op1_mode, addr_mode;
    4865              : 
    4866     43621833 :   switch (TREE_CODE_CLASS (TREE_CODE (exp)))
    4867              :     {
    4868      7180801 :     case tcc_expression:
    4869      7180801 :       switch (TREE_CODE (exp))
    4870              :         {
    4871         8331 :         case COND_EXPR:
    4872         8331 :         case DOT_PROD_EXPR:
    4873         8331 :         case SAD_EXPR:
    4874         8331 :         case WIDEN_MULT_PLUS_EXPR:
    4875         8331 :         case WIDEN_MULT_MINUS_EXPR:
    4876         8331 :           goto ternary;
    4877              : 
    4878            0 :         case TRUTH_ANDIF_EXPR:
    4879            0 :         case TRUTH_ORIF_EXPR:
    4880            0 :         case TRUTH_AND_EXPR:
    4881            0 :         case TRUTH_OR_EXPR:
    4882            0 :         case TRUTH_XOR_EXPR:
    4883            0 :           goto binary;
    4884              : 
    4885            0 :         case TRUTH_NOT_EXPR:
    4886            0 :           goto unary;
    4887              : 
    4888              :         default:
    4889              :           break;
    4890              :         }
    4891              :       break;
    4892              : 
    4893         8331 :     ternary:
    4894         8331 :       op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
    4895         8331 :       if (!op2)
    4896              :         return NULL_RTX;
    4897              :       /* Fall through.  */
    4898              : 
    4899         8331 :     binary:
    4900      1550158 :     case tcc_binary:
    4901      1550158 :       if (mode == BLKmode)
    4902              :         return NULL_RTX;
    4903      1550148 :       op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
    4904      1550148 :       if (!op1)
    4905              :         return NULL_RTX;
    4906      1549881 :       switch (TREE_CODE (exp))
    4907              :         {
    4908        83378 :         case LSHIFT_EXPR:
    4909        83378 :         case RSHIFT_EXPR:
    4910        83378 :         case LROTATE_EXPR:
    4911        83378 :         case RROTATE_EXPR:
    4912        83378 :         case WIDEN_LSHIFT_EXPR:
    4913              :           /* Ensure second operand isn't wider than the first one.  */
    4914        83378 :           inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
    4915        83378 :           if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
    4916        83370 :               && (GET_MODE_UNIT_PRECISION (mode)
    4917        83370 :                   < GET_MODE_PRECISION (op1_mode)))
    4918         5564 :             op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
    4919              :           break;
    4920              :         default:
    4921              :           break;
    4922              :         }
    4923              :       /* Fall through.  */
    4924              : 
    4925      2146665 :     unary:
    4926      2146665 :     case tcc_unary:
    4927      2146665 :       if (mode == BLKmode)
    4928              :         return NULL_RTX;
    4929      2146663 :       inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
    4930      2146663 :       op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
    4931      2146663 :       if (!op0)
    4932              :         return NULL_RTX;
    4933              :       break;
    4934              : 
    4935        73870 :     case tcc_comparison:
    4936        73870 :       unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
    4937        73870 :       goto binary;
    4938              : 
    4939            0 :     case tcc_type:
    4940            0 :     case tcc_statement:
    4941            0 :       gcc_unreachable ();
    4942              : 
    4943              :     case tcc_constant:
    4944              :     case tcc_exceptional:
    4945              :     case tcc_declaration:
    4946              :     case tcc_reference:
    4947              :     case tcc_vl_exp:
    4948              :       break;
    4949              :     }
    4950              : 
    4951     43617272 :   switch (TREE_CODE (exp))
    4952              :     {
    4953       276156 :     case STRING_CST:
    4954       276156 :       if (!lookup_constant_def (exp))
    4955              :         {
    4956        98746 :           if (strlen (TREE_STRING_POINTER (exp)) + 1
    4957        98746 :               != (size_t) TREE_STRING_LENGTH (exp))
    4958              :             return NULL_RTX;
    4959        98847 :           op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
    4960        98027 :           op0 = gen_rtx_MEM (BLKmode, op0);
    4961        98027 :           set_mem_attributes (op0, exp, 0);
    4962        98027 :           return op0;
    4963              :         }
    4964              :       /* Fall through.  */
    4965              : 
    4966      6252588 :     case INTEGER_CST:
    4967     12505134 :       if (BITINT_TYPE_P (TREE_TYPE (exp))
    4968      6252588 :           && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
    4969              :         return NULL;
    4970              :       /* FALLTHRU */
    4971      6325867 :     case REAL_CST:
    4972      6325867 :     case FIXED_CST:
    4973      6325867 :       op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
    4974      6325867 :       return op0;
    4975              : 
    4976            0 :     case POLY_INT_CST:
    4977            0 :       return immed_wide_int_const (poly_int_cst_value (exp), mode);
    4978              : 
    4979         4063 :     case COMPLEX_CST:
    4980         4063 :       gcc_assert (COMPLEX_MODE_P (mode));
    4981         4063 :       op0 = expand_debug_expr (TREE_REALPART (exp));
    4982         4063 :       op1 = expand_debug_expr (TREE_IMAGPART (exp));
    4983         4063 :       return gen_rtx_CONCAT (mode, op0, op1);
    4984              : 
    4985      6268160 :     case DEBUG_EXPR_DECL:
    4986      6268160 :       op0 = DECL_RTL_IF_SET (exp);
    4987              : 
    4988      5054229 :       if (op0)
    4989              :         {
    4990      5054229 :           if (GET_MODE (op0) != mode)
    4991            0 :             gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
    4992              :           else
    4993              :             return op0;
    4994              :         }
    4995              : 
    4996      1213931 :       op0 = gen_rtx_DEBUG_EXPR (mode);
    4997      1213931 :       DEBUG_EXPR_TREE_DECL (op0) = exp;
    4998      1213931 :       SET_DECL_RTL (exp, op0);
    4999              : 
    5000      1213931 :       return op0;
    5001              : 
    5002      5641778 :     case VAR_DECL:
    5003      5641778 :     case PARM_DECL:
    5004      5641778 :     case FUNCTION_DECL:
    5005      5641778 :     case LABEL_DECL:
    5006      5641778 :     case CONST_DECL:
    5007      5641778 :     case RESULT_DECL:
    5008      5641778 :       op0 = DECL_RTL_IF_SET (exp);
    5009              : 
    5010              :       /* This decl was probably optimized away.  */
    5011      3899295 :       if (!op0
    5012              :           /* At least label RTXen are sometimes replaced by
    5013              :              NOTE_INSN_DELETED_LABEL.  Any notes here are not
    5014              :              handled by copy_rtx.  */
    5015      3899295 :           || NOTE_P (op0))
    5016              :         {
    5017      1742484 :           if (!VAR_P (exp)
    5018      1729946 :               || DECL_EXTERNAL (exp)
    5019      1729323 :               || !TREE_STATIC (exp)
    5020       135300 :               || !DECL_NAME (exp)
    5021       135300 :               || DECL_HARD_REGISTER (exp)
    5022       135300 :               || DECL_IN_CONSTANT_POOL (exp)
    5023       135300 :               || mode == VOIDmode
    5024      1877784 :               || symtab_node::get (exp) == NULL)
    5025              :             return NULL;
    5026              : 
    5027        19596 :           op0 = make_decl_rtl_for_debug (exp);
    5028        19596 :           if (!MEM_P (op0)
    5029        19596 :               || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
    5030        39192 :               || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
    5031              :             return NULL;
    5032              :         }
    5033      3899294 :       else if (VAR_P (exp)
    5034      3770550 :                && is_global_var (exp)
    5035      4127376 :                && symtab_node::get (exp) == NULL)
    5036              :         return NULL;
    5037              :       else
    5038      3899294 :         op0 = copy_rtx (op0);
    5039              : 
    5040      3918890 :       if (GET_MODE (op0) == BLKmode
    5041              :           /* If op0 is not BLKmode, but mode is, adjust_mode
    5042              :              below would ICE.  While it is likely a FE bug,
    5043              :              try to be robust here.  See PR43166.  */
    5044       543488 :           || mode == BLKmode
    5045       543488 :           || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
    5046              :         {
    5047      3375402 :           gcc_assert (MEM_P (op0));
    5048      3375402 :           op0 = adjust_address_nv (op0, mode, 0);
    5049      3375402 :           return op0;
    5050              :         }
    5051              : 
    5052              :       /* Fall through.  */
    5053              : 
    5054       543488 :     adjust_mode:
    5055     10871139 :     case PAREN_EXPR:
    5056     10871139 :     CASE_CONVERT:
    5057     10871139 :       {
    5058     10871139 :         inner_mode = GET_MODE (op0);
    5059              : 
    5060     10871139 :         if (mode == inner_mode)
    5061              :           return op0;
    5062              : 
    5063       159463 :         if (inner_mode == VOIDmode)
    5064              :           {
    5065         8300 :             if (TREE_CODE (exp) == SSA_NAME)
    5066           37 :               inner_mode = TYPE_MODE (TREE_TYPE (exp));
    5067              :             else
    5068         8263 :               inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
    5069         8300 :             if (mode == inner_mode)
    5070              :               return op0;
    5071              :           }
    5072              : 
    5073       153884 :         if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
    5074              :           {
    5075         6302 :             if (GET_MODE_UNIT_BITSIZE (mode)
    5076        12604 :                 == GET_MODE_UNIT_BITSIZE (inner_mode))
    5077            4 :               op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
    5078         6298 :             else if (GET_MODE_UNIT_BITSIZE (mode)
    5079        12596 :                      < GET_MODE_UNIT_BITSIZE (inner_mode))
    5080         3579 :               op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
    5081              :             else
    5082         2719 :               op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
    5083              :           }
    5084       147582 :         else if (FLOAT_MODE_P (mode))
    5085              :           {
    5086            0 :             gcc_assert (TREE_CODE (exp) != SSA_NAME);
    5087            0 :             if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
    5088            0 :               op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
    5089              :             else
    5090            0 :               op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
    5091              :           }
    5092       147582 :         else if (FLOAT_MODE_P (inner_mode))
    5093              :           {
    5094            0 :             if (unsignedp)
    5095            0 :               op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
    5096              :             else
    5097            0 :               op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
    5098              :           }
    5099       295164 :         else if (GET_MODE_UNIT_PRECISION (mode)
    5100       147582 :                  == GET_MODE_UNIT_PRECISION (inner_mode))
    5101            0 :           op0 = lowpart_subreg (mode, op0, inner_mode);
    5102       295164 :         else if (GET_MODE_UNIT_PRECISION (mode)
    5103       147582 :                  < GET_MODE_UNIT_PRECISION (inner_mode))
    5104        71998 :           op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
    5105        75584 :         else if (UNARY_CLASS_P (exp)
    5106        75584 :                  ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
    5107            0 :                  : unsignedp)
    5108        50733 :           op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
    5109              :         else
    5110        24851 :           op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
    5111              : 
    5112              :         return op0;
    5113              :       }
    5114              : 
    5115      2240705 :     case MEM_REF:
    5116      2240705 :       if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
    5117              :         {
    5118       990155 :           tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
    5119              :                                      TREE_OPERAND (exp, 0),
    5120              :                                      TREE_OPERAND (exp, 1));
    5121       990155 :           if (newexp)
    5122            0 :             return expand_debug_expr (newexp);
    5123              :         }
    5124              :       /* FALLTHROUGH */
    5125      2240705 :     case INDIRECT_REF:
    5126      2240705 :       inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
    5127      2240705 :       op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
    5128      2240705 :       if (!op0)
    5129              :         return NULL;
    5130              : 
    5131      2234096 :       if (TREE_CODE (exp) == MEM_REF)
    5132              :         {
    5133      2234096 :           if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
    5134      2187269 :               || (GET_CODE (op0) == PLUS
    5135       286288 :                   && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
    5136              :             /* (mem (debug_implicit_ptr)) might confuse aliasing.
    5137              :                Instead just use get_inner_reference.  */
    5138        46827 :             goto component_ref;
    5139              : 
    5140      2187269 :           op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
    5141      2187269 :           poly_int64 offset;
    5142      2187269 :           if (!op1 || !poly_int_rtx_p (op1, &offset))
    5143            0 :             return NULL;
    5144              : 
    5145      2187269 :           op0 = plus_constant (inner_mode, op0, offset);
    5146              :         }
    5147              : 
    5148      2187269 :       as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
    5149              : 
    5150      2187269 :       op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
    5151              :                                           op0, as);
    5152      2187269 :       if (op0 == NULL_RTX)
    5153              :         return NULL;
    5154              : 
    5155      2187269 :       op0 = gen_rtx_MEM (mode, op0);
    5156      2187269 :       set_mem_attributes (op0, exp, 0);
    5157      2187269 :       if (TREE_CODE (exp) == MEM_REF
    5158      2187269 :           && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
    5159       990155 :         set_mem_expr (op0, NULL_TREE);
    5160      2187269 :       set_mem_addr_space (op0, as);
    5161              : 
    5162      2187269 :       return op0;
    5163              : 
    5164        56210 :     case TARGET_MEM_REF:
    5165        56210 :       if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
    5166        56210 :           && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
    5167              :         return NULL;
    5168              : 
    5169        56202 :       op0 = expand_debug_expr
    5170        56202 :             (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
    5171        56202 :       if (!op0)
    5172              :         return NULL;
    5173              : 
    5174        56202 :       as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
    5175        56202 :       op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
    5176              :                                           op0, as);
    5177        56202 :       if (op0 == NULL_RTX)
    5178              :         return NULL;
    5179              : 
    5180        56202 :       op0 = gen_rtx_MEM (mode, op0);
    5181              : 
    5182        56202 :       set_mem_attributes (op0, exp, 0);
    5183        56202 :       set_mem_addr_space (op0, as);
    5184              : 
    5185        56202 :       return op0;
    5186              : 
    5187        46827 :     component_ref:
    5188      3912734 :     case ARRAY_REF:
    5189      3912734 :     case ARRAY_RANGE_REF:
    5190      3912734 :     case COMPONENT_REF:
    5191      3912734 :     case BIT_FIELD_REF:
    5192      3912734 :     case REALPART_EXPR:
    5193      3912734 :     case IMAGPART_EXPR:
    5194      3912734 :     case VIEW_CONVERT_EXPR:
    5195      3912734 :       {
    5196      3912734 :         machine_mode mode1;
    5197      3912734 :         poly_int64 bitsize, bitpos;
    5198      3912734 :         tree offset;
    5199      3912734 :         int reversep, volatilep = 0;
    5200      3912734 :         tree tem
    5201      3912734 :           = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
    5202              :                                  &unsignedp, &reversep, &volatilep);
    5203      3912734 :         rtx orig_op0;
    5204              : 
    5205      3912734 :         if (known_eq (bitsize, 0))
    5206              :           return NULL;
    5207              : 
    5208      3853115 :         orig_op0 = op0 = expand_debug_expr (tem);
    5209              : 
    5210      3853115 :         if (!op0)
    5211              :           return NULL;
    5212              : 
    5213      3503536 :         if (offset)
    5214              :           {
    5215        12169 :             machine_mode addrmode, offmode;
    5216              : 
    5217        12169 :             if (!MEM_P (op0))
    5218              :               return NULL;
    5219              : 
    5220        12161 :             op0 = XEXP (op0, 0);
    5221        12161 :             addrmode = GET_MODE (op0);
    5222        12161 :             if (addrmode == VOIDmode)
    5223            0 :               addrmode = Pmode;
    5224              : 
    5225        12161 :             op1 = expand_debug_expr (offset);
    5226        12161 :             if (!op1)
    5227              :               return NULL;
    5228              : 
    5229        12159 :             offmode = GET_MODE (op1);
    5230        12159 :             if (offmode == VOIDmode)
    5231            6 :               offmode = TYPE_MODE (TREE_TYPE (offset));
    5232              : 
    5233        12159 :             if (addrmode != offmode)
    5234            0 :               op1 = lowpart_subreg (addrmode, op1, offmode);
    5235              : 
    5236              :             /* Don't use offset_address here, we don't need a
    5237              :                recognizable address, and we don't want to generate
    5238              :                code.  */
    5239        12159 :             op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
    5240              :                                                           op0, op1));
    5241              :           }
    5242              : 
    5243      3503526 :         if (MEM_P (op0))
    5244              :           {
    5245      3365542 :             if (mode1 == VOIDmode)
    5246              :               {
    5247         3437 :                 if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
    5248      3912734 :                   return NULL;
    5249              :                 /* Bitfield.  */
    5250         3437 :                 mode1 = smallest_int_mode_for_size (bitsize).require ();
    5251              :               }
    5252      3365542 :             poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
    5253      3365542 :             if (maybe_ne (bytepos, 0))
    5254              :               {
    5255      1385138 :                 op0 = adjust_address_nv (op0, mode1, bytepos);
    5256      1385138 :                 bitpos = num_trailing_bits (bitpos);
    5257              :               }
    5258      1980404 :             else if (known_eq (bitpos, 0)
    5259      3960180 :                      && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
    5260       515316 :               op0 = adjust_address_nv (op0, mode, 0);
    5261      1465088 :             else if (GET_MODE (op0) != mode1)
    5262         3111 :               op0 = adjust_address_nv (op0, mode1, 0);
    5263              :             else
    5264      1461977 :               op0 = copy_rtx (op0);
    5265      3365542 :             if (op0 == orig_op0)
    5266       152152 :               op0 = shallow_copy_rtx (op0);
    5267      3365542 :             if (TREE_CODE (tem) != SSA_NAME)
    5268      3363157 :               set_mem_attributes (op0, exp, 0);
    5269              :           }
    5270              : 
    5271      3503526 :         if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
    5272              :           return op0;
    5273              : 
    5274       103726 :         if (maybe_lt (bitpos, 0))
    5275              :           return NULL;
    5276              : 
    5277       103724 :         if (GET_MODE (op0) == BLKmode || mode == BLKmode)
    5278              :           return NULL;
    5279              : 
    5280       103687 :         poly_int64 bytepos;
    5281       103687 :         if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
    5282       204196 :             && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
    5283              :           {
    5284       101258 :             machine_mode opmode = GET_MODE (op0);
    5285              : 
    5286       101258 :             if (opmode == VOIDmode)
    5287            0 :               opmode = TYPE_MODE (TREE_TYPE (tem));
    5288              : 
    5289              :             /* This condition may hold if we're expanding the address
    5290              :                right past the end of an array that turned out not to
    5291              :                be addressable (i.e., the address was only computed in
    5292              :                debug stmts).  The gen_subreg below would rightfully
    5293              :                crash, and the address doesn't really exist, so just
    5294              :                drop it.  */
    5295       202516 :             if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
    5296              :               return NULL;
    5297              : 
    5298       204893 :             if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
    5299       101186 :               return simplify_gen_subreg (mode, op0, opmode, bytepos);
    5300              :           }
    5301              : 
    5302         4950 :         return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
    5303         1852 :                                      && TYPE_UNSIGNED (TREE_TYPE (exp))
    5304              :                                      ? SIGN_EXTRACT
    5305              :                                      : ZERO_EXTRACT, mode,
    5306         2475 :                                      GET_MODE (op0) != VOIDmode
    5307              :                                      ? GET_MODE (op0)
    5308            0 :                                      : TYPE_MODE (TREE_TYPE (tem)),
    5309              :                                      op0, gen_int_mode (bitsize, word_mode),
    5310         2475 :                                      gen_int_mode (bitpos, word_mode));
    5311              :       }
    5312              : 
    5313         3997 :     case ABS_EXPR:
    5314         3997 :     case ABSU_EXPR:
    5315         3997 :       return simplify_gen_unary (ABS, mode, op0, mode);
    5316              : 
    5317        14270 :     case NEGATE_EXPR:
    5318        14270 :       return simplify_gen_unary (NEG, mode, op0, mode);
    5319              : 
    5320        20945 :     case BIT_NOT_EXPR:
    5321        20945 :       return simplify_gen_unary (NOT, mode, op0, mode);
    5322              : 
    5323         3330 :     case FLOAT_EXPR:
    5324         3330 :       return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
    5325              :                                                                          0)))
    5326              :                                  ? UNSIGNED_FLOAT : FLOAT, mode, op0,
    5327         3330 :                                  inner_mode);
    5328              : 
    5329          456 :     case FIX_TRUNC_EXPR:
    5330          456 :       return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
    5331          456 :                                  inner_mode);
    5332              : 
    5333       239884 :     case POINTER_PLUS_EXPR:
    5334              :       /* For the rare target where pointers are not the same size as
    5335              :          size_t, we need to check for mis-matched modes and correct
    5336              :          the addend.  */
    5337       239884 :       if (op0 && op1
    5338       239884 :           && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
    5339       239782 :           && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
    5340       381679 :           && op0_mode != op1_mode)
    5341              :         {
    5342            0 :           if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
    5343              :               /* If OP0 is a partial mode, then we must truncate, even
    5344              :                  if it has the same bitsize as OP1 as GCC's
    5345              :                  representation of partial modes is opaque.  */
    5346            0 :               || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
    5347            0 :                   && (GET_MODE_BITSIZE (op0_mode)
    5348            0 :                       == GET_MODE_BITSIZE (op1_mode))))
    5349            0 :             op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
    5350              :           else
    5351              :             /* We always sign-extend, regardless of the signedness of
    5352              :                the operand, because the operand is always unsigned
    5353              :                here even if the original C expression is signed.  */
    5354            0 :             op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
    5355              :         }
    5356              :       /* Fall through.  */
    5357       737893 :     case PLUS_EXPR:
    5358       737893 :       return simplify_gen_binary (PLUS, mode, op0, op1);
    5359              : 
    5360       114655 :     case MINUS_EXPR:
    5361       114655 :     case POINTER_DIFF_EXPR:
    5362       114655 :       return simplify_gen_binary (MINUS, mode, op0, op1);
    5363              : 
    5364       179548 :     case MULT_EXPR:
    5365       179548 :       return simplify_gen_binary (MULT, mode, op0, op1);
    5366              : 
    5367         4145 :     case RDIV_EXPR:
    5368         4145 :       gcc_assert (FLOAT_MODE_P (mode)
    5369              :                   || ALL_FIXED_POINT_MODE_P (mode));
    5370              :       /* Fall through.  */
    5371       100982 :     case TRUNC_DIV_EXPR:
    5372       100982 :     case EXACT_DIV_EXPR:
    5373       100982 :       if (unsignedp)
    5374        10501 :         return simplify_gen_binary (UDIV, mode, op0, op1);
    5375              :       else
    5376        90481 :         return simplify_gen_binary (DIV, mode, op0, op1);
    5377              : 
    5378         4173 :     case TRUNC_MOD_EXPR:
    5379         8346 :       return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
    5380              : 
    5381            3 :     case FLOOR_DIV_EXPR:
    5382            3 :       if (unsignedp)
    5383            0 :         return simplify_gen_binary (UDIV, mode, op0, op1);
    5384              :       else
    5385              :         {
    5386            3 :           rtx div = simplify_gen_binary (DIV, mode, op0, op1);
    5387            3 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5388            3 :           rtx adj = floor_sdiv_adjust (mode, mod, op1);
    5389            3 :           return simplify_gen_binary (PLUS, mode, div, adj);
    5390              :         }
    5391              : 
    5392            1 :     case FLOOR_MOD_EXPR:
    5393            1 :       if (unsignedp)
    5394            0 :         return simplify_gen_binary (UMOD, mode, op0, op1);
    5395              :       else
    5396              :         {
    5397            1 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5398            1 :           rtx adj = floor_sdiv_adjust (mode, mod, op1);
    5399            1 :           adj = simplify_gen_unary (NEG, mode,
    5400              :                                     simplify_gen_binary (MULT, mode, adj, op1),
    5401              :                                     mode);
    5402            1 :           return simplify_gen_binary (PLUS, mode, mod, adj);
    5403              :         }
    5404              : 
    5405            3 :     case CEIL_DIV_EXPR:
    5406            3 :       if (unsignedp)
    5407              :         {
    5408            0 :           rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
    5409            0 :           rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
    5410            0 :           rtx adj = ceil_udiv_adjust (mode, mod, op1);
    5411            0 :           return simplify_gen_binary (PLUS, mode, div, adj);
    5412              :         }
    5413              :       else
    5414              :         {
    5415            3 :           rtx div = simplify_gen_binary (DIV, mode, op0, op1);
    5416            3 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5417            3 :           rtx adj = ceil_sdiv_adjust (mode, mod, op1);
    5418            3 :           return simplify_gen_binary (PLUS, mode, div, adj);
    5419              :         }
    5420              : 
    5421            0 :     case CEIL_MOD_EXPR:
    5422            0 :       if (unsignedp)
    5423              :         {
    5424            0 :           rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
    5425            0 :           rtx adj = ceil_udiv_adjust (mode, mod, op1);
    5426            0 :           adj = simplify_gen_unary (NEG, mode,
    5427              :                                     simplify_gen_binary (MULT, mode, adj, op1),
    5428              :                                     mode);
    5429            0 :           return simplify_gen_binary (PLUS, mode, mod, adj);
    5430              :         }
    5431              :       else
    5432              :         {
    5433            0 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5434            0 :           rtx adj = ceil_sdiv_adjust (mode, mod, op1);
    5435            0 :           adj = simplify_gen_unary (NEG, mode,
    5436              :                                     simplify_gen_binary (MULT, mode, adj, op1),
    5437              :                                     mode);
    5438            0 :           return simplify_gen_binary (PLUS, mode, mod, adj);
    5439              :         }
    5440              : 
    5441            0 :     case ROUND_DIV_EXPR:
    5442            0 :       if (unsignedp)
    5443              :         {
    5444            0 :           rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
    5445            0 :           rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
    5446            0 :           rtx adj = round_udiv_adjust (mode, mod, op1);
    5447            0 :           return simplify_gen_binary (PLUS, mode, div, adj);
    5448              :         }
    5449              :       else
    5450              :         {
    5451            0 :           rtx div = simplify_gen_binary (DIV, mode, op0, op1);
    5452            0 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5453            0 :           rtx adj = round_sdiv_adjust (mode, mod, op1);
    5454            0 :           return simplify_gen_binary (PLUS, mode, div, adj);
    5455              :         }
    5456              : 
    5457            0 :     case ROUND_MOD_EXPR:
    5458            0 :       if (unsignedp)
    5459              :         {
    5460            0 :           rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
    5461            0 :           rtx adj = round_udiv_adjust (mode, mod, op1);
    5462            0 :           adj = simplify_gen_unary (NEG, mode,
    5463              :                                     simplify_gen_binary (MULT, mode, adj, op1),
    5464              :                                     mode);
    5465            0 :           return simplify_gen_binary (PLUS, mode, mod, adj);
    5466              :         }
    5467              :       else
    5468              :         {
    5469            0 :           rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
    5470            0 :           rtx adj = round_sdiv_adjust (mode, mod, op1);
    5471            0 :           adj = simplify_gen_unary (NEG, mode,
    5472              :                                     simplify_gen_binary (MULT, mode, adj, op1),
    5473              :                                     mode);
    5474            0 :           return simplify_gen_binary (PLUS, mode, mod, adj);
    5475              :         }
    5476              : 
    5477        14865 :     case LSHIFT_EXPR:
    5478        14865 :       return simplify_gen_binary (ASHIFT, mode, op0, op1);
    5479              : 
    5480        67584 :     case RSHIFT_EXPR:
    5481        67584 :       if (unsignedp)
    5482        61295 :         return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
    5483              :       else
    5484         6289 :         return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
    5485              : 
    5486            2 :     case LROTATE_EXPR:
    5487            2 :       return simplify_gen_binary (ROTATE, mode, op0, op1);
    5488              : 
    5489          883 :     case RROTATE_EXPR:
    5490          883 :       return simplify_gen_binary (ROTATERT, mode, op0, op1);
    5491              : 
    5492        40034 :     case MIN_EXPR:
    5493        80068 :       return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
    5494              : 
    5495        39656 :     case MAX_EXPR:
    5496        79312 :       return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
    5497              : 
    5498        93454 :     case BIT_AND_EXPR:
    5499        93454 :     case TRUTH_AND_EXPR:
    5500        93454 :       return simplify_gen_binary (AND, mode, op0, op1);
    5501              : 
    5502        43158 :     case BIT_IOR_EXPR:
    5503        43158 :     case TRUTH_OR_EXPR:
    5504        43158 :       return simplify_gen_binary (IOR, mode, op0, op1);
    5505              : 
    5506        11273 :     case BIT_XOR_EXPR:
    5507        11273 :     case TRUTH_XOR_EXPR:
    5508        11273 :       return simplify_gen_binary (XOR, mode, op0, op1);
    5509              : 
    5510            0 :     case TRUTH_ANDIF_EXPR:
    5511            0 :       return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
    5512              : 
    5513            0 :     case TRUTH_ORIF_EXPR:
    5514            0 :       return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
    5515              : 
    5516            0 :     case TRUTH_NOT_EXPR:
    5517            0 :       return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
    5518              : 
    5519         5229 :     case LT_EXPR:
    5520         5229 :       return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
    5521         5229 :                                       op0, op1);
    5522              : 
    5523         6269 :     case LE_EXPR:
    5524         6269 :       return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
    5525         6269 :                                       op0, op1);
    5526              : 
    5527         9974 :     case GT_EXPR:
    5528         9974 :       return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
    5529         9974 :                                       op0, op1);
    5530              : 
    5531         3096 :     case GE_EXPR:
    5532         3096 :       return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
    5533         3096 :                                       op0, op1);
    5534              : 
    5535        30737 :     case EQ_EXPR:
    5536        30737 :       return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
    5537              : 
    5538        15368 :     case NE_EXPR:
    5539        15368 :       return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
    5540              : 
    5541         1587 :     case UNORDERED_EXPR:
    5542         1587 :       return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
    5543              : 
    5544            1 :     case ORDERED_EXPR:
    5545            1 :       return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
    5546              : 
    5547            0 :     case UNLT_EXPR:
    5548            0 :       return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
    5549              : 
    5550         1576 :     case UNLE_EXPR:
    5551         1576 :       return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
    5552              : 
    5553            0 :     case UNGT_EXPR:
    5554            0 :       return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
    5555              : 
    5556            0 :     case UNGE_EXPR:
    5557            0 :       return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
    5558              : 
    5559            0 :     case UNEQ_EXPR:
    5560            0 :       return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
    5561              : 
    5562            0 :     case LTGT_EXPR:
    5563            0 :       return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
    5564              : 
    5565         8331 :     case COND_EXPR:
    5566         8331 :       return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
    5567              : 
    5568        10745 :     case COMPLEX_EXPR:
    5569        10745 :       gcc_assert (COMPLEX_MODE_P (mode));
    5570        10745 :       if (GET_MODE (op0) == VOIDmode)
    5571            4 :         op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
    5572        10745 :       if (GET_MODE (op1) == VOIDmode)
    5573            0 :         op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
    5574        10745 :       return gen_rtx_CONCAT (mode, op0, op1);
    5575              : 
    5576            0 :     case CONJ_EXPR:
    5577            0 :       if (GET_CODE (op0) == CONCAT)
    5578            0 :         return gen_rtx_CONCAT (mode, XEXP (op0, 0),
    5579              :                                simplify_gen_unary (NEG, GET_MODE_INNER (mode),
    5580              :                                                    XEXP (op0, 1),
    5581              :                                                    GET_MODE_INNER (mode)));
    5582              :       else
    5583              :         {
    5584            0 :           scalar_mode imode = GET_MODE_INNER (mode);
    5585            0 :           rtx re, im;
    5586              : 
    5587            0 :           if (MEM_P (op0))
    5588              :             {
    5589            0 :               re = adjust_address_nv (op0, imode, 0);
    5590            0 :               im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
    5591              :             }
    5592              :           else
    5593              :             {
    5594            0 :               scalar_int_mode ifmode;
    5595            0 :               scalar_int_mode ihmode;
    5596            0 :               rtx halfsize;
    5597            0 :               if (!int_mode_for_mode (mode).exists (&ifmode)
    5598            0 :                   || !int_mode_for_mode (imode).exists (&ihmode))
    5599     43621833 :                 return NULL;
    5600            0 :               halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
    5601            0 :               re = op0;
    5602            0 :               if (mode != ifmode)
    5603            0 :                 re = gen_rtx_SUBREG (ifmode, re, 0);
    5604            0 :               re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
    5605            0 :               if (imode != ihmode)
    5606            0 :                 re = gen_rtx_SUBREG (imode, re, 0);
    5607            0 :               im = copy_rtx (op0);
    5608            0 :               if (mode != ifmode)
    5609            0 :                 im = gen_rtx_SUBREG (ifmode, im, 0);
    5610            0 :               im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
    5611            0 :               if (imode != ihmode)
    5612            0 :                 im = gen_rtx_SUBREG (imode, im, 0);
    5613              :             }
    5614            0 :           im = gen_rtx_NEG (imode, im);
    5615            0 :           return gen_rtx_CONCAT (mode, re, im);
    5616              :         }
    5617              : 
    5618      7164396 :     case ADDR_EXPR:
    5619      7164396 :       op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
    5620      7164396 :       if (!op0 || !MEM_P (op0))
    5621              :         {
    5622      1897270 :           if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
    5623       437895 :                || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
    5624       388945 :                || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
    5625      1946220 :               && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
    5626       190234 :                   || target_for_debug_bind (TREE_OPERAND (exp, 0))))
    5627      1329481 :             return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
    5628              : 
    5629       567789 :           if (handled_component_p (TREE_OPERAND (exp, 0)))
    5630              :             {
    5631       363698 :               poly_int64 bitoffset, bitsize, maxsize, byteoffset;
    5632       363698 :               bool reverse;
    5633       363698 :               tree decl
    5634       363698 :                 = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
    5635              :                                            &bitsize, &maxsize, &reverse);
    5636       363698 :               if ((VAR_P (decl)
    5637              :                    || TREE_CODE (decl) == PARM_DECL
    5638              :                    || TREE_CODE (decl) == RESULT_DECL)
    5639       332044 :                   && (!TREE_ADDRESSABLE (decl)
    5640        30216 :                       || target_for_debug_bind (decl))
    5641        78963 :                   && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
    5642       301828 :                   && known_gt (bitsize, 0)
    5643       284738 :                   && known_eq (bitsize, maxsize))
    5644              :                 {
    5645       284735 :                   rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
    5646       284735 :                   return plus_constant (mode, base, byteoffset);
    5647              :                 }
    5648              :             }
    5649              : 
    5650       283054 :           if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
    5651       283054 :               && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
    5652              :                  == ADDR_EXPR)
    5653              :             {
    5654        12144 :               op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
    5655              :                                                      0));
    5656        12144 :               if (op0 != NULL
    5657         6360 :                   && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
    5658            0 :                       || (GET_CODE (op0) == PLUS
    5659            0 :                           && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
    5660            0 :                           && CONST_INT_P (XEXP (op0, 1)))))
    5661              :                 {
    5662         6360 :                   op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
    5663              :                                                          1));
    5664         6360 :                   poly_int64 offset;
    5665        12720 :                   if (!op1 || !poly_int_rtx_p (op1, &offset))
    5666              :                     return NULL;
    5667              : 
    5668         6360 :                   return plus_constant (mode, op0, offset);
    5669              :                 }
    5670              :             }
    5671              : 
    5672              :           return NULL;
    5673              :         }
    5674              : 
    5675      5267126 :       as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
    5676      5267126 :       addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
    5677      5267126 :       op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
    5678              : 
    5679      5267126 :       return op0;
    5680              : 
    5681        11837 :     case VECTOR_CST:
    5682        11837 :       {
    5683        11837 :         unsigned HOST_WIDE_INT i, nelts;
    5684              : 
    5685        11837 :         if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
    5686              :           return NULL;
    5687              : 
    5688        11837 :         op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
    5689              : 
    5690        65873 :         for (i = 0; i < nelts; ++i)
    5691              :           {
    5692        54036 :             op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
    5693        54036 :             if (!op1)
    5694              :               return NULL;
    5695        54036 :             XVECEXP (op0, 0, i) = op1;
    5696              :           }
    5697              : 
    5698              :         return op0;
    5699              :       }
    5700              : 
    5701         1128 :     case CONSTRUCTOR:
    5702         1128 :       if (TREE_CLOBBER_P (exp))
    5703              :         return NULL;
    5704         1128 :       else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
    5705              :         {
    5706         1128 :           unsigned i;
    5707         1128 :           poly_uint64 elts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp));
    5708         1128 :           unsigned HOST_WIDE_INT nelts = constant_lower_bound (elts);
    5709         1128 :           tree val;
    5710              : 
    5711         1128 :           op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
    5712              : 
    5713         5215 :           FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
    5714              :             {
    5715         4220 :               op1 = expand_debug_expr (val);
    5716         4220 :               if (!op1)
    5717              :                 return NULL;
    5718         4087 :               XVECEXP (op0, 0, i) = op1;
    5719              :             }
    5720              : 
    5721          995 :           if (i < nelts)
    5722              :             {
    5723          117 :               op1 = expand_debug_expr
    5724          117 :                 (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
    5725              : 
    5726          117 :               if (!op1)
    5727              :                 return NULL;
    5728              : 
    5729          384 :               for (; i < nelts; i++)
    5730          267 :                 XVECEXP (op0, 0, i) = op1;
    5731              :             }
    5732              : 
    5733              :           return op0;
    5734              :         }
    5735              :       else
    5736            0 :         goto flag_unsupported;
    5737              : 
    5738              :     case CALL_EXPR:
    5739              :       /* ??? Maybe handle some builtins?  */
    5740              :       return NULL;
    5741              : 
    5742      9787995 :     case SSA_NAME:
    5743      9787995 :       {
    5744      9787995 :         gimple *g = get_gimple_for_ssa_name (exp);
    5745      9787995 :         if (g)
    5746              :           {
    5747      1510015 :             tree t = NULL_TREE;
    5748      1510015 :             if (deep_ter_debug_map)
    5749              :               {
    5750        72607 :                 tree *slot = deep_ter_debug_map->get (exp);
    5751        72607 :                 if (slot)
    5752        19332 :                   t = *slot;
    5753              :               }
    5754        19332 :             if (t == NULL_TREE)
    5755              :               {
    5756      1496094 :                 if (is_gimple_assign (g))
    5757      1495649 :                   t = gimple_assign_rhs_to_tree (g);
    5758              :                 else
    5759              :                   /* expand_debug_expr doesn't handle CALL_EXPR right now.  */
    5760              :                   return NULL;
    5761              :               }
    5762      1509570 :             op0 = expand_debug_expr (t);
    5763      1509570 :             if (!op0)
    5764              :               return NULL;
    5765              :           }
    5766              :         else
    5767              :           {
    5768              :             /* If this is a reference to an incoming value of
    5769              :                parameter that is never used in the code or where the
    5770              :                incoming value is never used in the code, use
    5771              :                PARM_DECL's DECL_RTL if set.  */
    5772      8277980 :             if (SSA_NAME_IS_DEFAULT_DEF (exp)
    5773      1343655 :                 && SSA_NAME_VAR (exp)
    5774      1343655 :                 && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
    5775      9560679 :                 && has_zero_uses (exp))
    5776              :               {
    5777        18675 :                 op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
    5778        18675 :                 if (op0)
    5779        18667 :                   goto adjust_mode;
    5780            8 :                 op0 = expand_debug_expr (SSA_NAME_VAR (exp));
    5781            8 :                 if (op0)
    5782            8 :                   goto adjust_mode;
    5783              :               }
    5784              : 
    5785      8259305 :             int part = var_to_partition (SA.map, exp);
    5786              : 
    5787      8259305 :             if (part == NO_PARTITION)
    5788              :               return NULL;
    5789              : 
    5790      8254988 :             gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
    5791              : 
    5792      8254988 :             op0 = copy_rtx (SA.partition_to_pseudo[part]);
    5793              :           }
    5794      9758073 :         goto adjust_mode;
    5795              :       }
    5796              : 
    5797              :     case ERROR_MARK:
    5798              :       return NULL;
    5799              : 
    5800              :     /* Vector stuff.  For most of the codes we don't have rtl codes.  */
    5801              :     case REALIGN_LOAD_EXPR:
    5802              :     case VEC_COND_EXPR:
    5803              :     case VEC_PACK_FIX_TRUNC_EXPR:
    5804              :     case VEC_PACK_FLOAT_EXPR:
    5805              :     case VEC_PACK_SAT_EXPR:
    5806              :     case VEC_PACK_TRUNC_EXPR:
    5807              :     case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
    5808              :     case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
    5809              :     case VEC_UNPACK_FLOAT_HI_EXPR:
    5810              :     case VEC_UNPACK_FLOAT_LO_EXPR:
    5811              :     case VEC_UNPACK_HI_EXPR:
    5812              :     case VEC_UNPACK_LO_EXPR:
    5813              :     case VEC_WIDEN_MULT_HI_EXPR:
    5814              :     case VEC_WIDEN_MULT_LO_EXPR:
    5815              :     case VEC_WIDEN_MULT_EVEN_EXPR:
    5816              :     case VEC_WIDEN_MULT_ODD_EXPR:
    5817              :     case VEC_WIDEN_LSHIFT_HI_EXPR:
    5818              :     case VEC_WIDEN_LSHIFT_LO_EXPR:
    5819              :     case VEC_PERM_EXPR:
    5820              :     case VEC_DUPLICATE_EXPR:
    5821              :     case VEC_SERIES_EXPR:
    5822              :     case SAD_EXPR:
    5823              :       return NULL;
    5824              : 
    5825              :     /* Misc codes.  */
    5826              :     case ADDR_SPACE_CONVERT_EXPR:
    5827              :     case FIXED_CONVERT_EXPR:
    5828              :     case OBJ_TYPE_REF:
    5829              :     case WITH_SIZE_EXPR:
    5830              :     case BIT_INSERT_EXPR:
    5831              :       return NULL;
    5832              : 
    5833            0 :     case DOT_PROD_EXPR:
    5834            0 :       if (SCALAR_INT_MODE_P (GET_MODE (op0))
    5835            0 :           && SCALAR_INT_MODE_P (mode))
    5836              :         {
    5837            0 :           op0
    5838            0 :             = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
    5839              :                                                                           0)))
    5840              :                                   ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
    5841              :                                   inner_mode);
    5842            0 :           op1
    5843            0 :             = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
    5844              :                                                                           1)))
    5845              :                                   ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
    5846              :                                   inner_mode);
    5847            0 :           op0 = simplify_gen_binary (MULT, mode, op0, op1);
    5848            0 :           return simplify_gen_binary (PLUS, mode, op0, op2);
    5849              :         }
    5850              :       return NULL;
    5851              : 
    5852         5954 :     case WIDEN_MULT_EXPR:
    5853         5954 :     case WIDEN_MULT_PLUS_EXPR:
    5854         5954 :     case WIDEN_MULT_MINUS_EXPR:
    5855         5954 :       if (SCALAR_INT_MODE_P (GET_MODE (op0))
    5856         5954 :           && SCALAR_INT_MODE_P (mode))
    5857              :         {
    5858         5954 :           inner_mode = GET_MODE (op0);
    5859         5954 :           if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
    5860         5338 :             op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
    5861              :           else
    5862          616 :             op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
    5863         5954 :           if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
    5864         5338 :             op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
    5865              :           else
    5866          616 :             op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
    5867         5954 :           op0 = simplify_gen_binary (MULT, mode, op0, op1);
    5868         5954 :           if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
    5869              :             return op0;
    5870            0 :           else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
    5871            0 :             return simplify_gen_binary (PLUS, mode, op0, op2);
    5872              :           else
    5873            0 :             return simplify_gen_binary (MINUS, mode, op2, op0);
    5874              :         }
    5875              :       return NULL;
    5876              : 
    5877              :     case MULT_HIGHPART_EXPR:
    5878              :       /* ??? Similar to the above.  */
    5879              :       return NULL;
    5880              : 
    5881            0 :     case WIDEN_SUM_EXPR:
    5882            0 :     case WIDEN_LSHIFT_EXPR:
    5883            0 :       if (SCALAR_INT_MODE_P (GET_MODE (op0))
    5884            0 :           && SCALAR_INT_MODE_P (mode))
    5885              :         {
    5886            0 :           op0
    5887            0 :             = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
    5888              :                                                                           0)))
    5889              :                                   ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
    5890              :                                   inner_mode);
    5891            0 :           return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
    5892            0 :                                       ? ASHIFT : PLUS, mode, op0, op1);
    5893              :         }
    5894              :       return NULL;
    5895              : 
    5896            0 :     default:
    5897            0 :     flag_unsupported:
    5898            0 :       if (flag_checking)
    5899              :         {
    5900            0 :           debug_tree (exp);
    5901            0 :           gcc_unreachable ();
    5902              :         }
    5903              :       return NULL;
    5904              :     }
    5905              : }
    5906              : 
    5907              : /* Return an RTX equivalent to the source bind value of the tree expression
    5908              :    EXP.  */
    5909              : 
    5910              : static rtx
    5911       176022 : expand_debug_source_expr (tree exp)
    5912              : {
    5913       330589 :   rtx op0 = NULL_RTX;
    5914       330589 :   machine_mode mode = VOIDmode, inner_mode;
    5915              : 
    5916       330589 :   switch (TREE_CODE (exp))
    5917              :     {
    5918       154567 :     case VAR_DECL:
    5919       154567 :       if (DECL_ABSTRACT_ORIGIN (exp))
    5920       154567 :         return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
    5921              :       break;
    5922       170930 :     case PARM_DECL:
    5923       170930 :       {
    5924       170930 :         mode = DECL_MODE (exp);
    5925       170930 :         op0 = expand_debug_parm_decl (exp);
    5926       170930 :         if (op0)
    5927              :            break;
    5928              :         /* See if this isn't an argument that has been completely
    5929              :            optimized out.  */
    5930       169960 :         if (!DECL_RTL_SET_P (exp)
    5931       169944 :             && !DECL_INCOMING_RTL (exp)
    5932       339904 :             && DECL_ABSTRACT_ORIGIN (current_function_decl))
    5933              :           {
    5934        58482 :             tree aexp = DECL_ORIGIN (exp);
    5935        58482 :             if (DECL_CONTEXT (aexp)
    5936        58482 :                 == DECL_ABSTRACT_ORIGIN (current_function_decl))
    5937              :               {
    5938        19407 :                 vec<tree, va_gc> **debug_args;
    5939        19407 :                 unsigned int ix;
    5940        19407 :                 tree ddecl;
    5941        19407 :                 debug_args = decl_debug_args_lookup (current_function_decl);
    5942        19407 :                 if (debug_args != NULL)
    5943              :                   {
    5944        23943 :                     for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
    5945         4540 :                          ix += 2)
    5946        23935 :                       if (ddecl == aexp)
    5947        19395 :                         return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
    5948              :                   }
    5949              :               }
    5950              :           }
    5951              :         break;
    5952              :       }
    5953              :     default:
    5954              :       break;
    5955              :     }
    5956              : 
    5957              :   if (op0 == NULL_RTX)
    5958              :     return NULL_RTX;
    5959              : 
    5960          970 :   inner_mode = GET_MODE (op0);
    5961          970 :   if (mode == inner_mode)
    5962              :     return op0;
    5963              : 
    5964           21 :   if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
    5965              :     {
    5966            7 :       if (GET_MODE_UNIT_BITSIZE (mode)
    5967           14 :           == GET_MODE_UNIT_BITSIZE (inner_mode))
    5968            0 :         op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
    5969            7 :       else if (GET_MODE_UNIT_BITSIZE (mode)
    5970           14 :                < GET_MODE_UNIT_BITSIZE (inner_mode))
    5971            7 :         op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
    5972              :       else
    5973            0 :         op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
    5974              :     }
    5975           14 :   else if (FLOAT_MODE_P (mode))
    5976            0 :     gcc_unreachable ();
    5977           14 :   else if (FLOAT_MODE_P (inner_mode))
    5978              :     {
    5979            0 :       if (TYPE_UNSIGNED (TREE_TYPE (exp)))
    5980            0 :         op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
    5981              :       else
    5982            0 :         op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
    5983              :     }
    5984           28 :   else if (GET_MODE_UNIT_PRECISION (mode)
    5985           14 :            == GET_MODE_UNIT_PRECISION (inner_mode))
    5986            0 :     op0 = lowpart_subreg (mode, op0, inner_mode);
    5987           28 :   else if (GET_MODE_UNIT_PRECISION (mode)
    5988           14 :            < GET_MODE_UNIT_PRECISION (inner_mode))
    5989           14 :     op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
    5990            0 :   else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
    5991            0 :     op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
    5992              :   else
    5993            0 :     op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
    5994              : 
    5995              :   return op0;
    5996              : }
    5997              : 
    5998              : /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
    5999              :    Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
    6000              :    deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN.  */
    6001              : 
    6002              : static void
    6003     53520680 : avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
    6004              : {
    6005     53520680 :   rtx exp = *exp_p;
    6006              : 
    6007     53520680 :   if (exp == NULL_RTX)
    6008              :     return;
    6009              : 
    6010     53520680 :   if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
    6011              :     return;
    6012              : 
    6013      6942587 :   if (depth == 4)
    6014              :     {
    6015              :       /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL).  */
    6016        25984 :       rtx dval = make_debug_expr_from_rtl (exp);
    6017              : 
    6018              :       /* Emit a debug bind insn before INSN.  */
    6019        25984 :       rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
    6020              :                                        DEBUG_EXPR_TREE_DECL (dval), exp,
    6021              :                                        VAR_INIT_STATUS_INITIALIZED);
    6022              : 
    6023        25984 :       emit_debug_insn_before (bind, insn);
    6024        25984 :       *exp_p = dval;
    6025        25984 :       return;
    6026              :     }
    6027              : 
    6028      6916603 :   const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
    6029      6916603 :   int i, j;
    6030     20630892 :   for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
    6031     13714289 :     switch (*format_ptr++)
    6032              :       {
    6033     12635001 :       case 'e':
    6034     12635001 :         avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
    6035     12635001 :         break;
    6036              : 
    6037              :       case 'E':
    6038              :       case 'V':
    6039            0 :         for (j = 0; j < XVECLEN (exp, i); j++)
    6040            0 :           avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
    6041              :         break;
    6042              : 
    6043              :       default:
    6044              :         break;
    6045              :       }
    6046              : }
    6047              : 
    6048              : /* Expand the _LOCs in debug insns.  We run this after expanding all
    6049              :    regular insns, so that any variables referenced in the function
    6050              :    will have their DECL_RTLs set.  */
    6051              : 
    6052              : static void
    6053       503732 : expand_debug_locations (void)
    6054              : {
    6055       503732 :   rtx_insn *insn;
    6056       503732 :   rtx_insn *last = get_last_insn ();
    6057       503732 :   int save_strict_alias = flag_strict_aliasing;
    6058              : 
    6059              :   /* New alias sets while setting up memory attributes cause
    6060              :      -fcompare-debug failures, even though it doesn't bring about any
    6061              :      codegen changes.  */
    6062       503732 :   flag_strict_aliasing = 0;
    6063              : 
    6064    109334613 :   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
    6065    161487042 :     if (DEBUG_BIND_INSN_P (insn))
    6066              :       {
    6067     40859695 :         tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
    6068     40859695 :         rtx val;
    6069     40859695 :         rtx_insn *prev_insn, *insn2;
    6070     40859695 :         machine_mode mode;
    6071              : 
    6072     40859695 :         if (value == NULL_TREE)
    6073              :           val = NULL_RTX;
    6074              :         else
    6075              :           {
    6076     22902189 :             if (INSN_VAR_LOCATION_STATUS (insn)
    6077              :                 == VAR_INIT_STATUS_UNINITIALIZED)
    6078       176022 :               val = expand_debug_source_expr (value);
    6079              :             /* The avoid_deep_ter_for_debug function inserts
    6080              :                debug bind stmts after SSA_NAME definition, with the
    6081              :                SSA_NAME as the whole bind location.  Disable temporarily
    6082              :                expansion of that SSA_NAME into the DEBUG_EXPR_DECL
    6083              :                being defined in this DEBUG_INSN.  */
    6084     22726167 :             else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
    6085              :               {
    6086        75869 :                 tree *slot = deep_ter_debug_map->get (value);
    6087        75869 :                 if (slot)
    6088              :                   {
    6089         6173 :                     if (*slot == INSN_VAR_LOCATION_DECL (insn))
    6090         5411 :                       *slot = NULL_TREE;
    6091              :                     else
    6092              :                       slot = NULL;
    6093              :                   }
    6094        75869 :                 val = expand_debug_expr (value);
    6095        75869 :                 if (slot)
    6096         5411 :                   *slot = INSN_VAR_LOCATION_DECL (insn);
    6097              :               }
    6098              :             else
    6099     22650298 :               val = expand_debug_expr (value);
    6100     22902189 :             gcc_assert (last == get_last_insn ());
    6101              :           }
    6102              : 
    6103     22902189 :         if (!val)
    6104     18451085 :           val = gen_rtx_UNKNOWN_VAR_LOC ();
    6105              :         else
    6106              :           {
    6107     22408610 :             mode = GET_MODE (INSN_VAR_LOCATION (insn));
    6108              : 
    6109     22408610 :             gcc_assert (mode == GET_MODE (val)
    6110              :                         || (GET_MODE (val) == VOIDmode
    6111              :                             && (CONST_SCALAR_INT_P (val)
    6112              :                                 || GET_CODE (val) == CONST_FIXED
    6113              :                                 || GET_CODE (val) == LABEL_REF)));
    6114              :           }
    6115              : 
    6116     40859695 :         INSN_VAR_LOCATION_LOC (insn) = val;
    6117     40859695 :         prev_insn = PREV_INSN (insn);
    6118    122605069 :         for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
    6119     40885679 :           avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
    6120              :       }
    6121              : 
    6122       503732 :   flag_strict_aliasing = save_strict_alias;
    6123       503732 : }
    6124              : 
    6125              : /* Performs swapping operands of commutative operations to expand
    6126              :    the expensive one first.  */
    6127              : 
    6128              : static void
    6129      9540118 : reorder_operands (basic_block bb)
    6130              : {
    6131      9540118 :   unsigned int *lattice;  /* Hold cost of each statement.  */
    6132      9540118 :   unsigned int i = 0, n = 0;
    6133      9540118 :   gimple_stmt_iterator gsi;
    6134      9540118 :   gimple_seq stmts;
    6135      9540118 :   gimple *stmt;
    6136      9540118 :   bool swap;
    6137      9540118 :   tree op0, op1;
    6138      9540118 :   ssa_op_iter iter;
    6139      9540118 :   use_operand_p use_p;
    6140      9540118 :   gimple *def0, *def1;
    6141              : 
    6142              :   /* Compute cost of each statement using estimate_num_insns.  */
    6143      9540118 :   stmts = bb_seq (bb);
    6144     97732251 :   for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
    6145              :     {
    6146     88192133 :       stmt = gsi_stmt (gsi);
    6147     88192133 :       if (!is_gimple_debug (stmt))
    6148     35074596 :         gimple_set_uid (stmt, n++);
    6149              :     }
    6150      9540118 :   lattice = XNEWVEC (unsigned int, n);
    6151    107272369 :   for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
    6152              :     {
    6153     88192133 :       unsigned cost;
    6154     88192133 :       stmt = gsi_stmt (gsi);
    6155     88192133 :       if (is_gimple_debug (stmt))
    6156     53117537 :         continue;
    6157     35074596 :       cost = estimate_num_insns (stmt, &eni_size_weights);
    6158     35074596 :       lattice[i] = cost;
    6159     66672726 :       FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
    6160              :         {
    6161     31598130 :           tree use = USE_FROM_PTR (use_p);
    6162     31598130 :           gimple *def_stmt;
    6163     31598130 :           if (TREE_CODE (use) != SSA_NAME)
    6164            0 :             continue;
    6165     31598130 :           def_stmt = get_gimple_for_ssa_name (use);
    6166     31598130 :           if (!def_stmt)
    6167     22492850 :             continue;
    6168      9105280 :           lattice[i] += lattice[gimple_uid (def_stmt)];
    6169              :         }
    6170     35074596 :       i++;
    6171     35074596 :       if (!is_gimple_assign (stmt)
    6172     35074596 :           || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
    6173     30750904 :         continue;
    6174      4323692 :       op0 = gimple_op (stmt, 1);
    6175      4323692 :       op1 = gimple_op (stmt, 2);
    6176      4323692 :       if (TREE_CODE (op0) != SSA_NAME
    6177      4323427 :           || TREE_CODE (op1) != SSA_NAME)
    6178      2745280 :         continue;
    6179              :       /* Swap operands if the second one is more expensive.  */
    6180      1578412 :       def0 = get_gimple_for_ssa_name (op0);
    6181      1578412 :       def1 = get_gimple_for_ssa_name (op1);
    6182      1578412 :       if (!def1)
    6183       872828 :         continue;
    6184       705584 :       swap = false;
    6185       705584 :       if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
    6186       330540 :         swap = true;
    6187       330540 :       if (swap)
    6188              :         {
    6189       330540 :           if (dump_file && (dump_flags & TDF_DETAILS))
    6190              :             {
    6191            2 :               fprintf (dump_file, "Swap operands in stmt:\n");
    6192            2 :               print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
    6193            2 :               fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
    6194            1 :                        def0 ? lattice[gimple_uid (def0)] : 0,
    6195            2 :                        lattice[gimple_uid (def1)]);
    6196              :             }
    6197       330540 :           swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
    6198              :                              gimple_assign_rhs2_ptr (stmt));
    6199              :         }
    6200              :     }
    6201      9540118 :   XDELETE (lattice);
    6202      9540118 : }
    6203              : 
    6204              : /* Expand basic block BB from GIMPLE trees to RTL.  */
    6205              : 
    6206              : static basic_block
    6207     12929173 : expand_gimple_basic_block (basic_block bb, rtx_insn *asan_epilog_seq)
    6208              : {
    6209     12929173 :   gimple_stmt_iterator gsi;
    6210     12929173 :   gimple *stmt = NULL;
    6211     12929173 :   rtx_note *note = NULL;
    6212     12929173 :   rtx_insn *last;
    6213     12929173 :   edge e;
    6214     12929173 :   edge_iterator ei;
    6215     12929173 :   bool nondebug_stmt_seen = false;
    6216              : 
    6217     12929173 :   if (dump_file)
    6218          826 :     fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
    6219              :              bb->index);
    6220              : 
    6221              :   /* Note that since we are now transitioning from GIMPLE to RTL, we
    6222              :      cannot use the gsi_*_bb() routines because they expect the basic
    6223              :      block to be in GIMPLE, instead of RTL.  Therefore, we need to
    6224              :      access the BB sequence directly.  */
    6225     12929173 :   if (optimize)
    6226      9540118 :     reorder_operands (bb);
    6227     12929173 :   rtl_profile_for_bb (bb);
    6228              : 
    6229              :   /* Remove the RETURN_EXPR if we may fall though to the exit
    6230              :      instead.  */
    6231     12929173 :   gsi = gsi_last_bb (bb);
    6232     12929173 :   if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
    6233              :     {
    6234      1485203 :       greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
    6235              : 
    6236      1485203 :       gcc_assert (single_succ_p (bb));
    6237      1485203 :       gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
    6238              : 
    6239      1485203 :       if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
    6240      1485203 :           && !gimple_return_retval (ret_stmt))
    6241              :         {
    6242       622200 :           gsi_remove (&gsi, false);
    6243       622200 :           single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
    6244              :         }
    6245              :     }
    6246              : 
    6247     12929173 :   gsi = gsi_start_bb (bb);
    6248     12929173 :   if (!gsi_end_p (gsi))
    6249              :     {
    6250     12381747 :       stmt = gsi_stmt (gsi);
    6251     12381747 :       if (gimple_code (stmt) != GIMPLE_LABEL)
    6252     12279562 :         stmt = NULL;
    6253              :     }
    6254              : 
    6255     12929173 :   rtx_code_label **elt = lab_rtx_for_bb->get (bb);
    6256     12929173 :   if ((unsigned) bb->index >= head_end_for_bb.length ())
    6257     10934849 :     head_end_for_bb.safe_grow_cleared (bb->index + 1);
    6258              : 
    6259     12929173 :   if (stmt || elt)
    6260              :     {
    6261      5355922 :       gcc_checking_assert (!note);
    6262      5355922 :       last = get_last_insn ();
    6263              : 
    6264      5355922 :       if (stmt)
    6265              :         {
    6266       649611 :           expand_gimple_stmt (stmt);
    6267       649611 :           gsi_next (&gsi);
    6268              :         }
    6269              : 
    6270      5355922 :       if (elt)
    6271      4706311 :         emit_label (*elt);
    6272              : 
    6273      5355922 :       head_end_for_bb[bb->index].first = NEXT_INSN (last);
    6274      5355922 :       if (NOTE_P (head_end_for_bb[bb->index].first))
    6275            0 :         head_end_for_bb[bb->index].first
    6276            0 :           = NEXT_INSN (head_end_for_bb[bb->index].first);
    6277      5355922 :       gcc_assert (LABEL_P (head_end_for_bb[bb->index].first));
    6278     10711844 :       note = emit_note_after (NOTE_INSN_BASIC_BLOCK,
    6279      5355922 :                               head_end_for_bb[bb->index].first);
    6280              : 
    6281      5355922 :       maybe_dump_rtl_for_gimple_stmt (stmt, last);
    6282              :     }
    6283              :   else
    6284      7573251 :     head_end_for_bb[bb->index].first = note = emit_note (NOTE_INSN_BASIC_BLOCK);
    6285              : 
    6286     12929173 :   if (note)
    6287     12929173 :     NOTE_BASIC_BLOCK (note) = bb;
    6288              : 
    6289     67620576 :   for (; !gsi_end_p (gsi); gsi_next (&gsi))
    6290              :     {
    6291     55222209 :       basic_block new_bb;
    6292              : 
    6293     55222209 :       stmt = gsi_stmt (gsi);
    6294     55222209 :       if (!is_gimple_debug (stmt))
    6295     46825570 :         nondebug_stmt_seen = true;
    6296              : 
    6297              :       /* If this statement is a non-debug one, and we generate debug
    6298              :          insns, then this one might be the last real use of a TERed
    6299              :          SSA_NAME, but where there are still some debug uses further
    6300              :          down.  Expanding the current SSA name in such further debug
    6301              :          uses by their RHS might lead to wrong debug info, as coalescing
    6302              :          might make the operands of such RHS be placed into the same
    6303              :          pseudo as something else.  Like so:
    6304              :            a_1 = a_0 + 1;   // Assume a_1 is TERed and a_0 is dead
    6305              :            use(a_1);
    6306              :            a_2 = ...
    6307              :            #DEBUG ... => a_1
    6308              :          As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
    6309              :          If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
    6310              :          the write to a_2 would actually have clobbered the place which
    6311              :          formerly held a_0.
    6312              : 
    6313              :          So, instead of that, we recognize the situation, and generate
    6314              :          debug temporaries at the last real use of TERed SSA names:
    6315              :            a_1 = a_0 + 1;
    6316              :            #DEBUG #D1 => a_1
    6317              :            use(a_1);
    6318              :            a_2 = ...
    6319              :            #DEBUG ... => #D1
    6320              :          */
    6321     55222209 :       if (MAY_HAVE_DEBUG_BIND_INSNS
    6322     29840996 :           && SA.values
    6323     83844093 :           && !is_gimple_debug (stmt))
    6324              :         {
    6325     20611675 :           ssa_op_iter iter;
    6326     20611675 :           tree op;
    6327     20611675 :           gimple *def;
    6328              : 
    6329     20611675 :           location_t sloc = curr_insn_location ();
    6330              : 
    6331              :           /* Look for SSA names that have their last use here (TERed
    6332              :              names always have only one real use).  */
    6333     39514652 :           FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
    6334     18902977 :             if ((def = get_gimple_for_ssa_name (op))
    6335     18902977 :                 && is_gimple_assign (def))
    6336              :               {
    6337      5129431 :                 imm_use_iterator imm_iter;
    6338      5129431 :                 use_operand_p use_p;
    6339      5129431 :                 bool have_debug_uses = false;
    6340              : 
    6341     10231068 :                 FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
    6342              :                   {
    6343      5152718 :                     if (gimple_debug_bind_p (USE_STMT (use_p)))
    6344              :                       {
    6345              :                         have_debug_uses = true;
    6346              :                         break;
    6347              :                       }
    6348      5129431 :                   }
    6349              : 
    6350      5129431 :                 if (have_debug_uses)
    6351              :                   {
    6352              :                     /* OP is a TERed SSA name, with DEF its defining
    6353              :                        statement, and where OP is used in further debug
    6354              :                        instructions.  Generate a debug temporary, and
    6355              :                        replace all uses of OP in debug insns with that
    6356              :                        temporary.  */
    6357        51081 :                     gimple *debugstmt;
    6358        51081 :                     tree value = gimple_assign_rhs_to_tree (def);
    6359        51081 :                     tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
    6360        51081 :                     rtx val;
    6361        51081 :                     machine_mode mode;
    6362              : 
    6363        51081 :                     set_curr_insn_location (gimple_location (def));
    6364              : 
    6365        51081 :                     if (DECL_P (value))
    6366          364 :                       mode = DECL_MODE (value);
    6367              :                     else
    6368        50717 :                       mode = TYPE_MODE (TREE_TYPE (value));
    6369              :                     /* FIXME: Is setting the mode really necessary? */
    6370        51081 :                     SET_DECL_MODE (vexpr, mode);
    6371              : 
    6372        51081 :                     val = gen_rtx_VAR_LOCATION
    6373        51081 :                         (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
    6374              : 
    6375        51081 :                     emit_debug_insn (val);
    6376              : 
    6377       191714 :                     FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
    6378              :                       {
    6379       140633 :                         if (!gimple_debug_bind_p (debugstmt))
    6380        51081 :                           continue;
    6381              : 
    6382       180906 :                         FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
    6383        90453 :                           SET_USE (use_p, vexpr);
    6384              : 
    6385        89552 :                         update_stmt (debugstmt);
    6386        51081 :                       }
    6387              :                   }
    6388              :               }
    6389     20611675 :           set_curr_insn_location (sloc);
    6390              :         }
    6391              : 
    6392     55222209 :       currently_expanding_gimple_stmt = stmt;
    6393              : 
    6394              :       /* Expand this statement, then evaluate the resulting RTL and
    6395              :          fixup the CFG accordingly.  */
    6396     55222209 :       if (gimple_code (stmt) == GIMPLE_COND)
    6397              :         {
    6398      5780708 :           new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
    6399      5780708 :           if (new_bb)
    6400              :             {
    6401       402370 :               currently_expanding_gimple_stmt = NULL;
    6402       402370 :               return new_bb;
    6403              :             }
    6404              :         }
    6405     49441501 :       else if (is_gimple_debug (stmt))
    6406              :         {
    6407      8396639 :           location_t sloc = curr_insn_location ();
    6408      8396639 :           gimple_stmt_iterator nsi = gsi;
    6409              : 
    6410     52990328 :           for (;;)
    6411              :             {
    6412     52990328 :               tree var;
    6413     52990328 :               tree value = NULL_TREE;
    6414     52990328 :               rtx val = NULL_RTX;
    6415     52990328 :               machine_mode mode;
    6416              : 
    6417     52990328 :               if (!gimple_debug_nonbind_marker_p (stmt))
    6418              :                 {
    6419     41193720 :                   if (gimple_debug_bind_p (stmt))
    6420              :                     {
    6421     41017698 :                       var = gimple_debug_bind_get_var (stmt);
    6422              : 
    6423     41017698 :                       if (TREE_CODE (var) != DEBUG_EXPR_DECL
    6424     37971998 :                           && TREE_CODE (var) != LABEL_DECL
    6425     78979659 :                           && !target_for_debug_bind (var))
    6426       385106 :                         goto delink_debug_stmt;
    6427              : 
    6428     40632592 :                       if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
    6429     40545457 :                         mode = DECL_MODE (var);
    6430              :                       else
    6431        87135 :                         mode = TYPE_MODE (TREE_TYPE (var));
    6432              : 
    6433     40632592 :                       if (gimple_debug_bind_has_value_p (stmt))
    6434     22675086 :                         value = gimple_debug_bind_get_value (stmt);
    6435              : 
    6436     40632592 :                       val = gen_rtx_VAR_LOCATION
    6437     40632592 :                         (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
    6438              :                     }
    6439       176022 :                   else if (gimple_debug_source_bind_p (stmt))
    6440              :                     {
    6441       176022 :                       var = gimple_debug_source_bind_get_var (stmt);
    6442              : 
    6443       176022 :                       value = gimple_debug_source_bind_get_value (stmt);
    6444              : 
    6445       176022 :                       if (!VECTOR_TYPE_P (TREE_TYPE (var)))
    6446       176022 :                         mode = DECL_MODE (var);
    6447              :                       else
    6448            0 :                         mode = TYPE_MODE (TREE_TYPE (var));
    6449              : 
    6450       176022 :                       val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
    6451              :                                                   VAR_INIT_STATUS_UNINITIALIZED);
    6452              :                     }
    6453              :                   else
    6454            0 :                     gcc_unreachable ();
    6455              :                 }
    6456              :               /* If this function was first compiled with markers
    6457              :                  enabled, but they're now disable (e.g. LTO), drop
    6458              :                  them on the floor.  */
    6459     11796608 :               else if (gimple_debug_nonbind_marker_p (stmt)
    6460     11796608 :                        && !MAY_HAVE_DEBUG_MARKER_INSNS)
    6461            0 :                 goto delink_debug_stmt;
    6462     11796608 :               else if (gimple_debug_begin_stmt_p (stmt))
    6463      4103506 :                 val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
    6464      7693102 :               else if (gimple_debug_inline_entry_p (stmt))
    6465      7693102 :                 val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
    6466              :               else
    6467              :                 gcc_unreachable ();
    6468              : 
    6469     52605222 :               last = get_last_insn ();
    6470              : 
    6471     52605222 :               set_curr_insn_location (gimple_location (stmt));
    6472              : 
    6473     52605222 :               emit_debug_insn (val);
    6474              : 
    6475     52605222 :               if (dump_file && (dump_flags & TDF_DETAILS))
    6476              :                 {
    6477              :                   /* We can't dump the insn with a TREE where an RTX
    6478              :                      is expected.  */
    6479            1 :                   if (GET_CODE (val) == VAR_LOCATION)
    6480              :                     {
    6481            0 :                       gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
    6482            0 :                       PAT_VAR_LOCATION_LOC (val) = const0_rtx;
    6483              :                     }
    6484            1 :                   maybe_dump_rtl_for_gimple_stmt (stmt, last);
    6485            1 :                   if (GET_CODE (val) == VAR_LOCATION)
    6486            0 :                     PAT_VAR_LOCATION_LOC (val) = (rtx)value;
    6487              :                 }
    6488              : 
    6489     52990328 :             delink_debug_stmt:
    6490              :               /* In order not to generate too many debug temporaries,
    6491              :                  we delink all uses of debug statements we already expanded.
    6492              :                  Therefore debug statements between definition and real
    6493              :                  use of TERed SSA names will continue to use the SSA name,
    6494              :                  and not be replaced with debug temps.  */
    6495     52990328 :               delink_stmt_imm_use (stmt);
    6496              : 
    6497     52990328 :               gsi = nsi;
    6498     52990328 :               gsi_next (&nsi);
    6499     52990328 :               if (gsi_end_p (nsi))
    6500              :                 break;
    6501     52310795 :               stmt = gsi_stmt (nsi);
    6502     52310795 :               if (!is_gimple_debug (stmt))
    6503              :                 break;
    6504              :             }
    6505              : 
    6506      8396639 :           set_curr_insn_location (sloc);
    6507              :         }
    6508              :       else
    6509              :         {
    6510     41044862 :           gcall *call_stmt = dyn_cast <gcall *> (stmt);
    6511     41044862 :           if (call_stmt
    6512     41044862 :               && asan_epilog_seq
    6513        25284 :               && gimple_call_tail_p (call_stmt)
    6514     41045012 :               && !gimple_call_must_tail_p (call_stmt))
    6515           33 :             gimple_call_set_tail (call_stmt, false);
    6516              : 
    6517     41044862 :           if (call_stmt && gimple_call_tail_p (call_stmt))
    6518              :             {
    6519       178659 :               bool can_fallthru;
    6520       178659 :               new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru,
    6521              :                                                asan_epilog_seq);
    6522       178659 :               if (new_bb)
    6523              :                 {
    6524       128434 :                   if (can_fallthru)
    6525            0 :                     bb = new_bb;
    6526              :                   else
    6527              :                     {
    6528       128434 :                       currently_expanding_gimple_stmt = NULL;
    6529       128434 :                       return new_bb;
    6530              :                     }
    6531              :                 }
    6532              :             }
    6533              :           else
    6534              :             {
    6535     40866203 :               def_operand_p def_p;
    6536     40866203 :               def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
    6537              : 
    6538     40866203 :               if (def_p != NULL)
    6539              :                 {
    6540              :                   /* Ignore this stmt if it is in the list of
    6541              :                      replaceable expressions.  */
    6542     34477799 :                   if (SA.values
    6543     42832426 :                       && bitmap_bit_p (SA.values,
    6544     17459774 :                                        SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
    6545      9105147 :                     continue;
    6546              :                 }
    6547     31761056 :               last = expand_gimple_stmt (stmt);
    6548     31761054 :               maybe_dump_rtl_for_gimple_stmt (stmt, last);
    6549              :             }
    6550              :         }
    6551              :     }
    6552              : 
    6553     12398367 :   currently_expanding_gimple_stmt = NULL;
    6554              : 
    6555              :   /* Expand implicit goto and convert goto_locus.  */
    6556     30112926 :   FOR_EACH_EDGE (e, ei, bb->succs)
    6557              :     {
    6558     17714559 :       if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
    6559      3488995 :         set_curr_insn_location (e->goto_locus);
    6560     17714559 :       if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
    6561              :         {
    6562      1663113 :           emit_jump (label_rtx_for_bb (e->dest));
    6563      1663113 :           e->flags &= ~EDGE_FALLTHRU;
    6564              :         }
    6565              :     }
    6566              : 
    6567              :   /* Expanded RTL can create a jump in the last instruction of block.
    6568              :      This later might be assumed to be a jump to successor and break edge insertion.
    6569              :      We need to insert dummy move to prevent this. PR41440. */
    6570     12398367 :   if (single_succ_p (bb)
    6571      5590981 :       && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
    6572      3162038 :       && (last = get_last_insn ())
    6573     15370448 :       && (JUMP_P (last)
    6574      3161967 :           || (DEBUG_INSN_P (last)
    6575       378430 :               && JUMP_P (prev_nondebug_insn (last)))))
    6576              :     {
    6577           84 :       rtx dummy = gen_reg_rtx (SImode);
    6578           84 :       emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
    6579              :     }
    6580              : 
    6581              :   /* A __builtin_unreachable () will insert a barrier that should end
    6582              :      the basic block.  In gimple, any code after it will have already
    6583              :      deleted, even without optimization.  If we emit additional code
    6584              :      here, as we would to adjust the stack after a call, it should be
    6585              :      eventually deleted, but it confuses internal checkers (PR118006)
    6586              :      and optimizers before it does, because we don't expect to find
    6587              :      barriers inside basic blocks.  */
    6588     12398367 :   if (!BARRIER_P (get_last_insn ()))
    6589      9041933 :     do_pending_stack_adjust ();
    6590              :   else
    6591      3356434 :     discard_pending_stack_adjust ();
    6592              : 
    6593              :   /* Find the block tail.  The last insn in the block is the insn
    6594              :      before a barrier and/or table jump insn.  */
    6595     12398367 :   last = get_last_insn ();
    6596     12398367 :   if (BARRIER_P (last))
    6597      3356434 :     last = PREV_INSN (last);
    6598     12398367 :   if (JUMP_TABLE_DATA_P (last))
    6599         7659 :     last = PREV_INSN (PREV_INSN (last));
    6600     12398367 :   if (BARRIER_P (last))
    6601         6449 :     last = PREV_INSN (last);
    6602     12398367 :   head_end_for_bb[bb->index].second = last;
    6603              : 
    6604     24796734 :   update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
    6605     12398367 :                             head_end_for_bb[bb->index].second, bb);
    6606              : 
    6607     12398367 :   return bb;
    6608              : }
    6609              : 
    6610              : 
    6611              : /* Create a basic block for initialization code.  */
    6612              : 
    6613              : static basic_block
    6614      1517908 : construct_init_block (void)
    6615              : {
    6616      1517908 :   basic_block init_block;
    6617      1517908 :   edge e = NULL;
    6618      1517908 :   int flags;
    6619              : 
    6620              :   /* Multiple entry points not supported yet.  */
    6621      1517908 :   gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
    6622      1517908 :   init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
    6623      1517908 :   init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
    6624      1517908 :   ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
    6625      1517908 :   EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
    6626              : 
    6627      1517908 :   e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
    6628              : 
    6629              :   /* When entry edge points to first basic block, we don't need jump,
    6630              :      otherwise we have to jump into proper target.  */
    6631      1517908 :   if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
    6632              :     {
    6633            0 :       tree label = gimple_block_label (e->dest);
    6634              : 
    6635            0 :       emit_jump (jump_target_rtx (label));
    6636            0 :       flags = 0;
    6637            0 :     }
    6638              :   else
    6639              :     flags = EDGE_FALLTHRU;
    6640              : 
    6641      1517908 :   init_block = create_basic_block (NEXT_INSN (get_insns ()),
    6642              :                                    get_last_insn (),
    6643      1517908 :                                    ENTRY_BLOCK_PTR_FOR_FN (cfun));
    6644      1517908 :   init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    6645      1517908 :   add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
    6646      1517908 :   if (e)
    6647      1517908 :     expand_split_edge (e, init_block, flags);
    6648              :   else
    6649            0 :     make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
    6650              :                            EDGE_FALLTHRU);
    6651              : 
    6652      1517908 :   update_bb_for_insn (init_block);
    6653      1517908 :   return init_block;
    6654              : }
    6655              : 
    6656              : /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
    6657              :    found in the block tree.  */
    6658              : 
    6659              : static void
    6660     18757806 : set_block_levels (tree block, int level)
    6661              : {
    6662     35997706 :   while (block)
    6663              :     {
    6664     17239900 :       BLOCK_NUMBER (block) = level;
    6665     17239900 :       set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
    6666     17239900 :       block = BLOCK_CHAIN (block);
    6667              :     }
    6668     18757806 : }
    6669              : 
    6670              : /* Create a block containing landing pads and similar stuff.  */
    6671              : 
    6672              : static void
    6673      1517906 : construct_exit_block (void)
    6674              : {
    6675      1517906 :   rtx_insn *head = get_last_insn ();
    6676      1517906 :   rtx_insn *end;
    6677      1517906 :   basic_block exit_block;
    6678      1517906 :   edge e, e2;
    6679      1517906 :   unsigned ix;
    6680      1517906 :   edge_iterator ei;
    6681      1517906 :   basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
    6682      1517906 :   rtx_insn *orig_end = BB_END (prev_bb);
    6683              : 
    6684      1517906 :   rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
    6685              : 
    6686              :   /* Make sure the locus is set to the end of the function, so that
    6687              :      epilogue line numbers and warnings are set properly.  */
    6688      1517906 :   if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
    6689      1196186 :     input_location = cfun->function_end_locus;
    6690              : 
    6691              :   /* Generate rtl for function exit.  */
    6692      1517906 :   expand_function_end ();
    6693              : 
    6694      1517906 :   end = get_last_insn ();
    6695      1517906 :   if (head == end)
    6696            0 :     return;
    6697              :   /* While emitting the function end we could move end of the last basic
    6698              :      block.  */
    6699      1517906 :   BB_END (prev_bb) = orig_end;
    6700      1517906 :   while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
    6701              :     head = NEXT_INSN (head);
    6702              :   /* But make sure exit_block starts with RETURN_LABEL, otherwise the
    6703              :      bb count counting will be confused.  Any instructions before that
    6704              :      label are emitted for the case where PREV_BB falls through into the
    6705              :      exit block, so append those instructions to prev_bb in that case.  */
    6706      1517906 :   if (NEXT_INSN (head) != return_label)
    6707              :     {
    6708        14993 :       while (NEXT_INSN (head) != return_label)
    6709              :         {
    6710         9947 :           if (!NOTE_P (NEXT_INSN (head)))
    6711         9947 :             BB_END (prev_bb) = NEXT_INSN (head);
    6712         9947 :           head = NEXT_INSN (head);
    6713              :         }
    6714              :     }
    6715      1517906 :   exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
    6716      1517906 :   exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
    6717      1517906 :   add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
    6718              : 
    6719      1517906 :   ix = 0;
    6720      4581923 :   while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
    6721              :     {
    6722      1546111 :       e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
    6723      1546111 :       if (!(e->flags & EDGE_ABNORMAL))
    6724      1417677 :         redirect_edge_succ (e, exit_block);
    6725              :       else
    6726       128434 :         ix++;
    6727              :     }
    6728              : 
    6729      1517906 :   e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
    6730              :                              EDGE_FALLTHRU);
    6731      3164246 :   FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    6732      1646340 :     if (e2 != e)
    6733              :       {
    6734       128434 :         exit_block->count -= e2->count ();
    6735              :       }
    6736      1517906 :   update_bb_for_insn (exit_block);
    6737              : }
    6738              : 
    6739              : /* Helper function for discover_nonconstant_array_refs.
    6740              :    Look for ARRAY_REF nodes with non-constant indexes and mark them
    6741              :    addressable.  */
    6742              : 
    6743              : static tree
    6744    168368762 : discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
    6745              :                                    void *data)
    6746              : {
    6747    168368762 :   tree t = *tp;
    6748    168368762 :   bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
    6749              : 
    6750    168368762 :   if (IS_TYPE_OR_DECL_P (t))
    6751     34549104 :     *walk_subtrees = 0;
    6752    133819658 :   else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
    6753              :     {
    6754       138560 :       t = get_base_address (t);
    6755       138560 :       if (t && DECL_P (t)
    6756       125441 :           && DECL_MODE (t) != BLKmode
    6757       176194 :           && !TREE_ADDRESSABLE (t))
    6758        32735 :         bitmap_set_bit (forced_stack_vars, DECL_UID (t));
    6759       138560 :       *walk_subtrees = 0;
    6760              :     }
    6761    133681098 :   else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
    6762              :     {
    6763      6507672 :       while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
    6764      3107353 :               && is_gimple_min_invariant (TREE_OPERAND (t, 1))
    6765      2626823 :               && (!TREE_OPERAND (t, 2)
    6766        37161 :                   || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
    6767      3914871 :              || (TREE_CODE (t) == COMPONENT_REF
    6768       968364 :                  && (!TREE_OPERAND (t,2)
    6769            0 :                      || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
    6770      2946507 :              || TREE_CODE (t) == BIT_FIELD_REF
    6771              :              || TREE_CODE (t) == REALPART_EXPR
    6772              :              || TREE_CODE (t) == IMAGPART_EXPR
    6773              :              || TREE_CODE (t) == VIEW_CONVERT_EXPR
    6774      6507672 :              || CONVERT_EXPR_P (t))
    6775      3571362 :         t = TREE_OPERAND (t, 0);
    6776              : 
    6777      2936310 :       if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
    6778              :         {
    6779       514552 :           t = get_base_address (t);
    6780       514552 :           if (t && DECL_P (t)
    6781       412547 :               && DECL_MODE (t) != BLKmode
    6782       537101 :               && !TREE_ADDRESSABLE (t))
    6783        13048 :             bitmap_set_bit (forced_stack_vars, DECL_UID (t));
    6784              :         }
    6785              : 
    6786      2936310 :       *walk_subtrees = 0;
    6787              :     }
    6788              :   /* References of size POLY_INT_CST to a fixed-size object must go
    6789              :      through memory.  It's more efficient to force that here than
    6790              :      to create temporary slots on the fly.
    6791              :      RTL expansion expects TARGET_MEM_REF to always address actual memory.
    6792              :      Also, force to stack non-BLKmode vars accessed through VIEW_CONVERT_EXPR
    6793              :      to BLKmode type.  */
    6794    130744788 :   else if (TREE_CODE (t) == TARGET_MEM_REF
    6795              :            || (TREE_CODE (t) == MEM_REF
    6796    129886432 :                && TYPE_SIZE (TREE_TYPE (t))
    6797              :                && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
    6798    260631220 :            || (TREE_CODE (t) == VIEW_CONVERT_EXPR
    6799       332530 :                && TYPE_MODE (TREE_TYPE (t)) == BLKmode))
    6800              :     {
    6801       936980 :       tree base = get_base_address (t);
    6802       936980 :       if (base
    6803       936980 :           && DECL_P (base)
    6804       260572 :           && !TREE_ADDRESSABLE (base)
    6805       119675 :           && DECL_MODE (base) != BLKmode
    6806       942993 :           && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
    6807         6013 :         bitmap_set_bit (forced_stack_vars, DECL_UID (base));
    6808       936980 :       *walk_subtrees = 0;
    6809              :     }
    6810              : 
    6811    168368762 :   return NULL_TREE;
    6812              : }
    6813              : 
    6814              : /* If there's a chance to get a pseudo for t then if it would be of float mode
    6815              :    and the actual access is via an integer mode (lowered memcpy or similar
    6816              :    access) then avoid the register expansion if the mode likely is not storage
    6817              :    suitable for raw bits processing (like XFmode on i?86).  */
    6818              : 
    6819              : static void
    6820      6877315 : avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
    6821              : {
    6822      6877315 :   machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
    6823      6877315 :   if (access_mode != BLKmode
    6824      6699910 :       && !SCALAR_INT_MODE_P (access_mode))
    6825              :     return;
    6826      5884767 :   tree base = get_base_address (t);
    6827      5884767 :   if (DECL_P (base)
    6828      4351946 :       && !TREE_ADDRESSABLE (base)
    6829      1301907 :       && FLOAT_MODE_P (DECL_MODE (base))
    6830           80 :       && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
    6831          240 :                    GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
    6832              :       /* Double check in the expensive way we really would get a pseudo.  */
    6833      5884772 :       && use_register_for_decl (base))
    6834            5 :     bitmap_set_bit (forced_stack_vars, DECL_UID (base));
    6835              : }
    6836              : 
    6837              : /* RTL expansion is not able to compile array references with variable
    6838              :    offsets for arrays stored in single register.  Discover such
    6839              :    expressions and mark variables as addressable to avoid this
    6840              :    scenario.  */
    6841              : 
    6842              : static void
    6843      1517908 : discover_nonconstant_array_refs (bitmap forced_stack_vars)
    6844              : {
    6845      1517908 :   basic_block bb;
    6846      1517908 :   gimple_stmt_iterator gsi;
    6847              : 
    6848      1517908 :   walk_stmt_info wi = {};
    6849      1517908 :   wi.info = forced_stack_vars;
    6850     14443998 :   FOR_EACH_BB_FN (bb, cfun)
    6851    127086899 :     for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    6852              :       {
    6853    101234719 :         gimple *stmt = gsi_stmt (gsi);
    6854    101234719 :         if (!is_gimple_debug (stmt))
    6855              :           {
    6856     48117166 :             walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
    6857     48117166 :             gcall *call = dyn_cast <gcall *> (stmt);
    6858      7106994 :             if (call && gimple_call_internal_p (call))
    6859              :               {
    6860       232950 :                 tree cand = NULL_TREE;
    6861       232950 :                 switch (gimple_call_internal_fn (call))
    6862              :                   {
    6863            0 :                   case IFN_LOAD_LANES:
    6864              :                     /* The source must be a MEM.  */
    6865            0 :                     cand = gimple_call_arg (call, 0);
    6866            0 :                     break;
    6867            0 :                   case IFN_STORE_LANES:
    6868              :                     /* The destination must be a MEM.  */
    6869            0 :                     cand = gimple_call_lhs (call);
    6870            0 :                     break;
    6871              :                   default:
    6872              :                     break;
    6873              :                   }
    6874            0 :                 if (cand)
    6875            0 :                   cand = get_base_address (cand);
    6876            0 :                 if (cand
    6877            0 :                     && DECL_P (cand)
    6878            0 :                     && use_register_for_decl (cand))
    6879            0 :                   bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
    6880              :               }
    6881    158925246 :             if (gimple_vdef (stmt))
    6882              :               {
    6883     16025475 :                 tree t = gimple_get_lhs (stmt);
    6884     16025475 :                 if (t && REFERENCE_CLASS_P (t))
    6885      6877315 :                   avoid_type_punning_on_regs (t, forced_stack_vars);
    6886              :               }
    6887              :           }
    6888              :       }
    6889      1517908 : }
    6890              : 
    6891              : /* This function sets crtl->args.internal_arg_pointer to a virtual
    6892              :    register if DRAP is needed.  Local register allocator will replace
    6893              :    virtual_incoming_args_rtx with the virtual register.  */
    6894              : 
    6895              : static void
    6896      1517906 : expand_stack_alignment (void)
    6897              : {
    6898      1517906 :   rtx drap_rtx;
    6899      1517906 :   unsigned int preferred_stack_boundary;
    6900              : 
    6901      1517906 :   if (! SUPPORTS_STACK_ALIGNMENT)
    6902              :     return;
    6903              : 
    6904      1517906 :   if (cfun->calls_alloca
    6905      1502392 :       || cfun->has_nonlocal_label
    6906      1501576 :       || crtl->has_nonlocal_goto)
    6907        16745 :     crtl->need_drap = true;
    6908              : 
    6909              :   /* Call update_stack_boundary here again to update incoming stack
    6910              :      boundary.  It may set incoming stack alignment to a different
    6911              :      value after RTL expansion.  TARGET_FUNCTION_OK_FOR_SIBCALL may
    6912              :      use the minimum incoming stack alignment to check if it is OK
    6913              :      to perform sibcall optimization since sibcall optimization will
    6914              :      only align the outgoing stack to incoming stack boundary.  */
    6915      1517906 :   if (targetm.calls.update_stack_boundary)
    6916      1517906 :     targetm.calls.update_stack_boundary ();
    6917              : 
    6918              :   /* The incoming stack frame has to be aligned at least at
    6919              :      parm_stack_boundary.  */
    6920      1517906 :   gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
    6921              : 
    6922              :   /* Update crtl->stack_alignment_estimated and use it later to align
    6923              :      stack.  We check PREFERRED_STACK_BOUNDARY if there may be non-call
    6924              :      exceptions since callgraph doesn't collect incoming stack alignment
    6925              :      in this case.  */
    6926      1517906 :   if (cfun->can_throw_non_call_exceptions
    6927       263025 :       && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
    6928              :     preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
    6929              :   else
    6930      1489928 :     preferred_stack_boundary = crtl->preferred_stack_boundary;
    6931      1517906 :   if (preferred_stack_boundary > crtl->stack_alignment_estimated)
    6932       729861 :     crtl->stack_alignment_estimated = preferred_stack_boundary;
    6933      1517906 :   if (preferred_stack_boundary > crtl->stack_alignment_needed)
    6934       680468 :     crtl->stack_alignment_needed = preferred_stack_boundary;
    6935              : 
    6936      1517906 :   gcc_assert (crtl->stack_alignment_needed
    6937              :               <= crtl->stack_alignment_estimated);
    6938              : 
    6939      1517906 :   crtl->stack_realign_needed
    6940      1517906 :     = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
    6941      1517906 :   crtl->stack_realign_tried = crtl->stack_realign_needed;
    6942              : 
    6943      1517906 :   crtl->stack_realign_processed = true;
    6944              : 
    6945              :   /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
    6946              :      alignment.  */
    6947      1517906 :   gcc_assert (targetm.calls.get_drap_rtx != NULL);
    6948      1517906 :   drap_rtx = targetm.calls.get_drap_rtx ();
    6949              : 
    6950              :   /* stack_realign_drap and drap_rtx must match.  */
    6951      1517906 :   gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
    6952              : 
    6953              :   /* Do nothing if NULL is returned, which means DRAP is not needed.  */
    6954      1517906 :   if (drap_rtx != NULL)
    6955              :     {
    6956         7406 :       crtl->args.internal_arg_pointer = drap_rtx;
    6957              : 
    6958              :       /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
    6959              :          needed. */
    6960         7406 :       fixup_tail_calls ();
    6961              :     }
    6962              : }
    6963              : 
    6964              : 
    6965              : static void
    6966            0 : expand_main_function (void)
    6967              : {
    6968              : #if (defined(INVOKE__main)                              \
    6969              :      || (!defined(HAS_INIT_SECTION)                     \
    6970              :          && !defined(INIT_SECTION_ASM_OP)               \
    6971              :          && !defined(INIT_ARRAY_SECTION_ASM_OP)))
    6972              :   emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
    6973              : #endif
    6974            0 : }
    6975              : 
    6976              : 
    6977              : /* Expand code to initialize the stack_protect_guard.  This is invoked at
    6978              :    the beginning of a function to be protected.  */
    6979              : 
    6980              : static void
    6981          295 : stack_protect_prologue (void)
    6982              : {
    6983          295 :   tree guard_decl = targetm.stack_protect_guard ();
    6984          295 :   rtx x, y;
    6985              : 
    6986          295 :   crtl->stack_protect_guard_decl = guard_decl;
    6987          295 :   x = expand_normal (crtl->stack_protect_guard);
    6988              : 
    6989          295 :   if (targetm.have_stack_protect_combined_set () && guard_decl)
    6990              :     {
    6991            0 :       gcc_assert (DECL_P (guard_decl));
    6992            0 :       y = DECL_RTL (guard_decl);
    6993              : 
    6994              :       /* Allow the target to compute address of Y and copy it to X without
    6995              :          leaking Y into a register.  This combined address + copy pattern
    6996              :          allows the target to prevent spilling of any intermediate results by
    6997              :          splitting it after register allocator.  */
    6998            0 :       if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
    6999              :         {
    7000            0 :           emit_insn (insn);
    7001            0 :           return;
    7002              :         }
    7003              :     }
    7004              : 
    7005          295 :   if (guard_decl)
    7006          295 :     y = expand_normal (guard_decl);
    7007              :   else
    7008            0 :     y = const0_rtx;
    7009              : 
    7010              :   /* Allow the target to copy from Y to X without leaking Y into a
    7011              :      register.  */
    7012          295 :   if (targetm.have_stack_protect_set ())
    7013          295 :     if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
    7014              :       {
    7015          295 :         emit_insn (insn);
    7016          295 :         return;
    7017              :       }
    7018              : 
    7019              :   /* Otherwise do a straight move.  */
    7020            0 :   emit_move_insn (x, y);
    7021              : }
    7022              : 
    7023              : /* Translate the intermediate representation contained in the CFG
    7024              :    from GIMPLE trees to RTL.
    7025              : 
    7026              :    We do conversion per basic block and preserve/update the tree CFG.
    7027              :    This implies we have to do some magic as the CFG can simultaneously
    7028              :    consist of basic blocks containing RTL and GIMPLE trees.  This can
    7029              :    confuse the CFG hooks, so be careful to not manipulate CFG during
    7030              :    the expansion.  */
    7031              : 
    7032              : namespace {
    7033              : 
    7034              : const pass_data pass_data_expand =
    7035              : {
    7036              :   RTL_PASS, /* type */
    7037              :   "expand", /* name */
    7038              :   OPTGROUP_NONE, /* optinfo_flags */
    7039              :   TV_EXPAND, /* tv_id */
    7040              :   ( PROP_ssa | PROP_gimple_leh | PROP_cfg
    7041              :     | PROP_gimple_lcx
    7042              :     | PROP_gimple_lvec
    7043              :     | PROP_gimple_lva), /* properties_required */
    7044              :   PROP_rtl, /* properties_provided */
    7045              :   ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
    7046              :   0, /* todo_flags_start */
    7047              :   0, /* todo_flags_finish */
    7048              : };
    7049              : 
    7050              : class pass_expand : public rtl_opt_pass
    7051              : {
    7052              : public:
    7053       297184 :   pass_expand (gcc::context *ctxt)
    7054       594368 :     : rtl_opt_pass (pass_data_expand, ctxt)
    7055              :   {}
    7056              : 
    7057              :   /* opt_pass methods: */
    7058              :   unsigned int execute (function *) final override;
    7059              : 
    7060              : }; // class pass_expand
    7061              : 
    7062              : unsigned int
    7063      1517908 : pass_expand::execute (function *fun)
    7064              : {
    7065      1517908 :   basic_block bb, init_block;
    7066      1517908 :   edge_iterator ei;
    7067      1517908 :   edge e;
    7068      1517908 :   rtx_insn *var_seq, *var_ret_seq;
    7069      1517908 :   unsigned i;
    7070              : 
    7071      1517908 :   timevar_push (TV_OUT_OF_SSA);
    7072      1517908 :   rewrite_out_of_ssa (&SA);
    7073      1517908 :   timevar_pop (TV_OUT_OF_SSA);
    7074      1517908 :   SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
    7075              : 
    7076      1517908 :   if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
    7077              :     {
    7078       503707 :       gimple_stmt_iterator gsi;
    7079      6913843 :       FOR_EACH_BB_FN (bb, cfun)
    7080     87836256 :         for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
    7081    116142129 :           if (gimple_debug_bind_p (gsi_stmt (gsi)))
    7082     41126145 :             avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
    7083              :     }
    7084              : 
    7085              :   /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE.  */
    7086      1517908 :   auto_bitmap forced_stack_vars;
    7087      1517908 :   discover_nonconstant_array_refs (forced_stack_vars);
    7088              : 
    7089              :   /* Make sure all values used by the optimization passes have sane
    7090              :      defaults.  */
    7091      1517908 :   reg_renumber = 0;
    7092              : 
    7093              :   /* Some backends want to know that we are expanding to RTL.  */
    7094      1517908 :   currently_expanding_to_rtl = 1;
    7095              :   /* Dominators are not kept up-to-date as we may create new basic-blocks.  */
    7096      1517908 :   free_dominance_info (CDI_DOMINATORS);
    7097              : 
    7098      1517908 :   rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
    7099              : 
    7100      1517908 :   insn_locations_init ();
    7101      1517908 :   if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
    7102              :     {
    7103              :       /* Eventually, all FEs should explicitly set function_start_locus.  */
    7104      1395529 :       if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
    7105       656700 :         set_curr_insn_location
    7106       656700 :           (DECL_SOURCE_LOCATION (current_function_decl));
    7107              :       else
    7108       738829 :         set_curr_insn_location (fun->function_start_locus);
    7109              :     }
    7110              :   else
    7111       122379 :     set_curr_insn_location (UNKNOWN_LOCATION);
    7112      1517908 :   prologue_location = curr_insn_location ();
    7113              : 
    7114              : #ifdef INSN_SCHEDULING
    7115      1517908 :   init_sched_attrs ();
    7116              : #endif
    7117              : 
    7118              :   /* Make sure first insn is a note even if we don't want linenums.
    7119              :      This makes sure the first insn will never be deleted.
    7120              :      Also, final expects a note to appear there.  */
    7121      1517908 :   emit_note (NOTE_INSN_DELETED);
    7122              : 
    7123      1517908 :   targetm.expand_to_rtl_hook ();
    7124      1517908 :   crtl->init_stack_alignment ();
    7125      1517908 :   fun->cfg->max_jumptable_ents = 0;
    7126              : 
    7127              :   /* Resolve the function section.  Some targets, like ARM EABI rely on knowledge
    7128              :      of the function section at expansion time to predict distance of calls.  */
    7129      1517908 :   resolve_unique_section (current_function_decl, 0, flag_function_sections);
    7130              : 
    7131              :   /* Expand the variables recorded during gimple lowering.  */
    7132      1517908 :   timevar_push (TV_VAR_EXPAND);
    7133      1517908 :   start_sequence ();
    7134              : 
    7135      1517908 :   var_ret_seq = expand_used_vars (forced_stack_vars);
    7136              : 
    7137      1517908 :   var_seq = end_sequence ();
    7138      1517908 :   timevar_pop (TV_VAR_EXPAND);
    7139              : 
    7140              :   /* Honor stack protection warnings.  */
    7141      1517908 :   if (warn_stack_protect)
    7142              :     {
    7143            0 :       if (fun->calls_alloca)
    7144            0 :         warning (OPT_Wstack_protector,
    7145              :                  "stack protector not protecting local variables: "
    7146              :                  "variable length buffer");
    7147            0 :       if (has_short_buffer && !crtl->stack_protect_guard)
    7148            0 :         warning (OPT_Wstack_protector,
    7149              :                  "stack protector not protecting function: "
    7150              :                  "all local arrays are less than %d bytes long",
    7151            0 :                  (int) param_ssp_buffer_size);
    7152              :     }
    7153              : 
    7154              :   /* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
    7155              :      memory addressable so expand_function_start can emit the required
    7156              :      copies.  */
    7157      1517908 :   auto_vec<tree, 16> marked_parms;
    7158      4685110 :   for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
    7159      3167202 :        parm = DECL_CHAIN (parm))
    7160      3167202 :     if (!TREE_ADDRESSABLE (parm)
    7161      3167202 :         && bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
    7162              :       {
    7163          189 :         TREE_ADDRESSABLE (parm) = 1;
    7164          189 :         marked_parms.safe_push (parm);
    7165              :       }
    7166      1517908 :   if (DECL_RESULT (current_function_decl)
    7167      1517908 :       && !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
    7168      3028755 :       && bitmap_bit_p (forced_stack_vars,
    7169      1510847 :                        DECL_UID (DECL_RESULT (current_function_decl))))
    7170              :     {
    7171            0 :       TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
    7172            0 :       marked_parms.safe_push (DECL_RESULT (current_function_decl));
    7173              :     }
    7174              : 
    7175              :   /* Set up parameters and prepare for return, for the function.  */
    7176      1517908 :   expand_function_start (current_function_decl);
    7177              : 
    7178              :   /* Clear TREE_ADDRESSABLE again.  */
    7179      3036005 :   while (!marked_parms.is_empty ())
    7180          189 :     TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
    7181              : 
    7182              :   /* If we emitted any instructions for setting up the variables,
    7183              :      emit them before the FUNCTION_START note.  */
    7184      1517908 :   if (var_seq)
    7185              :     {
    7186         1798 :       emit_insn_before (var_seq, parm_birth_insn);
    7187              : 
    7188              :       /* In expand_function_end we'll insert the alloca save/restore
    7189              :          before parm_birth_insn.  We've just insertted an alloca call.
    7190              :          Adjust the pointer to match.  */
    7191         1798 :       parm_birth_insn = var_seq;
    7192              :     }
    7193              : 
    7194              :   /* Now propagate the RTL assignment of each partition to the
    7195              :      underlying var of each SSA_NAME.  */
    7196      1517908 :   tree name;
    7197              : 
    7198     75416766 :   FOR_EACH_SSA_NAME (i, name, cfun)
    7199              :     {
    7200              :       /* We might have generated new SSA names in
    7201              :          update_alias_info_with_stack_vars.  They will have a NULL
    7202              :          defining statements, and won't be part of the partitioning,
    7203              :          so ignore those.  */
    7204     50275795 :       if (!SSA_NAME_DEF_STMT (name))
    7205        72380 :         continue;
    7206              : 
    7207     50203415 :       adjust_one_expanded_partition_var (name);
    7208              :     }
    7209              : 
    7210              :   /* Clean up RTL of variables that straddle across multiple
    7211              :      partitions, and check that the rtl of any PARM_DECLs that are not
    7212              :      cleaned up is that of their default defs.  */
    7213     75416766 :   FOR_EACH_SSA_NAME (i, name, cfun)
    7214              :     {
    7215     50275795 :       int part;
    7216              : 
    7217              :       /* We might have generated new SSA names in
    7218              :          update_alias_info_with_stack_vars.  They will have a NULL
    7219              :          defining statements, and won't be part of the partitioning,
    7220              :          so ignore those.  */
    7221     50275795 :       if (!SSA_NAME_DEF_STMT (name))
    7222        72380 :         continue;
    7223     50203415 :       part = var_to_partition (SA.map, name);
    7224     50203415 :       if (part == NO_PARTITION)
    7225     17956056 :         continue;
    7226              : 
    7227              :       /* If this decl was marked as living in multiple places, reset
    7228              :          this now to NULL.  */
    7229     32247359 :       tree var = SSA_NAME_VAR (name);
    7230     10713814 :       if (var && DECL_RTL_IF_SET (var) == pc_rtx)
    7231       248538 :         SET_DECL_RTL (var, NULL);
    7232              :       /* Check that the pseudos chosen by assign_parms are those of
    7233              :          the corresponding default defs.  */
    7234     31998821 :       else if (SSA_NAME_IS_DEFAULT_DEF (name)
    7235     31998821 :                && (TREE_CODE (var) == PARM_DECL
    7236      3749895 :                    || TREE_CODE (var) == RESULT_DECL))
    7237              :         {
    7238      3687179 :           rtx in = DECL_RTL_IF_SET (var);
    7239      3687179 :           gcc_assert (in);
    7240      3687179 :           rtx out = SA.partition_to_pseudo[part];
    7241      3687179 :           gcc_assert (in == out);
    7242              : 
    7243              :           /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
    7244              :              those expected by debug backends for each parm and for
    7245              :              the result.  This is particularly important for stabs,
    7246              :              whose register elimination from parm's DECL_RTL may cause
    7247              :              -fcompare-debug differences as SET_DECL_RTL changes reg's
    7248              :              attrs.  So, make sure the RTL already has the parm as the
    7249              :              EXPR, so that it won't change.  */
    7250      3687179 :           SET_DECL_RTL (var, NULL_RTX);
    7251      3687179 :           if (MEM_P (in))
    7252       788171 :             set_mem_attributes (in, var, true);
    7253      3687179 :           SET_DECL_RTL (var, in);
    7254              :         }
    7255              :     }
    7256              : 
    7257      1517908 :   if (flag_checking)
    7258      1517888 :     verify_partition_mem_exprs ();
    7259              : 
    7260              :   /* If this function is `main', emit a call to `__main'
    7261              :      to run global initializers, etc.  */
    7262      1517908 :   if (DECL_NAME (current_function_decl)
    7263      1517908 :       && MAIN_NAME_P (DECL_NAME (current_function_decl))
    7264      1633636 :       && DECL_FILE_SCOPE_P (current_function_decl))
    7265      1517908 :     expand_main_function ();
    7266              : 
    7267              :   /* Initialize the stack_protect_guard field.  This must happen after the
    7268              :      call to __main (if any) so that the external decl is initialized.  */
    7269      1517908 :   if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
    7270          295 :     stack_protect_prologue ();
    7271              : 
    7272      1517908 :   expand_phi_nodes (&SA);
    7273              : 
    7274              :   /* Release any stale SSA redirection data.  */
    7275      1517908 :   redirect_edge_var_map_empty ();
    7276              : 
    7277              :   /* Register rtl specific functions for cfg.  */
    7278      1517908 :   rtl_register_cfg_hooks ();
    7279              : 
    7280      1517908 :   init_block = construct_init_block ();
    7281              : 
    7282              :   /* Clear EDGE_EXECUTABLE on the entry edge(s).  It is cleaned from the
    7283              :      remaining edges later.  */
    7284      3035816 :   FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
    7285      1517908 :     e->flags &= ~EDGE_EXECUTABLE;
    7286              : 
    7287              :   /* If the function has too many markers, drop them while expanding.  */
    7288      1517908 :   if (cfun->debug_marker_count
    7289      1517908 :       >= param_max_debug_marker_count)
    7290            1 :     cfun->debug_nonbind_markers = false;
    7291              : 
    7292      1517908 :   if (optimize)
    7293      1065298 :     enable_ranger (fun);
    7294      1517908 :   lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
    7295      1517908 :   head_end_for_bb.create (last_basic_block_for_fn (fun));
    7296     14447079 :   FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
    7297              :                   next_bb)
    7298     12929173 :     bb = expand_gimple_basic_block (bb, var_ret_seq);
    7299      1517906 :   if (optimize)
    7300      1065298 :     disable_ranger (fun);
    7301     14849439 :   FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
    7302              :                   next_bb)
    7303              :     {
    7304     13331533 :       if ((bb->flags & BB_RTL) == 0)
    7305              :         {
    7306     12929163 :           bb->il.gimple.seq = NULL;
    7307     12929163 :           bb->il.gimple.phi_nodes = NULL;
    7308     12929163 :           init_rtl_bb_info (bb);
    7309     12929163 :           bb->flags |= BB_RTL;
    7310     12929163 :           BB_HEAD (bb) = head_end_for_bb[bb->index].first;
    7311     12929163 :           BB_END (bb) = head_end_for_bb[bb->index].second;
    7312              :         }
    7313              :       /* These flags have no purpose in RTL land.  */
    7314     13331533 :       if (EDGE_COUNT (bb->succs) == 2)
    7315              :         {
    7316      6420612 :           EDGE_SUCC (bb, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
    7317      6420612 :           EDGE_SUCC (bb, 1)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
    7318              :         }
    7319     19453308 :       else if (single_succ_p (bb))
    7320      6121775 :         single_succ_edge (bb)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
    7321              :     }
    7322      1517906 :   head_end_for_bb.release ();
    7323              : 
    7324      1517906 :   if (MAY_HAVE_DEBUG_BIND_INSNS)
    7325       503732 :     expand_debug_locations ();
    7326              : 
    7327      1517906 :   if (deep_ter_debug_map)
    7328              :     {
    7329          729 :       delete deep_ter_debug_map;
    7330          729 :       deep_ter_debug_map = NULL;
    7331              :     }
    7332              : 
    7333              :   /* Free stuff we no longer need after GIMPLE optimizations.  */
    7334      1517906 :   free_dominance_info (CDI_DOMINATORS);
    7335      1517906 :   free_dominance_info (CDI_POST_DOMINATORS);
    7336      1517906 :   delete_tree_cfg_annotations (fun);
    7337              : 
    7338      1517906 :   timevar_push (TV_OUT_OF_SSA);
    7339      1517906 :   finish_out_of_ssa (&SA);
    7340      1517906 :   timevar_pop (TV_OUT_OF_SSA);
    7341              : 
    7342      1517906 :   timevar_push (TV_POST_EXPAND);
    7343              :   /* We are no longer in SSA form.  */
    7344      1517906 :   fun->gimple_df->in_ssa_p = false;
    7345      1517906 :   loops_state_clear (LOOP_CLOSED_SSA);
    7346              : 
    7347              :   /* Expansion is used by optimization passes too, set maybe_hot_insn_p
    7348              :      conservatively to true until they are all profile aware.  */
    7349      3035812 :   delete lab_rtx_for_bb;
    7350      1517906 :   free_histograms (fun);
    7351              : 
    7352      1517906 :   construct_exit_block ();
    7353      1517906 :   insn_locations_finalize ();
    7354              : 
    7355      1517906 :   if (var_ret_seq)
    7356              :     {
    7357         1938 :       rtx_insn *after = return_label;
    7358         1938 :       rtx_insn *next = NEXT_INSN (after);
    7359         1938 :       if (next && NOTE_INSN_BASIC_BLOCK_P (next))
    7360              :         after = next;
    7361         1938 :       emit_insn_after (var_ret_seq, after);
    7362              :     }
    7363              : 
    7364      1517906 :   if (hwassist_sanitize_stack_p ())
    7365          368 :     hwasan_maybe_emit_frame_base_init ();
    7366              : 
    7367              :   /* Zap the tree EH table.  */
    7368      1517906 :   set_eh_throw_stmt_table (fun, NULL);
    7369              : 
    7370              :   /* We need JUMP_LABEL be set in order to redirect jumps, and hence
    7371              :      split edges which edge insertions might do.  */
    7372      1517906 :   rebuild_jump_labels (get_insns ());
    7373              : 
    7374              :   /* If we have a single successor to the entry block, put the pending insns
    7375              :      after parm birth, but before NOTE_INSNS_FUNCTION_BEG.  */
    7376      1517906 :   if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
    7377              :     {
    7378      1517906 :       edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
    7379      1517906 :       if (e->insns.r)
    7380              :         {
    7381           11 :           rtx_insn *insns = e->insns.r;
    7382           11 :           e->insns.r = NULL;
    7383           11 :           rebuild_jump_labels_chain (insns);
    7384           11 :           if (NOTE_P (parm_birth_insn)
    7385           11 :               && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
    7386           11 :             emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
    7387              :           else
    7388            0 :             emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
    7389              :         }
    7390              :     }
    7391              : 
    7392              :   /* Otherwise, as well as for other edges, take the usual way.  */
    7393      1517906 :   commit_edge_insertions ();
    7394              : 
    7395              :   /* We're done expanding trees to RTL.  */
    7396      1517906 :   currently_expanding_to_rtl = 0;
    7397              : 
    7398      1517906 :   flush_mark_addressable_queue ();
    7399              : 
    7400     18498289 :   FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
    7401              :                   EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
    7402              :     {
    7403     16980383 :       edge e;
    7404     16980383 :       edge_iterator ei;
    7405     39679330 :       for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
    7406              :         {
    7407              :           /* Clear EDGE_EXECUTABLE.  This flag is never used in the backend.  */
    7408     22698947 :           e->flags &= ~EDGE_EXECUTABLE;
    7409              : 
    7410              :           /* At the moment not all abnormal edges match the RTL
    7411              :              representation.  It is safe to remove them here as
    7412              :              find_many_sub_basic_blocks will rediscover them.
    7413              :              In the future we should get this fixed properly.  */
    7414     22698947 :           if ((e->flags & EDGE_ABNORMAL)
    7415       139066 :               && !(e->flags & EDGE_SIBCALL))
    7416        10632 :             remove_edge (e);
    7417              :           else
    7418     22688315 :             ei_next (&ei);
    7419              :         }
    7420              :     }
    7421              : 
    7422      1517906 :   auto_sbitmap blocks (last_basic_block_for_fn (fun));
    7423      1517906 :   bitmap_ones (blocks);
    7424      1517906 :   find_many_sub_basic_blocks (blocks);
    7425      1517906 :   purge_all_dead_edges ();
    7426              : 
    7427              :   /* After initial rtl generation, call back to finish generating
    7428              :      exception support code.  We need to do this before cleaning up
    7429              :      the CFG as the code does not expect dead landing pads.  */
    7430      1517906 :   if (fun->eh->region_tree != NULL)
    7431        66023 :     finish_eh_generation ();
    7432              : 
    7433              :   /* Call expand_stack_alignment after finishing all
    7434              :      updates to crtl->preferred_stack_boundary.  */
    7435      1517906 :   expand_stack_alignment ();
    7436              : 
    7437              :   /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
    7438              :      function.  */
    7439      1517906 :   if (crtl->tail_call_emit)
    7440       115810 :     fixup_tail_calls ();
    7441              : 
    7442      1517906 :   HOST_WIDE_INT patch_area_size, patch_area_entry;
    7443      1517906 :   parse_and_check_patch_area (flag_patchable_function_entry, false,
    7444              :                               &patch_area_size, &patch_area_entry);
    7445              : 
    7446      1517906 :   tree patchable_function_entry_attr
    7447      1517906 :     = lookup_attribute ("patchable_function_entry",
    7448      1517906 :                         DECL_ATTRIBUTES (cfun->decl));
    7449      1517906 :   if (patchable_function_entry_attr)
    7450              :     {
    7451           16 :       tree pp_val = TREE_VALUE (patchable_function_entry_attr);
    7452           16 :       tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
    7453              : 
    7454           16 :       patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
    7455           16 :       patch_area_entry = 0;
    7456           16 :       if (TREE_CHAIN (pp_val) != NULL_TREE)
    7457              :         {
    7458            8 :           tree patchable_function_entry_value2
    7459            8 :             = TREE_VALUE (TREE_CHAIN (pp_val));
    7460            8 :           patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
    7461              :         }
    7462              :     }
    7463              : 
    7464      1517906 :   if (patch_area_entry > patch_area_size)
    7465              :     {
    7466            0 :       if (patch_area_size > 0)
    7467            0 :         warning (OPT_Wattributes,
    7468              :                  "patchable function entry %wu exceeds size %wu",
    7469              :                  patch_area_entry, patch_area_size);
    7470            0 :       patch_area_entry = 0;
    7471              :     }
    7472              : 
    7473      1517906 :   crtl->patch_area_size = patch_area_size;
    7474      1517906 :   crtl->patch_area_entry = patch_area_entry;
    7475              : 
    7476              :   /* BB subdivision may have created basic blocks that are only reachable
    7477              :      from unlikely bbs but not marked as such in the profile.  */
    7478      1517906 :   if (optimize)
    7479      1065298 :     propagate_unlikely_bbs_forward ();
    7480              : 
    7481              :   /* Remove unreachable blocks, otherwise we cannot compute dominators
    7482              :      which are needed for loop state verification.  As a side-effect
    7483              :      this also compacts blocks.
    7484              :      ???  We cannot remove trivially dead insns here as for example
    7485              :      the DRAP reg on i?86 is not magically live at this point.
    7486              :      gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise.  */
    7487      1517906 :   cleanup_cfg (CLEANUP_NO_INSN_DEL);
    7488              : 
    7489      1517906 :   checking_verify_flow_info ();
    7490              : 
    7491              :   /* Initialize pseudos allocated for hard registers.  */
    7492      1517906 :   emit_initial_value_sets ();
    7493              : 
    7494              :   /* And finally unshare all RTL.  */
    7495      1517906 :   unshare_all_rtl ();
    7496              : 
    7497              :   /* There's no need to defer outputting this function any more; we
    7498              :      know we want to output it.  */
    7499      1517906 :   DECL_DEFER_OUTPUT (current_function_decl) = 0;
    7500              : 
    7501              :   /* Now that we're done expanding trees to RTL, we shouldn't have any
    7502              :      more CONCATs anywhere.  */
    7503      1517906 :   generating_concat_p = 0;
    7504              : 
    7505      1517906 :   if (dump_file)
    7506              :     {
    7507          404 :       fprintf (dump_file,
    7508              :                "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
    7509              :       /* And the pass manager will dump RTL for us.  */
    7510              :     }
    7511              : 
    7512              :   /* If we're emitting a nested function, make sure its parent gets
    7513              :      emitted as well.  Doing otherwise confuses debug info.  */
    7514      1517906 :     {
    7515      1517906 :       tree parent;
    7516      1517906 :       for (parent = DECL_CONTEXT (current_function_decl);
    7517      3263571 :            parent != NULL_TREE;
    7518      1745665 :            parent = get_containing_scope (parent))
    7519      1745665 :         if (TREE_CODE (parent) == FUNCTION_DECL)
    7520        78086 :           TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
    7521              :     }
    7522              : 
    7523      1517906 :   TREE_ASM_WRITTEN (current_function_decl) = 1;
    7524              : 
    7525              :   /* After expanding, the return labels are no longer needed. */
    7526      1517906 :   return_label = NULL;
    7527      1517906 :   naked_return_label = NULL;
    7528              : 
    7529              :   /* After expanding, the tm_restart map is no longer needed.  */
    7530      1517906 :   if (fun->gimple_df->tm_restart)
    7531          265 :     fun->gimple_df->tm_restart = NULL;
    7532              : 
    7533              :   /* Tag the blocks with a depth number so that change_scope can find
    7534              :      the common parent easily.  */
    7535      1517906 :   set_block_levels (DECL_INITIAL (fun->decl), 0);
    7536      1517906 :   default_rtl_profile ();
    7537              : 
    7538              :   /* For -dx discard loops now, otherwise IL verify in clean_state will
    7539              :      ICE.  */
    7540      1517906 :   if (rtl_dump_and_exit)
    7541              :     {
    7542           79 :       cfun->curr_properties &= ~PROP_loops;
    7543           79 :       loop_optimizer_finalize ();
    7544              :     }
    7545              : 
    7546      1517906 :   timevar_pop (TV_POST_EXPAND);
    7547              : 
    7548      1517906 :   return 0;
    7549      1517906 : }
    7550              : 
    7551              : } // anon namespace
    7552              : 
    7553              : rtl_opt_pass *
    7554       297184 : make_pass_expand (gcc::context *ctxt)
    7555              : {
    7556       297184 :   return new pass_expand (ctxt);
    7557              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.