LCOV - code coverage report
Current view: top level - gcc - function.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 83.0 % 2868 2380
Test Date: 2026-11-14 16:39:50 Functions: 90.4 % 166 150
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Expands front end tree to back end RTL for GCC.
       2              :    Copyright (C) 1987-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify it under
       7              : the terms of the GNU General Public License as published by the Free
       8              : Software Foundation; either version 3, or (at your option) any later
       9              : version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      12              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14              : for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : /* This file handles the generation of rtl code from tree structure
      21              :    at the level of the function as a whole.
      22              :    It creates the rtl expressions for parameters and auto variables
      23              :    and has full responsibility for allocating stack slots.
      24              : 
      25              :    `expand_function_start' is called at the beginning of a function,
      26              :    before the function body is parsed, and `expand_function_end' is
      27              :    called after parsing the body.
      28              : 
      29              :    Call `assign_stack_local' to allocate a stack slot for a local variable.
      30              :    This is usually done during the RTL generation for the function body,
      31              :    but it can also be done in the reload pass when a pseudo-register does
      32              :    not get a hard register.  */
      33              : 
      34              : #include "config.h"
      35              : #include "system.h"
      36              : #include "coretypes.h"
      37              : #include "backend.h"
      38              : #include "target.h"
      39              : #include "rtl.h"
      40              : #include "tree.h"
      41              : #include "gimple-expr.h"
      42              : #include "cfghooks.h"
      43              : #include "df.h"
      44              : #include "memmodel.h"
      45              : #include "tm_p.h"
      46              : #include "stringpool.h"
      47              : #include "expmed.h"
      48              : #include "optabs.h"
      49              : #include "opts.h"
      50              : #include "regs.h"
      51              : #include "emit-rtl.h"
      52              : #include "recog.h"
      53              : #include "rtl-error.h"
      54              : #include "hard-reg-set.h"
      55              : #include "alias.h"
      56              : #include "fold-const.h"
      57              : #include "stor-layout.h"
      58              : #include "varasm.h"
      59              : #include "except.h"
      60              : #include "dojump.h"
      61              : #include "explow.h"
      62              : #include "calls.h"
      63              : #include "expr.h"
      64              : #include "optabs-tree.h"
      65              : #include "output.h"
      66              : #include "langhooks.h"
      67              : #include "common/common-target.h"
      68              : #include "gimplify.h"
      69              : #include "tree-pass.h"
      70              : #include "cfgrtl.h"
      71              : #include "cfganal.h"
      72              : #include "cfgbuild.h"
      73              : #include "cfgcleanup.h"
      74              : #include "cfgexpand.h"
      75              : #include "shrink-wrap.h"
      76              : #include "toplev.h"
      77              : #include "rtl-iter.h"
      78              : #include "tree-dfa.h"
      79              : #include "tree-ssa.h"
      80              : #include "attribs.h"
      81              : #include "gimple.h"
      82              : #include "options.h"
      83              : #include "function-abi.h"
      84              : #include "value-range.h"
      85              : #include "gimple-range.h"
      86              : #include "insn-attr.h"
      87              : #include "hierarchical_discriminator.h"
      88              : 
      89              : /* So we can assign to cfun in this file.  */
      90              : #undef cfun
      91              : 
      92              : #ifndef STACK_ALIGNMENT_NEEDED
      93              : #define STACK_ALIGNMENT_NEEDED 1
      94              : #endif
      95              : 
      96              : #define STACK_BYTES (STACK_BOUNDARY / BITS_PER_UNIT)
      97              : 
      98              : /* Round a value to the lowest integer less than it that is a multiple of
      99              :    the required alignment.  Avoid using division in case the value is
     100              :    negative.  Assume the alignment is a power of two.  */
     101              : #define FLOOR_ROUND(VALUE,ALIGN) ((VALUE) & ~((ALIGN) - 1))
     102              : 
     103              : /* Similar, but round to the next highest integer that meets the
     104              :    alignment.  */
     105              : #define CEIL_ROUND(VALUE,ALIGN) (((VALUE) + (ALIGN) - 1) & ~((ALIGN)- 1))
     106              : 
     107              : /* Nonzero once virtual register instantiation has been done.
     108              :    assign_stack_local uses frame_pointer_rtx when this is nonzero.
     109              :    calls.cc:emit_library_call_value_1 uses it to set up
     110              :    post-instantiation libcalls.  */
     111              : int virtuals_instantiated;
     112              : 
     113              : /* Assign unique numbers to labels generated for profiling, debugging, etc.  */
     114              : static GTY(()) int funcdef_no;
     115              : 
     116              : /* These variables hold pointers to functions to create and destroy
     117              :    target specific, per-function data structures.  */
     118              : struct machine_function * (*init_machine_status) (void);
     119              : 
     120              : /* The currently compiled function.  */
     121              : struct function *cfun = 0;
     122              : 
     123              : /* These hashes record the prologue and epilogue insns.  */
     124              : 
     125              : struct insn_cache_hasher : ggc_cache_ptr_hash<rtx_def>
     126              : {
     127   1194590993 :   static hashval_t hash (rtx x) { return htab_hash_pointer (x); }
     128              :   static bool equal (rtx a, rtx b) { return a == b; }
     129              : };
     130              : 
     131              : static GTY((cache))
     132              :   hash_table<insn_cache_hasher> *prologue_insn_hash;
     133              : static GTY((cache))
     134              :   hash_table<insn_cache_hasher> *epilogue_insn_hash;
     135              : 
     136              : 
     137              : hash_table<used_type_hasher> *types_used_by_vars_hash = NULL;
     138              : vec<tree, va_gc> *types_used_by_cur_var_decl;
     139              : 
     140              : /* Forward declarations.  */
     141              : 
     142              : static class temp_slot *find_temp_slot_from_address (rtx);
     143              : static void pad_to_arg_alignment (struct args_size *, int, struct args_size *);
     144              : static void pad_below (struct args_size *, machine_mode, tree);
     145              : static void reorder_blocks_1 (rtx_insn *, tree, vec<tree> *);
     146              : static int all_blocks (tree, tree *);
     147              : static tree *get_block_vector (tree, int *);
     148              : extern tree debug_find_var_in_block_tree (tree, tree);
     149              : /* We always define `record_insns' even if it's not used so that we
     150              :    can always export `prologue_epilogue_contains'.  */
     151              : static void record_insns (rtx_insn *, rtx_insn *,
     152              :                           hash_table<insn_cache_hasher> **)
     153              :      ATTRIBUTE_UNUSED;
     154              : static bool contains (const rtx_insn *, hash_table<insn_cache_hasher> *);
     155              : static void prepare_function_start (void);
     156              : static void do_clobber_return_reg (rtx, void *);
     157              : static void do_use_return_reg (rtx, void *);
     158              : static rtx assign_stack_local_2 (machine_mode, poly_int64, int, int, rtx *);
     159              : 
     160              : 
     161              : /* Stack of nested functions.  */
     162              : /* Keep track of the cfun stack.  */
     163              : 
     164              : static vec<function *> function_context_stack;
     165              : 
     166              : /* Save the current context for compilation of a nested function.
     167              :    This is called from language-specific code.  */
     168              : 
     169              : void
     170     97440165 : push_function_context (void)
     171              : {
     172     97440165 :   if (cfun == 0)
     173           25 :     allocate_struct_function (NULL, false);
     174              : 
     175     97440165 :   function_context_stack.safe_push (cfun);
     176     97440165 :   set_cfun (NULL);
     177     97440165 : }
     178              : 
     179              : /* Restore the last saved context, at the end of a nested function.
     180              :    This function is called from language-specific code.  */
     181              : 
     182              : void
     183     97440142 : pop_function_context (void)
     184              : {
     185     97440142 :   struct function *p = function_context_stack.pop ();
     186     97440142 :   set_cfun (p);
     187     97440142 :   current_function_decl = p->decl;
     188              : 
     189              :   /* Reset variables that have known state during rtx generation.  */
     190     97440142 :   virtuals_instantiated = 0;
     191     97440142 :   generating_concat_p = 1;
     192     97440142 : }
     193              : 
     194              : /* Clear out all parts of the state in F that can safely be discarded
     195              :    after the function has been parsed, but not compiled, to let
     196              :    garbage collection reclaim the memory.  */
     197              : 
     198              : void
     199      1740090 : free_after_parsing (struct function *f)
     200              : {
     201      1740090 :   f->language = 0;
     202      1740090 : }
     203              : 
     204              : /* Clear out all parts of the state in F that can safely be discarded
     205              :    after the function has been compiled, to let garbage collection
     206              :    reclaim the memory.  */
     207              : 
     208              : void
     209      1745746 : free_after_compilation (struct function *f)
     210              : {
     211      1745746 :   prologue_insn_hash = NULL;
     212      1745746 :   epilogue_insn_hash = NULL;
     213              : 
     214      1745746 :   free (crtl->emit.regno_pointer_align);
     215              : 
     216      1745746 :   memset (crtl, 0, sizeof (struct rtl_data));
     217      1745746 :   f->eh = NULL;
     218      1745746 :   f->machine = NULL;
     219      1745746 :   f->cfg = NULL;
     220      1745746 :   f->curr_properties &= ~PROP_cfg;
     221      1745890 :   delete f->cond_uids;
     222      1745746 :   free_copyid_allocator (f);
     223              : 
     224      1745746 :   regno_reg_rtx = NULL;
     225      1745746 : }
     226              : 
     227              : /* Return size needed for stack frame based on slots so far allocated.
     228              :    This size counts from zero.  It is not rounded to PREFERRED_STACK_BOUNDARY;
     229              :    the caller may have to do that.  */
     230              : 
     231              : poly_int64
     232    149263738 : get_frame_size (void)
     233              : {
     234    149263738 :   if (FRAME_GROWS_DOWNWARD)
     235    149263738 :     return -frame_offset;
     236              :   else
     237              :     return frame_offset;
     238              : }
     239              : 
     240              : /* Issue an error message and return TRUE if frame OFFSET overflows in
     241              :    the signed target pointer arithmetics for function FUNC.  Otherwise
     242              :    return FALSE.  */
     243              : 
     244              : bool
     245      4063289 : frame_offset_overflow (poly_int64 offset, tree func)
     246              : {
     247      4063289 :   poly_uint64 size = FRAME_GROWS_DOWNWARD ? -offset : offset;
     248      4063289 :   unsigned HOST_WIDE_INT limit
     249      4063289 :     = ((HOST_WIDE_INT_1U << (GET_MODE_BITSIZE (Pmode) - 1))
     250              :        /* Leave room for the fixed part of the frame.  */
     251      4063289 :        - 64 * UNITS_PER_WORD);
     252              : 
     253      8126578 :   if (!coeffs_in_range_p (size, 0U, limit))
     254              :     {
     255            0 :       unsigned HOST_WIDE_INT hwisize;
     256            0 :       if (size.is_constant (&hwisize))
     257            0 :         error_at (DECL_SOURCE_LOCATION (func),
     258              :                   "total size of local objects %wu exceeds maximum %wu",
     259              :                   hwisize, limit);
     260              :       else
     261              :         error_at (DECL_SOURCE_LOCATION (func),
     262              :                   "total size of local objects exceeds maximum %wu",
     263              :                   limit);
     264            0 :       return true;
     265              :     }
     266              : 
     267              :   return false;
     268              : }
     269              : 
     270              : /* Return the minimum spill slot alignment for a register of mode MODE.  */
     271              : 
     272              : unsigned int
     273      1452724 : spill_slot_alignment (machine_mode mode ATTRIBUTE_UNUSED)
     274              : {
     275      1452724 :   return STACK_SLOT_ALIGNMENT (NULL_TREE, mode, GET_MODE_ALIGNMENT (mode));
     276              : }
     277              : 
     278              : /* Return stack slot alignment in bits for TYPE and MODE.  */
     279              : 
     280              : static unsigned int
     281       149651 : get_stack_local_alignment (tree type, machine_mode mode)
     282              : {
     283       149651 :   unsigned int alignment;
     284              : 
     285       149651 :   if (mode == BLKmode)
     286        22621 :     alignment = BIGGEST_ALIGNMENT;
     287              :   else
     288       127030 :     alignment = GET_MODE_ALIGNMENT (mode);
     289              : 
     290              :   /* Allow the frond-end to (possibly) increase the alignment of this
     291              :      stack slot.  */
     292       149651 :   if (! type)
     293        51042 :     type = lang_hooks.types.type_for_mode (mode, 0);
     294              : 
     295       149651 :   return STACK_SLOT_ALIGNMENT (type, mode, alignment);
     296              : }
     297              : 
     298              : /* Determine whether it is possible to fit a stack slot of size SIZE and
     299              :    alignment ALIGNMENT into an area in the stack frame that starts at
     300              :    frame offset START and has a length of LENGTH.  If so, store the frame
     301              :    offset to be used for the stack slot in *POFFSET and return true;
     302              :    return false otherwise.  This function will extend the frame size when
     303              :    given a start/length pair that lies at the end of the frame.  */
     304              : 
     305              : static bool
     306      2426922 : try_fit_stack_local (poly_int64 start, poly_int64 length,
     307              :                      poly_int64 size, unsigned int alignment,
     308              :                      poly_int64 *poffset)
     309              : {
     310      2426922 :   poly_int64 this_frame_offset;
     311      2426922 :   int frame_off, frame_alignment, frame_phase;
     312              : 
     313              :   /* Calculate how many bytes the start of local variables is off from
     314              :      stack alignment.  */
     315      2426922 :   frame_alignment = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
     316      2426922 :   frame_off = targetm.starting_frame_offset () % frame_alignment;
     317      2426922 :   frame_phase = frame_off ? frame_alignment - frame_off : 0;
     318              : 
     319              :   /* Round the frame offset to the specified alignment.  */
     320              : 
     321      2426922 :   if (FRAME_GROWS_DOWNWARD)
     322      2426922 :     this_frame_offset
     323      2426922 :       = (aligned_lower_bound (start + length - size - frame_phase, alignment)
     324      2426922 :          + frame_phase);
     325              :   else
     326              :     this_frame_offset
     327              :       = aligned_upper_bound (start - frame_phase, alignment) + frame_phase;
     328              : 
     329              :   /* See if it fits.  If this space is at the edge of the frame,
     330              :      consider extending the frame to make it fit.  Our caller relies on
     331              :      this when allocating a new slot.  */
     332      2426922 :   if (maybe_lt (this_frame_offset, start))
     333              :     {
     334       504826 :       if (known_eq (frame_offset, start))
     335       375169 :         frame_offset = this_frame_offset;
     336              :       else
     337              :         return false;
     338              :     }
     339      1922096 :   else if (maybe_gt (this_frame_offset + size, start + length))
     340              :     {
     341            0 :       if (known_eq (frame_offset, start + length))
     342            0 :         frame_offset = this_frame_offset + size;
     343              :       else
     344              :         return false;
     345              :     }
     346              : 
     347      2297265 :   *poffset = this_frame_offset;
     348      2297265 :   return true;
     349              : }
     350              : 
     351              : /* Create a new frame_space structure describing free space in the stack
     352              :    frame beginning at START and ending at END, and chain it into the
     353              :    function's frame_space_list.  */
     354              : 
     355              : static void
     356       400619 : add_frame_space (poly_int64 start, poly_int64 end)
     357              : {
     358       400619 :   class frame_space *space = ggc_alloc<frame_space> ();
     359       400619 :   space->next = crtl->frame_space_list;
     360       400619 :   crtl->frame_space_list = space;
     361       400619 :   space->start = start;
     362       400619 :   space->length = end - start;
     363       400619 : }
     364              : 
     365              : /* Allocate a stack slot of SIZE bytes and return a MEM rtx for it
     366              :    with machine mode MODE.
     367              : 
     368              :    ALIGN controls the amount of alignment for the address of the slot:
     369              :    0 means according to MODE,
     370              :    -1 means use BIGGEST_ALIGNMENT and round size to multiple of that,
     371              :    -2 means use BITS_PER_UNIT,
     372              :    positive specifies alignment boundary in bits.
     373              : 
     374              :    A positive ALIGN that exceeds MAX_SUPPORTED_STACK_ALIGNMENT is honoured
     375              :    by overallocating the slot and aligning its address at run time, with
     376              :    insns emitted at the current position, unless KIND has ASLK_REDUCE_ALIGN.
     377              : 
     378              :    KIND has ASLK_REDUCE_ALIGN bit set if it is OK to reduce
     379              :    alignment and ASLK_RECORD_PAD bit set if we should remember
     380              :    extra space we allocated for alignment purposes.  When we are
     381              :    called from assign_stack_temp_for_type, it is not set so we don't
     382              :    track the same stack slot in two independent lists.
     383              : 
     384              :    We do not round to stack_boundary here.  */
     385              : 
     386              : rtx
     387      2218819 : assign_stack_local_1 (machine_mode mode, poly_int64 size,
     388              :                       int align, int kind)
     389              : {
     390      2218819 :   return assign_stack_local_2 (mode, size, align, kind, NULL);
     391              : }
     392              : 
     393              : /* Like assign_stack_local_1.  If BASE_ADDR is nonnull, set it to the slot
     394              :    address before run-time alignment, or to NULL_RTX if none was needed.  */
     395              : 
     396              : static rtx
     397      2297265 : assign_stack_local_2 (machine_mode mode, poly_int64 size,
     398              :                       int align, int kind, rtx *base_addr)
     399              : {
     400      2297265 :   rtx x, addr;
     401      2297265 :   poly_int64 bigend_correction = 0;
     402      2297265 :   poly_int64 slot_offset = 0, old_frame_offset;
     403      2297265 :   unsigned int alignment, alignment_in_bits, dynamic_align = 0;
     404              : 
     405      2297265 :   if (base_addr)
     406        78446 :     *base_addr = NULL_RTX;
     407              : 
     408      2297265 :   if (align == 0)
     409              :     {
     410         7209 :       alignment = get_stack_local_alignment (NULL, mode);
     411         7209 :       alignment /= BITS_PER_UNIT;
     412              :     }
     413      2290056 :   else if (align == -1)
     414              :     {
     415          947 :       alignment = BIGGEST_ALIGNMENT / BITS_PER_UNIT;
     416          947 :       size = aligned_upper_bound (size, alignment);
     417              :     }
     418      2289109 :   else if (align == -2)
     419              :     alignment = 1; /* BITS_PER_UNIT / BITS_PER_UNIT */
     420              :   else
     421      2289109 :     alignment = align / BITS_PER_UNIT;
     422              : 
     423      2297265 :   alignment_in_bits = alignment * BITS_PER_UNIT;
     424              : 
     425              :   /* The frame cannot be aligned beyond MAX_SUPPORTED_STACK_ALIGNMENT.  If more
     426              :      is requested, pad the slot so that an over-aligned object still fits.  */
     427      2297265 :   if (alignment_in_bits > MAX_SUPPORTED_STACK_ALIGNMENT)
     428              :     {
     429            0 :       if (align > 0 && !(kind & ASLK_REDUCE_ALIGN) && maybe_ne (size, 0))
     430              :         {
     431            0 :           gcc_assert (currently_expanding_to_rtl);
     432            0 :           dynamic_align = alignment_in_bits;
     433              : 
     434            0 :           if (crtl->preferred_stack_boundary < PREFERRED_STACK_BOUNDARY)
     435            0 :             crtl->preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
     436              : 
     437              :           /* Pad from the alignment the frame base is guaranteed to have.  */
     438            0 :           size += ((alignment_in_bits
     439            0 :                     - REGNO_POINTER_ALIGN (VIRTUAL_STACK_VARS_REGNUM))
     440            0 :                    / BITS_PER_UNIT);
     441              :         }
     442              :       alignment_in_bits = MAX_SUPPORTED_STACK_ALIGNMENT;
     443              :       alignment = MAX_SUPPORTED_STACK_ALIGNMENT / BITS_PER_UNIT;
     444              :     }
     445              : 
     446      2297265 :   if (SUPPORTS_STACK_ALIGNMENT)
     447              :     {
     448      2297265 :       if (crtl->stack_alignment_estimated < alignment_in_bits)
     449              :         {
     450         4426 :           if (!crtl->stack_realign_processed)
     451         4405 :             crtl->stack_alignment_estimated = alignment_in_bits;
     452              :           else
     453              :             {
     454              :               /* If stack is realigned and stack alignment value
     455              :                  hasn't been finalized, it is OK not to increase
     456              :                  stack_alignment_estimated.  The bigger alignment
     457              :                  requirement is recorded in stack_alignment_needed
     458              :                  below.  */
     459           21 :               gcc_assert (!crtl->stack_realign_finalized);
     460           21 :               if (!crtl->stack_realign_needed)
     461              :                 {
     462              :                   /* It is OK to reduce the alignment as long as the
     463              :                      requested size is 0 or the estimated stack
     464              :                      alignment >= mode alignment.  */
     465           21 :                   gcc_assert ((kind & ASLK_REDUCE_ALIGN)
     466              :                               || known_eq (size, 0)
     467              :                               || (crtl->stack_alignment_estimated
     468              :                                   >= GET_MODE_ALIGNMENT (mode)));
     469           21 :                   alignment_in_bits = crtl->stack_alignment_estimated;
     470           21 :                   alignment = alignment_in_bits / BITS_PER_UNIT;
     471              :                 }
     472              :             }
     473              :         }
     474              :     }
     475              : 
     476      2297265 :   if (crtl->stack_alignment_needed < alignment_in_bits)
     477        20649 :     crtl->stack_alignment_needed = alignment_in_bits;
     478      2297265 :   if (crtl->max_used_stack_slot_alignment < alignment_in_bits)
     479       300900 :     crtl->max_used_stack_slot_alignment = alignment_in_bits;
     480              : 
     481      2297265 :   if (mode != BLKmode || maybe_ne (size, 0))
     482              :     {
     483      1481813 :       if (kind & ASLK_RECORD_PAD)
     484              :         {
     485              :           class frame_space **psp;
     486              : 
     487      1533024 :           for (psp = &crtl->frame_space_list; *psp; psp = &(*psp)->next)
     488              :             {
     489       189211 :               class frame_space *space = *psp;
     490       189211 :               if (!try_fit_stack_local (space->start, space->length, size,
     491              :                                         alignment, &slot_offset))
     492       129657 :                 continue;
     493        59554 :               *psp = space->next;
     494        59554 :               if (known_gt (slot_offset, space->start))
     495        23443 :                 add_frame_space (space->start, slot_offset);
     496        59554 :               if (known_lt (slot_offset + size, space->start + space->length))
     497        11192 :                 add_frame_space (slot_offset + size,
     498        11192 :                                  space->start + space->length);
     499        59554 :               goto found_space;
     500              :             }
     501              :         }
     502              :     }
     503              :   else if (!STACK_ALIGNMENT_NEEDED)
     504              :     {
     505              :       slot_offset = frame_offset;
     506              :       goto found_space;
     507              :     }
     508              : 
     509      2237711 :   old_frame_offset = frame_offset;
     510              : 
     511      2237711 :   if (FRAME_GROWS_DOWNWARD)
     512              :     {
     513      2237711 :       frame_offset -= size;
     514      2237711 :       try_fit_stack_local (frame_offset, size, size, alignment, &slot_offset);
     515              : 
     516      2237711 :       if (kind & ASLK_RECORD_PAD)
     517              :         {
     518      2159265 :           if (known_gt (slot_offset, frame_offset))
     519            0 :             add_frame_space (frame_offset, slot_offset);
     520      2159265 :           if (known_lt (slot_offset + size, old_frame_offset))
     521       365984 :             add_frame_space (slot_offset + size, old_frame_offset);
     522              :         }
     523              :     }
     524              :   else
     525              :     {
     526              :       frame_offset += size;
     527              :       try_fit_stack_local (old_frame_offset, size, size, alignment, &slot_offset);
     528              : 
     529              :       if (kind & ASLK_RECORD_PAD)
     530              :         {
     531              :           if (known_gt (slot_offset, old_frame_offset))
     532              :             add_frame_space (old_frame_offset, slot_offset);
     533              :           if (known_lt (slot_offset + size, frame_offset))
     534              :             add_frame_space (slot_offset + size, frame_offset);
     535              :         }
     536              :     }
     537              : 
     538      2297265 :  found_space:
     539              :   /* On a big-endian machine, if we are allocating more space than we will use,
     540              :      use the least significant bytes of those that are allocated.  */
     541      2297265 :   if (mode != BLKmode && !dynamic_align)
     542              :     {
     543              :       /* The slot size can sometimes be smaller than the mode size;
     544              :          e.g. the rs6000 port allocates slots with a vector mode
     545              :          that have the size of only one element.  However, the slot
     546              :          size must always be ordered wrt to the mode size, in the
     547              :          same way as for a subreg.  */
     548       691804 :       gcc_checking_assert (ordered_p (GET_MODE_SIZE (mode), size));
     549              :       if (BYTES_BIG_ENDIAN && maybe_lt (GET_MODE_SIZE (mode), size))
     550              :         bigend_correction = size - GET_MODE_SIZE (mode);
     551              :     }
     552              : 
     553              :   /* If we have already instantiated virtual registers, return the actual
     554              :      address relative to the frame pointer.  */
     555      2297265 :   if (virtuals_instantiated)
     556      1885333 :     addr = plus_constant (Pmode, frame_pointer_rtx,
     557              :                           trunc_int_for_mode
     558      1591508 :                           (slot_offset + bigend_correction
     559      1885333 :                            + targetm.starting_frame_offset (), Pmode));
     560              :   else
     561       734824 :     addr = plus_constant (Pmode, virtual_stack_vars_rtx,
     562              :                           trunc_int_for_mode
     563              :                           (slot_offset + bigend_correction,
     564       705757 :                            Pmode));
     565              : 
     566      2297265 :   if (dynamic_align)
     567              :     {
     568            0 :       if (base_addr)
     569            0 :         *base_addr = addr;
     570            0 :       addr = align_dynamic_address (addr, dynamic_align);
     571            0 :       mark_reg_pointer (addr, dynamic_align);
     572            0 :       alignment_in_bits = dynamic_align;
     573              :     }
     574              : 
     575      2297265 :   x = gen_rtx_MEM (mode, addr);
     576      2297265 :   set_mem_align (x, alignment_in_bits);
     577      2297265 :   MEM_NOTRAP_P (x) = 1;
     578              : 
     579      2297265 :   vec_safe_push (stack_slot_list, x);
     580              : 
     581      2297265 :   if (frame_offset_overflow (frame_offset, current_function_decl))
     582            0 :     frame_offset = 0;
     583              : 
     584      2297265 :   return x;
     585              : }
     586              : 
     587              : /* Wrap up assign_stack_local_1 with last parameter as false.  */
     588              : 
     589              : rtx
     590      1445297 : assign_stack_local (machine_mode mode, poly_int64 size, int align)
     591              : {
     592      1445297 :   return assign_stack_local_1 (mode, size, align, ASLK_RECORD_PAD);
     593              : }
     594              : 
     595              : /* In order to evaluate some expressions, such as function calls returning
     596              :    structures in memory, we need to temporarily allocate stack locations.
     597              :    We record each allocated temporary in the following structure.
     598              : 
     599              :    Associated with each temporary slot is a nesting level.  When we pop up
     600              :    one level, all temporaries associated with the previous level are freed.
     601              :    Normally, all temporaries are freed after the execution of the statement
     602              :    in which they were created.  However, if we are inside a ({...}) grouping,
     603              :    the result may be in a temporary and hence must be preserved.  If the
     604              :    result could be in a temporary, we preserve it if we can determine which
     605              :    one it is in.  If we cannot determine which temporary may contain the
     606              :    result, all temporaries are preserved.  A temporary is preserved by
     607              :    pretending it was allocated at the previous nesting level.  */
     608              : 
     609              : class GTY(()) temp_slot {
     610              : public:
     611              :   /* Points to next temporary slot.  */
     612              :   class temp_slot *next;
     613              :   /* Points to previous temporary slot.  */
     614              :   class temp_slot *prev;
     615              :   /* The rtx to used to reference the slot.  */
     616              :   rtx slot;
     617              :   /* The size, in units, of the slot.  */
     618              :   poly_int64 size;
     619              :   /* The type of the object in the slot, or zero if it doesn't correspond
     620              :      to a type.  We use this to determine whether a slot can be reused.
     621              :      It can be reused if objects of the type of the new slot will always
     622              :      conflict with objects of the type of the old slot.  */
     623              :   tree type;
     624              :   /* The alignment (in bits) of the slot.  */
     625              :   unsigned int align;
     626              :   /* True if this temporary is currently in use.  */
     627              :   bool in_use;
     628              :   /* Nesting level at which this slot is being used.  */
     629              :   int level;
     630              :   /* The offset of the slot from the frame_pointer, including extra space
     631              :      for alignment.  This info is for combine_temp_slots.  */
     632              :   poly_int64 base_offset;
     633              :   /* The size of the slot, including extra space for alignment.  This
     634              :      info is for combine_temp_slots.  */
     635              :   poly_int64 full_size;
     636              :   /* The address before run-time alignment, or NULL_RTX if the slot does not
     637              :      need run-time alignment.  */
     638              :   rtx base_addr;
     639              : };
     640              : 
     641              : /* Entry for the below hash table.  */
     642              : struct GTY((for_user)) temp_slot_address_entry {
     643              :   hashval_t hash;
     644              :   rtx address;
     645              :   class temp_slot *temp_slot;
     646              : };
     647              : 
     648              : struct temp_address_hasher : ggc_ptr_hash<temp_slot_address_entry>
     649              : {
     650              :   static hashval_t hash (temp_slot_address_entry *);
     651              :   static bool equal (temp_slot_address_entry *, temp_slot_address_entry *);
     652              : };
     653              : 
     654              : /* A table of addresses that represent a stack slot.  The table is a mapping
     655              :    from address RTXen to a temp slot.  */
     656              : static GTY(()) hash_table<temp_address_hasher> *temp_slot_address_table;
     657              : static size_t n_temp_slots_in_use;
     658              : 
     659              : /* Removes temporary slot TEMP from LIST.  */
     660              : 
     661              : static void
     662       208197 : cut_slot_from_list (class temp_slot *temp, class temp_slot **list)
     663              : {
     664            0 :   if (temp->next)
     665        25516 :     temp->next->prev = temp->prev;
     666       208197 :   if (temp->prev)
     667         7166 :     temp->prev->next = temp->next;
     668              :   else
     669       201031 :     *list = temp->next;
     670              : 
     671       208197 :   temp->prev = temp->next = NULL;
     672            0 : }
     673              : 
     674              : /* Inserts temporary slot TEMP to LIST.  */
     675              : 
     676              : static void
     677       286541 : insert_slot_to_list (class temp_slot *temp, class temp_slot **list)
     678              : {
     679       286541 :   temp->next = *list;
     680            0 :   if (*list)
     681        73856 :     (*list)->prev = temp;
     682       286541 :   temp->prev = NULL;
     683       286541 :   *list = temp;
     684            0 : }
     685              : 
     686              : /* Returns the list of used temp slots at LEVEL.  */
     687              : 
     688              : static class temp_slot **
     689     68335100 : temp_slots_at_level (int level)
     690              : {
     691    135219685 :   if (level >= (int) vec_safe_length (used_temp_slots))
     692      1953400 :     vec_safe_grow_cleared (used_temp_slots, level + 1, true);
     693              : 
     694     68335100 :   return &(*used_temp_slots)[level];
     695              : }
     696              : 
     697              : /* Returns the maximal temporary slot level.  */
     698              : 
     699              : static int
     700      1387360 : max_slot_level (void)
     701              : {
     702            0 :   if (!used_temp_slots)
     703              :     return -1;
     704              : 
     705      1327864 :   return used_temp_slots->length () - 1;
     706              : }
     707              : 
     708              : /* Moves temporary slot TEMP to LEVEL.  */
     709              : 
     710              : static void
     711         1228 : move_slot_to_level (class temp_slot *temp, int level)
     712              : {
     713         1228 :   cut_slot_from_list (temp, temp_slots_at_level (temp->level));
     714         1228 :   insert_slot_to_list (temp, temp_slots_at_level (level));
     715         1228 :   temp->level = level;
     716         1228 : }
     717              : 
     718              : /* Make temporary slot TEMP available.  */
     719              : 
     720              : static void
     721       142426 : make_slot_available (class temp_slot *temp)
     722              : {
     723       142426 :   cut_slot_from_list (temp, temp_slots_at_level (temp->level));
     724       142426 :   insert_slot_to_list (temp, &avail_temp_slots);
     725       142426 :   temp->in_use = false;
     726       142426 :   temp->level = -1;
     727       142426 :   n_temp_slots_in_use--;
     728       142426 : }
     729              : 
     730              : /* Compute the hash value for an address -> temp slot mapping.
     731              :    The value is cached on the mapping entry.  */
     732              : static hashval_t
     733      8425526 : temp_slot_address_compute_hash (struct temp_slot_address_entry *t)
     734              : {
     735      8425526 :   int do_not_record = 0;
     736      8425526 :   return hash_rtx (t->address, GET_MODE (t->address),
     737      8425526 :                    &do_not_record, NULL, false);
     738              : }
     739              : 
     740              : /* Return the hash value for an address -> temp slot mapping.  */
     741              : hashval_t
     742        30371 : temp_address_hasher::hash (temp_slot_address_entry *t)
     743              : {
     744        30371 :   return t->hash;
     745              : }
     746              : 
     747              : /* Compare two address -> temp slot mapping entries.  */
     748              : bool
     749        30784 : temp_address_hasher::equal (temp_slot_address_entry *t1,
     750              :                             temp_slot_address_entry *t2)
     751              : {
     752        30784 :   return exp_equiv_p (t1->address, t2->address, 0, true);
     753              : }
     754              : 
     755              : /* Add ADDRESS as an alias of TEMP_SLOT to the address -> temp slot mapping.  */
     756              : static void
     757       142480 : insert_temp_slot_address (rtx address, class temp_slot *temp_slot)
     758              : {
     759       142480 :   struct temp_slot_address_entry *t = ggc_alloc<temp_slot_address_entry> ();
     760       142480 :   t->address = copy_rtx (address);
     761       142480 :   t->temp_slot = temp_slot;
     762       142480 :   t->hash = temp_slot_address_compute_hash (t);
     763       142480 :   *temp_slot_address_table->find_slot_with_hash (t, t->hash, INSERT) = t;
     764       142480 : }
     765              : 
     766              : /* Remove an address -> temp slot mapping entry if the temp slot is
     767              :    not in use anymore.  Callback for remove_unused_temp_slot_addresses.  */
     768              : int
     769          886 : remove_unused_temp_slot_addresses_1 (temp_slot_address_entry **slot, void *)
     770              : {
     771          886 :   const struct temp_slot_address_entry *t = *slot;
     772          886 :   if (! t->temp_slot->in_use)
     773          520 :     temp_slot_address_table->clear_slot (slot);
     774          886 :   return 1;
     775              : }
     776              : 
     777              : /* Remove all mappings of addresses to unused temp slots.  */
     778              : static void
     779       134950 : remove_unused_temp_slot_addresses (void)
     780              : {
     781              :   /* Use quicker clearing if there aren't any active temp slots.  */
     782       134950 :   if (n_temp_slots_in_use)
     783          350 :     temp_slot_address_table->traverse
     784         1236 :       <void *, remove_unused_temp_slot_addresses_1> (NULL);
     785              :   else
     786       134600 :     temp_slot_address_table->empty ();
     787       134950 : }
     788              : 
     789              : /* Find the temp slot corresponding to the object at address X.  */
     790              : 
     791              : static class temp_slot *
     792      8283046 : find_temp_slot_from_address (rtx x)
     793              : {
     794      8283046 :   class temp_slot *p;
     795      8283046 :   struct temp_slot_address_entry tmp, *t;
     796              : 
     797              :   /* First try the easy way:
     798              :      See if X exists in the address -> temp slot mapping.  */
     799      8283046 :   tmp.address = x;
     800      8283046 :   tmp.temp_slot = NULL;
     801      8283046 :   tmp.hash = temp_slot_address_compute_hash (&tmp);
     802      8283046 :   t = temp_slot_address_table->find_with_hash (&tmp, tmp.hash);
     803      8283046 :   if (t)
     804         1260 :     return t->temp_slot;
     805              : 
     806              :   /* If we have a sum involving a register, see if it points to a temp
     807              :      slot.  */
     808      1342136 :   if (GET_CODE (x) == PLUS && REG_P (XEXP (x, 0))
     809      9325522 :       && (p = find_temp_slot_from_address (XEXP (x, 0))) != 0)
     810              :     return p;
     811      1342136 :   else if (GET_CODE (x) == PLUS && REG_P (XEXP (x, 1))
     812      8480373 :            && (p = find_temp_slot_from_address (XEXP (x, 1))) != 0)
     813              :     return p;
     814              : 
     815              :   /* Last resort: Address is a virtual stack var address.  */
     816      8281786 :   poly_int64 offset;
     817      8281786 :   if (strip_offset (x, &offset) == virtual_stack_vars_rtx)
     818              :     {
     819      1387360 :       int i;
     820      6062228 :       for (i = max_slot_level (); i >= 0; i--)
     821      3348490 :         for (p = *temp_slots_at_level (i); p; p = p->next)
     822         2972 :           if (known_in_range_p (offset, p->base_offset, p->full_size))
     823              :             return p;
     824              :     }
     825              : 
     826              :   return NULL;
     827              : }
     828              : 
     829              : /* Allocate a temporary stack slot and record it for possible later
     830              :    reuse.
     831              : 
     832              :    MODE is the machine mode to be given to the returned rtx.
     833              : 
     834              :    SIZE is the size in units of the space required.  We do no rounding here
     835              :    since assign_stack_local will do any required rounding.
     836              : 
     837              :    TYPE is the type that will be used for the stack slot.  */
     838              : 
     839              : rtx
     840       142442 : assign_stack_temp_for_type (machine_mode mode, poly_int64 size, tree type)
     841              : {
     842       142442 :   unsigned int align;
     843       142442 :   class temp_slot *p, *best_p = 0, *selected = NULL, **pp;
     844       142442 :   rtx slot;
     845              : 
     846       142442 :   gcc_assert (known_size_p (size));
     847              : 
     848       142442 :   align = get_stack_local_alignment (type, mode);
     849              : 
     850       142442 :   unsigned int required_align
     851       142442 :     = (mode == BLKmode ? BITS_PER_UNIT : GET_MODE_ALIGNMENT (mode));
     852       142442 :   if (type)
     853        98609 :     required_align = MAX (required_align, TYPE_ALIGN (type));
     854              : 
     855       284884 :   bool dynamic_align = required_align > MAX_SUPPORTED_STACK_ALIGNMENT;
     856              : 
     857       142442 :   if (dynamic_align)
     858              :     align = required_align;
     859              :   else
     860       142442 :     align = MAX (required_align, MIN (align, MAX_SUPPORTED_STACK_ALIGNMENT));
     861              : 
     862              :   /* Try to find an available, already-allocated temporary of the proper
     863              :      mode which meets the size and alignment requirements.  Choose the
     864              :      smallest one with the closest alignment.
     865              : 
     866              :      If assign_stack_temp is called outside of the tree->rtl expansion,
     867              :      we cannot reuse the stack slots (that may still refer to
     868              :      VIRTUAL_STACK_VARS_REGNUM).  */
     869       142442 :   if (!virtuals_instantiated)
     870              :     {
     871     11032614 :       for (p = avail_temp_slots; p; p = p->next)
     872              :         {
     873     10940626 :           if (p->align >= align
     874     10382745 :               && known_ge (p->size, size)
     875     10366147 :               && GET_MODE (p->slot) == mode
     876     10233817 :               && objects_must_conflict_p (p->type, type)
     877     11004856 :               && (best_p == 0
     878          234 :                   || (known_eq (best_p->size, p->size)
     879          120 :                       ? best_p->align > p->align
     880          114 :                       : known_ge (best_p->size, p->size))))
     881              :             {
     882        64078 :               if (p->align == align && known_eq (p->size, size))
     883              :                 {
     884        50454 :                   selected = p;
     885        50454 :                   cut_slot_from_list (selected, &avail_temp_slots);
     886        50454 :                   best_p = 0;
     887        50454 :                   break;
     888              :                 }
     889              :               best_p = p;
     890              :             }
     891              :         }
     892              :     }
     893              : 
     894              :   /* Make our best, if any, the one to use.  */
     895       142442 :   if (best_p)
     896              :     {
     897        13542 :       selected = best_p;
     898        13542 :       cut_slot_from_list (selected, &avail_temp_slots);
     899              : 
     900              :       /* If there are enough aligned bytes left over, make them into a new
     901              :          temp_slot so that the extra bytes don't get wasted.  Do this only
     902              :          for BLKmode slots, so that we can be sure of the alignment.  */
     903        13542 :       if (GET_MODE (best_p->slot) == BLKmode
     904        11220 :           && best_p->align <= MAX_SUPPORTED_STACK_ALIGNMENT)
     905              :         {
     906        11220 :           int alignment = best_p->align / BITS_PER_UNIT;
     907        11220 :           poly_int64 rounded_size = aligned_upper_bound (size, alignment);
     908              : 
     909        11220 :           if (known_ge (best_p->size - rounded_size, alignment))
     910              :             {
     911          445 :               p = ggc_alloc<temp_slot> ();
     912          445 :               p->base_addr = NULL_RTX;
     913          445 :               p->in_use = false;
     914          445 :               p->size = best_p->size - rounded_size;
     915          445 :               p->base_offset = best_p->base_offset + rounded_size;
     916          445 :               p->full_size = best_p->full_size - rounded_size;
     917          445 :               p->slot = adjust_address_nv (best_p->slot, BLKmode, rounded_size);
     918          445 :               p->align = best_p->align;
     919          445 :               p->type = best_p->type;
     920          445 :               insert_slot_to_list (p, &avail_temp_slots);
     921              : 
     922          445 :               vec_safe_push (stack_slot_list, p->slot);
     923              : 
     924          445 :               best_p->size = rounded_size;
     925          445 :               best_p->full_size = rounded_size;
     926              :             }
     927              :         }
     928              :     }
     929              : 
     930       282562 :   bool reused_p = selected != 0;
     931              : 
     932              :   /* If we still didn't find one, make a new temporary.  */
     933       140120 :   if (selected == 0)
     934              :     {
     935        78446 :       poly_int64 frame_offset_old = frame_offset;
     936              : 
     937        78446 :       p = ggc_alloc<temp_slot> ();
     938        78446 :       p->base_addr = NULL_RTX;
     939              : 
     940        78446 :       poly_int64 slot_size
     941              :         = (mode == BLKmode
     942        78446 :              ? aligned_upper_bound (size, (int) align / BITS_PER_UNIT)
     943              :              : size);
     944              : 
     945              :       /* We are passing an explicit alignment request to assign_stack_local.
     946              :          One side effect of that is assign_stack_local will not round SIZE
     947              :          to ensure the frame offset remains suitably aligned.
     948              : 
     949              :          So for requests which depended on the rounding of SIZE, we go ahead
     950              :          and round it now.  We also make sure ALIGNMENT is at least
     951              :          BIGGEST_ALIGNMENT.  */
     952        88470 :       gcc_assert (mode != BLKmode
     953              :                   || dynamic_align
     954              :                   || (align
     955              :                       >= MIN (BIGGEST_ALIGNMENT,
     956              :                               MAX_SUPPORTED_STACK_ALIGNMENT)));
     957              : 
     958        78446 :       rtx base_addr;
     959        78446 :       p->slot = assign_stack_local_2 (mode, slot_size, align, 0, &base_addr);
     960        78446 :       p->base_addr = base_addr;
     961              : 
     962        78446 :       p->align = align;
     963              : 
     964              :       /* The following slot size computation is necessary because we don't
     965              :          know the actual size of the temporary slot until assign_stack_local
     966              :          has performed all the frame alignment and size rounding for the
     967              :          requested temporary.  Note that extra space added for alignment
     968              :          can be either above or below this stack slot depending on which
     969              :          way the frame grows.  We include the extra space if and only if it
     970              :          is above this slot.  */
     971        78446 :       if (dynamic_align)
     972            0 :         p->size = slot_size;
     973              :       else if (FRAME_GROWS_DOWNWARD)
     974        78446 :         p->size = frame_offset_old - frame_offset;
     975              :       else
     976              :         p->size = size;
     977              : 
     978              :       /* Now define the fields used by combine_temp_slots.  */
     979        78446 :       if (FRAME_GROWS_DOWNWARD)
     980              :         {
     981        78446 :           p->base_offset = frame_offset;
     982        78446 :           p->full_size = frame_offset_old - frame_offset;
     983              :         }
     984              :       else
     985              :         {
     986              :           p->base_offset = frame_offset_old;
     987              :           p->full_size = frame_offset - frame_offset_old;
     988              :         }
     989              : 
     990        78446 :       selected = p;
     991              :     }
     992              : 
     993       142442 :   p = selected;
     994       142442 :   p->in_use = true;
     995       142442 :   p->type = type;
     996       142442 :   p->level = temp_slot_level;
     997       142442 :   n_temp_slots_in_use++;
     998              : 
     999       142442 :   pp = temp_slots_at_level (p->level);
    1000       142442 :   insert_slot_to_list (p, pp);
    1001              : 
    1002       142442 :   rtx addr = XEXP (p->slot, 0);
    1003              : 
    1004       142442 :   if (reused_p && p->align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1005              :     {
    1006            0 :       addr = align_dynamic_address (p->base_addr, p->align);
    1007            0 :       mark_reg_pointer (addr, p->align);
    1008              :     }
    1009              : 
    1010       142442 :   insert_temp_slot_address (addr, p);
    1011              : 
    1012              :   /* Create a new MEM rtx to avoid clobbering MEM flags of old slots.  */
    1013       142442 :   slot = gen_rtx_MEM (mode, addr);
    1014       142442 :   vec_safe_push (stack_slot_list, slot);
    1015              : 
    1016              :   /* If we know the alias set for the memory that will be used, use
    1017              :      it.  If there's no TYPE, then we don't know anything about the
    1018              :      alias set for the memory.  */
    1019       142442 :   set_mem_alias_set (slot, type ? get_alias_set (type) : 0);
    1020       142442 :   set_mem_align (slot, align);
    1021              : 
    1022              :   /* If a type is specified, set the relevant flags.  */
    1023       142442 :   if (type != 0)
    1024        98609 :     MEM_VOLATILE_P (slot) = TYPE_VOLATILE (type);
    1025       142442 :   MEM_NOTRAP_P (slot) = 1;
    1026              : 
    1027       142442 :   return slot;
    1028              : }
    1029              : 
    1030              : /* Allocate a temporary stack slot and record it for possible later
    1031              :    reuse.  First two arguments are same as in preceding function.  */
    1032              : 
    1033              : rtx
    1034        43833 : assign_stack_temp (machine_mode mode, poly_int64 size)
    1035              : {
    1036        43833 :   return assign_stack_temp_for_type (mode, size, NULL_TREE);
    1037              : }
    1038              : 
    1039              : /* Assign a temporary.
    1040              :    If TYPE_OR_DECL is a decl, then we are doing it on behalf of the decl
    1041              :    and so that should be used in error messages.  In either case, we
    1042              :    allocate of the given type.
    1043              :    MEMORY_REQUIRED is 1 if the result must be addressable stack memory;
    1044              :    it is 0 if a register is OK.
    1045              :    DONT_PROMOTE is 1 if we should not promote values in register
    1046              :    to wider modes.  */
    1047              : 
    1048              : rtx
    1049      1029593 : assign_temp (tree type_or_decl, int memory_required,
    1050              :              int dont_promote ATTRIBUTE_UNUSED)
    1051              : {
    1052      1029593 :   tree type, decl;
    1053      1029593 :   machine_mode mode;
    1054              : #ifdef PROMOTE_MODE
    1055      1029593 :   int unsignedp;
    1056              : #endif
    1057              : 
    1058      1029593 :   if (DECL_P (type_or_decl))
    1059            0 :     decl = type_or_decl, type = TREE_TYPE (decl);
    1060              :   else
    1061              :     decl = NULL, type = type_or_decl;
    1062              : 
    1063      1029593 :   mode = TYPE_MODE (type);
    1064              : #ifdef PROMOTE_MODE
    1065      1029593 :   unsignedp = TYPE_UNSIGNED (type);
    1066              : #endif
    1067              : 
    1068              :   /* Allocating temporaries of TREE_ADDRESSABLE type must be done in the front
    1069              :      end.  See also create_tmp_var for the gimplification-time check.  */
    1070      1029593 :   gcc_assert (!TREE_ADDRESSABLE (type) && COMPLETE_TYPE_P (type));
    1071              : 
    1072      1029593 :   if (mode == BLKmode || memory_required)
    1073              :     {
    1074        93054 :       poly_int64 size;
    1075        93054 :       rtx tmp;
    1076              : 
    1077              :       /* Unfortunately, we don't yet know how to allocate variable-sized
    1078              :          temporaries.  However, sometimes we can find a fixed upper limit on
    1079              :          the size, so try that instead.  */
    1080        93054 :       if (!poly_int_tree_p (TYPE_SIZE_UNIT (type), &size))
    1081            0 :         size = max_int_size_in_bytes (type);
    1082              : 
    1083              :       /* Zero sized arrays are a GNU C extension.  Set size to 1 to avoid
    1084              :          problems with allocating the stack space.  */
    1085        93054 :       if (known_eq (size, 0))
    1086            0 :         size = 1;
    1087              : 
    1088              :       /* The size of the temporary may be too large to fit into an integer.  */
    1089              :       /* ??? Not sure this should happen except for user silliness, so limit
    1090              :          this to things that aren't compiler-generated temporaries.  The
    1091              :          rest of the time we'll die in assign_stack_temp_for_type.  */
    1092        93054 :       if (decl
    1093            0 :           && !known_size_p (size)
    1094        93054 :           && TREE_CODE (TYPE_SIZE_UNIT (type)) == INTEGER_CST)
    1095              :         {
    1096            0 :           error ("size of variable %q+D is too large", decl);
    1097            0 :           size = 1;
    1098              :         }
    1099              : 
    1100        93054 :       tmp = assign_stack_temp_for_type (mode, size, type);
    1101        93054 :       return tmp;
    1102              :     }
    1103              : 
    1104              : #ifdef PROMOTE_MODE
    1105       936539 :   if (! dont_promote)
    1106            0 :     mode = promote_mode (type, mode, &unsignedp);
    1107              : #endif
    1108              : 
    1109       936539 :   return gen_reg_rtx (mode);
    1110              : }
    1111              : 
    1112              : /* Combine temporary stack slots which are adjacent on the stack.
    1113              : 
    1114              :    This allows for better use of already allocated stack space.  This is only
    1115              :    done for BLKmode slots because we can be sure that we won't have alignment
    1116              :    problems in this case.  */
    1117              : 
    1118              : static void
    1119       134950 : combine_temp_slots (void)
    1120              : {
    1121       134950 :   class temp_slot *p, *q, *next, *next_q;
    1122       134950 :   int num_slots;
    1123              : 
    1124              :   /* We can't combine slots, because the information about which slot
    1125              :      is in which alias set will be lost.  */
    1126       134950 :   if (flag_strict_aliasing)
    1127              :     return;
    1128              : 
    1129              :   /* If there are a lot of temp slots, don't do anything unless
    1130              :      high levels of optimization.  */
    1131        93231 :   if (! flag_expensive_optimizations)
    1132       423913 :     for (p = avail_temp_slots, num_slots = 0; p; p = p->next, num_slots++)
    1133       415239 :       if (num_slots > 100 || (num_slots > 10 && optimize == 0))
    1134              :         return;
    1135              : 
    1136       244957 :   for (p = avail_temp_slots; p; p = next)
    1137              :     {
    1138       164154 :       int delete_p = 0;
    1139              : 
    1140       164154 :       next = p->next;
    1141              : 
    1142       164154 :       if (GET_MODE (p->slot) != BLKmode
    1143        23544 :           || p->align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1144       140610 :         continue;
    1145              : 
    1146        28174 :       for (q = p->next; q; q = next_q)
    1147              :         {
    1148         4641 :           int delete_q = 0;
    1149              : 
    1150         4641 :           next_q = q->next;
    1151              : 
    1152         4641 :           if (GET_MODE (q->slot) != BLKmode
    1153          610 :               || q->align > MAX_SUPPORTED_STACK_ALIGNMENT)
    1154         4031 :             continue;
    1155              : 
    1156          610 :           if (known_eq (p->base_offset + p->full_size, q->base_offset))
    1157              :             {
    1158              :               /* Q comes after P; combine Q into P.  */
    1159          536 :               p->size += q->size;
    1160         4630 :               p->full_size += q->full_size;
    1161              :               delete_q = 1;
    1162              :             }
    1163           74 :           else if (known_eq (q->base_offset + q->full_size, p->base_offset))
    1164              :             {
    1165              :               /* P comes after Q; combine P into Q.  */
    1166           11 :               q->size += p->size;
    1167           11 :               q->full_size += p->full_size;
    1168              :               delete_p = 1;
    1169              :               break;
    1170              :             }
    1171          536 :           if (delete_q)
    1172         1072 :             cut_slot_from_list (q, &avail_temp_slots);
    1173              :         }
    1174              : 
    1175              :       /* Either delete P or advance past it.  */
    1176        23544 :       if (delete_p)
    1177           22 :         cut_slot_from_list (p, &avail_temp_slots);
    1178              :     }
    1179              : }
    1180              : 
    1181              : /* Indicate that NEW_RTX is an alternate way of referring to the temp
    1182              :    slot that previously was known by OLD_RTX.  */
    1183              : 
    1184              : void
    1185     16558311 : update_temp_slot_address (rtx old_rtx, rtx new_rtx)
    1186              : {
    1187     17176850 :   class temp_slot *p;
    1188              : 
    1189     17176850 :   if (rtx_equal_p (old_rtx, new_rtx))
    1190              :     return;
    1191              : 
    1192      4530604 :   p = find_temp_slot_from_address (old_rtx);
    1193              : 
    1194              :   /* If we didn't find one, see if both OLD_RTX is a PLUS.  If so, and
    1195              :      NEW_RTX is a register, see if one operand of the PLUS is a
    1196              :      temporary location.  If so, NEW_RTX points into it.  Otherwise,
    1197              :      if both OLD_RTX and NEW_RTX are a PLUS and if there is a register
    1198              :      in common between them.  If so, try a recursive call on those
    1199              :      values.  */
    1200      4530604 :   if (p == 0)
    1201              :     {
    1202      4530566 :       if (GET_CODE (old_rtx) != PLUS)
    1203              :         return;
    1204              : 
    1205       699879 :       if (REG_P (new_rtx))
    1206              :         {
    1207       228074 :           update_temp_slot_address (XEXP (old_rtx, 0), new_rtx);
    1208       228074 :           update_temp_slot_address (XEXP (old_rtx, 1), new_rtx);
    1209       228074 :           return;
    1210              :         }
    1211       471805 :       else if (GET_CODE (new_rtx) != PLUS)
    1212              :         return;
    1213              : 
    1214       471805 :       if (rtx_equal_p (XEXP (old_rtx, 0), XEXP (new_rtx, 0)))
    1215       217146 :         update_temp_slot_address (XEXP (old_rtx, 1), XEXP (new_rtx, 1));
    1216       254659 :       else if (rtx_equal_p (XEXP (old_rtx, 1), XEXP (new_rtx, 0)))
    1217            0 :         update_temp_slot_address (XEXP (old_rtx, 0), XEXP (new_rtx, 1));
    1218       254659 :       else if (rtx_equal_p (XEXP (old_rtx, 0), XEXP (new_rtx, 1)))
    1219        33099 :         update_temp_slot_address (XEXP (old_rtx, 1), XEXP (new_rtx, 0));
    1220       221560 :       else if (rtx_equal_p (XEXP (old_rtx, 1), XEXP (new_rtx, 1)))
    1221       140220 :         update_temp_slot_address (XEXP (old_rtx, 0), XEXP (new_rtx, 0));
    1222              : 
    1223              :       return;
    1224              :     }
    1225              : 
    1226              :   /* Otherwise add an alias for the temp's address.  */
    1227           38 :   insert_temp_slot_address (new_rtx, p);
    1228              : }
    1229              : 
    1230              : /* If X could be a reference to a temporary slot, mark that slot as
    1231              :    belonging to the to one level higher than the current level.  If X
    1232              :    matched one of our slots, just mark that one.  Otherwise, we can't
    1233              :    easily predict which it is, so upgrade all of them.
    1234              : 
    1235              :    This is called when an ({...}) construct occurs and a statement
    1236              :    returns a value in memory.  */
    1237              : 
    1238              : void
    1239     24770238 : preserve_temp_slots (rtx x)
    1240              : {
    1241     24770238 :   class temp_slot *p = 0, *next;
    1242              : 
    1243     24770238 :   if (x == 0)
    1244              :     return;
    1245              : 
    1246              :   /* If X is a register that is being used as a pointer, see if we have
    1247              :      a temporary slot we know it points to.  */
    1248     11137685 :   if (REG_P (x) && REG_POINTER (x))
    1249      1808322 :     p = find_temp_slot_from_address (x);
    1250              : 
    1251              :   /* If X is not in memory or is at a constant address, it cannot be in
    1252              :      a temporary slot.  */
    1253     11137685 :   if (p == 0 && (!MEM_P (x) || CONSTANT_P (XEXP (x, 0))))
    1254              :     return;
    1255              : 
    1256              :   /* First see if we can find a match.  */
    1257       701797 :   if (p == 0)
    1258       701797 :     p = find_temp_slot_from_address (XEXP (x, 0));
    1259              : 
    1260       701797 :   if (p != 0)
    1261              :     {
    1262         1222 :       if (p->level == temp_slot_level)
    1263         1222 :         move_slot_to_level (p, temp_slot_level - 1);
    1264              :       return;
    1265              :     }
    1266              : 
    1267              :   /* Otherwise, preserve all non-kept slots at this level.  */
    1268       700581 :   for (p = *temp_slots_at_level (temp_slot_level); p; p = next)
    1269              :     {
    1270            6 :       next = p->next;
    1271            6 :       move_slot_to_level (p, temp_slot_level - 1);
    1272              :     }
    1273              : }
    1274              : 
    1275              : /* Free all temporaries used so far.  This is normally called at the
    1276              :    end of generating code for a statement.  */
    1277              : 
    1278              : void
    1279     64000197 : free_temp_slots (void)
    1280              : {
    1281     64000197 :   class temp_slot *p, *next;
    1282     64000197 :   bool some_available = false;
    1283              : 
    1284     64142623 :   for (p = *temp_slots_at_level (temp_slot_level); p; p = next)
    1285              :     {
    1286       142426 :       next = p->next;
    1287       142426 :       make_slot_available (p);
    1288       142426 :       some_available = true;
    1289              :     }
    1290              : 
    1291     64000197 :   if (some_available)
    1292              :     {
    1293       134950 :       remove_unused_temp_slot_addresses ();
    1294       134950 :       combine_temp_slots ();
    1295              :     }
    1296     64000197 : }
    1297              : 
    1298              : /* Push deeper into the nesting level for stack temporaries.  */
    1299              : 
    1300              : void
    1301     31309090 : push_temp_slots (void)
    1302              : {
    1303     31309090 :   temp_slot_level++;
    1304     31309090 : }
    1305              : 
    1306              : /* Pop a temporary nesting level.  All slots in use in the current level
    1307              :    are freed.  */
    1308              : 
    1309              : void
    1310     31309086 : pop_temp_slots (void)
    1311              : {
    1312     31309086 :   free_temp_slots ();
    1313     31309086 :   temp_slot_level--;
    1314     31309086 : }
    1315              : 
    1316              : /* Initialize temporary slots.  */
    1317              : 
    1318              : void
    1319      3263784 : init_temp_slots (void)
    1320              : {
    1321              :   /* We have not allocated any temporaries yet.  */
    1322      3263784 :   avail_temp_slots = 0;
    1323      3263784 :   vec_alloc (used_temp_slots, 0);
    1324      3263784 :   temp_slot_level = 0;
    1325      3263784 :   n_temp_slots_in_use = 0;
    1326              : 
    1327              :   /* Set up the table to map addresses to temp slots.  */
    1328      3263784 :   if (! temp_slot_address_table)
    1329       217848 :     temp_slot_address_table = hash_table<temp_address_hasher>::create_ggc (32);
    1330              :   else
    1331      3045936 :     temp_slot_address_table->empty ();
    1332      3263784 : }
    1333              : 
    1334              : /* Functions and data structures to keep track of the values hard regs
    1335              :    had at the start of the function.  */
    1336              : 
    1337              : /* Private type used by get_hard_reg_initial_reg, get_hard_reg_initial_val,
    1338              :    and has_hard_reg_initial_val..  */
    1339              : struct GTY(()) initial_value_pair {
    1340              :   rtx hard_reg;
    1341              :   rtx pseudo;
    1342              : };
    1343              : /* ???  This could be a VEC but there is currently no way to define an
    1344              :    opaque VEC type.  This could be worked around by defining struct
    1345              :    initial_value_pair in function.h.  */
    1346              : struct GTY(()) initial_value_struct {
    1347              :   int num_entries;
    1348              :   int max_entries;
    1349              :   initial_value_pair * GTY ((length ("%h.num_entries"))) entries;
    1350              : };
    1351              : 
    1352              : /* If a pseudo represents an initial hard reg (or expression), return
    1353              :    it, else return NULL_RTX.  */
    1354              : 
    1355              : rtx
    1356            0 : get_hard_reg_initial_reg (rtx reg)
    1357              : {
    1358            0 :   struct initial_value_struct *ivs = crtl->hard_reg_initial_vals;
    1359            0 :   int i;
    1360              : 
    1361            0 :   if (ivs == 0)
    1362              :     return NULL_RTX;
    1363              : 
    1364            0 :   for (i = 0; i < ivs->num_entries; i++)
    1365            0 :     if (rtx_equal_p (ivs->entries[i].pseudo, reg))
    1366            0 :       return ivs->entries[i].hard_reg;
    1367              : 
    1368              :   return NULL_RTX;
    1369              : }
    1370              : 
    1371              : /* Make sure that there's a pseudo register of mode MODE that stores the
    1372              :    initial value of hard register REGNO.  Return an rtx for such a pseudo.  */
    1373              : 
    1374              : rtx
    1375            0 : get_hard_reg_initial_val (machine_mode mode, unsigned int regno)
    1376              : {
    1377            0 :   struct initial_value_struct *ivs;
    1378            0 :   rtx rv;
    1379              : 
    1380            0 :   rv = has_hard_reg_initial_val (mode, regno);
    1381            0 :   if (rv)
    1382              :     return rv;
    1383              : 
    1384            0 :   ivs = crtl->hard_reg_initial_vals;
    1385            0 :   if (ivs == 0)
    1386              :     {
    1387            0 :       ivs = ggc_alloc<initial_value_struct> ();
    1388            0 :       ivs->num_entries = 0;
    1389            0 :       ivs->max_entries = 5;
    1390            0 :       ivs->entries = ggc_vec_alloc<initial_value_pair> (5);
    1391            0 :       crtl->hard_reg_initial_vals = ivs;
    1392              :     }
    1393              : 
    1394            0 :   if (ivs->num_entries >= ivs->max_entries)
    1395              :     {
    1396            0 :       ivs->max_entries += 5;
    1397            0 :       ivs->entries = GGC_RESIZEVEC (initial_value_pair, ivs->entries,
    1398              :                                     ivs->max_entries);
    1399              :     }
    1400              : 
    1401            0 :   ivs->entries[ivs->num_entries].hard_reg = gen_rtx_REG (mode, regno);
    1402            0 :   ivs->entries[ivs->num_entries].pseudo = gen_reg_rtx (mode);
    1403              : 
    1404            0 :   return ivs->entries[ivs->num_entries++].pseudo;
    1405              : }
    1406              : 
    1407              : /* See if get_hard_reg_initial_val has been used to create a pseudo
    1408              :    for the initial value of hard register REGNO in mode MODE.  Return
    1409              :    the associated pseudo if so, otherwise return NULL.  */
    1410              : 
    1411              : rtx
    1412            0 : has_hard_reg_initial_val (machine_mode mode, unsigned int regno)
    1413              : {
    1414            0 :   struct initial_value_struct *ivs;
    1415            0 :   int i;
    1416              : 
    1417            0 :   ivs = crtl->hard_reg_initial_vals;
    1418            0 :   if (ivs != 0)
    1419            0 :     for (i = 0; i < ivs->num_entries; i++)
    1420            0 :       if (GET_MODE (ivs->entries[i].hard_reg) == mode
    1421            0 :           && REGNO (ivs->entries[i].hard_reg) == regno)
    1422            0 :         return ivs->entries[i].pseudo;
    1423              : 
    1424              :   return NULL_RTX;
    1425              : }
    1426              : 
    1427              : void
    1428      1517906 : emit_initial_value_sets (void)
    1429              : {
    1430      1517906 :   struct initial_value_struct *ivs = crtl->hard_reg_initial_vals;
    1431      1517906 :   int i;
    1432      1517906 :   rtx_insn *seq;
    1433              : 
    1434      1517906 :   if (ivs == 0)
    1435              :     return;
    1436              : 
    1437            0 :   start_sequence ();
    1438            0 :   for (i = 0; i < ivs->num_entries; i++)
    1439            0 :     emit_move_insn (ivs->entries[i].pseudo, ivs->entries[i].hard_reg);
    1440            0 :   seq = end_sequence ();
    1441              : 
    1442            0 :   emit_insn_at_entry (seq);
    1443              : }
    1444              : 
    1445              : /* Return the hardreg-pseudoreg initial values pair entry I and
    1446              :    TRUE if I is a valid entry, or FALSE if I is not a valid entry.  */
    1447              : bool
    1448            0 : initial_value_entry (int i, rtx *hreg, rtx *preg)
    1449              : {
    1450            0 :   struct initial_value_struct *ivs = crtl->hard_reg_initial_vals;
    1451            0 :   if (!ivs || i >= ivs->num_entries)
    1452              :     return false;
    1453              : 
    1454            0 :   *hreg = ivs->entries[i].hard_reg;
    1455            0 :   *preg = ivs->entries[i].pseudo;
    1456            0 :   return true;
    1457              : }
    1458              : 
    1459              : /* These routines are responsible for converting virtual register references
    1460              :    to the actual hard register references once RTL generation is complete.
    1461              : 
    1462              :    The following four variables are used for communication between the
    1463              :    routines.  They contain the offsets of the virtual registers from their
    1464              :    respective hard registers.  */
    1465              : 
    1466              : static poly_int64 in_arg_offset;
    1467              : static poly_int64 var_offset;
    1468              : static poly_int64 dynamic_offset;
    1469              : static poly_int64 out_arg_offset;
    1470              : static poly_int64 cfa_offset;
    1471              : 
    1472              : /* In most machines, the stack pointer register is equivalent to the bottom
    1473              :    of the stack.  */
    1474              : 
    1475              : #ifndef STACK_POINTER_OFFSET
    1476              : #define STACK_POINTER_OFFSET    0
    1477              : #endif
    1478              : 
    1479              : #if defined (REG_PARM_STACK_SPACE) && !defined (INCOMING_REG_PARM_STACK_SPACE)
    1480              : #define INCOMING_REG_PARM_STACK_SPACE REG_PARM_STACK_SPACE
    1481              : #endif
    1482              : 
    1483              : /* If not defined, pick an appropriate default for the offset of dynamically
    1484              :    allocated memory depending on the value of ACCUMULATE_OUTGOING_ARGS,
    1485              :    INCOMING_REG_PARM_STACK_SPACE, and OUTGOING_REG_PARM_STACK_SPACE.  */
    1486              : 
    1487              : #ifndef STACK_DYNAMIC_OFFSET
    1488              : 
    1489              : /* The bottom of the stack points to the actual arguments.  If
    1490              :    REG_PARM_STACK_SPACE is defined, this includes the space for the register
    1491              :    parameters.  However, if OUTGOING_REG_PARM_STACK space is not defined,
    1492              :    stack space for register parameters is not pushed by the caller, but
    1493              :    rather part of the fixed stack areas and hence not included in
    1494              :    `crtl->outgoing_args_size'.  Nevertheless, we must allow
    1495              :    for it when allocating stack dynamic objects.  */
    1496              : 
    1497              : #ifdef INCOMING_REG_PARM_STACK_SPACE
    1498              : #define STACK_DYNAMIC_OFFSET(FNDECL)    \
    1499              : ((ACCUMULATE_OUTGOING_ARGS                                                    \
    1500              :   ? (crtl->outgoing_args_size                                      \
    1501              :      + (OUTGOING_REG_PARM_STACK_SPACE ((!(FNDECL) ? NULL_TREE : TREE_TYPE (FNDECL))) ? 0 \
    1502              :                                                : INCOMING_REG_PARM_STACK_SPACE (FNDECL))) \
    1503              :   : 0) + (STACK_POINTER_OFFSET))
    1504              : #else
    1505              : #define STACK_DYNAMIC_OFFSET(FNDECL)    \
    1506              :   ((ACCUMULATE_OUTGOING_ARGS ? crtl->outgoing_args_size : poly_int64 (0)) \
    1507              :  + (STACK_POINTER_OFFSET))
    1508              : #endif
    1509              : #endif
    1510              : 
    1511              : 
    1512              : /* Given a piece of RTX and a pointer to a HOST_WIDE_INT, if the RTX
    1513              :    is a virtual register, return the equivalent hard register and set the
    1514              :    offset indirectly through the pointer.  Otherwise, return 0.  */
    1515              : 
    1516              : static rtx
    1517    375063560 : instantiate_new_reg (rtx x, poly_int64 *poffset)
    1518              : {
    1519    375063560 :   rtx new_rtx;
    1520    375063560 :   poly_int64 offset;
    1521              : 
    1522    375063560 :   if (x == virtual_incoming_args_rtx)
    1523              :     {
    1524      3765536 :       if (stack_realign_drap)
    1525              :         {
    1526              :           /* Replace virtual_incoming_args_rtx with internal arg
    1527              :              pointer if DRAP is used to realign stack.  */
    1528        16472 :           new_rtx = crtl->args.internal_arg_pointer;
    1529        16472 :           offset = 0;
    1530              :         }
    1531              :       else
    1532      3749064 :         new_rtx = arg_pointer_rtx, offset = in_arg_offset;
    1533              :     }
    1534    371298024 :   else if (x == virtual_stack_vars_rtx)
    1535     18997913 :     new_rtx = frame_pointer_rtx, offset = var_offset;
    1536    352300111 :   else if (x == virtual_stack_dynamic_rtx)
    1537        52995 :     new_rtx = stack_pointer_rtx, offset = dynamic_offset;
    1538    352247116 :   else if (x == virtual_outgoing_args_rtx)
    1539      1572314 :     new_rtx = stack_pointer_rtx, offset = out_arg_offset;
    1540    350674802 :   else if (x == virtual_cfa_rtx)
    1541              :     {
    1542              : #ifdef FRAME_POINTER_CFA_OFFSET
    1543              :       new_rtx = frame_pointer_rtx;
    1544              : #else
    1545         1842 :       new_rtx = arg_pointer_rtx;
    1546              : #endif
    1547         1842 :       offset = cfa_offset;
    1548              :     }
    1549    350672960 :   else if (x == virtual_preferred_stack_boundary_rtx)
    1550              :     {
    1551       112484 :       new_rtx = GEN_INT (crtl->preferred_stack_boundary / BITS_PER_UNIT);
    1552       112484 :       offset = 0;
    1553              :     }
    1554              :   else
    1555              :     return NULL_RTX;
    1556              : 
    1557     24503084 :   *poffset = offset;
    1558     24503084 :   return new_rtx;
    1559              : }
    1560              : 
    1561              : /* A subroutine of instantiate_virtual_regs.  Instantiate any virtual
    1562              :    registers present inside of *LOC.  The expression is simplified,
    1563              :    as much as possible, but is not to be considered "valid" in any sense
    1564              :    implied by the target.  Return true if any change is made.  */
    1565              : 
    1566              : static bool
    1567    216215160 : instantiate_virtual_regs_in_rtx (rtx *loc)
    1568              : {
    1569    216215160 :   if (!*loc)
    1570              :     return false;
    1571     97600388 :   bool changed = false;
    1572     97600388 :   subrtx_ptr_iterator::array_type array;
    1573    362340036 :   FOR_EACH_SUBRTX_PTR (iter, array, loc, NONCONST)
    1574              :     {
    1575    264739648 :       rtx *loc = *iter;
    1576    264739648 :       if (rtx x = *loc)
    1577              :         {
    1578    240476017 :           rtx new_rtx;
    1579    240476017 :           poly_int64 offset;
    1580    240476017 :           switch (GET_CODE (x))
    1581              :             {
    1582     35886929 :             case REG:
    1583     35886929 :               new_rtx = instantiate_new_reg (x, &offset);
    1584     35886929 :               if (new_rtx)
    1585              :                 {
    1586      1670833 :                   *loc = plus_constant (GET_MODE (x), new_rtx, offset);
    1587      1670833 :                   changed = true;
    1588              :                 }
    1589     35886929 :               iter.skip_subrtxes ();
    1590     35886929 :               break;
    1591              : 
    1592     28781964 :             case PLUS:
    1593     28781964 :               new_rtx = instantiate_new_reg (XEXP (x, 0), &offset);
    1594     28781964 :               if (new_rtx)
    1595              :                 {
    1596     19627866 :                   XEXP (x, 0) = new_rtx;
    1597     19627866 :                   *loc = plus_constant (GET_MODE (x), x, offset, true);
    1598     19627866 :                   changed = true;
    1599     19627866 :                   iter.skip_subrtxes ();
    1600     19627866 :                   break;
    1601              :                 }
    1602              : 
    1603              :               /* FIXME -- from old code */
    1604              :               /* If we have (plus (subreg (virtual-reg)) (const_int)), we know
    1605              :                  we can commute the PLUS and SUBREG because pointers into the
    1606              :                  frame are well-behaved.  */
    1607              :               break;
    1608              : 
    1609              :             default:
    1610              :               break;
    1611              :             }
    1612              :         }
    1613              :     }
    1614     97600388 :   return changed;
    1615     97600388 : }
    1616              : 
    1617              : /* A subroutine of instantiate_virtual_regs_in_insn.  Return true if X
    1618              :    matches the predicate for insn CODE operand OPERAND.  */
    1619              : 
    1620              : static bool
    1621     30673270 : safe_insn_predicate (int code, int operand, rtx x)
    1622              : {
    1623     30673270 :   return code < 0 || insn_operand_matches ((enum insn_code) code, operand, x);
    1624              : }
    1625              : 
    1626              : /* A subroutine of instantiate_virtual_regs.  Instantiate any virtual
    1627              :    registers present inside of insn.  The result will be a valid insn.  */
    1628              : 
    1629              : static void
    1630     95871193 : instantiate_virtual_regs_in_insn (rtx_insn *insn)
    1631              : {
    1632     95871193 :   poly_int64 offset;
    1633     95871193 :   int insn_code, i;
    1634     95871193 :   bool any_change = false;
    1635     95871193 :   rtx set, new_rtx, x;
    1636     95871193 :   rtx_insn *seq;
    1637              : 
    1638              :   /* There are some special cases to be handled first.  */
    1639     95871193 :   set = single_set (insn);
    1640     95871193 :   if (set)
    1641              :     {
    1642              :       /* We're allowed to assign to a virtual register.  This is interpreted
    1643              :          to mean that the underlying register gets assigned the inverse
    1644              :          transformation.  This is used, for example, in the handling of
    1645              :          non-local gotos.  */
    1646     91535455 :       new_rtx = instantiate_new_reg (SET_DEST (set), &offset);
    1647     91535455 :       if (new_rtx)
    1648              :         {
    1649            0 :           start_sequence ();
    1650              : 
    1651            0 :           instantiate_virtual_regs_in_rtx (&SET_SRC (set));
    1652            0 :           x = simplify_gen_binary (PLUS, GET_MODE (new_rtx), SET_SRC (set),
    1653            0 :                                    gen_int_mode (-offset, GET_MODE (new_rtx)));
    1654            0 :           x = force_operand (x, new_rtx);
    1655            0 :           if (x != new_rtx)
    1656            0 :             emit_move_insn (new_rtx, x);
    1657              : 
    1658            0 :           seq = end_sequence ();
    1659              : 
    1660            0 :           emit_insn_before (seq, insn);
    1661            0 :           delete_insn (insn);
    1662        18766 :           return;
    1663              :         }
    1664              : 
    1665              :       /* Handle a straight copy from a virtual register by generating a
    1666              :          new add insn.  The difference between this and falling through
    1667              :          to the generic case is avoiding a new pseudo and eliminating a
    1668              :          move insn in the initial rtl stream.  */
    1669     91535455 :       new_rtx = instantiate_new_reg (SET_SRC (set), &offset);
    1670     91535455 :       if (new_rtx
    1671       322833 :           && maybe_ne (offset, 0)
    1672         3841 :           && REG_P (SET_DEST (set))
    1673     91539296 :           && REGNO (SET_DEST (set)) > LAST_VIRTUAL_REGISTER)
    1674              :         {
    1675         3841 :           start_sequence ();
    1676              : 
    1677         3841 :           x = expand_simple_binop (GET_MODE (SET_DEST (set)), PLUS, new_rtx,
    1678              :                                    gen_int_mode (offset,
    1679         3841 :                                                  GET_MODE (SET_DEST (set))),
    1680              :                                    SET_DEST (set), 1, OPTAB_LIB_WIDEN);
    1681         3841 :           if (x != SET_DEST (set))
    1682            0 :             emit_move_insn (SET_DEST (set), x);
    1683              : 
    1684         3841 :           seq = end_sequence ();
    1685              : 
    1686         3841 :           emit_insn_before (seq, insn);
    1687         3841 :           delete_insn (insn);
    1688         3841 :           return;
    1689              :         }
    1690              : 
    1691     91531614 :       extract_insn (insn);
    1692     91531614 :       insn_code = INSN_CODE (insn);
    1693              : 
    1694              :       /* Handle a plus involving a virtual register by determining if the
    1695              :          operands remain valid if they're modified in place.  */
    1696     91531614 :       poly_int64 delta;
    1697     91531614 :       if (GET_CODE (SET_SRC (set)) == PLUS
    1698     10331445 :           && recog_data.n_operands >= 3
    1699     10265921 :           && recog_data.operand_loc[1] == &XEXP (SET_SRC (set), 0)
    1700     10265293 :           && recog_data.operand_loc[2] == &XEXP (SET_SRC (set), 1)
    1701     10265293 :           && poly_int_rtx_p (recog_data.operand[2], &delta)
    1702     99616255 :           && (new_rtx = instantiate_new_reg (recog_data.operand[1], &offset)))
    1703              :         {
    1704      2465027 :           offset += delta;
    1705              : 
    1706              :           /* If the sum is zero, then replace with a plain move.  */
    1707      2465027 :           if (known_eq (offset, 0)
    1708        14925 :               && REG_P (SET_DEST (set))
    1709      2479952 :               && REGNO (SET_DEST (set)) > LAST_VIRTUAL_REGISTER)
    1710              :             {
    1711        14925 :               start_sequence ();
    1712        14925 :               emit_move_insn (SET_DEST (set), new_rtx);
    1713        14925 :               seq = end_sequence ();
    1714              : 
    1715        14925 :               emit_insn_before (seq, insn);
    1716        14925 :               delete_insn (insn);
    1717        14925 :               return;
    1718              :             }
    1719              : 
    1720      2450102 :           x = gen_int_mode (offset, recog_data.operand_mode[2]);
    1721              : 
    1722              :           /* Using validate_change and apply_change_group here leaves
    1723              :              recog_data in an invalid state.  Since we know exactly what
    1724              :              we want to check, do those two by hand.  */
    1725      2450102 :           if (safe_insn_predicate (insn_code, 1, new_rtx)
    1726      2450102 :               && safe_insn_predicate (insn_code, 2, x))
    1727              :             {
    1728      2421981 :               *recog_data.operand_loc[1] = recog_data.operand[1] = new_rtx;
    1729      2421981 :               *recog_data.operand_loc[2] = recog_data.operand[2] = x;
    1730      2421981 :               any_change = true;
    1731              : 
    1732              :               /* Fall through into the regular operand fixup loop in
    1733              :                  order to take care of operands other than 1 and 2.  */
    1734              :             }
    1735              :         }
    1736              :     }
    1737              :   else
    1738              :     {
    1739      4335738 :       extract_insn (insn);
    1740      4335738 :       insn_code = INSN_CODE (insn);
    1741              :     }
    1742              : 
    1743              :   /* In the general case, we expect virtual registers to appear only in
    1744              :      operands, and then only as either bare registers or inside memories.  */
    1745    303648185 :   for (i = 0; i < recog_data.n_operands; ++i)
    1746              :     {
    1747    207795758 :       x = recog_data.operand[i];
    1748    207795758 :       switch (GET_CODE (x))
    1749              :         {
    1750     29784421 :         case MEM:
    1751     29784421 :           {
    1752     29784421 :             rtx addr = XEXP (x, 0);
    1753              : 
    1754     29784421 :             if (!instantiate_virtual_regs_in_rtx (&addr))
    1755     17092090 :               continue;
    1756              : 
    1757     12692331 :             start_sequence ();
    1758     12692331 :             x = replace_equiv_address (x, addr, true);
    1759              :             /* It may happen that the address with the virtual reg
    1760              :                was valid (e.g. based on the virtual stack reg, which might
    1761              :                be acceptable to the predicates with all offsets), whereas
    1762              :                the address now isn't anymore, for instance when the address
    1763              :                is still offsetted, but the base reg isn't virtual-stack-reg
    1764              :                anymore.  Below we would do a force_reg on the whole operand,
    1765              :                but this insn might actually only accept memory.  Hence,
    1766              :                before doing that last resort, try to reload the address into
    1767              :                a register, so this operand stays a MEM.  */
    1768     12692331 :             if (!safe_insn_predicate (insn_code, i, x))
    1769              :               {
    1770            0 :                 addr = force_reg (GET_MODE (addr), addr);
    1771            0 :                 x = replace_equiv_address (x, addr, true);
    1772              :               }
    1773     12692331 :             seq = end_sequence ();
    1774     12692331 :             if (seq)
    1775            0 :               emit_insn_before (seq, insn);
    1776              :           }
    1777     12692331 :           break;
    1778              : 
    1779    116237647 :         case REG:
    1780    116237647 :           new_rtx = instantiate_new_reg (x, &offset);
    1781    116237647 :           if (new_rtx == NULL)
    1782    115821128 :             continue;
    1783       416519 :           if (known_eq (offset, 0))
    1784              :             x = new_rtx;
    1785              :           else
    1786              :             {
    1787            0 :               start_sequence ();
    1788              : 
    1789              :               /* Careful, special mode predicates may have stuff in
    1790              :                  insn_data[insn_code].operand[i].mode that isn't useful
    1791              :                  to us for computing a new value.  */
    1792              :               /* ??? Recognize address_operand and/or "p" constraints
    1793              :                  to see if (plus new offset) is a valid before we put
    1794              :                  this through expand_simple_binop.  */
    1795            0 :               x = expand_simple_binop (GET_MODE (x), PLUS, new_rtx,
    1796            0 :                                        gen_int_mode (offset, GET_MODE (x)),
    1797              :                                        NULL_RTX, 1, OPTAB_LIB_WIDEN);
    1798            0 :               seq = end_sequence ();
    1799            0 :               emit_insn_before (seq, insn);
    1800              :             }
    1801              :           break;
    1802              : 
    1803      3001469 :         case SUBREG:
    1804      3001469 :           new_rtx = instantiate_new_reg (SUBREG_REG (x), &offset);
    1805      3001469 :           if (new_rtx == NULL)
    1806      3001463 :             continue;
    1807            6 :           start_sequence ();
    1808            6 :           if (maybe_ne (offset, 0))
    1809            0 :             new_rtx = expand_simple_binop
    1810            0 :               (GET_MODE (new_rtx), PLUS, new_rtx,
    1811            0 :                gen_int_mode (offset, GET_MODE (new_rtx)),
    1812              :                NULL_RTX, 1, OPTAB_LIB_WIDEN);
    1813           12 :           x = force_subreg (recog_data.operand_mode[i], new_rtx,
    1814            6 :                             GET_MODE (new_rtx), SUBREG_BYTE (x));
    1815            6 :           gcc_assert (x);
    1816            6 :           seq = end_sequence ();
    1817            6 :           emit_insn_before (seq, insn);
    1818            6 :           break;
    1819              : 
    1820     58772221 :         default:
    1821     58772221 :           continue;
    1822     58772221 :         }
    1823              : 
    1824              :       /* At this point, X contains the new value for the operand.
    1825              :          Validate the new value vs the insn predicate.  Note that
    1826              :          asm insns will have insn_code -1 here.  */
    1827     13108856 :       if (!safe_insn_predicate (insn_code, i, x))
    1828              :         {
    1829        56242 :           start_sequence ();
    1830        56242 :           if (REG_P (x))
    1831              :             {
    1832            0 :               gcc_assert (REGNO (x) <= LAST_VIRTUAL_REGISTER);
    1833            0 :               x = copy_to_reg (x);
    1834              :             }
    1835              :           else
    1836        56242 :             x = force_reg (insn_data[insn_code].operand[i].mode, x);
    1837        56242 :           seq = end_sequence ();
    1838        56242 :           if (seq)
    1839        56242 :             emit_insn_before (seq, insn);
    1840              :         }
    1841              : 
    1842     13108856 :       *recog_data.operand_loc[i] = recog_data.operand[i] = x;
    1843     13108856 :       any_change = true;
    1844              :     }
    1845              : 
    1846     95852427 :   if (any_change)
    1847              :     {
    1848              :       /* Propagate operand changes into the duplicates.  */
    1849     15441159 :       for (i = 0; i < recog_data.n_dups; ++i)
    1850        88527 :         *recog_data.dup_loc[i]
    1851        88527 :           = copy_rtx (recog_data.operand[(unsigned)recog_data.dup_num[i]]);
    1852              : 
    1853              :       /* Force re-recognition of the instruction for validation.  */
    1854     15352632 :       INSN_CODE (insn) = -1;
    1855              :     }
    1856              : 
    1857     95852427 :   if (asm_noperands (PATTERN (insn)) >= 0)
    1858              :     {
    1859        94601 :       if (!check_asm_operands (PATTERN (insn)))
    1860              :         {
    1861           23 :           error_for_asm (insn, "impossible constraint in %<asm%>");
    1862              :           /* For asm goto, instead of fixing up all the edges
    1863              :              just clear the template and clear input and output operands
    1864              :              and strip away clobbers.  */
    1865           23 :           if (JUMP_P (insn))
    1866              :             {
    1867           14 :               rtx asm_op = extract_asm_operands (PATTERN (insn));
    1868           14 :               PATTERN (insn) = asm_op;
    1869           14 :               PUT_MODE (asm_op, VOIDmode);
    1870           14 :               ASM_OPERANDS_TEMPLATE (asm_op) = ggc_strdup ("");
    1871           14 :               ASM_OPERANDS_OUTPUT_CONSTRAINT (asm_op) = "";
    1872           14 :               ASM_OPERANDS_OUTPUT_IDX (asm_op) = 0;
    1873           14 :               ASM_OPERANDS_INPUT_VEC (asm_op) = rtvec_alloc (0);
    1874           14 :               ASM_OPERANDS_INPUT_CONSTRAINT_VEC (asm_op) = rtvec_alloc (0);
    1875              :             }
    1876              :           else
    1877            9 :             delete_insn (insn);
    1878              :         }
    1879              :     }
    1880              :   else
    1881              :     {
    1882     95757826 :       if (recog_memoized (insn) < 0)
    1883            0 :         fatal_insn_not_found (insn);
    1884              :     }
    1885              : }
    1886              : 
    1887              : /* Subroutine of instantiate_decls.  Given RTL representing a decl,
    1888              :    do any instantiation required.  */
    1889              : 
    1890              : void
    1891      9812001 : instantiate_decl_rtl (rtx x)
    1892              : {
    1893      9817609 :   rtx addr;
    1894              : 
    1895      9817609 :   if (x == 0)
    1896              :     return;
    1897              : 
    1898              :   /* If this is a CONCAT, recurse for the pieces.  */
    1899      9817609 :   if (GET_CODE (x) == CONCAT)
    1900              :     {
    1901         5608 :       instantiate_decl_rtl (XEXP (x, 0));
    1902         5608 :       instantiate_decl_rtl (XEXP (x, 1));
    1903         5608 :       return;
    1904              :     }
    1905              : 
    1906              :   /* If this is not a MEM, no need to do anything.  Similarly if the
    1907              :      address is a constant or a register that is not a virtual register.  */
    1908      9812001 :   if (!MEM_P (x))
    1909              :     return;
    1910              : 
    1911      3291877 :   addr = XEXP (x, 0);
    1912      3291877 :   if (CONSTANT_P (addr)
    1913      3291877 :       || (REG_P (addr)
    1914       306051 :           && !VIRTUAL_REGISTER_P (addr)))
    1915              :     return;
    1916              : 
    1917      3067604 :   instantiate_virtual_regs_in_rtx (&XEXP (x, 0));
    1918              : }
    1919              : 
    1920              : /* Helper for instantiate_decls called via walk_tree: Process all decls
    1921              :    in the given DECL_VALUE_EXPR.  */
    1922              : 
    1923              : static tree
    1924      1585162 : instantiate_expr (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
    1925              : {
    1926      1585162 :   tree t = *tp;
    1927      1585162 :   if (! EXPR_P (t))
    1928              :     {
    1929       851735 :       *walk_subtrees = 0;
    1930       851735 :       if (DECL_P (t))
    1931              :         {
    1932       712470 :           if (DECL_RTL_SET_P (t))
    1933       175571 :             instantiate_decl_rtl (DECL_RTL (t));
    1934       138688 :           if (TREE_CODE (t) == PARM_DECL && DECL_NAMELESS (t)
    1935       825248 :               && DECL_INCOMING_RTL (t))
    1936       112778 :             instantiate_decl_rtl (DECL_INCOMING_RTL (t));
    1937       411114 :           if ((VAR_P (t) || TREE_CODE (t) == RESULT_DECL)
    1938       719933 :               && DECL_HAS_VALUE_EXPR_P (t))
    1939              :             {
    1940         7736 :               tree v = DECL_VALUE_EXPR (t);
    1941         7736 :               walk_tree (&v, instantiate_expr, NULL, NULL);
    1942              :             }
    1943              :         }
    1944              :     }
    1945      1585162 :   return NULL;
    1946              : }
    1947              : 
    1948              : /* Subroutine of instantiate_decls: Process all decls in the given
    1949              :    BLOCK node and all its subblocks.  */
    1950              : 
    1951              : static void
    1952     17236783 : instantiate_decls_1 (tree let)
    1953              : {
    1954     17236783 :   tree t;
    1955              : 
    1956     37554198 :   for (t = BLOCK_VARS (let); t; t = DECL_CHAIN (t))
    1957              :     {
    1958     20317415 :       if (DECL_RTL_SET_P (t))
    1959      2343318 :         instantiate_decl_rtl (DECL_RTL (t));
    1960     20317415 :       if (VAR_P (t) && DECL_HAS_VALUE_EXPR_P (t))
    1961              :         {
    1962       388198 :           tree v = DECL_VALUE_EXPR (t);
    1963       388198 :           walk_tree (&v, instantiate_expr, NULL, NULL);
    1964              :         }
    1965              :     }
    1966              : 
    1967              :   /* Process all subblocks.  */
    1968     32956444 :   for (t = BLOCK_SUBBLOCKS (let); t; t = BLOCK_CHAIN (t))
    1969     15719661 :     instantiate_decls_1 (t);
    1970     17236783 : }
    1971              : 
    1972              : /* Scan all decls in FNDECL (both variables and parameters) and instantiate
    1973              :    all virtual registers in their DECL_RTL's.  */
    1974              : 
    1975              : static void
    1976      1517122 : instantiate_decls (tree fndecl)
    1977              : {
    1978      1517122 :   tree decl;
    1979      1517122 :   unsigned ix;
    1980              : 
    1981              :   /* Process all parameters of the function.  */
    1982      4683753 :   for (decl = DECL_ARGUMENTS (fndecl); decl; decl = DECL_CHAIN (decl))
    1983              :     {
    1984      3166631 :       instantiate_decl_rtl (DECL_RTL (decl));
    1985      3166631 :       instantiate_decl_rtl (DECL_INCOMING_RTL (decl));
    1986      3166631 :       if (DECL_HAS_VALUE_EXPR_P (decl))
    1987              :         {
    1988          132 :           tree v = DECL_VALUE_EXPR (decl);
    1989          132 :           walk_tree (&v, instantiate_expr, NULL, NULL);
    1990              :         }
    1991              :     }
    1992              : 
    1993      1517122 :   if ((decl = DECL_RESULT (fndecl))
    1994      1517122 :       && TREE_CODE (decl) == RESULT_DECL)
    1995              :     {
    1996      1517122 :       if (DECL_RTL_SET_P (decl))
    1997       808923 :         instantiate_decl_rtl (DECL_RTL (decl));
    1998      1517122 :       if (DECL_HAS_VALUE_EXPR_P (decl))
    1999              :         {
    2000        70108 :           tree v = DECL_VALUE_EXPR (decl);
    2001        70108 :           walk_tree (&v, instantiate_expr, NULL, NULL);
    2002              :         }
    2003              :     }
    2004              : 
    2005              :   /* Process the saved static chain if it exists.  */
    2006      1517122 :   decl = DECL_STRUCT_FUNCTION (fndecl)->static_chain_decl;
    2007      1517122 :   if (decl && DECL_HAS_VALUE_EXPR_P (decl))
    2008         4114 :     instantiate_decl_rtl (DECL_RTL (DECL_VALUE_EXPR (decl)));
    2009              : 
    2010              :   /* Now process all variables defined in the function or its subblocks.  */
    2011      1517122 :   if (DECL_INITIAL (fndecl))
    2012      1517122 :     instantiate_decls_1 (DECL_INITIAL (fndecl));
    2013              : 
    2014      2860334 :   FOR_EACH_LOCAL_DECL (cfun, ix, decl)
    2015        71596 :     if (DECL_RTL_SET_P (decl))
    2016        28427 :       instantiate_decl_rtl (DECL_RTL (decl));
    2017      1517122 :   vec_free (cfun->local_decls);
    2018      1517122 : }
    2019              : 
    2020              : /* Return the value of STACK_DYNAMIC_OFFSET for the current function.
    2021              :    This is done through a function wrapper so that the macro sees a
    2022              :    predictable set of included files.  */
    2023              : 
    2024              : poly_int64
    2025      1517122 : get_stack_dynamic_offset ()
    2026              : {
    2027      1517122 :   return STACK_DYNAMIC_OFFSET (current_function_decl);
    2028              : }
    2029              : 
    2030              : /* Pass through the INSNS of function FNDECL and convert virtual register
    2031              :    references to hard register references.  */
    2032              : 
    2033              : static void
    2034      1517122 : instantiate_virtual_regs (void)
    2035              : {
    2036      1517122 :   rtx_insn *insn;
    2037              : 
    2038              :   /* Compute the offsets to use for this function.  */
    2039      1517122 :   in_arg_offset = FIRST_PARM_OFFSET (current_function_decl);
    2040      1517122 :   var_offset = targetm.starting_frame_offset ();
    2041      1517122 :   dynamic_offset = get_stack_dynamic_offset ();
    2042      1517122 :   out_arg_offset = STACK_POINTER_OFFSET;
    2043              : #ifdef FRAME_POINTER_CFA_OFFSET
    2044              :   cfa_offset = FRAME_POINTER_CFA_OFFSET (current_function_decl);
    2045              : #else
    2046      1517122 :   cfa_offset = ARG_POINTER_CFA_OFFSET (current_function_decl);
    2047              : #endif
    2048              : 
    2049              :   /* Initialize recognition, indicating that volatile is OK.  */
    2050      1517122 :   init_recog ();
    2051              : 
    2052              :   /* Scan through all the insns, instantiating every virtual register still
    2053              :      present.  */
    2054    180615442 :   for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
    2055    179098320 :     if (INSN_P (insn))
    2056              :       {
    2057              :         /* These patterns in the instruction stream can never be recognized.
    2058              :            Fortunately, they shouldn't contain virtual registers either.  */
    2059    162010674 :         if (GET_CODE (PATTERN (insn)) == USE
    2060    148494749 :             || GET_CODE (PATTERN (insn)) == CLOBBER
    2061    148312558 :             || GET_CODE (PATTERN (insn)) == ASM_INPUT
    2062    297587582 :             || DEBUG_MARKER_INSN_P (insn))
    2063     12733632 :           continue;
    2064    136543410 :         else if (DEBUG_BIND_INSN_P (insn))
    2065     40672217 :           instantiate_virtual_regs_in_rtx (INSN_VAR_LOCATION_PTR (insn));
    2066              :         else
    2067     95871193 :           instantiate_virtual_regs_in_insn (insn);
    2068              : 
    2069    136543410 :         if (insn->deleted ())
    2070        18775 :           continue;
    2071              : 
    2072    136524635 :         instantiate_virtual_regs_in_rtx (&REG_NOTES (insn));
    2073              : 
    2074              :         /* Instantiate any virtual registers in CALL_INSN_FUNCTION_USAGE.  */
    2075    185264603 :         if (auto call = dyn_cast<rtx_call_insn *> (insn))
    2076      6166283 :           instantiate_virtual_regs_in_rtx (&CALL_INSN_FUNCTION_USAGE (call));
    2077              :       }
    2078              : 
    2079              :   /* Instantiate the virtual registers in the DECLs for debugging purposes.  */
    2080      1517122 :   instantiate_decls (current_function_decl);
    2081              : 
    2082      1517122 :   targetm.instantiate_decls ();
    2083              : 
    2084              :   /* Indicate that, from now on, assign_stack_local should use
    2085              :      frame_pointer_rtx.  */
    2086      1517122 :   virtuals_instantiated = 1;
    2087      1517122 : }
    2088              : 
    2089              : namespace {
    2090              : 
    2091              : const pass_data pass_data_instantiate_virtual_regs =
    2092              : {
    2093              :   RTL_PASS, /* type */
    2094              :   "vregs", /* name */
    2095              :   OPTGROUP_NONE, /* optinfo_flags */
    2096              :   TV_NONE, /* tv_id */
    2097              :   0, /* properties_required */
    2098              :   0, /* properties_provided */
    2099              :   0, /* properties_destroyed */
    2100              :   0, /* todo_flags_start */
    2101              :   0, /* todo_flags_finish */
    2102              : };
    2103              : 
    2104              : class pass_instantiate_virtual_regs : public rtl_opt_pass
    2105              : {
    2106              : public:
    2107       297180 :   pass_instantiate_virtual_regs (gcc::context *ctxt)
    2108       594360 :     : rtl_opt_pass (pass_data_instantiate_virtual_regs, ctxt)
    2109              :   {}
    2110              : 
    2111              :   /* opt_pass methods: */
    2112      1517122 :   unsigned int execute (function *) final override
    2113              :     {
    2114      1517122 :       instantiate_virtual_regs ();
    2115      1517122 :       return 0;
    2116              :     }
    2117              : 
    2118              : }; // class pass_instantiate_virtual_regs
    2119              : 
    2120              : } // anon namespace
    2121              : 
    2122              : rtl_opt_pass *
    2123       297180 : make_pass_instantiate_virtual_regs (gcc::context *ctxt)
    2124              : {
    2125       297180 :   return new pass_instantiate_virtual_regs (ctxt);
    2126              : }
    2127              : 
    2128              : 
    2129              : /* Return true if EXP is an aggregate type (or a value with aggregate type).
    2130              :    This means a type for which function calls must pass an address to the
    2131              :    function or get an address back from the function.
    2132              :    EXP may be a type node or an expression (whose type is tested).  */
    2133              : 
    2134              : bool
    2135    153959875 : aggregate_value_p (const_tree exp, const_tree fntype)
    2136              : {
    2137    153972886 :   const_tree type = (TYPE_P (exp)) ? exp : TREE_TYPE (exp);
    2138    153972886 :   int i, regno, nregs;
    2139    153972886 :   rtx reg;
    2140              : 
    2141    153972886 :   if (fntype)
    2142    153647951 :     switch (TREE_CODE (fntype))
    2143              :       {
    2144     12243983 :       case CALL_EXPR:
    2145     12243983 :         {
    2146     12243983 :           tree fndecl = get_callee_fndecl (fntype);
    2147     12243983 :           if (fndecl)
    2148     11228017 :             fntype = TREE_TYPE (fndecl);
    2149      1015966 :           else if (CALL_EXPR_FN (fntype))
    2150       327094 :             fntype = TREE_TYPE (TREE_TYPE (CALL_EXPR_FN (fntype)));
    2151              :           else
    2152              :             /* For internal functions, assume nothing needs to be
    2153              :                returned in memory.  */
    2154              :             return false;
    2155              :         }
    2156              :         break;
    2157    131220298 :       case FUNCTION_DECL:
    2158    131220298 :         fntype = TREE_TYPE (fntype);
    2159    131220298 :         break;
    2160              :       case FUNCTION_TYPE:
    2161              :       case METHOD_TYPE:
    2162              :         break;
    2163              :       case IDENTIFIER_NODE:
    2164       338172 :         fntype = NULL_TREE;
    2165              :         break;
    2166            0 :       default:
    2167              :         /* We don't expect other tree types here.  */
    2168            0 :         gcc_unreachable ();
    2169              :       }
    2170              : 
    2171    153284014 :   if (VOID_TYPE_P (type))
    2172              :     return false;
    2173              : 
    2174    112657593 :   if (error_operand_p (fntype))
    2175              :     return false;
    2176              : 
    2177              :   /* If a record should be passed the same as its first (and only) member
    2178              :      don't pass it as an aggregate.  */
    2179    112657592 :   if (TREE_CODE (type) == RECORD_TYPE && TYPE_TRANSPARENT_AGGR (type))
    2180        13011 :     return aggregate_value_p (first_field (type), fntype);
    2181              : 
    2182              :   /* If the front end has decided that this needs to be passed by
    2183              :      reference, do so.  */
    2184    112644484 :   if ((TREE_CODE (exp) == PARM_DECL || TREE_CODE (exp) == RESULT_DECL)
    2185    205565398 :       && DECL_BY_REFERENCE (exp))
    2186              :     return true;
    2187              : 
    2188              :   /* Function types that are TREE_ADDRESSABLE force return in memory.  */
    2189    112479717 :   if (fntype && TREE_ADDRESSABLE (fntype))
    2190              :     return true;
    2191              : 
    2192              :   /* Types that are TREE_ADDRESSABLE must be constructed in memory,
    2193              :      and thus can't be returned in registers.  */
    2194    112479717 :   if (TREE_ADDRESSABLE (type))
    2195              :     return true;
    2196              : 
    2197    111099651 :   if (TYPE_EMPTY_P (type))
    2198              :     return false;
    2199              : 
    2200    110112399 :   if (flag_pcc_struct_return && AGGREGATE_TYPE_P (type))
    2201              :     return true;
    2202              : 
    2203    109360907 :   if (targetm.calls.return_in_memory (type, fntype))
    2204              :     return true;
    2205              : 
    2206              :   /* Make sure we have suitable call-clobbered regs to return
    2207              :      the value in; if not, we must return it in memory.  */
    2208    103279572 :   reg = hard_function_value (type, 0, fntype, 0);
    2209              : 
    2210              :   /* If we have something other than a REG (e.g. a PARALLEL), then assume
    2211              :      it is OK.  */
    2212    103279572 :   if (!REG_P (reg))
    2213              :     return false;
    2214              : 
    2215              :   /* Use the default ABI if the type of the function isn't known.
    2216              :      The scheme for handling interoperability between different ABIs
    2217              :      requires us to be able to tell when we're calling a function with
    2218              :      a nondefault ABI.  */
    2219    102350590 :   const predefined_function_abi &abi = (fntype
    2220    102350590 :                                         ? fntype_abi (fntype)
    2221       292141 :                                         : default_function_abi);
    2222    102350590 :   regno = REGNO (reg);
    2223    102350590 :   nregs = hard_regno_nregs (regno, TYPE_MODE (type));
    2224    207721568 :   for (i = 0; i < nregs; i++)
    2225    105370978 :     if (!fixed_regs[regno + i] && !abi.clobbers_full_reg_p (regno + i))
    2226              :       return true;
    2227              : 
    2228              :   return false;
    2229              : }
    2230              : 
    2231              : /* Return true if we should assign DECL a pseudo register; false if it
    2232              :    should live on the local stack.  */
    2233              : 
    2234              : bool
    2235    176492276 : use_register_for_decl (const_tree decl)
    2236              : {
    2237    176492276 :   if (TREE_CODE (decl) == SSA_NAME)
    2238              :     {
    2239              :       /* We often try to use the SSA_NAME, instead of its underlying
    2240              :          decl, to get type information and guide decisions, to avoid
    2241              :          differences of behavior between anonymous and named
    2242              :          variables, but in this one case we have to go for the actual
    2243              :          variable if there is one.  The main reason is that, at least
    2244              :          at -O0, we want to place user variables on the stack, but we
    2245              :          don't mind using pseudos for anonymous or ignored temps.
    2246              :          Should we take the SSA_NAME, we'd conclude all SSA_NAMEs
    2247              :          should go in pseudos, whereas their corresponding variables
    2248              :          might have to go on the stack.  So, disregarding the decl
    2249              :          here would negatively impact debug info at -O0, enable
    2250              :          coalescing between SSA_NAMEs that ought to get different
    2251              :          stack/pseudo assignments, and get the incoming argument
    2252              :          processing thoroughly confused by PARM_DECLs expected to live
    2253              :          in stack slots but assigned to pseudos.  */
    2254    157575406 :       if (!SSA_NAME_VAR (decl))
    2255    103905920 :         return TYPE_MODE (TREE_TYPE (decl)) != BLKmode
    2256    103905920 :           && !(flag_float_store && FLOAT_TYPE_P (TREE_TYPE (decl)));
    2257              : 
    2258              :       decl = SSA_NAME_VAR (decl);
    2259              :     }
    2260              : 
    2261              :   /* Honor volatile.  */
    2262     72586356 :   if (TREE_SIDE_EFFECTS (decl))
    2263              :     return false;
    2264              : 
    2265              :   /* Honor addressability.  */
    2266     72463881 :   if (TREE_ADDRESSABLE (decl))
    2267              :     return false;
    2268              : 
    2269              :   /* RESULT_DECLs are a bit special in that they're assigned without
    2270              :      regard to use_register_for_decl, but we generally only store in
    2271              :      them.  If we coalesce their SSA NAMEs, we'd better return a
    2272              :      result that matches the assignment in expand_function_start.  */
    2273     68011631 :   if (TREE_CODE (decl) == RESULT_DECL)
    2274              :     {
    2275              :       /* If it's not an aggregate, we're going to use a REG or a
    2276              :          PARALLEL containing a REG.  */
    2277      3526271 :       if (!aggregate_value_p (decl, current_function_decl))
    2278              :         return true;
    2279              : 
    2280              :       /* If expand_function_start determines the return value, we'll
    2281              :          use MEM if it's not by reference.  */
    2282        41295 :       if (cfun->returns_pcc_struct
    2283        82590 :           || (targetm.calls.struct_value_rtx
    2284        41295 :               (TREE_TYPE (current_function_decl), 1)))
    2285            0 :         return DECL_BY_REFERENCE (decl);
    2286              : 
    2287              :       /* Otherwise, we're taking an extra all.function_result_decl
    2288              :          argument.  It's set up in assign_parms_augmented_arg_list,
    2289              :          under the (negated) conditions above, and then it's used to
    2290              :          set up the RESULT_DECL rtl in assign_params, after looping
    2291              :          over all parameters.  Now, if the RESULT_DECL is not by
    2292              :          reference, we'll use a MEM either way.  */
    2293        41295 :       if (!DECL_BY_REFERENCE (decl))
    2294              :         return false;
    2295              : 
    2296              :       /* Otherwise, if RESULT_DECL is DECL_BY_REFERENCE, it will take
    2297              :          the function_result_decl's assignment.  Since it's a pointer,
    2298              :          we can short-circuit a number of the tests below, and we must
    2299              :          duplicate them because we don't have the function_result_decl
    2300              :          to test.  */
    2301        41295 :       if (!targetm.calls.allocate_stack_slots_for_args ())
    2302              :         return true;
    2303              :       /* We don't set DECL_IGNORED_P for the function_result_decl.  */
    2304        41295 :       if (optimize)
    2305              :         return true;
    2306              :       /* Needed for [[musttail]] which can operate even at -O0 */
    2307         5883 :       if (cfun->tail_call_marked)
    2308              :         return true;
    2309              :       /* We don't set DECL_REGISTER for the function_result_decl.  */
    2310              :       return false;
    2311              :     }
    2312              : 
    2313              :   /* Only register-like things go in registers.  */
    2314     64485360 :   if (DECL_MODE (decl) == BLKmode)
    2315              :     return false;
    2316              : 
    2317              :   /* If -ffloat-store specified, don't put explicit float variables
    2318              :      into registers.  */
    2319              :   /* ??? This should be checked after DECL_ARTIFICIAL, but tree-ssa
    2320              :      propagates values across these stores, and it probably shouldn't.  */
    2321     62705312 :   if (flag_float_store && FLOAT_TYPE_P (TREE_TYPE (decl)))
    2322              :     return false;
    2323              : 
    2324     62702508 :   if (!targetm.calls.allocate_stack_slots_for_args ())
    2325              :     return true;
    2326              : 
    2327              :   /* If we're not interested in tracking debugging information for
    2328              :      this decl, then we can certainly put it in a register.  */
    2329     62702145 :   if (DECL_IGNORED_P (decl))
    2330              :     return true;
    2331              : 
    2332     42689564 :   if (optimize)
    2333              :     return true;
    2334              : 
    2335              :   /* Thunks force a tail call even at -O0 so we need to avoid creating a
    2336              :      dangling reference in case the parameter is passed by reference.  */
    2337      8711428 :   if (TREE_CODE (decl) == PARM_DECL && cfun->tail_call_marked)
    2338              :     return true;
    2339              : 
    2340      8710632 :   if (!DECL_REGISTER (decl))
    2341              :     return false;
    2342              : 
    2343              :   /* When not optimizing, disregard register keyword for types that
    2344              :      could have methods, otherwise the methods won't be callable from
    2345              :      the debugger.  */
    2346        13318 :   if (RECORD_OR_UNION_TYPE_P (TREE_TYPE (decl)))
    2347              :     return false;
    2348              : 
    2349              :   return true;
    2350              : }
    2351              : 
    2352              : /* Structures to communicate between the subroutines of assign_parms.
    2353              :    The first holds data persistent across all parameters, the second
    2354              :    is cleared out for each parameter.  */
    2355              : 
    2356              : struct assign_parm_data_all
    2357              : {
    2358              :   /* When INIT_CUMULATIVE_ARGS gets revamped, allocating CUMULATIVE_ARGS
    2359              :      should become a job of the target or otherwise encapsulated.  */
    2360              :   CUMULATIVE_ARGS args_so_far_v;
    2361              :   cumulative_args_t args_so_far;
    2362              :   struct args_size stack_args_size;
    2363              :   tree function_result_decl;
    2364              :   tree orig_fnargs;
    2365              :   rtx_insn *first_conversion_insn;
    2366              :   rtx_insn *last_conversion_insn;
    2367              :   HOST_WIDE_INT pretend_args_size;
    2368              :   HOST_WIDE_INT extra_pretend_bytes;
    2369              :   int reg_parm_stack_space;
    2370              : };
    2371              : 
    2372     18886186 : struct assign_parm_data_one
    2373              : {
    2374              :   tree nominal_type;
    2375              :   function_arg_info arg;
    2376              :   rtx entry_parm;
    2377              :   rtx stack_parm;
    2378              :   machine_mode nominal_mode;
    2379              :   machine_mode passed_mode;
    2380              :   struct locate_and_pad_arg_data locate;
    2381              :   int partial;
    2382              : };
    2383              : 
    2384              : /* A subroutine of assign_parms.  Initialize ALL.  */
    2385              : 
    2386              : static void
    2387      4528193 : assign_parms_initialize_all (struct assign_parm_data_all *all)
    2388              : {
    2389      4528193 :   tree fntype ATTRIBUTE_UNUSED;
    2390              : 
    2391      4528193 :   memset (all, 0, sizeof (*all));
    2392              : 
    2393      4528193 :   fntype = TREE_TYPE (current_function_decl);
    2394              : 
    2395              : #ifdef INIT_CUMULATIVE_INCOMING_ARGS
    2396              :   INIT_CUMULATIVE_INCOMING_ARGS (all->args_so_far_v, fntype, NULL_RTX);
    2397              : #else
    2398      4528193 :   INIT_CUMULATIVE_ARGS (all->args_so_far_v, fntype, NULL_RTX,
    2399              :                         current_function_decl, -1);
    2400              : #endif
    2401      4528193 :   all->args_so_far = pack_cumulative_args (&all->args_so_far_v);
    2402              : 
    2403              : #ifdef INCOMING_REG_PARM_STACK_SPACE
    2404      4528193 :   all->reg_parm_stack_space
    2405      4528193 :     = INCOMING_REG_PARM_STACK_SPACE (current_function_decl);
    2406              : #endif
    2407      4528193 : }
    2408              : 
    2409              : /* If ARGS contains entries with complex types, split the entry into two
    2410              :    entries of the component type.  Return a new list of substitutions are
    2411              :    needed, else the old list.  */
    2412              : 
    2413              : static void
    2414            0 : split_complex_args (vec<tree> *args)
    2415              : {
    2416            0 :   unsigned i;
    2417            0 :   tree p;
    2418              : 
    2419            0 :   FOR_EACH_VEC_ELT (*args, i, p)
    2420              :     {
    2421            0 :       tree type = TREE_TYPE (p);
    2422            0 :       if (TREE_CODE (type) == COMPLEX_TYPE
    2423            0 :           && targetm.calls.split_complex_arg (type))
    2424              :         {
    2425            0 :           tree decl;
    2426            0 :           tree subtype = TREE_TYPE (type);
    2427            0 :           bool addressable = TREE_ADDRESSABLE (p);
    2428              : 
    2429              :           /* Rewrite the PARM_DECL's type with its component.  */
    2430            0 :           p = copy_node (p);
    2431            0 :           TREE_TYPE (p) = subtype;
    2432            0 :           DECL_ARG_TYPE (p) = TREE_TYPE (DECL_ARG_TYPE (p));
    2433            0 :           SET_DECL_MODE (p, VOIDmode);
    2434            0 :           DECL_SIZE (p) = NULL;
    2435            0 :           DECL_SIZE_UNIT (p) = NULL;
    2436              :           /* If this arg must go in memory, put it in a pseudo here.
    2437              :              We can't allow it to go in memory as per normal parms,
    2438              :              because the usual place might not have the imag part
    2439              :              adjacent to the real part.  */
    2440            0 :           DECL_ARTIFICIAL (p) = addressable;
    2441            0 :           DECL_IGNORED_P (p) = addressable;
    2442            0 :           TREE_ADDRESSABLE (p) = 0;
    2443            0 :           layout_decl (p, 0);
    2444            0 :           (*args)[i] = p;
    2445              : 
    2446              :           /* Build a second synthetic decl.  */
    2447            0 :           decl = build_decl (EXPR_LOCATION (p),
    2448              :                              PARM_DECL, NULL_TREE, subtype);
    2449            0 :           DECL_ARG_TYPE (decl) = DECL_ARG_TYPE (p);
    2450            0 :           DECL_ARTIFICIAL (decl) = addressable;
    2451            0 :           DECL_IGNORED_P (decl) = addressable;
    2452            0 :           layout_decl (decl, 0);
    2453            0 :           args->safe_insert (++i, decl);
    2454              :         }
    2455              :     }
    2456            0 : }
    2457              : 
    2458              : /* A subroutine of assign_parms.  Adjust the parameter list to incorporate
    2459              :    the hidden struct return argument, and (abi willing) complex args.
    2460              :    Return the new parameter list.  */
    2461              : 
    2462              : static vec<tree>
    2463      4528193 : assign_parms_augmented_arg_list (struct assign_parm_data_all *all)
    2464              : {
    2465      4528193 :   tree fndecl = current_function_decl;
    2466      4528193 :   tree fntype = TREE_TYPE (fndecl);
    2467      4528193 :   vec<tree> fnargs = vNULL;
    2468      4528193 :   tree arg;
    2469              : 
    2470     13770431 :   for (arg = DECL_ARGUMENTS (fndecl); arg; arg = DECL_CHAIN (arg))
    2471      9242238 :     fnargs.safe_push (arg);
    2472              : 
    2473      4528193 :   all->orig_fnargs = DECL_ARGUMENTS (fndecl);
    2474              : 
    2475              :   /* If struct value address is treated as the first argument, make it so.  */
    2476      4528193 :   if (aggregate_value_p (DECL_RESULT (fndecl), fndecl)
    2477       200855 :       && ! cfun->returns_pcc_struct
    2478      4729048 :       && targetm.calls.struct_value_rtx (TREE_TYPE (fndecl), 1) == 0)
    2479              :     {
    2480       200855 :       tree type = build_pointer_type (TREE_TYPE (fntype));
    2481       200855 :       tree decl;
    2482              : 
    2483       200855 :       decl = build_decl (DECL_SOURCE_LOCATION (fndecl),
    2484              :                          PARM_DECL, get_identifier (".result_ptr"), type);
    2485       200855 :       DECL_ARG_TYPE (decl) = type;
    2486       200855 :       DECL_ARTIFICIAL (decl) = 1;
    2487       200855 :       DECL_NAMELESS (decl) = 1;
    2488       200855 :       TREE_CONSTANT (decl) = 1;
    2489              :       /* We don't set DECL_IGNORED_P or DECL_REGISTER here.  If this
    2490              :          changes, the end of the RESULT_DECL handling block in
    2491              :          use_register_for_decl must be adjusted to match.  */
    2492              : 
    2493       200855 :       DECL_CHAIN (decl) = all->orig_fnargs;
    2494       200855 :       all->orig_fnargs = decl;
    2495       200855 :       fnargs.safe_insert (0, decl);
    2496              : 
    2497       200855 :       all->function_result_decl = decl;
    2498              :     }
    2499              : 
    2500              :   /* If the target wants to split complex arguments into scalars, do so.  */
    2501      4528193 :   if (targetm.calls.split_complex_arg)
    2502            0 :     split_complex_args (&fnargs);
    2503              : 
    2504      4528193 :   return fnargs;
    2505              : }
    2506              : 
    2507              : /* A subroutine of assign_parms.  Examine PARM and pull out type and mode
    2508              :    data for the parameter.  Incorporate ABI specifics such as pass-by-
    2509              :    reference and type promotion.  */
    2510              : 
    2511              : static void
    2512      9443093 : assign_parm_find_data_types (struct assign_parm_data_all *all, tree parm,
    2513              :                              struct assign_parm_data_one *data)
    2514              : {
    2515      9443093 :   int unsignedp;
    2516              : 
    2517      9443093 :   *data = assign_parm_data_one ();
    2518              : 
    2519              :   /* NAMED_ARG is a misnomer.  We really mean 'non-variadic'. */
    2520      9443093 :   if (!cfun->stdarg)
    2521      9361204 :     data->arg.named = 1;  /* No variadic parms.  */
    2522        81889 :   else if (DECL_CHAIN (parm))
    2523        38905 :     data->arg.named = 1;  /* Not the last non-variadic parm. */
    2524        42984 :   else if (targetm.calls.strict_argument_naming (all->args_so_far))
    2525        42984 :     data->arg.named = 1;  /* Only variadic ones are unnamed.  */
    2526              :   else
    2527            0 :     data->arg.named = 0;  /* Treat as variadic.  */
    2528              : 
    2529      9443093 :   data->nominal_type = TREE_TYPE (parm);
    2530      9443093 :   data->arg.type = DECL_ARG_TYPE (parm);
    2531              : 
    2532              :   /* Look out for errors propagating this far.  Also, if the parameter's
    2533              :      type is void then its value doesn't matter.  */
    2534      9443093 :   if (TREE_TYPE (parm) == error_mark_node
    2535              :       /* This can happen after weird syntax errors
    2536              :          or if an enum type is defined among the parms.  */
    2537      9443000 :       || TREE_CODE (parm) != PARM_DECL
    2538      9443000 :       || data->arg.type == NULL
    2539     18886093 :       || VOID_TYPE_P (data->nominal_type))
    2540              :     {
    2541           93 :       data->nominal_type = data->arg.type = void_type_node;
    2542           93 :       data->nominal_mode = data->passed_mode = data->arg.mode = VOIDmode;
    2543           93 :       return;
    2544              :     }
    2545              : 
    2546              :   /* Find mode of arg as it is passed, and mode of arg as it should be
    2547              :      during execution of this function.  */
    2548      9443000 :   data->passed_mode = data->arg.mode = TYPE_MODE (data->arg.type);
    2549      9443000 :   data->nominal_mode = TYPE_MODE (data->nominal_type);
    2550              : 
    2551              :   /* If the parm is to be passed as a transparent union or record, use the
    2552              :      type of the first field for the tests below.  We have already verified
    2553              :      that the modes are the same.  */
    2554      9443000 :   if (RECORD_OR_UNION_TYPE_P (data->arg.type)
    2555      9443000 :       && TYPE_TRANSPARENT_AGGR (data->arg.type))
    2556         1482 :     data->arg.type = TREE_TYPE (first_field (data->arg.type));
    2557              : 
    2558              :   /* See if this arg was passed by invisible reference.  */
    2559      9443000 :   if (apply_pass_by_reference_rules (&all->args_so_far_v, data->arg))
    2560              :     {
    2561         9940 :       data->nominal_type = data->arg.type;
    2562         9940 :       data->passed_mode = data->nominal_mode = data->arg.mode;
    2563              :     }
    2564              : 
    2565              :   /* Find mode as it is passed by the ABI.  */
    2566      9443000 :   unsignedp = TYPE_UNSIGNED (data->arg.type);
    2567      9443000 :   data->arg.mode
    2568      9443000 :     = promote_function_mode (data->arg.type, data->arg.mode, &unsignedp,
    2569      9443000 :                              TREE_TYPE (current_function_decl), 0);
    2570              : }
    2571              : 
    2572              : /* A subroutine of assign_parms.  Invoke setup_incoming_varargs.  */
    2573              : 
    2574              : static void
    2575        21594 : assign_parms_setup_varargs (struct assign_parm_data_all *all,
    2576              :                             struct assign_parm_data_one *data, bool no_rtl)
    2577              : {
    2578        21594 :   int varargs_pretend_bytes = 0;
    2579              : 
    2580        21594 :   function_arg_info last_named_arg = data->arg;
    2581        21594 :   last_named_arg.named = true;
    2582        21594 :   targetm.calls.setup_incoming_varargs (all->args_so_far, last_named_arg,
    2583              :                                         &varargs_pretend_bytes, no_rtl);
    2584              : 
    2585              :   /* If the back-end has requested extra stack space, record how much is
    2586              :      needed.  Do not change pretend_args_size otherwise since it may be
    2587              :      nonzero from an earlier partial argument.  */
    2588        21594 :   if (varargs_pretend_bytes > 0)
    2589            0 :     all->pretend_args_size = varargs_pretend_bytes;
    2590        21594 : }
    2591              : 
    2592              : /* A subroutine of assign_parms.  Set DATA->ENTRY_PARM corresponding to
    2593              :    the incoming location of the current parameter.  */
    2594              : 
    2595              : static void
    2596      3237317 : assign_parm_find_entry_rtl (struct assign_parm_data_all *all,
    2597              :                             struct assign_parm_data_one *data)
    2598              : {
    2599      3237317 :   HOST_WIDE_INT pretend_bytes = 0;
    2600      3237317 :   rtx entry_parm;
    2601      3237317 :   bool in_regs;
    2602              : 
    2603      3237317 :   if (data->arg.mode == VOIDmode)
    2604              :     {
    2605            0 :       data->entry_parm = data->stack_parm = const0_rtx;
    2606            0 :       return;
    2607              :     }
    2608              : 
    2609      3237317 :   targetm.calls.warn_parameter_passing_abi (all->args_so_far,
    2610              :                                             data->arg.type);
    2611              : 
    2612      6474634 :   entry_parm = targetm.calls.function_incoming_arg (all->args_so_far,
    2613      3237317 :                                                     data->arg);
    2614      3237317 :   if (entry_parm == 0)
    2615      1073399 :     data->arg.mode = data->passed_mode;
    2616              : 
    2617              :   /* Determine parm's home in the stack, in case it arrives in the stack
    2618              :      or we should pretend it did.  Compute the stack position and rtx where
    2619              :      the argument arrives and its size.
    2620              : 
    2621              :      There is one complexity here:  If this was a parameter that would
    2622              :      have been passed in registers, but wasn't only because it is
    2623              :      __builtin_va_alist, we want locate_and_pad_parm to treat it as if
    2624              :      it came in a register so that REG_PARM_STACK_SPACE isn't skipped.
    2625              :      In this case, we call FUNCTION_ARG with NAMED set to 1 instead of 0
    2626              :      as it was the previous time.  */
    2627      1073399 :   in_regs = (entry_parm != 0);
    2628              : #ifdef STACK_PARMS_IN_REG_PARM_AREA
    2629              :   in_regs = true;
    2630              : #endif
    2631      1073399 :   if (!in_regs && !data->arg.named)
    2632              :     {
    2633            0 :       if (targetm.calls.pretend_outgoing_varargs_named (all->args_so_far))
    2634              :         {
    2635            0 :           rtx tem;
    2636            0 :           function_arg_info named_arg = data->arg;
    2637            0 :           named_arg.named = true;
    2638            0 :           tem = targetm.calls.function_incoming_arg (all->args_so_far,
    2639              :                                                      named_arg);
    2640            0 :           in_regs = tem != NULL;
    2641              :         }
    2642              :     }
    2643              : 
    2644              :   /* If this parameter was passed both in registers and in the stack, use
    2645              :      the copy on the stack.  */
    2646      3237317 :   if (targetm.calls.must_pass_in_stack (data->arg))
    2647              :     entry_parm = 0;
    2648              : 
    2649      3237317 :   if (entry_parm)
    2650              :     {
    2651      2163918 :       int partial;
    2652              : 
    2653      2163918 :       partial = targetm.calls.arg_partial_bytes (all->args_so_far, data->arg);
    2654      2163918 :       data->partial = partial;
    2655              : 
    2656              :       /* The caller might already have allocated stack space for the
    2657              :          register parameters.  */
    2658      2163918 :       if (partial != 0 && all->reg_parm_stack_space == 0)
    2659              :         {
    2660              :           /* Part of this argument is passed in registers and part
    2661              :              is passed on the stack.  Ask the prologue code to extend
    2662              :              the stack part so that we can recreate the full value.
    2663              : 
    2664              :              PRETEND_BYTES is the size of the registers we need to store.
    2665              :              CURRENT_FUNCTION_PRETEND_ARGS_SIZE is the amount of extra
    2666              :              stack space that the prologue should allocate.
    2667              : 
    2668              :              Internally, gcc assumes that the argument pointer is aligned
    2669              :              to STACK_BOUNDARY bits.  This is used both for alignment
    2670              :              optimizations (see init_emit) and to locate arguments that are
    2671              :              aligned to more than PARM_BOUNDARY bits.  We must preserve this
    2672              :              invariant by rounding CURRENT_FUNCTION_PRETEND_ARGS_SIZE up to
    2673              :              a stack boundary.  */
    2674              : 
    2675              :           /* We assume at most one partial arg, and it must be the first
    2676              :              argument on the stack.  */
    2677            0 :           gcc_assert (!all->extra_pretend_bytes && !all->pretend_args_size);
    2678              : 
    2679            0 :           pretend_bytes = partial;
    2680            0 :           all->pretend_args_size = CEIL_ROUND (pretend_bytes, STACK_BYTES);
    2681              : 
    2682              :           /* We want to align relative to the actual stack pointer, so
    2683              :              don't include this in the stack size until later.  */
    2684            0 :           all->extra_pretend_bytes = all->pretend_args_size;
    2685              :         }
    2686              :     }
    2687              : 
    2688      3237317 :   locate_and_pad_parm (data->arg.mode, data->arg.type, in_regs,
    2689              :                        all->reg_parm_stack_space,
    2690              :                        entry_parm ? data->partial : 0, current_function_decl,
    2691              :                        &all->stack_args_size, &data->locate);
    2692              : 
    2693              :   /* Update parm_stack_boundary if this parameter is passed in the
    2694              :      stack.  */
    2695      3237317 :   if (!in_regs && crtl->parm_stack_boundary < data->locate.boundary)
    2696       199378 :     crtl->parm_stack_boundary = data->locate.boundary;
    2697              : 
    2698              :   /* Adjust offsets to include the pretend args.  */
    2699      3237317 :   pretend_bytes = all->extra_pretend_bytes - pretend_bytes;
    2700      3237317 :   data->locate.slot_offset.constant += pretend_bytes;
    2701      3237317 :   data->locate.offset.constant += pretend_bytes;
    2702              : 
    2703      3237317 :   data->entry_parm = entry_parm;
    2704              : }
    2705              : 
    2706              : /* A subroutine of assign_parms.  If there is actually space on the stack
    2707              :    for this parm, count it in stack_args_size and return true.  */
    2708              : 
    2709              : static bool
    2710      3237317 : assign_parm_is_stack_parm (struct assign_parm_data_all *all,
    2711              :                            struct assign_parm_data_one *data)
    2712              : {
    2713              :   /* Trivially true if we've no incoming register.  */
    2714      3237317 :   if (data->entry_parm == NULL)
    2715              :     ;
    2716              :   /* Also true if we're partially in registers and partially not,
    2717              :      since we've arranged to drop the entire argument on the stack.  */
    2718      2163918 :   else if (data->partial != 0)
    2719              :     ;
    2720              :   /* Also true if the target says that it's passed in both registers
    2721              :      and on the stack.  */
    2722      2163918 :   else if (GET_CODE (data->entry_parm) == PARALLEL
    2723        54752 :            && XEXP (XVECEXP (data->entry_parm, 0, 0), 0) == NULL_RTX)
    2724              :     ;
    2725              :   /* Also true if the target says that there's stack allocated for
    2726              :      all register parameters.  */
    2727      2163918 :   else if (all->reg_parm_stack_space > 0)
    2728              :     ;
    2729              :   /* Otherwise, no, this parameter has no ABI defined stack slot.  */
    2730              :   else
    2731              :     return false;
    2732              : 
    2733      1183734 :   all->stack_args_size.constant += data->locate.size.constant;
    2734      1183734 :   if (data->locate.size.var)
    2735            0 :     ADD_PARM_SIZE (all->stack_args_size, data->locate.size.var);
    2736              : 
    2737              :   return true;
    2738              : }
    2739              : 
    2740              : /* A subroutine of assign_parms.  Given that this parameter is allocated
    2741              :    stack space by the ABI, find it.  */
    2742              : 
    2743              : static void
    2744      1183734 : assign_parm_find_stack_rtl (tree parm, struct assign_parm_data_one *data)
    2745              : {
    2746      1183734 :   rtx offset_rtx, stack_parm;
    2747      1183734 :   unsigned int align, boundary;
    2748              : 
    2749              :   /* If we're passing this arg using a reg, make its stack home the
    2750              :      aligned stack slot.  */
    2751      1183734 :   if (data->entry_parm)
    2752       110335 :     offset_rtx = ARGS_SIZE_RTX (data->locate.slot_offset);
    2753              :   else
    2754      1316768 :     offset_rtx = ARGS_SIZE_RTX (data->locate.offset);
    2755              : 
    2756      1183734 :   stack_parm = crtl->args.internal_arg_pointer;
    2757      1183734 :   if (offset_rtx != const0_rtx)
    2758      1094142 :     stack_parm = gen_rtx_PLUS (Pmode, stack_parm, offset_rtx);
    2759      1183734 :   stack_parm = gen_rtx_MEM (data->arg.mode, stack_parm);
    2760              : 
    2761      1183734 :   if (!data->arg.pass_by_reference)
    2762              :     {
    2763      1178804 :       set_mem_attributes (stack_parm, parm, 1);
    2764              :       /* set_mem_attributes could set MEM_SIZE to the passed mode's size,
    2765              :          while promoted mode's size is needed.  */
    2766      1178804 :       if (data->arg.mode != BLKmode
    2767      1178804 :           && data->arg.mode != DECL_MODE (parm))
    2768              :         {
    2769            0 :           set_mem_size (stack_parm, GET_MODE_SIZE (data->arg.mode));
    2770            0 :           if (MEM_EXPR (stack_parm) && MEM_OFFSET_KNOWN_P (stack_parm))
    2771              :             {
    2772            0 :               poly_int64 offset = subreg_lowpart_offset (DECL_MODE (parm),
    2773            0 :                                                          data->arg.mode);
    2774            0 :               if (maybe_ne (offset, 0))
    2775            0 :                 set_mem_offset (stack_parm, MEM_OFFSET (stack_parm) - offset);
    2776              :             }
    2777              :         }
    2778              :     }
    2779              : 
    2780      1183734 :   boundary = data->locate.boundary;
    2781      1183734 :   align = BITS_PER_UNIT;
    2782              : 
    2783              :   /* If we're padding upward, we know that the alignment of the slot
    2784              :      is TARGET_FUNCTION_ARG_BOUNDARY.  If we're using slot_offset, we're
    2785              :      intentionally forcing upward padding.  Otherwise we have to come
    2786              :      up with a guess at the alignment based on OFFSET_RTX.  */
    2787      1183734 :   poly_int64 offset;
    2788      1183734 :   if (data->locate.where_pad == PAD_NONE || data->entry_parm)
    2789              :     align = boundary;
    2790      1073399 :   else if (data->locate.where_pad == PAD_UPWARD)
    2791              :     {
    2792      1073399 :       align = boundary;
    2793              :       /* If the argument offset is actually more aligned than the nominal
    2794              :          stack slot boundary, take advantage of that excess alignment.
    2795              :          Don't make any assumptions if STACK_POINTER_OFFSET is in use.  */
    2796      1073399 :       if (poly_int_rtx_p (offset_rtx, &offset)
    2797              :           && known_eq (STACK_POINTER_OFFSET, 0))
    2798              :         {
    2799      1073399 :           unsigned int offset_align = known_alignment (offset) * BITS_PER_UNIT;
    2800      1202209 :           if (offset_align == 0 || offset_align > STACK_BOUNDARY)
    2801       592206 :             offset_align = STACK_BOUNDARY;
    2802      1073399 :           align = MAX (align, offset_align);
    2803              :         }
    2804              :     }
    2805            0 :   else if (poly_int_rtx_p (offset_rtx, &offset))
    2806              :     {
    2807            0 :       align = least_bit_hwi (boundary);
    2808            0 :       unsigned int offset_align = known_alignment (offset) * BITS_PER_UNIT;
    2809            0 :       if (offset_align != 0)
    2810            0 :         align = MIN (align, offset_align);
    2811              :     }
    2812              : 
    2813              :   /* If part of the argument is passed in registers and the rest on the
    2814              :      stack, the value is reconstructed by spilling the register part just
    2815              :      below the aligned stack slot.  The full value therefore starts at an
    2816              :      address that is only aligned to the largest power of two dividing the
    2817              :      size of the register part, not to FUNCTION_ARG_BOUNDARY.  */
    2818      1183734 :   if (data->partial)
    2819            0 :     align = MIN (align,
    2820              :                  least_bit_hwi (data->partial) * BITS_PER_UNIT);
    2821              : 
    2822      1183734 :   set_mem_align (stack_parm, align);
    2823              : 
    2824      1183734 :   if (data->entry_parm)
    2825       110335 :     set_reg_attrs_for_parm (data->entry_parm, stack_parm);
    2826              : 
    2827      1183734 :   data->stack_parm = stack_parm;
    2828      1183734 : }
    2829              : 
    2830              : /* A subroutine of assign_parms.  Adjust DATA->ENTRY_RTL such that it's
    2831              :    always valid and contiguous.  */
    2832              : 
    2833              : static void
    2834      1183734 : assign_parm_adjust_entry_rtl (struct assign_parm_data_one *data)
    2835              : {
    2836      1183734 :   rtx entry_parm = data->entry_parm;
    2837      1183734 :   rtx stack_parm = data->stack_parm;
    2838              : 
    2839              :   /* If this parm was passed part in regs and part in memory, pretend it
    2840              :      arrived entirely in memory by pushing the register-part onto the stack.
    2841              :      In the special case of a DImode or DFmode that is split, we could put
    2842              :      it together in a pseudoreg directly, but for now that's not worth
    2843              :      bothering with.  */
    2844      1183734 :   if (data->partial != 0)
    2845              :     {
    2846              :       /* Handle calls that pass values in multiple non-contiguous
    2847              :          locations.  The Irix 6 ABI has examples of this.  */
    2848            0 :       if (GET_CODE (entry_parm) == PARALLEL)
    2849            0 :         emit_group_store (validize_mem (copy_rtx (stack_parm)), entry_parm,
    2850            0 :                           data->arg.type, int_size_in_bytes (data->arg.type));
    2851              :       else
    2852              :         {
    2853            0 :           gcc_assert (data->partial % UNITS_PER_WORD == 0);
    2854            0 :           move_block_from_reg (REGNO (entry_parm),
    2855              :                                validize_mem (copy_rtx (stack_parm)),
    2856              :                                data->partial / UNITS_PER_WORD);
    2857              :         }
    2858              : 
    2859              :       entry_parm = stack_parm;
    2860              :     }
    2861              : 
    2862              :   /* If we didn't decide this parm came in a register, by default it came
    2863              :      on the stack.  */
    2864      1183734 :   else if (entry_parm == NULL)
    2865              :     entry_parm = stack_parm;
    2866              : 
    2867              :   /* When an argument is passed in multiple locations, we can't make use
    2868              :      of this information, but we can save some copying if the whole argument
    2869              :      is passed in a single register.  */
    2870       110335 :   else if (GET_CODE (entry_parm) == PARALLEL
    2871            0 :            && data->nominal_mode != BLKmode
    2872            0 :            && data->passed_mode != BLKmode)
    2873              :     {
    2874            0 :       size_t i, len = XVECLEN (entry_parm, 0);
    2875              : 
    2876            0 :       for (i = 0; i < len; i++)
    2877            0 :         if (XEXP (XVECEXP (entry_parm, 0, i), 0) != NULL_RTX
    2878            0 :             && REG_P (XEXP (XVECEXP (entry_parm, 0, i), 0))
    2879            0 :             && (GET_MODE (XEXP (XVECEXP (entry_parm, 0, i), 0))
    2880            0 :                 == data->passed_mode)
    2881            0 :             && INTVAL (XEXP (XVECEXP (entry_parm, 0, i), 1)) == 0)
    2882              :           {
    2883              :             entry_parm = XEXP (XVECEXP (entry_parm, 0, i), 0);
    2884              :             break;
    2885              :           }
    2886              :     }
    2887              : 
    2888      1183734 :   data->entry_parm = entry_parm;
    2889      1183734 : }
    2890              : 
    2891              : /* A subroutine of assign_parms.  Reconstitute any values which were
    2892              :    passed in multiple registers and would fit in a single register.  */
    2893              : 
    2894              : static void
    2895      3162813 : assign_parm_remove_parallels (struct assign_parm_data_one *data)
    2896              : {
    2897      3162813 :   rtx entry_parm = data->entry_parm;
    2898              : 
    2899              :   /* Convert the PARALLEL to a REG of the same mode as the parallel.
    2900              :      This can be done with register operations rather than on the
    2901              :      stack, even if we will store the reconstituted parameter on the
    2902              :      stack later.  */
    2903      3162813 :   if (GET_CODE (entry_parm) == PARALLEL && GET_MODE (entry_parm) != BLKmode)
    2904              :     {
    2905        50894 :       rtx parmreg = gen_reg_rtx (GET_MODE (entry_parm));
    2906        50894 :       emit_group_store (parmreg, entry_parm, data->arg.type,
    2907       101788 :                         GET_MODE_SIZE (GET_MODE (entry_parm)));
    2908        50894 :       entry_parm = parmreg;
    2909              :     }
    2910              : 
    2911      3162813 :   data->entry_parm = entry_parm;
    2912      3162813 : }
    2913              : 
    2914              : /* A subroutine of assign_parms.  Adjust DATA->STACK_RTL such that it's
    2915              :    always valid and properly aligned.  */
    2916              : 
    2917              : static void
    2918      3237317 : assign_parm_adjust_stack_rtl (tree parm, struct assign_parm_data_one *data)
    2919              : {
    2920      3237317 :   rtx stack_parm = data->stack_parm;
    2921              : 
    2922              :   /* If we can't trust the parm stack slot to be aligned enough for its
    2923              :      ultimate type, don't use that slot after entry.  We'll make another
    2924              :      stack slot, if we need one.  */
    2925      3237317 :   if (stack_parm
    2926      3237317 :       && ((GET_MODE_ALIGNMENT (data->nominal_mode) > MEM_ALIGN (stack_parm)
    2927        24925 :            && ((optab_handler (movmisalign_optab, data->nominal_mode)
    2928              :                 != CODE_FOR_nothing)
    2929        49842 :                || targetm.slow_unaligned_access (data->nominal_mode,
    2930        24921 :                                                  MEM_ALIGN (stack_parm))))
    2931      1182188 :           || (data->nominal_type
    2932      2364376 :               && TYPE_ALIGN (data->nominal_type) > MEM_ALIGN (stack_parm)
    2933         4510 :               && ((MEM_ALIGN (stack_parm)
    2934         4510 :                    < MIN (BIGGEST_ALIGNMENT, MAX_SUPPORTED_STACK_ALIGNMENT))
    2935              :                   /* If its address is taken, make a local copy whose
    2936              :                      maximum alignment is MAX_SUPPORTED_STACK_ALIGNMENT.
    2937              :                    */
    2938            4 :                   || (TREE_ADDRESSABLE (parm)
    2939              :                       && (MEM_ALIGN (stack_parm)
    2940              :                           < MAX_SUPPORTED_STACK_ALIGNMENT))))))
    2941              :     stack_parm = NULL;
    2942              : 
    2943              :   /* If parm was passed in memory, and we need to convert it on entry,
    2944              :      don't store it back in that same slot.  */
    2945      3232806 :   else if (data->entry_parm == stack_parm
    2946      1067346 :            && data->nominal_mode != BLKmode
    2947       998336 :            && data->nominal_mode != data->passed_mode)
    2948              :     stack_parm = NULL;
    2949              : 
    2950              :   /* If stack protection is in effect for this function, don't leave any
    2951              :      pointers in their passed stack slots.  */
    2952      3232806 :   else if (crtl->stack_protect_guard
    2953          224 :            && (flag_stack_protect == SPCT_FLAG_ALL
    2954          163 :                || data->arg.pass_by_reference
    2955          163 :                || POINTER_TYPE_P (data->nominal_type)))
    2956      3237317 :     stack_parm = NULL;
    2957              : 
    2958      3237317 :   data->stack_parm = stack_parm;
    2959      3237317 : }
    2960              : 
    2961              : /* A subroutine of assign_parms.  Return true if the current parameter
    2962              :    should be stored as a BLKmode in the current frame.  */
    2963              : 
    2964              : static bool
    2965      3237317 : assign_parm_setup_block_p (struct assign_parm_data_one *data)
    2966              : {
    2967            0 :   if (data->nominal_mode == BLKmode)
    2968              :     return true;
    2969      3162813 :   if (GET_MODE (data->entry_parm) == BLKmode)
    2970            0 :     return true;
    2971              : 
    2972              : #ifdef BLOCK_REG_PADDING
    2973              :   /* Only assign_parm_setup_block knows how to deal with register arguments
    2974              :      that are padded at the least significant end.  */
    2975              :   if (REG_P (data->entry_parm)
    2976              :       && known_lt (GET_MODE_SIZE (data->arg.mode), UNITS_PER_WORD)
    2977              :       && (BLOCK_REG_PADDING (data->passed_mode, data->arg.type, 1)
    2978              :           == (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)))
    2979              :     return true;
    2980              : #endif
    2981              : 
    2982              :   return false;
    2983              : }
    2984              : 
    2985              : /* A subroutine of assign_parms.  Arrange for the parameter to be
    2986              :    present and valid in DATA->STACK_RTL.  */
    2987              : 
    2988              : static void
    2989        74504 : assign_parm_setup_block (struct assign_parm_data_all *all,
    2990              :                          tree parm, struct assign_parm_data_one *data)
    2991              : {
    2992        74504 :   rtx entry_parm = data->entry_parm;
    2993        74504 :   rtx stack_parm = data->stack_parm;
    2994        74504 :   rtx target_reg = NULL_RTX;
    2995        74504 :   bool in_conversion_seq = false;
    2996        74504 :   HOST_WIDE_INT size;
    2997        74504 :   HOST_WIDE_INT size_stored;
    2998              : 
    2999        74504 :   if (GET_CODE (entry_parm) == PARALLEL)
    3000         3858 :     entry_parm = emit_group_move_into_temps (entry_parm);
    3001              : 
    3002              :   /* If we want the parameter in a pseudo, don't use a stack slot.  */
    3003        74504 :   if (is_gimple_reg (parm) && use_register_for_decl (parm))
    3004              :     {
    3005            0 :       tree def = ssa_default_def (cfun, parm);
    3006            0 :       gcc_assert (def);
    3007            0 :       machine_mode mode = promote_ssa_mode (def, NULL);
    3008            0 :       rtx reg = gen_reg_rtx (mode);
    3009            0 :       if (GET_CODE (reg) != CONCAT)
    3010              :         stack_parm = reg;
    3011              :       else
    3012              :         {
    3013            0 :           target_reg = reg;
    3014              :           /* Avoid allocating a stack slot, if there isn't one
    3015              :              preallocated by the ABI.  It might seem like we should
    3016              :              always prefer a pseudo, but converting between
    3017              :              floating-point and integer modes goes through the stack
    3018              :              on various machines, so it's better to use the reserved
    3019              :              stack slot than to risk wasting it and allocating more
    3020              :              for the conversion.  */
    3021            0 :           if (stack_parm == NULL_RTX)
    3022              :             {
    3023            0 :               int save = generating_concat_p;
    3024            0 :               generating_concat_p = 0;
    3025            0 :               stack_parm = gen_reg_rtx (mode);
    3026            0 :               generating_concat_p = save;
    3027              :             }
    3028              :         }
    3029            0 :       data->stack_parm = NULL;
    3030              :     }
    3031              : 
    3032        74504 :   size = int_size_in_bytes (data->arg.type);
    3033        87127 :   size_stored = CEIL_ROUND (size, UNITS_PER_WORD);
    3034        74504 :   if (stack_parm == 0)
    3035              :     {
    3036         5497 :       HOST_WIDE_INT parm_align
    3037              :         = ((STRICT_ALIGNMENT || BITS_PER_WORD <= MAX_SUPPORTED_STACK_ALIGNMENT)
    3038         5709 :            ? MAX (DECL_ALIGN (parm), BITS_PER_WORD) : DECL_ALIGN (parm));
    3039              : 
    3040         5497 :       SET_DECL_ALIGN (parm, parm_align);
    3041         5497 :       stack_parm = assign_stack_local (BLKmode, size_stored,
    3042         5497 :                                        DECL_ALIGN (parm));
    3043        10994 :       if (known_eq (GET_MODE_SIZE (GET_MODE (entry_parm)), size))
    3044           77 :         PUT_MODE (stack_parm, GET_MODE (entry_parm));
    3045         5497 :       set_mem_attributes (stack_parm, parm, 1);
    3046              :     }
    3047              : 
    3048              :   /* If a BLKmode arrives in registers, copy it to a stack slot.  Handle
    3049              :      calls that pass values in multiple non-contiguous locations.  */
    3050        74504 :   if (REG_P (entry_parm) || GET_CODE (entry_parm) == PARALLEL)
    3051              :     {
    3052         5194 :       rtx mem;
    3053              : 
    3054              :       /* Note that we will be storing an integral number of words.
    3055              :          So we have to be careful to ensure that we allocate an
    3056              :          integral number of words.  We do this above when we call
    3057              :          assign_stack_local if space was not allocated in the argument
    3058              :          list.  If it was, this will not work if PARM_BOUNDARY is not
    3059              :          a multiple of BITS_PER_WORD.  It isn't clear how to fix this
    3060              :          if it becomes a problem.  Exception is when BLKmode arrives
    3061              :          with arguments not conforming to word_mode.  */
    3062              : 
    3063         5194 :       if (data->stack_parm == 0)
    3064              :         ;
    3065         5194 :       else if (GET_CODE (entry_parm) == PARALLEL)
    3066              :         ;
    3067              :       else
    3068         5194 :         gcc_assert (!size || !(PARM_BOUNDARY % BITS_PER_WORD));
    3069              : 
    3070         5194 :       mem = validize_mem (copy_rtx (stack_parm));
    3071              : 
    3072              :       /* Handle values in multiple non-contiguous locations.  */
    3073         5194 :       if (GET_CODE (entry_parm) == PARALLEL && !MEM_P (mem))
    3074            0 :         emit_group_store (mem, entry_parm, data->arg.type, size);
    3075         5194 :       else if (GET_CODE (entry_parm) == PARALLEL)
    3076              :         {
    3077         3858 :           push_to_sequence2 (all->first_conversion_insn,
    3078              :                              all->last_conversion_insn);
    3079         3858 :           emit_group_store (mem, entry_parm, data->arg.type, size);
    3080         3858 :           all->first_conversion_insn = get_insns ();
    3081         3858 :           all->last_conversion_insn = get_last_insn ();
    3082         3858 :           end_sequence ();
    3083         3858 :           in_conversion_seq = true;
    3084              :         }
    3085              : 
    3086         1336 :       else if (size == 0)
    3087              :         ;
    3088              : 
    3089              :       /* If SIZE is that of a mode no bigger than a word, just use
    3090              :          that mode's store operation.  */
    3091         1320 :       else if (size <= UNITS_PER_WORD)
    3092              :         {
    3093         1316 :           unsigned int bits = size * BITS_PER_UNIT;
    3094         1316 :           machine_mode mode = int_mode_for_size (bits, 0).else_blk ();
    3095              : 
    3096         1316 :           if (mode != BLKmode
    3097              : #ifdef BLOCK_REG_PADDING
    3098              :               && (size == UNITS_PER_WORD
    3099              :                   || (BLOCK_REG_PADDING (mode, data->arg.type, 1)
    3100              :                       != (BYTES_BIG_ENDIAN ? PAD_UPWARD : PAD_DOWNWARD)))
    3101              : #endif
    3102              :               )
    3103              :             {
    3104          102 :               rtx reg;
    3105              : 
    3106              :               /* We are really truncating a word_mode value containing
    3107              :                  SIZE bytes into a value of mode MODE.  If such an
    3108              :                  operation requires no actual instructions, we can refer
    3109              :                  to the value directly in mode MODE, otherwise we must
    3110              :                  start with the register in word_mode and explicitly
    3111              :                  convert it.  */
    3112          102 :               if (mode == word_mode
    3113          102 :                   || TRULY_NOOP_TRUNCATION_MODES_P (mode, word_mode))
    3114          102 :                 reg = gen_rtx_REG (mode, REGNO (entry_parm));
    3115              :               else
    3116              :                 {
    3117            0 :                   reg = gen_rtx_REG (word_mode, REGNO (entry_parm));
    3118            0 :                   reg = convert_to_mode (mode, copy_to_reg (reg), 1);
    3119              :                 }
    3120              : 
    3121              :               /* We use adjust_address to get a new MEM with the mode
    3122              :                  changed.  adjust_address is better than change_address
    3123              :                  for this purpose because adjust_address does not lose
    3124              :                  the MEM_EXPR associated with the MEM.
    3125              : 
    3126              :                  If the MEM_EXPR is lost, then optimizations like DSE
    3127              :                  assume the MEM escapes and thus is not subject to DSE.  */
    3128          102 :               emit_move_insn (adjust_address (mem, mode, 0), reg);
    3129              :             }
    3130              : 
    3131              : #ifdef BLOCK_REG_PADDING
    3132              :           /* Storing the register in memory as a full word, as
    3133              :              move_block_from_reg below would do, and then using the
    3134              :              MEM in a smaller mode, has the effect of shifting right
    3135              :              if BYTES_BIG_ENDIAN.  If we're bypassing memory, the
    3136              :              shifting must be explicit.  */
    3137              :           else if (!MEM_P (mem))
    3138              :             {
    3139              :               rtx x;
    3140              : 
    3141              :               /* If the assert below fails, we should have taken the
    3142              :                  mode != BLKmode path above, unless we have downward
    3143              :                  padding of smaller-than-word arguments on a machine
    3144              :                  with little-endian bytes, which would likely require
    3145              :                  additional changes to work correctly.  */
    3146              :               gcc_checking_assert (BYTES_BIG_ENDIAN
    3147              :                                    && (BLOCK_REG_PADDING (mode,
    3148              :                                                           data->arg.type, 1)
    3149              :                                        == PAD_UPWARD));
    3150              : 
    3151              :               int by = (UNITS_PER_WORD - size) * BITS_PER_UNIT;
    3152              : 
    3153              :               x = gen_rtx_REG (word_mode, REGNO (entry_parm));
    3154              :               x = expand_shift (RSHIFT_EXPR, word_mode, x, by,
    3155              :                                 NULL_RTX, 1);
    3156              :               x = force_reg (word_mode, x);
    3157              :               x = gen_lowpart_SUBREG (GET_MODE (mem), x);
    3158              : 
    3159              :               emit_move_insn (mem, x);
    3160              :             }
    3161              : #endif
    3162              : 
    3163              :           /* Blocks smaller than a word on a BYTES_BIG_ENDIAN
    3164              :              machine must be aligned to the left before storing
    3165              :              to memory.  Note that the previous test doesn't
    3166              :              handle all cases (e.g. SIZE == 3).  */
    3167         1214 :           else if (size != UNITS_PER_WORD
    3168              : #ifdef BLOCK_REG_PADDING
    3169              :                    && (BLOCK_REG_PADDING (mode, data->arg.type, 1)
    3170              :                        == PAD_DOWNWARD)
    3171              : #else
    3172              :                    && BYTES_BIG_ENDIAN
    3173              : #endif
    3174              :                    )
    3175              :             {
    3176              :               rtx tem, x;
    3177              :               int by = (UNITS_PER_WORD - size) * BITS_PER_UNIT;
    3178              :               rtx reg = gen_rtx_REG (word_mode, REGNO (entry_parm));
    3179              : 
    3180              :               x = expand_shift (LSHIFT_EXPR, word_mode, reg, by, NULL_RTX, 1);
    3181              :               tem = change_address (mem, word_mode, 0);
    3182              :               emit_move_insn (tem, x);
    3183              :             }
    3184              :           else
    3185         2428 :             move_block_from_reg (REGNO (entry_parm), mem,
    3186         1214 :                                  size_stored / UNITS_PER_WORD);
    3187              :         }
    3188            2 :       else if (!MEM_P (mem))
    3189              :         {
    3190            0 :           gcc_checking_assert (size > UNITS_PER_WORD);
    3191              : #ifdef BLOCK_REG_PADDING
    3192              :           gcc_checking_assert (BLOCK_REG_PADDING (GET_MODE (mem),
    3193              :                                                   data->arg.type, 0)
    3194              :                                == PAD_UPWARD);
    3195              : #endif
    3196            0 :           emit_move_insn (mem, entry_parm);
    3197              :         }
    3198              :       else
    3199            2 :         move_block_from_reg (REGNO (entry_parm), mem,
    3200            2 :                              size_stored / UNITS_PER_WORD);
    3201              :     }
    3202        69310 :   else if (data->stack_parm == 0 && !TYPE_EMPTY_P (data->arg.type))
    3203              :     {
    3204          207 :       push_to_sequence2 (all->first_conversion_insn, all->last_conversion_insn);
    3205          207 :       emit_block_move (stack_parm, data->entry_parm, GEN_INT (size),
    3206              :                        BLOCK_OP_NORMAL);
    3207          207 :       all->first_conversion_insn = get_insns ();
    3208          207 :       all->last_conversion_insn = get_last_insn ();
    3209          207 :       end_sequence ();
    3210          207 :       in_conversion_seq = true;
    3211              :     }
    3212              : 
    3213        74504 :   if (target_reg)
    3214              :     {
    3215            0 :       if (!in_conversion_seq)
    3216            0 :         emit_move_insn (target_reg, stack_parm);
    3217              :       else
    3218              :         {
    3219            0 :           push_to_sequence2 (all->first_conversion_insn,
    3220              :                              all->last_conversion_insn);
    3221            0 :           emit_move_insn (target_reg, stack_parm);
    3222            0 :           all->first_conversion_insn = get_insns ();
    3223            0 :           all->last_conversion_insn = get_last_insn ();
    3224            0 :           end_sequence ();
    3225              :         }
    3226              :       stack_parm = target_reg;
    3227              :     }
    3228              : 
    3229        74504 :   data->stack_parm = stack_parm;
    3230        74504 :   set_parm_rtl (parm, stack_parm);
    3231        74504 : }
    3232              : 
    3233              : /* A subroutine of assign_parms.  Allocate a pseudo to hold the current
    3234              :    parameter.  Get it there.  Perform all ABI specified conversions.  */
    3235              : 
    3236              : static void
    3237      2320046 : assign_parm_setup_reg (struct assign_parm_data_all *all, tree parm,
    3238              :                        struct assign_parm_data_one *data)
    3239              : {
    3240      2320046 :   rtx parmreg, validated_mem;
    3241      2320046 :   rtx equiv_stack_parm;
    3242      2320046 :   machine_mode promoted_nominal_mode;
    3243      2320046 :   int unsignedp = TYPE_UNSIGNED (TREE_TYPE (parm));
    3244      2320046 :   bool did_conversion = false;
    3245      2320046 :   bool need_conversion, moved;
    3246      2320046 :   enum insn_code icode;
    3247      2320046 :   rtx rtl;
    3248              : 
    3249              :   /* Store the parm in a pseudoregister during the function, but we may
    3250              :      need to do it in a wider mode.  Using 2 here makes the result
    3251              :      consistent with promote_decl_mode and thus expand_expr_real_1.  */
    3252      2320046 :   promoted_nominal_mode
    3253      4640092 :     = promote_function_mode (data->nominal_type, data->nominal_mode, &unsignedp,
    3254      2320046 :                              TREE_TYPE (current_function_decl), 2);
    3255              : 
    3256      2320046 :   parmreg = gen_reg_rtx (promoted_nominal_mode);
    3257      2320046 :   if (!DECL_ARTIFICIAL (parm))
    3258      2092957 :     mark_user_reg (parmreg);
    3259              : 
    3260              :   /* If this was an item that we received a pointer to,
    3261              :      set rtl appropriately.  */
    3262      2320046 :   if (data->arg.pass_by_reference)
    3263              :     {
    3264         4966 :       rtl = gen_rtx_MEM (TYPE_MODE (TREE_TYPE (data->arg.type)), parmreg);
    3265         4966 :       set_mem_attributes (rtl, parm, 1);
    3266              :     }
    3267              :   else
    3268              :     rtl = parmreg;
    3269              : 
    3270      2320046 :   assign_parm_remove_parallels (data);
    3271              : 
    3272              :   /* Copy the value into the register, thus bridging between
    3273              :      assign_parm_find_data_types and expand_expr_real_1.  */
    3274              : 
    3275      2320046 :   equiv_stack_parm = data->stack_parm;
    3276      2320046 :   validated_mem = validize_mem (copy_rtx (data->entry_parm));
    3277              : 
    3278      2320046 :   need_conversion = (data->nominal_mode != data->passed_mode
    3279      2320046 :                      || promoted_nominal_mode != data->arg.mode);
    3280         4171 :   moved = false;
    3281              : 
    3282              :   if (need_conversion
    3283         4171 :       && GET_MODE_CLASS (data->nominal_mode) == MODE_INT
    3284         3660 :       && data->nominal_mode == data->passed_mode
    3285            0 :       && data->nominal_mode == GET_MODE (data->entry_parm))
    3286              :     {
    3287              :       /* ENTRY_PARM has been converted to PROMOTED_MODE, its
    3288              :          mode, by the caller.  We now have to convert it to
    3289              :          NOMINAL_MODE, if different.  However, PARMREG may be in
    3290              :          a different mode than NOMINAL_MODE if it is being stored
    3291              :          promoted.
    3292              : 
    3293              :          If ENTRY_PARM is a hard register, it might be in a register
    3294              :          not valid for operating in its mode (e.g., an odd-numbered
    3295              :          register for a DFmode).  In that case, moves are the only
    3296              :          thing valid, so we can't do a convert from there.  This
    3297              :          occurs when the calling sequence allow such misaligned
    3298              :          usages.
    3299              : 
    3300              :          In addition, the conversion may involve a call, which could
    3301              :          clobber parameters which haven't been copied to pseudo
    3302              :          registers yet.
    3303              : 
    3304              :          First, we try to emit an insn which performs the necessary
    3305              :          conversion.  We verify that this insn does not clobber any
    3306              :          hard registers.  */
    3307              : 
    3308            0 :       rtx op0, op1;
    3309              : 
    3310            0 :       icode = can_extend_p (promoted_nominal_mode, data->passed_mode,
    3311              :                             unsignedp);
    3312              : 
    3313            0 :       op0 = parmreg;
    3314            0 :       op1 = validated_mem;
    3315            0 :       if (icode != CODE_FOR_nothing
    3316            0 :           && insn_operand_matches (icode, 0, op0)
    3317            0 :           && insn_operand_matches (icode, 1, op1))
    3318              :         {
    3319            0 :           enum rtx_code code = unsignedp ? ZERO_EXTEND : SIGN_EXTEND;
    3320            0 :           rtx_insn *insn, *insns;
    3321            0 :           rtx t = op1;
    3322            0 :           HARD_REG_SET hardregs;
    3323              : 
    3324            0 :           start_sequence ();
    3325              :           /* If op1 is a hard register that is likely spilled, first
    3326              :              force it into a pseudo, otherwise combiner might extend
    3327              :              its lifetime too much.  */
    3328            0 :           if (GET_CODE (t) == SUBREG)
    3329            0 :             t = SUBREG_REG (t);
    3330            0 :           if (REG_P (t)
    3331            0 :               && HARD_REGISTER_P (t)
    3332            0 :               && ! TEST_HARD_REG_BIT (fixed_reg_set, REGNO (t))
    3333            0 :               && targetm.class_likely_spilled_p (REGNO_REG_CLASS (REGNO (t))))
    3334              :             {
    3335            0 :               t = gen_reg_rtx (GET_MODE (op1));
    3336            0 :               emit_move_insn (t, op1);
    3337              :             }
    3338              :           else
    3339              :             t = op1;
    3340            0 :           rtx_insn *pat = gen_extend_insn (op0, t, promoted_nominal_mode,
    3341              :                                            data->passed_mode, unsignedp);
    3342            0 :           emit_insn (pat);
    3343            0 :           insns = get_insns ();
    3344              : 
    3345            0 :           moved = true;
    3346            0 :           CLEAR_HARD_REG_SET (hardregs);
    3347            0 :           for (insn = insns; insn && moved; insn = NEXT_INSN (insn))
    3348              :             {
    3349            0 :               if (INSN_P (insn))
    3350            0 :                 note_stores (insn, record_hard_reg_sets, &hardregs);
    3351            0 :               if (!hard_reg_set_empty_p (hardregs))
    3352            0 :                 moved = false;
    3353              :             }
    3354              : 
    3355            0 :           end_sequence ();
    3356              : 
    3357            0 :           if (moved)
    3358              :             {
    3359            0 :               emit_insn (insns);
    3360            0 :               if (equiv_stack_parm != NULL_RTX)
    3361            0 :                 equiv_stack_parm = gen_rtx_fmt_e (code, GET_MODE (parmreg),
    3362              :                                                   equiv_stack_parm);
    3363              :             }
    3364              :         }
    3365              :     }
    3366              : 
    3367            0 :   if (moved)
    3368              :     /* Nothing to do.  */
    3369              :     ;
    3370      2320046 :   else if (need_conversion)
    3371              :     {
    3372              :       /* We did not have an insn to convert directly, or the sequence
    3373              :          generated appeared unsafe.  We must first copy the parm to a
    3374              :          pseudo reg, and save the conversion until after all
    3375              :          parameters have been moved.  */
    3376              : 
    3377         4171 :       int save_tree_used;
    3378         4171 :       rtx tempreg = gen_reg_rtx (GET_MODE (data->entry_parm));
    3379              : 
    3380         4171 :       emit_move_insn (tempreg, validated_mem);
    3381              : 
    3382         4171 :       push_to_sequence2 (all->first_conversion_insn, all->last_conversion_insn);
    3383         4171 :       tempreg = convert_to_mode (data->nominal_mode, tempreg, unsignedp);
    3384              : 
    3385         4171 :       if (partial_subreg_p (tempreg)
    3386         3660 :           && GET_MODE (tempreg) == data->nominal_mode
    3387         3660 :           && REG_P (SUBREG_REG (tempreg))
    3388         3660 :           && data->nominal_mode == data->passed_mode
    3389         3660 :           && GET_MODE (SUBREG_REG (tempreg)) == GET_MODE (data->entry_parm))
    3390              :         {
    3391              :           /* The argument is already sign/zero extended, so note it
    3392              :              into the subreg.  */
    3393            0 :           SUBREG_PROMOTED_VAR_P (tempreg) = 1;
    3394            0 :           SUBREG_PROMOTED_SET (tempreg, unsignedp);
    3395              :         }
    3396              : 
    3397              :       /* TREE_USED gets set erroneously during expand_assignment.  */
    3398         4171 :       save_tree_used = TREE_USED (parm);
    3399         4171 :       SET_DECL_RTL (parm, rtl);
    3400         4171 :       expand_assignment (parm, make_tree (data->nominal_type, tempreg), false);
    3401         4171 :       SET_DECL_RTL (parm, NULL_RTX);
    3402         4171 :       TREE_USED (parm) = save_tree_used;
    3403         4171 :       all->first_conversion_insn = get_insns ();
    3404         4171 :       all->last_conversion_insn = get_last_insn ();
    3405         4171 :       end_sequence ();
    3406              : 
    3407         4171 :       did_conversion = true;
    3408              :     }
    3409      2315875 :   else if (MEM_P (data->entry_parm)
    3410       824413 :            && GET_MODE_ALIGNMENT (promoted_nominal_mode)
    3411       824471 :               > MEM_ALIGN (data->entry_parm)
    3412      2339787 :            && (((icode = optab_handler (movmisalign_optab,
    3413              :                                         promoted_nominal_mode))
    3414              :                 != CODE_FOR_nothing)
    3415        23908 :                || targetm.slow_unaligned_access (promoted_nominal_mode,
    3416        23966 :                                                  MEM_ALIGN (data->entry_parm))))
    3417              :     {
    3418            4 :       if (icode != CODE_FOR_nothing)
    3419            4 :         emit_insn (GEN_FCN (icode) (parmreg, validated_mem));
    3420              :       else
    3421            0 :         rtl = parmreg = extract_bit_field (validated_mem,
    3422            0 :                         GET_MODE_BITSIZE (promoted_nominal_mode), 0,
    3423              :                         unsignedp, parmreg,
    3424              :                         promoted_nominal_mode, VOIDmode, false, NULL);
    3425              :     }
    3426              :   else
    3427      2315871 :     emit_move_insn (parmreg, validated_mem);
    3428              : 
    3429              :   /* If we were passed a pointer but the actual value can live in a register,
    3430              :      retrieve it and use it directly.  Note that we cannot use nominal_mode,
    3431              :      because it will have been set to Pmode above, we must use the actual mode
    3432              :      of the parameter instead.  */
    3433      2320046 :   if (data->arg.pass_by_reference && TYPE_MODE (TREE_TYPE (parm)) != BLKmode)
    3434              :     {
    3435              :       /* Use a stack slot for debugging purposes if possible.  */
    3436          715 :       if (use_register_for_decl (parm))
    3437              :         {
    3438          377 :           parmreg = gen_reg_rtx (TYPE_MODE (TREE_TYPE (parm)));
    3439          377 :           mark_user_reg (parmreg);
    3440              :         }
    3441              :       else
    3442              :         {
    3443          338 :           int align = STACK_SLOT_ALIGNMENT (TREE_TYPE (parm),
    3444              :                                             TYPE_MODE (TREE_TYPE (parm)),
    3445              :                                             TYPE_ALIGN (TREE_TYPE (parm)));
    3446          338 :           parmreg
    3447          338 :             = assign_stack_local (TYPE_MODE (TREE_TYPE (parm)),
    3448          676 :                                   GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (parm))),
    3449              :                                   align);
    3450          338 :           set_mem_attributes (parmreg, parm, 1);
    3451              :         }
    3452              : 
    3453              :       /* We need to preserve an address based on VIRTUAL_STACK_VARS_REGNUM for
    3454              :          the debug info in case it is not legitimate.  */
    3455          715 :       if (GET_MODE (parmreg) != GET_MODE (rtl))
    3456              :         {
    3457            0 :           rtx tempreg = gen_reg_rtx (GET_MODE (rtl));
    3458            0 :           int unsigned_p = TYPE_UNSIGNED (TREE_TYPE (parm));
    3459              : 
    3460            0 :           push_to_sequence2 (all->first_conversion_insn,
    3461              :                              all->last_conversion_insn);
    3462            0 :           emit_move_insn (tempreg, rtl);
    3463            0 :           tempreg = convert_to_mode (GET_MODE (parmreg), tempreg, unsigned_p);
    3464            0 :           emit_move_insn (MEM_P (parmreg) ? copy_rtx (parmreg) : parmreg,
    3465              :                           tempreg);
    3466            0 :           all->first_conversion_insn = get_insns ();
    3467            0 :           all->last_conversion_insn = get_last_insn ();
    3468            0 :           end_sequence ();
    3469              : 
    3470            0 :           did_conversion = true;
    3471              :         }
    3472              :       else
    3473          715 :         emit_move_insn (MEM_P (parmreg) ? copy_rtx (parmreg) : parmreg, rtl);
    3474              : 
    3475          715 :       rtl = parmreg;
    3476              : 
    3477              :       /* STACK_PARM is the pointer, not the parm, and PARMREG is
    3478              :          now the parm.  */
    3479          715 :       data->stack_parm = NULL;
    3480              :     }
    3481              : 
    3482      2320046 :   set_parm_rtl (parm, rtl);
    3483              : 
    3484              :   /* Mark the register as eliminable if we did no conversion and it was
    3485              :      copied from memory at a fixed offset, and the arg pointer was not
    3486              :      copied to a pseudo-reg.  If the arg pointer is a pseudo reg or the
    3487              :      offset formed an invalid address, such memory-equivalences as we
    3488              :      make here would screw up life analysis for it.  */
    3489      2320046 :   if (data->nominal_mode == data->passed_mode
    3490      2315875 :       && !did_conversion
    3491      2315875 :       && data->stack_parm != 0
    3492       880015 :       && MEM_P (data->stack_parm)
    3493       880015 :       && data->locate.offset.var == 0
    3494      3200061 :       && reg_mentioned_p (virtual_incoming_args_rtx,
    3495       880015 :                           XEXP (data->stack_parm, 0)))
    3496              :     {
    3497       880015 :       rtx_insn *linsn = get_last_insn ();
    3498       880015 :       rtx_insn *sinsn;
    3499       880015 :       rtx set;
    3500              : 
    3501              :       /* Mark complex types separately.  */
    3502       880015 :       if (GET_CODE (parmreg) == CONCAT)
    3503              :         {
    3504         1071 :           scalar_mode submode = GET_MODE_INNER (GET_MODE (parmreg));
    3505         1071 :           int regnor = REGNO (XEXP (parmreg, 0));
    3506         1071 :           int regnoi = REGNO (XEXP (parmreg, 1));
    3507         1071 :           rtx stackr = adjust_address_nv (data->stack_parm, submode, 0);
    3508         2142 :           rtx stacki = adjust_address_nv (data->stack_parm, submode,
    3509              :                                           GET_MODE_SIZE (submode));
    3510              : 
    3511              :           /* Scan backwards for the set of the real and
    3512              :              imaginary parts.  */
    3513         5773 :           for (sinsn = linsn; sinsn != 0;
    3514         4702 :                sinsn = prev_nonnote_insn (sinsn))
    3515              :             {
    3516         4702 :               set = single_set (sinsn);
    3517         4702 :               if (set == 0)
    3518            0 :                 continue;
    3519              : 
    3520         4702 :               if (SET_DEST (set) == regno_reg_rtx [regnoi])
    3521         1071 :                 set_unique_reg_note (sinsn, REG_EQUIV, stacki);
    3522         3631 :               else if (SET_DEST (set) == regno_reg_rtx [regnor])
    3523         1071 :                 set_unique_reg_note (sinsn, REG_EQUIV, stackr);
    3524              :             }
    3525              :         }
    3526              :       else
    3527       878944 :         set_dst_reg_note (linsn, REG_EQUIV, equiv_stack_parm, parmreg);
    3528              :     }
    3529              : 
    3530              :   /* For pointer data type, suggest pointer register.  */
    3531      2320046 :   if (POINTER_TYPE_P (TREE_TYPE (parm)))
    3532       951940 :     mark_reg_pointer (parmreg,
    3533       951940 :                       TYPE_ALIGN (TREE_TYPE (TREE_TYPE (parm))));
    3534      2320046 : }
    3535              : 
    3536              : /* A subroutine of assign_parms.  Allocate stack space to hold the current
    3537              :    parameter.  Get it there.  Perform all ABI specified conversions.  */
    3538              : 
    3539              : static void
    3540       842767 : assign_parm_setup_stack (struct assign_parm_data_all *all, tree parm,
    3541              :                          struct assign_parm_data_one *data)
    3542              : {
    3543              :   /* Value must be stored in the stack slot STACK_PARM during function
    3544              :      execution.  */
    3545       842767 :   bool to_conversion = false;
    3546              : 
    3547       842767 :   assign_parm_remove_parallels (data);
    3548              : 
    3549       842767 :   if (data->arg.mode != data->nominal_mode)
    3550              :     {
    3551              :       /* Conversion is required.  */
    3552          835 :       rtx tempreg = gen_reg_rtx (GET_MODE (data->entry_parm));
    3553              : 
    3554          835 :       emit_move_insn (tempreg, validize_mem (copy_rtx (data->entry_parm)));
    3555              : 
    3556              :       /* Some ABIs require scalar floating point modes to be passed
    3557              :          in a wider scalar integer mode.  We need to explicitly
    3558              :          truncate to an integer mode of the correct precision before
    3559              :          using a SUBREG to reinterpret as a floating point value.  */
    3560          835 :       if (SCALAR_FLOAT_MODE_P (data->nominal_mode)
    3561          101 :           && SCALAR_INT_MODE_P (data->arg.mode)
    3562          835 :           && known_lt (GET_MODE_SIZE (data->nominal_mode),
    3563              :                        GET_MODE_SIZE (data->arg.mode)))
    3564            0 :         tempreg = convert_wider_int_to_float (data->nominal_mode,
    3565              :                                               data->arg.mode, tempreg);
    3566              : 
    3567          835 :       push_to_sequence2 (all->first_conversion_insn, all->last_conversion_insn);
    3568          835 :       to_conversion = true;
    3569              : 
    3570         2505 :       data->entry_parm = convert_to_mode (data->nominal_mode, tempreg,
    3571          835 :                                           TYPE_UNSIGNED (TREE_TYPE (parm)));
    3572              : 
    3573          835 :       if (data->stack_parm)
    3574              :         {
    3575            0 :           poly_int64 offset
    3576            0 :             = subreg_lowpart_offset (data->nominal_mode,
    3577            0 :                                      GET_MODE (data->stack_parm));
    3578              :           /* ??? This may need a big-endian conversion on sparc64.  */
    3579            0 :           data->stack_parm
    3580            0 :             = adjust_address (data->stack_parm, data->nominal_mode, 0);
    3581            0 :           if (maybe_ne (offset, 0) && MEM_OFFSET_KNOWN_P (data->stack_parm))
    3582            0 :             set_mem_offset (data->stack_parm,
    3583            0 :                             MEM_OFFSET (data->stack_parm) + offset);
    3584              :         }
    3585              :     }
    3586              : 
    3587       842767 :   if (data->entry_parm != data->stack_parm)
    3588              :     {
    3589       664623 :       rtx src, dest;
    3590              : 
    3591       664623 :       if (data->stack_parm == 0)
    3592              :         {
    3593       614836 :           int align = STACK_SLOT_ALIGNMENT (data->arg.type,
    3594              :                                             GET_MODE (data->entry_parm),
    3595              :                                             TYPE_ALIGN (data->arg.type));
    3596       614836 :           if (align < (int)GET_MODE_ALIGNMENT (GET_MODE (data->entry_parm))
    3597       617486 :               && ((optab_handler (movmisalign_optab,
    3598         2650 :                                   GET_MODE (data->entry_parm))
    3599              :                    != CODE_FOR_nothing)
    3600         2650 :                   || targetm.slow_unaligned_access (GET_MODE (data->entry_parm),
    3601              :                                                     align)))
    3602            0 :             align = GET_MODE_ALIGNMENT (GET_MODE (data->entry_parm));
    3603       614836 :           data->stack_parm
    3604       614836 :             = assign_stack_local (GET_MODE (data->entry_parm),
    3605      1229672 :                                   GET_MODE_SIZE (GET_MODE (data->entry_parm)),
    3606              :                                   align);
    3607       614836 :           align = MEM_ALIGN (data->stack_parm);
    3608       614836 :           set_mem_attributes (data->stack_parm, parm, 1);
    3609       614836 :           set_mem_align (data->stack_parm, align);
    3610              :         }
    3611              : 
    3612       664623 :       dest = validize_mem (copy_rtx (data->stack_parm));
    3613       664623 :       src = validize_mem (copy_rtx (data->entry_parm));
    3614              : 
    3615       664623 :       if (TYPE_EMPTY_P (data->arg.type))
    3616              :         /* Empty types don't really need to be copied.  */;
    3617       663074 :       else if (MEM_P (src))
    3618              :         {
    3619              :           /* Use a block move to handle potentially misaligned entry_parm.  */
    3620           63 :           if (!to_conversion)
    3621           63 :             push_to_sequence2 (all->first_conversion_insn,
    3622              :                                all->last_conversion_insn);
    3623           63 :           to_conversion = true;
    3624              : 
    3625           63 :           emit_block_move (dest, src,
    3626           63 :                            GEN_INT (int_size_in_bytes (data->arg.type)),
    3627              :                            BLOCK_OP_NORMAL);
    3628              :         }
    3629              :       else
    3630              :         {
    3631       663011 :           if (!REG_P (src))
    3632         1829 :             src = force_reg (GET_MODE (src), src);
    3633       663011 :           emit_move_insn (dest, src);
    3634              :         }
    3635              :     }
    3636              : 
    3637       842767 :   if (to_conversion)
    3638              :     {
    3639          898 :       all->first_conversion_insn = get_insns ();
    3640          898 :       all->last_conversion_insn = get_last_insn ();
    3641          898 :       end_sequence ();
    3642              :     }
    3643              : 
    3644       842767 :   set_parm_rtl (parm, data->stack_parm);
    3645       842767 : }
    3646              : 
    3647              : /* A subroutine of assign_parms.  If the ABI splits complex arguments, then
    3648              :    undo the frobbing that we did in assign_parms_augmented_arg_list.  */
    3649              : 
    3650              : static void
    3651            0 : assign_parms_unsplit_complex (struct assign_parm_data_all *all,
    3652              :                               vec<tree> fnargs)
    3653              : {
    3654            0 :   tree parm;
    3655            0 :   tree orig_fnargs = all->orig_fnargs;
    3656            0 :   unsigned i = 0;
    3657              : 
    3658            0 :   for (parm = orig_fnargs; parm; parm = TREE_CHAIN (parm), ++i)
    3659              :     {
    3660            0 :       if (TREE_CODE (TREE_TYPE (parm)) == COMPLEX_TYPE
    3661            0 :           && targetm.calls.split_complex_arg (TREE_TYPE (parm)))
    3662              :         {
    3663            0 :           rtx tmp, real, imag;
    3664            0 :           scalar_mode inner = GET_MODE_INNER (DECL_MODE (parm));
    3665              : 
    3666            0 :           real = DECL_RTL (fnargs[i]);
    3667            0 :           imag = DECL_RTL (fnargs[i + 1]);
    3668            0 :           if (inner != GET_MODE (real))
    3669              :             {
    3670            0 :               real = gen_lowpart_SUBREG (inner, real);
    3671            0 :               imag = gen_lowpart_SUBREG (inner, imag);
    3672              :             }
    3673              : 
    3674            0 :           if (TREE_ADDRESSABLE (parm))
    3675              :             {
    3676            0 :               rtx rmem, imem;
    3677            0 :               HOST_WIDE_INT size = int_size_in_bytes (TREE_TYPE (parm));
    3678            0 :               int align = STACK_SLOT_ALIGNMENT (TREE_TYPE (parm),
    3679              :                                                 DECL_MODE (parm),
    3680              :                                                 TYPE_ALIGN (TREE_TYPE (parm)));
    3681              : 
    3682              :               /* split_complex_arg put the real and imag parts in
    3683              :                  pseudos.  Move them to memory.  */
    3684            0 :               tmp = assign_stack_local (DECL_MODE (parm), size, align);
    3685            0 :               set_mem_attributes (tmp, parm, 1);
    3686            0 :               rmem = adjust_address_nv (tmp, inner, 0);
    3687            0 :               imem = adjust_address_nv (tmp, inner, GET_MODE_SIZE (inner));
    3688            0 :               push_to_sequence2 (all->first_conversion_insn,
    3689              :                                  all->last_conversion_insn);
    3690            0 :               emit_move_insn (rmem, real);
    3691            0 :               emit_move_insn (imem, imag);
    3692            0 :               all->first_conversion_insn = get_insns ();
    3693            0 :               all->last_conversion_insn = get_last_insn ();
    3694            0 :               end_sequence ();
    3695              :             }
    3696              :           else
    3697            0 :             tmp = gen_rtx_CONCAT (DECL_MODE (parm), real, imag);
    3698            0 :           set_parm_rtl (parm, tmp);
    3699              : 
    3700            0 :           real = DECL_INCOMING_RTL (fnargs[i]);
    3701            0 :           imag = DECL_INCOMING_RTL (fnargs[i + 1]);
    3702            0 :           if (inner != GET_MODE (real))
    3703              :             {
    3704            0 :               real = gen_lowpart_SUBREG (inner, real);
    3705            0 :               imag = gen_lowpart_SUBREG (inner, imag);
    3706              :             }
    3707            0 :           tmp = gen_rtx_CONCAT (DECL_MODE (parm), real, imag);
    3708            0 :           set_decl_incoming_rtl (parm, tmp, false);
    3709            0 :           i++;
    3710              :         }
    3711              :     }
    3712            0 : }
    3713              : 
    3714              : /* Assign RTL expressions to the function's parameters.  This may involve
    3715              :    copying them into registers and using those registers as the DECL_RTL.  */
    3716              : 
    3717              : static void
    3718      1517908 : assign_parms (tree fndecl)
    3719              : {
    3720      1517908 :   struct assign_parm_data_all all;
    3721      1517908 :   tree parm;
    3722      1517908 :   vec<tree> fnargs;
    3723      1517908 :   unsigned i;
    3724              : 
    3725      1517908 :   crtl->args.internal_arg_pointer
    3726      1517908 :     = targetm.calls.internal_arg_pointer ();
    3727              : 
    3728      1517908 :   assign_parms_initialize_all (&all);
    3729      1517908 :   fnargs = assign_parms_augmented_arg_list (&all);
    3730              : 
    3731      1517908 :   if (TYPE_NO_NAMED_ARGS_STDARG_P (TREE_TYPE (fndecl))
    3732      1517908 :       && fnargs.is_empty ())
    3733              :     {
    3734          114 :       struct assign_parm_data_one data = {};
    3735          114 :       assign_parms_setup_varargs (&all, &data, false);
    3736              :     }
    3737              : 
    3738      4755225 :   FOR_EACH_VEC_ELT (fnargs, i, parm)
    3739              :     {
    3740      3237317 :       struct assign_parm_data_one data;
    3741              : 
    3742              :       /* Extract the type of PARM; adjust it according to ABI.  */
    3743      3237317 :       assign_parm_find_data_types (&all, parm, &data);
    3744              : 
    3745              :       /* Early out for errors and void parameters.  */
    3746      3237317 :       if (data.passed_mode == VOIDmode)
    3747              :         {
    3748            0 :           SET_DECL_RTL (parm, const0_rtx);
    3749            0 :           DECL_INCOMING_RTL (parm) = DECL_RTL (parm);
    3750            0 :           continue;
    3751              :         }
    3752              : 
    3753              :       /* Estimate stack alignment from parameter alignment.  */
    3754      3237317 :       if (SUPPORTS_STACK_ALIGNMENT)
    3755              :         {
    3756      3237317 :           unsigned int align
    3757      6474634 :             = targetm.calls.function_arg_boundary (data.arg.mode,
    3758      3237317 :                                                    data.arg.type);
    3759      3237317 :           align = MINIMUM_ALIGNMENT (data.arg.type, data.arg.mode, align);
    3760      3237317 :           if (TYPE_ALIGN (data.nominal_type) > align)
    3761         4738 :             align = MINIMUM_ALIGNMENT (data.nominal_type,
    3762              :                                        TYPE_MODE (data.nominal_type),
    3763              :                                        TYPE_ALIGN (data.nominal_type));
    3764      3237317 :           if (crtl->stack_alignment_estimated < align)
    3765              :             {
    3766       367325 :               gcc_assert (!crtl->stack_realign_processed);
    3767       367325 :               crtl->stack_alignment_estimated = align;
    3768              :             }
    3769              :         }
    3770              : 
    3771              :       /* Find out where the parameter arrives in this function.  */
    3772      3237317 :       assign_parm_find_entry_rtl (&all, &data);
    3773              : 
    3774              :       /* Find out where stack space for this parameter might be.  */
    3775      3237317 :       if (assign_parm_is_stack_parm (&all, &data))
    3776              :         {
    3777      1183734 :           assign_parm_find_stack_rtl (parm, &data);
    3778      1183734 :           assign_parm_adjust_entry_rtl (&data);
    3779              :           /* For arguments that occupy no space in the parameter
    3780              :              passing area, have non-zero size and have address taken,
    3781              :              force creation of a stack slot so that they have distinct
    3782              :              address from other parameters.  */
    3783      1183734 :           if (TYPE_EMPTY_P (data.arg.type)
    3784         6899 :               && TREE_ADDRESSABLE (parm)
    3785         1710 :               && data.entry_parm == data.stack_parm
    3786         1710 :               && MEM_P (data.entry_parm)
    3787      1185444 :               && int_size_in_bytes (data.arg.type))
    3788         1542 :             data.stack_parm = NULL_RTX;
    3789              :         }
    3790              :       /* Record permanently how this parm was passed.  */
    3791      3237317 :       if (data.arg.pass_by_reference)
    3792              :         {
    3793         4966 :           rtx incoming_rtl
    3794         4966 :             = gen_rtx_MEM (TYPE_MODE (TREE_TYPE (data.arg.type)),
    3795              :                            data.entry_parm);
    3796         4966 :           set_decl_incoming_rtl (parm, incoming_rtl, true);
    3797              :         }
    3798              :       else
    3799      3232351 :         set_decl_incoming_rtl (parm, data.entry_parm, false);
    3800              : 
    3801      3237317 :       assign_parm_adjust_stack_rtl (parm, &data);
    3802              : 
    3803      3237317 :       if (assign_parm_setup_block_p (&data))
    3804        74504 :         assign_parm_setup_block (&all, parm, &data);
    3805      3162813 :       else if (data.arg.pass_by_reference || use_register_for_decl (parm))
    3806      2320046 :         assign_parm_setup_reg (&all, parm, &data);
    3807              :       else
    3808       842767 :         assign_parm_setup_stack (&all, parm, &data);
    3809              : 
    3810      3237317 :       if (cfun->stdarg && !DECL_CHAIN (parm))
    3811        21480 :         assign_parms_setup_varargs (&all, &data, false);
    3812              : 
    3813              :       /* Update info on where next arg arrives in registers.  */
    3814      3237317 :       targetm.calls.function_arg_advance (all.args_so_far, data.arg);
    3815              :     }
    3816              : 
    3817      1517908 :   if (targetm.calls.split_complex_arg)
    3818            0 :     assign_parms_unsplit_complex (&all, fnargs);
    3819              : 
    3820      1517908 :   fnargs.release ();
    3821              : 
    3822              :   /* Output all parameter conversion instructions (possibly including calls)
    3823              :      now that all parameters have been copied out of hard registers.  */
    3824      1517908 :   emit_insn (all.first_conversion_insn);
    3825              : 
    3826      1517908 :   do_pending_stack_adjust ();
    3827              : 
    3828              :   /* Estimate reload stack alignment from scalar return mode.  */
    3829      1517908 :   if (SUPPORTS_STACK_ALIGNMENT)
    3830              :     {
    3831      1517908 :       if (DECL_RESULT (fndecl))
    3832              :         {
    3833      1517908 :           tree type = TREE_TYPE (DECL_RESULT (fndecl));
    3834      1517908 :           machine_mode mode = TYPE_MODE (type);
    3835              : 
    3836      1517908 :           if (mode != BLKmode
    3837      1466177 :               && mode != VOIDmode
    3838       757393 :               && !AGGREGATE_TYPE_P (type))
    3839              :             {
    3840       694500 :               unsigned int align = GET_MODE_ALIGNMENT (mode);
    3841       694500 :               if (crtl->stack_alignment_estimated < align)
    3842              :                 {
    3843           10 :                   gcc_assert (!crtl->stack_realign_processed);
    3844           10 :                   crtl->stack_alignment_estimated = align;
    3845              :                 }
    3846              :             }
    3847              :         }
    3848              :     }
    3849              : 
    3850              :   /* If we are receiving a struct value address as the first argument, set up
    3851              :      the RTL for the function result. As this might require code to convert
    3852              :      the transmitted address to Pmode, we do this here to ensure that possible
    3853              :      preliminary conversions of the address have been emitted already.  */
    3854      1517908 :   if (all.function_result_decl)
    3855              :     {
    3856        70115 :       tree result = DECL_RESULT (current_function_decl);
    3857        70115 :       rtx addr = DECL_RTL (all.function_result_decl);
    3858        70115 :       rtx x;
    3859              : 
    3860        70115 :       if (DECL_BY_REFERENCE (result))
    3861              :         {
    3862         9038 :           SET_DECL_VALUE_EXPR (result, all.function_result_decl);
    3863         9038 :           x = addr;
    3864              :         }
    3865              :       else
    3866              :         {
    3867        61077 :           SET_DECL_VALUE_EXPR (result,
    3868              :                                build1 (INDIRECT_REF, TREE_TYPE (result),
    3869              :                                        all.function_result_decl));
    3870        87911 :           addr = convert_memory_address (Pmode, addr);
    3871        61077 :           x = gen_rtx_MEM (DECL_MODE (result), addr);
    3872        61077 :           set_mem_attributes (x, result, 1);
    3873              :         }
    3874              : 
    3875        70115 :       DECL_HAS_VALUE_EXPR_P (result) = 1;
    3876              : 
    3877        70115 :       set_parm_rtl (result, x);
    3878              :     }
    3879              : 
    3880              :   /* We have aligned all the args, so add space for the pretend args.  */
    3881      1517908 :   crtl->args.pretend_args_size = all.pretend_args_size;
    3882      1517908 :   all.stack_args_size.constant += all.extra_pretend_bytes;
    3883      1517908 :   crtl->args.size = all.stack_args_size.constant;
    3884              : 
    3885              :   /* Adjust function incoming argument size for alignment and
    3886              :      minimum length.  */
    3887              : 
    3888      1517908 :   crtl->args.size = upper_bound (crtl->args.size, all.reg_parm_stack_space);
    3889      3035816 :   crtl->args.size = aligned_upper_bound (crtl->args.size,
    3890      1517908 :                                          PARM_BOUNDARY / BITS_PER_UNIT);
    3891              : 
    3892      1517908 :   if (ARGS_GROW_DOWNWARD)
    3893              :     {
    3894              :       crtl->args.arg_offset_rtx
    3895              :         = (all.stack_args_size.var == 0
    3896              :            ? gen_int_mode (-all.stack_args_size.constant, Pmode)
    3897              :            : expand_expr (size_diffop (all.stack_args_size.var,
    3898              :                                        size_int (-all.stack_args_size.constant)),
    3899              :                           NULL_RTX, VOIDmode, EXPAND_NORMAL));
    3900              :     }
    3901              :   else
    3902      1644781 :     crtl->args.arg_offset_rtx = ARGS_SIZE_RTX (all.stack_args_size);
    3903              : 
    3904              :   /* See how many bytes, if any, of its args a function should try to pop
    3905              :      on return.  */
    3906              : 
    3907      1517908 :   crtl->args.pops_args = targetm.calls.return_pops_args (fndecl,
    3908      1517908 :                                                          TREE_TYPE (fndecl),
    3909              :                                                          crtl->args.size);
    3910              : 
    3911              :   /* For stdarg.h function, save info about
    3912              :      regs and stack space used by the named args.  */
    3913              : 
    3914      1517908 :   crtl->args.info = all.args_so_far_v;
    3915              : 
    3916              :   /* Set the rtx used for the function return value.  Put this in its
    3917              :      own variable so any optimizers that need this information don't have
    3918              :      to include tree.h.  Do this here so it gets done when an inlined
    3919              :      function gets output.  */
    3920              : 
    3921      1517908 :   crtl->return_rtx
    3922      1517908 :     = (DECL_RTL_SET_P (DECL_RESULT (fndecl))
    3923      2327032 :        ? DECL_RTL (DECL_RESULT (fndecl)) : NULL_RTX);
    3924              : 
    3925              :   /* If scalar return value was computed in a pseudo-reg, or was a named
    3926              :      return value that got dumped to the stack, copy that to the hard
    3927              :      return register.  */
    3928      1517908 :   if (DECL_RTL_SET_P (DECL_RESULT (fndecl)))
    3929              :     {
    3930       809124 :       tree decl_result = DECL_RESULT (fndecl);
    3931       809124 :       rtx decl_rtl = DECL_RTL (decl_result);
    3932              : 
    3933       809124 :       if (REG_P (decl_rtl)
    3934       809124 :           ? REGNO (decl_rtl) >= FIRST_PSEUDO_REGISTER
    3935        68059 :           : DECL_REGISTER (decl_result))
    3936              :         {
    3937       746089 :           rtx real_decl_rtl;
    3938              : 
    3939              :           /* Unless the psABI says not to.  */
    3940       746089 :           if (TYPE_EMPTY_P (TREE_TYPE (decl_result)))
    3941              :             real_decl_rtl = NULL_RTX;
    3942              :           else
    3943              :             {
    3944       740949 :               real_decl_rtl
    3945       740949 :                 = targetm.calls.function_value (TREE_TYPE (decl_result),
    3946              :                                                 fndecl, true);
    3947       740949 :               REG_FUNCTION_VALUE_P (real_decl_rtl) = 1;
    3948              :             }
    3949              :           /* The delay slot scheduler assumes that crtl->return_rtx
    3950              :              holds the hard register containing the return value, not a
    3951              :              temporary pseudo.  */
    3952       746089 :           crtl->return_rtx = real_decl_rtl;
    3953              :         }
    3954              :     }
    3955      1517908 : }
    3956              : 
    3957              : /* Gimplify the parameter list for current_function_decl.  This involves
    3958              :    evaluating SAVE_EXPRs of variable sized parameters and generating code
    3959              :    to implement callee-copies reference parameters.  Returns a sequence of
    3960              :    statements to add to the beginning of the function.  */
    3961              : 
    3962              : gimple_seq
    3963      3010285 : gimplify_parameters (gimple_seq *cleanup)
    3964              : {
    3965      3010285 :   struct assign_parm_data_all all;
    3966      3010285 :   tree parm;
    3967      3010285 :   gimple_seq stmts = NULL;
    3968      3010285 :   vec<tree> fnargs;
    3969      3010285 :   unsigned i;
    3970              : 
    3971      3010285 :   assign_parms_initialize_all (&all);
    3972      3010285 :   fnargs = assign_parms_augmented_arg_list (&all);
    3973              : 
    3974     12226346 :   FOR_EACH_VEC_ELT (fnargs, i, parm)
    3975              :     {
    3976      6205776 :       struct assign_parm_data_one data;
    3977              : 
    3978              :       /* Extract the type of PARM; adjust it according to ABI.  */
    3979      6205776 :       assign_parm_find_data_types (&all, parm, &data);
    3980              : 
    3981              :       /* Early out for errors and void parameters.  */
    3982      6205776 :       if (data.passed_mode == VOIDmode || DECL_SIZE (parm) == NULL)
    3983           93 :         continue;
    3984              : 
    3985              :       /* Update info on where next arg arrives in registers.  */
    3986      6205683 :       targetm.calls.function_arg_advance (all.args_so_far, data.arg);
    3987              : 
    3988              :       /* ??? Once upon a time variable_size stuffed parameter list
    3989              :          SAVE_EXPRs (amongst others) onto a pending sizes list.  This
    3990              :          turned out to be less than manageable in the gimple world.
    3991              :          Now we have to hunt them down ourselves.  */
    3992      6205683 :       gimplify_type_sizes (TREE_TYPE (parm), &stmts);
    3993              : 
    3994      6205683 :       if (TREE_CODE (DECL_SIZE_UNIT (parm)) != INTEGER_CST)
    3995              :         {
    3996           41 :           gimplify_one_sizepos (&DECL_SIZE (parm), &stmts);
    3997           41 :           gimplify_one_sizepos (&DECL_SIZE_UNIT (parm), &stmts);
    3998              :         }
    3999              : 
    4000      6205683 :       if (data.arg.pass_by_reference)
    4001              :         {
    4002         4974 :           tree type = TREE_TYPE (data.arg.type);
    4003         4974 :           function_arg_info orig_arg (type, data.arg.named);
    4004         4974 :           if (reference_callee_copied (&all.args_so_far_v, orig_arg))
    4005              :             {
    4006            0 :               tree local, t;
    4007              : 
    4008              :               /* For constant-sized objects, this is trivial; for
    4009              :                  variable-sized objects, we have to play games.  */
    4010            0 :               if (TREE_CODE (DECL_SIZE_UNIT (parm)) == INTEGER_CST
    4011            0 :                   && !(flag_stack_check == GENERIC_STACK_CHECK
    4012            0 :                        && compare_tree_int (DECL_SIZE_UNIT (parm),
    4013              :                                             STACK_CHECK_MAX_VAR_SIZE) > 0))
    4014              :                 {
    4015            0 :                   local = create_tmp_var (type, get_name (parm));
    4016            0 :                   DECL_IGNORED_P (local) = 0;
    4017              :                   /* If PARM was addressable, move that flag over
    4018              :                      to the local copy, as its address will be taken,
    4019              :                      not the PARMs.  Keep the parms address taken
    4020              :                      as we'll query that flag during gimplification.  */
    4021            0 :                   if (TREE_ADDRESSABLE (parm))
    4022            0 :                     TREE_ADDRESSABLE (local) = 1;
    4023            0 :                   if (DECL_NOT_GIMPLE_REG_P (parm))
    4024            0 :                     DECL_NOT_GIMPLE_REG_P (local) = 1;
    4025              : 
    4026            0 :                   if (!is_gimple_reg (local)
    4027            0 :                       && flag_stack_reuse != SR_NONE)
    4028              :                     {
    4029            0 :                       tree clobber = build_clobber (type);
    4030            0 :                       gimple *clobber_stmt;
    4031            0 :                       clobber_stmt = gimple_build_assign (local, clobber);
    4032            0 :                       gimple_seq_add_stmt (cleanup, clobber_stmt);
    4033              :                     }
    4034              :                 }
    4035              :               else
    4036              :                 {
    4037            0 :                   tree ptr_type, addr;
    4038              : 
    4039            0 :                   ptr_type = build_pointer_type (type);
    4040            0 :                   addr = create_tmp_reg (ptr_type, get_name (parm));
    4041            0 :                   DECL_IGNORED_P (addr) = 0;
    4042            0 :                   local = build_fold_indirect_ref (addr);
    4043              : 
    4044            0 :                   t = build_alloca_call_expr (DECL_SIZE_UNIT (parm),
    4045            0 :                                               DECL_ALIGN (parm),
    4046              :                                               max_int_size_in_bytes (type));
    4047              :                   /* The call has been built for a variable-sized object.  */
    4048            0 :                   CALL_ALLOCA_FOR_VAR_P (t) = 1;
    4049            0 :                   t = fold_convert (ptr_type, t);
    4050            0 :                   t = build2 (MODIFY_EXPR, TREE_TYPE (addr), addr, t);
    4051            0 :                   gimplify_and_add (t, &stmts);
    4052              :                 }
    4053              : 
    4054            0 :               gimplify_assign (local, parm, &stmts);
    4055              : 
    4056            0 :               SET_DECL_VALUE_EXPR (parm, local);
    4057            0 :               DECL_HAS_VALUE_EXPR_P (parm) = 1;
    4058              :             }
    4059              :         }
    4060              :     }
    4061              : 
    4062      3010285 :   fnargs.release ();
    4063              : 
    4064      3010285 :   return stmts;
    4065              : }
    4066              : 
    4067              : /* Compute the size and offset from the start of the stacked arguments for a
    4068              :    parm passed in mode PASSED_MODE and with type TYPE.
    4069              : 
    4070              :    INITIAL_OFFSET_PTR points to the current offset into the stacked
    4071              :    arguments.
    4072              : 
    4073              :    The starting offset and size for this parm are returned in
    4074              :    LOCATE->OFFSET and LOCATE->SIZE, respectively.  When IN_REGS is
    4075              :    nonzero, the offset is that of stack slot, which is returned in
    4076              :    LOCATE->SLOT_OFFSET.  LOCATE->ALIGNMENT_PAD is the amount of
    4077              :    padding required from the initial offset ptr to the stack slot.
    4078              : 
    4079              :    IN_REGS is nonzero if the argument will be passed in registers.  It will
    4080              :    never be set if REG_PARM_STACK_SPACE is not defined.
    4081              : 
    4082              :    REG_PARM_STACK_SPACE is the number of bytes of stack space reserved
    4083              :    for arguments which are passed in registers.
    4084              : 
    4085              :    FNDECL is the function in which the argument was defined.
    4086              : 
    4087              :    There are two types of rounding that are done.  The first, controlled by
    4088              :    TARGET_FUNCTION_ARG_BOUNDARY, forces the offset from the start of the
    4089              :    argument list to be aligned to the specific boundary (in bits).  This
    4090              :    rounding affects the initial and starting offsets, but not the argument
    4091              :    size.
    4092              : 
    4093              :    The second, controlled by TARGET_FUNCTION_ARG_PADDING and PARM_BOUNDARY,
    4094              :    optionally rounds the size of the parm to PARM_BOUNDARY.  The
    4095              :    initial offset is not affected by this rounding, while the size always
    4096              :    is and the starting offset may be.  */
    4097              : 
    4098              : /*  LOCATE->OFFSET will be negative for ARGS_GROW_DOWNWARD case;
    4099              :     INITIAL_OFFSET_PTR is positive because locate_and_pad_parm's
    4100              :     callers pass in the total size of args so far as
    4101              :     INITIAL_OFFSET_PTR.  LOCATE->SIZE is always positive.  */
    4102              : 
    4103              : void
    4104      5497310 : locate_and_pad_parm (machine_mode passed_mode, tree type, int in_regs,
    4105              :                      int reg_parm_stack_space, int partial,
    4106              :                      tree fndecl ATTRIBUTE_UNUSED,
    4107              :                      struct args_size *initial_offset_ptr,
    4108              :                      struct locate_and_pad_arg_data *locate)
    4109              : {
    4110      5497310 :   tree sizetree;
    4111      5497310 :   pad_direction where_pad;
    4112      5497310 :   unsigned int boundary, round_boundary;
    4113      5497310 :   int part_size_in_regs;
    4114              : 
    4115              :   /* If we have found a stack parm before we reach the end of the
    4116              :      area reserved for registers, skip that area.  */
    4117      5497310 :   if (! in_regs)
    4118              :     {
    4119      3267208 :       if (reg_parm_stack_space > 0)
    4120              :         {
    4121        83845 :           if (initial_offset_ptr->var
    4122        83845 :               || !ordered_p (initial_offset_ptr->constant,
    4123              :                              reg_parm_stack_space))
    4124              :             {
    4125            0 :               initial_offset_ptr->var
    4126            0 :                 = size_binop (MAX_EXPR, ARGS_SIZE_TREE (*initial_offset_ptr),
    4127              :                               ssize_int (reg_parm_stack_space));
    4128            0 :               initial_offset_ptr->constant = 0;
    4129              :             }
    4130              :           else
    4131        83845 :             initial_offset_ptr->constant
    4132        83845 :               = ordered_max (initial_offset_ptr->constant,
    4133              :                              reg_parm_stack_space);
    4134              :         }
    4135              :     }
    4136              : 
    4137      5497310 :   part_size_in_regs = (reg_parm_stack_space == 0 ? partial : 0);
    4138              : 
    4139      5497310 :   sizetree = (type
    4140      5497310 :               ? arg_size_in_bytes (type)
    4141        38021 :               : size_int (GET_MODE_SIZE (passed_mode)));
    4142      5497310 :   where_pad = targetm.calls.function_arg_padding (passed_mode, type);
    4143      5497310 :   boundary = targetm.calls.function_arg_boundary (passed_mode, type);
    4144      5497310 :   round_boundary = targetm.calls.function_arg_round_boundary (passed_mode,
    4145              :                                                               type);
    4146      5497310 :   locate->where_pad = where_pad;
    4147              : 
    4148              :   /* Alignment can't exceed MAX_SUPPORTED_STACK_ALIGNMENT.  */
    4149      5497310 :   if (boundary > MAX_SUPPORTED_STACK_ALIGNMENT)
    4150              :     boundary = MAX_SUPPORTED_STACK_ALIGNMENT;
    4151              : 
    4152      5497310 :   locate->boundary = boundary;
    4153              : 
    4154      5497310 :   if (SUPPORTS_STACK_ALIGNMENT)
    4155              :     {
    4156              :       /* stack_alignment_estimated can't change after stack has been
    4157              :          realigned.  */
    4158      5497310 :       if (crtl->stack_alignment_estimated < boundary)
    4159              :         {
    4160         5129 :           if (!crtl->stack_realign_processed)
    4161         5129 :             crtl->stack_alignment_estimated = boundary;
    4162              :           else
    4163              :             {
    4164              :               /* If stack is realigned and stack alignment value
    4165              :                  hasn't been finalized, it is OK not to increase
    4166              :                  stack_alignment_estimated.  The bigger alignment
    4167              :                  requirement is recorded in stack_alignment_needed
    4168              :                  below.  */
    4169            0 :               gcc_assert (!crtl->stack_realign_finalized
    4170              :                           && crtl->stack_realign_needed);
    4171              :             }
    4172              :         }
    4173              :     }
    4174              : 
    4175      5497310 :   if (ARGS_GROW_DOWNWARD)
    4176              :     {
    4177              :       locate->slot_offset.constant = -initial_offset_ptr->constant;
    4178              :       if (initial_offset_ptr->var)
    4179              :         locate->slot_offset.var = size_binop (MINUS_EXPR, ssize_int (0),
    4180              :                                               initial_offset_ptr->var);
    4181              : 
    4182              :       {
    4183              :         tree s2 = sizetree;
    4184              :         if (where_pad != PAD_NONE
    4185              :             && (!tree_fits_uhwi_p (sizetree)
    4186              :                 || (tree_to_uhwi (sizetree) * BITS_PER_UNIT) % round_boundary))
    4187              :           s2 = round_up (s2, round_boundary / BITS_PER_UNIT);
    4188              :         SUB_PARM_SIZE (locate->slot_offset, s2);
    4189              :       }
    4190              : 
    4191              :       locate->slot_offset.constant += part_size_in_regs;
    4192              : 
    4193              :       if (!in_regs || reg_parm_stack_space > 0)
    4194              :         pad_to_arg_alignment (&locate->slot_offset, boundary,
    4195              :                               &locate->alignment_pad);
    4196              : 
    4197              :       locate->size.constant = (-initial_offset_ptr->constant
    4198              :                                - locate->slot_offset.constant);
    4199              :       if (initial_offset_ptr->var)
    4200              :         locate->size.var = size_binop (MINUS_EXPR,
    4201              :                                        size_binop (MINUS_EXPR,
    4202              :                                                    ssize_int (0),
    4203              :                                                    initial_offset_ptr->var),
    4204              :                                        locate->slot_offset.var);
    4205              : 
    4206              :       /* Pad_below needs the pre-rounded size to know how much to pad
    4207              :          below.  */
    4208              :       locate->offset = locate->slot_offset;
    4209              :       if (where_pad == PAD_DOWNWARD)
    4210              :         pad_below (&locate->offset, passed_mode, sizetree);
    4211              : 
    4212              :     }
    4213              :   else
    4214              :     {
    4215      5497310 :       if (!in_regs || reg_parm_stack_space > 0)
    4216      3443727 :         pad_to_arg_alignment (initial_offset_ptr, boundary,
    4217              :                               &locate->alignment_pad);
    4218      5497310 :       locate->slot_offset = *initial_offset_ptr;
    4219              : 
    4220              : #ifdef PUSH_ROUNDING
    4221      5497310 :       if (passed_mode != BLKmode)
    4222      5151416 :         sizetree = size_int (PUSH_ROUNDING (TREE_INT_CST_LOW (sizetree)));
    4223              : #endif
    4224              : 
    4225              :       /* Pad_below needs the pre-rounded size to know how much to pad below
    4226              :          so this must be done before rounding up.  */
    4227      5497310 :       locate->offset = locate->slot_offset;
    4228      5497310 :       if (where_pad == PAD_DOWNWARD)
    4229            0 :         pad_below (&locate->offset, passed_mode, sizetree);
    4230              : 
    4231      5497310 :       if (where_pad != PAD_NONE
    4232      5497310 :           && (!tree_fits_uhwi_p (sizetree)
    4233      5497310 :               || (tree_to_uhwi (sizetree) * BITS_PER_UNIT) % round_boundary))
    4234        18581 :         sizetree = round_up (sizetree, round_boundary / BITS_PER_UNIT);
    4235              : 
    4236      5497310 :       ADD_PARM_SIZE (locate->size, sizetree);
    4237              : 
    4238      5497310 :       locate->size.constant -= part_size_in_regs;
    4239              :     }
    4240              : 
    4241      5497310 :   locate->offset.constant
    4242      5497310 :     += targetm.calls.function_arg_offset (passed_mode, type);
    4243      5497310 : }
    4244              : 
    4245              : /* Round the stack offset in *OFFSET_PTR up to a multiple of BOUNDARY.
    4246              :    BOUNDARY is measured in bits, but must be a multiple of a storage unit.  */
    4247              : 
    4248              : static void
    4249      3443727 : pad_to_arg_alignment (struct args_size *offset_ptr, int boundary,
    4250              :                       struct args_size *alignment_pad)
    4251              : {
    4252      3443727 :   tree save_var = NULL_TREE;
    4253      3443727 :   poly_int64 save_constant = 0;
    4254      3443727 :   int boundary_in_bytes = boundary / BITS_PER_UNIT;
    4255      3443727 :   poly_int64 sp_offset = STACK_POINTER_OFFSET;
    4256              : 
    4257              : #ifdef SPARC_STACK_BOUNDARY_HACK
    4258              :   /* ??? The SPARC port may claim a STACK_BOUNDARY higher than
    4259              :      the real alignment of %sp.  However, when it does this, the
    4260              :      alignment of %sp+STACK_POINTER_OFFSET is STACK_BOUNDARY.  */
    4261              :   if (SPARC_STACK_BOUNDARY_HACK)
    4262              :     sp_offset = 0;
    4263              : #endif
    4264              : 
    4265      5208990 :   if (boundary > PARM_BOUNDARY)
    4266              :     {
    4267       141905 :       save_var = offset_ptr->var;
    4268       141905 :       save_constant = offset_ptr->constant;
    4269              :     }
    4270              : 
    4271      3443727 :   alignment_pad->var = NULL_TREE;
    4272      3443727 :   alignment_pad->constant = 0;
    4273              : 
    4274      3443727 :   if (boundary > BITS_PER_UNIT)
    4275              :     {
    4276      3443727 :       int misalign;
    4277      3443727 :       if (offset_ptr->var
    4278      3443727 :           || !known_misalignment (offset_ptr->constant + sp_offset,
    4279              :                                   boundary_in_bytes, &misalign))
    4280              :         {
    4281            0 :           tree sp_offset_tree = ssize_int (sp_offset);
    4282            0 :           tree offset = size_binop (PLUS_EXPR,
    4283              :                                     ARGS_SIZE_TREE (*offset_ptr),
    4284              :                                     sp_offset_tree);
    4285            0 :           tree rounded;
    4286            0 :           if (ARGS_GROW_DOWNWARD)
    4287              :             rounded = round_down (offset, boundary / BITS_PER_UNIT);
    4288              :           else
    4289            0 :             rounded = round_up   (offset, boundary / BITS_PER_UNIT);
    4290              : 
    4291            0 :           offset_ptr->var = size_binop (MINUS_EXPR, rounded, sp_offset_tree);
    4292              :           /* ARGS_SIZE_TREE includes constant term.  */
    4293            0 :           offset_ptr->constant = 0;
    4294            0 :           if (boundary > PARM_BOUNDARY)
    4295            0 :             alignment_pad->var = size_binop (MINUS_EXPR, offset_ptr->var,
    4296              :                                              save_var);
    4297              :         }
    4298              :       else
    4299              :         {
    4300      3443727 :           if (ARGS_GROW_DOWNWARD)
    4301              :             offset_ptr->constant -= misalign;
    4302              :           else
    4303      3443727 :             offset_ptr->constant += -misalign & (boundary_in_bytes - 1);
    4304              : 
    4305      3443727 :           if (boundary > PARM_BOUNDARY)
    4306       141905 :             alignment_pad->constant = offset_ptr->constant - save_constant;
    4307              :         }
    4308              :     }
    4309      3443727 : }
    4310              : 
    4311              : static void
    4312            0 : pad_below (struct args_size *offset_ptr, machine_mode passed_mode, tree sizetree)
    4313              : {
    4314            0 :   unsigned int align = PARM_BOUNDARY / BITS_PER_UNIT;
    4315            0 :   int misalign;
    4316            0 :   if (passed_mode != BLKmode
    4317            0 :       && known_misalignment (GET_MODE_SIZE (passed_mode), align, &misalign))
    4318            0 :     offset_ptr->constant += -misalign & (align - 1);
    4319              :   else
    4320              :     {
    4321            0 :       if (TREE_CODE (sizetree) != INTEGER_CST
    4322            0 :           || (TREE_INT_CST_LOW (sizetree) & (align - 1)) != 0)
    4323              :         {
    4324              :           /* Round the size up to multiple of PARM_BOUNDARY bits.  */
    4325            0 :           tree s2 = round_up (sizetree, align);
    4326              :           /* Add it in.  */
    4327            0 :           ADD_PARM_SIZE (*offset_ptr, s2);
    4328            0 :           SUB_PARM_SIZE (*offset_ptr, sizetree);
    4329              :         }
    4330              :     }
    4331            0 : }
    4332              : 
    4333              : 
    4334              : /* True if register REGNO was alive at a place where `setjmp' was
    4335              :    called and was set more than once or is an argument.  Such regs may
    4336              :    be clobbered by `longjmp'.  */
    4337              : 
    4338              : static bool
    4339           44 : regno_clobbered_at_setjmp (bitmap setjmp_crosses, int regno)
    4340              : {
    4341              :   /* There appear to be cases where some local vars never reach the
    4342              :      backend but have bogus regnos.  */
    4343           44 :   if (regno >= max_reg_num ())
    4344              :     return false;
    4345              : 
    4346           44 :   return ((REG_N_SETS (regno) > 1
    4347           42 :            || REGNO_REG_SET_P (df_get_live_out (ENTRY_BLOCK_PTR_FOR_FN (cfun)),
    4348              :                                regno))
    4349           44 :           && REGNO_REG_SET_P (setjmp_crosses, regno));
    4350              : }
    4351              : 
    4352              : /* Walk the tree of blocks describing the binding levels within a
    4353              :    function and warn about variables the might be killed by setjmp or
    4354              :    vfork.  This is done after calling flow_analysis before register
    4355              :    allocation since that will clobber the pseudo-regs to hard
    4356              :    regs.  */
    4357              : 
    4358              : static void
    4359           83 : setjmp_vars_warning (bitmap setjmp_crosses, tree block)
    4360              : {
    4361           83 :   tree decl, sub;
    4362              : 
    4363          212 :   for (decl = BLOCK_VARS (block); decl; decl = DECL_CHAIN (decl))
    4364              :     {
    4365          129 :       if (VAR_P (decl)
    4366          129 :           && DECL_RTL_SET_P (decl)
    4367           35 :           && REG_P (DECL_RTL (decl))
    4368          147 :           && regno_clobbered_at_setjmp (setjmp_crosses, REGNO (DECL_RTL (decl))))
    4369            1 :         warning (OPT_Wclobbered, "variable %q+D might be clobbered by"
    4370              :                  " %<longjmp%> or %<vfork%>", decl);
    4371              :     }
    4372              : 
    4373          142 :   for (sub = BLOCK_SUBBLOCKS (block); sub; sub = BLOCK_CHAIN (sub))
    4374           59 :     setjmp_vars_warning (setjmp_crosses, sub);
    4375           83 : }
    4376              : 
    4377              : /* Do the appropriate part of setjmp_vars_warning
    4378              :    but for arguments instead of local variables.  */
    4379              : 
    4380              : static void
    4381           24 : setjmp_args_warning (bitmap setjmp_crosses)
    4382              : {
    4383           24 :   tree decl;
    4384           24 :   for (decl = DECL_ARGUMENTS (current_function_decl);
    4385           50 :        decl; decl = DECL_CHAIN (decl))
    4386           26 :     if (DECL_RTL (decl) != 0
    4387           26 :         && REG_P (DECL_RTL (decl))
    4388           52 :         && regno_clobbered_at_setjmp (setjmp_crosses, REGNO (DECL_RTL (decl))))
    4389            0 :       warning (OPT_Wclobbered,
    4390              :                "argument %q+D might be clobbered by %<longjmp%> or %<vfork%>",
    4391              :                decl);
    4392           24 : }
    4393              : 
    4394              : /* Generate warning messages for variables live across setjmp.  */
    4395              : 
    4396              : void
    4397       135555 : generate_setjmp_warnings (void)
    4398              : {
    4399       135555 :   bitmap setjmp_crosses = regstat_get_setjmp_crosses ();
    4400              : 
    4401       135555 :   if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS
    4402       135555 :       || bitmap_empty_p (setjmp_crosses))
    4403              :     return;
    4404              : 
    4405           24 :   setjmp_vars_warning (setjmp_crosses, DECL_INITIAL (current_function_decl));
    4406           24 :   setjmp_args_warning (setjmp_crosses);
    4407              : }
    4408              : 
    4409              : 
    4410              : /* Reverse the order of elements in the fragment chain T of blocks,
    4411              :    and return the new head of the chain (old last element).
    4412              :    In addition to that clear BLOCK_SAME_RANGE flags when needed
    4413              :    and adjust BLOCK_SUPERCONTEXT from the super fragment to
    4414              :    its super fragment origin.  */
    4415              : 
    4416              : static tree
    4417      5852874 : block_fragments_nreverse (tree t)
    4418              : {
    4419      5852874 :   tree prev = 0, block, next, prev_super = 0;
    4420      5852874 :   tree super = BLOCK_SUPERCONTEXT (t);
    4421      5852874 :   if (BLOCK_FRAGMENT_ORIGIN (super))
    4422      4859746 :     super = BLOCK_FRAGMENT_ORIGIN (super);
    4423     17532473 :   for (block = t; block; block = next)
    4424              :     {
    4425     11679599 :       next = BLOCK_FRAGMENT_CHAIN (block);
    4426     11679599 :       BLOCK_FRAGMENT_CHAIN (block) = prev;
    4427      5826725 :       if ((prev && !BLOCK_SAME_RANGE (prev))
    4428     15035985 :           || (BLOCK_FRAGMENT_CHAIN (BLOCK_SUPERCONTEXT (block))
    4429              :               != prev_super))
    4430      3394951 :         BLOCK_SAME_RANGE (block) = 0;
    4431     11679599 :       prev_super = BLOCK_SUPERCONTEXT (block);
    4432     11679599 :       BLOCK_SUPERCONTEXT (block) = super;
    4433     11679599 :       prev = block;
    4434              :     }
    4435      5852874 :   t = BLOCK_FRAGMENT_ORIGIN (t);
    4436      5852874 :   if (BLOCK_FRAGMENT_CHAIN (BLOCK_SUPERCONTEXT (t))
    4437              :       != prev_super)
    4438      1860933 :     BLOCK_SAME_RANGE (t) = 0;
    4439      5852874 :   BLOCK_SUPERCONTEXT (t) = super;
    4440      5852874 :   return prev;
    4441              : }
    4442              : 
    4443              : /* Reverse the order of elements in the chain T of blocks,
    4444              :    and return the new head of the chain (old last element).
    4445              :    Also do the same on subblocks and reverse the order of elements
    4446              :    in BLOCK_FRAGMENT_CHAIN as well.  */
    4447              : 
    4448              : static tree
    4449     25065244 : blocks_nreverse_all (tree t)
    4450              : {
    4451     25065244 :   tree prev = 0, block, next;
    4452     49551921 :   for (block = t; block; block = next)
    4453              :     {
    4454     24486677 :       next = BLOCK_CHAIN (block);
    4455     24486677 :       BLOCK_CHAIN (block) = prev;
    4456     24486677 :       if (BLOCK_FRAGMENT_CHAIN (block)
    4457     24486677 :           && BLOCK_FRAGMENT_ORIGIN (block) == NULL_TREE)
    4458              :         {
    4459     11705748 :           BLOCK_FRAGMENT_CHAIN (block)
    4460      5852874 :             = block_fragments_nreverse (BLOCK_FRAGMENT_CHAIN (block));
    4461      5852874 :           if (!BLOCK_SAME_RANGE (BLOCK_FRAGMENT_CHAIN (block)))
    4462      2787668 :             BLOCK_SAME_RANGE (block) = 0;
    4463              :         }
    4464     24486677 :       BLOCK_SUBBLOCKS (block) = blocks_nreverse_all (BLOCK_SUBBLOCKS (block));
    4465     24486677 :       prev = block;
    4466              :     }
    4467     25065244 :   return prev;
    4468              : }
    4469              : 
    4470              : 
    4471              : /* Identify BLOCKs referenced by more than one NOTE_INSN_BLOCK_{BEG,END},
    4472              :    and create duplicate blocks.  */
    4473              : /* ??? Need an option to either create block fragments or to create
    4474              :    abstract origin duplicates of a source block.  It really depends
    4475              :    on what optimization has been performed.  */
    4476              : 
    4477              : void
    4478       578567 : reorder_blocks (void)
    4479              : {
    4480       578567 :   tree block = DECL_INITIAL (current_function_decl);
    4481              : 
    4482       578567 :   if (block == NULL_TREE)
    4483            0 :     return;
    4484              : 
    4485       578567 :   auto_vec<tree, 10> block_stack;
    4486              : 
    4487              :   /* Reset the TREE_ASM_WRITTEN bit for all blocks.  */
    4488       578567 :   clear_block_marks (block);
    4489              : 
    4490              :   /* Prune the old trees away, so that they don't get in the way.  */
    4491       578567 :   BLOCK_SUBBLOCKS (block) = NULL_TREE;
    4492       578567 :   BLOCK_CHAIN (block) = NULL_TREE;
    4493              : 
    4494              :   /* Recreate the block tree from the note nesting.  */
    4495       578567 :   reorder_blocks_1 (get_insns (), block, &block_stack);
    4496       578567 :   BLOCK_SUBBLOCKS (block) = blocks_nreverse_all (BLOCK_SUBBLOCKS (block));
    4497       578567 : }
    4498              : 
    4499              : /* Helper function for reorder_blocks.  Reset TREE_ASM_WRITTEN.  */
    4500              : 
    4501              : void
    4502     25617660 : clear_block_marks (tree block)
    4503              : {
    4504     47646600 :   while (block)
    4505              :     {
    4506     22028940 :       TREE_ASM_WRITTEN (block) = 0;
    4507     22028940 :       clear_block_marks (BLOCK_SUBBLOCKS (block));
    4508     22028940 :       block = BLOCK_CHAIN (block);
    4509              :     }
    4510     25617660 : }
    4511              : 
    4512              : static void
    4513       578567 : reorder_blocks_1 (rtx_insn *insns, tree current_block,
    4514              :                   vec<tree> *p_block_stack)
    4515              : {
    4516       578567 :   rtx_insn *insn;
    4517       578567 :   tree prev_beg = NULL_TREE, prev_end = NULL_TREE;
    4518              : 
    4519    203905199 :   for (insn = insns; insn; insn = NEXT_INSN (insn))
    4520              :     {
    4521    203326632 :       if (NOTE_P (insn))
    4522              :         {
    4523    149244059 :           if (NOTE_KIND (insn) == NOTE_INSN_BLOCK_BEG)
    4524              :             {
    4525     24486677 :               tree block = NOTE_BLOCK (insn);
    4526     24486677 :               tree origin;
    4527              : 
    4528     24486677 :               gcc_assert (BLOCK_FRAGMENT_ORIGIN (block) == NULL_TREE);
    4529     24486677 :               origin = block;
    4530              : 
    4531     24486677 :               if (prev_end)
    4532      1518512 :                 BLOCK_SAME_RANGE (prev_end) = 0;
    4533     24486677 :               prev_end = NULL_TREE;
    4534              : 
    4535              :               /* If we have seen this block before, that means it now
    4536              :                  spans multiple address regions.  Create a new fragment.  */
    4537     24486677 :               if (TREE_ASM_WRITTEN (block))
    4538              :                 {
    4539     11679599 :                   tree new_block = copy_node (block);
    4540              : 
    4541     11679599 :                   BLOCK_SAME_RANGE (new_block) = 0;
    4542     11679599 :                   BLOCK_FRAGMENT_ORIGIN (new_block) = origin;
    4543     11679599 :                   BLOCK_FRAGMENT_CHAIN (new_block)
    4544     11679599 :                     = BLOCK_FRAGMENT_CHAIN (origin);
    4545     11679599 :                   BLOCK_FRAGMENT_CHAIN (origin) = new_block;
    4546              : 
    4547     11679599 :                   NOTE_BLOCK (insn) = new_block;
    4548     11679599 :                   block = new_block;
    4549              :                 }
    4550              : 
    4551     24486677 :               if (prev_beg == current_block && prev_beg)
    4552     16061104 :                 BLOCK_SAME_RANGE (block) = 1;
    4553              : 
    4554     24486677 :               prev_beg = origin;
    4555              : 
    4556     24486677 :               BLOCK_SUBBLOCKS (block) = 0;
    4557     24486677 :               TREE_ASM_WRITTEN (block) = 1;
    4558              :               /* When there's only one block for the entire function,
    4559              :                  current_block == block and we mustn't do this, it
    4560              :                  will cause infinite recursion.  */
    4561     24486677 :               if (block != current_block)
    4562              :                 {
    4563     24486677 :                   tree super;
    4564     24486677 :                   if (block != origin)
    4565     11679599 :                     gcc_assert (BLOCK_SUPERCONTEXT (origin) == current_block
    4566              :                                 || BLOCK_FRAGMENT_ORIGIN (BLOCK_SUPERCONTEXT
    4567              :                                                                       (origin))
    4568              :                                    == current_block);
    4569     24486677 :                   if (p_block_stack->is_empty ())
    4570              :                     super = current_block;
    4571              :                   else
    4572              :                     {
    4573     22109707 :                       super = p_block_stack->last ();
    4574     34013298 :                       gcc_assert (super == current_block
    4575              :                                   || BLOCK_FRAGMENT_ORIGIN (super)
    4576              :                                      == current_block);
    4577              :                     }
    4578     24486677 :                   BLOCK_SUPERCONTEXT (block) = super;
    4579     24486677 :                   BLOCK_CHAIN (block) = BLOCK_SUBBLOCKS (current_block);
    4580     24486677 :                   BLOCK_SUBBLOCKS (current_block) = block;
    4581     24486677 :                   current_block = origin;
    4582              :                 }
    4583     24486677 :               p_block_stack->safe_push (block);
    4584              :             }
    4585    124757382 :           else if (NOTE_KIND (insn) == NOTE_INSN_BLOCK_END)
    4586              :             {
    4587     24486677 :               NOTE_BLOCK (insn) = p_block_stack->pop ();
    4588     24486677 :               current_block = BLOCK_SUPERCONTEXT (current_block);
    4589     24486677 :               if (BLOCK_FRAGMENT_ORIGIN (current_block))
    4590      3936189 :                 current_block = BLOCK_FRAGMENT_ORIGIN (current_block);
    4591     24486677 :               prev_beg = NULL_TREE;
    4592     40547781 :               prev_end = BLOCK_SAME_RANGE (NOTE_BLOCK (insn))
    4593     24486677 :                          ? NOTE_BLOCK (insn) : NULL_TREE;
    4594              :             }
    4595              :         }
    4596              :       else
    4597              :         {
    4598     54082573 :           prev_beg = NULL_TREE;
    4599     54082573 :           if (prev_end)
    4600       769155 :             BLOCK_SAME_RANGE (prev_end) = 0;
    4601              :           prev_end = NULL_TREE;
    4602              :         }
    4603              :     }
    4604       578567 : }
    4605              : 
    4606              : /* Reverse the order of elements in the chain T of blocks,
    4607              :    and return the new head of the chain (old last element).  */
    4608              : 
    4609              : tree
    4610     34093565 : blocks_nreverse (tree t)
    4611              : {
    4612     34093565 :   tree prev = 0, block, next;
    4613     60304245 :   for (block = t; block; block = next)
    4614              :     {
    4615     26210680 :       next = BLOCK_CHAIN (block);
    4616     26210680 :       BLOCK_CHAIN (block) = prev;
    4617     26210680 :       prev = block;
    4618              :     }
    4619     34093565 :   return prev;
    4620              : }
    4621              : 
    4622              : /* Concatenate two chains of blocks (chained through BLOCK_CHAIN)
    4623              :    by modifying the last node in chain 1 to point to chain 2.  */
    4624              : 
    4625              : tree
    4626     87363932 : block_chainon (tree op1, tree op2)
    4627              : {
    4628     87363932 :   tree t1;
    4629              : 
    4630     87363932 :   if (!op1)
    4631              :     return op2;
    4632      4395287 :   if (!op2)
    4633              :     return op1;
    4634              : 
    4635     26134750 :   for (t1 = op1; BLOCK_CHAIN (t1); t1 = BLOCK_CHAIN (t1))
    4636     21739463 :     continue;
    4637      4395287 :   BLOCK_CHAIN (t1) = op2;
    4638              : 
    4639              : #ifdef ENABLE_TREE_CHECKING
    4640      4395287 :   {
    4641      4395287 :     tree t2;
    4642      8806957 :     for (t2 = op2; t2; t2 = BLOCK_CHAIN (t2))
    4643      4411670 :       gcc_assert (t2 != t1);
    4644              :   }
    4645              : #endif
    4646              : 
    4647              :   return op1;
    4648     21739463 : }
    4649              : 
    4650              : /* Count the subblocks of the list starting with BLOCK.  If VECTOR is
    4651              :    non-NULL, list them all into VECTOR, in a depth-first preorder
    4652              :    traversal of the block tree.  Also clear TREE_ASM_WRITTEN in all
    4653              :    blocks.  */
    4654              : 
    4655              : static int
    4656    132801624 : all_blocks (tree block, tree *vector)
    4657              : {
    4658    132801624 :   int n_blocks = 0;
    4659              : 
    4660    261106790 :   while (block)
    4661              :     {
    4662    128305166 :       TREE_ASM_WRITTEN (block) = 0;
    4663              : 
    4664              :       /* Record this block.  */
    4665    128305166 :       if (vector)
    4666     64152583 :         vector[n_blocks] = block;
    4667              : 
    4668    128305166 :       ++n_blocks;
    4669              : 
    4670              :       /* Record the subblocks, and their subblocks...  */
    4671    192457749 :       n_blocks += all_blocks (BLOCK_SUBBLOCKS (block),
    4672     64152583 :                               vector ? vector + n_blocks : 0);
    4673    128305166 :       block = BLOCK_CHAIN (block);
    4674              :     }
    4675              : 
    4676    132801624 :   return n_blocks;
    4677              : }
    4678              : 
    4679              : /* Return a vector containing all the blocks rooted at BLOCK.  The
    4680              :    number of elements in the vector is stored in N_BLOCKS_P.  The
    4681              :    vector is dynamically allocated; it is the caller's responsibility
    4682              :    to call `free' on the pointer returned.  */
    4683              : 
    4684              : static tree *
    4685      2248229 : get_block_vector (tree block, int *n_blocks_p)
    4686              : {
    4687      2248229 :   tree *block_vector;
    4688              : 
    4689      2248229 :   *n_blocks_p = all_blocks (block, NULL);
    4690      2248229 :   block_vector = XNEWVEC (tree, *n_blocks_p);
    4691      2248229 :   all_blocks (block, block_vector);
    4692              : 
    4693      2248229 :   return block_vector;
    4694              : }
    4695              : 
    4696              : static GTY(()) int next_block_index = 2;
    4697              : 
    4698              : /* Set BLOCK_NUMBER for all the blocks in FN.  */
    4699              : 
    4700              : void
    4701      2248229 : number_blocks (tree fn)
    4702              : {
    4703      2248229 :   int i;
    4704      2248229 :   int n_blocks;
    4705      2248229 :   tree *block_vector;
    4706              : 
    4707      2248229 :   block_vector = get_block_vector (DECL_INITIAL (fn), &n_blocks);
    4708              : 
    4709              :   /* The top-level BLOCK isn't numbered at all.  */
    4710     66400812 :   for (i = 1; i < n_blocks; ++i)
    4711              :     /* We number the blocks from two.  */
    4712     61904354 :     BLOCK_NUMBER (block_vector[i]) = next_block_index++;
    4713              : 
    4714      2248229 :   free (block_vector);
    4715              : 
    4716      2248229 :   return;
    4717              : }
    4718              : 
    4719              : /* If VAR is present in a subblock of BLOCK, return the subblock.  */
    4720              : 
    4721              : DEBUG_FUNCTION tree
    4722            0 : debug_find_var_in_block_tree (tree var, tree block)
    4723              : {
    4724            0 :   tree t;
    4725              : 
    4726            0 :   for (t = BLOCK_VARS (block); t; t = TREE_CHAIN (t))
    4727            0 :     if (t == var)
    4728              :       return block;
    4729              : 
    4730            0 :   for (t = BLOCK_SUBBLOCKS (block); t; t = TREE_CHAIN (t))
    4731              :     {
    4732            0 :       tree ret = debug_find_var_in_block_tree (var, t);
    4733            0 :       if (ret)
    4734              :         return ret;
    4735              :     }
    4736              : 
    4737              :   return NULL_TREE;
    4738              : }
    4739              : 
    4740              : /* Keep track of whether we're in a dummy function context.  If we are,
    4741              :    we don't want to invoke the set_current_function hook, because we'll
    4742              :    get into trouble if the hook calls target_reinit () recursively or
    4743              :    when the initial initialization is not yet complete.  */
    4744              : 
    4745              : static bool in_dummy_function;
    4746              : 
    4747              : /* Invoke the target hook when setting cfun.  Update the optimization options
    4748              :    if the function uses different options than the default.  */
    4749              : 
    4750              : static void
    4751    813916746 : invoke_set_current_function_hook (tree fndecl)
    4752              : {
    4753    813916746 :   if (!in_dummy_function)
    4754              :     {
    4755    813472444 :       tree opts = ((fndecl)
    4756    813472444 :                    ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (fndecl)
    4757    813472444 :                    : optimization_default_node);
    4758              : 
    4759    813472444 :       if (!opts)
    4760    376505579 :         opts = optimization_default_node;
    4761              : 
    4762              :       /* Change optimization options if needed.  */
    4763    813472444 :       if (optimization_current_node != opts)
    4764              :         {
    4765      3260099 :           optimization_current_node = opts;
    4766      3260099 :           cl_optimization_restore (&global_options, &global_options_set,
    4767      3260099 :                                    TREE_OPTIMIZATION (opts));
    4768              :         }
    4769              : 
    4770    813472444 :       targetm.set_current_function (fndecl);
    4771    813472444 :       this_fn_optabs = this_target_optabs;
    4772              : 
    4773              :       /* Initialize global alignment variables after op.  */
    4774    813472444 :       parse_alignment_opts ();
    4775              : 
    4776    813472444 :       if (opts != optimization_default_node)
    4777              :         {
    4778      1666180 :           init_tree_optimization_optabs (opts);
    4779      1666180 :           if (TREE_OPTIMIZATION_OPTABS (opts))
    4780       115765 :             this_fn_optabs = (struct target_optabs *)
    4781       115765 :               TREE_OPTIMIZATION_OPTABS (opts);
    4782              :         }
    4783              :     }
    4784    813916746 : }
    4785              : 
    4786              : /* Set cfun to NEW_CFUN and switch to the optimization and target options
    4787              :    associated with NEW_FNDECL.
    4788              : 
    4789              :    FORCE says whether we should do the switch even if NEW_CFUN is the current
    4790              :    function, e.g. because there has been a change in optimization or target
    4791              :    options.  */
    4792              : 
    4793              : static void
    4794   1894555311 : set_function_decl (function *new_cfun, tree new_fndecl, bool force)
    4795              : {
    4796   1894555311 :   if (cfun != new_cfun || force)
    4797              :     {
    4798    607782836 :       cfun = new_cfun;
    4799    607782836 :       invoke_set_current_function_hook (new_fndecl);
    4800    607782836 :       redirect_edge_var_map_empty ();
    4801              :     }
    4802   1894555311 : }
    4803              : 
    4804              : /* cfun should never be set directly; use this function.  */
    4805              : 
    4806              : void
    4807   1148177119 : set_cfun (struct function *new_cfun, bool force)
    4808              : {
    4809   1148177119 :   set_function_decl (new_cfun, new_cfun ? new_cfun->decl : NULL_TREE, force);
    4810   1148177119 : }
    4811              : 
    4812              : /* Initialized with NOGC, making this poisonous to the garbage collector.  */
    4813              : 
    4814              : static vec<function *> cfun_stack;
    4815              : 
    4816              : /* Push the current cfun onto the stack, then switch to function NEW_CFUN
    4817              :    and FUNCTION_DECL NEW_FNDECL.  FORCE is as for set_function_decl.  */
    4818              : 
    4819              : static void
    4820    746378192 : push_function_decl (function *new_cfun, tree new_fndecl, bool force)
    4821              : {
    4822    746378192 :   gcc_assert ((!cfun && !current_function_decl)
    4823              :               || (cfun && current_function_decl == cfun->decl));
    4824    746378192 :   cfun_stack.safe_push (cfun);
    4825    746378192 :   current_function_decl = new_fndecl;
    4826    746378192 :   set_function_decl (new_cfun, new_fndecl, force);
    4827    746378192 : }
    4828              : 
    4829              : /* Push the current cfun onto the stack and switch to function declaration
    4830              :    NEW_FNDECL, which might or might not have a function body.  FORCE is as for
    4831              :    set_function_decl.  */
    4832              : 
    4833              : void
    4834            0 : push_function_decl (tree new_fndecl, bool force)
    4835              : {
    4836            0 :   force |= current_function_decl != new_fndecl;
    4837            0 :   push_function_decl (DECL_STRUCT_FUNCTION (new_fndecl), new_fndecl, force);
    4838            0 : }
    4839              : 
    4840              : /* Push the current cfun onto the stack, and set cfun to new_cfun.  Also set
    4841              :    current_function_decl accordingly.  */
    4842              : 
    4843              : void
    4844    746378192 : push_cfun (struct function *new_cfun)
    4845              : {
    4846    746378192 :   push_function_decl (new_cfun, new_cfun ? new_cfun->decl : NULL_TREE, false);
    4847    746378192 : }
    4848              : 
    4849              : /* A common subroutine for pop_cfun and pop_function_decl.  FORCE is as
    4850              :    for set_function_decl.  */
    4851              : 
    4852              : static void
    4853    747268170 : pop_cfun_1 (bool force)
    4854              : {
    4855    747268170 :   struct function *new_cfun = cfun_stack.pop ();
    4856              :   /* When in_dummy_function, we do have a cfun but current_function_decl is
    4857              :      NULL.  We also allow pushing NULL cfun and subsequently changing
    4858              :      current_function_decl to something else and have both restored by
    4859              :      pop_cfun.  */
    4860    747268170 :   gcc_checking_assert (in_dummy_function
    4861              :                        || !cfun
    4862              :                        || current_function_decl == cfun->decl);
    4863    747268170 :   set_cfun (new_cfun, force);
    4864    747268170 :   current_function_decl = new_cfun ? new_cfun->decl : NULL_TREE;
    4865    747268170 : }
    4866              : 
    4867              : /* Pop cfun from the stack.  Also set current_function_decl accordingly.  */
    4868              : 
    4869              : void
    4870    747268170 : pop_cfun (void)
    4871              : {
    4872    747268170 :   pop_cfun_1 (false);
    4873    747268170 : }
    4874              : 
    4875              : /* Undo push_function_decl.  */
    4876              : 
    4877              : void
    4878            0 : pop_function_decl (void)
    4879              : {
    4880              :   /* If the previous cfun was null, the options should be reset to the
    4881              :      global set.  Checking the current cfun against the new (popped) cfun
    4882              :      wouldn't catch this if the current function decl has no function
    4883              :      struct.  */
    4884            0 :   pop_cfun_1 (!cfun_stack.last ());
    4885            0 : }
    4886              : 
    4887              : /* Return value of funcdef and increase it.  */
    4888              : int
    4889    205911739 : get_next_funcdef_no (void)
    4890              : {
    4891    205911739 :   return funcdef_no++;
    4892              : }
    4893              : 
    4894              : /* Return value of funcdef.  */
    4895              : int
    4896            0 : get_last_funcdef_no (void)
    4897              : {
    4898            0 :   return funcdef_no;
    4899              : }
    4900              : 
    4901              : /* Allocate and initialize the stack usage info data structure for the
    4902              :    current function.  */
    4903              : static void
    4904          690 : allocate_stack_usage_info (void)
    4905              : {
    4906          690 :   gcc_assert (!cfun->su);
    4907          690 :   cfun->su = ggc_cleared_alloc<stack_usage> ();
    4908          690 :   cfun->su->static_stack_size = -1;
    4909          690 : }
    4910              : 
    4911              : /* Allocate a function structure for FNDECL and set its contents
    4912              :    to the defaults.  Set cfun to the newly-allocated object.
    4913              :    Some of the helper functions invoked during initialization assume
    4914              :    that cfun has already been set.  Therefore, assign the new object
    4915              :    directly into cfun and invoke the back end hook explicitly at the
    4916              :    very end, rather than initializing a temporary and calling set_cfun
    4917              :    on it.
    4918              : 
    4919              :    ABSTRACT_P is true if this is a function that will never be seen by
    4920              :    the middle-end.  Such functions are front-end concepts (like C++
    4921              :    function templates) that do not correspond directly to functions
    4922              :    placed in object files.  */
    4923              : 
    4924              : void
    4925    206133910 : allocate_struct_function (tree fndecl, bool abstract_p)
    4926              : {
    4927    206133910 :   tree fntype = fndecl ? TREE_TYPE (fndecl) : NULL_TREE;
    4928              : 
    4929    206133910 :   cfun = ggc_cleared_alloc<function> ();
    4930              : 
    4931    206133910 :   init_eh_for_function ();
    4932              : 
    4933    206133910 :   if (init_machine_status)
    4934    206133910 :     cfun->machine = (*init_machine_status) ();
    4935              : 
    4936              : #ifdef OVERRIDE_ABI_FORMAT
    4937    206133910 :   OVERRIDE_ABI_FORMAT (fndecl);
    4938              : #endif
    4939              : 
    4940    206133910 :   if (fndecl != NULL_TREE)
    4941              :     {
    4942    205911739 :       DECL_STRUCT_FUNCTION (fndecl) = cfun;
    4943    205911739 :       cfun->decl = fndecl;
    4944    205911739 :       current_function_funcdef_no = get_next_funcdef_no ();
    4945              :     }
    4946              : 
    4947    206133910 :   invoke_set_current_function_hook (fndecl);
    4948              : 
    4949    206133910 :   if (fndecl != NULL_TREE)
    4950              :     {
    4951    205911739 :       tree result = DECL_RESULT (fndecl);
    4952              : 
    4953    205911739 :       if (!abstract_p)
    4954              :         {
    4955              :           /* Now that we have activated any function-specific attributes
    4956              :              that might affect layout, particularly vector modes, relayout
    4957              :              each of the parameters and the result.  */
    4958    107488584 :           relayout_decl (result);
    4959    324110345 :           for (tree parm = DECL_ARGUMENTS (fndecl); parm;
    4960    216621761 :                parm = DECL_CHAIN (parm))
    4961    216621761 :             relayout_decl (parm);
    4962              : 
    4963              :           /* Similarly relayout the function decl.  */
    4964    107488584 :           targetm.target_option.relayout_function (fndecl);
    4965              :         }
    4966              : 
    4967    107488584 :       if (!abstract_p && aggregate_value_p (result, fndecl))
    4968              :         {
    4969              : #ifdef PCC_STATIC_STRUCT_RETURN
    4970              :           cfun->returns_pcc_struct = 1;
    4971              : #endif
    4972      3170507 :           cfun->returns_struct = 1;
    4973              :         }
    4974              : 
    4975    205911739 :       cfun->stdarg = stdarg_p (fntype);
    4976              : 
    4977              :       /* Assume all registers in stdarg functions need to be saved.  */
    4978    205911739 :       cfun->va_list_gpr_size = VA_LIST_MAX_GPR_SIZE;
    4979    205911739 :       cfun->va_list_fpr_size = VA_LIST_MAX_FPR_SIZE;
    4980              : 
    4981              :       /* ??? This could be set on a per-function basis by the front-end
    4982              :          but is this worth the hassle?  */
    4983    205911739 :       cfun->can_throw_non_call_exceptions = flag_non_call_exceptions;
    4984    205911739 :       cfun->can_delete_dead_exceptions = flag_delete_dead_exceptions;
    4985              : 
    4986    205911739 :       if (!profile_flag && !flag_instrument_function_entry_exit)
    4987    205911297 :         DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (fndecl) = 1;
    4988              : 
    4989    205911739 :       if (flag_callgraph_info)
    4990            1 :         allocate_stack_usage_info ();
    4991              :     }
    4992              : 
    4993              :   /* Don't enable begin stmt markers if var-tracking at assignments is
    4994              :      disabled.  The markers make little sense without the variable
    4995              :      binding annotations among them.  */
    4996    412267820 :   cfun->debug_nonbind_markers = lang_hooks.emits_begin_stmt
    4997    206133910 :     && MAY_HAVE_DEBUG_MARKER_STMTS;
    4998    206133910 : }
    4999              : 
    5000              : /* This is like allocate_struct_function, but pushes a new cfun for FNDECL
    5001              :    instead of just setting it.  */
    5002              : 
    5003              : void
    5004       890004 : push_struct_function (tree fndecl, bool abstract_p)
    5005              : {
    5006              :   /* When in_dummy_function we might be in the middle of a pop_cfun and
    5007              :      current_function_decl and cfun may not match.  */
    5008       890004 :   gcc_assert (in_dummy_function
    5009              :               || (!cfun && !current_function_decl)
    5010              :               || (cfun && current_function_decl == cfun->decl));
    5011       890004 :   cfun_stack.safe_push (cfun);
    5012       890004 :   current_function_decl = fndecl;
    5013       890004 :   allocate_struct_function (fndecl, abstract_p);
    5014       890004 : }
    5015              : 
    5016              : /* Reset crtl and other non-struct-function variables to defaults as
    5017              :    appropriate for emitting rtl at the start of a function.  */
    5018              : 
    5019              : static void
    5020      1745840 : prepare_function_start (void)
    5021              : {
    5022      1745840 :   gcc_assert (!get_last_insn ());
    5023              : 
    5024      1745840 :   if (in_dummy_function)
    5025       222146 :     crtl->abi = &default_function_abi;
    5026              :   else
    5027      1523694 :     crtl->abi = &fndecl_abi (cfun->decl).base_abi ();
    5028              : 
    5029      1745840 :   init_temp_slots ();
    5030      1745840 :   init_emit ();
    5031      1745840 :   init_varasm_status ();
    5032      1745840 :   init_expr ();
    5033      1745840 :   default_rtl_profile ();
    5034              : 
    5035      1745840 :   if (flag_stack_usage_info && !flag_callgraph_info)
    5036          689 :     allocate_stack_usage_info ();
    5037              : 
    5038      1745840 :   cse_not_expected = ! optimize;
    5039              : 
    5040              :   /* Caller save not needed yet.  */
    5041      1745840 :   caller_save_needed = 0;
    5042              : 
    5043              :   /* We haven't done register allocation yet.  */
    5044      1745840 :   reg_renumber = 0;
    5045              : 
    5046              :   /* Indicate that we have not instantiated virtual registers yet.  */
    5047      1745840 :   virtuals_instantiated = 0;
    5048              : 
    5049              :   /* Indicate that we want CONCATs now.  */
    5050      1745840 :   generating_concat_p = 1;
    5051              : 
    5052              :   /* Indicate we have no need of a frame pointer yet.  */
    5053      1745840 :   frame_pointer_needed = 0;
    5054              : 
    5055              :   /* Reset the cache of the "extended" flag in the target's
    5056              :      _BitInt info struct.  */
    5057      1745840 :   bitint_extended = -1;
    5058              : 
    5059              :   /* Reset the divisor of the last division expanded, so that a function's
    5060              :      code does not depend on what was expanded before it.  */
    5061      1745840 :   last_div_const = 0;
    5062      1745840 : }
    5063              : 
    5064              : void
    5065       222151 : push_dummy_function (bool with_decl)
    5066              : {
    5067       222151 :   tree fn_decl, fn_type, fn_result_decl;
    5068              : 
    5069       222151 :   gcc_assert (!in_dummy_function);
    5070       222151 :   in_dummy_function = true;
    5071              : 
    5072       222151 :   if (with_decl)
    5073              :     {
    5074            5 :       fn_type = build_function_type_list (void_type_node, NULL_TREE);
    5075            5 :       fn_decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL, NULL_TREE,
    5076              :                             fn_type);
    5077            5 :       fn_result_decl = build_decl (UNKNOWN_LOCATION, RESULT_DECL,
    5078              :                                          NULL_TREE, void_type_node);
    5079            5 :       DECL_RESULT (fn_decl) = fn_result_decl;
    5080            5 :       DECL_ARTIFICIAL (fn_decl) = 1;
    5081            5 :       tree fn_name = get_identifier (" ");
    5082            5 :       SET_DECL_ASSEMBLER_NAME (fn_decl, fn_name);
    5083              :     }
    5084              :   else
    5085              :     fn_decl = NULL_TREE;
    5086              : 
    5087       222151 :   push_struct_function (fn_decl);
    5088       222151 : }
    5089              : 
    5090              : /* Initialize the rtl expansion mechanism so that we can do simple things
    5091              :    like generate sequences.  This is used to provide a context during global
    5092              :    initialization of some passes.  You must call expand_dummy_function_end
    5093              :    to exit this context.  */
    5094              : 
    5095              : void
    5096       222146 : init_dummy_function_start (void)
    5097              : {
    5098       222146 :   push_dummy_function (false);
    5099       222146 :   prepare_function_start ();
    5100       222146 : }
    5101              : 
    5102              : /* Generate RTL for the start of the function SUBR (a FUNCTION_DECL tree node)
    5103              :    and initialize static variables for generating RTL for the statements
    5104              :    of the function.  */
    5105              : 
    5106              : void
    5107      1523694 : init_function_start (tree subr)
    5108              : {
    5109              :   /* Initialize backend, if needed.  */
    5110      1523694 :   initialize_rtl ();
    5111              : 
    5112      1523694 :   prepare_function_start ();
    5113      1523694 :   decide_function_section (subr);
    5114              : 
    5115              :   /* Warn if this value is an aggregate type,
    5116              :      regardless of which calling convention we are using for it.  */
    5117      1523694 :   if (AGGREGATE_TYPE_P (TREE_TYPE (DECL_RESULT (subr))))
    5118       110348 :     warning_at (DECL_SOURCE_LOCATION (DECL_RESULT (subr)),
    5119       110348 :                 OPT_Waggregate_return, "function returns an aggregate");
    5120      1523694 : }
    5121              : 
    5122              : /* Expand code to verify the stack_protect_guard.  This is invoked at
    5123              :    the end of a function to be protected.  */
    5124              : 
    5125              : void
    5126          328 : stack_protect_epilogue (void)
    5127              : {
    5128          328 :   tree guard_decl = crtl->stack_protect_guard_decl;
    5129          328 :   rtx_code_label *label = gen_label_rtx ();
    5130          328 :   rtx x, y;
    5131          328 :   rtx_insn *seq = NULL;
    5132              : 
    5133          328 :   x = expand_normal (crtl->stack_protect_guard);
    5134              : 
    5135          328 :   if (targetm.have_stack_protect_combined_test () && guard_decl)
    5136              :     {
    5137            0 :       gcc_assert (DECL_P (guard_decl));
    5138            0 :       y = DECL_RTL (guard_decl);
    5139              :       /* Allow the target to compute address of Y and compare it with X without
    5140              :          leaking Y into a register.  This combined address + compare pattern
    5141              :          allows the target to prevent spilling of any intermediate results by
    5142              :          splitting it after register allocator.  */
    5143            0 :       seq = targetm.gen_stack_protect_combined_test (x, y, label);
    5144              :     }
    5145              :   else
    5146              :     {
    5147          328 :       if (guard_decl)
    5148          328 :         y = expand_normal (guard_decl);
    5149              :       else
    5150            0 :         y = const0_rtx;
    5151              : 
    5152              :       /* Allow the target to compare Y with X without leaking either into
    5153              :          a register.  */
    5154          328 :       if (targetm.have_stack_protect_test ())
    5155          328 :         seq = targetm.gen_stack_protect_test (x, y, label);
    5156              :     }
    5157              : 
    5158          328 :   if (seq)
    5159          328 :     emit_insn (seq);
    5160              :   else
    5161            0 :     emit_cmp_and_jump_insns (x, y, EQ, NULL_RTX, ptr_mode, 1, label);
    5162              : 
    5163              :   /* The noreturn predictor has been moved to the tree level.  The rtl-level
    5164              :      predictors estimate this branch about 20%, which isn't enough to get
    5165              :      things moved out of line.  Since this is the only extant case of adding
    5166              :      a noreturn function at the rtl level, it doesn't seem worth doing ought
    5167              :      except adding the prediction by hand.  */
    5168          328 :   rtx_insn *tmp = get_last_insn ();
    5169          328 :   if (JUMP_P (tmp))
    5170          328 :     predict_insn_def (tmp, PRED_NORETURN, TAKEN);
    5171              : 
    5172          328 :   expand_call (targetm.stack_protect_fail (), NULL_RTX, /*ignore=*/true);
    5173          328 :   free_temp_slots ();
    5174          328 :   emit_label (label);
    5175          328 : }
    5176              : 
    5177              : /* Start the RTL for a new function, and set variables used for
    5178              :    emitting RTL.
    5179              :    SUBR is the FUNCTION_DECL node.
    5180              :    PARMS_HAVE_CLEANUPS is nonzero if there are cleanups associated with
    5181              :    the function's parameters, which must be run at any return statement.  */
    5182              : 
    5183              : bool currently_expanding_function_start;
    5184              : void
    5185      1517908 : expand_function_start (tree subr)
    5186              : {
    5187      1517908 :   currently_expanding_function_start = true;
    5188              : 
    5189              :   /* Make sure volatile mem refs aren't considered
    5190              :      valid operands of arithmetic insns.  */
    5191      1517908 :   init_recog_no_volatile ();
    5192              : 
    5193      1517908 :   crtl->profile
    5194      3035816 :     = (profile_flag
    5195      1518225 :        && ! DECL_NO_INSTRUMENT_FUNCTION_ENTRY_EXIT (subr));
    5196              : 
    5197      1517908 :   crtl->limit_stack
    5198      1517908 :     = (stack_limit_rtx != NULL_RTX && ! DECL_NO_LIMIT_STACK (subr));
    5199              : 
    5200              :   /* Make the label for return statements to jump to.  Do not special
    5201              :      case machines with special return instructions -- they will be
    5202              :      handled later during jump, ifcvt, or epilogue creation.  */
    5203      1517908 :   return_label = gen_label_rtx ();
    5204              : 
    5205              :   /* Initialize rtx used to return the value.  */
    5206              :   /* Do this before assign_parms so that we copy the struct value address
    5207              :      before any library calls that assign parms might generate.  */
    5208              : 
    5209              :   /* Decide whether to return the value in memory or in a register.  */
    5210      1517908 :   tree res = DECL_RESULT (subr);
    5211      1517908 :   if (aggregate_value_p (res, subr))
    5212              :     {
    5213              :       /* Returning something that won't go in a register.  */
    5214        70115 :       rtx value_address = 0;
    5215              : 
    5216              : #ifdef PCC_STATIC_STRUCT_RETURN
    5217              :       if (cfun->returns_pcc_struct)
    5218              :         {
    5219              :           int size = int_size_in_bytes (TREE_TYPE (res));
    5220              :           value_address = assemble_static_space (size);
    5221              :         }
    5222              :       else
    5223              : #endif
    5224        70115 :         {
    5225        70115 :           rtx sv = targetm.calls.struct_value_rtx (TREE_TYPE (subr), 2);
    5226              :           /* Expect to be passed the address of a place to store the value.
    5227              :              If it is passed as an argument, assign_parms will take care of
    5228              :              it.  */
    5229        70115 :           if (sv)
    5230              :             {
    5231            0 :               value_address = gen_reg_rtx (Pmode);
    5232            0 :               emit_move_insn (value_address, sv);
    5233              :             }
    5234              :         }
    5235            0 :       if (value_address)
    5236              :         {
    5237            0 :           rtx x = value_address;
    5238            0 :           if (!DECL_BY_REFERENCE (res))
    5239              :             {
    5240            0 :               x = gen_rtx_MEM (DECL_MODE (res), x);
    5241            0 :               set_mem_attributes (x, res, 1);
    5242              :             }
    5243            0 :           set_parm_rtl (res, x);
    5244              :         }
    5245              :     }
    5246      1447793 :   else if (DECL_MODE (res) == VOIDmode)
    5247              :     /* If return mode is void, this decl rtl should not be used.  */
    5248       708784 :     set_parm_rtl (res, NULL_RTX);
    5249              :   else
    5250              :     {
    5251              :       /* Compute the return values into a pseudo reg, which we will copy
    5252              :          into the true return register after the cleanups are done.  */
    5253       739009 :       tree return_type = TREE_TYPE (res);
    5254              : 
    5255              :       /* If we may coalesce this result, make sure it has the expected mode
    5256              :          in case it was promoted.  But we need not bother about BLKmode.  */
    5257       739009 :       machine_mode promoted_mode
    5258       543898 :         = flag_tree_coalesce_vars && is_gimple_reg (res)
    5259      1245485 :           ? promote_ssa_mode (ssa_default_def (cfun, res), NULL)
    5260              :           : BLKmode;
    5261              : 
    5262       506476 :       if (promoted_mode != BLKmode)
    5263       506467 :         set_parm_rtl (res, gen_reg_rtx (promoted_mode));
    5264       232542 :       else if (TYPE_MODE (return_type) != BLKmode
    5265       232542 :                && targetm.calls.return_in_msb (return_type))
    5266              :         /* expand_function_end will insert the appropriate padding in
    5267              :            this case.  Use the return value's natural (unpadded) mode
    5268              :            within the function proper.  */
    5269            0 :         set_parm_rtl (res, gen_reg_rtx (TYPE_MODE (return_type)));
    5270              :       else
    5271              :         {
    5272              :           /* In order to figure out what mode to use for the pseudo, we
    5273              :              figure out what the mode of the eventual return register will
    5274              :              actually be, and use that.  */
    5275       232542 :           rtx hard_reg = hard_function_value (return_type, subr, 0, 1);
    5276              : 
    5277              :           /* Structures that are returned in registers are not
    5278              :              aggregate_value_p, so we may see a PARALLEL or a REG.  */
    5279       232542 :           if (REG_P (hard_reg))
    5280       229220 :             set_parm_rtl (res, gen_reg_rtx (GET_MODE (hard_reg)));
    5281              :           else
    5282              :             {
    5283         3322 :               gcc_assert (GET_CODE (hard_reg) == PARALLEL);
    5284         3322 :               set_parm_rtl (res, gen_group_rtx (hard_reg));
    5285              :             }
    5286              :         }
    5287              : 
    5288              :       /* Set DECL_REGISTER flag so that expand_function_end will copy the
    5289              :          result to the real return register(s).  */
    5290       739009 :       DECL_REGISTER (res) = 1;
    5291              :     }
    5292              : 
    5293              :   /* Initialize rtx for parameters and local variables.
    5294              :      In some cases this requires emitting insns.  */
    5295      1517908 :   assign_parms (subr);
    5296              : 
    5297              :   /* If function gets a static chain arg, store it.  */
    5298      1517908 :   if (cfun->static_chain_decl)
    5299              :     {
    5300        19568 :       tree parm = cfun->static_chain_decl;
    5301        19568 :       rtx local, chain;
    5302        19568 :       rtx_insn *insn;
    5303        19568 :       int unsignedp;
    5304              : 
    5305        19568 :       local = gen_reg_rtx (promote_decl_mode (parm, &unsignedp));
    5306        19568 :       chain = targetm.calls.static_chain (current_function_decl, true);
    5307              : 
    5308        19568 :       set_decl_incoming_rtl (parm, chain, false);
    5309        19568 :       set_parm_rtl (parm, local);
    5310        19568 :       mark_reg_pointer (local, TYPE_ALIGN (TREE_TYPE (TREE_TYPE (parm))));
    5311              : 
    5312        19568 :       if (GET_MODE (local) != GET_MODE (chain))
    5313              :         {
    5314            1 :           convert_move (local, chain, unsignedp);
    5315            1 :           insn = get_last_insn ();
    5316              :         }
    5317              :       else
    5318        19567 :         insn = emit_move_insn (local, chain);
    5319              : 
    5320              :       /* Mark the register as eliminable, similar to parameters.  */
    5321        19568 :       if (MEM_P (chain)
    5322        19568 :           && reg_mentioned_p (arg_pointer_rtx, XEXP (chain, 0)))
    5323            0 :         set_dst_reg_note (insn, REG_EQUIV, chain, local);
    5324              : 
    5325              :       /* If we aren't optimizing, save the static chain onto the stack.  */
    5326        19568 :       if (!optimize)
    5327              :         {
    5328         4115 :           tree saved_static_chain_decl
    5329         4115 :             = build_decl (DECL_SOURCE_LOCATION (parm), VAR_DECL,
    5330         4115 :                           DECL_NAME (parm), TREE_TYPE (parm));
    5331         4115 :           rtx saved_static_chain_rtx
    5332         8230 :             = assign_stack_local (Pmode, GET_MODE_SIZE (Pmode), 0);
    5333         4115 :           SET_DECL_RTL (saved_static_chain_decl, saved_static_chain_rtx);
    5334         4115 :           emit_move_insn (saved_static_chain_rtx, chain);
    5335         4115 :           SET_DECL_VALUE_EXPR (parm, saved_static_chain_decl);
    5336         4115 :           DECL_HAS_VALUE_EXPR_P (parm) = 1;
    5337              :         }
    5338              :     }
    5339              : 
    5340              :   /* The following was moved from init_function_start.
    5341              :      The move was supposed to make sdb output more accurate.  */
    5342              :   /* Indicate the beginning of the function body,
    5343              :      as opposed to parm setup.  */
    5344      1517908 :   emit_note (NOTE_INSN_FUNCTION_BEG);
    5345              : 
    5346      1517908 :   gcc_assert (NOTE_P (get_last_insn ()));
    5347              : 
    5348      1517908 :   function_beg_insn = parm_birth_insn = get_last_insn ();
    5349              : 
    5350              :   /* If the function receives a non-local goto, then store the
    5351              :      bits we need to restore the frame pointer.  */
    5352      1517908 :   if (cfun->nonlocal_goto_save_area)
    5353              :     {
    5354          405 :       tree t_save;
    5355          405 :       rtx r_save;
    5356              : 
    5357          405 :       tree var = TREE_OPERAND (cfun->nonlocal_goto_save_area, 0);
    5358          405 :       gcc_assert (DECL_RTL_SET_P (var));
    5359              : 
    5360          405 :       t_save = build4 (ARRAY_REF,
    5361          405 :                        TREE_TYPE (TREE_TYPE (cfun->nonlocal_goto_save_area)),
    5362              :                        cfun->nonlocal_goto_save_area,
    5363              :                        integer_zero_node, NULL_TREE, NULL_TREE);
    5364          405 :       r_save = expand_expr (t_save, NULL_RTX, VOIDmode, EXPAND_WRITE);
    5365          405 :       gcc_assert (GET_MODE (r_save) == Pmode);
    5366              : 
    5367          405 :       emit_move_insn (r_save, hard_frame_pointer_rtx);
    5368          405 :       update_nonlocal_goto_save_area ();
    5369              :     }
    5370              : 
    5371      1517908 :   if (crtl->profile)
    5372              :     {
    5373              : #ifdef PROFILE_HOOK
    5374              :       PROFILE_HOOK (current_function_funcdef_no);
    5375              : #endif
    5376              :     }
    5377              : 
    5378              :   /* If we are doing generic stack checking, the probe should go here.  */
    5379      1517908 :   if (flag_stack_check == GENERIC_STACK_CHECK)
    5380           49 :     stack_check_probe_note = emit_note (NOTE_INSN_DELETED);
    5381              : 
    5382      1517908 :   currently_expanding_function_start = false;
    5383      1517908 : }
    5384              : 
    5385              : void
    5386       222151 : pop_dummy_function (void)
    5387              : {
    5388       222151 :   pop_cfun ();
    5389       222151 :   in_dummy_function = false;
    5390       222151 : }
    5391              : 
    5392              : /* Undo the effects of init_dummy_function_start.  */
    5393              : void
    5394       222146 : expand_dummy_function_end (void)
    5395              : {
    5396       222146 :   gcc_assert (in_dummy_function);
    5397              : 
    5398              :   /* End any sequences that failed to be closed due to syntax errors.  */
    5399       222146 :   while (in_sequence_p ())
    5400            0 :     end_sequence ();
    5401              : 
    5402              :   /* Outside function body, can't compute type's actual size
    5403              :      until next function's body starts.  */
    5404              : 
    5405       222146 :   free_after_parsing (cfun);
    5406       222146 :   free_after_compilation (cfun);
    5407       222146 :   pop_dummy_function ();
    5408       222146 : }
    5409              : 
    5410              : /* Helper for diddle_return_value.  */
    5411              : 
    5412              : void
    5413     20688333 : diddle_return_value_1 (void (*doit) (rtx, void *), void *arg, rtx outgoing)
    5414              : {
    5415     20688333 :   if (! outgoing)
    5416              :     return;
    5417              : 
    5418     10810011 :   if (REG_P (outgoing))
    5419     10753682 :     (*doit) (outgoing, arg);
    5420        56329 :   else if (GET_CODE (outgoing) == PARALLEL)
    5421              :     {
    5422              :       int i;
    5423              : 
    5424       132656 :       for (i = 0; i < XVECLEN (outgoing, 0); i++)
    5425              :         {
    5426        78762 :           rtx x = XEXP (XVECEXP (outgoing, 0, i), 0);
    5427              : 
    5428        78762 :           if (REG_P (x) && REGNO (x) < FIRST_PSEUDO_REGISTER)
    5429        78762 :             (*doit) (x, arg);
    5430              :         }
    5431              :     }
    5432              : }
    5433              : 
    5434              : /* Call DOIT for each hard register used as a return value from
    5435              :    the current function.  */
    5436              : 
    5437              : void
    5438     20688333 : diddle_return_value (void (*doit) (rtx, void *), void *arg)
    5439              : {
    5440     20688333 :   diddle_return_value_1 (doit, arg, crtl->return_rtx);
    5441     20688333 : }
    5442              : 
    5443              : static void
    5444        13017 : do_clobber_return_reg (rtx reg, void *arg ATTRIBUTE_UNUSED)
    5445              : {
    5446         5192 :   emit_clobber (reg);
    5447         5192 : }
    5448              : 
    5449              : void
    5450       641847 : clobber_return_register (void)
    5451              : {
    5452       641847 :   diddle_return_value (do_clobber_return_reg, NULL);
    5453              : 
    5454              :   /* In case we do use pseudo to return value, clobber it too.  */
    5455       641847 :   if (DECL_RTL_SET_P (DECL_RESULT (current_function_decl)))
    5456              :     {
    5457         7858 :       tree decl_result = DECL_RESULT (current_function_decl);
    5458         7858 :       rtx decl_rtl = DECL_RTL (decl_result);
    5459         7858 :       if (REG_P (decl_rtl) && REGNO (decl_rtl) >= FIRST_PSEUDO_REGISTER)
    5460              :         {
    5461         7825 :           do_clobber_return_reg (decl_rtl, NULL);
    5462              :         }
    5463              :     }
    5464       641847 : }
    5465              : 
    5466              : static void
    5467       805711 : do_use_return_reg (rtx reg, void *arg ATTRIBUTE_UNUSED)
    5468              : {
    5469       805711 :   emit_use (reg);
    5470       805711 : }
    5471              : 
    5472              : static void
    5473      1517906 : use_return_register (void)
    5474              : {
    5475            0 :   diddle_return_value (do_use_return_reg, NULL);
    5476            0 : }
    5477              : 
    5478              : /* Generate RTL for the end of the current function.  */
    5479              : 
    5480              : void
    5481      1517906 : expand_function_end (void)
    5482              : {
    5483              :   /* If arg_pointer_save_area was referenced only from a nested
    5484              :      function, we will not have initialized it yet.  Do that now.  */
    5485      1517906 :   if (arg_pointer_save_area && ! crtl->arg_pointer_save_area_init)
    5486            0 :     get_arg_pointer_save_area ();
    5487              : 
    5488              :   /* If we are doing generic stack checking and this function makes calls,
    5489              :      do a stack probe at the start of the function to ensure we have enough
    5490              :      space for another stack frame.  */
    5491      1517906 :   if (flag_stack_check == GENERIC_STACK_CHECK)
    5492              :     {
    5493           49 :       rtx_insn *insn, *seq;
    5494              : 
    5495          645 :       for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
    5496          628 :         if (CALL_P (insn))
    5497              :           {
    5498           32 :             rtx max_frame_size = GEN_INT (STACK_CHECK_MAX_FRAME_SIZE);
    5499           32 :             start_sequence ();
    5500           32 :             if (STACK_CHECK_MOVING_SP)
    5501           32 :               anti_adjust_stack_and_probe (max_frame_size, true);
    5502              :             else
    5503              :               probe_stack_range (STACK_OLD_CHECK_PROTECT, max_frame_size);
    5504           32 :             seq = end_sequence ();
    5505           32 :             set_insn_locations (seq, prologue_location);
    5506           32 :             emit_insn_before (seq, stack_check_probe_note);
    5507           32 :             break;
    5508              :           }
    5509              :     }
    5510              : 
    5511              :   /* End any sequences that failed to be closed due to syntax errors.  */
    5512      1517906 :   while (in_sequence_p ())
    5513            0 :     end_sequence ();
    5514              : 
    5515      1517906 :   clear_pending_stack_adjust ();
    5516      1517906 :   do_pending_stack_adjust ();
    5517              : 
    5518              :   /* Output a linenumber for the end of the function.
    5519              :      SDB depended on this.  */
    5520      1517906 :   set_curr_insn_location (input_location);
    5521              : 
    5522              :   /* Before the return label (if any), clobber the return
    5523              :      registers so that they are not propagated live to the rest of
    5524              :      the function.  This can only happen with functions that drop
    5525              :      through; if there had been a return statement, there would
    5526              :      have either been a return rtx, or a jump to the return label.
    5527              : 
    5528              :      We delay actual code generation after the current_function_value_rtx
    5529              :      is computed.  */
    5530      1517906 :   rtx_insn *clobber_after = get_last_insn ();
    5531              : 
    5532              :   /* Output the label for the actual return from the function.  */
    5533      1517906 :   emit_label (return_label);
    5534              : 
    5535      1517906 :   if (targetm_common.except_unwind_info (&global_options) == UI_SJLJ)
    5536              :     {
    5537              :       /* Let except.cc know where it should emit the call to unregister
    5538              :          the function context for sjlj exceptions.  */
    5539            0 :       if (flag_exceptions)
    5540            0 :         sjlj_emit_function_exit_after (get_last_insn ());
    5541              :     }
    5542              : 
    5543              :   /* If this is an implementation of throw, do what's necessary to
    5544              :      communicate between __builtin_eh_return and the epilogue.  */
    5545      1517906 :   expand_eh_return ();
    5546              : 
    5547              :   /* If stack protection is enabled for this function, check the guard.  */
    5548      1517906 :   if (crtl->stack_protect_guard
    5549          296 :       && targetm.stack_protect_runtime_enabled_p ()
    5550      1518201 :       && naked_return_label == NULL_RTX)
    5551          295 :     stack_protect_epilogue ();
    5552              : 
    5553              :   /* If scalar return value was computed in a pseudo-reg, or was a named
    5554              :      return value that got dumped to the stack, copy that to the hard
    5555              :      return register.  */
    5556      1517906 :   if (DECL_RTL_SET_P (DECL_RESULT (current_function_decl)))
    5557              :     {
    5558       809122 :       tree decl_result = DECL_RESULT (current_function_decl);
    5559       809122 :       rtx decl_rtl = DECL_RTL (decl_result);
    5560              : 
    5561       809122 :       if ((REG_P (decl_rtl)
    5562       809122 :            ? REGNO (decl_rtl) >= FIRST_PSEUDO_REGISTER
    5563        68059 :            : DECL_REGISTER (decl_result))
    5564              :           /* Unless the psABI says not to.  */
    5565       809122 :           && !TYPE_EMPTY_P (TREE_TYPE (decl_result)))
    5566              :         {
    5567       740949 :           rtx real_decl_rtl = crtl->return_rtx;
    5568       740949 :           complex_mode cmode;
    5569              : 
    5570              :           /* This should be set in assign_parms.  */
    5571       740949 :           gcc_assert (REG_FUNCTION_VALUE_P (real_decl_rtl));
    5572              : 
    5573              :           /* If this is a BLKmode structure being returned in registers,
    5574              :              then use the mode computed in expand_return.  Note that if
    5575              :              decl_rtl is memory, then its mode may have been changed,
    5576              :              but that crtl->return_rtx has not.  */
    5577       740949 :           if (GET_MODE (real_decl_rtl) == BLKmode)
    5578         2722 :             PUT_MODE (real_decl_rtl, GET_MODE (decl_rtl));
    5579              : 
    5580              :           /* If a non-BLKmode return value should be padded at the least
    5581              :              significant end of the register, shift it left by the appropriate
    5582              :              amount.  BLKmode results are handled using the group load/store
    5583              :              machinery.  */
    5584       740949 :           if (TYPE_MODE (TREE_TYPE (decl_result)) != BLKmode
    5585       738224 :               && REG_P (real_decl_rtl)
    5586      1476383 :               && targetm.calls.return_in_msb (TREE_TYPE (decl_result)))
    5587              :             {
    5588            0 :               emit_move_insn (gen_rtx_REG (GET_MODE (decl_rtl),
    5589              :                                            REGNO (real_decl_rtl)),
    5590              :                               decl_rtl);
    5591            0 :               shift_return_value (GET_MODE (decl_rtl), true, real_decl_rtl);
    5592              :             }
    5593       740949 :           else if (GET_CODE (real_decl_rtl) == PARALLEL)
    5594              :             {
    5595              :               /* If expand_function_start has created a PARALLEL for decl_rtl,
    5596              :                  move the result to the real return registers.  Otherwise, do
    5597              :                  a group load from decl_rtl for a named return.  */
    5598         4288 :               if (GET_CODE (decl_rtl) == PARALLEL)
    5599         3322 :                 emit_group_move (real_decl_rtl, decl_rtl);
    5600              :               else
    5601          966 :                 emit_group_load (real_decl_rtl, decl_rtl,
    5602          966 :                                  TREE_TYPE (decl_result),
    5603          966 :                                  int_size_in_bytes (TREE_TYPE (decl_result)));
    5604              :             }
    5605              :           /* In the case of complex integer modes smaller than a word, we'll
    5606              :              need to generate some non-trivial bitfield insertions.  Do that
    5607              :              on a pseudo and not the hard register.  */
    5608       736661 :           else if (GET_CODE (decl_rtl) == CONCAT
    5609          736 :                    && is_complex_int_mode (GET_MODE (decl_rtl), &cmode)
    5610       736809 :                    && GET_MODE_BITSIZE (cmode) <= BITS_PER_WORD)
    5611              :             {
    5612           94 :               int old_generating_concat_p;
    5613           94 :               rtx tmp;
    5614              : 
    5615           94 :               old_generating_concat_p = generating_concat_p;
    5616           94 :               generating_concat_p = 0;
    5617           94 :               tmp = gen_reg_rtx (GET_MODE (decl_rtl));
    5618           94 :               generating_concat_p = old_generating_concat_p;
    5619              : 
    5620           94 :               emit_move_insn (tmp, decl_rtl);
    5621           94 :               emit_move_insn (real_decl_rtl, tmp);
    5622              :             }
    5623              :           /* If a named return value dumped decl_return to memory, then
    5624              :              we may need to re-do the PROMOTE_MODE signed/unsigned
    5625              :              extension.  */
    5626       736567 :           else if (GET_MODE (real_decl_rtl) != GET_MODE (decl_rtl))
    5627              :             {
    5628            0 :               int unsignedp = TYPE_UNSIGNED (TREE_TYPE (decl_result));
    5629            0 :               promote_function_mode (TREE_TYPE (decl_result),
    5630              :                                      GET_MODE (decl_rtl), &unsignedp,
    5631            0 :                                      TREE_TYPE (current_function_decl), 1);
    5632              : 
    5633            0 :               convert_move (real_decl_rtl, decl_rtl, unsignedp);
    5634              :             }
    5635              :           else
    5636       736567 :             emit_move_insn (real_decl_rtl, decl_rtl);
    5637              :         }
    5638              :     }
    5639              : 
    5640              :   /* If returning a structure, arrange to return the address of the value
    5641              :      in a place where debuggers expect to find it.
    5642              : 
    5643              :      If returning a structure PCC style,
    5644              :      the caller also depends on this value.
    5645              :      And cfun->returns_pcc_struct is not necessarily set.  */
    5646      1517906 :   if ((cfun->returns_struct || cfun->returns_pcc_struct)
    5647        69906 :       && !targetm.calls.omit_struct_return_reg)
    5648              :     {
    5649        69906 :       rtx value_address = DECL_RTL (DECL_RESULT (current_function_decl));
    5650        69906 :       tree type = TREE_TYPE (DECL_RESULT (current_function_decl));
    5651        69906 :       rtx outgoing;
    5652              : 
    5653        69906 :       if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl)))
    5654         9038 :         type = TREE_TYPE (type);
    5655              :       else
    5656        60868 :         value_address = XEXP (value_address, 0);
    5657              : 
    5658        69906 :       outgoing = targetm.calls.function_value (build_pointer_type (type),
    5659              :                                                current_function_decl, true);
    5660              : 
    5661              :       /* Mark this as a function return value so integrate will delete the
    5662              :          assignment and USE below when inlining this function.  */
    5663        69906 :       REG_FUNCTION_VALUE_P (outgoing) = 1;
    5664              : 
    5665              :       /* The address may be ptr_mode and OUTGOING may be Pmode.  */
    5666        69906 :       scalar_int_mode mode = as_a <scalar_int_mode> (GET_MODE (outgoing));
    5667        69906 :       value_address = convert_memory_address (mode, value_address);
    5668              : 
    5669        69906 :       emit_move_insn (outgoing, value_address);
    5670              : 
    5671              :       /* Show return register used to hold result (in this case the address
    5672              :          of the result.  */
    5673        69906 :       crtl->return_rtx = outgoing;
    5674              :     }
    5675              : 
    5676              :   /* Emit the actual code to clobber return register.  Don't emit
    5677              :      it if clobber_after is a barrier, then the previous basic block
    5678              :      certainly doesn't fall thru into the exit block.  */
    5679      1517906 :   if (!BARRIER_P (clobber_after))
    5680              :     {
    5681       570823 :       start_sequence ();
    5682       570823 :       clobber_return_register ();
    5683       570823 :       rtx_insn *seq = end_sequence ();
    5684              : 
    5685       570823 :       emit_insn_after (seq, clobber_after);
    5686              :     }
    5687              : 
    5688              :   /* Output the label for the naked return from the function.  */
    5689      1517906 :   if (naked_return_label)
    5690          379 :     emit_label (naked_return_label);
    5691              : 
    5692              :   /* @@@ This is a kludge.  We want to ensure that instructions that
    5693              :      may trap are not moved into the epilogue by scheduling, because
    5694              :      we don't always emit unwind information for the epilogue.  */
    5695      1517906 :   if (cfun->can_throw_non_call_exceptions
    5696      1517906 :       && targetm_common.except_unwind_info (&global_options) != UI_SJLJ)
    5697       263025 :     emit_insn (gen_blockage ());
    5698              : 
    5699              :   /* If stack protection is enabled for this function, check the guard.  */
    5700      1517906 :   if (crtl->stack_protect_guard
    5701          296 :       && targetm.stack_protect_runtime_enabled_p ()
    5702      1518201 :       && naked_return_label)
    5703            0 :     stack_protect_epilogue ();
    5704              : 
    5705              :   /* If we had calls to alloca, and this machine needs
    5706              :      an accurate stack pointer to exit the function,
    5707              :      insert some code to save and restore the stack pointer.  */
    5708      1517906 :   if (! EXIT_IGNORE_STACK
    5709              :       && cfun->calls_alloca)
    5710              :     {
    5711              :       rtx tem = 0;
    5712              : 
    5713              :       start_sequence ();
    5714              :       emit_stack_save (SAVE_FUNCTION, &tem);
    5715              :       rtx_insn *seq = end_sequence ();
    5716              :       emit_insn_before (seq, parm_birth_insn);
    5717              : 
    5718              :       emit_stack_restore (SAVE_FUNCTION, tem);
    5719              :     }
    5720              : 
    5721              :   /* ??? This should no longer be necessary since stupid is no longer with
    5722              :      us, but there are some parts of the compiler (eg reload_combine, and
    5723              :      sh mach_dep_reorg) that still try and compute their own lifetime info
    5724              :      instead of using the general framework.  */
    5725      1517906 :   use_return_register ();
    5726      1517906 : }
    5727              : 
    5728              : rtx
    5729            0 : get_arg_pointer_save_area (void)
    5730              : {
    5731            0 :   rtx ret = arg_pointer_save_area;
    5732              : 
    5733            0 :   if (! ret)
    5734              :     {
    5735            0 :       ret = assign_stack_local (Pmode, GET_MODE_SIZE (Pmode), 0);
    5736            0 :       arg_pointer_save_area = ret;
    5737              :     }
    5738              : 
    5739            0 :   if (! crtl->arg_pointer_save_area_init)
    5740              :     {
    5741              :       /* Save the arg pointer at the beginning of the function.  The
    5742              :          generated stack slot may not be a valid memory address, so we
    5743              :          have to check it and fix it if necessary.  */
    5744            0 :       start_sequence ();
    5745            0 :       emit_move_insn (validize_mem (copy_rtx (ret)),
    5746              :                       crtl->args.internal_arg_pointer);
    5747            0 :       rtx_insn *seq = end_sequence ();
    5748              : 
    5749            0 :       push_topmost_sequence ();
    5750            0 :       emit_insn_after (seq, entry_of_function ());
    5751            0 :       pop_topmost_sequence ();
    5752              : 
    5753            0 :       crtl->arg_pointer_save_area_init = true;
    5754              :     }
    5755              : 
    5756            0 :   return ret;
    5757              : }
    5758              : 
    5759              : 
    5760              : /* If debugging dumps are requested, dump information about how the
    5761              :    target handled -fstack-check=clash for the prologue.
    5762              : 
    5763              :    PROBES describes what if any probes were emitted.
    5764              : 
    5765              :    RESIDUALS indicates if the prologue had any residual allocation
    5766              :    (i.e. total allocation was not a multiple of PROBE_INTERVAL).  */
    5767              : 
    5768              : void
    5769          126 : dump_stack_clash_frame_info (enum stack_clash_probes probes, bool residuals)
    5770              : {
    5771          126 :   if (!dump_file)
    5772              :     return;
    5773              : 
    5774           17 :   switch (probes)
    5775              :     {
    5776            1 :     case NO_PROBE_NO_FRAME:
    5777            1 :       fprintf (dump_file,
    5778              :                "Stack clash no probe no stack adjustment in prologue.\n");
    5779            1 :       break;
    5780           11 :     case NO_PROBE_SMALL_FRAME:
    5781           11 :       fprintf (dump_file,
    5782              :                "Stack clash no probe small stack adjustment in prologue.\n");
    5783           11 :       break;
    5784            3 :     case PROBE_INLINE:
    5785            3 :       fprintf (dump_file, "Stack clash inline probes in prologue.\n");
    5786            3 :       break;
    5787            2 :     case PROBE_LOOP:
    5788            2 :       fprintf (dump_file, "Stack clash probe loop in prologue.\n");
    5789            2 :       break;
    5790              :     }
    5791              : 
    5792           17 :   if (residuals)
    5793           16 :     fprintf (dump_file, "Stack clash residual allocation in prologue.\n");
    5794              :   else
    5795            1 :     fprintf (dump_file, "Stack clash no residual allocation in prologue.\n");
    5796              : 
    5797           17 :   if (frame_pointer_needed)
    5798            0 :     fprintf (dump_file, "Stack clash frame pointer needed.\n");
    5799              :   else
    5800           17 :     fprintf (dump_file, "Stack clash no frame pointer needed.\n");
    5801              : 
    5802           17 :   if (TREE_THIS_VOLATILE (cfun->decl))
    5803            1 :     fprintf (dump_file,
    5804              :              "Stack clash noreturn prologue, assuming no implicit"
    5805              :              " probes in caller.\n");
    5806              :   else
    5807           16 :     fprintf (dump_file,
    5808              :              "Stack clash not noreturn prologue.\n");
    5809              : }
    5810              : 
    5811              : /* Add a list of INSNS to the hash HASHP, possibly allocating HASHP
    5812              :    for the first time.  */
    5813              : 
    5814              : static void
    5815      3743564 : record_insns (rtx_insn *insns, rtx_insn *end,
    5816              :               hash_table<insn_cache_hasher> **hashp)
    5817              : {
    5818      3743564 :   rtx_insn *tmp;
    5819      3743564 :   hash_table<insn_cache_hasher> *hash = *hashp;
    5820              : 
    5821      3743564 :   if (hash == NULL)
    5822      3034256 :     *hashp = hash = hash_table<insn_cache_hasher>::create_ggc (17);
    5823              : 
    5824     14495405 :   for (tmp = insns; tmp != end; tmp = NEXT_INSN (tmp))
    5825              :     {
    5826     10751841 :       rtx *slot = hash->find_slot (tmp, INSERT);
    5827     10751841 :       gcc_assert (*slot == NULL);
    5828     10751841 :       *slot = tmp;
    5829              :     }
    5830      3743564 : }
    5831              : 
    5832              : /* INSN has been duplicated or replaced by as COPY, perhaps by duplicating a
    5833              :    basic block, splitting or peepholes.  If INSN is a prologue or epilogue
    5834              :    insn, then record COPY as well.  */
    5835              : 
    5836              : void
    5837      3780335 : maybe_copy_prologue_epilogue_insn (rtx_insn *insn, rtx_insn *copy)
    5838              : {
    5839      3780335 :   hash_table<insn_cache_hasher> *hash;
    5840      3780335 :   rtx *slot;
    5841              : 
    5842      3780335 :   hash = epilogue_insn_hash;
    5843      3780335 :   if (!hash || !hash->find (insn))
    5844              :     {
    5845      3265612 :       hash = prologue_insn_hash;
    5846      3265612 :       if (!hash || !hash->find (insn))
    5847              :         return;
    5848              :     }
    5849              : 
    5850       590984 :   slot = hash->find_slot (copy, INSERT);
    5851       590984 :   gcc_assert (*slot == NULL);
    5852       590984 :   *slot = copy;
    5853              : }
    5854              : 
    5855              : /* Determine if any INSNs in HASH are, or are part of, INSN.  Because
    5856              :    we can be running after reorg, SEQUENCE rtl is possible.  */
    5857              : 
    5858              : static bool
    5859    296106181 : contains (const rtx_insn *insn, hash_table<insn_cache_hasher> *hash)
    5860              : {
    5861    296106181 :   if (hash == NULL)
    5862              :     return false;
    5863              : 
    5864    296050067 :   if (NONJUMP_INSN_P (insn) && GET_CODE (PATTERN (insn)) == SEQUENCE)
    5865              :     {
    5866            0 :       rtx_sequence *seq = as_a <rtx_sequence *> (PATTERN (insn));
    5867            0 :       int i;
    5868            0 :       for (i = seq->len () - 1; i >= 0; i--)
    5869            0 :         if (hash->find (seq->element (i)))
    5870              :           return true;
    5871              :       return false;
    5872              :     }
    5873              : 
    5874    296050067 :   return hash->find (const_cast<rtx_insn *> (insn)) != NULL;
    5875              : }
    5876              : 
    5877              : bool
    5878    113668968 : prologue_contains (const rtx_insn *insn)
    5879              : {
    5880    113668968 :   return contains (insn, prologue_insn_hash);
    5881              : }
    5882              : 
    5883              : bool
    5884    113668968 : epilogue_contains (const rtx_insn *insn)
    5885              : {
    5886    113668968 :   return contains (insn, epilogue_insn_hash);
    5887              : }
    5888              : 
    5889              : bool
    5890         4355 : prologue_epilogue_contains (const rtx_insn *insn)
    5891              : {
    5892         4355 :   if (contains (insn, prologue_insn_hash))
    5893              :     return true;
    5894         4170 :   if (contains (insn, epilogue_insn_hash))
    5895              :     return true;
    5896              :   return false;
    5897              : }
    5898              : 
    5899              : void
    5900       158628 : record_prologue_seq (rtx_insn *seq)
    5901              : {
    5902       158628 :   record_insns (seq, NULL, &prologue_insn_hash);
    5903       158628 : }
    5904              : 
    5905              : void
    5906       140794 : record_epilogue_seq (rtx_insn *seq)
    5907              : {
    5908       140794 :   record_insns (seq, NULL, &epilogue_insn_hash);
    5909       140794 : }
    5910              : 
    5911              : /* Set JUMP_LABEL for a return insn.  */
    5912              : 
    5913              : void
    5914      1563039 : set_return_jump_label (rtx_insn *returnjump)
    5915              : {
    5916      1563039 :   rtx pat = PATTERN (returnjump);
    5917      1563039 :   if (GET_CODE (pat) == PARALLEL)
    5918        27216 :     pat = XVECEXP (pat, 0, 0);
    5919      1563039 :   if (ANY_RETURN_P (pat))
    5920              :     JUMP_LABEL (returnjump) = pat;
    5921              :   else
    5922            0 :     JUMP_LABEL (returnjump) = ret_rtx;
    5923      1563039 : }
    5924              : 
    5925              : /* Return a sequence to be used as the split prologue for the current
    5926              :    function, or NULL.  */
    5927              : 
    5928              : static rtx_insn *
    5929      1562535 : make_split_prologue_seq (void)
    5930              : {
    5931      1562535 :   if (!flag_split_stack
    5932      1562535 :       || lookup_attribute ("no_split_stack", DECL_ATTRIBUTES (cfun->decl)))
    5933              :     return NULL;
    5934              : 
    5935       259994 :   start_sequence ();
    5936       259994 :   emit_insn (targetm.gen_split_stack_prologue ());
    5937       259994 :   rtx_insn *seq = end_sequence ();
    5938              : 
    5939       259994 :   record_insns (seq, NULL, &prologue_insn_hash);
    5940       259994 :   set_insn_locations (seq, prologue_location);
    5941              : 
    5942       259994 :   return seq;
    5943              : }
    5944              : 
    5945              : /* Return a sequence to be used as the prologue for the current function,
    5946              :    or NULL.  */
    5947              : 
    5948              : static rtx_insn *
    5949      1562535 : make_prologue_seq (void)
    5950              : {
    5951      1562535 :   if (!targetm.have_prologue ())
    5952              :     return NULL;
    5953              : 
    5954      1562535 :   start_sequence ();
    5955      1562535 :   rtx_insn *seq = targetm.gen_prologue ();
    5956      1562535 :   emit_insn (seq);
    5957              : 
    5958              :   /* Insert an explicit USE for the frame pointer
    5959              :      if the profiling is on and the frame pointer is required.  */
    5960      1562535 :   if (crtl->profile && frame_pointer_needed)
    5961          276 :     emit_use (hard_frame_pointer_rtx);
    5962              : 
    5963              :   /* Retain a map of the prologue insns.  */
    5964      1562535 :   record_insns (seq, NULL, &prologue_insn_hash);
    5965      1562535 :   emit_note (NOTE_INSN_PROLOGUE_END);
    5966              : 
    5967              :   /* Ensure that instructions are not moved into the prologue when
    5968              :      profiling is on.  The call to the profiling routine can be
    5969              :      emitted within the live range of a call-clobbered register.  */
    5970      1562535 :   if (!targetm.profile_before_prologue () && crtl->profile)
    5971           12 :     emit_insn (gen_blockage ());
    5972              : 
    5973      1562535 :   seq = end_sequence ();
    5974      1562535 :   set_insn_locations (seq, prologue_location);
    5975              : 
    5976      1562535 :   return seq;
    5977              : }
    5978              : 
    5979              : /* Emit a sequence of insns to zero the call-used registers before RET
    5980              :    according to ZERO_REGS_TYPE.  Return true if a sequence was emitted, in
    5981              :    which case crtl->must_be_zero_on_return has grown and the caller has to
    5982              :    refresh the data flow information.  */
    5983              : 
    5984              : static bool
    5985          177 : gen_call_used_regs_seq (rtx_insn *ret, unsigned int zero_regs_type)
    5986              : {
    5987          177 :   bool only_gpr = true;
    5988          177 :   bool only_used = true;
    5989          177 :   bool only_arg = true;
    5990              : 
    5991              :   /* No need to zero call-used-regs in main ().  */
    5992          177 :   if (MAIN_NAME_P (DECL_NAME (current_function_decl)))
    5993              :     return false;
    5994              : 
    5995              :   /* No need to zero call-used-regs if __builtin_eh_return is called
    5996              :      since it isn't a normal function return.  */
    5997          135 :   if (crtl->calls_eh_return)
    5998              :     return false;
    5999              : 
    6000              :   /* If only_gpr is true, only zero call-used registers that are
    6001              :      general-purpose registers; if only_used is true, only zero
    6002              :      call-used registers that are used in the current function;
    6003              :      if only_arg is true, only zero call-used registers that pass
    6004              :      parameters defined by the flatform's calling conversion.  */
    6005              : 
    6006          135 :   using namespace zero_regs_flags;
    6007              : 
    6008          135 :   only_gpr = zero_regs_type & ONLY_GPR;
    6009          135 :   only_used = zero_regs_type & ONLY_USED;
    6010          135 :   only_arg = zero_regs_type & ONLY_ARG;
    6011              : 
    6012          135 :   if ((zero_regs_type & LEAFY_MODE) && leaf_function_p ())
    6013              :     only_used = true;
    6014              : 
    6015              :   /* For each of the hard registers, we should zero it if:
    6016              :             1. it is a call-used register;
    6017              :         and 2. it is not a fixed register;
    6018              :         and 3. it is not live at the return of the routine;
    6019              :         and 4. it is general register if only_gpr is true;
    6020              :         and 5. it is used in the routine if only_used is true;
    6021              :         and 6. it is a register that passes parameter if only_arg is true.  */
    6022              : 
    6023              :   /* First, prepare the data flow information.  */
    6024          135 :   basic_block bb = BLOCK_FOR_INSN (ret);
    6025          135 :   auto_bitmap live_out;
    6026          135 :   bitmap_copy (live_out, df_get_live_out (bb));
    6027          135 :   df_simulate_initialize_backwards (bb, live_out);
    6028          135 :   df_simulate_one_insn_backwards (bb, ret, live_out);
    6029              : 
    6030          135 :   HARD_REG_SET selected_hardregs;
    6031          135 :   HARD_REG_SET all_call_used_regs;
    6032          540 :   CLEAR_HARD_REG_SET (selected_hardregs);
    6033        12825 :   CLEAR_HARD_REG_SET (all_call_used_regs);
    6034        12825 :   for (unsigned int regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    6035              :     {
    6036        12690 :       if (!crtl->abi->clobbers_full_reg_p (regno))
    6037         1215 :         continue;
    6038        11475 :       if (fixed_regs[regno])
    6039         5884 :         continue;
    6040         5591 :       if (REGNO_REG_SET_P (live_out, regno))
    6041          124 :         continue;
    6042              : #ifdef LEAF_REG_REMAP
    6043              :       if (crtl->uses_only_leaf_regs && LEAF_REG_REMAP (regno) < 0)
    6044              :         continue;
    6045              : #endif
    6046              :       /* This is a call used register that is dead at return.  */
    6047         5467 :       SET_HARD_REG_BIT (all_call_used_regs, regno);
    6048              : 
    6049         7387 :       if (only_gpr
    6050         5467 :           && !TEST_HARD_REG_BIT (reg_class_contents[GENERAL_REGS], regno))
    6051         1920 :         continue;
    6052         3547 :       if (only_used && !df_regs_ever_live_p (regno))
    6053         1806 :         continue;
    6054         1741 :       if (only_arg && !FUNCTION_ARG_REGNO_P (regno))
    6055          364 :         continue;
    6056              : 
    6057              :       /* Now this is a register that we might want to zero.  */
    6058         1377 :       SET_HARD_REG_BIT (selected_hardregs, regno);
    6059              :     }
    6060              : 
    6061          135 :   if (hard_reg_set_empty_p (selected_hardregs))
    6062              :     return false;
    6063              : 
    6064              :   /* Now that we have a hard register set that needs to be zeroed, pass it to
    6065              :      target to generate zeroing sequence.  */
    6066          131 :   HARD_REG_SET zeroed_hardregs;
    6067          131 :   start_sequence ();
    6068          131 :   zeroed_hardregs = targetm.calls.zero_call_used_regs (selected_hardregs);
    6069              : 
    6070              :   /* For most targets, the returned set of registers is a subset of
    6071              :      selected_hardregs, however, for some of the targets (for example MIPS),
    6072              :      clearing some registers that are in selected_hardregs requires clearing
    6073              :      other call used registers that are not in the selected_hardregs, under
    6074              :      such situation, the returned set of registers must be a subset of
    6075              :      all call used registers.  */
    6076          262 :   gcc_assert (hard_reg_set_subset_p (zeroed_hardregs, all_call_used_regs));
    6077              : 
    6078          131 :   rtx_insn *seq = end_sequence ();
    6079          131 :   if (!seq)
    6080              :     return false;
    6081              : 
    6082              :   /* Emit the memory blockage and register clobber asm volatile before
    6083              :      the whole sequence.  */
    6084          131 :   start_sequence ();
    6085          131 :   expand_asm_reg_clobber_mem_blockage (zeroed_hardregs);
    6086          131 :   rtx_insn *seq_barrier = end_sequence ();
    6087              : 
    6088          131 :   emit_insn_before (seq_barrier, ret);
    6089          131 :   emit_insn_before (seq, ret);
    6090              : 
    6091          266 :   crtl->must_be_zero_on_return |= zeroed_hardregs;
    6092              :   return true;
    6093          135 : }
    6094              : 
    6095              : 
    6096              : /* Return a sequence to be used as the epilogue for the current function,
    6097              :    or NULL.  */
    6098              : 
    6099              : static rtx_insn *
    6100      1562535 : make_epilogue_seq (void)
    6101              : {
    6102      1562535 :   if (!targetm.have_epilogue ())
    6103              :     return NULL;
    6104              : 
    6105      1562535 :   start_sequence ();
    6106      1562535 :   emit_note (NOTE_INSN_EPILOGUE_BEG);
    6107      1562535 :   rtx_insn *seq = targetm.gen_epilogue ();
    6108      1562535 :   if (seq)
    6109      1562535 :     emit_jump_insn (seq);
    6110              : 
    6111              :   /* Retain a map of the epilogue insns.  */
    6112      1562535 :   record_insns (seq, NULL, &epilogue_insn_hash);
    6113      1562535 :   set_insn_locations (seq, epilogue_location);
    6114              : 
    6115      1562535 :   seq = get_insns ();
    6116      1562535 :   rtx_insn *returnjump = get_last_insn ();
    6117      1562535 :   end_sequence ();
    6118              : 
    6119      1562535 :   if (JUMP_P (returnjump))
    6120      1562459 :     set_return_jump_label (returnjump);
    6121              : 
    6122              :   return seq;
    6123              : }
    6124              : 
    6125              : 
    6126              : /* Generate the prologue and epilogue RTL if the machine supports it.  Thread
    6127              :    this into place with notes indicating where the prologue ends and where
    6128              :    the epilogue begins.  Update the basic block information when possible.
    6129              : 
    6130              :    Notes on epilogue placement:
    6131              :    There are several kinds of edges to the exit block:
    6132              :    * a single fallthru edge from LAST_BB
    6133              :    * possibly, edges from blocks containing sibcalls
    6134              :    * possibly, fake edges from infinite loops
    6135              : 
    6136              :    The epilogue is always emitted on the fallthru edge from the last basic
    6137              :    block in the function, LAST_BB, into the exit block.
    6138              : 
    6139              :    If LAST_BB is empty except for a label, it is the target of every
    6140              :    other basic block in the function that ends in a return.  If a
    6141              :    target has a return or simple_return pattern (possibly with
    6142              :    conditional variants), these basic blocks can be changed so that a
    6143              :    return insn is emitted into them, and their target is adjusted to
    6144              :    the real exit block.
    6145              : 
    6146              :    Notes on shrink wrapping: We implement a fairly conservative
    6147              :    version of shrink-wrapping rather than the textbook one.  We only
    6148              :    generate a single prologue and a single epilogue.  This is
    6149              :    sufficient to catch a number of interesting cases involving early
    6150              :    exits.
    6151              : 
    6152              :    First, we identify the blocks that require the prologue to occur before
    6153              :    them.  These are the ones that modify a call-saved register, or reference
    6154              :    any of the stack or frame pointer registers.  To simplify things, we then
    6155              :    mark everything reachable from these blocks as also requiring a prologue.
    6156              :    This takes care of loops automatically, and avoids the need to examine
    6157              :    whether MEMs reference the frame, since it is sufficient to check for
    6158              :    occurrences of the stack or frame pointer.
    6159              : 
    6160              :    We then compute the set of blocks for which the need for a prologue
    6161              :    is anticipatable (borrowing terminology from the shrink-wrapping
    6162              :    description in Muchnick's book).  These are the blocks which either
    6163              :    require a prologue themselves, or those that have only successors
    6164              :    where the prologue is anticipatable.  The prologue needs to be
    6165              :    inserted on all edges from BB1->BB2 where BB2 is in ANTIC and BB1
    6166              :    is not.  For the moment, we ensure that only one such edge exists.
    6167              : 
    6168              :    The epilogue is placed as described above, but we make a
    6169              :    distinction between inserting return and simple_return patterns
    6170              :    when modifying other blocks that end in a return.  Blocks that end
    6171              :    in a sibcall omit the sibcall_epilogue if the block is not in
    6172              :    ANTIC.  */
    6173              : 
    6174              : void
    6175      1517128 : thread_prologue_and_epilogue_insns (void)
    6176              : {
    6177      1517128 :   df_analyze ();
    6178              : 
    6179              :   /* Can't deal with multiple successors of the entry block at the
    6180              :      moment.  Function should always have at least one entry
    6181              :      point.  */
    6182      1517128 :   gcc_assert (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
    6183              : 
    6184      1517128 :   edge entry_edge = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
    6185      1517128 :   edge orig_entry_edge = entry_edge;
    6186              : 
    6187      1517128 :   rtx_insn *split_prologue_seq = make_split_prologue_seq ();
    6188      1517128 :   rtx_insn *prologue_seq = make_prologue_seq ();
    6189      1517128 :   rtx_insn *epilogue_seq = make_epilogue_seq ();
    6190              : 
    6191              :   /* Try to perform a kind of shrink-wrapping, making sure the
    6192              :      prologue/epilogue is emitted only around those parts of the
    6193              :      function that require it.  */
    6194      1517128 :   try_shrink_wrapping (&entry_edge, prologue_seq);
    6195              : 
    6196              :   /* If the target can handle splitting the prologue/epilogue into separate
    6197              :      components, try to shrink-wrap these components separately.  */
    6198      1517128 :   try_shrink_wrapping_separate (entry_edge->dest);
    6199              : 
    6200              :   /* If that did anything for any component we now need the generate the
    6201              :      "main" prologue again.  Because some targets require some of these
    6202              :      to be called in a specific order (i386 requires the split prologue
    6203              :      to be first, for example), we create all three sequences again here.
    6204              :      If this does not work for some target, that target should not enable
    6205              :      separate shrink-wrapping.  */
    6206      1517128 :   if (crtl->shrink_wrapped_separate)
    6207              :     {
    6208        45407 :       split_prologue_seq = make_split_prologue_seq ();
    6209        45407 :       prologue_seq = make_prologue_seq ();
    6210        45407 :       epilogue_seq = make_epilogue_seq ();
    6211              :     }
    6212              : 
    6213      1517128 :   rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
    6214              : 
    6215              :   /* A small fib -- epilogue is not yet completed, but we wish to re-use
    6216              :      this marker for the splits of EH_RETURN patterns, and nothing else
    6217              :      uses the flag in the meantime.  */
    6218      1517128 :   epilogue_completed = 1;
    6219              : 
    6220              :   /* Find non-fallthru edges that end with EH_RETURN instructions.  On
    6221              :      some targets, these get split to a special version of the epilogue
    6222              :      code.  In order to be able to properly annotate these with unwind
    6223              :      info, try to split them now.  If we get a valid split, drop an
    6224              :      EPILOGUE_BEG note and mark the insns as epilogue insns.  */
    6225      1517128 :   edge e;
    6226      1517128 :   edge_iterator ei;
    6227      3088223 :   FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    6228              :     {
    6229      1571095 :       rtx_insn *prev, *last, *trial;
    6230              : 
    6231      1571095 :       if (e->flags & EDGE_FALLTHRU)
    6232      1386726 :         continue;
    6233       184369 :       last = BB_END (e->src);
    6234       184369 :       if (!eh_returnjump_p (last))
    6235       184338 :         continue;
    6236              : 
    6237           31 :       prev = PREV_INSN (last);
    6238           31 :       trial = try_split (PATTERN (last), last, 1);
    6239           31 :       if (trial == last)
    6240            0 :         continue;
    6241              : 
    6242           31 :       record_insns (NEXT_INSN (prev), NEXT_INSN (trial), &epilogue_insn_hash);
    6243           31 :       emit_note_after (NOTE_INSN_EPILOGUE_BEG, prev);
    6244              :     }
    6245              : 
    6246      1517128 :   edge exit_fallthru_edge = find_fallthru_edge (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds);
    6247              : 
    6248      1517128 :   if (exit_fallthru_edge)
    6249              :     {
    6250      1386726 :       if (epilogue_seq)
    6251              :         {
    6252      1386726 :           insert_insn_on_edge (epilogue_seq, exit_fallthru_edge);
    6253      1386726 :           commit_edge_insertions ();
    6254              : 
    6255              :           /* The epilogue insns we inserted may cause the exit edge to no longer
    6256              :              be fallthru.  */
    6257      2854347 :           FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    6258              :             {
    6259      1467621 :               if (((e->flags & EDGE_FALLTHRU) != 0)
    6260      1467621 :                   && returnjump_p (BB_END (e->src)))
    6261            0 :                 e->flags &= ~EDGE_FALLTHRU;
    6262              :             }
    6263              : 
    6264      1386726 :           find_sub_basic_blocks (BLOCK_FOR_INSN (epilogue_seq));
    6265              :         }
    6266            0 :       else if (next_active_insn (BB_END (exit_fallthru_edge->src)))
    6267              :         {
    6268              :           /* We have a fall-through edge to the exit block, the source is not
    6269              :              at the end of the function, and there will be an assembler epilogue
    6270              :              at the end of the function.
    6271              :              We can't use force_nonfallthru here, because that would try to
    6272              :              use return.  Inserting a jump 'by hand' is extremely messy, so
    6273              :              we take advantage of cfg_layout_finalize using
    6274              :              fixup_fallthru_exit_predecessor.  */
    6275            0 :           cfg_layout_initialize (0);
    6276            0 :           basic_block cur_bb;
    6277            0 :           FOR_EACH_BB_FN (cur_bb, cfun)
    6278            0 :             if (cur_bb->index >= NUM_FIXED_BLOCKS
    6279            0 :                 && cur_bb->next_bb->index >= NUM_FIXED_BLOCKS)
    6280            0 :               cur_bb->aux = cur_bb->next_bb;
    6281            0 :           cfg_layout_finalize ();
    6282              :         }
    6283              :     }
    6284              : 
    6285              :   /* Insert the prologue.  */
    6286              : 
    6287      1517128 :   rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (cfun));
    6288              : 
    6289      1517128 :   if (split_prologue_seq || prologue_seq)
    6290              :     {
    6291      1517128 :       rtx_insn *split_prologue_insn = split_prologue_seq;
    6292      1517128 :       if (split_prologue_seq)
    6293              :         {
    6294       259990 :           while (split_prologue_insn && !NONDEBUG_INSN_P (split_prologue_insn))
    6295            0 :             split_prologue_insn = NEXT_INSN (split_prologue_insn);
    6296       259990 :           insert_insn_on_edge (split_prologue_seq, orig_entry_edge);
    6297              :         }
    6298              : 
    6299      1517128 :       rtx_insn *prologue_insn = prologue_seq;
    6300      1517128 :       if (prologue_seq)
    6301              :         {
    6302      1916802 :           while (prologue_insn && !NONDEBUG_INSN_P (prologue_insn))
    6303       399674 :             prologue_insn = NEXT_INSN (prologue_insn);
    6304      1517128 :           insert_insn_on_edge (prologue_seq, entry_edge);
    6305              :         }
    6306              : 
    6307      1517128 :       commit_edge_insertions ();
    6308              : 
    6309              :       /* Look for basic blocks within the prologue insns.  */
    6310      1517128 :       if (split_prologue_insn
    6311      1517128 :           && BLOCK_FOR_INSN (split_prologue_insn) == NULL)
    6312              :         split_prologue_insn = NULL;
    6313      1517128 :       if (prologue_insn
    6314      1517128 :           && BLOCK_FOR_INSN (prologue_insn) == NULL)
    6315              :         prologue_insn = NULL;
    6316      1517128 :       if (split_prologue_insn || prologue_insn)
    6317              :         {
    6318      1141082 :           auto_sbitmap blocks (last_basic_block_for_fn (cfun));
    6319      1141082 :           bitmap_clear (blocks);
    6320      1141082 :           if (split_prologue_insn)
    6321       259990 :             bitmap_set_bit (blocks,
    6322       259990 :                             BLOCK_FOR_INSN (split_prologue_insn)->index);
    6323      1141082 :           if (prologue_insn)
    6324      1117454 :             bitmap_set_bit (blocks, BLOCK_FOR_INSN (prologue_insn)->index);
    6325      1141082 :           find_many_sub_basic_blocks (blocks);
    6326      1141082 :         }
    6327              :     }
    6328              : 
    6329      1517128 :   default_rtl_profile ();
    6330              : 
    6331              :   /* Emit sibling epilogues before any sibling call sites.  */
    6332      1517128 :   for (ei = ei_start (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds);
    6333      3088223 :        (e = ei_safe_edge (ei));
    6334      1571095 :        ei_next (&ei))
    6335              :     {
    6336              :       /* Skip those already handled, the ones that run without prologue.  */
    6337      1571095 :       if (e->flags & EDGE_IGNORE)
    6338              :         {
    6339         5063 :           e->flags &= ~EDGE_IGNORE;
    6340         5063 :           continue;
    6341              :         }
    6342              : 
    6343      1566032 :       rtx_insn *insn = BB_END (e->src);
    6344              : 
    6345      1566032 :       auto call = dyn_cast<rtx_call_insn *> (insn);
    6346       124434 :       if (!(call && SIBLING_CALL_P (call)))
    6347      1441598 :         continue;
    6348              : 
    6349       124434 :       rtx_insn *ep_seq;
    6350       124434 :       if (targetm.emit_epilogue_for_sibcall)
    6351              :         {
    6352            0 :           start_sequence ();
    6353            0 :           targetm.emit_epilogue_for_sibcall (as_a<rtx_call_insn *> (insn));
    6354            0 :           ep_seq = end_sequence ();
    6355              :         }
    6356              :       else
    6357       124434 :         ep_seq = targetm.gen_sibcall_epilogue ();
    6358       124434 :       if (ep_seq)
    6359              :         {
    6360        59047 :           start_sequence ();
    6361        59047 :           emit_note (NOTE_INSN_EPILOGUE_BEG);
    6362        59047 :           emit_insn (ep_seq);
    6363        59047 :           rtx_insn *seq = end_sequence ();
    6364              : 
    6365              :           /* Retain a map of the epilogue insns.  Used in life analysis to
    6366              :              avoid getting rid of sibcall epilogue insns.  Do this before we
    6367              :              actually emit the sequence.  */
    6368        59047 :           record_insns (seq, NULL, &epilogue_insn_hash);
    6369        59047 :           set_insn_locations (seq, epilogue_location);
    6370              : 
    6371        59047 :           emit_insn_before (seq, insn);
    6372              : 
    6373        59047 :           find_sub_basic_blocks (BLOCK_FOR_INSN (insn));
    6374              :         }
    6375              :     }
    6376              : 
    6377      1517128 :   if (epilogue_seq)
    6378              :     {
    6379              :       rtx_insn *insn, *next;
    6380              : 
    6381              :       /* Similarly, move any line notes that appear after the epilogue.
    6382              :          There is no need, however, to be quite so anal about the existence
    6383              :          of such a note.  Also possibly move
    6384              :          NOTE_INSN_FUNCTION_BEG notes, as those can be relevant for debug
    6385              :          info generation.  */
    6386     11634304 :       for (insn = epilogue_seq; insn; insn = next)
    6387              :         {
    6388     10117176 :           next = NEXT_INSN (insn);
    6389     10117176 :           if (NOTE_P (insn)
    6390      3180941 :               && (NOTE_KIND (insn) == NOTE_INSN_FUNCTION_BEG))
    6391            0 :             reorder_insns (insn, insn, PREV_INSN (epilogue_seq));
    6392              :         }
    6393              :     }
    6394              : 
    6395              :   /* Threading the prologue and epilogue changes the artificial refs in the
    6396              :      entry and exit blocks, and may invalidate DF info for tail calls.
    6397              :      This is also needed for [[musttail]] conversion even when not
    6398              :      optimizing.  */
    6399      1517128 :   if (optimize
    6400       452165 :       || cfun->tail_call_marked
    6401       451997 :       || flag_optimize_sibling_calls
    6402       451956 :       || flag_ipa_icf_functions
    6403       451899 :       || in_lto_p)
    6404      1073715 :     df_update_entry_exit_and_calls ();
    6405              :   else
    6406              :     {
    6407       443413 :       df_update_entry_block_defs ();
    6408       443413 :       df_update_exit_block_uses ();
    6409              :     }
    6410      1517128 : }
    6411              : 
    6412              : /* Reposition the prologue-end and epilogue-begin notes after
    6413              :    instruction scheduling.  */
    6414              : 
    6415              : void
    6416       982732 : reposition_prologue_and_epilogue_notes (void)
    6417              : {
    6418       982732 :   if (!targetm.have_prologue ()
    6419            0 :       && !targetm.have_epilogue ()
    6420            0 :       && !targetm.have_sibcall_epilogue ()
    6421       982732 :       && !targetm.emit_epilogue_for_sibcall)
    6422              :     return;
    6423              : 
    6424              :   /* Since the hash table is created on demand, the fact that it is
    6425              :      non-null is a signal that it is non-empty.  */
    6426       982732 :   if (prologue_insn_hash != NULL)
    6427              :     {
    6428       982732 :       size_t len = prologue_insn_hash->elements ();
    6429       982732 :       rtx_insn *insn, *last = NULL, *note = NULL;
    6430              : 
    6431              :       /* Scan from the beginning until we reach the last prologue insn.  */
    6432              :       /* ??? While we do have the CFG intact, there are two problems:
    6433              :          (1) The prologue can contain loops (typically probing the stack),
    6434              :              which means that the end of the prologue isn't in the first bb.
    6435              :          (2) Sometimes the PROLOGUE_END note gets pushed into the next bb.  */
    6436     69548794 :       for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
    6437              :         {
    6438     69004155 :           if (NOTE_P (insn))
    6439              :             {
    6440     12945091 :               if (NOTE_KIND (insn) == NOTE_INSN_PROLOGUE_END)
    6441     68566062 :                 note = insn;
    6442              :             }
    6443     56059064 :           else if (contains (insn, prologue_insn_hash))
    6444              :             {
    6445      3479302 :               last = insn;
    6446      3479302 :               if (--len == 0)
    6447              :                 break;
    6448              :             }
    6449              :         }
    6450              : 
    6451       982732 :       if (last)
    6452              :         {
    6453       636606 :           if (note == NULL)
    6454              :             {
    6455              :               /* Scan forward looking for the PROLOGUE_END note.  It should
    6456              :                  be right at the beginning of the block, possibly with other
    6457              :                  insn notes that got moved there.  */
    6458         1452 :               for (note = NEXT_INSN (last); ; note = NEXT_INSN (note))
    6459              :                 {
    6460         1452 :                   if (NOTE_P (note)
    6461         1334 :                       && NOTE_KIND (note) == NOTE_INSN_PROLOGUE_END)
    6462              :                     break;
    6463              :                 }
    6464              :             }
    6465              : 
    6466              :           /* Avoid placing note between CODE_LABEL and BASIC_BLOCK note.  */
    6467       636606 :           if (LABEL_P (last))
    6468          229 :             last = NEXT_INSN (last);
    6469       636606 :           reorder_insns (note, note, last);
    6470              :         }
    6471              :     }
    6472              : 
    6473       982732 :   if (epilogue_insn_hash != NULL)
    6474              :     {
    6475       982732 :       edge_iterator ei;
    6476       982732 :       edge e;
    6477              : 
    6478      2230939 :       FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    6479              :         {
    6480      1248207 :           rtx_insn *insn, *first = NULL, *note = NULL;
    6481      1248207 :           basic_block bb = e->src;
    6482              : 
    6483              :           /* Scan from the beginning until we reach the first epilogue insn. */
    6484     16940461 :           FOR_BB_INSNS (bb, insn)
    6485              :             {
    6486     16789380 :               if (NOTE_P (insn))
    6487              :                 {
    6488      4036348 :                   if (NOTE_KIND (insn) == NOTE_INSN_EPILOGUE_BEG)
    6489              :                     {
    6490      1097126 :                       note = insn;
    6491      1097126 :                       if (first != NULL)
    6492              :                         break;
    6493              :                     }
    6494              :                 }
    6495     12753032 :               else if (first == NULL && contains (insn, epilogue_insn_hash))
    6496              :                 {
    6497      1097130 :                   first = insn;
    6498      1097130 :                   if (note != NULL)
    6499              :                     break;
    6500              :                 }
    6501              :             }
    6502              : 
    6503      1248207 :           if (note)
    6504              :             {
    6505              :               /* If the function has a single basic block, and no real
    6506              :                  epilogue insns (e.g. sibcall with no cleanup), the
    6507              :                  epilogue note can get scheduled before the prologue
    6508              :                  note.  If we have frame related prologue insns, having
    6509              :                  them scanned during the epilogue will result in a crash.
    6510              :                  In this case re-order the epilogue note to just before
    6511              :                  the last insn in the block.  */
    6512      1097126 :               if (first == NULL)
    6513            0 :                 first = BB_END (bb);
    6514              : 
    6515      1097126 :               if (PREV_INSN (first) != note)
    6516        32048 :                 reorder_insns (note, note, PREV_INSN (first));
    6517              :             }
    6518              :         }
    6519              :     }
    6520              : }
    6521              : 
    6522              : /* Returns the name of function declared by FNDECL.  */
    6523              : const char *
    6524       158620 : fndecl_name (tree fndecl)
    6525              : {
    6526       158620 :   if (fndecl == NULL)
    6527              :     return "(nofn)";
    6528       158600 :   return lang_hooks.decl_printable_name (fndecl, 1);
    6529              : }
    6530              : 
    6531              : /* Returns the name of function FN.  */
    6532              : const char *
    6533       158490 : function_name (const function *fn)
    6534              : {
    6535       158490 :   tree fndecl = (fn == NULL) ? NULL : fn->decl;
    6536       158490 :   return fndecl_name (fndecl);
    6537              : }
    6538              : 
    6539              : /* Returns the name of the current function.  */
    6540              : const char *
    6541         8783 : current_function_name (void)
    6542              : {
    6543         8783 :   return function_name (cfun);
    6544              : }
    6545              : 
    6546              : 
    6547              : static void
    6548            0 : rest_of_handle_check_leaf_regs (void)
    6549              : {
    6550              : #ifdef LEAF_REGISTERS
    6551              :   crtl->uses_only_leaf_regs
    6552              :     = optimize > 0 && only_leaf_regs_used () && leaf_function_p ();
    6553              : #endif
    6554            0 : }
    6555              : 
    6556              : /* Insert a TYPE into the used types hash table of CFUN.  */
    6557              : 
    6558              : static void
    6559     58938036 : used_types_insert_helper (tree type, struct function *func)
    6560              : {
    6561     58938036 :   if (type != NULL && func != NULL)
    6562              :     {
    6563     58938036 :       if (func->used_types_hash == NULL)
    6564     17828622 :         func->used_types_hash = hash_set<tree>::create_ggc (37);
    6565              : 
    6566     58938036 :       func->used_types_hash->add (type);
    6567              :     }
    6568     58938036 : }
    6569              : 
    6570              : /* Given a type, insert it into the used hash table in cfun.  */
    6571              : void
    6572    216633357 : used_types_insert (tree t)
    6573              : {
    6574    226708263 :   while (POINTER_TYPE_P (t) || TREE_CODE (t) == ARRAY_TYPE)
    6575     10340482 :     if (TYPE_NAME (t))
    6576              :       break;
    6577              :     else
    6578     10074906 :       t = TREE_TYPE (t);
    6579    216633357 :   if (TREE_CODE (t) == ERROR_MARK)
    6580              :     return;
    6581    216633352 :   if (TYPE_NAME (t) == NULL_TREE
    6582    216633352 :       || TYPE_NAME (t) == TYPE_NAME (TYPE_MAIN_VARIANT (t)))
    6583     80880019 :     t = TYPE_MAIN_VARIANT (t);
    6584    216633352 :   if (debug_info_level > DINFO_LEVEL_NONE)
    6585              :     {
    6586     86475434 :       if (cfun)
    6587     58938036 :         used_types_insert_helper (t, cfun);
    6588              :       else
    6589              :         {
    6590              :           /* So this might be a type referenced by a global variable.
    6591              :              Record that type so that we can later decide to emit its
    6592              :              debug information.  */
    6593     27537398 :           vec_safe_push (types_used_by_cur_var_decl, t);
    6594              :         }
    6595              :     }
    6596              : }
    6597              : 
    6598              : /* Helper to Hash a struct types_used_by_vars_entry.  */
    6599              : 
    6600              : static hashval_t
    6601    180703524 : hash_types_used_by_vars_entry (const struct types_used_by_vars_entry *entry)
    6602              : {
    6603    180703524 :   gcc_assert (entry && entry->var_decl && entry->type);
    6604              : 
    6605    180703524 :   return iterative_hash_object (entry->type,
    6606              :                                 iterative_hash_object (entry->var_decl, 0));
    6607              : }
    6608              : 
    6609              : /* Hash function of the types_used_by_vars_entry hash table.  */
    6610              : 
    6611              : hashval_t
    6612    180703524 : used_type_hasher::hash (types_used_by_vars_entry *entry)
    6613              : {
    6614    180703524 :   return hash_types_used_by_vars_entry (entry);
    6615              : }
    6616              : 
    6617              : /*Equality function of the types_used_by_vars_entry hash table.  */
    6618              : 
    6619              : bool
    6620    197071050 : used_type_hasher::equal (types_used_by_vars_entry *e1,
    6621              :                          types_used_by_vars_entry *e2)
    6622              : {
    6623    197071050 :   return (e1->var_decl == e2->var_decl && e1->type == e2->type);
    6624              : }
    6625              : 
    6626              : /* Inserts an entry into the types_used_by_vars_hash hash table. */
    6627              : 
    6628              : void
    6629     27465341 : types_used_by_var_decl_insert (tree type, tree var_decl)
    6630              : {
    6631     27465341 :   if (type != NULL && var_decl != NULL)
    6632              :     {
    6633     27465341 :       types_used_by_vars_entry **slot;
    6634     27465341 :       struct types_used_by_vars_entry e;
    6635     27465341 :       e.var_decl = var_decl;
    6636     27465341 :       e.type = type;
    6637     27465341 :       if (types_used_by_vars_hash == NULL)
    6638        14649 :         types_used_by_vars_hash
    6639        14649 :           = hash_table<used_type_hasher>::create_ggc (37);
    6640              : 
    6641     27465341 :       slot = types_used_by_vars_hash->find_slot (&e, INSERT);
    6642     27465341 :       if (*slot == NULL)
    6643              :         {
    6644      7851901 :           struct types_used_by_vars_entry *entry;
    6645      7851901 :           entry = ggc_alloc<types_used_by_vars_entry> ();
    6646      7851901 :           entry->type = type;
    6647      7851901 :           entry->var_decl = var_decl;
    6648      7851901 :           *slot = entry;
    6649              :         }
    6650              :     }
    6651     27465341 : }
    6652              : 
    6653              : namespace {
    6654              : 
    6655              : const pass_data pass_data_leaf_regs =
    6656              : {
    6657              :   RTL_PASS, /* type */
    6658              :   "*leaf_regs", /* name */
    6659              :   OPTGROUP_NONE, /* optinfo_flags */
    6660              :   TV_NONE, /* tv_id */
    6661              :   0, /* properties_required */
    6662              :   0, /* properties_provided */
    6663              :   0, /* properties_destroyed */
    6664              :   0, /* todo_flags_start */
    6665              :   0, /* todo_flags_finish */
    6666              : };
    6667              : 
    6668              : class pass_leaf_regs : public rtl_opt_pass
    6669              : {
    6670              : public:
    6671       297180 :   pass_leaf_regs (gcc::context *ctxt)
    6672       594360 :     : rtl_opt_pass (pass_data_leaf_regs, ctxt)
    6673              :   {}
    6674              : 
    6675              :   /* opt_pass methods: */
    6676      1517128 :   unsigned int execute (function *) final override
    6677              :     {
    6678      1517128 :       rest_of_handle_check_leaf_regs ();
    6679      1517128 :       return 0;
    6680              :     }
    6681              : 
    6682              : }; // class pass_leaf_regs
    6683              : 
    6684              : } // anon namespace
    6685              : 
    6686              : rtl_opt_pass *
    6687       297180 : make_pass_leaf_regs (gcc::context *ctxt)
    6688              : {
    6689       297180 :   return new pass_leaf_regs (ctxt);
    6690              : }
    6691              : 
    6692              : static void
    6693      1517128 : rest_of_handle_thread_prologue_and_epilogue (function *fun)
    6694              : {
    6695              :   /* prepare_shrink_wrap is sensitive to the block structure of the control
    6696              :      flow graph, so clean it up first.  */
    6697      1517128 :   if (cfun->tail_call_marked || optimize)
    6698      1065131 :     cleanup_cfg (0);
    6699              : 
    6700              :   /* On some machines, the prologue and epilogue code, or parts thereof,
    6701              :      can be represented as RTL.  Doing so lets us schedule insns between
    6702              :      it and the rest of the code and also allows delayed branch
    6703              :      scheduling to operate in the epilogue.  */
    6704      1517128 :   thread_prologue_and_epilogue_insns ();
    6705              : 
    6706              :   /* Some non-cold blocks may now be only reachable from cold blocks.
    6707              :      Fix that up.  */
    6708      1517128 :   fixup_partitions ();
    6709              : 
    6710              :   /* After prologue and epilogue generation, the judgement on whether
    6711              :      one memory access onto stack frame may trap or not could change,
    6712              :      since we get more exact stack information by now.  So try to
    6713              :      remove any EH edges here, see PR90259.  */
    6714      1517128 :   if (fun->can_throw_non_call_exceptions)
    6715       263022 :     purge_all_dead_edges ();
    6716              : 
    6717              :   /* Shrink-wrapping can result in unreachable edges in the epilogue,
    6718              :      see PR57320.  */
    6719      1969293 :   cleanup_cfg (optimize ? CLEANUP_EXPENSIVE : 0);
    6720              : 
    6721              :   /* The stack usage info is finalized during prologue expansion.  */
    6722      1517128 :   if (flag_stack_usage_info || flag_callgraph_info)
    6723          356 :     output_stack_usage ();
    6724      1517128 : }
    6725              : 
    6726              : /* Record a final call to CALLEE at LOCATION.  */
    6727              : 
    6728              : void
    6729            0 : record_final_call (tree callee, location_t location)
    6730              : {
    6731            0 :   struct callinfo_callee datum = { location, callee };
    6732            0 :   vec_safe_push (cfun->su->callees, datum);
    6733            0 : }
    6734              : 
    6735              : /* Record a dynamic allocation made for DECL_OR_EXP.  */
    6736              : 
    6737              : void
    6738            0 : record_dynamic_alloc (tree decl_or_exp)
    6739              : {
    6740            0 :   struct callinfo_dalloc datum;
    6741              : 
    6742            0 :   if (DECL_P (decl_or_exp))
    6743              :     {
    6744            0 :       datum.location = DECL_SOURCE_LOCATION (decl_or_exp);
    6745            0 :       const char *name = lang_hooks.decl_printable_name (decl_or_exp, 2);
    6746            0 :       const char *dot = strrchr (name, '.');
    6747            0 :       if (dot)
    6748            0 :         name = dot + 1;
    6749            0 :       datum.name = ggc_strdup (name);
    6750              :     }
    6751              :   else
    6752              :     {
    6753            0 :       datum.location = EXPR_LOCATION (decl_or_exp);
    6754            0 :       datum.name = NULL;
    6755              :     }
    6756              : 
    6757            0 :   vec_safe_push (cfun->su->dallocs, datum);
    6758            0 : }
    6759              : 
    6760              : namespace {
    6761              : 
    6762              : const pass_data pass_data_thread_prologue_and_epilogue =
    6763              : {
    6764              :   RTL_PASS, /* type */
    6765              :   "pro_and_epilogue", /* name */
    6766              :   OPTGROUP_NONE, /* optinfo_flags */
    6767              :   TV_THREAD_PROLOGUE_AND_EPILOGUE, /* tv_id */
    6768              :   0, /* properties_required */
    6769              :   0, /* properties_provided */
    6770              :   0, /* properties_destroyed */
    6771              :   0, /* todo_flags_start */
    6772              :   ( TODO_df_verify | TODO_df_finish ), /* todo_flags_finish */
    6773              : };
    6774              : 
    6775              : class pass_thread_prologue_and_epilogue : public rtl_opt_pass
    6776              : {
    6777              : public:
    6778       297180 :   pass_thread_prologue_and_epilogue (gcc::context *ctxt)
    6779       594360 :     : rtl_opt_pass (pass_data_thread_prologue_and_epilogue, ctxt)
    6780              :   {}
    6781              : 
    6782              :   /* opt_pass methods: */
    6783      1517135 :   bool gate (function *) final override
    6784              :     {
    6785      1517135 :       return !targetm.use_late_prologue_epilogue ();
    6786              :     }
    6787              : 
    6788      1517128 :   unsigned int execute (function * fun) final override
    6789              :     {
    6790      1517128 :       rest_of_handle_thread_prologue_and_epilogue (fun);
    6791      1517128 :       return 0;
    6792              :     }
    6793              : 
    6794              : }; // class pass_thread_prologue_and_epilogue
    6795              : 
    6796              : const pass_data pass_data_late_thread_prologue_and_epilogue =
    6797              : {
    6798              :   RTL_PASS, /* type */
    6799              :   "late_pro_and_epilogue", /* name */
    6800              :   OPTGROUP_NONE, /* optinfo_flags */
    6801              :   TV_THREAD_PROLOGUE_AND_EPILOGUE, /* tv_id */
    6802              :   0, /* properties_required */
    6803              :   0, /* properties_provided */
    6804              :   0, /* properties_destroyed */
    6805              :   0, /* todo_flags_start */
    6806              :   ( TODO_df_verify | TODO_df_finish ), /* todo_flags_finish */
    6807              : };
    6808              : 
    6809              : class pass_late_thread_prologue_and_epilogue : public rtl_opt_pass
    6810              : {
    6811              : public:
    6812       297180 :   pass_late_thread_prologue_and_epilogue (gcc::context *ctxt)
    6813       594360 :     : rtl_opt_pass (pass_data_late_thread_prologue_and_epilogue, ctxt)
    6814              :   {}
    6815              : 
    6816              :   /* opt_pass methods: */
    6817      1517135 :   bool gate (function *) final override
    6818              :     {
    6819      1517135 :       return targetm.use_late_prologue_epilogue ();
    6820              :     }
    6821              : 
    6822            0 :   unsigned int execute (function *fn) final override
    6823              :     {
    6824              :       /* It's not currently possible to have both delay slots and
    6825              :          late prologue/epilogue, since the latter has to run before
    6826              :          the former, and the former won't honor whatever restrictions
    6827              :          the latter is trying to enforce.  */
    6828            0 :       gcc_assert (!DELAY_SLOTS);
    6829            0 :       rest_of_handle_thread_prologue_and_epilogue (fn);
    6830            0 :       return 0;
    6831              :     }
    6832              : }; // class pass_late_thread_prologue_and_epilogue
    6833              : 
    6834              : } // anon namespace
    6835              : 
    6836              : rtl_opt_pass *
    6837       297180 : make_pass_thread_prologue_and_epilogue (gcc::context *ctxt)
    6838              : {
    6839       297180 :   return new pass_thread_prologue_and_epilogue (ctxt);
    6840              : }
    6841              : 
    6842              : rtl_opt_pass *
    6843       297180 : make_pass_late_thread_prologue_and_epilogue (gcc::context *ctxt)
    6844              : {
    6845       297180 :   return new pass_late_thread_prologue_and_epilogue (ctxt);
    6846              : }
    6847              : 
    6848              : namespace {
    6849              : 
    6850              : const pass_data pass_data_zero_call_used_regs =
    6851              : {
    6852              :   RTL_PASS, /* type */
    6853              :   "zero_call_used_regs", /* name */
    6854              :   OPTGROUP_NONE, /* optinfo_flags */
    6855              :   TV_NONE, /* tv_id */
    6856              :   0, /* properties_required */
    6857              :   0, /* properties_provided */
    6858              :   0, /* properties_destroyed */
    6859              :   0, /* todo_flags_start */
    6860              :   0, /* todo_flags_finish */
    6861              : };
    6862              : 
    6863              : class pass_zero_call_used_regs: public rtl_opt_pass
    6864              : {
    6865              : public:
    6866       297180 :   pass_zero_call_used_regs (gcc::context *ctxt)
    6867       594360 :     : rtl_opt_pass (pass_data_zero_call_used_regs, ctxt)
    6868              :   {}
    6869              : 
    6870              :   /* opt_pass methods: */
    6871              :   unsigned int execute (function *) final override;
    6872              : 
    6873              : }; // class pass_zero_call_used_regs
    6874              : 
    6875              : unsigned int
    6876      1517128 : pass_zero_call_used_regs::execute (function *fun)
    6877              : {
    6878      1517128 :   using namespace zero_regs_flags;
    6879      1517128 :   unsigned int zero_regs_type = UNSET;
    6880              : 
    6881      1517128 :   tree attr_zero_regs = lookup_attribute ("zero_call_used_regs",
    6882      1517128 :                                           DECL_ATTRIBUTES (fun->decl));
    6883              : 
    6884              :   /* Get the type of zero_call_used_regs from function attribute.
    6885              :      We have filtered out invalid attribute values already at this point.  */
    6886      1517128 :   if (attr_zero_regs)
    6887              :     {
    6888              :       /* The TREE_VALUE of an attribute is a TREE_LIST whose TREE_VALUE
    6889              :          is the attribute argument's value.  */
    6890           88 :       attr_zero_regs = TREE_VALUE (attr_zero_regs);
    6891           88 :       gcc_assert (TREE_CODE (attr_zero_regs) == TREE_LIST);
    6892           88 :       attr_zero_regs = TREE_VALUE (attr_zero_regs);
    6893           88 :       gcc_assert (TREE_CODE (attr_zero_regs) == STRING_CST);
    6894              : 
    6895          496 :       for (unsigned int i = 0; zero_call_used_regs_opts[i].name != NULL; ++i)
    6896          496 :         if (strcmp (TREE_STRING_POINTER (attr_zero_regs),
    6897          496 :                      zero_call_used_regs_opts[i].name) == 0)
    6898              :           {
    6899           88 :             zero_regs_type = zero_call_used_regs_opts[i].flag;
    6900           88 :             break;
    6901              :           }
    6902              :     }
    6903              : 
    6904           88 :   if (!zero_regs_type)
    6905      1517040 :     zero_regs_type = flag_zero_call_used_regs;
    6906              : 
    6907              :   /* No need to zero call-used-regs when no user request is present.  */
    6908      1517128 :   if (!(zero_regs_type & ENABLED))
    6909              :     return 0;
    6910              : 
    6911          183 :   edge_iterator ei;
    6912          183 :   edge e;
    6913              : 
    6914              :   /* This pass needs data flow information.  */
    6915          183 :   df_analyze ();
    6916              : 
    6917              :   /* Iterate over the function's return instructions and insert any
    6918              :      register zeroing required by the -fzero-call-used-regs command-line
    6919              :      option or the "zero_call_used_regs" function attribute.  */
    6920          183 :   bool zeroed = false;
    6921          367 :   FOR_EACH_EDGE (e, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
    6922              :     {
    6923          184 :       rtx_insn *insn = BB_END (e->src);
    6924          184 :       if (JUMP_P (insn) && ANY_RETURN_P (JUMP_LABEL (insn)))
    6925          177 :         zeroed |= gen_call_used_regs_seq (insn, zero_regs_type);
    6926              :     }
    6927              : 
    6928              :   /* The registers in must_be_zero_on_return are uses of the exit block, see
    6929              :      df_epilogue_uses_p.  Growing that set also changes which registers a
    6930              :      sibling call is recorded as clobbering, see df_get_call_refs, so the
    6931              :      calls have to be rescanned as well.  */
    6932          183 :   if (zeroed)
    6933          131 :     df_update_entry_exit_and_calls ();
    6934              : 
    6935              :   return 0;
    6936              : }
    6937              : 
    6938              : } // anon namespace
    6939              : 
    6940              : rtl_opt_pass *
    6941       297180 : make_pass_zero_call_used_regs (gcc::context *ctxt)
    6942              : {
    6943       297180 :   return new pass_zero_call_used_regs (ctxt);
    6944              : }
    6945              : 
    6946              : /* If CONSTRAINT is a matching constraint, then return its number.
    6947              :    Otherwise, return -1.  */
    6948              : 
    6949              : static int
    6950        46420 : matching_constraint_num (const char *constraint)
    6951              : {
    6952        46420 :   if (*constraint == '%')
    6953         1140 :     constraint++;
    6954              : 
    6955        46420 :   if (IN_RANGE (*constraint, '0', '9'))
    6956        33400 :     return strtoul (constraint, NULL, 10);
    6957              : 
    6958              :   return -1;
    6959              : }
    6960              : 
    6961              : /* This mini-pass fixes fall-out from SSA in asm statements that have
    6962              :    in-out constraints.  Say you start with
    6963              : 
    6964              :      orig = inout;
    6965              :      asm ("": "+mr" (inout));
    6966              :      use (orig);
    6967              : 
    6968              :    which is transformed very early to use explicit output and match operands:
    6969              : 
    6970              :      orig = inout;
    6971              :      asm ("": "=mr" (inout) : "0" (inout));
    6972              :      use (orig);
    6973              : 
    6974              :    Or, after SSA and copyprop,
    6975              : 
    6976              :      asm ("": "=mr" (inout_2) : "0" (inout_1));
    6977              :      use (inout_1);
    6978              : 
    6979              :    Clearly inout_2 and inout_1 can't be coalesced easily anymore, as
    6980              :    they represent two separate values, so they will get different pseudo
    6981              :    registers during expansion.  Then, since the two operands need to match
    6982              :    per the constraints, but use different pseudo registers, reload can
    6983              :    only register a reload for these operands.  But reloads can only be
    6984              :    satisfied by hardregs, not by memory, so we need a register for this
    6985              :    reload, just because we are presented with non-matching operands.
    6986              :    So, even though we allow memory for this operand, no memory can be
    6987              :    used for it, just because the two operands don't match.  This can
    6988              :    cause reload failures on register-starved targets.
    6989              : 
    6990              :    So it's a symptom of reload not being able to use memory for reloads
    6991              :    or, alternatively it's also a symptom of both operands not coming into
    6992              :    reload as matching (in which case the pseudo could go to memory just
    6993              :    fine, as the alternative allows it, and no reload would be necessary).
    6994              :    We fix the latter problem here, by transforming
    6995              : 
    6996              :      asm ("": "=mr" (inout_2) : "0" (inout_1));
    6997              : 
    6998              :    back to
    6999              : 
    7000              :      inout_2 = inout_1;
    7001              :      asm ("": "=mr" (inout_2) : "0" (inout_2));  */
    7002              : 
    7003              : static void
    7004        36053 : match_asm_constraints_1 (rtx_insn *insn, rtx *p_sets, int noutputs)
    7005              : {
    7006        36053 :   int i;
    7007        36053 :   bool changed = false;
    7008        36053 :   rtx op = SET_SRC (p_sets[0]);
    7009        36053 :   int ninputs = ASM_OPERANDS_INPUT_LENGTH (op);
    7010        36053 :   rtvec inputs = ASM_OPERANDS_INPUT_VEC (op);
    7011        36053 :   bool *output_matched = XALLOCAVEC (bool, noutputs);
    7012              : 
    7013        36053 :   memset (output_matched, 0, noutputs * sizeof (bool));
    7014        81544 :   for (i = 0; i < ninputs; i++)
    7015              :     {
    7016        45491 :       rtx input, output;
    7017        45491 :       rtx_insn *insns;
    7018        45491 :       const char *constraint = ASM_OPERANDS_INPUT_CONSTRAINT (op, i);
    7019        45491 :       int match, j;
    7020              : 
    7021        45491 :       match = matching_constraint_num (constraint);
    7022        45491 :       if (match < 0)
    7023        13016 :         continue;
    7024              : 
    7025        32475 :       gcc_assert (match < noutputs);
    7026        32475 :       output = SET_DEST (p_sets[match]);
    7027        32475 :       input = RTVEC_ELT (inputs, i);
    7028              :       /* Only do the transformation for pseudos.  */
    7029        33508 :       if (! REG_P (output)
    7030        32298 :           || rtx_equal_p (output, input)
    7031        31637 :           || !(REG_P (input) || SUBREG_P (input)
    7032         3160 :                || MEM_P (input) || CONSTANT_P (input))
    7033        64111 :           || !general_operand (input, GET_MODE (output)))
    7034         1033 :         continue;
    7035              : 
    7036              :       /* We can't do anything if the output is also used as input,
    7037              :          as we're going to overwrite it.  */
    7038        86149 :       for (j = 0; j < ninputs; j++)
    7039        54707 :         if (reg_overlap_mentioned_p (output, RTVEC_ELT (inputs, j)))
    7040              :           break;
    7041        31442 :       if (j != ninputs)
    7042            0 :         continue;
    7043              : 
    7044              :       /* Avoid changing the same input several times.  For
    7045              :          asm ("" : "=mr" (out1), "=mr" (out2) : "0" (in), "1" (in));
    7046              :          only change it once (to out1), rather than changing it
    7047              :          first to out1 and afterwards to out2.  */
    7048        31442 :       if (i > 0)
    7049              :         {
    7050        40954 :           for (j = 0; j < noutputs; j++)
    7051        33460 :             if (output_matched[j] && input == SET_DEST (p_sets[j]))
    7052              :               break;
    7053         7557 :           if (j != noutputs)
    7054           63 :             continue;
    7055              :         }
    7056        31379 :       output_matched[match] = true;
    7057              : 
    7058        31379 :       start_sequence ();
    7059        31379 :       emit_move_insn (output, copy_rtx (input));
    7060        31379 :       insns = end_sequence ();
    7061        31379 :       emit_insn_before (insns, insn);
    7062              : 
    7063        31379 :       constraint = ASM_OPERANDS_OUTPUT_CONSTRAINT(SET_SRC(p_sets[match]));
    7064        31379 :       bool early_clobber_p = strchr (constraint, '&') != NULL;
    7065              : 
    7066              :       /* Now replace all mentions of the input with output.  We can't
    7067              :          just replace the occurrence in inputs[i], as the register might
    7068              :          also be used in some other input (or even in an address of an
    7069              :          output), which would mean possibly increasing the number of
    7070              :          inputs by one (namely 'output' in addition), which might pose
    7071              :          a too complicated problem for reload to solve.  E.g. this situation:
    7072              : 
    7073              :            asm ("" : "=r" (output), "=m" (input) : "0" (input))
    7074              : 
    7075              :          Here 'input' is used in two occurrences as input (once for the
    7076              :          input operand, once for the address in the second output operand).
    7077              :          If we would replace only the occurrence of the input operand (to
    7078              :          make the matching) we would be left with this:
    7079              : 
    7080              :            output = input
    7081              :            asm ("" : "=r" (output), "=m" (input) : "0" (output))
    7082              : 
    7083              :          Now we suddenly have two different input values (containing the same
    7084              :          value, but different pseudos) where we formerly had only one.
    7085              :          With more complicated asms this might lead to reload failures
    7086              :          which wouldn't have happen without this pass.  So, iterate over
    7087              :          all operands and replace all occurrences of the register used.
    7088              : 
    7089              :          However, if one or more of the 'input' uses have a non-matching
    7090              :          constraint and the matched output operand is an early clobber
    7091              :          operand, then do not replace the input operand, since by definition
    7092              :          it conflicts with the output operand and cannot share the same
    7093              :          register.  See PR89313 for details.  */
    7094              : 
    7095       153500 :       for (j = 0; j < noutputs; j++)
    7096       122121 :         if (!rtx_equal_p (SET_DEST (p_sets[j]), input)
    7097       122121 :             && reg_overlap_mentioned_p (input, SET_DEST (p_sets[j])))
    7098         1766 :           SET_DEST (p_sets[j]) = replace_rtx (SET_DEST (p_sets[j]),
    7099              :                                               input, output);
    7100        85834 :       for (j = 0; j < ninputs; j++)
    7101        54455 :         if (reg_overlap_mentioned_p (input, RTVEC_ELT (inputs, j)))
    7102              :           {
    7103        30709 :             if (!early_clobber_p
    7104        31638 :                 || match == matching_constraint_num
    7105          929 :                               (ASM_OPERANDS_INPUT_CONSTRAINT (op, j)))
    7106        30705 :               RTVEC_ELT (inputs, j) = replace_rtx (RTVEC_ELT (inputs, j),
    7107              :                                                    input, output);
    7108              :           }
    7109              : 
    7110              :       changed = true;
    7111              :     }
    7112              : 
    7113        36053 :   if (changed)
    7114        24228 :     df_insn_rescan (insn);
    7115        36053 : }
    7116              : 
    7117              : /* It is expected and desired that optimizations coalesce multiple pseudos into
    7118              :    one whenever possible.  However, in case of hard register constraints we may
    7119              :    have to undo this and introduce copies since otherwise we could constraint a
    7120              :    single pseudo to different hard registers.  For example, during register
    7121              :    allocation the following insn would be unsatisfiable since pseudo 60 is
    7122              :    constrained to hard register r5 and r6 at the same time.
    7123              : 
    7124              :    (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
    7125              :                (reg:DI 60) repeated x2
    7126              :            ]
    7127              :             [
    7128              :                (asm_input:DI ("{r5}") t.c:4)
    7129              :                (asm_input:DI ("{r6}") t.c:4)
    7130              :            ]
    7131              :             [] t.c:4) "t.c":4:3 -1
    7132              :         (expr_list:REG_DEAD (reg:DI 60)
    7133              :            (nil)))
    7134              : 
    7135              :    Therefore, introduce a copy of pseudo 60 and transform it into
    7136              : 
    7137              :    (insn 10 5 7 2 (set (reg:DI 62)
    7138              :            (reg:DI 60)) "t.c":4:3 1503 {*movdi_64}
    7139              :         (nil))
    7140              :    (insn 7 10 11 2 (asm_operands/v ("foo") ("") 0 [
    7141              :                (reg:DI 60)
    7142              :                (reg:DI 62)
    7143              :            ]
    7144              :             [
    7145              :                (asm_input:DI ("{r5}") t.c:4)
    7146              :                (asm_input:DI ("{r6}") t.c:4)
    7147              :            ]
    7148              :             [] t.c:4) "t.c":4:3 -1
    7149              :         (expr_list:REG_DEAD (reg:DI 62)
    7150              :            (expr_list:REG_DEAD (reg:DI 60)
    7151              :                (nil))))
    7152              : 
    7153              :    Now, LRA can assign pseudo 60 to r5, and pseudo 62 to r6.
    7154              : 
    7155              :    TODO: The current implementation is conservative and we could do a bit
    7156              :    better in case of alternatives.  For example
    7157              : 
    7158              :    (insn 7 5 0 2 (asm_operands/v ("foo") ("") 0 [
    7159              :                (reg:DI 60) repeated x2
    7160              :            ]
    7161              :             [
    7162              :                (asm_input:DI ("r,{r5}") t.c:4)
    7163              :                (asm_input:DI ("{r6},r") t.c:4)
    7164              :            ]
    7165              :             [] t.c:4) "t.c":4:3 -1
    7166              :         (expr_list:REG_DEAD (reg:DI 60)
    7167              :            (nil)))
    7168              : 
    7169              :    For this insn we wouldn't need to come up with a copy of pseudo 60 since in
    7170              :    each alternative pseudo 60 is constrained exactly one time.  */
    7171              : 
    7172              : static void
    7173      3459378 : match_asm_constraints_2 (rtx_insn *insn, rtx pat)
    7174              : {
    7175      3459378 :   rtx op;
    7176      3459378 :   if (GET_CODE (pat) == SET && GET_CODE (SET_SRC (pat)) == ASM_OPERANDS)
    7177              :     op = SET_SRC (pat);
    7178      3379832 :   else if (GET_CODE (pat) == ASM_OPERANDS)
    7179              :     op = pat;
    7180              :   else
    7181      3322296 :     return;
    7182       137082 :   int ninputs = ASM_OPERANDS_INPUT_LENGTH (op);
    7183       137082 :   rtvec inputs = ASM_OPERANDS_INPUT_VEC (op);
    7184       137082 :   bool changed = false;
    7185       137082 :   auto_bitmap constrained_regs;
    7186              : 
    7187       263483 :   for (int i = 0; i < ninputs; ++i)
    7188              :     {
    7189       126401 :       rtx input = RTVEC_ELT (inputs, i);
    7190       126401 :       const char *constraint = ASM_OPERANDS_INPUT_CONSTRAINT (op, i);
    7191        25279 :       if ((!REG_P (input) && !SUBREG_P (input))
    7192       101239 :           || (REG_P (input) && HARD_REGISTER_P (input))
    7193       226536 :           || strchr (constraint, '{') == nullptr)
    7194       126353 :         continue;
    7195           48 :       int regno;
    7196           48 :       if (SUBREG_P (input))
    7197              :         {
    7198            0 :           if (REG_P (SUBREG_REG (input)))
    7199            0 :             regno = REGNO (SUBREG_REG (input));
    7200              :           else
    7201            0 :             continue;
    7202              :         }
    7203              :       else
    7204           48 :         regno = REGNO (input);
    7205              :       /* Keep the first usage of a constrained pseudo as is and only
    7206              :          introduce copies for subsequent usages.  */
    7207           48 :       if (! bitmap_bit_p (constrained_regs, regno))
    7208              :         {
    7209           48 :           bitmap_set_bit (constrained_regs, regno);
    7210           48 :           continue;
    7211              :         }
    7212            0 :       rtx tmp = gen_reg_rtx (GET_MODE (input));
    7213            0 :       start_sequence ();
    7214            0 :       emit_move_insn (tmp, input);
    7215            0 :       rtx_insn *insns = get_insns ();
    7216            0 :       end_sequence ();
    7217            0 :       emit_insn_before (insns, insn);
    7218            0 :       RTVEC_ELT (inputs, i) = tmp;
    7219            0 :       changed = true;
    7220              :     }
    7221              : 
    7222       137082 :   if (changed)
    7223            0 :     df_insn_rescan (insn);
    7224       137082 : }
    7225              : 
    7226              : /* Add the decl D to the local_decls list of FUN.  */
    7227              : 
    7228              : void
    7229     39541660 : add_local_decl (struct function *fun, tree d)
    7230              : {
    7231     39541660 :   gcc_assert (VAR_P (d));
    7232     39541660 :   vec_safe_push (fun->local_decls, d);
    7233     39541660 : }
    7234              : 
    7235              : namespace {
    7236              : 
    7237              : const pass_data pass_data_match_asm_constraints =
    7238              : {
    7239              :   RTL_PASS, /* type */
    7240              :   "asmcons", /* name */
    7241              :   OPTGROUP_NONE, /* optinfo_flags */
    7242              :   TV_NONE, /* tv_id */
    7243              :   0, /* properties_required */
    7244              :   0, /* properties_provided */
    7245              :   0, /* properties_destroyed */
    7246              :   0, /* todo_flags_start */
    7247              :   0, /* todo_flags_finish */
    7248              : };
    7249              : 
    7250              : class pass_match_asm_constraints : public rtl_opt_pass
    7251              : {
    7252              : public:
    7253       297180 :   pass_match_asm_constraints (gcc::context *ctxt)
    7254       594360 :     : rtl_opt_pass (pass_data_match_asm_constraints, ctxt)
    7255              :   {}
    7256              : 
    7257              :   /* opt_pass methods: */
    7258              :   unsigned int execute (function *) final override;
    7259              : 
    7260              : }; // class pass_match_asm_constraints
    7261              : 
    7262              : unsigned
    7263      1517126 : pass_match_asm_constraints::execute (function *fun)
    7264              : {
    7265      1517126 :   basic_block bb;
    7266      1517126 :   rtx_insn *insn;
    7267      1517126 :   rtx pat, *p_sets;
    7268      1517126 :   int noutputs;
    7269              : 
    7270      1517126 :   if (!crtl->has_asm_statement)
    7271              :     return 0;
    7272              : 
    7273        34909 :   df_set_flags (DF_DEFER_INSN_RESCAN);
    7274       337863 :   FOR_EACH_BB_FN (bb, fun)
    7275              :     {
    7276      3916918 :       FOR_BB_INSNS (bb, insn)
    7277              :         {
    7278      3613964 :           if (!INSN_P (insn))
    7279       675168 :             continue;
    7280              : 
    7281      2938796 :           pat = PATTERN (insn);
    7282              : 
    7283      2938796 :           if (GET_CODE (pat) == PARALLEL)
    7284      1298930 :             for (int i = XVECLEN (pat, 0) - 1; i >= 0; --i)
    7285       909756 :               match_asm_constraints_2 (insn, XVECEXP (pat, 0, i));
    7286              :           else
    7287      2549622 :             match_asm_constraints_2 (insn, pat);
    7288              : 
    7289      2938796 :           if (GET_CODE (pat) == PARALLEL)
    7290       389174 :             p_sets = &XVECEXP (pat, 0, 0), noutputs = XVECLEN (pat, 0);
    7291      2549622 :           else if (GET_CODE (pat) == SET)
    7292      1549159 :             p_sets = &PATTERN (insn), noutputs = 1;
    7293              :           else
    7294      1000463 :             continue;
    7295              : 
    7296      1938333 :           if (GET_CODE (*p_sets) == SET
    7297      1877013 :               && GET_CODE (SET_SRC (*p_sets)) == ASM_OPERANDS)
    7298        36053 :             match_asm_constraints_1 (insn, p_sets, noutputs);
    7299              :          }
    7300              :     }
    7301              : 
    7302              :   return TODO_df_finish;
    7303              : }
    7304              : 
    7305              : } // anon namespace
    7306              : 
    7307              : rtl_opt_pass *
    7308       297180 : make_pass_match_asm_constraints (gcc::context *ctxt)
    7309              : {
    7310       297180 :   return new pass_match_asm_constraints (ctxt);
    7311              : }
    7312              : 
    7313              : 
    7314              : #include "gt-function.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.