LCOV - code coverage report
Current view: top level - gcc - emit-rtl.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 84.3 % 2841 2394
Test Date: 2026-11-14 16:39:50 Functions: 86.6 % 268 232
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Emit RTL for the GCC expander.
       2              :    Copyright (C) 1987-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify it under
       7              : the terms of the GNU General Public License as published by the Free
       8              : Software Foundation; either version 3, or (at your option) any later
       9              : version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      12              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14              : for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : 
      21              : /* Middle-to-low level generation of rtx code and insns.
      22              : 
      23              :    This file contains support functions for creating rtl expressions
      24              :    and manipulating them in the doubly-linked chain of insns.
      25              : 
      26              :    The patterns of the insns are created by machine-dependent
      27              :    routines in insn-emit.cc, which is generated automatically from
      28              :    the machine description.  These routines make the individual rtx's
      29              :    of the pattern with `gen_rtx_fmt_ee' and others in genrtl.[ch],
      30              :    which are automatically generated from rtl.def; what is machine
      31              :    dependent is the kind of rtx's they make and what arguments they
      32              :    use.  */
      33              : 
      34              : #include "config.h"
      35              : #include "system.h"
      36              : #include "coretypes.h"
      37              : #include "memmodel.h"
      38              : #include "backend.h"
      39              : #include "target.h"
      40              : #include "rtl.h"
      41              : #include "tree.h"
      42              : #include "df.h"
      43              : #include "tm_p.h"
      44              : #include "stringpool.h"
      45              : #include "insn-config.h"
      46              : #include "regs.h"
      47              : #include "emit-rtl.h"
      48              : #include "recog.h"
      49              : #include "diagnostic-core.h"
      50              : #include "alias.h"
      51              : #include "fold-const.h"
      52              : #include "varasm.h"
      53              : #include "cfgrtl.h"
      54              : #include "tree-eh.h"
      55              : #include "explow.h"
      56              : #include "expr.h"
      57              : #include "builtins.h"
      58              : #include "rtl-iter.h"
      59              : #include "stor-layout.h"
      60              : #include "opts.h"
      61              : #include "optabs.h"
      62              : #include "predict.h"
      63              : #include "rtx-vector-builder.h"
      64              : #include "gimple.h"
      65              : #include "gimple-ssa.h"
      66              : #include "bbitmap.h"
      67              : 
      68              : struct target_rtl default_target_rtl;
      69              : #if SWITCHABLE_TARGET
      70              : struct target_rtl *this_target_rtl = &default_target_rtl;
      71              : #endif
      72              : 
      73              : #define initial_regno_reg_rtx (this_target_rtl->x_initial_regno_reg_rtx)
      74              : 
      75              : /* Commonly used modes.  */
      76              : 
      77              : scalar_int_mode byte_mode;      /* Mode whose width is BITS_PER_UNIT.  */
      78              : scalar_int_mode word_mode;      /* Mode whose width is BITS_PER_WORD.  */
      79              : scalar_int_mode ptr_mode;       /* Mode whose width is POINTER_SIZE.  */
      80              : 
      81              : /* Datastructures maintained for currently processed function in RTL form.  */
      82              : 
      83              : struct rtl_data x_rtl;
      84              : 
      85              : /* Indexed by pseudo register number, gives the rtx for that pseudo.
      86              :    Allocated in parallel with regno_pointer_align.
      87              :    FIXME: We could put it into emit_status struct, but gengtype is not able to deal
      88              :    with length attribute nested in top level structures.  */
      89              : 
      90              : rtx * regno_reg_rtx;
      91              : 
      92              : /* This is *not* reset after each function.  It gives each CODE_LABEL
      93              :    in the entire compilation a unique label number.  */
      94              : 
      95              : static GTY(()) int label_num = 1;
      96              : 
      97              : /* We record floating-point CONST_DOUBLEs in each floating-point mode for
      98              :    the values of 0, 1, and 2.  For the integer entries and VOIDmode, we
      99              :    record a copy of const[012]_rtx and constm1_rtx.  CONSTM1_RTX
     100              :    is set only for MODE_INT and MODE_VECTOR_INT modes.  */
     101              : 
     102              : rtx const_tiny_rtx[4][(int) MAX_MACHINE_MODE];
     103              : 
     104              : rtx const_true_rtx;
     105              : 
     106              : REAL_VALUE_TYPE dconst0;
     107              : REAL_VALUE_TYPE dconst1;
     108              : REAL_VALUE_TYPE dconst2;
     109              : REAL_VALUE_TYPE dconstm0;
     110              : REAL_VALUE_TYPE dconstm1;
     111              : REAL_VALUE_TYPE dconsthalf;
     112              : REAL_VALUE_TYPE dconstinf;
     113              : REAL_VALUE_TYPE dconstninf;
     114              : 
     115              : /* Record fixed-point constant 0 and 1.  */
     116              : FIXED_VALUE_TYPE fconst0[MAX_FCONST0];
     117              : FIXED_VALUE_TYPE fconst1[MAX_FCONST1];
     118              : 
     119              : /* We make one copy of (const_int C) where C is in
     120              :    [- MAX_SAVED_CONST_INT, MAX_SAVED_CONST_INT]
     121              :    to save space during the compilation and simplify comparisons of
     122              :    integers.  */
     123              : 
     124              : rtx const_int_rtx[MAX_SAVED_CONST_INT * 2 + 1];
     125              : 
     126              : /* Standard pieces of rtx, to be substituted directly into things.  */
     127              : rtx pc_rtx;
     128              : rtx ret_rtx;
     129              : rtx simple_return_rtx;
     130              : 
     131              : /* Marker used for denoting an INSN, which should never be accessed (i.e.,
     132              :    this pointer should normally never be dereferenced), but is required to be
     133              :    distinct from NULL_RTX.  Currently used by peephole2 pass.  */
     134              : rtx_insn *invalid_insn_rtx;
     135              : 
     136              : /* A hash table storing CONST_INTs whose absolute value is greater
     137              :    than MAX_SAVED_CONST_INT.  */
     138              : 
     139              : struct const_int_hasher : ggc_cache_ptr_hash<rtx_def>
     140              : {
     141              :   typedef HOST_WIDE_INT compare_type;
     142              : 
     143              :   static hashval_t hash (rtx i);
     144              :   static bool equal (rtx i, HOST_WIDE_INT h);
     145              : };
     146              : 
     147              : static GTY ((cache)) hash_table<const_int_hasher> *const_int_htab;
     148              : 
     149              : struct const_wide_int_hasher : ggc_cache_ptr_hash<rtx_def>
     150              : {
     151              :   static hashval_t hash (rtx x);
     152              :   static bool equal (rtx x, rtx y);
     153              : };
     154              : 
     155              : static GTY ((cache)) hash_table<const_wide_int_hasher> *const_wide_int_htab;
     156              : 
     157              : struct const_poly_int_hasher : ggc_cache_ptr_hash<rtx_def>
     158              : {
     159              :   typedef std::pair<machine_mode, poly_wide_int_ref> compare_type;
     160              : 
     161              :   static hashval_t hash (rtx x);
     162              :   static bool equal (rtx x, const compare_type &y);
     163              : };
     164              : 
     165              : static GTY ((cache)) hash_table<const_poly_int_hasher> *const_poly_int_htab;
     166              : 
     167              : /* A hash table storing register attribute structures.  */
     168              : struct reg_attr_hasher : ggc_cache_ptr_hash<reg_attrs>
     169              : {
     170              :   static hashval_t hash (reg_attrs *x);
     171              :   static bool equal (reg_attrs *a, reg_attrs *b);
     172              : };
     173              : 
     174              : static GTY ((cache)) hash_table<reg_attr_hasher> *reg_attrs_htab;
     175              : 
     176              : /* A hash table storing all CONST_DOUBLEs.  */
     177              : struct const_double_hasher : ggc_cache_ptr_hash<rtx_def>
     178              : {
     179              :   static hashval_t hash (rtx x);
     180              :   static bool equal (rtx x, rtx y);
     181              : };
     182              : 
     183              : static GTY ((cache)) hash_table<const_double_hasher> *const_double_htab;
     184              : 
     185              : /* A hash table storing all CONST_FIXEDs.  */
     186              : struct const_fixed_hasher : ggc_cache_ptr_hash<rtx_def>
     187              : {
     188              :   static hashval_t hash (rtx x);
     189              :   static bool equal (rtx x, rtx y);
     190              : };
     191              : 
     192              : static GTY ((cache)) hash_table<const_fixed_hasher> *const_fixed_htab;
     193              : 
     194              : #define cur_insn_uid (crtl->emit.x_cur_insn_uid)
     195              : #define cur_debug_insn_uid (crtl->emit.x_cur_debug_insn_uid)
     196              : #define first_label_num (crtl->emit.x_first_label_num)
     197              : 
     198              : static void set_used_decls (tree);
     199              : static void mark_label_nuses (rtx);
     200              : #if TARGET_SUPPORTS_WIDE_INT
     201              : static rtx lookup_const_wide_int (rtx);
     202              : #endif
     203              : static rtx lookup_const_double (rtx);
     204              : static rtx lookup_const_fixed (rtx);
     205              : static rtx gen_const_vector (machine_mode, int);
     206              : static void copy_rtx_if_shared_1 (rtx *orig);
     207              : 
     208              : /* Probability of the conditional branch currently proceeded by try_split.  */
     209              : profile_probability split_branch_probability;
     210              : 
     211              : /* Returns a hash code for X (which is a really a CONST_INT).  */
     212              : 
     213              : hashval_t
     214   3448211539 : const_int_hasher::hash (rtx x)
     215              : {
     216   3448211539 :   return (hashval_t) INTVAL (x);
     217              : }
     218              : 
     219              : /* Returns true if the value represented by X (which is really a
     220              :    CONST_INT) is the same as that given by Y (which is really a
     221              :    HOST_WIDE_INT *).  */
     222              : 
     223              : bool
     224   4359677659 : const_int_hasher::equal (rtx x, HOST_WIDE_INT y)
     225              : {
     226   4359677659 :   return (INTVAL (x) == y);
     227              : }
     228              : 
     229              : #if TARGET_SUPPORTS_WIDE_INT
     230              : /* Returns a hash code for X (which is a really a CONST_WIDE_INT).  */
     231              : 
     232              : hashval_t
     233      1471787 : const_wide_int_hasher::hash (rtx x)
     234              : {
     235      1471787 :   int i;
     236      1471787 :   unsigned HOST_WIDE_INT hash = 0;
     237      1471787 :   const_rtx xr = x;
     238              : 
     239      4440840 :   for (i = 0; i < CONST_WIDE_INT_NUNITS (xr); i++)
     240      2969053 :     hash += CONST_WIDE_INT_ELT (xr, i);
     241              : 
     242      1471787 :   return (hashval_t) hash;
     243              : }
     244              : 
     245              : /* Returns true if the value represented by X (which is really a
     246              :    CONST_WIDE_INT) is the same as that given by Y (which is really a
     247              :    CONST_WIDE_INT).  */
     248              : 
     249              : bool
     250      1502821 : const_wide_int_hasher::equal (rtx x, rtx y)
     251              : {
     252      1502821 :   int i;
     253      1502821 :   const_rtx xr = x;
     254      1502821 :   const_rtx yr = y;
     255      1502821 :   if (CONST_WIDE_INT_NUNITS (xr) != CONST_WIDE_INT_NUNITS (yr))
     256              :     return false;
     257              : 
     258      2624200 :   for (i = 0; i < CONST_WIDE_INT_NUNITS (xr); i++)
     259      2093149 :     if (CONST_WIDE_INT_ELT (xr, i) != CONST_WIDE_INT_ELT (yr, i))
     260              :       return false;
     261              : 
     262              :   return true;
     263              : }
     264              : #endif
     265              : 
     266              : /* Returns a hash code for CONST_POLY_INT X.  */
     267              : 
     268              : hashval_t
     269            0 : const_poly_int_hasher::hash (rtx x)
     270              : {
     271            0 :   inchash::hash h;
     272            0 :   h.add_int (GET_MODE (x));
     273            0 :   for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
     274            0 :     h.add_wide_int (CONST_POLY_INT_COEFFS (x)[i]);
     275            0 :   return h.end ();
     276              : }
     277              : 
     278              : /* Returns true if CONST_POLY_INT X is an rtx representation of Y.  */
     279              : 
     280              : bool
     281            0 : const_poly_int_hasher::equal (rtx x, const compare_type &y)
     282              : {
     283            0 :   if (GET_MODE (x) != y.first)
     284              :     return false;
     285            0 :   for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
     286            0 :     if (CONST_POLY_INT_COEFFS (x)[i] != y.second.coeffs[i])
     287              :       return false;
     288              :   return true;
     289              : }
     290              : 
     291              : /* Returns a hash code for X (which is really a CONST_DOUBLE).  */
     292              : hashval_t
     293     18879216 : const_double_hasher::hash (rtx x)
     294              : {
     295     18879216 :   const_rtx const value = x;
     296     18879216 :   hashval_t h;
     297              : 
     298     18879216 :   if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (value) == VOIDmode)
     299              :     h = CONST_DOUBLE_LOW (value) ^ CONST_DOUBLE_HIGH (value);
     300              :   else
     301              :     {
     302     18879216 :       h = real_hash (CONST_DOUBLE_REAL_VALUE (value));
     303              :       /* MODE is used in the comparison, so it should be in the hash.  */
     304     18879216 :       h ^= GET_MODE (value);
     305              :     }
     306     18879216 :   return h;
     307              : }
     308              : 
     309              : /* Returns true if the value represented by X (really a ...)
     310              :    is the same as that represented by Y (really a ...) */
     311              : bool
     312     12992864 : const_double_hasher::equal (rtx x, rtx y)
     313              : {
     314     12992864 :   const_rtx const a = x, b = y;
     315              : 
     316     12992864 :   if (GET_MODE (a) != GET_MODE (b))
     317              :     return false;
     318      4676606 :   if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (a) == VOIDmode)
     319              :     return (CONST_DOUBLE_LOW (a) == CONST_DOUBLE_LOW (b)
     320              :             && CONST_DOUBLE_HIGH (a) == CONST_DOUBLE_HIGH (b));
     321              :   else
     322      4676606 :     return real_identical (CONST_DOUBLE_REAL_VALUE (a),
     323      4676606 :                            CONST_DOUBLE_REAL_VALUE (b));
     324              : }
     325              : 
     326              : /* Returns a hash code for X (which is really a CONST_FIXED).  */
     327              : 
     328              : hashval_t
     329     12172113 : const_fixed_hasher::hash (rtx x)
     330              : {
     331     12172113 :   const_rtx const value = x;
     332     12172113 :   hashval_t h;
     333              : 
     334     12172113 :   h = fixed_hash (CONST_FIXED_VALUE (value));
     335              :   /* MODE is used in the comparison, so it should be in the hash.  */
     336     12172113 :   h ^= GET_MODE (value);
     337     12172113 :   return h;
     338              : }
     339              : 
     340              : /* Returns true if the value represented by X is the same as that
     341              :    represented by Y.  */
     342              : 
     343              : bool
     344      6089598 : const_fixed_hasher::equal (rtx x, rtx y)
     345              : {
     346      6089598 :   const_rtx const a = x, b = y;
     347              : 
     348      6089598 :   if (GET_MODE (a) != GET_MODE (b))
     349              :     return false;
     350       579444 :   return fixed_identical (CONST_FIXED_VALUE (a), CONST_FIXED_VALUE (b));
     351              : }
     352              : 
     353              : /* Return true if the given memory attributes are equal.  */
     354              : 
     355              : bool
     356    234822459 : mem_attrs_eq_p (const class mem_attrs *p, const class mem_attrs *q)
     357              : {
     358    234822459 :   if (p == q)
     359              :     return true;
     360    231795526 :   if (!p || !q)
     361              :     return false;
     362    231188602 :   return (p->alias == q->alias
     363    145257709 :           && p->offset_known_p == q->offset_known_p
     364    115504815 :           && (!p->offset_known_p || known_eq (p->offset, q->offset))
     365     99568707 :           && p->size_known_p == q->size_known_p
     366     96210664 :           && (!p->size_known_p || known_eq (p->size, q->size))
     367     82710886 :           && p->align == q->align
     368     68065353 :           && p->addrspace == q->addrspace
     369    298970995 :           && (p->expr == q->expr
     370     32144120 :               || (p->expr != NULL_TREE && q->expr != NULL_TREE
     371     24961135 :                   && operand_equal_p (p->expr, q->expr, 0))));
     372              : }
     373              : 
     374              : /* Set MEM's memory attributes so that they are the same as ATTRS.  */
     375              : 
     376              : static void
     377    102696285 : set_mem_attrs (rtx mem, mem_attrs *attrs)
     378              : {
     379              :   /* If everything is the default, we can just clear the attributes.  */
     380    102696285 :   if (mem_attrs_eq_p (attrs, mode_mem_attrs[(int) GET_MODE (mem)]))
     381              :     {
     382      3356048 :       MEM_ATTRS (mem) = 0;
     383      3356048 :       return;
     384              :     }
     385              : 
     386     99340237 :   if (!MEM_ATTRS (mem)
     387     99340237 :       || !mem_attrs_eq_p (attrs, MEM_ATTRS (mem)))
     388              :     {
     389     75835600 :       MEM_ATTRS (mem) = ggc_alloc<mem_attrs> ();
     390     75835600 :       memcpy (MEM_ATTRS (mem), attrs, sizeof (mem_attrs));
     391              :     }
     392              : }
     393              : 
     394              : /* Returns a hash code for X (which is a really a reg_attrs *).  */
     395              : 
     396              : hashval_t
     397    322111355 : reg_attr_hasher::hash (reg_attrs *x)
     398              : {
     399    322111355 :   const reg_attrs *const p = x;
     400              : 
     401    322111355 :   inchash::hash h;
     402    322111355 :   h.add_ptr (p->decl);
     403    322111355 :   h.add_poly_hwi (p->offset);
     404    322111355 :   return h.end ();
     405              : }
     406              : 
     407              : /* Returns true if the value represented by X  is the same as that given by
     408              :    Y.  */
     409              : 
     410              : bool
     411    311133396 : reg_attr_hasher::equal (reg_attrs *x, reg_attrs *y)
     412              : {
     413    311133396 :   const reg_attrs *const p = x;
     414    311133396 :   const reg_attrs *const q = y;
     415              : 
     416    311133396 :   return (p->decl == q->decl && known_eq (p->offset, q->offset));
     417              : }
     418              : /* Allocate a new reg_attrs structure and insert it into the hash table if
     419              :    one identical to it is not already in the table.  We are doing this for
     420              :    MEM of mode MODE.  */
     421              : 
     422              : static reg_attrs *
     423     66593083 : get_reg_attrs (tree decl, poly_int64 offset)
     424              : {
     425     66593083 :   reg_attrs attrs;
     426              : 
     427              :   /* If everything is the default, we can just return zero.  */
     428     66593083 :   if (decl == 0 && known_eq (offset, 0))
     429              :     return 0;
     430              : 
     431     60872185 :   attrs.decl = decl;
     432     60872185 :   attrs.offset = offset;
     433              : 
     434     60872185 :   reg_attrs **slot = reg_attrs_htab->find_slot (&attrs, INSERT);
     435     60872185 :   if (*slot == 0)
     436              :     {
     437     30749154 :       *slot = ggc_alloc<reg_attrs> ();
     438     30749154 :       memcpy (*slot, &attrs, sizeof (reg_attrs));
     439              :     }
     440              : 
     441     60872185 :   return *slot;
     442              : }
     443              : 
     444              : 
     445              : #if !HAVE_blockage
     446              : /* Generate an empty ASM_INPUT, which is used to block attempts to schedule,
     447              :    and to block register equivalences to be seen across this insn.  */
     448              : 
     449              : rtx
     450              : gen_blockage (void)
     451              : {
     452              :   rtx x = gen_rtx_ASM_INPUT (VOIDmode, "");
     453              :   MEM_VOLATILE_P (x) = true;
     454              :   return x;
     455              : }
     456              : #endif
     457              : 
     458              : 
     459              : /* Set the mode and register number of X to MODE and REGNO.  */
     460              : 
     461              : void
     462   1775267869 : set_mode_and_regno (rtx x, machine_mode mode, unsigned int regno)
     463              : {
     464   1775267869 :   unsigned int nregs = (HARD_REGISTER_NUM_P (regno)
     465   1775267869 :                         ? hard_regno_nregs (regno, mode)
     466   1339107683 :                         : 1);
     467   1775267869 :   PUT_MODE_RAW (x, mode);
     468   1775267869 :   set_regno_raw (x, regno, nregs);
     469   1775267869 : }
     470              : 
     471              : /* Initialize a fresh REG rtx with mode MODE and register REGNO.  */
     472              : 
     473              : rtx
     474    359932105 : init_raw_REG (rtx x, machine_mode mode, unsigned int regno)
     475              : {
     476    359932105 :   set_mode_and_regno (x, mode, regno);
     477    359932105 :   REG_ATTRS (x) = NULL;
     478    359932105 :   ORIGINAL_REGNO (x) = regno;
     479    359932105 :   return x;
     480              : }
     481              : 
     482              : /* Generate a new REG rtx.  Make sure ORIGINAL_REGNO is set properly, and
     483              :    don't attempt to share with the various global pieces of rtl (such as
     484              :    frame_pointer_rtx).  */
     485              : 
     486              : rtx
     487    357764614 : gen_raw_REG (machine_mode mode, unsigned int regno)
     488              : {
     489    357764614 :   rtx x = rtx_alloc (REG MEM_STAT_INFO);
     490    357764614 :   init_raw_REG (x, mode, regno);
     491    357764614 :   return x;
     492              : }
     493              : 
     494              : /* There are some RTL codes that require special attention; the generation
     495              :    functions do the raw handling.  If you add to this list, modify
     496              :    special_rtx in gengenrtl.cc as well.  */
     497              : 
     498              : rtx_expr_list *
     499    209576912 : gen_rtx_EXPR_LIST (machine_mode mode, rtx expr, rtx expr_list)
     500              : {
     501    209576912 :   return as_a <rtx_expr_list *> (gen_rtx_fmt_ee (EXPR_LIST, mode, expr,
     502    209576912 :                                                  expr_list));
     503              : }
     504              : 
     505              : rtx_insn_list *
     506    349038142 : gen_rtx_INSN_LIST (machine_mode mode, rtx insn, rtx insn_list)
     507              : {
     508    349038142 :   return as_a <rtx_insn_list *> (gen_rtx_fmt_ue (INSN_LIST, mode, insn,
     509    349038142 :                                                  insn_list));
     510              : }
     511              : 
     512              : rtx_insn *
     513       847948 : gen_rtx_INSN (machine_mode mode, rtx_insn *prev_insn, rtx_insn *next_insn,
     514              :               basic_block bb, rtx pattern, location_t location, int code,
     515              :               rtx reg_notes)
     516              : {
     517       847948 :   return as_a <rtx_insn *> (gen_rtx_fmt_uuBeLie (INSN, mode,
     518              :                                                  prev_insn, next_insn,
     519              :                                                  bb, pattern, location, code,
     520       847948 :                                                  reg_notes));
     521              : }
     522              : 
     523              : rtx
     524   1326144646 : gen_rtx_CONST_INT (machine_mode mode ATTRIBUTE_UNUSED, HOST_WIDE_INT arg)
     525              : {
     526   1326144646 :   if (arg >= - MAX_SAVED_CONST_INT && arg <= MAX_SAVED_CONST_INT)
     527    852526930 :     return const_int_rtx[arg + MAX_SAVED_CONST_INT];
     528              : 
     529              : #if STORE_FLAG_VALUE != 1 && STORE_FLAG_VALUE != -1
     530              :   if (const_true_rtx && arg == STORE_FLAG_VALUE)
     531              :     return const_true_rtx;
     532              : #endif
     533              : 
     534              :   /* Look up the CONST_INT in the hash table.  */
     535    473617716 :   rtx *slot = const_int_htab->find_slot_with_hash (arg, (hashval_t) arg,
     536              :                                                    INSERT);
     537    473617716 :   if (*slot == 0)
     538     36182592 :     *slot = gen_rtx_raw_CONST_INT (VOIDmode, arg);
     539              : 
     540    473617716 :   return *slot;
     541              : }
     542              : 
     543              : /* Return a CONST_INT, CONST_POLY_INT, or CONST_VECTOR containing the given
     544              :    value.  In the case of a vector, the value will be duplicated.  */
     545              : 
     546              : rtx
     547   1239837570 : gen_int_mode (poly_int64 c, machine_mode mode)
     548              : {
     549   1239837570 :   machine_mode s_mode = (VECTOR_MODE_P (mode) ? GET_MODE_INNER (mode) : mode);
     550   1239837570 :   rtx val;
     551              : 
     552   1239837570 :   c = trunc_int_for_mode (c, s_mode);
     553   1239837570 :   if (c.is_constant ())
     554   1239837570 :     val = GEN_INT (c.coeffs[0]);
     555              :   else
     556              :     {
     557              :       unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (s_mode));
     558              :       val = immed_wide_int_const (poly_wide_int::from (c, prec, SIGNED),
     559              :                                   s_mode);
     560              :     }
     561              : 
     562   1239837570 :   if (VECTOR_MODE_P (mode))
     563            0 :     val = gen_const_vec_duplicate (mode, val);
     564              : 
     565   1239837570 :   return val;
     566              : }
     567              : 
     568              : /* CONST_DOUBLEs might be created from pairs of integers, or from
     569              :    REAL_VALUE_TYPEs.  Also, their length is known only at run time,
     570              :    so we cannot use gen_rtx_raw_CONST_DOUBLE.  */
     571              : 
     572              : /* Determine whether REAL, a CONST_DOUBLE, already exists in the
     573              :    hash table.  If so, return its counterpart; otherwise add it
     574              :    to the hash table and return it.  */
     575              : static rtx
     576     10047643 : lookup_const_double (rtx real)
     577              : {
     578     10047643 :   rtx *slot = const_double_htab->find_slot (real, INSERT);
     579     10047643 :   if (*slot == 0)
     580      8028268 :     *slot = real;
     581              : 
     582     10047643 :   return *slot;
     583              : }
     584              : 
     585              : /* Return a CONST_DOUBLE rtx for a floating-point value specified by
     586              :    VALUE in mode MODE.  */
     587              : rtx
     588     10047643 : const_double_from_real_value (REAL_VALUE_TYPE value, machine_mode mode)
     589              : {
     590     10047643 :   rtx real = rtx_alloc (CONST_DOUBLE);
     591     10047643 :   PUT_MODE (real, mode);
     592              : 
     593     10047643 :   real->u.rv = value;
     594              : 
     595     10047643 :   return lookup_const_double (real);
     596              : }
     597              : 
     598              : /* Determine whether FIXED, a CONST_FIXED, already exists in the
     599              :    hash table.  If so, return its counterpart; otherwise add it
     600              :    to the hash table and return it.  */
     601              : 
     602              : static rtx
     603      7530666 : lookup_const_fixed (rtx fixed)
     604              : {
     605      7530666 :   rtx *slot = const_fixed_htab->find_slot (fixed, INSERT);
     606      7530666 :   if (*slot == 0)
     607      7530666 :     *slot = fixed;
     608              : 
     609      7530666 :   return *slot;
     610              : }
     611              : 
     612              : /* Return a CONST_FIXED rtx for a fixed-point value specified by
     613              :    VALUE in mode MODE.  */
     614              : 
     615              : rtx
     616      7530666 : const_fixed_from_fixed_value (FIXED_VALUE_TYPE value, machine_mode mode)
     617              : {
     618      7530666 :   rtx fixed = rtx_alloc (CONST_FIXED);
     619      7530666 :   PUT_MODE (fixed, mode);
     620              : 
     621      7530666 :   fixed->u.fv = value;
     622              : 
     623      7530666 :   return lookup_const_fixed (fixed);
     624              : }
     625              : 
     626              : #if TARGET_SUPPORTS_WIDE_INT == 0
     627              : /* Constructs double_int from rtx CST.  */
     628              : 
     629              : double_int
     630              : rtx_to_double_int (const_rtx cst)
     631              : {
     632              :   double_int r;
     633              : 
     634              :   if (CONST_INT_P (cst))
     635              :       r = double_int::from_shwi (INTVAL (cst));
     636              :   else if (CONST_DOUBLE_AS_INT_P (cst))
     637              :     {
     638              :       r.low = CONST_DOUBLE_LOW (cst);
     639              :       r.high = CONST_DOUBLE_HIGH (cst);
     640              :     }
     641              :   else
     642              :     gcc_unreachable ();
     643              : 
     644              :   return r;
     645              : }
     646              : #endif
     647              : 
     648              : #if TARGET_SUPPORTS_WIDE_INT
     649              : /* Determine whether CONST_WIDE_INT WINT already exists in the hash table.
     650              :    If so, return its counterpart; otherwise add it to the hash table and
     651              :    return it.  */
     652              : 
     653              : static rtx
     654       591175 : lookup_const_wide_int (rtx wint)
     655              : {
     656       591175 :   rtx *slot = const_wide_int_htab->find_slot (wint, INSERT);
     657       591175 :   if (*slot == 0)
     658        60124 :     *slot = wint;
     659              : 
     660       591175 :   return *slot;
     661              : }
     662              : #endif
     663              : 
     664              : /* Return an rtx constant for V, given that the constant has mode MODE.
     665              :    The returned rtx will be a CONST_INT if V fits, otherwise it will be
     666              :    a CONST_DOUBLE (if !TARGET_SUPPORTS_WIDE_INT) or a CONST_WIDE_INT
     667              :    (if TARGET_SUPPORTS_WIDE_INT).  */
     668              : 
     669              : static rtx
     670    639499693 : immed_wide_int_const_1 (const wide_int_ref &v, machine_mode mode)
     671              : {
     672    639499693 :   unsigned int len = v.get_len ();
     673              :   /* Not scalar_int_mode because we also allow pointer bound modes.  */
     674    639499693 :   unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (mode));
     675              : 
     676              :   /* Allow truncation but not extension since we do not know if the
     677              :      number is signed or unsigned.  */
     678    639499693 :   gcc_assert (prec <= v.get_precision ());
     679              : 
     680    639499693 :   if (len < 2 || prec <= HOST_BITS_PER_WIDE_INT)
     681    638908518 :     return gen_int_mode (v.elt (0), mode);
     682              : 
     683              : #if TARGET_SUPPORTS_WIDE_INT
     684       591175 :   {
     685       591175 :     unsigned int i;
     686       591175 :     rtx value;
     687       591175 :     unsigned int blocks_needed
     688       591175 :       = (prec + HOST_BITS_PER_WIDE_INT - 1) / HOST_BITS_PER_WIDE_INT;
     689              : 
     690       591175 :     if (len > blocks_needed)
     691              :       len = blocks_needed;
     692              : 
     693       591175 :     value = const_wide_int_alloc (len);
     694              : 
     695              :     /* It is so tempting to just put the mode in here.  Must control
     696              :        myself ... */
     697       591175 :     PUT_MODE (value, VOIDmode);
     698       591175 :     CWI_PUT_NUM_ELEM (value, len);
     699              : 
     700      1781224 :     for (i = 0; i < len; i++)
     701      1190049 :       CONST_WIDE_INT_ELT (value, i) = v.elt (i);
     702              : 
     703       591175 :     return lookup_const_wide_int (value);
     704              :   }
     705              : #else
     706              :   return immed_double_const (v.elt (0), v.elt (1), mode);
     707              : #endif
     708              : }
     709              : 
     710              : #if TARGET_SUPPORTS_WIDE_INT == 0
     711              : /* Return a CONST_DOUBLE or CONST_INT for a value specified as a pair
     712              :    of ints: I0 is the low-order word and I1 is the high-order word.
     713              :    For values that are larger than HOST_BITS_PER_DOUBLE_INT, the
     714              :    implied upper bits are copies of the high bit of i1.  The value
     715              :    itself is neither signed nor unsigned.  Do not use this routine for
     716              :    non-integer modes; convert to REAL_VALUE_TYPE and use
     717              :    const_double_from_real_value.  */
     718              : 
     719              : rtx
     720              : immed_double_const (HOST_WIDE_INT i0, HOST_WIDE_INT i1, machine_mode mode)
     721              : {
     722              :   rtx value;
     723              :   unsigned int i;
     724              : 
     725              :   /* There are the following cases (note that there are no modes with
     726              :      HOST_BITS_PER_WIDE_INT < GET_MODE_BITSIZE (mode) < HOST_BITS_PER_DOUBLE_INT):
     727              : 
     728              :      1) If GET_MODE_BITSIZE (mode) <= HOST_BITS_PER_WIDE_INT, then we use
     729              :         gen_int_mode.
     730              :      2) If the value of the integer fits into HOST_WIDE_INT anyway
     731              :         (i.e., i1 consists only from copies of the sign bit, and sign
     732              :         of i0 and i1 are the same), then we return a CONST_INT for i0.
     733              :      3) Otherwise, we create a CONST_DOUBLE for i0 and i1.  */
     734              :   scalar_mode smode;
     735              :   if (is_a <scalar_mode> (mode, &smode)
     736              :       && GET_MODE_BITSIZE (smode) <= HOST_BITS_PER_WIDE_INT)
     737              :     return gen_int_mode (i0, mode);
     738              : 
     739              :   /* If this integer fits in one word, return a CONST_INT.  */
     740              :   if ((i1 == 0 && i0 >= 0) || (i1 == ~0 && i0 < 0))
     741              :     return GEN_INT (i0);
     742              : 
     743              :   /* We use VOIDmode for integers.  */
     744              :   value = rtx_alloc (CONST_DOUBLE);
     745              :   PUT_MODE (value, VOIDmode);
     746              : 
     747              :   CONST_DOUBLE_LOW (value) = i0;
     748              :   CONST_DOUBLE_HIGH (value) = i1;
     749              : 
     750              :   for (i = 2; i < (sizeof CONST_DOUBLE_FORMAT - 1); i++)
     751              :     XWINT (value, i) = 0;
     752              : 
     753              :   return lookup_const_double (value);
     754              : }
     755              : #endif
     756              : 
     757              : /* Return an rtx representation of C in mode MODE.  */
     758              : 
     759              : rtx
     760    639499693 : immed_wide_int_const (const poly_wide_int_ref &c, machine_mode mode)
     761              : {
     762    639499693 :   if (c.is_constant ())
     763    639499693 :     return immed_wide_int_const_1 (c.coeffs[0], mode);
     764              : 
     765              :   /* Not scalar_int_mode because we also allow pointer bound modes.  */
     766              :   unsigned int prec = GET_MODE_PRECISION (as_a <scalar_mode> (mode));
     767              : 
     768              :   /* Allow truncation but not extension since we do not know if the
     769              :      number is signed or unsigned.  */
     770              :   gcc_assert (prec <= c.coeffs[0].get_precision ());
     771              :   poly_wide_int newc = poly_wide_int::from (c, prec, SIGNED);
     772              : 
     773              :   /* See whether we already have an rtx for this constant.  */
     774              :   inchash::hash h;
     775              :   h.add_int (mode);
     776              :   for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
     777              :     h.add_wide_int (newc.coeffs[i]);
     778              :   const_poly_int_hasher::compare_type typed_value (mode, newc);
     779              :   rtx *slot = const_poly_int_htab->find_slot_with_hash (typed_value,
     780              :                                                         h.end (), INSERT);
     781              :   rtx x = *slot;
     782              :   if (x)
     783              :     return x;
     784              : 
     785              :   /* Create a new rtx.  There's a choice to be made here between installing
     786              :      the actual mode of the rtx or leaving it as VOIDmode (for consistency
     787              :      with CONST_INT).  In practice the handling of the codes is different
     788              :      enough that we get no benefit from using VOIDmode, and various places
     789              :      assume that VOIDmode implies CONST_INT.  Using the real mode seems like
     790              :      the right long-term direction anyway.  */
     791              :   typedef trailing_wide_ints<NUM_POLY_INT_COEFFS> twi;
     792              :   size_t extra_size = twi::extra_size (prec);
     793              :   x = rtx_alloc_v (CONST_POLY_INT,
     794              :                    sizeof (struct const_poly_int_def) + extra_size);
     795              :   PUT_MODE (x, mode);
     796              :   CONST_POLY_INT_COEFFS (x).set_precision (prec);
     797              :   for (unsigned int i = 0; i < NUM_POLY_INT_COEFFS; ++i)
     798              :     CONST_POLY_INT_COEFFS (x)[i] = newc.coeffs[i];
     799              : 
     800              :   *slot = x;
     801              :   return x;
     802              : }
     803              : 
     804              : rtx
     805    196193859 : gen_rtx_REG (machine_mode mode, unsigned int regno)
     806              : {
     807              :   /* In case the MD file explicitly references the frame pointer, have
     808              :      all such references point to the same frame pointer.  This is
     809              :      used during frame pointer elimination to distinguish the explicit
     810              :      references to these registers from pseudos that happened to be
     811              :      assigned to them.
     812              : 
     813              :      If we have eliminated the frame pointer or arg pointer, we will
     814              :      be using it as a normal register, for example as a spill
     815              :      register.  In such cases, we might be accessing it in a mode that
     816              :      is not Pmode and therefore cannot use the pre-allocated rtx.
     817              : 
     818              :      Also don't do this when we are making new REGs in reload, since
     819              :      we don't want to get confused with the real pointers.  */
     820              : 
     821    210799344 :   if (mode == Pmode && !reload_in_progress && !lra_in_progress)
     822              :     {
     823     63321546 :       if (regno == FRAME_POINTER_REGNUM
     824      3041024 :           && (!reload_completed || frame_pointer_needed))
     825      3041024 :         return frame_pointer_rtx;
     826              : 
     827     60280522 :       if (!HARD_FRAME_POINTER_IS_FRAME_POINTER
     828              :           && regno == HARD_FRAME_POINTER_REGNUM
     829      4354495 :           && (!reload_completed || frame_pointer_needed))
     830      3829973 :         return hard_frame_pointer_rtx;
     831              : #if !HARD_FRAME_POINTER_IS_ARG_POINTER
     832     56450549 :       if (FRAME_POINTER_REGNUM != ARG_POINTER_REGNUM
     833              :           && regno == ARG_POINTER_REGNUM)
     834      3045493 :         return arg_pointer_rtx;
     835              : #endif
     836              : #ifdef RETURN_ADDRESS_POINTER_REGNUM
     837              :       if (regno == RETURN_ADDRESS_POINTER_REGNUM)
     838              :         return return_address_pointer_rtx;
     839              : #endif
     840     53405056 :       if (regno == (unsigned) PIC_OFFSET_TABLE_REGNUM
     841            0 :           && PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
     842     53405056 :           && fixed_regs[PIC_OFFSET_TABLE_REGNUM])
     843            0 :         return pic_offset_table_rtx;
     844     53405056 :       if (regno == STACK_POINTER_REGNUM)
     845      4074853 :         return stack_pointer_rtx;
     846              :     }
     847              : 
     848              : #if 0
     849              :   /* If the per-function register table has been set up, try to re-use
     850              :      an existing entry in that table to avoid useless generation of RTL.
     851              : 
     852              :      This code is disabled for now until we can fix the various backends
     853              :      which depend on having non-shared hard registers in some cases.   Long
     854              :      term we want to re-enable this code as it can significantly cut down
     855              :      on the amount of useless RTL that gets generated.
     856              : 
     857              :      We'll also need to fix some code that runs after reload that wants to
     858              :      set ORIGINAL_REGNO.  */
     859              : 
     860              :   if (cfun
     861              :       && cfun->emit
     862              :       && regno_reg_rtx
     863              :       && regno < FIRST_PSEUDO_REGISTER
     864              :       && reg_raw_mode[regno] == mode)
     865              :     return regno_reg_rtx[regno];
     866              : #endif
     867              : 
     868    182202516 :   return gen_raw_REG (mode, regno);
     869              : }
     870              : 
     871              : rtx
     872    254480401 : gen_rtx_MEM (machine_mode mode, rtx addr)
     873              : {
     874    254480401 :   rtx rt = gen_rtx_raw_MEM (mode, addr);
     875              : 
     876              :   /* This field is not cleared by the mere allocation of the rtx, so
     877              :      we clear it here.  */
     878    254480401 :   MEM_ATTRS (rt) = 0;
     879              : 
     880    254480401 :   return rt;
     881              : }
     882              : 
     883              : /* Generate a memory referring to non-trapping constant memory.  */
     884              : 
     885              : rtx
     886      2173724 : gen_const_mem (machine_mode mode, rtx addr)
     887              : {
     888      2173724 :   rtx mem = gen_rtx_MEM (mode, addr);
     889      2173724 :   MEM_READONLY_P (mem) = 1;
     890      2173724 :   MEM_NOTRAP_P (mem) = 1;
     891      2173724 :   return mem;
     892              : }
     893              : 
     894              : /* Generate a MEM referring to fixed portions of the frame, e.g., register
     895              :    save areas.  */
     896              : 
     897              : rtx
     898      1226188 : gen_frame_mem (machine_mode mode, rtx addr)
     899              : {
     900      1226188 :   rtx mem = gen_rtx_MEM (mode, addr);
     901      1226188 :   MEM_NOTRAP_P (mem) = 1;
     902      1226188 :   set_mem_alias_set (mem, get_frame_alias_set ());
     903      1226188 :   return mem;
     904              : }
     905              : 
     906              : /* Generate a MEM referring to a temporary use of the stack, not part
     907              :     of the fixed stack frame.  For example, something which is pushed
     908              :     by a target splitter.  */
     909              : rtx
     910            0 : gen_tmp_stack_mem (machine_mode mode, rtx addr)
     911              : {
     912            0 :   rtx mem = gen_rtx_MEM (mode, addr);
     913            0 :   MEM_NOTRAP_P (mem) = 1;
     914            0 :   if (!cfun->calls_alloca)
     915            0 :     set_mem_alias_set (mem, get_frame_alias_set ());
     916            0 :   return mem;
     917              : }
     918              : 
     919              : /* We want to create (subreg:OMODE (obj:IMODE) OFFSET).  Return true if
     920              :    this construct would be valid, and false otherwise.  */
     921              : 
     922              : bool
     923     52488818 : validate_subreg (machine_mode omode, machine_mode imode,
     924              :                  const_rtx reg, poly_uint64 offset)
     925              : {
     926    104977636 :   poly_uint64 isize = GET_MODE_SIZE (imode);
     927    104977636 :   poly_uint64 osize = GET_MODE_SIZE (omode);
     928              : 
     929              :   /* The sizes must be ordered, so that we know whether the subreg
     930              :      is partial, paradoxical or complete.  */
     931     52488818 :   if (!ordered_p (isize, osize))
     932              :     return false;
     933              : 
     934              :   /* All subregs must be aligned.  */
     935     52488818 :   if (!multiple_p (offset, osize))
     936              :     return false;
     937              : 
     938              :   /* The subreg offset cannot be outside the inner object.  */
     939     52488798 :   if (maybe_ge (offset, isize))
     940              :     return false;
     941              : 
     942     52488798 :   poly_uint64 regsize = REGMODE_NATURAL_SIZE (imode);
     943              : 
     944              :   /* ??? This should not be here.  Temporarily continue to allow word_mode
     945              :      subregs of anything.  The most common offender is (subreg:SI (reg:DF)).
     946              :      Generally, backends are doing something sketchy but it'll take time to
     947              :      fix them all.  */
     948     52488798 :   if (omode == word_mode)
     949              :     ;
     950              :   /* ??? Similarly, e.g. with (subreg:DF (reg:TI)).  Though store_bit_field
     951              :      is the culprit here, and not the backends.  */
     952     31941115 :   else if (known_ge (osize, regsize) && known_ge (isize, osize))
     953              :     ;
     954              :   /* Allow component subregs of complex and vector.  Though given the below
     955              :      extraction rules, it's not always clear what that means.  */
     956     25616590 :   else if ((COMPLEX_MODE_P (imode) || VECTOR_MODE_P (imode))
     957     26815869 :            && GET_MODE_INNER (imode) == omode)
     958              :     ;
     959              :   /* ??? x86 sse code makes heavy use of *paradoxical* vector subregs,
     960              :      i.e. (subreg:V4SF (reg:SF) 0) or (subreg:V4SF (reg:V2SF) 0).  This
     961              :      surely isn't the cleanest way to represent this.  It's questionable
     962              :      if this ought to be represented at all -- why can't this all be hidden
     963              :      in post-reload splitters that make arbitrarily mode changes to the
     964              :      registers themselves.  */
     965     24167588 :   else if (VECTOR_MODE_P (omode)
     966     26449960 :            && GET_MODE_UNIT_SIZE (omode) == GET_MODE_UNIT_SIZE (imode))
     967              :     ;
     968              :   /* Subregs involving floating point modes are not allowed to
     969              :      change size unless it's an insert into a complex mode.
     970              :      Therefore (subreg:DI (reg:DF) 0) and (subreg:CS (reg:SF) 0) are fine, but
     971              :      (subreg:SI (reg:DF) 0) isn't.  */
     972     24063523 :   else if ((FLOAT_MODE_P (imode) || FLOAT_MODE_P (omode))
     973       326423 :            && !COMPLEX_MODE_P (omode))
     974              :     {
     975       326261 :       if (! (known_eq (isize, osize)
     976              :              /* LRA can use subreg to store a floating point value in
     977              :                 an integer mode.  Although the floating point and the
     978              :                 integer modes need the same number of hard registers,
     979              :                 the size of floating point mode can be less than the
     980              :                 integer mode.  LRA also uses subregs for a register
     981              :                 should be used in different mode in on insn.  */
     982        73114 :              || lra_in_progress))
     983              :         return false;
     984              :     }
     985              : 
     986              :   /* Paradoxical subregs must have offset zero.  */
     987     52417711 :   if (maybe_gt (osize, isize) && !known_eq (offset, 0U))
     988              :     return false;
     989              : 
     990              :   /* Verify that the offset is representable.  */
     991              : 
     992              :   /* Ensure that subregs of hard registers can be folded.  In other words,
     993              :      the hardware register must be valid in the subreg's outer mode,
     994              :      and consequently the subreg can be replaced with a hardware register.  */
     995     52417711 :   if (reg && REG_P (reg) && HARD_REGISTER_P (reg))
     996              :     {
     997       316348 :       unsigned int regno = REGNO (reg);
     998              : 
     999       316348 :       if ((COMPLEX_MODE_P (imode) || VECTOR_MODE_P (imode))
    1000       611920 :           && GET_MODE_INNER (imode) == omode)
    1001              :         ;
    1002       116822 :       else if (!REG_CAN_CHANGE_MODE_P (regno, imode, omode))
    1003              :         return false;
    1004              : 
    1005              :       /* Pass true to allow_stack_regs because targets like x86
    1006              :          expect to be able to take subregs of the stack pointer.  */
    1007       316251 :       return simplify_subreg_regno (regno, imode, offset, omode, true) >= 0;
    1008              :     }
    1009              :   /* Do not allow normal SUBREG with stricter alignment than the inner MEM.
    1010              : 
    1011              :      PR120329: Combine can create paradoxical mem subregs even for
    1012              :      strict-alignment targets.  Allow it until combine is fixed.  */
    1013     52101363 :   else if (reg && MEM_P (reg) && STRICT_ALIGNMENT
    1014              :            && MEM_ALIGN (reg) < GET_MODE_ALIGNMENT (omode)
    1015              :            && known_le (osize, isize))
    1016              :     return false;
    1017              : 
    1018              :   /* If ISIZE is greater than REGSIZE, the inner value is split into blocks
    1019              :      of size REGSIZE.  The outer size must then be ordered wrt REGSIZE,
    1020              :      otherwise we wouldn't know at compile time how many blocks the
    1021              :      outer mode occupies.  */
    1022     52101363 :   if (maybe_gt (isize, regsize) && !ordered_p (osize, regsize))
    1023              :     return false;
    1024              : 
    1025              :   /* For normal pseudo registers, we want most of the same checks.  Namely:
    1026              : 
    1027              :      Assume that the pseudo register will be allocated to hard registers
    1028              :      that can hold REGSIZE bytes each.  If OSIZE is not a multiple of REGSIZE,
    1029              :      the remainder must correspond to the lowpart of the containing hard
    1030              :      register.  If BYTES_BIG_ENDIAN, the lowpart is at the highest offset,
    1031              :      otherwise it is at the lowest offset.
    1032              : 
    1033              :      Given that we've already checked the mode and offset alignment,
    1034              :      we only have to check subblock subregs here.
    1035              : 
    1036              :      For paradoxical little-endian registers, this check is redundant.  The
    1037              :      offset has already been validated to be zero.
    1038              : 
    1039              :      For paradoxical big-endian registers, this check is not valid
    1040              :      because the offset is zero.  */
    1041     52101363 :   if (maybe_lt (osize, regsize)
    1042     24726343 :       && known_le (osize, isize)
    1043     69333628 :       && ! (lra_in_progress && (FLOAT_MODE_P (imode) || FLOAT_MODE_P (omode))))
    1044              :     {
    1045              :       /* It is invalid for the target to pick a register size for a mode
    1046              :          that isn't ordered wrt to the size of that mode.  */
    1047     17223839 :       poly_uint64 block_size = ordered_min (isize, regsize);
    1048     17223839 :       unsigned int start_reg;
    1049     17223839 :       poly_uint64 offset_within_reg;
    1050     17223839 :       if (!can_div_trunc_p (offset, block_size, &start_reg, &offset_within_reg)
    1051     17223839 :           || (BYTES_BIG_ENDIAN
    1052              :               ? maybe_ne (offset_within_reg, block_size - osize)
    1053     17223839 :               : maybe_ne (offset_within_reg, 0U)))
    1054         4437 :         return false;
    1055              :     }
    1056              :   return true;
    1057              : }
    1058              : 
    1059              : rtx
    1060     26035543 : gen_rtx_SUBREG (machine_mode mode, rtx reg, poly_uint64 offset)
    1061              : {
    1062     26035543 :   gcc_assert (validate_subreg (mode, GET_MODE (reg), reg, offset));
    1063     26035543 :   return gen_rtx_raw_SUBREG (mode, reg, offset);
    1064              : }
    1065              : 
    1066              : /* Generate a SUBREG representing the least-significant part of REG if MODE
    1067              :    is smaller than mode of REG, otherwise paradoxical SUBREG.  */
    1068              : 
    1069              : rtx
    1070       598796 : gen_lowpart_SUBREG (machine_mode mode, rtx reg)
    1071              : {
    1072       598796 :   machine_mode inmode;
    1073              : 
    1074       598796 :   inmode = GET_MODE (reg);
    1075       598796 :   if (inmode == VOIDmode)
    1076            0 :     inmode = mode;
    1077       598796 :   return gen_rtx_SUBREG (mode, reg,
    1078       598796 :                          subreg_lowpart_offset (mode, inmode));
    1079              : }
    1080              : 
    1081              : rtx
    1082    107375816 : gen_rtx_VAR_LOCATION (machine_mode mode, tree decl, rtx loc,
    1083              :                       enum var_init_status status)
    1084              : {
    1085    107375816 :   rtx x = gen_rtx_fmt_te (VAR_LOCATION, mode, decl, loc);
    1086    107375816 :   PAT_VAR_LOCATION_STATUS (x) = status;
    1087    107375816 :   return x;
    1088              : }
    1089              : 
    1090              : 
    1091              : /* Create an rtvec and stores within it the RTXen passed in the arguments.  */
    1092              : 
    1093              : rtvec
    1094     25156843 : gen_rtvec (int n, ...)
    1095              : {
    1096     25156843 :   int i;
    1097     25156843 :   rtvec rt_val;
    1098     25156843 :   va_list p;
    1099              : 
    1100     25156843 :   va_start (p, n);
    1101              : 
    1102              :   /* Don't allocate an empty rtvec...  */
    1103     25156843 :   if (n == 0)
    1104              :     {
    1105            0 :       va_end (p);
    1106            0 :       return NULL_RTVEC;
    1107              :     }
    1108              : 
    1109     25156843 :   rt_val = rtvec_alloc (n);
    1110              : 
    1111     95754323 :   for (i = 0; i < n; i++)
    1112     45440637 :     rt_val->elem[i] = va_arg (p, rtx);
    1113              : 
    1114     25156843 :   va_end (p);
    1115     25156843 :   return rt_val;
    1116              : }
    1117              : 
    1118              : rtvec
    1119       472345 : gen_rtvec_v (int n, rtx *argp)
    1120              : {
    1121       472345 :   int i;
    1122       472345 :   rtvec rt_val;
    1123              : 
    1124              :   /* Don't allocate an empty rtvec...  */
    1125       472345 :   if (n == 0)
    1126              :     return NULL_RTVEC;
    1127              : 
    1128       472345 :   rt_val = rtvec_alloc (n);
    1129              : 
    1130      2149051 :   for (i = 0; i < n; i++)
    1131      1204361 :     rt_val->elem[i] = *argp++;
    1132              : 
    1133              :   return rt_val;
    1134              : }
    1135              : 
    1136              : rtvec
    1137            0 : gen_rtvec_v (int n, rtx_insn **argp)
    1138              : {
    1139            0 :   int i;
    1140            0 :   rtvec rt_val;
    1141              : 
    1142              :   /* Don't allocate an empty rtvec...  */
    1143            0 :   if (n == 0)
    1144              :     return NULL_RTVEC;
    1145              : 
    1146            0 :   rt_val = rtvec_alloc (n);
    1147              : 
    1148            0 :   for (i = 0; i < n; i++)
    1149            0 :     rt_val->elem[i] = *argp++;
    1150              : 
    1151              :   return rt_val;
    1152              : }
    1153              : 
    1154              : 
    1155              : /* Return the number of bytes between the start of an OUTER_MODE
    1156              :    in-memory value and the start of an INNER_MODE in-memory value,
    1157              :    given that the former is a lowpart of the latter.  It may be a
    1158              :    paradoxical lowpart, in which case the offset will be negative
    1159              :    on big-endian targets.  */
    1160              : 
    1161              : poly_int64
    1162     75259848 : byte_lowpart_offset (machine_mode outer_mode,
    1163              :                      machine_mode inner_mode)
    1164              : {
    1165     75259848 :   if (paradoxical_subreg_p (outer_mode, inner_mode))
    1166      3925209 :     return -subreg_lowpart_offset (inner_mode, outer_mode);
    1167              :   else
    1168     71334639 :     return subreg_lowpart_offset (outer_mode, inner_mode);
    1169              : }
    1170              : 
    1171              : /* Return the offset of (subreg:OUTER_MODE (mem:INNER_MODE X) OFFSET)
    1172              :    from address X.  For paradoxical big-endian subregs this is a
    1173              :    negative value, otherwise it's the same as OFFSET.  */
    1174              : 
    1175              : poly_int64
    1176     12562971 : subreg_memory_offset (machine_mode outer_mode, machine_mode inner_mode,
    1177              :                       poly_uint64 offset)
    1178              : {
    1179     12562971 :   if (paradoxical_subreg_p (outer_mode, inner_mode))
    1180              :     {
    1181      3461269 :       gcc_assert (known_eq (offset, 0U));
    1182      3461269 :       return -subreg_lowpart_offset (inner_mode, outer_mode);
    1183              :     }
    1184      9101702 :   return offset;
    1185              : }
    1186              : 
    1187              : /* As above, but return the offset that existing subreg X would have
    1188              :    if SUBREG_REG (X) were stored in memory.  The only significant thing
    1189              :    about the current SUBREG_REG is its mode.  */
    1190              : 
    1191              : poly_int64
    1192       682354 : subreg_memory_offset (const_rtx x)
    1193              : {
    1194      1364708 :   return subreg_memory_offset (GET_MODE (x), GET_MODE (SUBREG_REG (x)),
    1195       682354 :                                SUBREG_BYTE (x));
    1196              : }
    1197              : 
    1198              : /* Generate a REG rtx for a new pseudo register of mode MODE.
    1199              :    This pseudo is assigned the next sequential register number.  */
    1200              : 
    1201              : rtx
    1202     69368741 : gen_reg_rtx (machine_mode mode)
    1203              : {
    1204     69368741 :   rtx val;
    1205     69368741 :   unsigned int align = GET_MODE_ALIGNMENT (mode);
    1206              : 
    1207     69368741 :   gcc_assert (can_create_pseudo_p ());
    1208              : 
    1209              :   /* If a virtual register with bigger mode alignment is generated,
    1210              :      increase stack alignment estimation because it might be spilled
    1211              :      to stack later.  */
    1212     69368741 :   if (SUPPORTS_STACK_ALIGNMENT
    1213     69368741 :       && crtl->stack_alignment_estimated < align
    1214       376532 :       && !crtl->stack_realign_processed)
    1215              :     {
    1216       375458 :       unsigned int min_align = MINIMUM_ALIGNMENT (NULL, mode, align);
    1217       375458 :       if (crtl->stack_alignment_estimated < min_align)
    1218       375458 :         crtl->stack_alignment_estimated = min_align;
    1219              :     }
    1220              : 
    1221     69368741 :   if (generating_concat_p
    1222     52320428 :       && (GET_MODE_CLASS (mode) == MODE_COMPLEX_FLOAT
    1223     52320428 :           || GET_MODE_CLASS (mode) == MODE_COMPLEX_INT))
    1224              :     {
    1225              :       /* For complex modes, don't make a single pseudo.
    1226              :          Instead, make a CONCAT of two pseudos.
    1227              :          This allows noncontiguous allocation of the real and imaginary parts,
    1228              :          which makes much better code.  Besides, allocating DCmode
    1229              :          pseudos overstrains reload on some machines like the 386.  */
    1230       226007 :       rtx realpart, imagpart;
    1231       226007 :       machine_mode partmode = GET_MODE_INNER (mode);
    1232              : 
    1233       226007 :       realpart = gen_reg_rtx (partmode);
    1234       226007 :       imagpart = gen_reg_rtx (partmode);
    1235       226007 :       return gen_rtx_CONCAT (mode, realpart, imagpart);
    1236              :     }
    1237              : 
    1238              :   /* Do not call gen_reg_rtx with uninitialized crtl.  */
    1239     69142734 :   gcc_assert (crtl->emit.regno_pointer_align_length);
    1240              : 
    1241     69142734 :   crtl->emit.ensure_regno_capacity ();
    1242     69142734 :   gcc_assert (reg_rtx_no < crtl->emit.regno_pointer_align_length);
    1243              : 
    1244     69142734 :   val = gen_raw_REG (mode, reg_rtx_no);
    1245     69142734 :   regno_reg_rtx[reg_rtx_no++] = val;
    1246     69142734 :   return val;
    1247              : }
    1248              : 
    1249              : /* Make sure m_regno_pointer_align, and regno_reg_rtx are large
    1250              :    enough to have elements in the range 0 <= idx <= reg_rtx_no.  */
    1251              : 
    1252              : void
    1253     69143960 : emit_status::ensure_regno_capacity ()
    1254              : {
    1255     69143960 :   int old_size = regno_pointer_align_length;
    1256              : 
    1257     69143960 :   if (reg_rtx_no < old_size)
    1258              :     return;
    1259              : 
    1260       182447 :   int new_size = old_size * 2;
    1261       182447 :   while (reg_rtx_no >= new_size)
    1262            0 :     new_size *= 2;
    1263              : 
    1264       182447 :   char *tmp = XRESIZEVEC (char, regno_pointer_align, new_size);
    1265       182447 :   memset (tmp + old_size, 0, new_size - old_size);
    1266       182447 :   regno_pointer_align = (unsigned char *) tmp;
    1267              : 
    1268       182447 :   rtx *new1 = GGC_RESIZEVEC (rtx, regno_reg_rtx, new_size);
    1269       182447 :   memset (new1 + old_size, 0, (new_size - old_size) * sizeof (rtx));
    1270       182447 :   regno_reg_rtx = new1;
    1271              : 
    1272       182447 :   crtl->emit.regno_pointer_align_length = new_size;
    1273              : }
    1274              : 
    1275              : /* Return TRUE if REG is a PARM_DECL, FALSE otherwise.  */
    1276              : 
    1277              : bool
    1278        12103 : reg_is_parm_p (rtx reg)
    1279              : {
    1280        12103 :   tree decl;
    1281              : 
    1282        12103 :   gcc_assert (REG_P (reg));
    1283        12103 :   decl = REG_EXPR (reg);
    1284         9657 :   return (decl && TREE_CODE (decl) == PARM_DECL);
    1285              : }
    1286              : 
    1287              : /* Update NEW with the same attributes as REG, but with OFFSET added
    1288              :    to the REG_OFFSET.  */
    1289              : 
    1290              : static void
    1291     18675708 : update_reg_offset (rtx new_rtx, rtx reg, poly_int64 offset)
    1292              : {
    1293     18675708 :   REG_ATTRS (new_rtx) = get_reg_attrs (REG_EXPR (reg),
    1294     18675708 :                                        REG_OFFSET (reg) + offset);
    1295     18675708 : }
    1296              : 
    1297              : /* Generate a register with same attributes as REG, but with OFFSET
    1298              :    added to the REG_OFFSET.  */
    1299              : 
    1300              : rtx
    1301     11069598 : gen_rtx_REG_offset (rtx reg, machine_mode mode, unsigned int regno,
    1302              :                     poly_int64 offset)
    1303              : {
    1304              :   /* Use gen_raw_REG rather than gen_rtx_REG, because otherwise we'd
    1305              :      overwrite REG_ATTRS (and in the callers often ORIGINAL_REGNO too)
    1306              :      of the shared REG rtxes like stack_pointer_rtx etc.  This should
    1307              :      happen only for SUBREGs from DEBUG_INSNs, RA should ensure
    1308              :      multi-word registers don't overlap the special registers like
    1309              :      stack pointer.  */
    1310     11069598 :   rtx new_rtx = gen_raw_REG (mode, regno);
    1311              : 
    1312     11069598 :   update_reg_offset (new_rtx, reg, offset);
    1313     11069598 :   return new_rtx;
    1314              : }
    1315              : 
    1316              : /* Generate a new pseudo-register with the same attributes as REG, but
    1317              :    with OFFSET added to the REG_OFFSET.  */
    1318              : 
    1319              : rtx
    1320       630634 : gen_reg_rtx_offset (rtx reg, machine_mode mode, int offset)
    1321              : {
    1322       630634 :   rtx new_rtx = gen_reg_rtx (mode);
    1323              : 
    1324       630634 :   update_reg_offset (new_rtx, reg, offset);
    1325       630634 :   return new_rtx;
    1326              : }
    1327              : 
    1328              : /* Adjust REG in-place so that it has mode MODE.  It is assumed that the
    1329              :    new register is a (possibly paradoxical) lowpart of the old one.  */
    1330              : 
    1331              : void
    1332      2892516 : adjust_reg_mode (rtx reg, machine_mode mode)
    1333              : {
    1334      2892516 :   update_reg_offset (reg, reg, byte_lowpart_offset (mode, GET_MODE (reg)));
    1335      2892516 :   PUT_MODE (reg, mode);
    1336      2892516 : }
    1337              : 
    1338              : /* Copy REG's attributes from X, if X has any attributes.  If REG and X
    1339              :    have different modes, REG is a (possibly paradoxical) lowpart of X.  */
    1340              : 
    1341              : void
    1342     76120689 : set_reg_attrs_from_value (rtx reg, rtx x)
    1343              : {
    1344     76120689 :   poly_int64 offset;
    1345     76120689 :   bool can_be_reg_pointer = true;
    1346              : 
    1347              :   /* Don't call mark_reg_pointer for incompatible pointer sign
    1348              :      extension.  */
    1349     76120689 :   while (GET_CODE (x) == SIGN_EXTEND
    1350              :          || GET_CODE (x) == ZERO_EXTEND
    1351     77174359 :          || GET_CODE (x) == TRUNCATE
    1352     77174359 :          || (GET_CODE (x) == SUBREG && subreg_lowpart_p (x)))
    1353              :     {
    1354              : #if defined(POINTERS_EXTEND_UNSIGNED)
    1355      1053670 :       if (((GET_CODE (x) == SIGN_EXTEND && POINTERS_EXTEND_UNSIGNED)
    1356              :            || (GET_CODE (x) == ZERO_EXTEND && ! POINTERS_EXTEND_UNSIGNED)
    1357       491227 :            || (paradoxical_subreg_p (x)
    1358        18075 :                && ! (SUBREG_PROMOTED_VAR_P (x)
    1359            0 :                      && SUBREG_CHECK_PROMOTED_SIGN (x,
    1360              :                                                     POINTERS_EXTEND_UNSIGNED))))
    1361      1071745 :           && !targetm.have_ptr_extend ())
    1362              :         can_be_reg_pointer = false;
    1363              : #endif
    1364      1053670 :       x = XEXP (x, 0);
    1365              :     }
    1366              : 
    1367              :   /* Hard registers can be reused for multiple purposes within the same
    1368              :      function, so setting REG_ATTRS, REG_POINTER and REG_POINTER_ALIGN
    1369              :      on them is wrong.  */
    1370     76120689 :   if (HARD_REGISTER_P (reg))
    1371     49368739 :     return;
    1372              : 
    1373     26751950 :   offset = byte_lowpart_offset (GET_MODE (reg), GET_MODE (x));
    1374     26751950 :   if (MEM_P (x))
    1375              :     {
    1376      5723121 :       if (MEM_OFFSET_KNOWN_P (x))
    1377      2686248 :         REG_ATTRS (reg) = get_reg_attrs (MEM_EXPR (x),
    1378      2686248 :                                          MEM_OFFSET (x) + offset);
    1379      5262785 :       if (can_be_reg_pointer && MEM_POINTER (x))
    1380       663982 :         mark_reg_pointer (reg, 0);
    1381              :     }
    1382     21489165 :   else if (REG_P (x))
    1383              :     {
    1384      6960597 :       if (REG_ATTRS (x))
    1385      4082960 :         update_reg_offset (reg, x, offset);
    1386      6960597 :       if (can_be_reg_pointer && REG_POINTER (x))
    1387      1929262 :         mark_reg_pointer (reg, REGNO_POINTER_ALIGN (REGNO (x)));
    1388              :     }
    1389              : }
    1390              : 
    1391              : /* Generate a REG rtx for a new pseudo register, copying the mode
    1392              :    and attributes from X.  */
    1393              : 
    1394              : rtx
    1395       572418 : gen_reg_rtx_and_attrs (rtx x)
    1396              : {
    1397       572418 :   rtx reg = gen_reg_rtx (GET_MODE (x));
    1398       572418 :   set_reg_attrs_from_value (reg, x);
    1399       572418 :   return reg;
    1400              : }
    1401              : 
    1402              : /* Set the register attributes for registers contained in PARM_RTX.
    1403              :    Use needed values from memory attributes of MEM.  */
    1404              : 
    1405              : void
    1406       110335 : set_reg_attrs_for_parm (rtx parm_rtx, rtx mem)
    1407              : {
    1408       110335 :   if (REG_P (parm_rtx))
    1409       110335 :     set_reg_attrs_from_value (parm_rtx, mem);
    1410            0 :   else if (GET_CODE (parm_rtx) == PARALLEL)
    1411              :     {
    1412              :       /* Check for a NULL entry in the first slot, used to indicate that the
    1413              :          parameter goes both on the stack and in registers.  */
    1414            0 :       int i = XEXP (XVECEXP (parm_rtx, 0, 0), 0) ? 0 : 1;
    1415            0 :       for (; i < XVECLEN (parm_rtx, 0); i++)
    1416              :         {
    1417            0 :           rtx x = XVECEXP (parm_rtx, 0, i);
    1418            0 :           if (REG_P (XEXP (x, 0)))
    1419            0 :             REG_ATTRS (XEXP (x, 0))
    1420            0 :               = get_reg_attrs (MEM_EXPR (mem),
    1421            0 :                                INTVAL (XEXP (x, 1)));
    1422              :         }
    1423              :     }
    1424       110335 : }
    1425              : 
    1426              : /* Set the REG_ATTRS for registers in value X, given that X represents
    1427              :    decl T.  */
    1428              : 
    1429              : void
    1430     67882303 : set_reg_attrs_for_decl_rtl (tree t, rtx x)
    1431              : {
    1432     67882303 :   if (!t)
    1433              :     return;
    1434     67882107 :   tree tdecl = t;
    1435     67882107 :   if (GET_CODE (x) == SUBREG)
    1436              :     {
    1437          621 :       gcc_assert (subreg_lowpart_p (x));
    1438          621 :       x = SUBREG_REG (x);
    1439              :     }
    1440     67882107 :   if (REG_P (x))
    1441     44656942 :     REG_ATTRS (x)
    1442     44656942 :       = get_reg_attrs (t, byte_lowpart_offset (GET_MODE (x),
    1443     44656942 :                                                DECL_P (tdecl)
    1444     21824535 :                                                ? DECL_MODE (tdecl)
    1445     22832407 :                                                : TYPE_MODE (TREE_TYPE (tdecl))));
    1446     67882107 :   if (GET_CODE (x) == CONCAT)
    1447              :     {
    1448       229410 :       if (REG_P (XEXP (x, 0)))
    1449       229410 :         REG_ATTRS (XEXP (x, 0)) = get_reg_attrs (t, 0);
    1450       229410 :       if (REG_P (XEXP (x, 1)))
    1451       229410 :         REG_ATTRS (XEXP (x, 1))
    1452       458820 :           = get_reg_attrs (t, GET_MODE_UNIT_SIZE (GET_MODE (XEXP (x, 0))));
    1453              :     }
    1454     67882107 :   if (GET_CODE (x) == PARALLEL)
    1455              :     {
    1456        61912 :       int i, start;
    1457              : 
    1458              :       /* Check for a NULL entry, used to indicate that the parameter goes
    1459              :          both on the stack and in registers.  */
    1460        61912 :       if (XEXP (XVECEXP (x, 0, 0), 0))
    1461              :         start = 0;
    1462              :       else
    1463            0 :         start = 1;
    1464              : 
    1465       177277 :       for (i = start; i < XVECLEN (x, 0); i++)
    1466              :         {
    1467       115365 :           rtx y = XVECEXP (x, 0, i);
    1468       115365 :           if (REG_P (XEXP (y, 0)))
    1469       115365 :             REG_ATTRS (XEXP (y, 0)) = get_reg_attrs (t, INTVAL (XEXP (y, 1)));
    1470              :         }
    1471              :     }
    1472              : }
    1473              : 
    1474              : /* Assign the RTX X to declaration T.  */
    1475              : 
    1476              : void
    1477    643686011 : set_decl_rtl (tree t, rtx x)
    1478              : {
    1479    643686011 :   DECL_WRTL_CHECK (t)->decl_with_rtl.rtl = x;
    1480    643686011 :   if (x)
    1481     34206265 :     set_reg_attrs_for_decl_rtl (t, x);
    1482    643686011 : }
    1483              : 
    1484              : /* Assign the RTX X to parameter declaration T.  BY_REFERENCE_P is true
    1485              :    if the ABI requires the parameter to be passed by reference.  */
    1486              : 
    1487              : void
    1488      3256885 : set_decl_incoming_rtl (tree t, rtx x, bool by_reference_p)
    1489              : {
    1490      3256885 :   DECL_INCOMING_RTL (t) = x;
    1491      3256885 :   if (x && !by_reference_p)
    1492      3251919 :     set_reg_attrs_for_decl_rtl (t, x);
    1493      3256885 : }
    1494              : 
    1495              : /* Identify REG (which may be a CONCAT) as a user register.  */
    1496              : 
    1497              : void
    1498      6397059 : mark_user_reg (rtx reg)
    1499              : {
    1500      6397059 :   if (GET_CODE (reg) == CONCAT)
    1501              :     {
    1502         3346 :       REG_USERVAR_P (XEXP (reg, 0)) = 1;
    1503         3346 :       REG_USERVAR_P (XEXP (reg, 1)) = 1;
    1504              :     }
    1505              :   else
    1506              :     {
    1507      6393713 :       gcc_assert (REG_P (reg));
    1508      6393713 :       REG_USERVAR_P (reg) = 1;
    1509              :     }
    1510      6397059 : }
    1511              : 
    1512              : /* Identify REG as a probable pointer register and show its alignment
    1513              :    as ALIGN, if nonzero.  */
    1514              : 
    1515              : void
    1516     17563660 : mark_reg_pointer (rtx reg, int align)
    1517              : {
    1518     17563660 :   if (! REG_POINTER (reg))
    1519              :     {
    1520     11123029 :       REG_POINTER (reg) = 1;
    1521              : 
    1522     11123029 :       if (align)
    1523      9656670 :         REGNO_POINTER_ALIGN (REGNO (reg)) = align;
    1524              :     }
    1525      6440631 :   else if (align && align < REGNO_POINTER_ALIGN (REGNO (reg)))
    1526              :     /* We can no-longer be sure just how aligned this pointer is.  */
    1527      1325816 :     REGNO_POINTER_ALIGN (REGNO (reg)) = align;
    1528     17563660 : }
    1529              : 
    1530              : /* Return 1 plus largest pseudo reg number used in the current function.  */
    1531              : 
    1532              : int
    1533   7171036264 : max_reg_num (void)
    1534              : {
    1535   7171036264 :   return reg_rtx_no;
    1536              : }
    1537              : 
    1538              : /* Return 1 + the largest label number used so far in the current function.  */
    1539              : 
    1540              : int
    1541      4107363 : max_label_num (void)
    1542              : {
    1543      4107363 :   return label_num;
    1544              : }
    1545              : 
    1546              : /* Return first label number used in this function (if any were used).  */
    1547              : 
    1548              : int
    1549      2588324 : get_first_label_num (void)
    1550              : {
    1551      2588324 :   return first_label_num;
    1552              : }
    1553              : 
    1554              : /* If the rtx for label was created during the expansion of a nested
    1555              :    function, then first_label_num won't include this label number.
    1556              :    Fix this now so that array indices work later.  */
    1557              : 
    1558              : void
    1559        27192 : maybe_set_first_label_num (rtx_code_label *x)
    1560              : {
    1561        27192 :   if (CODE_LABEL_NUMBER (x) < first_label_num)
    1562          400 :     first_label_num = CODE_LABEL_NUMBER (x);
    1563        27192 : }
    1564              : 
    1565              : /* For use by the RTL function loader, when mingling with normal
    1566              :    functions.
    1567              :    Ensure that label_num is greater than the label num of X, to avoid
    1568              :    duplicate labels in the generated assembler.  */
    1569              : 
    1570              : void
    1571           28 : maybe_set_max_label_num (rtx_code_label *x)
    1572              : {
    1573           28 :   if (CODE_LABEL_NUMBER (x) >= label_num)
    1574           24 :     label_num = CODE_LABEL_NUMBER (x) + 1;
    1575           28 : }
    1576              : 
    1577              : 
    1578              : /* Return a value representing some low-order bits of X, where the number
    1579              :    of low-order bits is given by MODE.  Note that no conversion is done
    1580              :    between floating-point and fixed-point values, rather, the bit
    1581              :    representation is returned.
    1582              : 
    1583              :    This function handles the cases in common between gen_lowpart, below,
    1584              :    and two variants in cse.cc and combine.cc.  These are the cases that can
    1585              :    be safely handled at all points in the compilation.
    1586              : 
    1587              :    If this is not a case we can handle, return 0.  */
    1588              : 
    1589              : rtx
    1590     54314739 : gen_lowpart_common (machine_mode mode, rtx x)
    1591              : {
    1592    108655108 :   poly_uint64 msize = GET_MODE_SIZE (mode);
    1593     54327554 :   machine_mode innermode;
    1594              : 
    1595              :   /* Unfortunately, this routine doesn't take a parameter for the mode of X,
    1596              :      so we have to make one up.  Yuk.  */
    1597     54327554 :   innermode = GET_MODE (x);
    1598     54327554 :   if (CONST_INT_P (x)
    1599     54327554 :       && known_le (msize * BITS_PER_UNIT,
    1600              :                    (unsigned HOST_WIDE_INT) HOST_BITS_PER_WIDE_INT))
    1601     16042795 :     innermode = int_mode_for_size (HOST_BITS_PER_WIDE_INT, 0).require ();
    1602     38284759 :   else if (innermode == VOIDmode)
    1603       197634 :     innermode = int_mode_for_size (HOST_BITS_PER_DOUBLE_INT, 0).require ();
    1604              : 
    1605     54327554 :   gcc_assert (innermode != VOIDmode && innermode != BLKmode);
    1606              : 
    1607     54327554 :   if (innermode == mode)
    1608              :     return x;
    1609              : 
    1610              :   /* The size of the outer and inner modes must be ordered.  */
    1611     63009074 :   poly_uint64 xsize = GET_MODE_SIZE (innermode);
    1612     31504537 :   if (!ordered_p (msize, xsize))
    1613              :     return 0;
    1614              : 
    1615     31504537 :   if (SCALAR_FLOAT_MODE_P (mode))
    1616              :     {
    1617              :       /* Don't allow paradoxical FLOAT_MODE subregs.  */
    1618       267436 :       if (maybe_gt (msize, xsize))
    1619              :         return 0;
    1620              :     }
    1621              :   else
    1622              :     {
    1623              :       /* MODE must occupy no more of the underlying registers than X.  */
    1624     31237101 :       poly_uint64 regsize = REGMODE_NATURAL_SIZE (innermode);
    1625     31237101 :       unsigned int mregs, xregs;
    1626     31237101 :       if (!can_div_away_from_zero_p (msize, regsize, &mregs)
    1627     31237101 :           || !can_div_away_from_zero_p (xsize, regsize, &xregs)
    1628     31237101 :           || mregs > xregs)
    1629         1243 :         return 0;
    1630              :     }
    1631              : 
    1632     31503294 :   scalar_int_mode int_mode, int_innermode, from_mode;
    1633     31503294 :   if ((GET_CODE (x) == ZERO_EXTEND || GET_CODE (x) == SIGN_EXTEND)
    1634       171871 :       && is_a <scalar_int_mode> (mode, &int_mode)
    1635     31503294 :       && is_a <scalar_int_mode> (innermode, &int_innermode)
    1636     31675117 :       && is_a <scalar_int_mode> (GET_MODE (XEXP (x, 0)), &from_mode))
    1637              :     {
    1638              :       /* If we are getting the low-order part of something that has been
    1639              :          sign- or zero-extended, we can either just use the object being
    1640              :          extended or make a narrower extension.  If we want an even smaller
    1641              :          piece than the size of the object being extended, call ourselves
    1642              :          recursively.
    1643              : 
    1644              :          This case is used mostly by combine and cse.  */
    1645              : 
    1646       171823 :       if (from_mode == int_mode)
    1647              :         return XEXP (x, 0);
    1648        50262 :       else if (GET_MODE_SIZE (int_mode) < GET_MODE_SIZE (from_mode))
    1649              :         return gen_lowpart_common (int_mode, XEXP (x, 0));
    1650        11817 :       else if (GET_MODE_SIZE (int_mode) < GET_MODE_SIZE (int_innermode))
    1651         3624 :         return gen_rtx_fmt_e (GET_CODE (x), int_mode, XEXP (x, 0));
    1652              :     }
    1653     31331471 :   else if (GET_CODE (x) == SUBREG || REG_P (x)
    1654              :            || GET_CODE (x) == CONCAT || GET_CODE (x) == CONST_VECTOR
    1655     17608650 :            || CONST_DOUBLE_AS_FLOAT_P (x) || CONST_SCALAR_INT_P (x)
    1656              :            || CONST_POLY_INT_P (x))
    1657     20980445 :     return lowpart_subreg (mode, x, innermode);
    1658              : 
    1659              :   /* Otherwise, we can't do this.  */
    1660              :   return 0;
    1661              : }
    1662              : 
    1663              : rtx
    1664        11743 : gen_highpart (machine_mode mode, rtx x)
    1665              : {
    1666        23486 :   poly_uint64 msize = GET_MODE_SIZE (mode);
    1667        11743 :   rtx result;
    1668              : 
    1669              :   /* This case loses if X is a subreg.  To catch bugs early,
    1670              :      complain if an invalid MODE is used even in other cases.  */
    1671        13481 :   gcc_assert (known_le (msize, (unsigned int) UNITS_PER_WORD)
    1672              :               || known_eq (msize, GET_MODE_UNIT_SIZE (GET_MODE (x))));
    1673              : 
    1674              :   /* gen_lowpart_common handles a lot of special cases due to needing to handle
    1675              :      paradoxical subregs; it only calls simplify_gen_subreg when certain that
    1676              :      it will produce something meaningful.  The only case we need to handle
    1677              :      specially here is MEM.  */
    1678        11743 :   if (MEM_P (x))
    1679              :     {
    1680          111 :       poly_int64 offset = subreg_highpart_offset (mode, GET_MODE (x));
    1681          111 :       return adjust_address (x, mode, offset);
    1682              :     }
    1683              : 
    1684        11632 :   result = simplify_gen_subreg (mode, x, GET_MODE (x),
    1685        11632 :                                 subreg_highpart_offset (mode, GET_MODE (x)));
    1686              :   /* Since we handle MEM directly above, we should never get a MEM back
    1687              :      from simplify_gen_subreg.  */
    1688        11632 :   gcc_assert (result && !MEM_P (result));
    1689              : 
    1690              :   return result;
    1691              : }
    1692              : 
    1693              : /* Like gen_highpart, but accept mode of EXP operand in case EXP can
    1694              :    be VOIDmode constant.  */
    1695              : rtx
    1696            0 : gen_highpart_mode (machine_mode outermode, machine_mode innermode, rtx exp)
    1697              : {
    1698            0 :   if (GET_MODE (exp) != VOIDmode)
    1699              :     {
    1700            0 :       gcc_assert (GET_MODE (exp) == innermode);
    1701            0 :       return gen_highpart (outermode, exp);
    1702              :     }
    1703            0 :   return simplify_gen_subreg (outermode, exp, innermode,
    1704            0 :                               subreg_highpart_offset (outermode, innermode));
    1705              : }
    1706              : 
    1707              : /* Return the SUBREG_BYTE for a lowpart subreg whose outer mode has
    1708              :    OUTER_BYTES bytes and whose inner mode has INNER_BYTES bytes.  */
    1709              : 
    1710              : poly_uint64
    1711    239345036 : subreg_size_lowpart_offset (poly_uint64 outer_bytes, poly_uint64 inner_bytes)
    1712              : {
    1713    239345036 :   gcc_checking_assert (ordered_p (outer_bytes, inner_bytes));
    1714    239345036 :   if (maybe_gt (outer_bytes, inner_bytes))
    1715              :     /* Paradoxical subregs always have a SUBREG_BYTE of 0.  */
    1716              :     return 0;
    1717              : 
    1718    239345036 :   if (BYTES_BIG_ENDIAN && WORDS_BIG_ENDIAN)
    1719              :     return inner_bytes - outer_bytes;
    1720    239345036 :   else if (!BYTES_BIG_ENDIAN && !WORDS_BIG_ENDIAN)
    1721    239345036 :     return 0;
    1722              :   else
    1723              :     return subreg_size_offset_from_lsb (outer_bytes, inner_bytes, 0);
    1724              : }
    1725              : 
    1726              : /* Return the SUBREG_BYTE for a highpart subreg whose outer mode has
    1727              :    OUTER_BYTES bytes and whose inner mode has INNER_BYTES bytes.  */
    1728              : 
    1729              : poly_uint64
    1730        46603 : subreg_size_highpart_offset (poly_uint64 outer_bytes, poly_uint64 inner_bytes)
    1731              : {
    1732        46603 :   gcc_assert (known_ge (inner_bytes, outer_bytes));
    1733              : 
    1734              :   if (BYTES_BIG_ENDIAN && WORDS_BIG_ENDIAN)
    1735              :     return 0;
    1736              :   else if (!BYTES_BIG_ENDIAN && !WORDS_BIG_ENDIAN)
    1737        46603 :     return inner_bytes - outer_bytes;
    1738              :   else
    1739              :     return subreg_size_offset_from_lsb (outer_bytes, inner_bytes,
    1740              :                                         (inner_bytes - outer_bytes)
    1741              :                                         * BITS_PER_UNIT);
    1742              : }
    1743              : 
    1744              : /* Return true iff X, assumed to be a SUBREG,
    1745              :    refers to the least significant part of its containing reg.
    1746              :    If X is not a SUBREG, always return true (it is its own low part!).  */
    1747              : 
    1748              : bool
    1749     56062115 : subreg_lowpart_p (const_rtx x)
    1750              : {
    1751     56062115 :   if (GET_CODE (x) != SUBREG)
    1752              :     return true;
    1753     56062115 :   else if (GET_MODE (SUBREG_REG (x)) == VOIDmode)
    1754              :     return false;
    1755              : 
    1756     56062115 :   return known_eq (subreg_lowpart_offset (GET_MODE (x),
    1757              :                                           GET_MODE (SUBREG_REG (x))),
    1758              :                    SUBREG_BYTE (x));
    1759              : }
    1760              : 
    1761              : /* Return subword OFFSET of operand OP.
    1762              :    The word number, OFFSET, is interpreted as the word number starting
    1763              :    at the low-order address.  OFFSET 0 is the low-order word if not
    1764              :    WORDS_BIG_ENDIAN, otherwise it is the high-order word.
    1765              : 
    1766              :    If we cannot extract the required word, we return zero.  Otherwise,
    1767              :    an rtx corresponding to the requested word will be returned.
    1768              : 
    1769              :    VALIDATE_ADDRESS is nonzero if the address should be validated.  Before
    1770              :    reload has completed, a valid address will always be returned.  After
    1771              :    reload, if a valid address cannot be returned, we return zero.
    1772              : 
    1773              :    If VALIDATE_ADDRESS is zero, we simply form the required address; validating
    1774              :    it is the responsibility of the caller.
    1775              : 
    1776              :    MODE is the mode of OP in case it is a CONST_INT.
    1777              : 
    1778              :    ??? This is still rather broken for some cases.  The problem for the
    1779              :    moment is that all callers of this thing provide no 'goal mode' to
    1780              :    tell us to work with.  This exists because all callers were written
    1781              :    in a word based SUBREG world.
    1782              :    Now use of this function can be deprecated by simplify_subreg in most
    1783              :    cases.
    1784              :  */
    1785              : 
    1786              : rtx
    1787        89711 : operand_subword (rtx op, poly_uint64 offset, int validate_address,
    1788              :                  machine_mode mode)
    1789              : {
    1790        89711 :   if (mode == VOIDmode)
    1791         3038 :     mode = GET_MODE (op);
    1792              : 
    1793         3038 :   gcc_assert (mode != VOIDmode);
    1794              : 
    1795              :   /* If OP is narrower than a word, fail.  */
    1796        89711 :   if (mode != BLKmode
    1797       194559 :       && maybe_lt (GET_MODE_SIZE (mode), UNITS_PER_WORD))
    1798              :     return 0;
    1799              : 
    1800              :   /* If we want a word outside OP, return zero.  */
    1801        89711 :   if (mode != BLKmode
    1802       194559 :       && maybe_gt ((offset + 1) * UNITS_PER_WORD, GET_MODE_SIZE (mode)))
    1803            0 :     return const0_rtx;
    1804              : 
    1805              :   /* Form a new MEM at the requested address.  */
    1806        89711 :   if (MEM_P (op))
    1807              :     {
    1808        11285 :       rtx new_rtx = adjust_address_nv (op, word_mode, offset * UNITS_PER_WORD);
    1809              : 
    1810         9949 :       if (! validate_address)
    1811              :         return new_rtx;
    1812              : 
    1813         9949 :       else if (reload_completed)
    1814              :         {
    1815            0 :           if (! strict_memory_address_addr_space_p (word_mode,
    1816              :                                                     XEXP (new_rtx, 0),
    1817            0 :                                                     MEM_ADDR_SPACE (op)))
    1818              :             return 0;
    1819              :         }
    1820              :       else
    1821         9949 :         return replace_equiv_address (new_rtx, XEXP (new_rtx, 0));
    1822              :     }
    1823              : 
    1824              :   /* Rest can be handled by simplify_subreg.  */
    1825        96674 :   return simplify_gen_subreg (word_mode, op, mode, (offset * UNITS_PER_WORD));
    1826              : }
    1827              : 
    1828              : /* Similar to `operand_subword', but never return 0.  If we can't
    1829              :    extract the required subword, put OP into a register and try again.
    1830              :    The second attempt must succeed.  We always validate the address in
    1831              :    this case.
    1832              : 
    1833              :    MODE is the mode of OP, in case it is CONST_INT.  */
    1834              : 
    1835              : rtx
    1836        67688 : operand_subword_force (rtx op, poly_uint64 offset, machine_mode mode)
    1837              : {
    1838        67688 :   rtx result = operand_subword (op, offset, 1, mode);
    1839              : 
    1840        67688 :   if (result)
    1841              :     return result;
    1842              : 
    1843            0 :   if (mode != BLKmode && mode != VOIDmode)
    1844              :     {
    1845              :       /* If this is a register which cannot be accessed by words, copy it
    1846              :          to a pseudo register.  */
    1847            0 :       if (REG_P (op))
    1848            0 :         op = copy_to_reg (op);
    1849              :       else
    1850            0 :         op = force_reg (mode, op);
    1851              :     }
    1852              : 
    1853            0 :   result = operand_subword (op, offset, 1, mode);
    1854            0 :   gcc_assert (result);
    1855              : 
    1856              :   return result;
    1857              : }
    1858              : 
    1859     32448615 : mem_attrs::mem_attrs ()
    1860     32448615 :   : expr (NULL_TREE),
    1861     32448615 :     offset (0),
    1862     32448615 :     size (0),
    1863     32448615 :     alias (0),
    1864     32448615 :     align (0),
    1865     32448615 :     addrspace (ADDR_SPACE_GENERIC),
    1866     32448615 :     offset_known_p (false),
    1867     32448615 :     size_known_p (false)
    1868     32448615 : {}
    1869              : 
    1870              : /* Returns true if both MEM_EXPR can be considered equal
    1871              :    and false otherwise.  */
    1872              : 
    1873              : bool
    1874        45388 : mem_expr_equal_p (const_tree expr1, const_tree expr2)
    1875              : {
    1876        45388 :   if (expr1 == expr2)
    1877              :     return true;
    1878              : 
    1879        44783 :   if (! expr1 || ! expr2)
    1880              :     return false;
    1881              : 
    1882        34826 :   if (TREE_CODE (expr1) != TREE_CODE (expr2))
    1883              :     return false;
    1884              : 
    1885        33216 :   return operand_equal_p (expr1, expr2, 0);
    1886              : }
    1887              : 
    1888              : /* Return OFFSET if XEXP (MEM, 0) - OFFSET is known to be ALIGN
    1889              :    bits aligned for 0 <= OFFSET < ALIGN / BITS_PER_UNIT, or
    1890              :    -1 if not known.  */
    1891              : 
    1892              : int
    1893           22 : get_mem_align_offset (rtx mem, unsigned int align)
    1894              : {
    1895           22 :   tree expr;
    1896           22 :   poly_uint64 offset;
    1897              : 
    1898              :   /* This function can't use
    1899              :      if (!MEM_EXPR (mem) || !MEM_OFFSET_KNOWN_P (mem)
    1900              :          || (MAX (MEM_ALIGN (mem),
    1901              :                   MAX (align, get_object_alignment (MEM_EXPR (mem))))
    1902              :              < align))
    1903              :        return -1;
    1904              :      else
    1905              :        return (- MEM_OFFSET (mem)) & (align / BITS_PER_UNIT - 1);
    1906              :      for two reasons:
    1907              :      - COMPONENT_REFs in MEM_EXPR can have NULL first operand,
    1908              :        for <variable>.  get_inner_reference doesn't handle it and
    1909              :        even if it did, the alignment in that case needs to be determined
    1910              :        from DECL_FIELD_CONTEXT's TYPE_ALIGN.
    1911              :      - it would do suboptimal job for COMPONENT_REFs, even if MEM_EXPR
    1912              :        isn't sufficiently aligned, the object it is in might be.  */
    1913           22 :   gcc_assert (MEM_P (mem));
    1914           22 :   expr = MEM_EXPR (mem);
    1915           22 :   if (expr == NULL_TREE || !MEM_OFFSET_KNOWN_P (mem))
    1916              :     return -1;
    1917              : 
    1918           21 :   offset = MEM_OFFSET (mem);
    1919           21 :   if (DECL_P (expr))
    1920              :     {
    1921            2 :       if (DECL_ALIGN (expr) < align)
    1922              :         return -1;
    1923              :     }
    1924           19 :   else if (INDIRECT_REF_P (expr))
    1925              :     {
    1926            0 :       if (TYPE_ALIGN (TREE_TYPE (expr)) < (unsigned int) align)
    1927              :         return -1;
    1928              :     }
    1929           19 :   else if (TREE_CODE (expr) == COMPONENT_REF)
    1930              :     {
    1931            0 :       while (1)
    1932              :         {
    1933            0 :           tree inner = TREE_OPERAND (expr, 0);
    1934            0 :           tree field = TREE_OPERAND (expr, 1);
    1935            0 :           tree byte_offset = component_ref_field_offset (expr);
    1936            0 :           tree bit_offset = DECL_FIELD_BIT_OFFSET (field);
    1937              : 
    1938            0 :           poly_uint64 suboffset;
    1939            0 :           if (!byte_offset
    1940            0 :               || !poly_int_tree_p (byte_offset, &suboffset)
    1941            0 :               || !tree_fits_uhwi_p (bit_offset))
    1942            0 :             return -1;
    1943              : 
    1944            0 :           offset += suboffset;
    1945            0 :           offset += tree_to_uhwi (bit_offset) / BITS_PER_UNIT;
    1946              : 
    1947            0 :           if (inner == NULL_TREE)
    1948              :             {
    1949            0 :               if (TYPE_ALIGN (DECL_FIELD_CONTEXT (field))
    1950              :                   < (unsigned int) align)
    1951              :                 return -1;
    1952            0 :               break;
    1953              :             }
    1954            0 :           else if (DECL_P (inner))
    1955              :             {
    1956            0 :               if (DECL_ALIGN (inner) < align)
    1957              :                 return -1;
    1958              :               break;
    1959              :             }
    1960            0 :           else if (TREE_CODE (inner) != COMPONENT_REF)
    1961              :             return -1;
    1962            0 :           expr = inner;
    1963            0 :         }
    1964              :     }
    1965              :   else
    1966              :     return -1;
    1967              : 
    1968            0 :   HOST_WIDE_INT misalign;
    1969            0 :   if (!known_misalignment (offset, align / BITS_PER_UNIT, &misalign))
    1970              :     return -1;
    1971            0 :   return misalign;
    1972              : }
    1973              : 
    1974              : /* Given REF (a MEM) and T, either the type of X or the expression
    1975              :    corresponding to REF, set the memory attributes.  OBJECTP is nonzero
    1976              :    if we are making a new object of this type.  BITPOS is nonzero if
    1977              :    there is an offset outstanding on T that will be applied later.
    1978              :    MAY_STORE_P is true when REF can be the destination of a store.  */
    1979              : 
    1980              : void
    1981     32448615 : set_mem_attributes_minus_bitpos (rtx ref, tree t, int objectp,
    1982              :                                  poly_int64 bitpos, bool may_store_p)
    1983              : {
    1984     32448615 :   poly_int64 apply_bitpos = 0;
    1985     32448615 :   tree type;
    1986     32448615 :   class mem_attrs attrs, *defattrs, *refattrs;
    1987     32448615 :   addr_space_t as;
    1988              : 
    1989              :   /* It can happen that type_for_mode was given a mode for which there
    1990              :      is no language-level type.  In which case it returns NULL, which
    1991              :      we can see here.  */
    1992     32448615 :   if (t == NULL_TREE)
    1993            0 :     return;
    1994              : 
    1995     32448615 :   type = TYPE_P (t) ? t : TREE_TYPE (t);
    1996     32448615 :   if (type == error_mark_node)
    1997              :     return;
    1998              : 
    1999              :   /* If we have already set DECL_RTL = ref, get_alias_set will get the
    2000              :      wrong answer, as it assumes that DECL_RTL already has the right alias
    2001              :      info.  Callers should not set DECL_RTL until after the call to
    2002              :      set_mem_attributes.  */
    2003     32448615 :   gcc_assert (!DECL_P (t) || ref != DECL_RTL_IF_SET (t));
    2004              : 
    2005              :   /* Get the alias set from the expression or type (perhaps using a
    2006              :      front-end routine) and use it.  */
    2007     32448615 :   attrs.alias = get_alias_set (t);
    2008              : 
    2009     32448615 :   MEM_VOLATILE_P (ref) |= TYPE_VOLATILE (type);
    2010     32448615 :   MEM_POINTER (ref) = POINTER_TYPE_P (type);
    2011              : 
    2012              :   /* Default values from pre-existing memory attributes if present.  */
    2013     32448615 :   refattrs = MEM_ATTRS (ref);
    2014     32448615 :   if (refattrs)
    2015              :     {
    2016              :       /* ??? Can this ever happen?  Calling this routine on a MEM that
    2017              :          already carries memory attributes should probably be invalid.  */
    2018     13154327 :       attrs.expr = refattrs->expr;
    2019     13154327 :       attrs.offset_known_p = refattrs->offset_known_p;
    2020     13154327 :       attrs.offset = refattrs->offset;
    2021     13154327 :       attrs.size_known_p = refattrs->size_known_p;
    2022     13154327 :       attrs.size = refattrs->size;
    2023     13154327 :       attrs.align = refattrs->align;
    2024              :     }
    2025              : 
    2026              :   /* Otherwise, default values from the mode of the MEM reference.  */
    2027              :   else
    2028              :     {
    2029     19294288 :       defattrs = mode_mem_attrs[(int) GET_MODE (ref)];
    2030     19294288 :       gcc_assert (!defattrs->expr);
    2031     19294288 :       gcc_assert (!defattrs->offset_known_p);
    2032              : 
    2033              :       /* Respect mode size.  */
    2034     19294288 :       attrs.size_known_p = defattrs->size_known_p;
    2035     19294288 :       attrs.size = defattrs->size;
    2036              :       /* ??? Is this really necessary?  We probably should always get
    2037              :          the size from the type below.  */
    2038              : 
    2039              :       /* Respect mode alignment for STRICT_ALIGNMENT targets if T is a type;
    2040              :          if T is an object, always compute the object alignment below.  */
    2041     19294288 :       if (TYPE_P (t))
    2042      2041520 :         attrs.align = defattrs->align;
    2043              :       else
    2044     17252768 :         attrs.align = BITS_PER_UNIT;
    2045              :       /* ??? If T is a type, respecting mode alignment may *also* be wrong
    2046              :          e.g. if the type carries an alignment attribute.  Should we be
    2047              :          able to simply always use TYPE_ALIGN?  */
    2048              :     }
    2049              : 
    2050              :   /* We can set the alignment from the type if we are making an object or if
    2051              :      this is an INDIRECT_REF.  */
    2052     32448615 :   if (objectp || TREE_CODE (t) == INDIRECT_REF)
    2053     10000813 :     attrs.align = MAX (attrs.align, TYPE_ALIGN (type));
    2054              : 
    2055              :   /* If the size is known, we can set that.  */
    2056     32448615 :   tree new_size = TYPE_SIZE_UNIT (type);
    2057              : 
    2058              :   /* The address-space is that of the type.  */
    2059     32448615 :   as = TYPE_ADDR_SPACE (type);
    2060              : 
    2061              :   /* If T is not a type, we may be able to deduce some more information about
    2062              :      the expression.  */
    2063     32448615 :   if (! TYPE_P (t))
    2064              :     {
    2065     30400773 :       tree base;
    2066              : 
    2067     30400773 :       if (TREE_THIS_VOLATILE (t))
    2068       246920 :         MEM_VOLATILE_P (ref) = 1;
    2069              : 
    2070              :       /* Now remove any conversions: they don't change what the underlying
    2071              :          object is.  Likewise for SAVE_EXPR.  */
    2072     30405596 :       while (CONVERT_EXPR_P (t)
    2073              :              || TREE_CODE (t) == VIEW_CONVERT_EXPR
    2074     30405596 :              || TREE_CODE (t) == SAVE_EXPR)
    2075         4823 :         t = TREE_OPERAND (t, 0);
    2076              : 
    2077              :       /* Note whether this expression can trap.  */
    2078     30400773 :       MEM_NOTRAP_P (ref)
    2079     30400773 :         = !(may_store_p ? lhs_could_trap_p (t) : tree_could_trap_p (t));
    2080              : 
    2081     30400773 :       base = get_base_address (t);
    2082     30400773 :       if (base)
    2083              :         {
    2084     30400773 :           if (DECL_P (base)
    2085     17446761 :               && TREE_READONLY (base)
    2086      2247210 :               && (TREE_STATIC (base) || DECL_EXTERNAL (base))
    2087     32295733 :               && !TREE_THIS_VOLATILE (base))
    2088      1894581 :             MEM_READONLY_P (ref) = 1;
    2089              : 
    2090              :           /* Mark static const strings readonly as well.  */
    2091     30400773 :           if (TREE_CODE (base) == STRING_CST
    2092       287620 :               && TREE_READONLY (base)
    2093     30663238 :               && TREE_STATIC (base))
    2094       262465 :             MEM_READONLY_P (ref) = 1;
    2095              : 
    2096              :           /* Address-space information is on the base object.  */
    2097     30400773 :           if (TREE_CODE (base) == MEM_REF
    2098     30400773 :               || TREE_CODE (base) == TARGET_MEM_REF)
    2099     12666380 :             as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (base,
    2100              :                                                                       0))));
    2101              :           else
    2102     17734393 :             as = TYPE_ADDR_SPACE (TREE_TYPE (base));
    2103              :         }
    2104              : 
    2105              :       /* If this expression uses it's parent's alias set, mark it such
    2106              :          that we won't change it.  */
    2107     30400773 :       if (component_uses_parent_alias_set_from (t) != NULL_TREE)
    2108      1150829 :         MEM_KEEP_ALIAS_SET_P (ref) = 1;
    2109              : 
    2110              :       /* If this is a decl, set the attributes of the MEM from it.  */
    2111     30400773 :       if (DECL_P (t))
    2112              :         {
    2113      7986062 :           attrs.expr = t;
    2114      7986062 :           attrs.offset_known_p = true;
    2115      7986062 :           attrs.offset = 0;
    2116      7986062 :           apply_bitpos = bitpos;
    2117      7986062 :           new_size = DECL_SIZE_UNIT (t);
    2118              :         }
    2119              : 
    2120              :       /* ???  If we end up with a constant or a descriptor do not
    2121              :          record a MEM_EXPR.  */
    2122     22414711 :       else if (CONSTANT_CLASS_P (t)
    2123     22316684 :                || TREE_CODE (t) == CONSTRUCTOR)
    2124              :         ;
    2125              : 
    2126              :       /* If this is a field reference, record it.  */
    2127     22316684 :       else if (TREE_CODE (t) == COMPONENT_REF)
    2128              :         {
    2129     10067449 :           attrs.expr = t;
    2130     10067449 :           attrs.offset_known_p = true;
    2131     10067449 :           attrs.offset = 0;
    2132     10067449 :           apply_bitpos = bitpos;
    2133     10067449 :           if (DECL_BIT_FIELD (TREE_OPERAND (t, 1)))
    2134        66388 :             new_size = DECL_SIZE_UNIT (TREE_OPERAND (t, 1));
    2135              :         }
    2136              : 
    2137              :       /* Else record it.  */
    2138              :       else
    2139              :         {
    2140     12249235 :           gcc_assert (handled_component_p (t)
    2141              :                       || TREE_CODE (t) == MEM_REF
    2142              :                       || TREE_CODE (t) == TARGET_MEM_REF);
    2143     12249235 :           attrs.expr = t;
    2144     12249235 :           attrs.offset_known_p = true;
    2145     12249235 :           attrs.offset = 0;
    2146     12249235 :           apply_bitpos = bitpos;
    2147              :         }
    2148              : 
    2149              :       /* If this is a reference based on a partitioned decl replace the
    2150              :          base with a MEM_REF of the pointer representative we created
    2151              :          during stack slot partitioning.  */
    2152     30400773 :       if (attrs.expr
    2153     30302746 :           && VAR_P (base)
    2154     14322553 :           && ! is_global_var (base)
    2155     39254939 :           && cfun->gimple_df->decls_to_pointers != NULL)
    2156              :         {
    2157      4097450 :           tree *namep = cfun->gimple_df->decls_to_pointers->get (base);
    2158      4097450 :           if (namep)
    2159              :             {
    2160      2393778 :               attrs.expr = unshare_expr (attrs.expr);
    2161      2393778 :               tree *orig_base = &attrs.expr;
    2162      5020577 :               while (handled_component_p (*orig_base))
    2163      2626799 :                 orig_base = &TREE_OPERAND (*orig_base, 0);
    2164      2393778 :               if (TREE_CODE (*orig_base) == MEM_REF
    2165      2393778 :                   || TREE_CODE (*orig_base) == TARGET_MEM_REF)
    2166       727460 :                 TREE_OPERAND (*orig_base, 0) = *namep;
    2167              :               else
    2168              :                 {
    2169      1666318 :                   tree aptrt = reference_alias_ptr_type (*orig_base);
    2170      1666318 :                   *orig_base = build2 (MEM_REF, TREE_TYPE (*orig_base),
    2171              :                                        *namep, build_int_cst (aptrt, 0));
    2172              :                 }
    2173              :             }
    2174              :         }
    2175              : 
    2176              :       /* Compute the alignment.  */
    2177     30400773 :       unsigned int obj_align;
    2178     30400773 :       unsigned HOST_WIDE_INT obj_bitpos;
    2179     30400773 :       get_object_alignment_1 (t, &obj_align, &obj_bitpos);
    2180     30400773 :       unsigned int diff_align = known_alignment (obj_bitpos - bitpos);
    2181     30400773 :       if (diff_align != 0)
    2182      3996399 :         obj_align = MIN (obj_align, diff_align);
    2183     30400773 :       attrs.align = MAX (attrs.align, obj_align);
    2184              :     }
    2185              : 
    2186     32448615 :   poly_uint64 const_size;
    2187     32448615 :   if (poly_int_tree_p (new_size, &const_size))
    2188              :     {
    2189     31805003 :       attrs.size_known_p = true;
    2190     31805003 :       attrs.size = const_size;
    2191              :     }
    2192              : 
    2193              :   /* If we modified OFFSET based on T, then subtract the outstanding
    2194              :      bit position offset.  Similarly, increase the size of the accessed
    2195              :      object to contain the negative offset.  */
    2196     32448615 :   if (maybe_ne (apply_bitpos, 0))
    2197              :     {
    2198      2888752 :       gcc_assert (attrs.offset_known_p);
    2199      2888752 :       poly_int64 bytepos = bits_to_bytes_round_down (apply_bitpos);
    2200      2888752 :       attrs.offset -= bytepos;
    2201      2888752 :       if (attrs.size_known_p)
    2202     32448615 :         attrs.size += bytepos;
    2203              :     }
    2204              : 
    2205              :   /* Now set the attributes we computed above.  */
    2206     32448615 :   attrs.addrspace = as;
    2207     32448615 :   set_mem_attrs (ref, &attrs);
    2208              : }
    2209              : 
    2210              : void
    2211     28239550 : set_mem_attributes (rtx ref, tree t, int objectp, bool may_store_p)
    2212              : {
    2213     28239550 :   set_mem_attributes_minus_bitpos (ref, t, objectp, 0, may_store_p);
    2214     28239550 : }
    2215              : 
    2216              : /* Set the alias set of MEM to SET.  */
    2217              : 
    2218              : void
    2219      7929287 : set_mem_alias_set (rtx mem, alias_set_type set)
    2220              : {
    2221              :   /* If the new and old alias sets don't conflict, something is wrong.  */
    2222     11071570 :   gcc_checking_assert (alias_sets_conflict_p (set, MEM_ALIAS_SET (mem)));
    2223      7929287 :   mem_attrs attrs (*get_mem_attrs (mem));
    2224      7929287 :   attrs.alias = set;
    2225      7929287 :   set_mem_attrs (mem, &attrs);
    2226      7929287 : }
    2227              : 
    2228              : /* Set the address space of MEM to ADDRSPACE (target-defined).  */
    2229              : 
    2230              : void
    2231     10341562 : set_mem_addr_space (rtx mem, addr_space_t addrspace)
    2232              : {
    2233     10341562 :   mem_attrs attrs (*get_mem_attrs (mem));
    2234     10341562 :   attrs.addrspace = addrspace;
    2235     10341562 :   set_mem_attrs (mem, &attrs);
    2236     10341562 : }
    2237              : 
    2238              : /* Set the alignment of MEM to ALIGN bits.  */
    2239              : 
    2240              : void
    2241     12829456 : set_mem_align (rtx mem, unsigned int align)
    2242              : {
    2243     12829456 :   mem_attrs attrs (*get_mem_attrs (mem));
    2244     12829456 :   attrs.align = align;
    2245     12829456 :   set_mem_attrs (mem, &attrs);
    2246     12829456 : }
    2247              : 
    2248              : /* Set the expr for MEM to EXPR.  */
    2249              : 
    2250              : void
    2251      7126041 : set_mem_expr (rtx mem, tree expr)
    2252              : {
    2253      7126041 :   mem_attrs attrs (*get_mem_attrs (mem));
    2254      7126041 :   attrs.expr = expr;
    2255      7126041 :   set_mem_attrs (mem, &attrs);
    2256      7126041 : }
    2257              : 
    2258              : /* Set the offset of MEM to OFFSET.  */
    2259              : 
    2260              : void
    2261          134 : set_mem_offset (rtx mem, poly_int64 offset)
    2262              : {
    2263          134 :   mem_attrs attrs (*get_mem_attrs (mem));
    2264          134 :   attrs.offset_known_p = true;
    2265          134 :   attrs.offset = offset;
    2266          134 :   set_mem_attrs (mem, &attrs);
    2267          134 : }
    2268              : 
    2269              : /* Clear the offset of MEM.  */
    2270              : 
    2271              : void
    2272        91381 : clear_mem_offset (rtx mem)
    2273              : {
    2274        91381 :   mem_attrs attrs (*get_mem_attrs (mem));
    2275        91381 :   attrs.offset_known_p = false;
    2276        91381 :   set_mem_attrs (mem, &attrs);
    2277        91381 : }
    2278              : 
    2279              : /* Set the size of MEM to SIZE.  */
    2280              : 
    2281              : void
    2282      2503489 : set_mem_size (rtx mem, poly_int64 size)
    2283              : {
    2284      2503489 :   mem_attrs attrs (*get_mem_attrs (mem));
    2285      2503489 :   attrs.size_known_p = true;
    2286      2503489 :   attrs.size = size;
    2287      2503489 :   set_mem_attrs (mem, &attrs);
    2288      2503489 : }
    2289              : 
    2290              : /* Clear the size of MEM.  */
    2291              : 
    2292              : void
    2293          615 : clear_mem_size (rtx mem)
    2294              : {
    2295          615 :   mem_attrs attrs (*get_mem_attrs (mem));
    2296          615 :   attrs.size_known_p = false;
    2297          615 :   set_mem_attrs (mem, &attrs);
    2298          615 : }
    2299              : 
    2300              : /* Return a memory reference like MEMREF, but with its mode changed to MODE
    2301              :    and its address changed to ADDR.  (VOIDmode means don't change the mode.
    2302              :    NULL for ADDR means don't change the address.)  VALIDATE is nonzero if the
    2303              :    returned memory location is required to be valid.  INPLACE is true if any
    2304              :    changes can be made directly to MEMREF or false if MEMREF must be treated
    2305              :    as immutable.
    2306              : 
    2307              :    The memory attributes are not changed.  */
    2308              : 
    2309              : static rtx
    2310    507442941 : change_address_1 (rtx memref, machine_mode mode, rtx addr, int validate,
    2311              :                   bool inplace)
    2312              : {
    2313    507442941 :   addr_space_t as;
    2314    507442941 :   rtx new_rtx;
    2315              : 
    2316    507442941 :   gcc_assert (MEM_P (memref));
    2317    507442941 :   as = MEM_ADDR_SPACE (memref);
    2318    507442941 :   if (mode == VOIDmode)
    2319    480274519 :     mode = GET_MODE (memref);
    2320    507442941 :   if (addr == 0)
    2321           64 :     addr = XEXP (memref, 0);
    2322    486853137 :   if (mode == GET_MODE (memref) && addr == XEXP (memref, 0)
    2323    853231137 :       && (!validate || memory_address_addr_space_p (mode, addr, as)))
    2324              :     return memref;
    2325              : 
    2326              :   /* Don't validate address for LRA.  LRA can make the address valid
    2327              :      by itself in most efficient way.  */
    2328    161808474 :   if (validate && !lra_in_progress)
    2329              :     {
    2330     23496591 :       if (reload_in_progress || reload_completed)
    2331      3870853 :         gcc_assert (memory_address_addr_space_p (mode, addr, as));
    2332              :       else
    2333     19625738 :         addr = memory_address_addr_space (mode, addr, as);
    2334              :     }
    2335              : 
    2336    161808474 :   if (rtx_equal_p (addr, XEXP (memref, 0)) && mode == GET_MODE (memref))
    2337              :     return memref;
    2338              : 
    2339    157653543 :   if (inplace)
    2340              :     {
    2341       261256 :       XEXP (memref, 0) = addr;
    2342       261256 :       return memref;
    2343              :     }
    2344              : 
    2345    157392287 :   new_rtx = gen_rtx_MEM (mode, addr);
    2346    157392287 :   MEM_COPY_ATTRIBUTES (new_rtx, memref);
    2347    157392287 :   return new_rtx;
    2348              : }
    2349              : 
    2350              : /* Version of change_address that takes a validate parameter.  */
    2351              : 
    2352              : static rtx
    2353       254825 : change_address (rtx memref, machine_mode mode, rtx addr, int validate)
    2354              : {
    2355       254825 :   rtx new_rtx = change_address_1 (memref, mode, addr, validate, false);
    2356       254825 :   machine_mode mmode = GET_MODE (new_rtx);
    2357       254825 :   class mem_attrs *defattrs;
    2358              : 
    2359       254825 :   mem_attrs attrs (*get_mem_attrs (memref));
    2360       254825 :   defattrs = mode_mem_attrs[(int) mmode];
    2361       254825 :   attrs.expr = NULL_TREE;
    2362       254825 :   attrs.offset_known_p = false;
    2363       254825 :   attrs.size_known_p = defattrs->size_known_p;
    2364       254825 :   attrs.size = defattrs->size;
    2365       254825 :   attrs.align = defattrs->align;
    2366              : 
    2367              :   /* If there are no changes, just return the original memory reference.  */
    2368       254825 :   if (new_rtx == memref)
    2369              :     {
    2370        57235 :       if (mem_attrs_eq_p (get_mem_attrs (memref), &attrs))
    2371              :         return new_rtx;
    2372              : 
    2373        56137 :       new_rtx = gen_rtx_MEM (mmode, XEXP (memref, 0));
    2374        56137 :       MEM_COPY_ATTRIBUTES (new_rtx, memref);
    2375              :     }
    2376              : 
    2377       253727 :   set_mem_attrs (new_rtx, &attrs);
    2378       253727 :   return new_rtx;
    2379              : }
    2380              : 
    2381              : /* Like change_address_1 with VALIDATE nonzero, but we are not saying in what
    2382              :    way we are changing MEMREF, so we only preserve the alias set.  */
    2383              : 
    2384              : rtx
    2385       254825 : change_address (rtx memref, machine_mode mode, rtx addr)
    2386              : {
    2387       254825 :   return change_address (memref, mode, addr, true);
    2388              : }
    2389              : 
    2390              : /* Like change_address_1 with VALIDATE zero, but we are not saying in what
    2391              :    way we are changing MEMREF, so we only preserve the alias set.  */
    2392              : 
    2393              : rtx
    2394            0 : change_address_nv (rtx memref, machine_mode mode, rtx addr)
    2395              : {
    2396            0 :   return change_address (memref, mode, addr, false);
    2397              : }
    2398              : 
    2399              : /* Return a memory reference like MEMREF, but with its mode changed
    2400              :    to MODE and its address offset by OFFSET bytes.  If VALIDATE is
    2401              :    nonzero, the memory address is forced to be valid.
    2402              :    If ADJUST_ADDRESS is zero, OFFSET is only used to update MEM_ATTRS
    2403              :    and the caller is responsible for adjusting MEMREF base register.
    2404              :    If ADJUST_OBJECT is zero, the underlying object associated with the
    2405              :    memory reference is left unchanged and the caller is responsible for
    2406              :    dealing with it.  Otherwise, if the new memory reference is outside
    2407              :    the underlying object, even partially, then the object is dropped.
    2408              :    SIZE, if nonzero, is the size of an access in cases where MODE
    2409              :    has no inherent size.  */
    2410              : 
    2411              : rtx
    2412     36066682 : adjust_address_1 (rtx memref, machine_mode mode, poly_int64 offset,
    2413              :                   int validate, int adjust_address, int adjust_object,
    2414              :                   poly_int64 size)
    2415              : {
    2416     36066682 :   rtx addr = XEXP (memref, 0);
    2417     36066682 :   rtx new_rtx;
    2418     36066682 :   machine_mode address_mode;
    2419     36066682 :   class mem_attrs attrs (*get_mem_attrs (memref)), *defattrs;
    2420     36066682 :   unsigned HOST_WIDE_INT max_align;
    2421              : #ifdef POINTERS_EXTEND_UNSIGNED
    2422     36066682 :   scalar_int_mode pointer_mode
    2423     36066682 :     = targetm.addr_space.pointer_mode (attrs.addrspace);
    2424              : #endif
    2425              : 
    2426              :   /* VOIDmode means no mode change for change_address_1.  */
    2427     36066682 :   if (mode == VOIDmode)
    2428        17715 :     mode = GET_MODE (memref);
    2429              : 
    2430              :   /* Take the size of non-BLKmode accesses from the mode.  */
    2431     36066682 :   defattrs = mode_mem_attrs[(int) mode];
    2432     36066682 :   if (defattrs->size_known_p)
    2433     29855875 :     size = defattrs->size;
    2434              : 
    2435              :   /* If there are no changes, just return the original memory reference.  */
    2436     36066682 :   if (mode == GET_MODE (memref)
    2437     15602946 :       && known_eq (offset, 0)
    2438      9153743 :       && (known_eq (size, 0)
    2439      3818754 :           || (attrs.size_known_p && known_eq (attrs.size, size)))
    2440     50184103 :       && (!validate || memory_address_addr_space_p (mode, addr,
    2441      4964330 :                                                     attrs.addrspace)))
    2442              :     return memref;
    2443              : 
    2444              :   /* ??? Prefer to create garbage instead of creating shared rtl.
    2445              :      This may happen even if offset is nonzero -- consider
    2446              :      (plus (plus reg reg) const_int) -- so do this always.  */
    2447     26913597 :   addr = copy_rtx (addr);
    2448              : 
    2449              :   /* Convert a possibly large offset to a signed value within the
    2450              :      range of the target address space.  */
    2451     26913597 :   address_mode = get_address_mode (memref);
    2452     26913597 :   if (!VECTOR_MODE_P (address_mode))
    2453     26913597 :     offset = trunc_int_for_mode (offset, address_mode);
    2454              :   else
    2455            0 :     offset = trunc_int_for_mode (offset, GET_MODE_INNER (address_mode));
    2456              : 
    2457     26913597 :   if (adjust_address)
    2458              :     {
    2459              :       /* If MEMREF is a LO_SUM and the offset is within the alignment of the
    2460              :          object, we can merge it into the LO_SUM.  */
    2461     26882233 :       if (GET_MODE (memref) != BLKmode
    2462     12313451 :           && GET_CODE (addr) == LO_SUM
    2463     26882233 :           && known_in_range_p (offset,
    2464            0 :                                0, (GET_MODE_ALIGNMENT (GET_MODE (memref))
    2465            0 :                                    / BITS_PER_UNIT)))
    2466            0 :         addr = gen_rtx_LO_SUM (address_mode, XEXP (addr, 0),
    2467              :                                plus_constant (address_mode,
    2468              :                                               XEXP (addr, 1), offset));
    2469              : #ifdef POINTERS_EXTEND_UNSIGNED
    2470              :       /* If MEMREF is a ZERO_EXTEND from pointer_mode and the offset is valid
    2471              :          in that mode, we merge it into the ZERO_EXTEND.  We take advantage of
    2472              :          the fact that pointers are not allowed to overflow.  */
    2473     26882233 :       else if (POINTERS_EXTEND_UNSIGNED > 0
    2474     26882233 :                && GET_CODE (addr) == ZERO_EXTEND
    2475           12 :                && GET_MODE (XEXP (addr, 0)) == pointer_mode
    2476     26882235 :                && known_eq (trunc_int_for_mode (offset, pointer_mode), offset))
    2477            2 :         addr = gen_rtx_ZERO_EXTEND (address_mode,
    2478              :                                     plus_constant (pointer_mode,
    2479              :                                                    XEXP (addr, 0), offset));
    2480              : #endif
    2481              :       else
    2482     26882231 :         addr = plus_constant (address_mode, addr, offset);
    2483              :     }
    2484              : 
    2485     26913597 :   new_rtx = change_address_1 (memref, mode, addr, validate, false);
    2486              : 
    2487              :   /* If the address is a REG, change_address_1 rightfully returns memref,
    2488              :      but this would destroy memref's MEM_ATTRS.  */
    2489     26913597 :   if (new_rtx == memref && maybe_ne (offset, 0))
    2490        31319 :     new_rtx = copy_rtx (new_rtx);
    2491              : 
    2492              :   /* Conservatively drop the object if we don't know where we start from.  */
    2493     26913597 :   if (adjust_object && (!attrs.offset_known_p || !attrs.size_known_p))
    2494              :     {
    2495         1663 :       attrs.expr = NULL_TREE;
    2496         1663 :       attrs.alias = 0;
    2497              :     }
    2498              : 
    2499              :   /* Compute the new values of the memory attributes due to this adjustment.
    2500              :      We add the offsets and update the alignment.  */
    2501     26913597 :   if (attrs.offset_known_p)
    2502              :     {
    2503     23068170 :       attrs.offset += offset;
    2504              : 
    2505              :       /* Drop the object if the new left end is not within its bounds.  */
    2506     23068170 :       if (adjust_object && maybe_lt (attrs.offset, 0))
    2507              :         {
    2508        25715 :           attrs.expr = NULL_TREE;
    2509        25715 :           attrs.alias = 0;
    2510              :         }
    2511              :     }
    2512              : 
    2513              :   /* Compute the new alignment by taking the MIN of the alignment and the
    2514              :      lowest-order set bit in OFFSET, but don't change the alignment if OFFSET
    2515              :      if zero.  */
    2516     26913597 :   if (maybe_ne (offset, 0))
    2517              :     {
    2518     12319390 :       max_align = known_alignment (offset) * BITS_PER_UNIT;
    2519     12319390 :       attrs.align = MIN (attrs.align, max_align);
    2520              :     }
    2521              : 
    2522     26913597 :   if (maybe_ne (size, 0))
    2523              :     {
    2524              :       /* Drop the object if the new right end is not within its bounds.  */
    2525     26199283 :       if (adjust_object && maybe_gt (offset + size, attrs.size))
    2526              :         {
    2527        64969 :           attrs.expr = NULL_TREE;
    2528        64969 :           attrs.alias = 0;
    2529              :         }
    2530     26199283 :       attrs.size_known_p = true;
    2531     26199283 :       attrs.size = size;
    2532              :     }
    2533       714314 :   else if (attrs.size_known_p)
    2534              :     {
    2535       707866 :       gcc_assert (!adjust_object);
    2536     26913597 :       attrs.size -= offset;
    2537              :       /* ??? The store_by_pieces machinery generates negative sizes,
    2538              :          so don't assert for that here.  */
    2539              :     }
    2540              : 
    2541     26913597 :   set_mem_attrs (new_rtx, &attrs);
    2542              : 
    2543     26913597 :   return new_rtx;
    2544              : }
    2545              : 
    2546              : /* Return a memory reference like MEMREF, but with its mode changed
    2547              :    to MODE and its address changed to ADDR, which is assumed to be
    2548              :    MEMREF offset by OFFSET bytes.  If VALIDATE is
    2549              :    nonzero, the memory address is forced to be valid.  */
    2550              : 
    2551              : rtx
    2552        61370 : adjust_automodify_address_1 (rtx memref, machine_mode mode, rtx addr,
    2553              :                              poly_int64 offset, int validate)
    2554              : {
    2555        61370 :   memref = change_address_1 (memref, VOIDmode, addr, validate, false);
    2556        61370 :   return adjust_address_1 (memref, mode, offset, validate, 0, 0, 0);
    2557              : }
    2558              : 
    2559              : /* Return a memory reference like MEMREF, but whose address is changed by
    2560              :    adding OFFSET, an RTX, to it.  POW2 is the highest power of two factor
    2561              :    known to be in OFFSET (possibly 1).  */
    2562              : 
    2563              : rtx
    2564       774607 : offset_address (rtx memref, rtx offset, unsigned HOST_WIDE_INT pow2)
    2565              : {
    2566       774607 :   rtx new_rtx, addr = XEXP (memref, 0);
    2567       774607 :   machine_mode address_mode;
    2568       774607 :   class mem_attrs *defattrs;
    2569              : 
    2570       774607 :   mem_attrs attrs (*get_mem_attrs (memref));
    2571       774607 :   address_mode = get_address_mode (memref);
    2572       774607 :   new_rtx = simplify_gen_binary (PLUS, address_mode, addr, offset);
    2573              : 
    2574              :   /* At this point we don't know _why_ the address is invalid.  It
    2575              :      could have secondary memory references, multiplies or anything.
    2576              : 
    2577              :      However, if we did go and rearrange things, we can wind up not
    2578              :      being able to recognize the magic around pic_offset_table_rtx.
    2579              :      This stuff is fragile, and is yet another example of why it is
    2580              :      bad to expose PIC machinery too early.  */
    2581      1547477 :   if (! memory_address_addr_space_p (GET_MODE (memref), new_rtx,
    2582       774607 :                                      attrs.addrspace)
    2583       136799 :       && GET_CODE (addr) == PLUS
    2584       825500 :       && XEXP (addr, 0) == pic_offset_table_rtx)
    2585              :     {
    2586         1737 :       addr = force_reg (GET_MODE (addr), addr);
    2587         1737 :       new_rtx = simplify_gen_binary (PLUS, address_mode, addr, offset);
    2588              :     }
    2589              : 
    2590       774607 :   update_temp_slot_address (XEXP (memref, 0), new_rtx);
    2591       774607 :   new_rtx = change_address_1 (memref, VOIDmode, new_rtx, 1, false);
    2592              : 
    2593              :   /* If there are no changes, just return the original memory reference.  */
    2594       774607 :   if (new_rtx == memref)
    2595              :     return new_rtx;
    2596              : 
    2597              :   /* Update the alignment to reflect the offset.  Reset the offset, which
    2598              :      we don't know.  */
    2599       774607 :   defattrs = mode_mem_attrs[(int) GET_MODE (new_rtx)];
    2600       774607 :   attrs.offset_known_p = false;
    2601       774607 :   attrs.size_known_p = defattrs->size_known_p;
    2602       774607 :   attrs.size = defattrs->size;
    2603       774607 :   attrs.align = MIN (attrs.align, pow2 * BITS_PER_UNIT);
    2604       774607 :   set_mem_attrs (new_rtx, &attrs);
    2605       774607 :   return new_rtx;
    2606              : }
    2607              : 
    2608              : /* Return a memory reference like MEMREF, but with its address changed to
    2609              :    ADDR.  The caller is asserting that the actual piece of memory pointed
    2610              :    to is the same, just the form of the address is being changed, such as
    2611              :    by putting something into a register.  INPLACE is true if any changes
    2612              :    can be made directly to MEMREF or false if MEMREF must be treated as
    2613              :    immutable.  */
    2614              : 
    2615              : rtx
    2616     13061676 : replace_equiv_address (rtx memref, rtx addr, bool inplace)
    2617              : {
    2618              :   /* change_address_1 copies the memory attribute structure without change
    2619              :      and that's exactly what we want here.  */
    2620     13061676 :   update_temp_slot_address (XEXP (memref, 0), addr);
    2621     13061676 :   return change_address_1 (memref, VOIDmode, addr, 1, inplace);
    2622              : }
    2623              : 
    2624              : /* Likewise, but the reference is not required to be valid.  */
    2625              : 
    2626              : rtx
    2627    466376866 : replace_equiv_address_nv (rtx memref, rtx addr, bool inplace)
    2628              : {
    2629    466376866 :   return change_address_1 (memref, VOIDmode, addr, 0, inplace);
    2630              : }
    2631              : 
    2632              : 
    2633              : /* Emit insns to reload VALUE into a new register.  VALUE is an
    2634              :    auto-increment or auto-decrement RTX whose operand is a register or
    2635              :    memory location; so reloading involves incrementing that location.
    2636              : 
    2637              :    INC_AMOUNT is the number to increment or decrement by (always
    2638              :    positive and ignored for POST_MODIFY/PRE_MODIFY).
    2639              : 
    2640              :    Return a pseudo containing the result.  */
    2641              : rtx
    2642            0 : address_reload_context::emit_autoinc (rtx value, poly_int64 inc_amount)
    2643              : {
    2644              :   /* Since we're going to call recog, and might be called within recog,
    2645              :      we need to ensure we save and restore recog_data.  */
    2646            0 :   recog_state_saver recog_save;
    2647              : 
    2648              :   /* REG or MEM to be copied and incremented.  */
    2649            0 :   rtx incloc = XEXP (value, 0);
    2650              : 
    2651            0 :   const rtx_code code = GET_CODE (value);
    2652            0 :   const bool post_p
    2653            0 :     = code == POST_DEC || code == POST_INC || code == POST_MODIFY;
    2654              : 
    2655            0 :   bool plus_p = true;
    2656            0 :   rtx inc;
    2657            0 :   if (code == PRE_MODIFY || code == POST_MODIFY)
    2658              :     {
    2659            0 :       gcc_assert (GET_CODE (XEXP (value, 1)) == PLUS
    2660              :                   || GET_CODE (XEXP (value, 1)) == MINUS);
    2661            0 :       gcc_assert (rtx_equal_p (XEXP (XEXP (value, 1), 0), XEXP (value, 0)));
    2662            0 :       plus_p = GET_CODE (XEXP (value, 1)) == PLUS;
    2663            0 :       inc = XEXP (XEXP (value, 1), 1);
    2664              :     }
    2665              :   else
    2666              :     {
    2667            0 :       if (code == PRE_DEC || code == POST_DEC)
    2668            0 :         inc_amount = -inc_amount;
    2669              : 
    2670            0 :       inc = gen_int_mode (inc_amount, GET_MODE (value));
    2671              :     }
    2672              : 
    2673            0 :   rtx result;
    2674            0 :   if (!post_p && REG_P (incloc))
    2675              :     result = incloc;
    2676              :   else
    2677              :     {
    2678            0 :       result = get_reload_reg ();
    2679              :       /* First copy the location to the result register.  */
    2680            0 :       emit_insn (gen_move_insn (result, incloc));
    2681              :     }
    2682              : 
    2683              :   /* See if we can directly increment INCLOC.  */
    2684            0 :   rtx_insn *last = get_last_insn ();
    2685            0 :   rtx_insn *add_insn = emit_insn (plus_p
    2686            0 :                                   ? gen_add2_insn (incloc, inc)
    2687            0 :                                   : gen_sub2_insn (incloc, inc));
    2688            0 :   const int icode = recog_memoized (add_insn);
    2689            0 :   if (icode >= 0)
    2690              :     {
    2691            0 :       if (!post_p && result != incloc)
    2692            0 :         emit_insn (gen_move_insn (result, incloc));
    2693              :       return result;
    2694              :     }
    2695            0 :   delete_insns_since (last);
    2696              : 
    2697              :   /* If couldn't do the increment directly, must increment in RESULT.
    2698              :      The way we do this depends on whether this is pre- or
    2699              :      post-increment.  For pre-increment, copy INCLOC to the reload
    2700              :      register, increment it there, then save back.  */
    2701            0 :   if (!post_p)
    2702              :     {
    2703            0 :       if (incloc != result)
    2704            0 :         emit_insn (gen_move_insn (result, incloc));
    2705            0 :       if (plus_p)
    2706            0 :         emit_insn (gen_add2_insn (result, inc));
    2707              :       else
    2708            0 :         emit_insn (gen_sub2_insn (result, inc));
    2709            0 :       if (incloc != result)
    2710            0 :         emit_insn (gen_move_insn (incloc, result));
    2711              :     }
    2712              :   else
    2713              :     {
    2714              :       /* Post-increment.
    2715              : 
    2716              :          Because this might be a jump insn or a compare, and because
    2717              :          RESULT may not be available after the insn in an input
    2718              :          reload, we must do the incrementing before the insn being
    2719              :          reloaded for.
    2720              : 
    2721              :          We have already copied INCLOC to RESULT.  Increment the copy in
    2722              :          RESULT, save that back, then decrement RESULT so it has
    2723              :          the original value.  */
    2724            0 :       if (plus_p)
    2725            0 :         emit_insn (gen_add2_insn (result, inc));
    2726              :       else
    2727            0 :         emit_insn (gen_sub2_insn (result, inc));
    2728            0 :       emit_insn (gen_move_insn (incloc, result));
    2729              :       /* Restore non-modified value for the result.  We prefer this
    2730              :          way because it does not require an additional hard
    2731              :          register.  */
    2732            0 :       if (plus_p)
    2733              :         {
    2734            0 :           poly_int64 offset;
    2735            0 :           if (poly_int_rtx_p (inc, &offset))
    2736            0 :             emit_insn (gen_add2_insn (result,
    2737              :                                       gen_int_mode (-offset,
    2738            0 :                                                     GET_MODE (result))));
    2739              :           else
    2740            0 :             emit_insn (gen_sub2_insn (result, inc));
    2741              :         }
    2742              :       else
    2743            0 :         emit_insn (gen_add2_insn (result, inc));
    2744              :     }
    2745              :   return result;
    2746            0 : }
    2747              : 
    2748              : /* Return a memory reference like MEM, but with the address reloaded into a
    2749              :    pseudo register.  */
    2750              : 
    2751              : rtx
    2752            0 : force_reload_address (rtx mem)
    2753              : {
    2754            0 :   rtx addr = XEXP (mem, 0);
    2755            0 :   if (GET_RTX_CLASS (GET_CODE (addr)) == RTX_AUTOINC)
    2756              :     {
    2757            0 :       const auto size = GET_MODE_SIZE (GET_MODE (mem));
    2758            0 :       addr = address_reload_context ().emit_autoinc (addr, size);
    2759              :     }
    2760              :   else
    2761            0 :     addr = force_reg (Pmode, addr);
    2762              : 
    2763            0 :   return replace_equiv_address (mem, addr);
    2764              : }
    2765              : 
    2766              : /* Return a memory reference like MEMREF, but with its mode widened to
    2767              :    MODE and offset by OFFSET.  This would be used by targets that e.g.
    2768              :    cannot issue QImode memory operations and have to use SImode memory
    2769              :    operations plus masking logic.  */
    2770              : 
    2771              : rtx
    2772           47 : widen_memory_access (rtx memref, machine_mode mode, poly_int64 offset)
    2773              : {
    2774           47 :   rtx new_rtx = adjust_address_1 (memref, mode, offset, 1, 1, 0, 0);
    2775           94 :   poly_uint64 size = GET_MODE_SIZE (mode);
    2776              : 
    2777              :   /* If there are no changes, just return the original memory reference.  */
    2778           47 :   if (new_rtx == memref)
    2779              :     return new_rtx;
    2780              : 
    2781           47 :   mem_attrs attrs (*get_mem_attrs (new_rtx));
    2782              : 
    2783              :   /* If we don't know what offset we were at within the expression, then
    2784              :      we can't know if we've overstepped the bounds.  */
    2785           47 :   if (! attrs.offset_known_p)
    2786            5 :     attrs.expr = NULL_TREE;
    2787              : 
    2788           47 :   while (attrs.expr)
    2789              :     {
    2790           42 :       if (TREE_CODE (attrs.expr) == COMPONENT_REF)
    2791              :         {
    2792            0 :           tree field = TREE_OPERAND (attrs.expr, 1);
    2793            0 :           tree offset = component_ref_field_offset (attrs.expr);
    2794              : 
    2795            0 :           if (! DECL_SIZE_UNIT (field))
    2796              :             {
    2797            0 :               attrs.expr = NULL_TREE;
    2798            0 :               break;
    2799              :             }
    2800              : 
    2801              :           /* Is the field at least as large as the access?  If so, ok,
    2802              :              otherwise strip back to the containing structure.  */
    2803            0 :           if (poly_int_tree_p (DECL_SIZE_UNIT (field))
    2804            0 :               && known_ge (wi::to_poly_offset (DECL_SIZE_UNIT (field)), size)
    2805            0 :               && known_ge (attrs.offset, 0))
    2806              :             break;
    2807              : 
    2808            0 :           poly_uint64 suboffset;
    2809            0 :           if (!poly_int_tree_p (offset, &suboffset))
    2810              :             {
    2811            0 :               attrs.expr = NULL_TREE;
    2812            0 :               break;
    2813              :             }
    2814              : 
    2815            0 :           attrs.expr = TREE_OPERAND (attrs.expr, 0);
    2816            0 :           attrs.offset += suboffset;
    2817            0 :           attrs.offset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (field))
    2818            0 :                            / BITS_PER_UNIT);
    2819              :         }
    2820              :       /* Similarly for the decl.  */
    2821           42 :       else if (DECL_P (attrs.expr)
    2822           12 :                && DECL_SIZE_UNIT (attrs.expr)
    2823            7 :                && poly_int_tree_p (DECL_SIZE_UNIT (attrs.expr))
    2824            7 :                && known_ge (wi::to_poly_offset (DECL_SIZE_UNIT (attrs.expr)),
    2825              :                            size)
    2826           42 :                && known_ge (attrs.offset, 0))
    2827              :         break;
    2828              :       else
    2829              :         {
    2830              :           /* The widened memory access overflows the expression, which means
    2831              :              that it could alias another expression.  Zap it.  */
    2832           42 :           attrs.expr = NULL_TREE;
    2833           42 :           break;
    2834              :         }
    2835              :     }
    2836              : 
    2837           47 :   if (! attrs.expr)
    2838           47 :     attrs.offset_known_p = false;
    2839              : 
    2840              :   /* The widened memory may alias other stuff, so zap the alias set.  */
    2841              :   /* ??? Maybe use get_alias_set on any remaining expression.  */
    2842           47 :   attrs.alias = 0;
    2843           47 :   attrs.size_known_p = true;
    2844           47 :   attrs.size = size;
    2845           47 :   set_mem_attrs (new_rtx, &attrs);
    2846           47 :   return new_rtx;
    2847              : }
    2848              : 
    2849              : /* A fake decl that is used as the MEM_EXPR of spill slots.  */
    2850              : static GTY(()) tree spill_slot_decl;
    2851              : 
    2852              : tree
    2853    842101699 : get_spill_slot_decl (bool force_build_p)
    2854              : {
    2855    842101699 :   tree d = spill_slot_decl;
    2856    842101699 :   rtx rd;
    2857              : 
    2858    842101699 :   if (d || !force_build_p)
    2859              :     return d;
    2860              : 
    2861        31003 :   d = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
    2862              :                   VAR_DECL, get_identifier ("%sfp"), void_type_node);
    2863        31003 :   DECL_ARTIFICIAL (d) = 1;
    2864        31003 :   DECL_IGNORED_P (d) = 1;
    2865        31003 :   TREE_USED (d) = 1;
    2866        31003 :   spill_slot_decl = d;
    2867              : 
    2868        31003 :   rd = gen_rtx_MEM (BLKmode, frame_pointer_rtx);
    2869        31003 :   MEM_NOTRAP_P (rd) = 1;
    2870        31003 :   mem_attrs attrs (*mode_mem_attrs[(int) BLKmode]);
    2871        31003 :   attrs.alias = new_alias_set ();
    2872        31003 :   attrs.expr = d;
    2873        31003 :   set_mem_attrs (rd, &attrs);
    2874        31003 :   SET_DECL_RTL (d, rd);
    2875              : 
    2876        31003 :   return d;
    2877              : }
    2878              : 
    2879              : /* Given MEM, a result from assign_stack_local, fill in the memory
    2880              :    attributes as appropriate for a register allocator spill slot.
    2881              :    These slots are not aliasable by other memory.  We arrange for
    2882              :    them all to use a single MEM_EXPR, so that the aliasing code can
    2883              :    work properly in the case of shared spill slots.  */
    2884              : 
    2885              : void
    2886      1452724 : set_mem_attrs_for_spill (rtx mem)
    2887              : {
    2888      1452724 :   rtx addr;
    2889              : 
    2890      1452724 :   mem_attrs attrs (*get_mem_attrs (mem));
    2891      1452724 :   attrs.expr = get_spill_slot_decl (true);
    2892      1452724 :   attrs.alias = MEM_ALIAS_SET (DECL_RTL (attrs.expr));
    2893      1452724 :   attrs.addrspace = ADDR_SPACE_GENERIC;
    2894              : 
    2895              :   /* We expect the incoming memory to be of the form:
    2896              :         (mem:MODE (plus (reg sfp) (const_int offset)))
    2897              :      with perhaps the plus missing for offset = 0.  */
    2898      1452724 :   addr = XEXP (mem, 0);
    2899      1452724 :   attrs.offset_known_p = true;
    2900      1452724 :   strip_offset (addr, &attrs.offset);
    2901              : 
    2902      1452724 :   set_mem_attrs (mem, &attrs);
    2903      1452724 :   MEM_NOTRAP_P (mem) = 1;
    2904      1452724 : }
    2905              : 
    2906              : /* Return a newly created CODE_LABEL rtx with a unique label number.  */
    2907              : 
    2908              : rtx_code_label *
    2909     15646181 : gen_label_rtx (void)
    2910              : {
    2911     15646181 :   return as_a <rtx_code_label *> (
    2912              :             gen_rtx_CODE_LABEL (VOIDmode, NULL_RTX, NULL_RTX,
    2913     15646181 :                                 NULL, label_num++, NULL));
    2914              : }
    2915              : 
    2916              : /* For procedure integration.  */
    2917              : 
    2918              : /* Install new pointers to the first and last insns in the chain.
    2919              :    Also, set cur_insn_uid to one higher than the last in use.
    2920              :    Used for an inline-procedure after copying the insn chain.  */
    2921              : 
    2922              : void
    2923            8 : set_new_first_and_last_insn (rtx_insn *first, rtx_insn *last)
    2924              : {
    2925            8 :   rtx_insn *insn;
    2926              : 
    2927            8 :   set_first_insn (first);
    2928            8 :   set_last_insn (last);
    2929            8 :   cur_insn_uid = 0;
    2930              : 
    2931            8 :   if (param_min_nondebug_insn_uid || MAY_HAVE_DEBUG_INSNS)
    2932              :     {
    2933            0 :       int debug_count = 0;
    2934              : 
    2935            0 :       cur_insn_uid = param_min_nondebug_insn_uid - 1;
    2936            0 :       cur_debug_insn_uid = 0;
    2937              : 
    2938            0 :       for (insn = first; insn; insn = NEXT_INSN (insn))
    2939            0 :         if (INSN_UID (insn) < param_min_nondebug_insn_uid)
    2940            0 :           cur_debug_insn_uid = MAX (cur_debug_insn_uid, INSN_UID (insn));
    2941              :         else
    2942              :           {
    2943            0 :             cur_insn_uid = MAX (cur_insn_uid, INSN_UID (insn));
    2944            0 :             if (DEBUG_INSN_P (insn))
    2945            0 :               debug_count++;
    2946              :           }
    2947              : 
    2948            0 :       if (debug_count)
    2949            0 :         cur_debug_insn_uid = param_min_nondebug_insn_uid + debug_count;
    2950              :       else
    2951            0 :         cur_debug_insn_uid++;
    2952              :     }
    2953              :   else
    2954            8 :     for (insn = first; insn; insn = NEXT_INSN (insn))
    2955            0 :       cur_insn_uid = MAX (cur_insn_uid, INSN_UID (insn));
    2956              : 
    2957            8 :   cur_insn_uid++;
    2958            8 : }
    2959              : 
    2960              : /* Go through all the RTL insn bodies and copy any invalid shared
    2961              :    structure.  This routine should only be called once.  */
    2962              : 
    2963              : static void
    2964      3035033 : unshare_all_rtl_1 (rtx_insn *insn)
    2965              : {
    2966              :   /* Unshare just about everything else.  */
    2967      3035033 :   unshare_all_rtl_in_chain (insn);
    2968              : 
    2969              :   /* Make sure the addresses of stack slots found outside the insn chain
    2970              :      (such as, in DECL_RTL of a variable) are not shared
    2971              :      with the insn chain.
    2972              : 
    2973              :      This special care is necessary when the stack slot MEM does not
    2974              :      actually appear in the insn chain.  If it does appear, its address
    2975              :      is unshared from all else at that point.  */
    2976      3035033 :   unsigned int i;
    2977      3035033 :   rtx temp;
    2978      9358579 :   FOR_EACH_VEC_SAFE_ELT (stack_slot_list, i, temp)
    2979      3288513 :     (*stack_slot_list)[i] = copy_rtx_if_shared (temp);
    2980      3035033 : }
    2981              : 
    2982              : /* Go through all the RTL insn bodies and copy any invalid shared
    2983              :    structure, again.  This is a fairly expensive thing to do so it
    2984              :    should be done sparingly.  */
    2985              : 
    2986              : void
    2987      1517127 : unshare_all_rtl_again (rtx_insn *insn)
    2988              : {
    2989      1517127 :   rtx_insn *p;
    2990      1517127 :   tree decl;
    2991              : 
    2992    174512168 :   for (p = insn; p; p = NEXT_INSN (p))
    2993    172995041 :     if (INSN_P (p))
    2994              :       {
    2995    137122512 :         reset_used_flags (PATTERN (p));
    2996    137122512 :         reset_used_flags (REG_NOTES (p));
    2997    179168339 :         if (auto call = dyn_cast<rtx_call_insn *> (p))
    2998      6173298 :           reset_used_flags (CALL_INSN_FUNCTION_USAGE (call));
    2999              :       }
    3000              : 
    3001              :   /* Make sure that virtual stack slots are not shared.  */
    3002      1517127 :   set_used_decls (DECL_INITIAL (cfun->decl));
    3003              : 
    3004              :   /* Make sure that virtual parameters are not shared.  */
    3005      4683770 :   for (decl = DECL_ARGUMENTS (cfun->decl); decl; decl = DECL_CHAIN (decl))
    3006      3166643 :     set_used_flags (DECL_RTL (decl));
    3007              : 
    3008              :   rtx temp;
    3009              :   unsigned int i;
    3010      3956998 :   FOR_EACH_VEC_SAFE_ELT (stack_slot_list, i, temp)
    3011      2439871 :     reset_used_flags (temp);
    3012              : 
    3013      1517127 :   unshare_all_rtl_1 (insn);
    3014      1517127 : }
    3015              : 
    3016              : void
    3017      1517906 : unshare_all_rtl (void)
    3018              : {
    3019      1517906 :   unshare_all_rtl_1 (get_insns ());
    3020              : 
    3021      4685106 :   for (tree decl = DECL_ARGUMENTS (cfun->decl); decl; decl = DECL_CHAIN (decl))
    3022              :     {
    3023      3167200 :       if (DECL_RTL_SET_P (decl))
    3024      3167200 :         SET_DECL_RTL (decl, copy_rtx_if_shared (DECL_RTL (decl)));
    3025      3167200 :       DECL_INCOMING_RTL (decl) = copy_rtx_if_shared (DECL_INCOMING_RTL (decl));
    3026              :     }
    3027      1517906 : }
    3028              : 
    3029              : 
    3030              : /* Check that ORIG is not marked when it should not be and mark ORIG as in use,
    3031              :    Recursively does the same for subexpressions.  */
    3032              : 
    3033              : static void
    3034  57609826888 : verify_rtx_sharing (rtx orig, rtx_insn *insn)
    3035              : {
    3036  57609826888 :   rtx x = orig;
    3037  57609826888 :   int i;
    3038  57609826888 :   enum rtx_code code;
    3039  57609826888 :   const char *format_ptr;
    3040              : 
    3041  57609826888 :   if (x == 0)
    3042              :     return;
    3043              : 
    3044  47725773256 :   code = GET_CODE (x);
    3045              : 
    3046              :   /* These types may be freely shared.  */
    3047              : 
    3048  47725773256 :   switch (code)
    3049              :     {
    3050              :     case REG:
    3051              :     case DEBUG_EXPR:
    3052              :     case VALUE:
    3053              :     CASE_CONST_ANY:
    3054              :     case SYMBOL_REF:
    3055              :     case LABEL_REF:
    3056              :     case CODE_LABEL:
    3057              :     case PC:
    3058              :     case RETURN:
    3059              :     case SIMPLE_RETURN:
    3060              :     case SCRATCH:
    3061              :       /* SCRATCH must be shared because they represent distinct values.  */
    3062              :       return;
    3063   2046238132 :     case CLOBBER:
    3064              :       /* Share clobbers of hard registers, but do not share pseudo reg
    3065              :          clobbers or clobbers of hard registers that originated as pseudos.
    3066              :          This is needed to allow safe register renaming.  */
    3067   2046238132 :       if (REG_P (XEXP (x, 0))
    3068    740741167 :           && HARD_REGISTER_NUM_P (REGNO (XEXP (x, 0)))
    3069   2783839361 :           && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (x, 0))))
    3070              :         return;
    3071              :       break;
    3072              : 
    3073     36705766 :     case CONST:
    3074     36705766 :       if (shared_const_p (orig))
    3075              :         return;
    3076              :       break;
    3077              : 
    3078   2816687674 :     case MEM:
    3079              :       /* A MEM is allowed to be shared if its address is constant.  */
    3080   2816687674 :       if (CONSTANT_ADDRESS_P (XEXP (x, 0))
    3081   2816687674 :           || reload_completed || reload_in_progress)
    3082              :         return;
    3083              : 
    3084              :       break;
    3085              : 
    3086              :     default:
    3087              :       break;
    3088              :     }
    3089              : 
    3090              :   /* This rtx may not be shared.  If it has already been seen,
    3091              :      replace it with a copy of itself.  */
    3092  23164982267 :   if (flag_checking && RTX_FLAG (x, used))
    3093              :     {
    3094            0 :       error ("invalid rtl sharing found in the insn");
    3095            0 :       debug_rtx (insn);
    3096            0 :       error ("shared rtx");
    3097            0 :       debug_rtx (x);
    3098            0 :       internal_error ("internal consistency failure");
    3099              :     }
    3100  23164982267 :   gcc_assert (!RTX_FLAG (x, used));
    3101              : 
    3102  23164982267 :   RTX_FLAG (x, used) = 1;
    3103              : 
    3104              :   /* Now scan the subexpressions recursively.  */
    3105              : 
    3106  23164982267 :   format_ptr = GET_RTX_FORMAT (code);
    3107              : 
    3108  65004334632 :   for (i = 0; i < GET_RTX_LENGTH (code); i++)
    3109              :     {
    3110  41839352365 :       switch (*format_ptr++)
    3111              :         {
    3112  36495096425 :         case 'e':
    3113  36495096425 :           verify_rtx_sharing (XEXP (x, i), insn);
    3114  36495096425 :           break;
    3115              : 
    3116    921744775 :         case 'E':
    3117    921744775 :           if (XVEC (x, i) != NULL)
    3118              :             {
    3119    921744775 :               int j;
    3120    921744775 :               int len = XVECLEN (x, i);
    3121              : 
    3122   2791030904 :               for (j = 0; j < len; j++)
    3123              :                 {
    3124              :                   /* We allow sharing of ASM_OPERANDS inside single
    3125              :                      instruction.  */
    3126   1869286129 :                   if (j && GET_CODE (XVECEXP (x, i, j)) == SET
    3127     57047708 :                       && (GET_CODE (SET_SRC (XVECEXP (x, i, j)))
    3128              :                           == ASM_OPERANDS))
    3129      3260268 :                     verify_rtx_sharing (SET_DEST (XVECEXP (x, i, j)), insn);
    3130              :                   else
    3131   1866025861 :                     verify_rtx_sharing (XVECEXP (x, i, j), insn);
    3132              :                 }
    3133              :             }
    3134              :           break;
    3135              :         }
    3136              :     }
    3137              : }
    3138              : 
    3139              : /* Reset used-flags for INSN.  */
    3140              : 
    3141              : static void
    3142  18823174386 : reset_insn_used_flags (rtx_insn *insn)
    3143              : {
    3144  18823174386 :   gcc_assert (INSN_P (insn));
    3145  18823174386 :   reset_used_flags (PATTERN (insn));
    3146  18823174386 :   reset_used_flags (REG_NOTES (insn));
    3147  18823174386 :   if (auto call = dyn_cast<rtx_call_insn *> (insn))
    3148    844539896 :     reset_used_flags (CALL_INSN_FUNCTION_USAGE (call));
    3149  18823174386 : }
    3150              : 
    3151              : /* Go through all the RTL insn bodies and clear all the USED bits.  */
    3152              : 
    3153              : static void
    3154    192614712 : reset_all_used_flags (void)
    3155              : {
    3156    192614712 :   rtx_insn *p;
    3157              : 
    3158  25438973004 :   for (p = get_insns (); p; p = NEXT_INSN (p))
    3159  25246358292 :     if (INSN_P (p))
    3160              :       {
    3161  18823174386 :         rtx pat = PATTERN (p);
    3162  18823174386 :         if (rtx_sequence *seq = dyn_cast<rtx_sequence *> (pat))
    3163              :           {
    3164            0 :             gcc_assert (REG_NOTES (p) == NULL);
    3165            0 :             for (int i = 0; i < seq->len (); i++)
    3166              :               {
    3167            0 :                 rtx_insn *insn = seq->insn (i);
    3168            0 :                 if (INSN_P (insn))
    3169            0 :                   reset_insn_used_flags (insn);
    3170              :               }
    3171              :           }
    3172              :         else
    3173  18823174386 :           reset_insn_used_flags (p);
    3174              :       }
    3175    192614712 : }
    3176              : 
    3177              : /* Verify sharing in INSN.  */
    3178              : 
    3179              : static void
    3180   9411587193 : verify_insn_sharing (rtx_insn *insn)
    3181              : {
    3182   9411587193 :   gcc_assert (INSN_P (insn));
    3183   9411587193 :   verify_rtx_sharing (PATTERN (insn), insn);
    3184   9411587193 :   verify_rtx_sharing (REG_NOTES (insn), insn);
    3185   9411587193 :   if (auto call = dyn_cast<rtx_call_insn *> (insn))
    3186    422269948 :     verify_rtx_sharing (CALL_INSN_FUNCTION_USAGE (call), call);
    3187   9411587193 : }
    3188              : 
    3189              : /* Go through all the RTL insn bodies and check that there is no unexpected
    3190              :    sharing in between the subexpressions.  */
    3191              : 
    3192              : DEBUG_FUNCTION void
    3193     96307356 : verify_rtl_sharing (void)
    3194              : {
    3195     96307356 :   rtx_insn *p;
    3196              : 
    3197     96307356 :   timevar_push (TV_VERIFY_RTL_SHARING);
    3198              : 
    3199     96307356 :   reset_all_used_flags ();
    3200              : 
    3201  12719486502 :   for (p = get_insns (); p; p = NEXT_INSN (p))
    3202  12623179146 :     if (INSN_P (p))
    3203              :       {
    3204   9411587193 :         rtx pat = PATTERN (p);
    3205   9411587193 :         if (rtx_sequence *seq = dyn_cast<rtx_sequence *> (pat))
    3206            0 :           for (int i = 0; i < seq->len (); i++)
    3207              :               {
    3208            0 :                 rtx_insn *insn = seq->insn (i);
    3209            0 :                 if (INSN_P (insn))
    3210            0 :                   verify_insn_sharing (insn);
    3211              :               }
    3212              :         else
    3213   9411587193 :           verify_insn_sharing (p);
    3214              :       }
    3215              : 
    3216     96307356 :   reset_all_used_flags ();
    3217              : 
    3218     96307356 :   timevar_pop (TV_VERIFY_RTL_SHARING);
    3219     96307356 : }
    3220              : 
    3221              : /* Go through all the RTL insn bodies and copy any invalid shared structure.
    3222              :    Assumes the mark bits are cleared at entry.  */
    3223              : 
    3224              : void
    3225      9604035 : unshare_all_rtl_in_chain (rtx_insn *insn)
    3226              : {
    3227    372257464 :   for (; insn; insn = NEXT_INSN (insn))
    3228    362653429 :     if (INSN_P (insn))
    3229              :       {
    3230    296940761 :         PATTERN (insn) = copy_rtx_if_shared (PATTERN (insn));
    3231    296940761 :         REG_NOTES (insn) = copy_rtx_if_shared (REG_NOTES (insn));
    3232    374994024 :         if (auto call = dyn_cast<rtx_call_insn *> (insn))
    3233     12340595 :           CALL_INSN_FUNCTION_USAGE (call)
    3234     12340595 :             = copy_rtx_if_shared (CALL_INSN_FUNCTION_USAGE (call));
    3235              :       }
    3236      9604035 : }
    3237              : 
    3238              : /* Go through all virtual stack slots of a function and mark them as
    3239              :    shared.  We never replace the DECL_RTLs themselves with a copy,
    3240              :    but expressions mentioned into a DECL_RTL cannot be shared with
    3241              :    expressions in the instruction stream.
    3242              : 
    3243              :    Note that reload may convert pseudo registers into memories in-place.
    3244              :    Pseudo registers are always shared, but MEMs never are.  Thus if we
    3245              :    reset the used flags on MEMs in the instruction stream, we must set
    3246              :    them again on MEMs that appear in DECL_RTLs.  */
    3247              : 
    3248              : static void
    3249     17236788 : set_used_decls (tree blk)
    3250              : {
    3251     17236788 :   tree t;
    3252              : 
    3253              :   /* Mark decls.  */
    3254     37554203 :   for (t = BLOCK_VARS (blk); t; t = DECL_CHAIN (t))
    3255     20317415 :     if (DECL_RTL_SET_P (t))
    3256      2343318 :       set_used_flags (DECL_RTL (t));
    3257              : 
    3258              :   /* Now process sub-blocks.  */
    3259     32956449 :   for (t = BLOCK_SUBBLOCKS (blk); t; t = BLOCK_CHAIN (t))
    3260     15719661 :     set_used_decls (t);
    3261     17236788 : }
    3262              : 
    3263              : /* Mark ORIG as in use, and return a copy of it if it was already in use.
    3264              :    Recursively does the same for subexpressions.  Uses
    3265              :    copy_rtx_if_shared_1 to reduce stack space.  */
    3266              : 
    3267              : rtx
    3268    643979948 : copy_rtx_if_shared (rtx orig)
    3269              : {
    3270    643979948 :   copy_rtx_if_shared_1 (&orig);
    3271    643979948 :   return orig;
    3272              : }
    3273              : 
    3274              : /* Mark *ORIG1 as in use, and set it to a copy of it if it was already in
    3275              :    use.  Recursively does the same for subexpressions.  */
    3276              : 
    3277              : static void
    3278   1122833395 : copy_rtx_if_shared_1 (rtx *orig1)
    3279              : {
    3280   1122833395 :   rtx x;
    3281   1122833395 :   int i;
    3282   1122833395 :   enum rtx_code code;
    3283   1122833395 :   rtx *last_ptr;
    3284   1122833395 :   const char *format_ptr;
    3285   1122833395 :   int copied = 0;
    3286   1854823846 :   int length;
    3287              : 
    3288              :   /* Repeat is used to turn tail-recursion into iteration.  */
    3289   1854823846 : repeat:
    3290   1854823846 :   x = *orig1;
    3291              : 
    3292   1854823846 :   if (x == 0)
    3293              :     return;
    3294              : 
    3295   1524629854 :   code = GET_CODE (x);
    3296              : 
    3297              :   /* These types may be freely shared.  */
    3298              : 
    3299   1524629854 :   switch (code)
    3300              :     {
    3301              :     case REG:
    3302              :     case DEBUG_EXPR:
    3303              :     case VALUE:
    3304              :     CASE_CONST_ANY:
    3305              :     case SYMBOL_REF:
    3306              :     case LABEL_REF:
    3307              :     case CODE_LABEL:
    3308              :     case PC:
    3309              :     case RETURN:
    3310              :     case SIMPLE_RETURN:
    3311              :     case SCRATCH:
    3312              :       /* SCRATCH must be shared because they represent distinct values.  */
    3313              :       return;
    3314     63145096 :     case CLOBBER:
    3315              :       /* Share clobbers of hard registers, but do not share pseudo reg
    3316              :          clobbers or clobbers of hard registers that originated as pseudos.
    3317              :          This is needed to allow safe register renaming.  */
    3318     63145096 :       if (REG_P (XEXP (x, 0))
    3319     26455232 :           && HARD_REGISTER_NUM_P (REGNO (XEXP (x, 0)))
    3320     89420034 :           && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (x, 0))))
    3321              :         return;
    3322              :       break;
    3323              : 
    3324      5782776 :     case CONST:
    3325      5782776 :       if (shared_const_p (x))
    3326              :         return;
    3327              :       break;
    3328              : 
    3329              :     case DEBUG_INSN:
    3330              :     case INSN:
    3331              :     case JUMP_INSN:
    3332              :     case CALL_INSN:
    3333              :     case NOTE:
    3334              :     case BARRIER:
    3335              :       /* The chain of insns is not being copied.  */
    3336              :       return;
    3337              : 
    3338              :     default:
    3339              :       break;
    3340              :     }
    3341              : 
    3342              :   /* This rtx may not be shared.  If it has already been seen,
    3343              :      replace it with a copy of itself.  */
    3344              : 
    3345    759517086 :   if (RTX_FLAG (x, used))
    3346              :     {
    3347      4963348 :       x = shallow_copy_rtx (x);
    3348      4963348 :       copied = 1;
    3349              :     }
    3350    759517086 :   RTX_FLAG (x, used) = 1;
    3351              : 
    3352              :   /* Now scan the subexpressions recursively.
    3353              :      We can store any replaced subexpressions directly into X
    3354              :      since we know X is not shared!  Any vectors in X
    3355              :      must be copied if X was copied.  */
    3356              : 
    3357    759517086 :   format_ptr = GET_RTX_FORMAT (code);
    3358    759517086 :   length = GET_RTX_LENGTH (code);
    3359    759517086 :   last_ptr = NULL;
    3360              : 
    3361   2140739142 :   for (i = 0; i < length; i++)
    3362              :     {
    3363   1381222056 :       switch (*format_ptr++)
    3364              :         {
    3365   1147363805 :         case 'e':
    3366   1147363805 :           if (last_ptr)
    3367    446960251 :             copy_rtx_if_shared_1 (last_ptr);
    3368   1147363805 :           last_ptr = &XEXP (x, i);
    3369   1147363805 :           break;
    3370              : 
    3371     32250100 :         case 'E':
    3372     32250100 :           if (XVEC (x, i) != NULL)
    3373              :             {
    3374     32250100 :               int j;
    3375     32250100 :               int len = XVECLEN (x, i);
    3376              : 
    3377              :               /* Copy the vector iff I copied the rtx and the length
    3378              :                  is nonzero.  */
    3379     32250100 :               if (copied && len > 0)
    3380         1366 :                 XVEC (x, i) = gen_rtvec_v (len, XVEC (x, i)->elem);
    3381              : 
    3382              :               /* Call recursively on all inside the vector.  */
    3383     95730193 :               for (j = 0; j < len; j++)
    3384              :                 {
    3385     63480093 :                   if (last_ptr)
    3386     31893196 :                     copy_rtx_if_shared_1 (last_ptr);
    3387     63480093 :                   last_ptr = &XVECEXP (x, i, j);
    3388              :                 }
    3389              :             }
    3390              :           break;
    3391              :         }
    3392              :     }
    3393    759517086 :   *orig1 = x;
    3394    759517086 :   if (last_ptr)
    3395              :     {
    3396    731990451 :       orig1 = last_ptr;
    3397    731990451 :       goto repeat;
    3398              :     }
    3399              : }
    3400              : 
    3401              : /* Set the USED bit in X and its non-shareable subparts to FLAG.  */
    3402              : 
    3403              : static void
    3404  81161901879 : mark_used_flags (rtx x, int flag)
    3405              : {
    3406  >12623*10^7 :   int i, j;
    3407  >12623*10^7 :   enum rtx_code code;
    3408  >12623*10^7 :   const char *format_ptr;
    3409  >12623*10^7 :   int length;
    3410              : 
    3411              :   /* Repeat is used to turn tail-recursion into iteration.  */
    3412  >12623*10^7 : repeat:
    3413  >12623*10^7 :   if (x == 0)
    3414              :     return;
    3415              : 
    3416  >10630*10^7 :   code = GET_CODE (x);
    3417              : 
    3418              :   /* These types may be freely shared so we needn't do any resetting
    3419              :      for them.  */
    3420              : 
    3421  >10630*10^7 :   switch (code)
    3422              :     {
    3423              :     case REG:
    3424              :     case DEBUG_EXPR:
    3425              :     case VALUE:
    3426              :     CASE_CONST_ANY:
    3427              :     case SYMBOL_REF:
    3428              :     case CODE_LABEL:
    3429              :     case PC:
    3430              :     case RETURN:
    3431              :     case SIMPLE_RETURN:
    3432              :       return;
    3433              : 
    3434              :     case DEBUG_INSN:
    3435              :     case INSN:
    3436              :     case JUMP_INSN:
    3437              :     case CALL_INSN:
    3438              :     case NOTE:
    3439              :     case LABEL_REF:
    3440              :     case BARRIER:
    3441              :       /* The chain of insns is not being copied.  */
    3442              :       return;
    3443              : 
    3444  54910901119 :     default:
    3445  54910901119 :       break;
    3446              :     }
    3447              : 
    3448  54910901119 :   RTX_FLAG (x, used) = flag;
    3449              : 
    3450  54910901119 :   format_ptr = GET_RTX_FORMAT (code);
    3451  54910901119 :   length = GET_RTX_LENGTH (code);
    3452              : 
    3453  >10828*10^7 :   for (i = 0; i < length; i++)
    3454              :     {
    3455  98450647502 :       switch (*format_ptr++)
    3456              :         {
    3457  83619560292 :         case 'e':
    3458  83619560292 :           if (i == length-1)
    3459              :             {
    3460  45077057504 :               x = XEXP (x, i);
    3461  45077057504 :               goto repeat;
    3462              :             }
    3463  38542502788 :           mark_used_flags (XEXP (x, i), flag);
    3464  38542502788 :           break;
    3465              : 
    3466              :         case 'E':
    3467   5715724556 :           for (j = 0; j < XVECLEN (x, i); j++)
    3468   3813038040 :             mark_used_flags (XVECEXP (x, i, j), flag);
    3469              :           break;
    3470              :         }
    3471              :     }
    3472              : }
    3473              : 
    3474              : /* Clear all the USED bits in X to allow copy_rtx_if_shared to be used
    3475              :    to look for shared sub-parts.  */
    3476              : 
    3477              : void
    3478  38798827511 : reset_used_flags (rtx x)
    3479              : {
    3480  38798827511 :   mark_used_flags (x, 0);
    3481  38798827511 : }
    3482              : 
    3483              : /* Set all the USED bits in X to allow copy_rtx_if_shared to be used
    3484              :    to look for shared sub-parts.  */
    3485              : 
    3486              : void
    3487      7533540 : set_used_flags (rtx x)
    3488              : {
    3489      7533540 :   mark_used_flags (x, 1);
    3490      7533540 : }
    3491              : 
    3492              : /* Copy X if necessary so that it won't be altered by changes in OTHER.
    3493              :    Return X or the rtx for the pseudo reg the value of X was copied into.
    3494              :    OTHER must be valid as a SET_DEST.  */
    3495              : 
    3496              : rtx
    3497            0 : make_safe_from (rtx x, rtx other)
    3498              : {
    3499            0 :   while (1)
    3500            0 :     switch (GET_CODE (other))
    3501              :       {
    3502            0 :       case SUBREG:
    3503            0 :         other = SUBREG_REG (other);
    3504            0 :         break;
    3505            0 :       case STRICT_LOW_PART:
    3506            0 :       case SIGN_EXTEND:
    3507            0 :       case ZERO_EXTEND:
    3508            0 :         other = XEXP (other, 0);
    3509            0 :         break;
    3510            0 :       default:
    3511            0 :         goto done;
    3512              :       }
    3513            0 :  done:
    3514            0 :   if ((MEM_P (other)
    3515            0 :        && ! CONSTANT_P (x)
    3516            0 :        && !REG_P (x)
    3517            0 :        && GET_CODE (x) != SUBREG)
    3518            0 :       || (REG_P (other)
    3519            0 :           && (REGNO (other) < FIRST_PSEUDO_REGISTER
    3520            0 :               || reg_mentioned_p (other, x))))
    3521              :     {
    3522            0 :       rtx temp = gen_reg_rtx (GET_MODE (x));
    3523            0 :       emit_move_insn (temp, x);
    3524            0 :       return temp;
    3525              :     }
    3526              :   return x;
    3527              : }
    3528              : 
    3529              : /* Emission of insns (adding them to the doubly-linked list).  */
    3530              : 
    3531              : /* Return the last insn emitted, even if it is in a sequence now pushed.  */
    3532              : 
    3533              : rtx_insn *
    3534            0 : get_last_insn_anywhere (void)
    3535              : {
    3536            0 :   struct sequence_stack *seq;
    3537            0 :   for (seq = get_current_sequence (); seq; seq = seq->next)
    3538            0 :     if (seq->last != 0)
    3539              :       return seq->last;
    3540              :   return 0;
    3541              : }
    3542              : 
    3543              : /* Return the first nonnote insn emitted in current sequence or current
    3544              :    function.  This routine looks inside SEQUENCEs.  */
    3545              : 
    3546              : rtx_insn *
    3547            0 : get_first_nonnote_insn (void)
    3548              : {
    3549            0 :   rtx_insn *insn = get_insns ();
    3550              : 
    3551            0 :   if (insn)
    3552              :     {
    3553            0 :       if (NOTE_P (insn))
    3554            0 :         for (insn = next_insn (insn);
    3555            0 :              insn && NOTE_P (insn);
    3556            0 :              insn = next_insn (insn))
    3557            0 :           continue;
    3558              :       else
    3559              :         {
    3560            0 :           rtx_sequence *seq;
    3561            0 :           if (NONJUMP_INSN_P (insn)
    3562            0 :               && (seq = dyn_cast<rtx_sequence *> (PATTERN (insn))))
    3563            0 :             insn = seq->insn (0);
    3564              :         }
    3565              :     }
    3566              : 
    3567            0 :   return insn;
    3568              : }
    3569              : 
    3570              : /* Return the first nonnote, nondebug insn emitted in current sequence or
    3571              :    current function.  This routine follows get_first_nonnote_insn into a
    3572              :    leading SEQUENCE.  */
    3573              : 
    3574              : rtx_insn *
    3575            0 : get_first_nonnote_nondebug_insn (void)
    3576              : {
    3577            0 :   rtx_insn *insn = get_first_nonnote_insn ();
    3578            0 :   while (insn && (NOTE_P (insn) || DEBUG_INSN_P (insn)))
    3579            0 :     insn = next_insn (insn);
    3580            0 :   return insn;
    3581              : }
    3582              : 
    3583              : /* Return the last nonnote insn emitted in current sequence or current
    3584              :    function.  This routine looks inside SEQUENCEs.  */
    3585              : 
    3586              : rtx_insn *
    3587            0 : get_last_nonnote_insn (void)
    3588              : {
    3589            0 :   rtx_insn *insn = get_last_insn ();
    3590              : 
    3591            0 :   if (insn)
    3592              :     {
    3593            0 :       if (NOTE_P (insn))
    3594            0 :         for (insn = previous_insn (insn);
    3595            0 :              insn && NOTE_P (insn);
    3596            0 :              insn = previous_insn (insn))
    3597            0 :           continue;
    3598              :       else
    3599              :         {
    3600            0 :           if (NONJUMP_INSN_P (insn))
    3601            0 :             if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
    3602            0 :               insn = seq->insn (seq->len () - 1);
    3603              :         }
    3604              :     }
    3605              : 
    3606            0 :   return insn;
    3607              : }
    3608              : 
    3609              : /* Return the number of actual (non-debug) insns emitted in this
    3610              :    function.  */
    3611              : 
    3612              : int
    3613      3577947 : get_max_insn_count (void)
    3614              : {
    3615      3577947 :   int n = cur_insn_uid;
    3616              : 
    3617              :   /* The table size must be stable across -g, to avoid codegen
    3618              :      differences due to debug insns, and not be affected by
    3619              :      -fmin-insn-uid, to avoid excessive table size and to simplify
    3620              :      debugging of -fcompare-debug failures.  */
    3621      3577947 :   if (cur_debug_insn_uid > param_min_nondebug_insn_uid)
    3622      3577947 :     n -= cur_debug_insn_uid;
    3623              :   else
    3624            0 :     n -= param_min_nondebug_insn_uid;
    3625              : 
    3626      3577947 :   return n;
    3627              : }
    3628              : 
    3629              : 
    3630              : /* Return the next insn.  If it is a SEQUENCE, return the first insn
    3631              :    of the sequence.  */
    3632              : 
    3633              : rtx_insn *
    3634   1159725232 : next_insn (rtx_insn *insn)
    3635              : {
    3636   1159725232 :   if (insn)
    3637              :     {
    3638   1159725232 :       insn = NEXT_INSN (insn);
    3639   1159725232 :       if (insn && NONJUMP_INSN_P (insn))
    3640     14383213 :         if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
    3641            0 :           insn = seq->insn (0);
    3642              :     }
    3643              : 
    3644   1159725232 :   return insn;
    3645              : }
    3646              : 
    3647              : /* Return the previous insn.  If it is a SEQUENCE, return the last insn
    3648              :    of the sequence.  */
    3649              : 
    3650              : rtx_insn *
    3651        72462 : previous_insn (rtx_insn *insn)
    3652              : {
    3653        72462 :   if (insn)
    3654              :     {
    3655        72462 :       insn = PREV_INSN (insn);
    3656        72462 :       if (insn && NONJUMP_INSN_P (insn))
    3657         8737 :         if (rtx_sequence *seq = dyn_cast <rtx_sequence *> (PATTERN (insn)))
    3658            0 :           insn = seq->insn (seq->len () - 1);
    3659              :     }
    3660              : 
    3661        72462 :   return insn;
    3662              : }
    3663              : 
    3664              : /* Return the next insn after INSN that is not a NOTE.  This routine does not
    3665              :    look inside SEQUENCEs.  */
    3666              : 
    3667              : rtx_insn *
    3668       551040 : next_nonnote_insn (rtx_insn *insn)
    3669              : {
    3670       574634 :   while (insn)
    3671              :     {
    3672       574634 :       insn = NEXT_INSN (insn);
    3673       574634 :       if (insn == 0 || !NOTE_P (insn))
    3674              :         break;
    3675              :     }
    3676              : 
    3677       551040 :   return insn;
    3678              : }
    3679              : 
    3680              : /* Return the next insn after INSN that is not a DEBUG_INSN.  This
    3681              :    routine does not look inside SEQUENCEs.  */
    3682              : 
    3683              : rtx_insn *
    3684      6206106 : next_nondebug_insn (rtx_insn *insn)
    3685              : {
    3686      8581019 :   while (insn)
    3687              :     {
    3688      8581019 :       insn = NEXT_INSN (insn);
    3689      8581019 :       if (insn == 0 || !DEBUG_INSN_P (insn))
    3690              :         break;
    3691              :     }
    3692              : 
    3693      6206106 :   return insn;
    3694              : }
    3695              : 
    3696              : /* Return the previous insn before INSN that is not a NOTE.  This routine does
    3697              :    not look inside SEQUENCEs.  */
    3698              : 
    3699              : rtx_insn *
    3700     94517682 : prev_nonnote_insn (rtx_insn *insn)
    3701              : {
    3702     99494884 :   while (insn)
    3703              :     {
    3704     99494884 :       insn = PREV_INSN (insn);
    3705     99494884 :       if (insn == 0 || !NOTE_P (insn))
    3706              :         break;
    3707              :     }
    3708              : 
    3709     94517682 :   return insn;
    3710              : }
    3711              : 
    3712              : /* Return the previous insn before INSN that is not a DEBUG_INSN.
    3713              :    This routine does not look inside SEQUENCEs.  */
    3714              : 
    3715              : rtx_insn *
    3716      3176265 : prev_nondebug_insn (rtx_insn *insn)
    3717              : {
    3718     10935773 :   while (insn)
    3719              :     {
    3720     10935773 :       insn = PREV_INSN (insn);
    3721     10935773 :       if (insn == 0 || !DEBUG_INSN_P (insn))
    3722              :         break;
    3723              :     }
    3724              : 
    3725      3176265 :   return insn;
    3726              : }
    3727              : 
    3728              : /* Return the next insn after INSN that is not a NOTE nor DEBUG_INSN.
    3729              :    This routine does not look inside SEQUENCEs.  */
    3730              : 
    3731              : rtx_insn *
    3732     52589418 : next_nonnote_nondebug_insn (rtx_insn *insn)
    3733              : {
    3734     71512805 :   while (insn)
    3735              :     {
    3736     71512805 :       insn = NEXT_INSN (insn);
    3737     71512805 :       if (insn == 0 || (!NOTE_P (insn) && !DEBUG_INSN_P (insn)))
    3738              :         break;
    3739              :     }
    3740              : 
    3741     52589418 :   return insn;
    3742              : }
    3743              : 
    3744              : /* Return the next insn after INSN that is not a NOTE nor DEBUG_INSN,
    3745              :    but stop the search before we enter another basic block.  This
    3746              :    routine does not look inside SEQUENCEs.
    3747              :    NOTE: This can potentially bleed into next BB. If current insn is
    3748              :          last insn of BB, followed by a code_label before the start of
    3749              :          the next BB, code_label will be returned. But this is the
    3750              :          behavior rest of gcc assumes/relies on e.g. get_last_bb_insn.  */
    3751              : 
    3752              : rtx_insn *
    3753      7608296 : next_nonnote_nondebug_insn_bb (rtx_insn *insn)
    3754              : {
    3755      7608296 :   while (insn)
    3756              :     {
    3757      7641826 :       insn = NEXT_INSN (insn);
    3758      7641826 :       if (insn == 0)
    3759              :         break;
    3760      7413971 :       if (DEBUG_INSN_P (insn))
    3761            0 :         continue;
    3762      7413971 :       if (!NOTE_P (insn))
    3763              :         break;
    3764      9820328 :       if (NOTE_INSN_BASIC_BLOCK_P (insn))
    3765              :         return NULL;
    3766              :     }
    3767              : 
    3768              :   return insn;
    3769              : }
    3770              : 
    3771              : /* Return the previous insn before INSN that is not a NOTE nor DEBUG_INSN.
    3772              :    This routine does not look inside SEQUENCEs.  */
    3773              : 
    3774              : rtx_insn *
    3775    145585522 : prev_nonnote_nondebug_insn (rtx_insn *insn)
    3776              : {
    3777    271707960 :   while (insn)
    3778              :     {
    3779    271707960 :       insn = PREV_INSN (insn);
    3780    271707960 :       if (insn == 0 || (!NOTE_P (insn) && !DEBUG_INSN_P (insn)))
    3781              :         break;
    3782              :     }
    3783              : 
    3784    145585522 :   return insn;
    3785              : }
    3786              : 
    3787              : /* Return the previous insn before INSN that is not a NOTE nor
    3788              :    DEBUG_INSN, but stop the search before we enter another basic
    3789              :    block.  This routine does not look inside SEQUENCEs.  */
    3790              : 
    3791              : rtx_insn *
    3792     71214224 : prev_nonnote_nondebug_insn_bb (rtx_insn *insn)
    3793              : {
    3794     71214224 :   while (insn)
    3795              :     {
    3796    140542847 :       insn = PREV_INSN (insn);
    3797    140542847 :       if (insn == 0)
    3798              :         break;
    3799    140542843 :       if (DEBUG_INSN_P (insn))
    3800     59733577 :         continue;
    3801     80809266 :       if (!NOTE_P (insn))
    3802              :         break;
    3803    162112992 :       if (NOTE_INSN_BASIC_BLOCK_P (insn))
    3804              :         return NULL;
    3805              :     }
    3806              : 
    3807              :   return insn;
    3808              : }
    3809              : 
    3810              : /* Return the next INSN, CALL_INSN, JUMP_INSN or DEBUG_INSN after INSN;
    3811              :    or 0, if there is none.  This routine does not look inside
    3812              :    SEQUENCEs.  */
    3813              : 
    3814              : rtx_insn *
    3815      4411951 : next_real_insn (rtx_insn *insn)
    3816              : {
    3817      4478118 :   while (insn)
    3818              :     {
    3819      4478118 :       insn = NEXT_INSN (insn);
    3820      4478118 :       if (insn == 0 || INSN_P (insn))
    3821              :         break;
    3822              :     }
    3823              : 
    3824      4411951 :   return insn;
    3825              : }
    3826              : 
    3827              : /* Return the last INSN, CALL_INSN, JUMP_INSN or DEBUG_INSN before INSN;
    3828              :    or 0, if there is none.  This routine does not look inside
    3829              :    SEQUENCEs.  */
    3830              : 
    3831              : rtx_insn *
    3832       988342 : prev_real_insn (rtx_insn *insn)
    3833              : {
    3834      2923261 :   while (insn)
    3835              :     {
    3836      2923261 :       insn = PREV_INSN (insn);
    3837      2923261 :       if (insn == 0 || INSN_P (insn))
    3838              :         break;
    3839              :     }
    3840              : 
    3841       988342 :   return insn;
    3842              : }
    3843              : 
    3844              : /* Return the next INSN, CALL_INSN or JUMP_INSN after INSN;
    3845              :    or 0, if there is none.  This routine does not look inside
    3846              :    SEQUENCEs.  */
    3847              : 
    3848              : rtx_insn *
    3849            0 : next_real_nondebug_insn (rtx_insn *insn)
    3850              : {
    3851            0 :   while (insn)
    3852              :     {
    3853            0 :       insn = NEXT_INSN (insn);
    3854            0 :       if (insn == 0 || NONDEBUG_INSN_P (insn))
    3855              :         break;
    3856              :     }
    3857              : 
    3858            0 :   return insn;
    3859              : }
    3860              : 
    3861              : /* Return the last INSN, CALL_INSN or JUMP_INSN before INSN;
    3862              :    or 0, if there is none.  This routine does not look inside
    3863              :    SEQUENCEs.  */
    3864              : 
    3865              : rtx_insn *
    3866      5134812 : prev_real_nondebug_insn (rtx_insn *insn)
    3867              : {
    3868     15815901 :   while (insn)
    3869              :     {
    3870     15815901 :       insn = PREV_INSN (insn);
    3871     15815901 :       if (insn == 0 || NONDEBUG_INSN_P (insn))
    3872              :         break;
    3873              :     }
    3874              : 
    3875      5134812 :   return insn;
    3876              : }
    3877              : 
    3878              : /* Return the last CALL_INSN in the current list, or 0 if there is none.
    3879              :    This routine does not look inside SEQUENCEs.  */
    3880              : 
    3881              : rtx_call_insn *
    3882     12330731 : last_call_insn (void)
    3883              : {
    3884     12330731 :   rtx_insn *insn;
    3885              : 
    3886     12458208 :   for (insn = get_last_insn ();
    3887     12458208 :        insn && !CALL_P (insn);
    3888       127477 :        insn = PREV_INSN (insn))
    3889              :     ;
    3890              : 
    3891     12330731 :   return safe_as_a <rtx_call_insn *> (insn);
    3892              : }
    3893              : 
    3894              : bool
    3895   1187911227 : active_insn_p (const rtx_insn *insn)
    3896              : {
    3897   1187911227 :   return (CALL_P (insn) || JUMP_P (insn)
    3898   1187911227 :           || JUMP_TABLE_DATA_P (insn) /* FIXME */
    3899   1187911227 :           || (NONJUMP_INSN_P (insn)
    3900    702679318 :               && (! reload_completed
    3901    499441379 :                   || (GET_CODE (PATTERN (insn)) != USE
    3902    497823831 :                       && GET_CODE (PATTERN (insn)) != CLOBBER))));
    3903              : }
    3904              : 
    3905              : /* Find the next insn after INSN that really does something.  This routine
    3906              :    does not look inside SEQUENCEs.  After reload this also skips over
    3907              :    standalone USE and CLOBBER insn.  */
    3908              : 
    3909              : rtx_insn *
    3910    115679757 : next_active_insn (rtx_insn *insn)
    3911              : {
    3912    258346527 :   while (insn)
    3913              :     {
    3914    258346527 :       insn = NEXT_INSN (insn);
    3915    374026284 :       if (insn == 0 || active_insn_p (insn))
    3916              :         break;
    3917              :     }
    3918              : 
    3919    115679757 :   return insn;
    3920              : }
    3921              : 
    3922              : /* Find the last insn before INSN that really does something.  This routine
    3923              :    does not look inside SEQUENCEs.  After reload this also skips over
    3924              :    standalone USE and CLOBBER insn.  */
    3925              : 
    3926              : rtx_insn *
    3927        46323 : prev_active_insn (rtx_insn *insn)
    3928              : {
    3929       138046 :   while (insn)
    3930              :     {
    3931       138046 :       insn = PREV_INSN (insn);
    3932       184369 :       if (insn == 0 || active_insn_p (insn))
    3933              :         break;
    3934              :     }
    3935              : 
    3936        46323 :   return insn;
    3937              : }
    3938              : 
    3939              : /* Find a RTX_AUTOINC class rtx which matches DATA.  */
    3940              : 
    3941              : static int
    3942            0 : find_auto_inc (const_rtx x, const_rtx reg)
    3943              : {
    3944            0 :   subrtx_iterator::array_type array;
    3945            0 :   FOR_EACH_SUBRTX (iter, array, x, NONCONST)
    3946              :     {
    3947            0 :       const_rtx x = *iter;
    3948            0 :       if (GET_RTX_CLASS (GET_CODE (x)) == RTX_AUTOINC
    3949            0 :           && rtx_equal_p (reg, XEXP (x, 0)))
    3950            0 :         return true;
    3951              :     }
    3952            0 :   return false;
    3953            0 : }
    3954              : 
    3955              : /* Increment the label uses for all labels present in rtx.  */
    3956              : 
    3957              : static void
    3958     57166057 : mark_label_nuses (rtx x)
    3959              : {
    3960     57166057 :   enum rtx_code code;
    3961     57166057 :   int i, j;
    3962     57166057 :   const char *fmt;
    3963              : 
    3964     57166057 :   code = GET_CODE (x);
    3965     57166062 :   if (code == LABEL_REF && LABEL_P (label_ref_label (x)))
    3966            1 :     LABEL_NUSES (label_ref_label (x))++;
    3967              : 
    3968     57166057 :   fmt = GET_RTX_FORMAT (code);
    3969    140999719 :   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    3970              :     {
    3971     83833662 :       if (fmt[i] == 'e')
    3972     44884827 :         mark_label_nuses (XEXP (x, i));
    3973     38948835 :       else if (fmt[i] == 'E')
    3974      4013113 :         for (j = XVECLEN (x, i) - 1; j >= 0; j--)
    3975      2780303 :           mark_label_nuses (XVECEXP (x, i, j));
    3976              :     }
    3977     57166057 : }
    3978              : 
    3979              : 
    3980              : /* Try splitting insns that can be split for better scheduling.
    3981              :    PAT is the pattern which might split.
    3982              :    TRIAL is the insn providing PAT.
    3983              :    LAST is nonzero if we should return the last insn of the sequence produced.
    3984              : 
    3985              :    If this routine succeeds in splitting, it returns the first or last
    3986              :    replacement insn depending on the value of LAST.  Otherwise, it
    3987              :    returns TRIAL.  If the insn to be returned can be split, it will be.  */
    3988              : 
    3989              : rtx_insn *
    3990    404497076 : try_split (rtx pat, rtx_insn *trial, int last)
    3991              : {
    3992    404497076 :   rtx_insn *before, *after;
    3993    404497076 :   rtx note;
    3994    404497076 :   rtx_insn *seq, *tem;
    3995    404497076 :   profile_probability probability;
    3996    404497076 :   rtx_insn *insn_last, *insn;
    3997    404497076 :   int njumps = 0;
    3998    404497076 :   rtx_insn *call_insn = NULL;
    3999              : 
    4000    404497076 :   if (any_condjump_p (trial)
    4001    404497076 :       && (note = find_reg_note (trial, REG_BR_PROB, 0)))
    4002     14965041 :     split_branch_probability
    4003     14965041 :       = profile_probability::from_reg_br_prob_note (XINT (note, 0));
    4004              :   else
    4005    389532035 :     split_branch_probability = profile_probability::uninitialized ();
    4006              : 
    4007    404497076 :   probability = split_branch_probability;
    4008              : 
    4009    404497076 :   seq = split_insns (pat, trial);
    4010              : 
    4011    404497076 :   split_branch_probability = profile_probability::uninitialized ();
    4012              : 
    4013    404497076 :   if (!seq)
    4014              :     return trial;
    4015              : 
    4016              :   int split_insn_count = 0;
    4017              :   /* Avoid infinite loop if any insn of the result matches
    4018              :      the original pattern.  */
    4019              :   insn_last = seq;
    4020      9516808 :   while (1)
    4021              :     {
    4022      9516808 :       if (INSN_P (insn_last)
    4023      9516808 :           && rtx_equal_p (PATTERN (insn_last), pat))
    4024              :         return trial;
    4025      9515289 :       split_insn_count++;
    4026      9515289 :       if (!NEXT_INSN (insn_last))
    4027              :         break;
    4028              :       insn_last = NEXT_INSN (insn_last);
    4029              :     }
    4030              : 
    4031              :   /* We're not good at redistributing frame information if
    4032              :      the split occurs before reload or if it results in more
    4033              :      than one insn.  */
    4034      6287432 :   if (RTX_FRAME_RELATED_P (trial))
    4035              :     {
    4036           37 :       if (!reload_completed || split_insn_count != 1)
    4037              :         return trial;
    4038              : 
    4039           37 :       rtx_insn *new_insn = seq;
    4040           37 :       rtx_insn *old_insn = trial;
    4041           37 :       copy_frame_info_to_split_insn (old_insn, new_insn);
    4042              :     }
    4043              : 
    4044              :   /* We will be adding the new sequence to the function.  The splitters
    4045              :      may have introduced invalid RTL sharing, so unshare the sequence now.  */
    4046      6287432 :   unshare_all_rtl_in_chain (seq);
    4047              : 
    4048              :   /* Mark labels and copy flags.  */
    4049     22090153 :   for (insn = insn_last; insn ; insn = PREV_INSN (insn))
    4050              :     {
    4051      9515289 :       if (JUMP_P (insn))
    4052              :         {
    4053         6198 :           if (JUMP_P (trial))
    4054         5358 :             set_crossing_jump (insn, CROSSING_JUMP_P (trial));
    4055         6198 :           mark_jump_label (PATTERN (insn), insn, 0);
    4056         6198 :           njumps++;
    4057         6198 :           if (probability.initialized_p ()
    4058         5326 :               && any_condjump_p (insn)
    4059        11524 :               && !find_reg_note (insn, REG_BR_PROB, 0))
    4060              :             {
    4061              :               /* We can preserve the REG_BR_PROB notes only if exactly
    4062              :                  one jump is created, otherwise the machine description
    4063              :                  is responsible for this step using
    4064              :                  split_branch_probability variable.  */
    4065         5326 :               gcc_assert (njumps == 1);
    4066         5326 :               add_reg_br_prob_note (insn, probability);
    4067              :             }
    4068              :         }
    4069              :     }
    4070              : 
    4071              :   /* If we are splitting a CALL_INSN, look for the CALL_INSN
    4072              :      in SEQ and copy any additional information across.  */
    4073      6287432 :   if (auto call_trial = dyn_cast<rtx_call_insn *> (trial))
    4074              :     {
    4075            0 :       for (insn = insn_last; insn ; insn = PREV_INSN (insn))
    4076            0 :         if (auto call = dyn_cast<rtx_call_insn *> (insn))
    4077              :           {
    4078            0 :             gcc_assert (call_insn == NULL_RTX);
    4079            0 :             call_insn = insn;
    4080              : 
    4081              :             /* Add the old CALL_INSN_FUNCTION_USAGE to whatever the
    4082              :                target may have explicitly specified.  */
    4083            0 :             rtx *p = &CALL_INSN_FUNCTION_USAGE (call);
    4084            0 :             while (*p)
    4085            0 :               p = &XEXP (*p, 1);
    4086            0 :             *p = CALL_INSN_FUNCTION_USAGE (call_trial);
    4087              : 
    4088              :             /* Preserve the ABI information from the original call.  */
    4089            0 :             CALL_INSN_ABI_ID (call) = CALL_INSN_ABI_ID (call_trial);
    4090              : 
    4091              :             /* If the old call was a sibling call, the new one must
    4092              :                be too.  */
    4093            0 :             SIBLING_CALL_P (insn) = SIBLING_CALL_P (trial);
    4094              :           }
    4095              :     }
    4096              : 
    4097              :   /* Copy notes, particularly those related to the CFG.  */
    4098      8149981 :   for (note = REG_NOTES (trial); note; note = XEXP (note, 1))
    4099              :     {
    4100      1862549 :       switch (REG_NOTE_KIND (note))
    4101              :         {
    4102         2790 :         case REG_EH_REGION:
    4103         2790 :           copy_reg_eh_region_note_backward (note, insn_last, NULL);
    4104         2790 :           break;
    4105              : 
    4106              :         case REG_NORETURN:
    4107              :         case REG_SETJMP:
    4108              :         case REG_TM:
    4109              :         case REG_CALL_NOCF_CHECK:
    4110              :         case REG_CALL_ARG_LOCATION:
    4111            0 :           for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
    4112              :             {
    4113            0 :               if (CALL_P (insn))
    4114            0 :                 add_reg_note (insn, REG_NOTE_KIND (note), XEXP (note, 0));
    4115              :             }
    4116              :           break;
    4117              : 
    4118              :         case REG_NON_LOCAL_GOTO:
    4119              :         case REG_LABEL_TARGET:
    4120            0 :           for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
    4121              :             {
    4122            0 :               if (JUMP_P (insn))
    4123            0 :                 add_reg_note (insn, REG_NOTE_KIND (note), XEXP (note, 0));
    4124              :             }
    4125              :           break;
    4126              : 
    4127              :         case REG_INC:
    4128              :           if (!AUTO_INC_DEC)
    4129              :             break;
    4130              : 
    4131              :           for (insn = insn_last; insn != NULL_RTX; insn = PREV_INSN (insn))
    4132              :             {
    4133              :               rtx reg = XEXP (note, 0);
    4134              :               if (!FIND_REG_INC_NOTE (insn, reg)
    4135              :                   && find_auto_inc (PATTERN (insn), reg))
    4136              :                 add_reg_note (insn, REG_INC, reg);
    4137              :             }
    4138              :           break;
    4139              : 
    4140       306100 :         case REG_ARGS_SIZE:
    4141       306100 :           fixup_args_size_notes (NULL, insn_last, get_args_size (note));
    4142       306100 :           break;
    4143              : 
    4144            0 :         case REG_CALL_DECL:
    4145            0 :         case REG_UNTYPED_CALL:
    4146            0 :           gcc_assert (call_insn != NULL_RTX);
    4147            0 :           add_reg_note (call_insn, REG_NOTE_KIND (note), XEXP (note, 0));
    4148            0 :           break;
    4149              : 
    4150              :         default:
    4151              :           break;
    4152              :         }
    4153              :     }
    4154              : 
    4155              :   /* If there are LABELS inside the split insns increment the
    4156              :      usage count so we don't delete the label.  */
    4157      6287432 :   if (INSN_P (trial))
    4158              :     {
    4159              :       insn = insn_last;
    4160     15802721 :       while (insn != NULL_RTX)
    4161              :         {
    4162              :           /* JUMP_P insns have already been "marked" above.  */
    4163      9515289 :           if (NONJUMP_INSN_P (insn))
    4164      9500927 :             mark_label_nuses (PATTERN (insn));
    4165              : 
    4166      9515289 :           insn = PREV_INSN (insn);
    4167              :         }
    4168              :     }
    4169              : 
    4170      6287432 :   before = PREV_INSN (trial);
    4171      6287432 :   after = NEXT_INSN (trial);
    4172              : 
    4173      6287432 :   emit_insn_after_setloc (seq, trial, INSN_LOCATION (trial));
    4174              : 
    4175      6287432 :   delete_insn (trial);
    4176              : 
    4177              :   /* Recursively call try_split for each new insn created; by the
    4178              :      time control returns here that insn will be fully split, so
    4179              :      set LAST and continue from the insn after the one returned.
    4180              :      We can't use next_active_insn here since AFTER may be a note.
    4181              :      Ignore deleted insns, which can be occur if not optimizing.  */
    4182     22090153 :   for (tem = NEXT_INSN (before); tem != after; tem = NEXT_INSN (tem))
    4183      9515289 :     if (! tem->deleted () && INSN_P (tem))
    4184      9507125 :       tem = try_split (PATTERN (tem), tem, 1);
    4185              : 
    4186              :   /* Return either the first or the last insn, depending on which was
    4187              :      requested.  */
    4188      6287432 :   return last
    4189      6287432 :     ? (after ? PREV_INSN (after) : get_last_insn ())
    4190            0 :     : NEXT_INSN (before);
    4191              : }
    4192              : 
    4193              : /* Make and return an INSN rtx, initializing all its slots.
    4194              :    Store PATTERN in the pattern slots.  */
    4195              : 
    4196              : rtx_insn *
    4197    129320420 : make_insn_raw (rtx pattern)
    4198              : {
    4199    129320420 :   rtx_insn *insn;
    4200              : 
    4201    129320420 :   insn = as_a <rtx_insn *> (rtx_alloc (INSN));
    4202              : 
    4203    129320420 :   INSN_UID (insn) = cur_insn_uid++;
    4204    129320420 :   PATTERN (insn) = pattern;
    4205    129320420 :   INSN_CODE (insn) = -1;
    4206    129320420 :   REG_NOTES (insn) = NULL;
    4207    129320420 :   INSN_LOCATION (insn) = curr_insn_location ();
    4208    129320420 :   BLOCK_FOR_INSN (insn) = NULL;
    4209              : 
    4210              : #ifdef ENABLE_RTL_CHECKING
    4211              :   if (insn
    4212              :       && INSN_P (insn)
    4213              :       && (returnjump_p (insn)
    4214              :           || (GET_CODE (insn) == SET
    4215              :               && SET_DEST (insn) == pc_rtx)))
    4216              :     {
    4217              :       warning (0, "ICE: %<emit_insn%> used where %<emit_jump_insn%> needed:");
    4218              :       debug_rtx (insn);
    4219              :     }
    4220              : #endif
    4221              : 
    4222    129320420 :   return insn;
    4223              : }
    4224              : 
    4225              : /* Like `make_insn_raw' but make a DEBUG_INSN instead of an insn.  */
    4226              : 
    4227              : static rtx_insn *
    4228     54478789 : make_debug_insn_raw (rtx pattern)
    4229              : {
    4230     54478789 :   rtx_debug_insn *insn;
    4231              : 
    4232     54478789 :   insn = as_a <rtx_debug_insn *> (rtx_alloc (DEBUG_INSN));
    4233     54478789 :   INSN_UID (insn) = cur_debug_insn_uid++;
    4234     54478789 :   if (cur_debug_insn_uid > param_min_nondebug_insn_uid)
    4235     54478789 :     INSN_UID (insn) = cur_insn_uid++;
    4236              : 
    4237     54478789 :   PATTERN (insn) = pattern;
    4238     54478789 :   INSN_CODE (insn) = -1;
    4239     54478789 :   REG_NOTES (insn) = NULL;
    4240     54478789 :   INSN_LOCATION (insn) = curr_insn_location ();
    4241     54478789 :   BLOCK_FOR_INSN (insn) = NULL;
    4242              : 
    4243     54478789 :   return insn;
    4244              : }
    4245              : 
    4246              : /* Like `make_insn_raw' but make a JUMP_INSN instead of an insn.  */
    4247              : 
    4248              : static rtx_insn *
    4249     18485139 : make_jump_insn_raw (rtx pattern)
    4250              : {
    4251     18485139 :   rtx_jump_insn *insn;
    4252              : 
    4253     18485139 :   insn = as_a <rtx_jump_insn *> (rtx_alloc (JUMP_INSN));
    4254     18485139 :   INSN_UID (insn) = cur_insn_uid++;
    4255              : 
    4256     18485139 :   PATTERN (insn) = pattern;
    4257     18485139 :   INSN_CODE (insn) = -1;
    4258     18485139 :   REG_NOTES (insn) = NULL;
    4259     18485139 :   JUMP_LABEL (insn) = NULL;
    4260     18485139 :   INSN_LOCATION (insn) = curr_insn_location ();
    4261     18485139 :   BLOCK_FOR_INSN (insn) = NULL;
    4262              : 
    4263     18485139 :   return insn;
    4264              : }
    4265              : 
    4266              : /* Like `make_insn_raw' but make a CALL_INSN instead of an insn.  */
    4267              : 
    4268              : static rtx_insn *
    4269      6479861 : make_call_insn_raw (rtx pattern)
    4270              : {
    4271      6479861 :   rtx_call_insn *insn;
    4272              : 
    4273      6479861 :   insn = as_a <rtx_call_insn *> (rtx_alloc (CALL_INSN));
    4274      6479861 :   INSN_UID (insn) = cur_insn_uid++;
    4275              : 
    4276      6479861 :   PATTERN (insn) = pattern;
    4277      6479861 :   INSN_CODE (insn) = -1;
    4278      6479861 :   REG_NOTES (insn) = NULL;
    4279      6479861 :   CALL_INSN_FUNCTION_USAGE (insn) = NULL;
    4280      6479861 :   CALL_INSN_ABI_ID (insn) = 0;
    4281      6479861 :   INSN_LOCATION (insn) = curr_insn_location ();
    4282      6479861 :   BLOCK_FOR_INSN (insn) = NULL;
    4283              : 
    4284      6479861 :   return insn;
    4285              : }
    4286              : 
    4287              : /* Like `make_insn_raw' but make a NOTE instead of an insn.  */
    4288              : 
    4289              : static rtx_note *
    4290    176332756 : make_note_raw (enum insn_note subtype)
    4291              : {
    4292              :   /* Some notes are never created this way at all.  These notes are
    4293              :      only created by patching out insns.  */
    4294    176332756 :   gcc_assert (subtype != NOTE_INSN_DELETED_LABEL
    4295              :               && subtype != NOTE_INSN_DELETED_DEBUG_LABEL);
    4296              : 
    4297    176332756 :   rtx_note *note = as_a <rtx_note *> (rtx_alloc (NOTE));
    4298    176332756 :   INSN_UID (note) = cur_insn_uid++;
    4299    176332756 :   NOTE_KIND (note) = subtype;
    4300    176332756 :   BLOCK_FOR_INSN (note) = NULL;
    4301    176332756 :   memset (&NOTE_DATA (note), 0, sizeof (NOTE_DATA (note)));
    4302    176332756 :   return note;
    4303              : }
    4304              : 
    4305              : /* Add INSN to the end of the doubly-linked list, between PREV and NEXT.
    4306              :    INSN may be any object that can appear in the chain: INSN_P and NOTE_P objects,
    4307              :    but also BARRIERs and JUMP_TABLE_DATAs.  PREV and NEXT may be NULL.  */
    4308              : 
    4309              : static inline void
    4310    616051534 : link_insn_into_chain (rtx_insn *insn, rtx_insn *prev, rtx_insn *next)
    4311              : {
    4312    616051534 :   SET_PREV_INSN (insn) = prev;
    4313    616051534 :   SET_NEXT_INSN (insn) = next;
    4314    616051534 :   if (prev != NULL)
    4315              :     {
    4316    443573384 :       SET_NEXT_INSN (prev) = insn;
    4317    443573384 :       if (NONJUMP_INSN_P (prev))
    4318    177568425 :         if (rtx_sequence *sequence
    4319    177568425 :             = dyn_cast <rtx_sequence *> (PATTERN (prev)))
    4320            0 :           SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = insn;
    4321              :     }
    4322    616051534 :   if (next != NULL)
    4323              :     {
    4324    181391118 :       SET_PREV_INSN (next) = insn;
    4325    181391118 :       if (NONJUMP_INSN_P (next))
    4326     49288362 :         if (rtx_sequence *sequence
    4327     49288362 :             = dyn_cast <rtx_sequence *> (PATTERN (next)))
    4328            0 :           SET_PREV_INSN (sequence->insn (0)) = insn;
    4329              :     }
    4330              : 
    4331    616051534 :   if (NONJUMP_INSN_P (insn))
    4332    301126026 :     if (rtx_sequence *sequence
    4333    301126026 :         = dyn_cast <rtx_sequence *> (PATTERN (insn)))
    4334              :       {
    4335            0 :         SET_PREV_INSN (sequence->insn (0)) = prev;
    4336            0 :         SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = next;
    4337              :       }
    4338    616051534 : }
    4339              : 
    4340              : /* Add INSN to the end of the doubly-linked list.
    4341              :    INSN may be an INSN, JUMP_INSN, CALL_INSN, CODE_LABEL, BARRIER or NOTE.  */
    4342              : 
    4343              : void
    4344    423334214 : add_insn (rtx_insn *insn)
    4345              : {
    4346    423334214 :   rtx_insn *prev = get_last_insn ();
    4347    423334214 :   link_insn_into_chain (insn, prev, NULL);
    4348    423334214 :   if (get_insns () == NULL)
    4349    172288516 :     set_first_insn (insn);
    4350    423334214 :   set_last_insn (insn);
    4351    423334214 : }
    4352              : 
    4353              : /* Add INSN into the doubly-linked list after insn AFTER.  */
    4354              : 
    4355              : static void
    4356     71418036 : add_insn_after_nobb (rtx_insn *insn, rtx_insn *after)
    4357              : {
    4358     71418036 :   rtx_insn *next = NEXT_INSN (after);
    4359              : 
    4360     71418036 :   gcc_assert (!optimize || !after->deleted ());
    4361              : 
    4362     71418036 :   link_insn_into_chain (insn, after, next);
    4363              : 
    4364     71418036 :   if (next == NULL)
    4365              :     {
    4366              :       struct sequence_stack *seq;
    4367              : 
    4368     11326202 :       for (seq = get_current_sequence (); seq; seq = seq->next)
    4369     11326202 :         if (after == seq->last)
    4370              :           {
    4371     11326202 :             seq->last = insn;
    4372     11326202 :             break;
    4373              :           }
    4374              :     }
    4375     71418036 : }
    4376              : 
    4377              : /* Add INSN into the doubly-linked list before insn BEFORE.  */
    4378              : 
    4379              : static void
    4380    121299284 : add_insn_before_nobb (rtx_insn *insn, rtx_insn *before)
    4381              : {
    4382    121299284 :   rtx_insn *prev = PREV_INSN (before);
    4383              : 
    4384    121299284 :   gcc_assert (!optimize || !before->deleted ());
    4385              : 
    4386    121299284 :   link_insn_into_chain (insn, prev, before);
    4387              : 
    4388    121299284 :   if (prev == NULL)
    4389              :     {
    4390              :       struct sequence_stack *seq;
    4391              : 
    4392       189634 :       for (seq = get_current_sequence (); seq; seq = seq->next)
    4393       189634 :         if (before == seq->first)
    4394              :           {
    4395       189634 :             seq->first = insn;
    4396       189634 :             break;
    4397              :           }
    4398              : 
    4399       189634 :       gcc_assert (seq);
    4400              :     }
    4401    121299284 : }
    4402              : 
    4403              : /* Like add_insn_after_nobb, but try to set BLOCK_FOR_INSN.
    4404              :    If BB is NULL, an attempt is made to infer the bb from before.
    4405              : 
    4406              :    This and the next function should be the only functions called
    4407              :    to insert an insn once delay slots have been filled since only
    4408              :    they know how to update a SEQUENCE. */
    4409              : 
    4410              : void
    4411     69965800 : add_insn_after (rtx_insn *insn, rtx_insn *after, basic_block bb)
    4412              : {
    4413     69965800 :   add_insn_after_nobb (insn, after);
    4414     69965800 :   if (!BARRIER_P (after)
    4415     69934775 :       && !BARRIER_P (insn)
    4416    132080047 :       && (bb = BLOCK_FOR_INSN (after)))
    4417              :     {
    4418     51484385 :       set_block_for_insn (insn, bb);
    4419     51484385 :       if (INSN_P (insn))
    4420       842535 :         df_insn_rescan (insn);
    4421              :       /* Should not happen as first in the BB is always
    4422              :          either NOTE or LABEL.  */
    4423     51484385 :       if (BB_END (bb) == after
    4424              :           /* Avoid clobbering of structure when creating new BB.  */
    4425      1025066 :           && !BARRIER_P (insn)
    4426     51484385 :           && !NOTE_INSN_BASIC_BLOCK_P (insn))
    4427       334004 :         BB_END (bb) = insn;
    4428              :     }
    4429     69965800 : }
    4430              : 
    4431              : /* Like add_insn_before_nobb, but try to set BLOCK_FOR_INSN.
    4432              :    If BB is NULL, an attempt is made to infer the bb from before.
    4433              : 
    4434              :    This and the previous function should be the only functions called
    4435              :    to insert an insn once delay slots have been filled since only
    4436              :    they know how to update a SEQUENCE. */
    4437              : 
    4438              : void
    4439     97812595 : add_insn_before (rtx_insn *insn, rtx_insn *before, basic_block bb)
    4440              : {
    4441     97812595 :   add_insn_before_nobb (insn, before);
    4442              : 
    4443     97812595 :   if (BARRIER_P (insn))
    4444              :     return;
    4445              : 
    4446     97812478 :   if (!bb
    4447     23254396 :       && !BARRIER_P (before))
    4448     23254395 :     bb = BLOCK_FOR_INSN (before);
    4449              : 
    4450     23254396 :   if (bb)
    4451              :     {
    4452     94086585 :       set_block_for_insn (insn, bb);
    4453     94086585 :       if (INSN_P (insn))
    4454     13764508 :         df_insn_rescan (insn);
    4455              :       /* Should not happen as first in the BB is always either NOTE or
    4456              :          LABEL.  */
    4457     94086585 :       gcc_assert (BB_HEAD (bb) != insn
    4458              :                   /* Avoid clobbering of structure when creating new BB.  */
    4459              :                   || BARRIER_P (insn)
    4460              :                   || NOTE_INSN_BASIC_BLOCK_P (insn));
    4461              :     }
    4462              : }
    4463              : 
    4464              : /* Replace insn with an deleted instruction note.  */
    4465              : 
    4466              : void
    4467     14456498 : set_insn_deleted (rtx_insn *insn)
    4468              : {
    4469     14456498 :   if (INSN_P (insn))
    4470     14456498 :     df_insn_delete (insn);
    4471     14456498 :   PUT_CODE (insn, NOTE);
    4472     14456498 :   NOTE_KIND (insn) = NOTE_INSN_DELETED;
    4473     14456498 : }
    4474              : 
    4475              : 
    4476              : /* Unlink INSN from the insn chain.
    4477              : 
    4478              :    This function knows how to handle sequences.
    4479              : 
    4480              :    This function does not invalidate data flow information associated with
    4481              :    INSN (i.e. does not call df_insn_delete).  That makes this function
    4482              :    usable for only disconnecting an insn from the chain, and re-emit it
    4483              :    elsewhere later.
    4484              : 
    4485              :    To later insert INSN elsewhere in the insn chain via add_insn and
    4486              :    similar functions, PREV_INSN and NEXT_INSN must be nullified by
    4487              :    the caller.  Nullifying them here breaks many insn chain walks.
    4488              : 
    4489              :    To really delete an insn and related DF information, use delete_insn.  */
    4490              : 
    4491              : void
    4492    141962512 : remove_insn (rtx_insn *insn)
    4493              : {
    4494    141962512 :   rtx_insn *next = NEXT_INSN (insn);
    4495    141962512 :   rtx_insn *prev = PREV_INSN (insn);
    4496    141962512 :   basic_block bb;
    4497              : 
    4498    141962512 :   if (prev)
    4499              :     {
    4500    141962253 :       SET_NEXT_INSN (prev) = next;
    4501    141962253 :       if (NONJUMP_INSN_P (prev))
    4502     52317896 :         if (rtx_sequence *sequence
    4503     52317896 :             = dyn_cast <rtx_sequence *> (PATTERN (prev)))
    4504            0 :           SET_NEXT_INSN (sequence->insn (sequence->len () - 1)) = next;
    4505              :     }
    4506              :   else
    4507              :     {
    4508              :       struct sequence_stack *seq;
    4509              : 
    4510          259 :       for (seq = get_current_sequence (); seq; seq = seq->next)
    4511          259 :         if (insn == seq->first)
    4512              :           {
    4513          259 :             seq->first = next;
    4514          259 :             break;
    4515              :           }
    4516              : 
    4517          259 :       gcc_assert (seq);
    4518              :     }
    4519              : 
    4520    141962512 :   if (next)
    4521              :     {
    4522    139831287 :       SET_PREV_INSN (next) = prev;
    4523    139831287 :       if (NONJUMP_INSN_P (next))
    4524     47142166 :         if (rtx_sequence *sequence
    4525     47142166 :             = dyn_cast <rtx_sequence *> (PATTERN (next)))
    4526            0 :           SET_PREV_INSN (sequence->insn (0)) = prev;
    4527              :     }
    4528              :   else
    4529              :     {
    4530              :       struct sequence_stack *seq;
    4531              : 
    4532      2131225 :       for (seq = get_current_sequence (); seq; seq = seq->next)
    4533      2131225 :         if (insn == seq->last)
    4534              :           {
    4535      2131225 :             seq->last = prev;
    4536      2131225 :             break;
    4537              :           }
    4538              : 
    4539      2131225 :       gcc_assert (seq);
    4540              :     }
    4541              : 
    4542              :   /* Fix up basic block boundaries, if necessary.  */
    4543    141962512 :   if (!BARRIER_P (insn)
    4544    141962512 :       && (bb = BLOCK_FOR_INSN (insn)))
    4545              :     {
    4546    138892617 :       if (BB_HEAD (bb) == insn)
    4547              :         {
    4548              :           /* Never ever delete the basic block note without deleting whole
    4549              :              basic block.  */
    4550      2481381 :           gcc_assert (!NOTE_P (insn));
    4551      2481381 :           BB_HEAD (bb) = next;
    4552              :         }
    4553    138892617 :       if (BB_END (bb) == insn)
    4554     23873755 :         BB_END (bb) = prev;
    4555              :     }
    4556    141962512 : }
    4557              : 
    4558              : /* Append CALL_FUSAGE to the CALL_INSN_FUNCTION_USAGE for CALL_INSN.  */
    4559              : 
    4560              : void
    4561      6425601 : add_function_usage_to (rtx_insn *call_insn, rtx call_fusage)
    4562              : {
    4563      6425601 :   gcc_assert (call_insn && CALL_P (call_insn));
    4564              : 
    4565              :   /* Put the register usage information on the CALL.  If there is already
    4566              :      some usage information, put ours at the end.  */
    4567      6425601 :   if (CALL_INSN_FUNCTION_USAGE (call_insn))
    4568              :     {
    4569              :       rtx link;
    4570              : 
    4571       572385 :       for (link = CALL_INSN_FUNCTION_USAGE (call_insn); XEXP (link, 1) != 0;
    4572              :            link = XEXP (link, 1))
    4573              :         ;
    4574              : 
    4575       572384 :       XEXP (link, 1) = call_fusage;
    4576              :     }
    4577              :   else
    4578      5853217 :     CALL_INSN_FUNCTION_USAGE (call_insn) = call_fusage;
    4579      6425601 : }
    4580              : 
    4581              : /* Delete all insns made since FROM.
    4582              :    FROM becomes the new last instruction.  */
    4583              : 
    4584              : void
    4585      1332083 : delete_insns_since (rtx_insn *from)
    4586              : {
    4587      1332083 :   if (from == 0)
    4588       143483 :     set_first_insn (0);
    4589              :   else
    4590      1188600 :     SET_NEXT_INSN (from) = 0;
    4591      1332083 :   set_last_insn (from);
    4592      1332083 : }
    4593              : 
    4594              : /* This function is deprecated, please use sequences instead.
    4595              : 
    4596              :    Move a consecutive bunch of insns to a different place in the chain.
    4597              :    The insns to be moved are those between FROM and TO.
    4598              :    They are moved to a new position after the insn AFTER.
    4599              :    AFTER must not be FROM or TO or any insn in between.
    4600              : 
    4601              :    This function does not know about SEQUENCEs and hence should not be
    4602              :    called after delay-slot filling has been done.  */
    4603              : 
    4604              : void
    4605      1836303 : reorder_insns_nobb (rtx_insn *from, rtx_insn *to, rtx_insn *after)
    4606              : {
    4607      1836303 :   if (flag_checking)
    4608              :     {
    4609      2471295 :       for (rtx_insn *x = from; x != to; x = NEXT_INSN (x))
    4610       635015 :         gcc_assert (after != x);
    4611      1836280 :       gcc_assert (after != to);
    4612              :     }
    4613              : 
    4614              :   /* Splice this bunch out of where it is now.  */
    4615      1836303 :   if (PREV_INSN (from))
    4616      1836303 :     SET_NEXT_INSN (PREV_INSN (from)) = NEXT_INSN (to);
    4617      1836303 :   if (NEXT_INSN (to))
    4618      1826494 :     SET_PREV_INSN (NEXT_INSN (to)) = PREV_INSN (from);
    4619      1836303 :   if (get_last_insn () == to)
    4620         9809 :     set_last_insn (PREV_INSN (from));
    4621      1836303 :   if (get_insns () == from)
    4622            0 :     set_first_insn (NEXT_INSN (to));
    4623              : 
    4624              :   /* Make the new neighbors point to it and it to them.  */
    4625      1836303 :   if (NEXT_INSN (after))
    4626      1828396 :     SET_PREV_INSN (NEXT_INSN (after)) = to;
    4627              : 
    4628      1836303 :   SET_NEXT_INSN (to) = NEXT_INSN (after);
    4629      1836303 :   SET_PREV_INSN (from) = after;
    4630      1836303 :   SET_NEXT_INSN (after) = from;
    4631      1836303 :   if (after == get_last_insn ())
    4632         7907 :     set_last_insn (to);
    4633      1836303 : }
    4634              : 
    4635              : /* Same as function above, but take care to update BB boundaries.  */
    4636              : void
    4637      1192734 : reorder_insns (rtx_insn *from, rtx_insn *to, rtx_insn *after)
    4638              : {
    4639      1192734 :   rtx_insn *prev = PREV_INSN (from);
    4640      1192734 :   basic_block bb, bb2;
    4641              : 
    4642      1192734 :   reorder_insns_nobb (from, to, after);
    4643              : 
    4644      1192734 :   if (!BARRIER_P (after)
    4645      1192734 :       && (bb = BLOCK_FOR_INSN (after)))
    4646              :     {
    4647      1185075 :       rtx_insn *x;
    4648      1185075 :       df_set_bb_dirty (bb);
    4649              : 
    4650      1185075 :       if (!BARRIER_P (from)
    4651      1185075 :           && (bb2 = BLOCK_FOR_INSN (from)))
    4652              :         {
    4653      1185075 :           if (BB_END (bb2) == to)
    4654        19481 :             BB_END (bb2) = prev;
    4655      1185075 :           df_set_bb_dirty (bb2);
    4656              :         }
    4657              : 
    4658      1185075 :       if (BB_END (bb) == after)
    4659       339028 :         BB_END (bb) = to;
    4660              : 
    4661      2425575 :       for (x = from; x != NEXT_INSN (to); x = NEXT_INSN (x))
    4662      1240500 :         if (!BARRIER_P (x))
    4663      1240500 :           df_insn_change_bb (x, bb);
    4664              :     }
    4665      1192734 : }
    4666              : 
    4667              : 
    4668              : /* Emit insn(s) of given code and pattern
    4669              :    at a specified place within the doubly-linked list.
    4670              : 
    4671              :    All of the emit_foo global entry points accept an object
    4672              :    X which is either an insn list or a PATTERN of a single
    4673              :    instruction.
    4674              : 
    4675              :    There are thus a few canonical ways to generate code and
    4676              :    emit it at a specific place in the instruction stream.  For
    4677              :    example, consider the instruction named SPOT and the fact that
    4678              :    we would like to emit some instructions before SPOT.  We might
    4679              :    do it like this:
    4680              : 
    4681              :         start_sequence ();
    4682              :         ... emit the new instructions ...
    4683              :         insns_head = end_sequence ();
    4684              : 
    4685              :         emit_insn_before (insns_head, SPOT);
    4686              : 
    4687              :    It used to be common to generate SEQUENCE rtl instead, but that
    4688              :    is a relic of the past which no longer occurs.  The reason is that
    4689              :    SEQUENCE rtl results in much fragmented RTL memory since the SEQUENCE
    4690              :    generated would almost certainly die right after it was created.  */
    4691              : 
    4692              : static rtx_insn *
    4693     12008851 : emit_pattern_before_noloc (rtx x, rtx_insn *before, rtx_insn *last,
    4694              :                            basic_block bb,
    4695              :                            rtx_insn *(*make_raw) (rtx))
    4696              : {
    4697     12008851 :   rtx_insn *insn;
    4698              : 
    4699     12008851 :   gcc_assert (before);
    4700              : 
    4701     12008851 :   if (x == NULL_RTX)
    4702              :     return last;
    4703              : 
    4704     11993326 :   switch (GET_CODE (x))
    4705              :     {
    4706     10469251 :     case DEBUG_INSN:
    4707     10469251 :     case INSN:
    4708     10469251 :     case JUMP_INSN:
    4709     10469251 :     case CALL_INSN:
    4710     10469251 :     case CODE_LABEL:
    4711     10469251 :     case BARRIER:
    4712     10469251 :     case NOTE:
    4713     10469251 :       insn = as_a <rtx_insn *> (x);
    4714     33854264 :       while (insn)
    4715              :         {
    4716     12915762 :           rtx_insn *next = NEXT_INSN (insn);
    4717     12915762 :           add_insn_before (insn, before, bb);
    4718     12915762 :           last = insn;
    4719     12915762 :           insn = next;
    4720              :         }
    4721              :       break;
    4722              : 
    4723              : #ifdef ENABLE_RTL_CHECKING
    4724              :     case SEQUENCE:
    4725              :       gcc_unreachable ();
    4726              :       break;
    4727              : #endif
    4728              : 
    4729      1524075 :     default:
    4730      1524075 :       last = (*make_raw) (x);
    4731      1524075 :       add_insn_before (last, before, bb);
    4732      1524075 :       break;
    4733              :     }
    4734              : 
    4735              :   return last;
    4736              : }
    4737              : 
    4738              : /* Make X be output before the instruction BEFORE.  */
    4739              : 
    4740              : rtx_insn *
    4741       483938 : emit_insn_before_noloc (rtx x, rtx_insn *before, basic_block bb)
    4742              : {
    4743       483938 :   return emit_pattern_before_noloc (x, before, before, bb, make_insn_raw);
    4744              : }
    4745              : 
    4746              : /* Make an instruction with body X and code JUMP_INSN
    4747              :    and output it before the instruction BEFORE.  */
    4748              : 
    4749              : rtx_jump_insn *
    4750            0 : emit_jump_insn_before_noloc (rtx x, rtx_insn *before)
    4751              : {
    4752            0 :   return as_a <rtx_jump_insn *> (
    4753              :                 emit_pattern_before_noloc (x, before, NULL, NULL,
    4754            0 :                                            make_jump_insn_raw));
    4755              : }
    4756              : 
    4757              : /* Make an instruction with body X and code CALL_INSN
    4758              :    and output it before the instruction BEFORE.  */
    4759              : 
    4760              : rtx_insn *
    4761            0 : emit_call_insn_before_noloc (rtx x, rtx_insn *before)
    4762              : {
    4763            0 :   return emit_pattern_before_noloc (x, before, NULL, NULL,
    4764            0 :                                     make_call_insn_raw);
    4765              : }
    4766              : 
    4767              : /* Make an instruction with body X and code DEBUG_INSN
    4768              :    and output it before the instruction BEFORE.  */
    4769              : 
    4770              : rtx_insn *
    4771            0 : emit_debug_insn_before_noloc (rtx x, rtx_insn *before)
    4772              : {
    4773            0 :   return emit_pattern_before_noloc (x, before, NULL, NULL,
    4774            0 :                                     make_debug_insn_raw);
    4775              : }
    4776              : 
    4777              : /* Make an insn of code BARRIER
    4778              :    and output it before the insn BEFORE.  */
    4779              : 
    4780              : rtx_barrier *
    4781            0 : emit_barrier_before (rtx_insn *before)
    4782              : {
    4783            0 :   rtx_barrier *insn = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
    4784              : 
    4785            0 :   INSN_UID (insn) = cur_insn_uid++;
    4786              : 
    4787            0 :   add_insn_before (insn, before, NULL);
    4788            0 :   return insn;
    4789              : }
    4790              : 
    4791              : /* Emit the label LABEL before the insn BEFORE.  */
    4792              : 
    4793              : rtx_code_label *
    4794      6714161 : emit_label_before (rtx_code_label *label, rtx_insn *before)
    4795              : {
    4796      6714161 :   gcc_checking_assert (INSN_UID (label) == 0);
    4797      6714161 :   INSN_UID (label) = cur_insn_uid++;
    4798      6714161 :   add_insn_before (label, before, NULL);
    4799      6714161 :   return label;
    4800              : }
    4801              : 
    4802              : /* Helper for emit_insn_after, handles lists of instructions
    4803              :    efficiently.  */
    4804              : 
    4805              : static rtx_insn *
    4806     24959210 : emit_insn_after_1 (rtx_insn *first, rtx_insn *after, basic_block bb)
    4807              : {
    4808     24959210 :   rtx_insn *last;
    4809     24959210 :   rtx_insn *after_after;
    4810     24959210 :   if (!bb && !BARRIER_P (after))
    4811     20282827 :     bb = BLOCK_FOR_INSN (after);
    4812              : 
    4813     20282827 :   if (bb)
    4814              :     {
    4815     24959115 :       df_set_bb_dirty (bb);
    4816     68912616 :       for (last = first; NEXT_INSN (last); last = NEXT_INSN (last))
    4817     18994386 :         if (!BARRIER_P (last))
    4818              :           {
    4819     18992674 :             set_block_for_insn (last, bb);
    4820     18992674 :             df_insn_rescan (last);
    4821              :           }
    4822     24959115 :       if (!BARRIER_P (last))
    4823              :         {
    4824     24959115 :           set_block_for_insn (last, bb);
    4825     24959115 :           df_insn_rescan (last);
    4826              :         }
    4827     24959115 :       if (BB_END (bb) == after)
    4828      9234171 :         BB_END (bb) = last;
    4829              :     }
    4830              :   else
    4831           95 :     for (last = first; NEXT_INSN (last); last = NEXT_INSN (last))
    4832            0 :       continue;
    4833              : 
    4834     24959210 :   after_after = NEXT_INSN (after);
    4835              : 
    4836     24959210 :   SET_NEXT_INSN (after) = first;
    4837     24959210 :   SET_PREV_INSN (first) = after;
    4838     24959210 :   SET_NEXT_INSN (last) = after_after;
    4839     24959210 :   if (after_after)
    4840     24846455 :     SET_PREV_INSN (after_after) = last;
    4841              : 
    4842     24959210 :   if (after == get_last_insn ())
    4843       112755 :     set_last_insn (last);
    4844              : 
    4845     24959210 :   return last;
    4846              : }
    4847              : 
    4848              : static rtx_insn *
    4849     29989381 : emit_pattern_after_noloc (rtx x, rtx_insn *after, basic_block bb,
    4850              :                           rtx_insn *(*make_raw)(rtx))
    4851              : {
    4852     29989381 :   rtx_insn *last = after;
    4853              : 
    4854     29989381 :   gcc_assert (after);
    4855              : 
    4856     29989381 :   if (x == NULL_RTX)
    4857              :     return last;
    4858              : 
    4859     29423604 :   switch (GET_CODE (x))
    4860              :     {
    4861     24959210 :     case DEBUG_INSN:
    4862     24959210 :     case INSN:
    4863     24959210 :     case JUMP_INSN:
    4864     24959210 :     case CALL_INSN:
    4865     24959210 :     case CODE_LABEL:
    4866     24959210 :     case BARRIER:
    4867     24959210 :     case NOTE:
    4868     24959210 :       last = emit_insn_after_1 (as_a <rtx_insn *> (x), after, bb);
    4869     24959210 :       break;
    4870              : 
    4871              : #ifdef ENABLE_RTL_CHECKING
    4872              :     case SEQUENCE:
    4873              :       gcc_unreachable ();
    4874              :       break;
    4875              : #endif
    4876              : 
    4877      4464394 :     default:
    4878      4464394 :       last = (*make_raw) (x);
    4879      4464394 :       add_insn_after (last, after, bb);
    4880      4464394 :       break;
    4881              :     }
    4882              : 
    4883              :   return last;
    4884              : }
    4885              : 
    4886              : /* Make X be output after the insn AFTER and set the BB of insn.  If
    4887              :    BB is NULL, an attempt is made to infer the BB from AFTER.  */
    4888              : 
    4889              : rtx_insn *
    4890      4918061 : emit_insn_after_noloc (rtx x, rtx_insn *after, basic_block bb)
    4891              : {
    4892      4918061 :   return emit_pattern_after_noloc (x, after, bb, make_insn_raw);
    4893              : }
    4894              : 
    4895              : 
    4896              : /* Make an insn of code JUMP_INSN with body X
    4897              :    and output it after the insn AFTER.  */
    4898              : 
    4899              : rtx_jump_insn *
    4900         2313 : emit_jump_insn_after_noloc (rtx x, rtx_insn *after)
    4901              : {
    4902         2313 :   return as_a <rtx_jump_insn *> (
    4903         2313 :                 emit_pattern_after_noloc (x, after, NULL, make_jump_insn_raw));
    4904              : }
    4905              : 
    4906              : /* Make an instruction with body X and code CALL_INSN
    4907              :    and output it after the instruction AFTER.  */
    4908              : 
    4909              : rtx_insn *
    4910            0 : emit_call_insn_after_noloc (rtx x, rtx_insn *after)
    4911              : {
    4912            0 :   return emit_pattern_after_noloc (x, after, NULL, make_call_insn_raw);
    4913              : }
    4914              : 
    4915              : /* Make an instruction with body X and code CALL_INSN
    4916              :    and output it after the instruction AFTER.  */
    4917              : 
    4918              : rtx_insn *
    4919            0 : emit_debug_insn_after_noloc (rtx x, rtx_insn *after)
    4920              : {
    4921            0 :   return emit_pattern_after_noloc (x, after, NULL, make_debug_insn_raw);
    4922              : }
    4923              : 
    4924              : /* Make an insn of code BARRIER
    4925              :    and output it after the insn AFTER.  */
    4926              : 
    4927              : rtx_barrier *
    4928      7820528 : emit_barrier_after (rtx_insn *after)
    4929              : {
    4930      7820528 :   rtx_barrier *insn = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
    4931              : 
    4932      7820528 :   INSN_UID (insn) = cur_insn_uid++;
    4933              : 
    4934      7820528 :   add_insn_after (insn, after, NULL);
    4935      7820528 :   return insn;
    4936              : }
    4937              : 
    4938              : /* Emit the label LABEL after the insn AFTER.  */
    4939              : 
    4940              : rtx_insn *
    4941            0 : emit_label_after (rtx_insn *label, rtx_insn *after)
    4942              : {
    4943            0 :   gcc_checking_assert (INSN_UID (label) == 0);
    4944            0 :   INSN_UID (label) = cur_insn_uid++;
    4945            0 :   add_insn_after (label, after, NULL);
    4946            0 :   return label;
    4947              : }
    4948              : 
    4949              : /* Notes require a bit of special handling: Some notes need to have their
    4950              :    BLOCK_FOR_INSN set, others should never have it set, and some should
    4951              :    have it set or clear depending on the context.   */
    4952              : 
    4953              : /* Return true iff a note of kind SUBTYPE should be emitted with routines
    4954              :    that never set BLOCK_FOR_INSN on NOTE.  BB_BOUNDARY is true if the
    4955              :    caller is asked to emit a note before BB_HEAD, or after BB_END.  */
    4956              : 
    4957              : static bool
    4958    159253382 : note_outside_basic_block_p (enum insn_note subtype, bool on_bb_boundary_p)
    4959              : {
    4960            0 :   switch (subtype)
    4961              :     {
    4962              :       /* NOTE_INSN_SWITCH_TEXT_SECTIONS only appears between basic blocks.  */
    4963              :       case NOTE_INSN_SWITCH_TEXT_SECTIONS:
    4964              :         return true;
    4965              : 
    4966              :       /* Notes for var tracking and EH region markers can appear between or
    4967              :          inside basic blocks.  If the caller is emitting on the basic block
    4968              :          boundary, do not set BLOCK_FOR_INSN on the new note.  */
    4969     66960888 :       case NOTE_INSN_VAR_LOCATION:
    4970     66960888 :       case NOTE_INSN_EH_REGION_BEG:
    4971     66960888 :       case NOTE_INSN_EH_REGION_END:
    4972            0 :         return on_bb_boundary_p;
    4973              : 
    4974              :       /* Otherwise, BLOCK_FOR_INSN must be set.  */
    4975            0 :       default:
    4976            0 :         return false;
    4977              :     }
    4978              : }
    4979              : 
    4980              : /* Emit a note of subtype SUBTYPE after the insn AFTER.  */
    4981              : 
    4982              : rtx_note *
    4983     59108096 : emit_note_after (enum insn_note subtype, rtx_insn *after)
    4984              : {
    4985     59108096 :   rtx_note *note = make_note_raw (subtype);
    4986     59108096 :   basic_block bb = BARRIER_P (after) ? NULL : BLOCK_FOR_INSN (after);
    4987     59077071 :   bool on_bb_boundary_p = (bb != NULL && BB_END (bb) == after);
    4988              : 
    4989     59108096 :   if (note_outside_basic_block_p (subtype, on_bb_boundary_p))
    4990      1452236 :     add_insn_after_nobb (note, after);
    4991              :   else
    4992     57655860 :     add_insn_after (note, after, bb);
    4993     59108096 :   return note;
    4994              : }
    4995              : 
    4996              : /* Emit a note of subtype SUBTYPE before the insn BEFORE.  */
    4997              : 
    4998              : rtx_note *
    4999    100145286 : emit_note_before (enum insn_note subtype, rtx_insn *before)
    5000              : {
    5001    100145286 :   rtx_note *note = make_note_raw (subtype);
    5002    100145286 :   basic_block bb = BARRIER_P (before) ? NULL : BLOCK_FOR_INSN (before);
    5003    100145285 :   bool on_bb_boundary_p = (bb != NULL && BB_HEAD (bb) == before);
    5004              : 
    5005    100145286 :   if (note_outside_basic_block_p (subtype, on_bb_boundary_p))
    5006     23486689 :     add_insn_before_nobb (note, before);
    5007              :   else
    5008     76658597 :     add_insn_before (note, before, bb);
    5009    100145286 :   return note;
    5010              : }
    5011              : 
    5012              : /* Insert PATTERN after AFTER, setting its INSN_LOCATION to LOC.
    5013              :    MAKE_RAW indicates how to turn PATTERN into a real insn.  */
    5014              : 
    5015              : static rtx_insn *
    5016     22869349 : emit_pattern_after_setloc (rtx pattern, rtx_insn *after, location_t loc,
    5017              :                            rtx_insn *(*make_raw) (rtx))
    5018              : {
    5019     22869349 :   rtx_insn *last = emit_pattern_after_noloc (pattern, after, NULL, make_raw);
    5020              : 
    5021     22869349 :   if (pattern == NULL_RTX || !loc)
    5022              :     return last;
    5023              : 
    5024     16144285 :   after = NEXT_INSN (after);
    5025      4422025 :   while (1)
    5026              :     {
    5027     20566310 :       if (active_insn_p (after)
    5028     19420333 :           && !JUMP_TABLE_DATA_P (after) /* FIXME */
    5029     39986643 :           && !INSN_LOCATION (after))
    5030     19414208 :         INSN_LOCATION (after) = loc;
    5031     20566310 :       if (after == last)
    5032              :         break;
    5033      4422025 :       after = NEXT_INSN (after);
    5034              :     }
    5035              :   return last;
    5036              : }
    5037              : 
    5038              : /* Insert PATTERN after AFTER.  MAKE_RAW indicates how to turn PATTERN
    5039              :    into a real insn.  SKIP_DEBUG_INSNS indicates whether to insert after
    5040              :    any DEBUG_INSNs.  */
    5041              : 
    5042              : static rtx_insn *
    5043     11125350 : emit_pattern_after (rtx pattern, rtx_insn *after, bool skip_debug_insns,
    5044              :                     rtx_insn *(*make_raw) (rtx))
    5045              : {
    5046     11125350 :   rtx_insn *prev = after;
    5047              : 
    5048     11125350 :   if (skip_debug_insns)
    5049     11712370 :     while (DEBUG_INSN_P (prev))
    5050      2073175 :       prev = PREV_INSN (prev);
    5051              : 
    5052     11125350 :   if (INSN_P (prev))
    5053      8925692 :     return emit_pattern_after_setloc (pattern, after, INSN_LOCATION (prev),
    5054      8925692 :                                       make_raw);
    5055              :   else
    5056      2199658 :     return emit_pattern_after_noloc (pattern, after, NULL, make_raw);
    5057              : }
    5058              : 
    5059              : /* Like emit_insn_after_noloc, but set INSN_LOCATION according to LOC.  */
    5060              : rtx_insn *
    5061      8577765 : emit_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
    5062              : {
    5063      8577765 :   return emit_pattern_after_setloc (pattern, after, loc, make_insn_raw);
    5064              : }
    5065              : 
    5066              : /* Like emit_insn_after_noloc, but set INSN_LOCATION according to AFTER.  */
    5067              : rtx_insn *
    5068      8996612 : emit_insn_after (rtx pattern, rtx_insn *after)
    5069              : {
    5070      8996612 :   return emit_pattern_after (pattern, after, true, make_insn_raw);
    5071              : }
    5072              : 
    5073              : /* Like emit_jump_insn_after_noloc, but set INSN_LOCATION according to LOC.  */
    5074              : rtx_jump_insn *
    5075      5365892 : emit_jump_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
    5076              : {
    5077      5365892 :   return as_a <rtx_jump_insn *> (
    5078      5365892 :         emit_pattern_after_setloc (pattern, after, loc, make_jump_insn_raw));
    5079              : }
    5080              : 
    5081              : /* Like emit_jump_insn_after_noloc, but set INSN_LOCATION according to AFTER.  */
    5082              : rtx_jump_insn *
    5083       617873 : emit_jump_insn_after (rtx pattern, rtx_insn *after)
    5084              : {
    5085       617873 :   return as_a <rtx_jump_insn *> (
    5086       617873 :         emit_pattern_after (pattern, after, true, make_jump_insn_raw));
    5087              : }
    5088              : 
    5089              : /* Like emit_call_insn_after_noloc, but set INSN_LOCATION according to LOC.  */
    5090              : rtx_insn *
    5091            0 : emit_call_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
    5092              : {
    5093            0 :   return emit_pattern_after_setloc (pattern, after, loc, make_call_insn_raw);
    5094              : }
    5095              : 
    5096              : /* Like emit_call_insn_after_noloc, but set INSN_LOCATION according to AFTER.  */
    5097              : rtx_insn *
    5098        24710 : emit_call_insn_after (rtx pattern, rtx_insn *after)
    5099              : {
    5100        24710 :   return emit_pattern_after (pattern, after, true, make_call_insn_raw);
    5101              : }
    5102              : 
    5103              : /* Like emit_debug_insn_after_noloc, but set INSN_LOCATION according to LOC.  */
    5104              : rtx_insn *
    5105            0 : emit_debug_insn_after_setloc (rtx pattern, rtx_insn *after, location_t loc)
    5106              : {
    5107            0 :   return emit_pattern_after_setloc (pattern, after, loc, make_debug_insn_raw);
    5108              : }
    5109              : 
    5110              : /* Like emit_debug_insn_after_noloc, but set INSN_LOCATION according to AFTER.  */
    5111              : rtx_insn *
    5112      1486155 : emit_debug_insn_after (rtx pattern, rtx_insn *after)
    5113              : {
    5114      1486155 :   return emit_pattern_after (pattern, after, false, make_debug_insn_raw);
    5115              : }
    5116              : 
    5117              : /* Insert PATTERN before BEFORE, setting its INSN_LOCATION to LOC.
    5118              :    MAKE_RAW indicates how to turn PATTERN into a real insn.  INSNP
    5119              :    indicates if PATTERN is meant for an INSN as opposed to a JUMP_INSN,
    5120              :    CALL_INSN, etc.  */
    5121              : 
    5122              : static rtx_insn *
    5123     10933415 : emit_pattern_before_setloc (rtx pattern, rtx_insn *before, location_t loc,
    5124              :                             bool insnp, rtx_insn *(*make_raw) (rtx))
    5125              : {
    5126     10933415 :   rtx_insn *first = PREV_INSN (before);
    5127     11278680 :   rtx_insn *last = emit_pattern_before_noloc (pattern, before,
    5128              :                                               insnp ? before : NULL,
    5129              :                                               NULL, make_raw);
    5130              : 
    5131     10933415 :   if (pattern == NULL_RTX || !loc)
    5132              :     return last;
    5133              : 
    5134      9917432 :   if (!first)
    5135        91356 :     first = get_insns ();
    5136              :   else
    5137      9826076 :     first = NEXT_INSN (first);
    5138      1693777 :   while (1)
    5139              :     {
    5140     11611209 :       if (active_insn_p (first)
    5141     11028039 :           && !JUMP_TABLE_DATA_P (first) /* FIXME */
    5142     22639248 :           && !INSN_LOCATION (first))
    5143     10779807 :         INSN_LOCATION (first) = loc;
    5144     11611209 :       if (first == last)
    5145              :         break;
    5146      1693777 :       first = NEXT_INSN (first);
    5147              :     }
    5148              :   return last;
    5149              : }
    5150              : 
    5151              : /* Insert PATTERN before BEFORE.  MAKE_RAW indicates how to turn PATTERN
    5152              :    into a real insn.  SKIP_DEBUG_INSNS indicates whether to insert
    5153              :    before any DEBUG_INSNs.  INSNP indicates if PATTERN is meant for an
    5154              :    INSN as opposed to a JUMP_INSN, CALL_INSN, etc.  */
    5155              : 
    5156              : static rtx_insn *
    5157     11124363 : emit_pattern_before (rtx pattern, rtx_insn *before, bool skip_debug_insns,
    5158              :                      bool insnp, rtx_insn *(*make_raw) (rtx))
    5159              : {
    5160     11124363 :   rtx_insn *next = before;
    5161              : 
    5162     11124363 :   if (skip_debug_insns)
    5163     10788072 :     while (DEBUG_INSN_P (next))
    5164           34 :       next = PREV_INSN (next);
    5165              : 
    5166     11124363 :   if (INSN_P (next))
    5167     10532865 :     return emit_pattern_before_setloc (pattern, before, INSN_LOCATION (next),
    5168     10532865 :                                        insnp, make_raw);
    5169              :   else
    5170       592664 :     return emit_pattern_before_noloc (pattern, before,
    5171              :                                       insnp ? before : NULL,
    5172       591498 :                                       NULL, make_raw);
    5173              : }
    5174              : 
    5175              : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC.  */
    5176              : rtx_insn *
    5177       400550 : emit_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
    5178              : {
    5179       400550 :   return emit_pattern_before_setloc (pattern, before, loc, true,
    5180       400550 :                                      make_insn_raw);
    5181              : }
    5182              : 
    5183              : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to BEFORE.  */
    5184              : rtx_insn *
    5185     10777932 : emit_insn_before (rtx pattern, rtx_insn *before)
    5186              : {
    5187     10777932 :   return emit_pattern_before (pattern, before, true, true, make_insn_raw);
    5188              : }
    5189              : 
    5190              : /* like emit_insn_before_noloc, but set INSN_LOCATION according to LOC.  */
    5191              : rtx_jump_insn *
    5192            0 : emit_jump_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
    5193              : {
    5194            0 :   return as_a <rtx_jump_insn *> (
    5195              :         emit_pattern_before_setloc (pattern, before, loc, false,
    5196            0 :                                     make_jump_insn_raw));
    5197              : }
    5198              : 
    5199              : /* Like emit_jump_insn_before_noloc, but set INSN_LOCATION according to BEFORE.  */
    5200              : rtx_jump_insn *
    5201        10106 : emit_jump_insn_before (rtx pattern, rtx_insn *before)
    5202              : {
    5203        10106 :   return as_a <rtx_jump_insn *> (
    5204              :         emit_pattern_before (pattern, before, true, false,
    5205        10106 :                              make_jump_insn_raw));
    5206              : }
    5207              : 
    5208              : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC.  */
    5209              : rtx_insn *
    5210            0 : emit_call_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
    5211              : {
    5212            0 :   return emit_pattern_before_setloc (pattern, before, loc, false,
    5213            0 :                                      make_call_insn_raw);
    5214              : }
    5215              : 
    5216              : /* Like emit_call_insn_before_noloc,
    5217              :    but set insn_location according to BEFORE.  */
    5218              : rtx_insn *
    5219            0 : emit_call_insn_before (rtx pattern, rtx_insn *before)
    5220              : {
    5221            0 :   return emit_pattern_before (pattern, before, true, false,
    5222            0 :                               make_call_insn_raw);
    5223              : }
    5224              : 
    5225              : /* Like emit_insn_before_noloc, but set INSN_LOCATION according to LOC.  */
    5226              : rtx_insn *
    5227            0 : emit_debug_insn_before_setloc (rtx pattern, rtx_insn *before, location_t loc)
    5228              : {
    5229            0 :   return emit_pattern_before_setloc (pattern, before, loc, false,
    5230            0 :                                      make_debug_insn_raw);
    5231              : }
    5232              : 
    5233              : /* Like emit_debug_insn_before_noloc,
    5234              :    but set insn_location according to BEFORE.  */
    5235              : rtx_insn *
    5236       336325 : emit_debug_insn_before (rtx pattern, rtx_insn *before)
    5237              : {
    5238       336325 :   return emit_pattern_before (pattern, before, false, false,
    5239       336325 :                               make_debug_insn_raw);
    5240              : }
    5241              : 
    5242              : /* Take X and emit it at the end of the doubly-linked
    5243              :    INSN list.
    5244              : 
    5245              :    Returns the last insn emitted.  */
    5246              : 
    5247              : rtx_insn *
    5248    251106802 : emit_insn (rtx x)
    5249              : {
    5250    251106802 :   rtx_insn *last = get_last_insn ();
    5251    251106802 :   rtx_insn *insn;
    5252              : 
    5253    251106802 :   if (x == NULL_RTX)
    5254              :     return last;
    5255              : 
    5256    246372918 :   switch (GET_CODE (x))
    5257              :     {
    5258    120816983 :     case DEBUG_INSN:
    5259    120816983 :     case INSN:
    5260    120816983 :     case JUMP_INSN:
    5261    120816983 :     case CALL_INSN:
    5262    120816983 :     case CODE_LABEL:
    5263    120816983 :     case BARRIER:
    5264    120816983 :     case NOTE:
    5265    120816983 :       insn = as_a <rtx_insn *> (x);
    5266    410438312 :       while (insn)
    5267              :         {
    5268    168804346 :           rtx_insn *next = NEXT_INSN (insn);
    5269    168804346 :           add_insn (insn);
    5270    168804346 :           last = insn;
    5271    168804346 :           insn = next;
    5272              :         }
    5273              :       break;
    5274              : 
    5275              : #ifdef ENABLE_RTL_CHECKING
    5276              :     case JUMP_TABLE_DATA:
    5277              :     case SEQUENCE:
    5278              :       gcc_unreachable ();
    5279              :       break;
    5280              : #endif
    5281              : 
    5282    125555935 :     default:
    5283    125555935 :       last = make_insn_raw (x);
    5284    125555935 :       add_insn (last);
    5285    125555935 :       break;
    5286              :     }
    5287              : 
    5288              :   return last;
    5289              : }
    5290              : 
    5291              : /* Make an insn of code DEBUG_INSN with pattern X
    5292              :    and add it to the end of the doubly-linked list.  */
    5293              : 
    5294              : rtx_insn *
    5295     52656309 : emit_debug_insn (rtx x)
    5296              : {
    5297     52656309 :   rtx_insn *last = get_last_insn ();
    5298     52656309 :   rtx_insn *insn;
    5299              : 
    5300     52656309 :   if (x == NULL_RTX)
    5301              :     return last;
    5302              : 
    5303     52656309 :   switch (GET_CODE (x))
    5304              :     {
    5305            0 :     case DEBUG_INSN:
    5306            0 :     case INSN:
    5307            0 :     case JUMP_INSN:
    5308            0 :     case CALL_INSN:
    5309            0 :     case CODE_LABEL:
    5310            0 :     case BARRIER:
    5311            0 :     case NOTE:
    5312            0 :       insn = as_a <rtx_insn *> (x);
    5313            0 :       while (insn)
    5314              :         {
    5315            0 :           rtx_insn *next = NEXT_INSN (insn);
    5316            0 :           add_insn (insn);
    5317            0 :           last = insn;
    5318            0 :           insn = next;
    5319              :         }
    5320              :       break;
    5321              : 
    5322              : #ifdef ENABLE_RTL_CHECKING
    5323              :     case JUMP_TABLE_DATA:
    5324              :     case SEQUENCE:
    5325              :       gcc_unreachable ();
    5326              :       break;
    5327              : #endif
    5328              : 
    5329     52656309 :     default:
    5330     52656309 :       last = make_debug_insn_raw (x);
    5331     52656309 :       add_insn (last);
    5332     52656309 :       break;
    5333              :     }
    5334              : 
    5335              :   return last;
    5336              : }
    5337              : 
    5338              : /* Make an insn of code JUMP_INSN with pattern X
    5339              :    and add it to the end of the doubly-linked list.  */
    5340              : 
    5341              : rtx_insn *
    5342     30074868 : emit_jump_insn (rtx x)
    5343              : {
    5344     30074868 :   rtx_insn *last = NULL;
    5345     30074868 :   rtx_insn *insn;
    5346              : 
    5347     30074868 :   switch (GET_CODE (x))
    5348              :     {
    5349     12091069 :     case DEBUG_INSN:
    5350     12091069 :     case INSN:
    5351     12091069 :     case JUMP_INSN:
    5352     12091069 :     case CALL_INSN:
    5353     12091069 :     case CODE_LABEL:
    5354     12091069 :     case BARRIER:
    5355     12091069 :     case NOTE:
    5356     12091069 :       insn = as_a <rtx_insn *> (x);
    5357     45994435 :       while (insn)
    5358              :         {
    5359     21812297 :           rtx_insn *next = NEXT_INSN (insn);
    5360     21812297 :           add_insn (insn);
    5361     21812297 :           last = insn;
    5362     21812297 :           insn = next;
    5363              :         }
    5364              :       break;
    5365              : 
    5366              : #ifdef ENABLE_RTL_CHECKING
    5367              :     case JUMP_TABLE_DATA:
    5368              :     case SEQUENCE:
    5369              :       gcc_unreachable ();
    5370              :       break;
    5371              : #endif
    5372              : 
    5373     17983799 :     default:
    5374     17983799 :       last = make_jump_insn_raw (x);
    5375     17983799 :       add_insn (last);
    5376     17983799 :       break;
    5377              :     }
    5378              : 
    5379     30074868 :   return last;
    5380              : }
    5381              : 
    5382              : /* Make an insn of code JUMP_INSN with pattern X,
    5383              :    add a REG_BR_PROB note that indicates very likely probability,
    5384              :    and add it to the end of the doubly-linked list.  */
    5385              : 
    5386              : rtx_insn *
    5387            0 : emit_likely_jump_insn (rtx x)
    5388              : {
    5389            0 :   rtx_insn *jump = emit_jump_insn (x);
    5390            0 :   add_reg_br_prob_note (jump, profile_probability::very_likely ());
    5391            0 :   return jump;
    5392              : }
    5393              : 
    5394              : /* Make an insn of code JUMP_INSN with pattern X,
    5395              :    add a REG_BR_PROB note that indicates very unlikely probability,
    5396              :    and add it to the end of the doubly-linked list.  */
    5397              : 
    5398              : rtx_insn *
    5399            0 : emit_unlikely_jump_insn (rtx x)
    5400              : {
    5401            0 :   rtx_insn *jump = emit_jump_insn (x);
    5402            0 :   add_reg_br_prob_note (jump, profile_probability::very_unlikely ());
    5403            0 :   return jump;
    5404              : }
    5405              : 
    5406              : /* Make an insn of code CALL_INSN with pattern X
    5407              :    and add it to the end of the doubly-linked list.  */
    5408              : 
    5409              : rtx_insn *
    5410      6459576 : emit_call_insn (rtx x)
    5411              : {
    5412      6459576 :   rtx_insn *insn;
    5413              : 
    5414      6459576 :   switch (GET_CODE (x))
    5415              :     {
    5416         4425 :     case DEBUG_INSN:
    5417         4425 :     case INSN:
    5418         4425 :     case JUMP_INSN:
    5419         4425 :     case CALL_INSN:
    5420         4425 :     case CODE_LABEL:
    5421         4425 :     case BARRIER:
    5422         4425 :     case NOTE:
    5423         4425 :       insn = emit_insn (x);
    5424         4425 :       break;
    5425              : 
    5426              : #ifdef ENABLE_RTL_CHECKING
    5427              :     case SEQUENCE:
    5428              :     case JUMP_TABLE_DATA:
    5429              :       gcc_unreachable ();
    5430              :       break;
    5431              : #endif
    5432              : 
    5433      6455151 :     default:
    5434      6455151 :       insn = make_call_insn_raw (x);
    5435      6455151 :       add_insn (insn);
    5436      6455151 :       break;
    5437              :     }
    5438              : 
    5439      6459576 :   return insn;
    5440              : }
    5441              : 
    5442              : /* Add the label LABEL to the end of the doubly-linked list.  */
    5443              : 
    5444              : rtx_code_label *
    5445      8929409 : emit_label (rtx uncast_label)
    5446              : {
    5447      8929409 :   rtx_code_label *label = as_a <rtx_code_label *> (uncast_label);
    5448              : 
    5449      8929409 :   gcc_checking_assert (INSN_UID (label) == 0);
    5450      8929409 :   INSN_UID (label) = cur_insn_uid++;
    5451      8929409 :   add_insn (label);
    5452      8929409 :   return label;
    5453              : }
    5454              : 
    5455              : /* Make an insn of code JUMP_TABLE_DATA
    5456              :    and add it to the end of the doubly-linked list.  */
    5457              : 
    5458              : rtx_jump_table_data *
    5459         7659 : emit_jump_table_data (rtx table)
    5460              : {
    5461         7659 :   rtx_jump_table_data *jump_table_data =
    5462         7659 :     as_a <rtx_jump_table_data *> (rtx_alloc (JUMP_TABLE_DATA));
    5463         7659 :   INSN_UID (jump_table_data) = cur_insn_uid++;
    5464         7659 :   PATTERN (jump_table_data) = table;
    5465         7659 :   BLOCK_FOR_INSN (jump_table_data) = NULL;
    5466         7659 :   add_insn (jump_table_data);
    5467         7659 :   return jump_table_data;
    5468              : }
    5469              : 
    5470              : /* Make an insn of code BARRIER
    5471              :    and add it to the end of the doubly-linked list.  */
    5472              : 
    5473              : rtx_barrier *
    5474      3636539 : emit_barrier (void)
    5475              : {
    5476      3636539 :   rtx_barrier *barrier = as_a <rtx_barrier *> (rtx_alloc (BARRIER));
    5477      3636539 :   INSN_UID (barrier) = cur_insn_uid++;
    5478      3636539 :   add_insn (barrier);
    5479      3636539 :   return barrier;
    5480              : }
    5481              : 
    5482              : /* Emit a copy of note ORIG.  */
    5483              : 
    5484              : rtx_note *
    5485       176817 : emit_note_copy (rtx_note *orig)
    5486              : {
    5487       176817 :   enum insn_note kind = (enum insn_note) NOTE_KIND (orig);
    5488       176817 :   rtx_note *note = make_note_raw (kind);
    5489       176817 :   NOTE_DATA (note) = NOTE_DATA (orig);
    5490       176817 :   add_insn (note);
    5491       176817 :   return note;
    5492              : }
    5493              : 
    5494              : /* Make an insn of code NOTE or type NOTE_NO
    5495              :    and add it to the end of the doubly-linked list.  */
    5496              : 
    5497              : rtx_note *
    5498     16902557 : emit_note (enum insn_note kind)
    5499              : {
    5500     16902557 :   rtx_note *note = make_note_raw (kind);
    5501     16902557 :   add_insn (note);
    5502     16902557 :   return note;
    5503              : }
    5504              : 
    5505              : /* Emit a clobber of lvalue X.  */
    5506              : 
    5507              : rtx_insn *
    5508       552654 : emit_clobber (rtx x)
    5509              : {
    5510              :   /* CONCATs should not appear in the insn stream.  */
    5511       552654 :   if (GET_CODE (x) == CONCAT)
    5512              :     {
    5513            0 :       emit_clobber (XEXP (x, 0));
    5514            0 :       return emit_clobber (XEXP (x, 1));
    5515              :     }
    5516       552654 :   return emit_insn (gen_rtx_CLOBBER (VOIDmode, x));
    5517              : }
    5518              : 
    5519              : /* Return a sequence of insns to clobber lvalue X.  */
    5520              : 
    5521              : rtx_insn *
    5522            0 : gen_clobber (rtx x)
    5523              : {
    5524            0 :   rtx_insn *seq;
    5525              : 
    5526            0 :   start_sequence ();
    5527            0 :   emit_clobber (x);
    5528            0 :   seq = end_sequence ();
    5529            0 :   return seq;
    5530              : }
    5531              : 
    5532              : /* Emit a use of rvalue X.  */
    5533              : 
    5534              : rtx_insn *
    5535       811460 : emit_use (rtx x)
    5536              : {
    5537              :   /* CONCATs should not appear in the insn stream.  */
    5538       811460 :   if (GET_CODE (x) == CONCAT)
    5539              :     {
    5540            0 :       emit_use (XEXP (x, 0));
    5541            0 :       return emit_use (XEXP (x, 1));
    5542              :     }
    5543       811460 :   return emit_insn (gen_rtx_USE (VOIDmode, x));
    5544              : }
    5545              : 
    5546              : /* Return a sequence of insns to use rvalue X.  */
    5547              : 
    5548              : rtx_insn *
    5549            0 : gen_use (rtx x)
    5550              : {
    5551            0 :   rtx_insn *seq;
    5552              : 
    5553            0 :   start_sequence ();
    5554            0 :   emit_use (x);
    5555            0 :   seq = end_sequence ();
    5556            0 :   return seq;
    5557              : }
    5558              : 
    5559              : /* Notes like REG_EQUAL and REG_EQUIV refer to a set in an instruction.
    5560              :    Return the set in INSN that such notes describe, or NULL if the notes
    5561              :    have no meaning for INSN.  */
    5562              : 
    5563              : rtx
    5564    240038716 : set_for_reg_notes (rtx_insn *insn)
    5565              : {
    5566    240038716 :   rtx pat, reg;
    5567              : 
    5568    240038716 :   if (!INSN_P (insn))
    5569              :     return NULL_RTX;
    5570              : 
    5571    240033191 :   pat = PATTERN (insn);
    5572    240033191 :   if (GET_CODE (pat) == PARALLEL)
    5573              :     {
    5574              :       /* We do not use single_set because that ignores SETs of unused
    5575              :          registers.  REG_EQUAL and REG_EQUIV notes really do require the
    5576              :          PARALLEL to have a single SET.  */
    5577     19015698 :       if (multiple_sets (insn))
    5578              :         return NULL_RTX;
    5579     18292684 :       pat = XVECEXP (pat, 0, 0);
    5580              :     }
    5581              : 
    5582    239310177 :   if (GET_CODE (pat) != SET)
    5583              :     return NULL_RTX;
    5584              : 
    5585    132548293 :   reg = SET_DEST (pat);
    5586              : 
    5587              :   /* Notes apply to the contents of a STRICT_LOW_PART.  */
    5588    132548293 :   if (GET_CODE (reg) == STRICT_LOW_PART
    5589    132541858 :       || GET_CODE (reg) == ZERO_EXTRACT)
    5590         8564 :     reg = XEXP (reg, 0);
    5591              : 
    5592              :   /* Check that we have a register.  */
    5593    132548293 :   if (!(REG_P (reg) || GET_CODE (reg) == SUBREG))
    5594     35302957 :     return NULL_RTX;
    5595              : 
    5596              :   return pat;
    5597              : }
    5598              : 
    5599              : /* Place a note of KIND on insn INSN with DATUM as the datum. If a
    5600              :    note of this type already exists, remove it first.  */
    5601              : 
    5602              : rtx
    5603     23973855 : set_unique_reg_note (rtx_insn *insn, enum reg_note kind, rtx datum)
    5604              : {
    5605     23973855 :   rtx note = find_reg_note (insn, kind, NULL_RTX);
    5606              : 
    5607     23973855 :   switch (kind)
    5608              :     {
    5609     23973855 :     case REG_EQUAL:
    5610     23973855 :     case REG_EQUIV:
    5611              :       /* We need to support the REG_EQUAL on USE trick of find_reloads.  */
    5612     23973855 :       if (!set_for_reg_notes (insn) && GET_CODE (PATTERN (insn)) != USE)
    5613              :         return NULL_RTX;
    5614              : 
    5615              :       /* Don't add ASM_OPERAND REG_EQUAL/REG_EQUIV notes.
    5616              :          It serves no useful purpose and breaks eliminate_regs.  */
    5617     23939584 :       if (GET_CODE (datum) == ASM_OPERANDS)
    5618              :         return NULL_RTX;
    5619              : 
    5620              :       /* Notes with side effects are dangerous.  Even if the side-effect
    5621              :          initially mirrors one in PATTERN (INSN), later optimizations
    5622              :          might alter the way that the final register value is calculated
    5623              :          and so move or alter the side-effect in some way.  The note would
    5624              :          then no longer be a valid substitution for SET_SRC.  */
    5625     23929267 :       if (side_effects_p (datum))
    5626              :         return NULL_RTX;
    5627              :       break;
    5628              : 
    5629              :     default:
    5630              :       break;
    5631              :     }
    5632              : 
    5633     23928801 :   if (note)
    5634      8109476 :     XEXP (note, 0) = datum;
    5635              :   else
    5636              :     {
    5637     15819325 :       add_reg_note (insn, kind, datum);
    5638     15819325 :       note = REG_NOTES (insn);
    5639              :     }
    5640              : 
    5641     23928801 :   switch (kind)
    5642              :     {
    5643     23928801 :     case REG_EQUAL:
    5644     23928801 :     case REG_EQUIV:
    5645     23928801 :       df_notes_rescan (insn);
    5646     23928801 :       break;
    5647              :     default:
    5648              :       break;
    5649              :     }
    5650              : 
    5651              :   return note;
    5652              : }
    5653              : 
    5654              : /* Like set_unique_reg_note, but don't do anything unless INSN sets DST.  */
    5655              : rtx
    5656      1408109 : set_dst_reg_note (rtx_insn *insn, enum reg_note kind, rtx datum, rtx dst)
    5657              : {
    5658      1408109 :   rtx set = set_for_reg_notes (insn);
    5659              : 
    5660      1408109 :   if (set && SET_DEST (set) == dst)
    5661      1391222 :     return set_unique_reg_note (insn, kind, datum);
    5662              :   return NULL_RTX;
    5663              : }
    5664              : 
    5665              : /* Emit the rtl pattern X as an appropriate kind of insn.  Also emit a
    5666              :    following barrier if the instruction needs one and if ALLOW_BARRIER_P
    5667              :    is true.
    5668              : 
    5669              :    If X is a label, it is simply added into the insn chain.  */
    5670              : 
    5671              : rtx_insn *
    5672     19016275 : emit (rtx x, bool allow_barrier_p)
    5673              : {
    5674     19016275 :   enum rtx_code code = classify_insn (x);
    5675              : 
    5676     19016275 :   switch (code)
    5677              :     {
    5678            0 :     case CODE_LABEL:
    5679            0 :       return emit_label (x);
    5680      9402658 :     case INSN:
    5681      9402658 :       return emit_insn (x);
    5682      9607355 :     case  JUMP_INSN:
    5683      9607355 :       {
    5684      9607355 :         rtx_insn *insn = emit_jump_insn (x);
    5685      9607355 :         if (allow_barrier_p
    5686      9607355 :             && (any_uncondjump_p (insn) || GET_CODE (x) == RETURN))
    5687            0 :           return emit_barrier ();
    5688              :         return insn;
    5689              :       }
    5690         6262 :     case CALL_INSN:
    5691         6262 :       return emit_call_insn (x);
    5692            0 :     case DEBUG_INSN:
    5693            0 :       return emit_debug_insn (x);
    5694            0 :     default:
    5695            0 :       gcc_unreachable ();
    5696              :     }
    5697              : }
    5698              : 
    5699              : /* Space for free sequence stack entries.  */
    5700              : static GTY ((deletable)) struct sequence_stack *free_sequence_stack;
    5701              : 
    5702              : /* Begin emitting insns to a sequence.  If this sequence will contain
    5703              :    something that might cause the compiler to pop arguments to function
    5704              :    calls (because those pops have previously been deferred; see
    5705              :    INHIBIT_DEFER_POP for more details), use do_pending_stack_adjust
    5706              :    before calling this function.  That will ensure that the deferred
    5707              :    pops are not accidentally emitted in the middle of this sequence.  */
    5708              : 
    5709              : void
    5710    241225603 : start_sequence (void)
    5711              : {
    5712    241225603 :   struct sequence_stack *tem;
    5713              : 
    5714    241225603 :   if (free_sequence_stack != NULL)
    5715              :     {
    5716    240539070 :       tem = free_sequence_stack;
    5717    240539070 :       free_sequence_stack = tem->next;
    5718              :     }
    5719              :   else
    5720       686533 :     tem = ggc_alloc<sequence_stack> ();
    5721              : 
    5722    241225603 :   tem->next = get_current_sequence ()->next;
    5723    241225603 :   tem->first = get_insns ();
    5724    241225603 :   tem->last = get_last_insn ();
    5725    241225603 :   get_current_sequence ()->next = tem;
    5726              : 
    5727    241225603 :   set_first_insn (0);
    5728    241225603 :   set_last_insn (0);
    5729    241225603 : }
    5730              : 
    5731              : /* Set up the insn chain starting with FIRST as the current sequence,
    5732              :    saving the previously current one.  See the documentation for
    5733              :    start_sequence for more information about how to use this function.  */
    5734              : 
    5735              : void
    5736      5623604 : push_to_sequence (rtx_insn *first)
    5737              : {
    5738      5623604 :   rtx_insn *last;
    5739              : 
    5740      5623604 :   start_sequence ();
    5741              : 
    5742     14839759 :   for (last = first; last && NEXT_INSN (last); last = NEXT_INSN (last))
    5743              :     ;
    5744              : 
    5745      5623604 :   set_first_insn (first);
    5746      5623604 :   set_last_insn (last);
    5747      5623604 : }
    5748              : 
    5749              : /* Like push_to_sequence, but take the last insn as an argument to avoid
    5750              :    looping through the list.  */
    5751              : 
    5752              : void
    5753        79455 : push_to_sequence2 (rtx_insn *first, rtx_insn *last)
    5754              : {
    5755        79455 :   start_sequence ();
    5756              : 
    5757        79455 :   set_first_insn (first);
    5758        79455 :   set_last_insn (last);
    5759        79455 : }
    5760              : 
    5761              : /* Set up the outer-level insn chain
    5762              :    as the current sequence, saving the previously current one.  */
    5763              : 
    5764              : void
    5765          476 : push_topmost_sequence (void)
    5766              : {
    5767          476 :   struct sequence_stack *top;
    5768              : 
    5769          476 :   start_sequence ();
    5770              : 
    5771          476 :   top = get_topmost_sequence ();
    5772          476 :   set_first_insn (top->first);
    5773          476 :   set_last_insn (top->last);
    5774          476 : }
    5775              : 
    5776              : /* After emitting to the outer-level insn chain, update the outer-level
    5777              :    insn chain, and restore the previous saved state.  */
    5778              : 
    5779              : void
    5780          476 : pop_topmost_sequence (void)
    5781              : {
    5782          476 :   struct sequence_stack *top;
    5783              : 
    5784          476 :   top = get_topmost_sequence ();
    5785          476 :   top->first = get_insns ();
    5786          476 :   top->last = get_last_insn ();
    5787              : 
    5788          476 :   end_sequence ();
    5789          476 : }
    5790              : 
    5791              : /* After emitting to a sequence, restore the previous saved state and return
    5792              :    the start of the completed sequence.
    5793              : 
    5794              :    If the compiler might have deferred popping arguments while
    5795              :    generating this sequence, and this sequence will not be immediately
    5796              :    inserted into the instruction stream, use do_pending_stack_adjust
    5797              :    before calling this function.  That will ensure that the deferred
    5798              :    pops are inserted into this sequence, and not into some random
    5799              :    location in the instruction stream.  See INHIBIT_DEFER_POP for more
    5800              :    information about deferred popping of arguments.  */
    5801              : 
    5802              : rtx_insn *
    5803    241225601 : end_sequence (void)
    5804              : {
    5805    241225601 :   rtx_insn *insns = get_insns ();
    5806              : 
    5807    241225601 :   struct sequence_stack *tem = get_current_sequence ()->next;
    5808              : 
    5809    241225601 :   set_first_insn (tem->first);
    5810    241225601 :   set_last_insn (tem->last);
    5811    241225601 :   get_current_sequence ()->next = tem->next;
    5812              : 
    5813    241225601 :   memset (tem, 0, sizeof (*tem));
    5814    241225601 :   tem->next = free_sequence_stack;
    5815    241225601 :   free_sequence_stack = tem;
    5816              : 
    5817    241225601 :   return insns;
    5818              : }
    5819              : 
    5820              : /* Return true if currently emitting into a sequence.  */
    5821              : 
    5822              : bool
    5823      5060669 : in_sequence_p (void)
    5824              : {
    5825      5060669 :   return get_current_sequence ()->next != 0;
    5826              : }
    5827              : 
    5828              : /* Put the various virtual registers into REGNO_REG_RTX.  */
    5829              : 
    5830              : static void
    5831      1745944 : init_virtual_regs (void)
    5832              : {
    5833      1745944 :   regno_reg_rtx[VIRTUAL_INCOMING_ARGS_REGNUM] = virtual_incoming_args_rtx;
    5834      1745944 :   regno_reg_rtx[VIRTUAL_STACK_VARS_REGNUM] = virtual_stack_vars_rtx;
    5835      1745944 :   regno_reg_rtx[VIRTUAL_STACK_DYNAMIC_REGNUM] = virtual_stack_dynamic_rtx;
    5836      1745944 :   regno_reg_rtx[VIRTUAL_OUTGOING_ARGS_REGNUM] = virtual_outgoing_args_rtx;
    5837      1745944 :   regno_reg_rtx[VIRTUAL_CFA_REGNUM] = virtual_cfa_rtx;
    5838      1745944 :   regno_reg_rtx[VIRTUAL_PREFERRED_STACK_BOUNDARY_REGNUM]
    5839      1745944 :     = virtual_preferred_stack_boundary_rtx;
    5840      1745944 : }
    5841              : 
    5842              : 
    5843              : /* Used by copy_insn_1 to avoid copying SCRATCHes more than once.  */
    5844              : static rtx copy_insn_scratch_in[MAX_RECOG_OPERANDS];
    5845              : static rtx copy_insn_scratch_out[MAX_RECOG_OPERANDS];
    5846              : static int copy_insn_n_scratches;
    5847              : 
    5848              : /* When an insn is being copied by copy_insn_1, this is nonzero if we have
    5849              :    copied an ASM_OPERANDS.
    5850              :    In that case, it is the original input-operand vector.  */
    5851              : static rtvec orig_asm_operands_vector;
    5852              : 
    5853              : /* When an insn is being copied by copy_insn_1, this is nonzero if we have
    5854              :    copied an ASM_OPERANDS.
    5855              :    In that case, it is the copied input-operand vector.  */
    5856              : static rtvec copy_asm_operands_vector;
    5857              : 
    5858              : /* Likewise for the constraints vector.  */
    5859              : static rtvec orig_asm_constraints_vector;
    5860              : static rtvec copy_asm_constraints_vector;
    5861              : 
    5862              : /* Recursively create a new copy of an rtx for copy_insn.
    5863              :    This function differs from copy_rtx in that it handles SCRATCHes and
    5864              :    ASM_OPERANDs properly.
    5865              :    Normally, this function is not used directly; use copy_insn as front end.
    5866              :    However, you could first copy an insn pattern with copy_insn and then use
    5867              :    this function afterwards to properly copy any REG_NOTEs containing
    5868              :    SCRATCHes.  */
    5869              : 
    5870              : rtx
    5871     21676893 : copy_insn_1 (rtx orig)
    5872              : {
    5873     21676893 :   rtx copy;
    5874     21676893 :   int i, j;
    5875     21676893 :   RTX_CODE code;
    5876     21676893 :   const char *format_ptr;
    5877              : 
    5878     21676893 :   if (orig == NULL)
    5879              :     return NULL;
    5880              : 
    5881     21675270 :   code = GET_CODE (orig);
    5882              : 
    5883     21675270 :   switch (code)
    5884              :     {
    5885              :     case REG:
    5886              :     case DEBUG_EXPR:
    5887              :     CASE_CONST_ANY:
    5888              :     case SYMBOL_REF:
    5889              :     case CODE_LABEL:
    5890              :     case PC:
    5891              :     case RETURN:
    5892              :     case SIMPLE_RETURN:
    5893              :       return orig;
    5894      1052775 :     case CLOBBER:
    5895              :       /* Share clobbers of hard registers, but do not share pseudo reg
    5896              :          clobbers or clobbers of hard registers that originated as pseudos.
    5897              :          This is needed to allow safe register renaming.  */
    5898      1052775 :       if (REG_P (XEXP (orig, 0))
    5899       452804 :           && HARD_REGISTER_NUM_P (REGNO (XEXP (orig, 0)))
    5900      1505573 :           && HARD_REGISTER_NUM_P (ORIGINAL_REGNO (XEXP (orig, 0))))
    5901              :         return orig;
    5902              :       break;
    5903              : 
    5904              :     case SCRATCH:
    5905        73243 :       for (i = 0; i < copy_insn_n_scratches; i++)
    5906         1688 :         if (copy_insn_scratch_in[i] == orig)
    5907         1660 :           return copy_insn_scratch_out[i];
    5908              :       break;
    5909              : 
    5910       103897 :     case CONST:
    5911       103897 :       if (shared_const_p (orig))
    5912              :         return orig;
    5913              :       break;
    5914              : 
    5915              :       /* A MEM with a constant address is not sharable.  The problem is that
    5916              :          the constant address may need to be reloaded.  If the mem is shared,
    5917              :          then reloading one copy of this mem will cause all copies to appear
    5918              :          to have been reloaded.  */
    5919              : 
    5920              :     default:
    5921              :       break;
    5922              :     }
    5923              : 
    5924              :   /* Copy the various flags, fields, and other information.  We assume
    5925              :      that all fields need copying, and then clear the fields that should
    5926              :      not be copied.  That is the sensible default behavior, and forces
    5927              :      us to explicitly document why we are *not* copying a flag.  */
    5928      9665748 :   copy = shallow_copy_rtx (orig);
    5929              : 
    5930              :   /* We do not copy JUMP, CALL, or FRAME_RELATED for INSNs.  */
    5931      9665748 :   if (INSN_P (orig))
    5932              :     {
    5933            0 :       RTX_FLAG (copy, jump) = 0;
    5934            0 :       RTX_FLAG (copy, call) = 0;
    5935            0 :       RTX_FLAG (copy, frame_related) = 0;
    5936              :     }
    5937              : 
    5938      9665748 :   format_ptr = GET_RTX_FORMAT (GET_CODE (copy));
    5939              : 
    5940     26486732 :   for (i = 0; i < GET_RTX_LENGTH (GET_CODE (copy)); i++)
    5941     16820984 :     switch (*format_ptr++)
    5942              :       {
    5943     13731743 :       case 'e':
    5944     13731743 :         if (XEXP (orig, i) != NULL)
    5945     13709397 :           XEXP (copy, i) = copy_insn_1 (XEXP (orig, i));
    5946              :         break;
    5947              : 
    5948       640072 :       case 'E':
    5949       640072 :       case 'V':
    5950       640072 :         if (XVEC (orig, i) == orig_asm_constraints_vector)
    5951           12 :           XVEC (copy, i) = copy_asm_constraints_vector;
    5952       640060 :         else if (XVEC (orig, i) == orig_asm_operands_vector)
    5953           12 :           XVEC (copy, i) = copy_asm_operands_vector;
    5954       640048 :         else if (XVEC (orig, i) != NULL)
    5955              :           {
    5956       640048 :             XVEC (copy, i) = rtvec_alloc (XVECLEN (orig, i));
    5957      1889733 :             for (j = 0; j < XVECLEN (copy, i); j++)
    5958      1249685 :               XVECEXP (copy, i, j) = copy_insn_1 (XVECEXP (orig, i, j));
    5959              :           }
    5960              :         break;
    5961              : 
    5962              :       case 't':
    5963              :       case 'w':
    5964              :       case 'i':
    5965              :       case 'L':
    5966              :       case 'p':
    5967              :       case 's':
    5968              :       case 'S':
    5969              :       case 'u':
    5970              :       case '0':
    5971              :         /* These are left unchanged.  */
    5972              :         break;
    5973              : 
    5974            0 :       default:
    5975            0 :         gcc_unreachable ();
    5976              :       }
    5977              : 
    5978      9665748 :   if (code == SCRATCH)
    5979              :     {
    5980        71555 :       i = copy_insn_n_scratches++;
    5981        71555 :       gcc_assert (i < MAX_RECOG_OPERANDS);
    5982        71555 :       copy_insn_scratch_in[i] = orig;
    5983        71555 :       copy_insn_scratch_out[i] = copy;
    5984              :     }
    5985      9594193 :   else if (code == ASM_OPERANDS)
    5986              :     {
    5987          226 :       orig_asm_operands_vector = ASM_OPERANDS_INPUT_VEC (orig);
    5988          226 :       copy_asm_operands_vector = ASM_OPERANDS_INPUT_VEC (copy);
    5989          226 :       orig_asm_constraints_vector = ASM_OPERANDS_INPUT_CONSTRAINT_VEC (orig);
    5990          226 :       copy_asm_constraints_vector = ASM_OPERANDS_INPUT_CONSTRAINT_VEC (copy);
    5991              :     }
    5992              : 
    5993              :   return copy;
    5994              : }
    5995              : 
    5996              : /* Create a new copy of an rtx.
    5997              :    This function differs from copy_rtx in that it handles SCRATCHes and
    5998              :    ASM_OPERANDs properly.
    5999              :    INSN doesn't really have to be a full INSN; it could be just the
    6000              :    pattern.  */
    6001              : rtx
    6002      3714332 : copy_insn (rtx insn)
    6003              : {
    6004      3714332 :   copy_insn_n_scratches = 0;
    6005      3714332 :   orig_asm_operands_vector = 0;
    6006      3714332 :   orig_asm_constraints_vector = 0;
    6007      3714332 :   copy_asm_operands_vector = 0;
    6008      3714332 :   copy_asm_constraints_vector = 0;
    6009      3714332 :   return copy_insn_1 (insn);
    6010              : }
    6011              : 
    6012              : /* Return a copy of INSN that can be used in a SEQUENCE delay slot,
    6013              :    on that assumption that INSN itself remains in its original place.  */
    6014              : 
    6015              : rtx_insn *
    6016            0 : copy_delay_slot_insn (rtx_insn *insn)
    6017              : {
    6018              :   /* Copy INSN with its rtx_code, all its notes, location etc.  */
    6019            0 :   insn = as_a <rtx_insn *> (copy_rtx (insn));
    6020            0 :   INSN_UID (insn) = cur_insn_uid++;
    6021            0 :   return insn;
    6022              : }
    6023              : 
    6024              : /* Initialize data structures and variables in this file
    6025              :    before generating rtl for each function.  */
    6026              : 
    6027              : void
    6028      1745944 : init_emit (void)
    6029              : {
    6030      1745944 :   set_first_insn (NULL);
    6031      1745944 :   set_last_insn (NULL);
    6032      1745944 :   if (param_min_nondebug_insn_uid)
    6033              :     cur_insn_uid = param_min_nondebug_insn_uid;
    6034              :   else
    6035      1745944 :     cur_insn_uid = 1;
    6036      1745944 :   cur_debug_insn_uid = 1;
    6037      1745944 :   reg_rtx_no = LAST_VIRTUAL_REGISTER + 1;
    6038      1745944 :   first_label_num = label_num;
    6039      1745944 :   get_current_sequence ()->next = NULL;
    6040              : 
    6041              :   /* Init the tables that describe all the pseudo regs.  */
    6042              : 
    6043      1745944 :   crtl->emit.regno_pointer_align_length = LAST_VIRTUAL_REGISTER + 101;
    6044              : 
    6045      1745944 :   crtl->emit.regno_pointer_align
    6046      1745944 :     = XCNEWVEC (unsigned char, crtl->emit.regno_pointer_align_length);
    6047              : 
    6048      1745944 :   regno_reg_rtx
    6049      1745944 :     = ggc_cleared_vec_alloc<rtx> (crtl->emit.regno_pointer_align_length);
    6050              : 
    6051              :   /* Put copies of all the hard registers into regno_reg_rtx.  */
    6052      1745944 :   memcpy (regno_reg_rtx,
    6053      1745944 :           initial_regno_reg_rtx,
    6054              :           FIRST_PSEUDO_REGISTER * sizeof (rtx));
    6055              : 
    6056              :   /* Put copies of all the virtual register rtx into regno_reg_rtx.  */
    6057      1745944 :   init_virtual_regs ();
    6058              : 
    6059              :   /* Indicate that the virtual registers and stack locations are
    6060              :      all pointers.  */
    6061      1745944 :   REG_POINTER (stack_pointer_rtx) = 1;
    6062      1745944 :   REG_POINTER (frame_pointer_rtx) = 1;
    6063      1745944 :   REG_POINTER (hard_frame_pointer_rtx) = 1;
    6064      1745944 :   REG_POINTER (arg_pointer_rtx) = 1;
    6065              : 
    6066      1745944 :   REG_POINTER (virtual_incoming_args_rtx) = 1;
    6067      1745944 :   REG_POINTER (virtual_stack_vars_rtx) = 1;
    6068      1745944 :   REG_POINTER (virtual_stack_dynamic_rtx) = 1;
    6069      1745944 :   REG_POINTER (virtual_outgoing_args_rtx) = 1;
    6070      1745944 :   REG_POINTER (virtual_cfa_rtx) = 1;
    6071              : 
    6072              : #ifdef STACK_BOUNDARY
    6073      1745944 :   REGNO_POINTER_ALIGN (STACK_POINTER_REGNUM) = STACK_BOUNDARY;
    6074      1745944 :   REGNO_POINTER_ALIGN (FRAME_POINTER_REGNUM) = STACK_BOUNDARY;
    6075      1745944 :   REGNO_POINTER_ALIGN (HARD_FRAME_POINTER_REGNUM) = STACK_BOUNDARY;
    6076      1745944 :   REGNO_POINTER_ALIGN (ARG_POINTER_REGNUM) = STACK_BOUNDARY;
    6077              : 
    6078      1745944 :   REGNO_POINTER_ALIGN (VIRTUAL_INCOMING_ARGS_REGNUM) = STACK_BOUNDARY;
    6079      1745944 :   REGNO_POINTER_ALIGN (VIRTUAL_STACK_VARS_REGNUM) = STACK_BOUNDARY;
    6080      1745944 :   REGNO_POINTER_ALIGN (VIRTUAL_STACK_DYNAMIC_REGNUM) = STACK_BOUNDARY;
    6081      1745944 :   REGNO_POINTER_ALIGN (VIRTUAL_OUTGOING_ARGS_REGNUM) = STACK_BOUNDARY;
    6082              : 
    6083      1745944 :   REGNO_POINTER_ALIGN (VIRTUAL_CFA_REGNUM) = BITS_PER_WORD;
    6084              : #endif
    6085              : 
    6086              : #ifdef INIT_EXPANDERS
    6087              :   INIT_EXPANDERS;
    6088              : #endif
    6089      1745944 : }
    6090              : 
    6091              : /* Return the value of element I of CONST_VECTOR X as a wide_int.  */
    6092              : 
    6093              : wide_int
    6094         1068 : const_vector_int_elt (const_rtx x, unsigned int i)
    6095              : {
    6096              :   /* First handle elements that are directly encoded.  */
    6097         1068 :   machine_mode elt_mode = GET_MODE_INNER (GET_MODE (x));
    6098         1068 :   if (i < (unsigned int) XVECLEN (x, 0))
    6099            0 :     return rtx_mode_t (CONST_VECTOR_ENCODED_ELT (x, i), elt_mode);
    6100              : 
    6101              :   /* Identify the pattern that contains element I and work out the index of
    6102              :      the last encoded element for that pattern.  */
    6103         1068 :   unsigned int encoded_nelts = const_vector_encoded_nelts (x);
    6104         1068 :   unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
    6105         1068 :   unsigned int count = i / npatterns;
    6106         1068 :   unsigned int pattern = i % npatterns;
    6107         1068 :   unsigned int final_i = encoded_nelts - npatterns + pattern;
    6108              : 
    6109              :   /* If there are no steps, the final encoded value is the right one.  */
    6110         1068 :   if (!CONST_VECTOR_STEPPED_P (x))
    6111            0 :     return rtx_mode_t (CONST_VECTOR_ENCODED_ELT (x, final_i), elt_mode);
    6112              : 
    6113              :   /* Otherwise work out the value from the last two encoded elements.  */
    6114         1068 :   rtx v1 = CONST_VECTOR_ENCODED_ELT (x, final_i - npatterns);
    6115         1068 :   rtx v2 = CONST_VECTOR_ENCODED_ELT (x, final_i);
    6116         1068 :   wide_int diff = wi::sub (rtx_mode_t (v2, elt_mode),
    6117         1068 :                            rtx_mode_t (v1, elt_mode));
    6118         1068 :   return wi::add (rtx_mode_t (v2, elt_mode), (count - 2) * diff);
    6119         1068 : }
    6120              : 
    6121              : /* Return the value of element I of CONST_VECTOR X.  */
    6122              : 
    6123              : rtx
    6124      4904583 : const_vector_elt (const_rtx x, unsigned int i)
    6125              : {
    6126              :   /* First handle elements that are directly encoded.  */
    6127      4904583 :   if (i < (unsigned int) XVECLEN (x, 0))
    6128      4902447 :     return CONST_VECTOR_ENCODED_ELT (x, i);
    6129              : 
    6130              :   /* If there are no steps, the final encoded value is the right one.  */
    6131         2136 :   if (!CONST_VECTOR_STEPPED_P (x))
    6132              :     {
    6133              :       /* Identify the pattern that contains element I and work out the index of
    6134              :          the last encoded element for that pattern.  */
    6135         1068 :       unsigned int encoded_nelts = const_vector_encoded_nelts (x);
    6136         1068 :       unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
    6137         1068 :       unsigned int pattern = i % npatterns;
    6138         1068 :       unsigned int final_i = encoded_nelts - npatterns + pattern;
    6139         1068 :       return CONST_VECTOR_ENCODED_ELT (x, final_i);
    6140              :     }
    6141              : 
    6142              :   /* Otherwise work out the value from the last two encoded elements.  */
    6143         1068 :   return immed_wide_int_const (const_vector_int_elt (x, i),
    6144         2136 :                                GET_MODE_INNER (GET_MODE (x)));
    6145              : }
    6146              : 
    6147              : /* Return true if X is a valid element for a CONST_VECTOR of the given
    6148              :   mode.  */
    6149              : 
    6150              : bool
    6151       563234 : valid_for_const_vector_p (machine_mode, rtx x)
    6152              : {
    6153       563234 :   return (CONST_SCALAR_INT_P (x)
    6154              :           || CONST_POLY_INT_P (x)
    6155       164726 :           || CONST_DOUBLE_AS_FLOAT_P (x)
    6156       708274 :           || CONST_FIXED_P (x));
    6157              : }
    6158              : 
    6159              : /* Generate a vector constant of mode MODE in which every element has
    6160              :    value ELT.  */
    6161              : 
    6162              : rtx
    6163     42812169 : gen_const_vec_duplicate (machine_mode mode, rtx elt)
    6164              : {
    6165     42812169 :   rtx_vector_builder builder (mode, 1, 1);
    6166     42812169 :   builder.quick_push (elt);
    6167     42812169 :   return builder.build ();
    6168     42812169 : }
    6169              : 
    6170              : /* Return a vector rtx of mode MODE in which every element has value X.
    6171              :    The result will be a constant if X is constant.  */
    6172              : 
    6173              : rtx
    6174       255068 : gen_vec_duplicate (machine_mode mode, rtx x)
    6175              : {
    6176       255068 :   if (valid_for_const_vector_p (mode, x))
    6177       115812 :     return gen_const_vec_duplicate (mode, x);
    6178       139256 :   return gen_rtx_VEC_DUPLICATE (mode, x);
    6179              : }
    6180              : 
    6181              : /* A subroutine of const_vec_series_p that handles the case in which:
    6182              : 
    6183              :      (GET_CODE (X) == CONST_VECTOR
    6184              :       && CONST_VECTOR_NPATTERNS (X) == 1
    6185              :       && !CONST_VECTOR_DUPLICATE_P (X))
    6186              : 
    6187              :    is known to hold.  */
    6188              : 
    6189              : bool
    6190         5911 : const_vec_series_p_1 (const_rtx x, rtx *base_out, rtx *step_out)
    6191              : {
    6192              :   /* Stepped sequences are only defined for integers, to avoid specifying
    6193              :      rounding behavior.  */
    6194         5911 :   if (GET_MODE_CLASS (GET_MODE (x)) != MODE_VECTOR_INT)
    6195              :     return false;
    6196              : 
    6197              :   /* A non-duplicated vector with two elements can always be seen as a
    6198              :      series with a nonzero step.  Longer vectors must have a stepped
    6199              :      encoding.  */
    6200         5911 :   if (maybe_ne (CONST_VECTOR_NUNITS (x), 2)
    6201         5911 :       && !CONST_VECTOR_STEPPED_P (x))
    6202              :     return false;
    6203              : 
    6204              :   /* Calculate the step between the first and second elements.  */
    6205         5907 :   scalar_mode inner = GET_MODE_INNER (GET_MODE (x));
    6206         5907 :   rtx base = CONST_VECTOR_ELT (x, 0);
    6207        11814 :   rtx step = simplify_binary_operation (MINUS, inner,
    6208         5907 :                                         CONST_VECTOR_ENCODED_ELT (x, 1), base);
    6209         5907 :   if (rtx_equal_p (step, CONST0_RTX (inner)))
    6210              :     return false;
    6211              : 
    6212              :   /* If we have a stepped encoding, check that the step between the
    6213              :      second and third elements is the same as STEP.  */
    6214         5907 :   if (CONST_VECTOR_STEPPED_P (x))
    6215              :     {
    6216         9352 :       rtx diff = simplify_binary_operation (MINUS, inner,
    6217              :                                             CONST_VECTOR_ENCODED_ELT (x, 2),
    6218         4676 :                                             CONST_VECTOR_ENCODED_ELT (x, 1));
    6219         4676 :       if (!rtx_equal_p (step, diff))
    6220              :         return false;
    6221              :     }
    6222              : 
    6223         5907 :   *base_out = base;
    6224         5907 :   *step_out = step;
    6225         5907 :   return true;
    6226              : }
    6227              : 
    6228              : /* Generate a vector constant of mode MODE in which element I has
    6229              :    the value BASE + I * STEP.  */
    6230              : 
    6231              : rtx
    6232          645 : gen_const_vec_series (machine_mode mode, rtx base, rtx step)
    6233              : {
    6234          645 :   gcc_assert (valid_for_const_vector_p (mode, base)
    6235              :               && valid_for_const_vector_p (mode, step));
    6236              : 
    6237          645 :   rtx_vector_builder builder (mode, 1, 3);
    6238          645 :   builder.quick_push (base);
    6239         2580 :   for (int i = 1; i < 3; ++i)
    6240         1290 :     builder.quick_push (simplify_gen_binary (PLUS, GET_MODE_INNER (mode),
    6241         1290 :                                              builder[i - 1], step));
    6242          645 :   return builder.build ();
    6243          645 : }
    6244              : 
    6245              : /* Generate a vector of mode MODE in which element I has the value
    6246              :    BASE + I * STEP.  The result will be a constant if BASE and STEP
    6247              :    are both constants.  */
    6248              : 
    6249              : rtx
    6250         5056 : gen_vec_series (machine_mode mode, rtx base, rtx step)
    6251              : {
    6252         5056 :   if (step == const0_rtx)
    6253          211 :     return gen_vec_duplicate (mode, base);
    6254         4845 :   if (valid_for_const_vector_p (mode, base)
    6255         4845 :       && valid_for_const_vector_p (mode, step))
    6256            0 :     return gen_const_vec_series (mode, base, step);
    6257         4845 :   return gen_rtx_VEC_SERIES (mode, base, step);
    6258              : }
    6259              : 
    6260              : /* Generate a new vector constant for mode MODE and constant value
    6261              :    CONSTANT.  */
    6262              : 
    6263              : static rtx
    6264     41418663 : gen_const_vector (machine_mode mode, int constant)
    6265              : {
    6266     41418663 :   machine_mode inner = GET_MODE_INNER (mode);
    6267              : 
    6268     41418663 :   gcc_assert (!DECIMAL_FLOAT_MODE_P (inner));
    6269              : 
    6270     41418663 :   rtx el = const_tiny_rtx[constant][(int) inner];
    6271     41418663 :   gcc_assert (el);
    6272              : 
    6273     41418663 :   return gen_const_vec_duplicate (mode, el);
    6274              : }
    6275              : 
    6276              : /* Generate a vector like gen_rtx_raw_CONST_VEC, but use the zero vector when
    6277              :    all elements are zero, and the one vector when all elements are one.  */
    6278              : rtx
    6279       167995 : gen_rtx_CONST_VECTOR (machine_mode mode, rtvec v)
    6280              : {
    6281       335990 :   gcc_assert (known_eq (GET_MODE_NUNITS (mode), GET_NUM_ELEM (v)));
    6282              : 
    6283              :   /* If the values are all the same, check to see if we can use one of the
    6284              :      standard constant vectors.  */
    6285       167995 :   if (rtvec_all_equal_p (v))
    6286        48583 :     return gen_const_vec_duplicate (mode, RTVEC_ELT (v, 0));
    6287              : 
    6288       119412 :   unsigned int nunits = GET_NUM_ELEM (v);
    6289       119412 :   rtx_vector_builder builder (mode, nunits, 1);
    6290       815576 :   for (unsigned int i = 0; i < nunits; ++i)
    6291       576752 :     builder.quick_push (RTVEC_ELT (v, i));
    6292       119412 :   return builder.build (v);
    6293       119412 : }
    6294              : 
    6295              : /* Initialise global register information required by all functions.  */
    6296              : 
    6297              : void
    6298       801328 : init_emit_regs (void)
    6299              : {
    6300       801328 :   int i;
    6301       801328 :   machine_mode mode;
    6302       801328 :   mem_attrs *attrs;
    6303              : 
    6304              :   /* Reset register attributes */
    6305       801328 :   reg_attrs_htab->empty ();
    6306              : 
    6307              :   /* We need reg_raw_mode, so initialize the modes now.  */
    6308       801328 :   init_reg_modes_target ();
    6309              : 
    6310              :   /* Assign register numbers to the globally defined register rtx.  */
    6311       816523 :   stack_pointer_rtx = gen_raw_REG (Pmode, STACK_POINTER_REGNUM);
    6312       816523 :   frame_pointer_rtx = gen_raw_REG (Pmode, FRAME_POINTER_REGNUM);
    6313       816523 :   hard_frame_pointer_rtx = gen_raw_REG (Pmode, HARD_FRAME_POINTER_REGNUM);
    6314       816523 :   arg_pointer_rtx = gen_raw_REG (Pmode, ARG_POINTER_REGNUM);
    6315      2403984 :   virtual_incoming_args_rtx =
    6316       816523 :     gen_raw_REG (Pmode, VIRTUAL_INCOMING_ARGS_REGNUM);
    6317      2403984 :   virtual_stack_vars_rtx =
    6318       816523 :     gen_raw_REG (Pmode, VIRTUAL_STACK_VARS_REGNUM);
    6319      2403984 :   virtual_stack_dynamic_rtx =
    6320       816523 :     gen_raw_REG (Pmode, VIRTUAL_STACK_DYNAMIC_REGNUM);
    6321      2403984 :   virtual_outgoing_args_rtx =
    6322       816523 :     gen_raw_REG (Pmode, VIRTUAL_OUTGOING_ARGS_REGNUM);
    6323       816523 :   virtual_cfa_rtx = gen_raw_REG (Pmode, VIRTUAL_CFA_REGNUM);
    6324      2403984 :   virtual_preferred_stack_boundary_rtx =
    6325       816523 :     gen_raw_REG (Pmode, VIRTUAL_PREFERRED_STACK_BOUNDARY_REGNUM);
    6326              : 
    6327              :   /* Initialize RTL for commonly used hard registers.  These are
    6328              :      copied into regno_reg_rtx as we begin to compile each function.  */
    6329     76126160 :   for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
    6330     75324832 :     initial_regno_reg_rtx[i] = gen_raw_REG (reg_raw_mode[i], i);
    6331              : 
    6332              : #ifdef RETURN_ADDRESS_POINTER_REGNUM
    6333              :   return_address_pointer_rtx
    6334              :     = gen_raw_REG (Pmode, RETURN_ADDRESS_POINTER_REGNUM);
    6335              : #endif
    6336              : 
    6337       801328 :   pic_offset_table_rtx = NULL_RTX;
    6338       801328 :   if ((unsigned) PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM)
    6339        20661 :     pic_offset_table_rtx = gen_raw_REG (Pmode, PIC_OFFSET_TABLE_REGNUM);
    6340              : 
    6341              :   /* Process stack-limiting command-line options.  */
    6342       801328 :   if (opt_fstack_limit_symbol_arg != NULL)
    6343            0 :     stack_limit_rtx
    6344            0 :       = gen_rtx_SYMBOL_REF (Pmode, ggc_strdup (opt_fstack_limit_symbol_arg));
    6345       801328 :   if (opt_fstack_limit_register_no >= 0)
    6346            0 :     stack_limit_rtx = gen_rtx_REG (Pmode, opt_fstack_limit_register_no);
    6347              : 
    6348    100166000 :   for (i = 0; i < (int) MAX_MACHINE_MODE; i++)
    6349              :     {
    6350     99364672 :       mode = (machine_mode) i;
    6351     99364672 :       attrs = ggc_cleared_alloc<mem_attrs> ();
    6352     99364672 :       attrs->align = BITS_PER_UNIT;
    6353     99364672 :       attrs->addrspace = ADDR_SPACE_GENERIC;
    6354     99364672 :       if (mode != BLKmode && mode != VOIDmode)
    6355              :         {
    6356     97762016 :           attrs->size_known_p = true;
    6357    195524032 :           attrs->size = GET_MODE_SIZE (mode);
    6358     97762016 :           if (STRICT_ALIGNMENT)
    6359              :             attrs->align = GET_MODE_ALIGNMENT (mode);
    6360              :         }
    6361     99364672 :       mode_mem_attrs[i] = attrs;
    6362              :     }
    6363              : 
    6364       801328 :   split_branch_probability = profile_probability::uninitialized ();
    6365       801328 : }
    6366              : 
    6367              : /* Initialize global machine_mode variables.  */
    6368              : 
    6369              : void
    6370       296879 : init_derived_machine_modes (void)
    6371              : {
    6372       296879 :   opt_scalar_int_mode mode_iter, opt_byte_mode, opt_word_mode;
    6373      2375032 :   FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
    6374              :     {
    6375      2078153 :       scalar_int_mode mode = mode_iter.require ();
    6376              : 
    6377      2375032 :       if (GET_MODE_BITSIZE (mode) == BITS_PER_UNIT
    6378      2078153 :           && !opt_byte_mode.exists ())
    6379       296879 :         opt_byte_mode = mode;
    6380              : 
    6381      4156306 :       if (GET_MODE_BITSIZE (mode) == BITS_PER_WORD
    6382      2078153 :           && !opt_word_mode.exists ())
    6383       296879 :         opt_word_mode = mode;
    6384              :     }
    6385              : 
    6386       296879 :   byte_mode = opt_byte_mode.require ();
    6387       296879 :   word_mode = opt_word_mode.require ();
    6388       296879 :   ptr_mode = as_a <scalar_int_mode>
    6389       311323 :     (mode_for_size (POINTER_SIZE, GET_MODE_CLASS (Pmode), 0).require ());
    6390       296879 : }
    6391              : 
    6392              : /* Create some permanent unique rtl objects shared between all functions.  */
    6393              : 
    6394              : void
    6395       289641 : init_emit_once (void)
    6396              : {
    6397       289641 :   int i;
    6398       289641 :   machine_mode mode;
    6399       289641 :   scalar_float_mode double_mode;
    6400       289641 :   opt_scalar_mode smode_iter;
    6401              : 
    6402              :   /* Initialize the CONST_INT, CONST_WIDE_INT, CONST_DOUBLE,
    6403              :      CONST_FIXED, and memory attribute hash tables.  */
    6404       289641 :   const_int_htab = hash_table<const_int_hasher>::create_ggc (37);
    6405              : 
    6406              : #if TARGET_SUPPORTS_WIDE_INT
    6407       289641 :   const_wide_int_htab = hash_table<const_wide_int_hasher>::create_ggc (37);
    6408              : #endif
    6409       289641 :   const_double_htab = hash_table<const_double_hasher>::create_ggc (37);
    6410              : 
    6411       289641 :   if (NUM_POLY_INT_COEFFS > 1)
    6412              :     const_poly_int_htab = hash_table<const_poly_int_hasher>::create_ggc (37);
    6413              : 
    6414       289641 :   const_fixed_htab = hash_table<const_fixed_hasher>::create_ggc (37);
    6415              : 
    6416       289641 :   reg_attrs_htab = hash_table<reg_attr_hasher>::create_ggc (37);
    6417              : 
    6418              : #ifdef INIT_EXPANDERS
    6419              :   /* This is to initialize {init|mark|free}_machine_status before the first
    6420              :      call to push_function_context_to.  This is needed by the Chill front
    6421              :      end which calls push_function_context_to before the first call to
    6422              :      init_function_start.  */
    6423              :   INIT_EXPANDERS;
    6424              : #endif
    6425              : 
    6426              :   /* Create the unique rtx's for certain rtx codes and operand values.  */
    6427              : 
    6428              :   /* Don't use gen_rtx_CONST_INT here since gen_rtx_CONST_INT in this case
    6429              :      tries to use these variables.  */
    6430     37653330 :   for (i = - MAX_SAVED_CONST_INT; i <= MAX_SAVED_CONST_INT; i++)
    6431     74727378 :     const_int_rtx[i + MAX_SAVED_CONST_INT] =
    6432     37363689 :       gen_rtx_raw_CONST_INT (VOIDmode, (HOST_WIDE_INT) i);
    6433              : 
    6434       289641 :   if (STORE_FLAG_VALUE >= - MAX_SAVED_CONST_INT
    6435              :       && STORE_FLAG_VALUE <= MAX_SAVED_CONST_INT)
    6436       289641 :     const_true_rtx = const_int_rtx[STORE_FLAG_VALUE + MAX_SAVED_CONST_INT];
    6437              :   else
    6438              :     const_true_rtx = gen_rtx_CONST_INT (VOIDmode, STORE_FLAG_VALUE);
    6439              : 
    6440       289641 :   mode = targetm.c.mode_for_floating_type (TI_DOUBLE_TYPE);
    6441       289641 :   double_mode = as_a<scalar_float_mode> (mode);
    6442              : 
    6443       289641 :   real_from_integer (&dconst0, double_mode, 0, SIGNED);
    6444       289641 :   real_from_integer (&dconst1, double_mode, 1, SIGNED);
    6445       289641 :   real_from_integer (&dconst2, double_mode, 2, SIGNED);
    6446              : 
    6447       289641 :   dconstm0 = dconst0;
    6448       289641 :   dconstm0.sign = 1;
    6449              : 
    6450       289641 :   dconstm1 = dconst1;
    6451       289641 :   dconstm1.sign = 1;
    6452              : 
    6453       289641 :   dconsthalf = dconst1;
    6454       289641 :   SET_REAL_EXP (&dconsthalf, REAL_EXP (&dconsthalf) - 1);
    6455              : 
    6456       289641 :   real_inf (&dconstinf);
    6457       289641 :   real_inf (&dconstninf, true);
    6458              : 
    6459      1448205 :   for (i = 0; i < 3; i++)
    6460              :     {
    6461       868923 :       const REAL_VALUE_TYPE *const r =
    6462              :         (i == 0 ? &dconst0 : i == 1 ? &dconst1 : &dconst2);
    6463              : 
    6464      6082461 :       FOR_EACH_MODE_IN_CLASS (mode, MODE_FLOAT)
    6465      5213538 :         const_tiny_rtx[i][(int) mode] =
    6466      5213538 :           const_double_from_real_value (*r, mode);
    6467              : 
    6468      3475692 :       FOR_EACH_MODE_IN_CLASS (mode, MODE_DECIMAL_FLOAT)
    6469      2606769 :         const_tiny_rtx[i][(int) mode] =
    6470      2606769 :           const_double_from_real_value (*r, mode);
    6471              : 
    6472       868923 :       const_tiny_rtx[i][(int) VOIDmode] = GEN_INT (i);
    6473              : 
    6474      6951384 :       FOR_EACH_MODE_IN_CLASS (mode, MODE_INT)
    6475      6082461 :         const_tiny_rtx[i][(int) mode] = GEN_INT (i);
    6476              : 
    6477      2606769 :       for (mode = MIN_MODE_PARTIAL_INT;
    6478      3475692 :            mode <= MAX_MODE_PARTIAL_INT;
    6479      2606769 :            mode = (machine_mode)((int)(mode) + 1))
    6480      2606769 :         const_tiny_rtx[i][(int) mode] = GEN_INT (i);
    6481              :     }
    6482              : 
    6483       289641 :   const_tiny_rtx[3][(int) VOIDmode] = constm1_rtx;
    6484              : 
    6485      2317128 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_INT)
    6486      2027487 :     const_tiny_rtx[3][(int) mode] = constm1_rtx;
    6487              : 
    6488              :   /* For BImode, 1 and -1 are unsigned and signed interpretations
    6489              :      of the same value.  */
    6490       289641 :   for (mode = MIN_MODE_BOOL;
    6491       579282 :        mode <= MAX_MODE_BOOL;
    6492       289641 :        mode = (machine_mode)((int)(mode) + 1))
    6493              :     {
    6494       289641 :       const_tiny_rtx[0][(int) mode] = const0_rtx;
    6495       289641 :       if (mode == BImode)
    6496              :         {
    6497       289641 :           const_tiny_rtx[1][(int) mode] = const_true_rtx;
    6498       289641 :           const_tiny_rtx[3][(int) mode] = const_true_rtx;
    6499              :         }
    6500              :       else
    6501              :         {
    6502              :           const_tiny_rtx[1][(int) mode] = const1_rtx;
    6503              :           const_tiny_rtx[3][(int) mode] = constm1_rtx;
    6504              :         }
    6505              :     }
    6506              : 
    6507       868923 :   for (mode = MIN_MODE_PARTIAL_INT;
    6508      1158564 :        mode <= MAX_MODE_PARTIAL_INT;
    6509       868923 :        mode = (machine_mode)((int)(mode) + 1))
    6510       868923 :     const_tiny_rtx[3][(int) mode] = constm1_rtx;
    6511              : 
    6512      3186051 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_COMPLEX_INT)
    6513              :     {
    6514      2896410 :       rtx inner = const_tiny_rtx[0][(int)GET_MODE_INNER (mode)];
    6515      2896410 :       const_tiny_rtx[0][(int) mode] = gen_rtx_CONCAT (mode, inner, inner);
    6516              :     }
    6517              : 
    6518      2027487 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_COMPLEX_FLOAT)
    6519              :     {
    6520      1737846 :       rtx inner = const_tiny_rtx[0][(int)GET_MODE_INNER (mode)];
    6521      1737846 :       const_tiny_rtx[0][(int) mode] = gen_rtx_CONCAT (mode, inner, inner);
    6522              :     }
    6523              : 
    6524       289641 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_BOOL)
    6525              :     {
    6526            0 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6527            0 :       const_tiny_rtx[3][(int) mode] = gen_const_vector (mode, 3);
    6528            0 :       if (GET_MODE_INNER (mode) == BImode)
    6529              :         /* As for BImode, "all 1" and "all -1" are unsigned and signed
    6530              :            interpretations of the same value.  */
    6531              :         const_tiny_rtx[1][(int) mode] = const_tiny_rtx[3][(int) mode];
    6532              :       else
    6533            0 :         const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
    6534              :     }
    6535              : 
    6536      9268512 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_INT)
    6537              :     {
    6538      8978871 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6539      8978871 :       const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
    6540      8978871 :       const_tiny_rtx[3][(int) mode] = gen_const_vector (mode, 3);
    6541              :     }
    6542              : 
    6543      7530666 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_FLOAT)
    6544              :     {
    6545      7241025 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6546      7241025 :       const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
    6547              :     }
    6548              : 
    6549      1737846 :   FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_FRACT)
    6550              :     {
    6551      1448205 :       scalar_mode smode = smode_iter.require ();
    6552      1448205 :       FCONST0 (smode).data.high = 0;
    6553      1448205 :       FCONST0 (smode).data.low = 0;
    6554      1448205 :       FCONST0 (smode).mode = smode;
    6555      2896410 :       const_tiny_rtx[0][(int) smode]
    6556      1448205 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
    6557              :     }
    6558              : 
    6559      1737846 :   FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_UFRACT)
    6560              :     {
    6561      1448205 :       scalar_mode smode = smode_iter.require ();
    6562      1448205 :       FCONST0 (smode).data.high = 0;
    6563      1448205 :       FCONST0 (smode).data.low = 0;
    6564      1448205 :       FCONST0 (smode).mode = smode;
    6565      2896410 :       const_tiny_rtx[0][(int) smode]
    6566      1448205 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
    6567              :     }
    6568              : 
    6569      1448205 :   FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_ACCUM)
    6570              :     {
    6571      1158564 :       scalar_mode smode = smode_iter.require ();
    6572      1158564 :       FCONST0 (smode).data.high = 0;
    6573      1158564 :       FCONST0 (smode).data.low = 0;
    6574      1158564 :       FCONST0 (smode).mode = smode;
    6575      2317128 :       const_tiny_rtx[0][(int) smode]
    6576      1158564 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
    6577              : 
    6578              :       /* We store the value 1.  */
    6579      1158564 :       FCONST1 (smode).data.high = 0;
    6580      1158564 :       FCONST1 (smode).data.low = 0;
    6581      1158564 :       FCONST1 (smode).mode = smode;
    6582      1158564 :       FCONST1 (smode).data
    6583      1158564 :         = double_int_one.lshift (GET_MODE_FBIT (smode),
    6584              :                                  HOST_BITS_PER_DOUBLE_INT,
    6585      1158564 :                                  SIGNED_FIXED_POINT_MODE_P (smode));
    6586      2317128 :       const_tiny_rtx[1][(int) smode]
    6587      1158564 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST1 (smode), smode);
    6588              :     }
    6589              : 
    6590      1448205 :   FOR_EACH_MODE_IN_CLASS (smode_iter, MODE_UACCUM)
    6591              :     {
    6592      1158564 :       scalar_mode smode = smode_iter.require ();
    6593      1158564 :       FCONST0 (smode).data.high = 0;
    6594      1158564 :       FCONST0 (smode).data.low = 0;
    6595      1158564 :       FCONST0 (smode).mode = smode;
    6596      2317128 :       const_tiny_rtx[0][(int) smode]
    6597      1158564 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST0 (smode), smode);
    6598              : 
    6599              :       /* We store the value 1.  */
    6600      1158564 :       FCONST1 (smode).data.high = 0;
    6601      1158564 :       FCONST1 (smode).data.low = 0;
    6602      1158564 :       FCONST1 (smode).mode = smode;
    6603      1158564 :       FCONST1 (smode).data
    6604      1158564 :         = double_int_one.lshift (GET_MODE_FBIT (smode),
    6605              :                                  HOST_BITS_PER_DOUBLE_INT,
    6606      1158564 :                                  SIGNED_FIXED_POINT_MODE_P (smode));
    6607      2317128 :       const_tiny_rtx[1][(int) smode]
    6608      1158564 :         = CONST_FIXED_FROM_FIXED_VALUE (FCONST1 (smode), smode);
    6609              :     }
    6610              : 
    6611       289641 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_FRACT)
    6612              :     {
    6613            0 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6614              :     }
    6615              : 
    6616       289641 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_UFRACT)
    6617              :     {
    6618            0 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6619              :     }
    6620              : 
    6621       289641 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_ACCUM)
    6622              :     {
    6623            0 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6624            0 :       const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
    6625              :     }
    6626              : 
    6627       289641 :   FOR_EACH_MODE_IN_CLASS (mode, MODE_VECTOR_UACCUM)
    6628              :     {
    6629            0 :       const_tiny_rtx[0][(int) mode] = gen_const_vector (mode, 0);
    6630            0 :       const_tiny_rtx[1][(int) mode] = gen_const_vector (mode, 1);
    6631              :     }
    6632              : 
    6633     35625843 :   for (i = (int) CCmode; i < (int) MAX_MACHINE_MODE; ++i)
    6634     35336202 :     if (GET_MODE_CLASS ((machine_mode) i) == MODE_CC)
    6635      3475692 :       const_tiny_rtx[0][i] = const0_rtx;
    6636              : 
    6637       289641 :   pc_rtx = gen_rtx_fmt_ (PC, VOIDmode);
    6638       289641 :   ret_rtx = gen_rtx_fmt_ (RETURN, VOIDmode);
    6639       289641 :   simple_return_rtx = gen_rtx_fmt_ (SIMPLE_RETURN, VOIDmode);
    6640       289641 :   invalid_insn_rtx = gen_rtx_INSN (VOIDmode,
    6641              :                                    /*prev_insn=*/NULL,
    6642              :                                    /*next_insn=*/NULL,
    6643              :                                    /*bb=*/NULL,
    6644              :                                    /*pattern=*/NULL_RTX,
    6645              :                                    /*location=*/-1,
    6646              :                                    CODE_FOR_nothing,
    6647              :                                    /*reg_notes=*/NULL_RTX);
    6648       289641 : }
    6649              : 
    6650              : /* Produce exact duplicate of insn INSN after AFTER.
    6651              :    Care updating of libcall regions if present.  */
    6652              : 
    6653              : rtx_insn *
    6654      3646492 : emit_copy_of_insn_after (rtx_insn *insn, rtx_insn *after)
    6655              : {
    6656      3646492 :   rtx_insn *new_rtx;
    6657      3646492 :   rtx link;
    6658              : 
    6659      3646492 :   switch (GET_CODE (insn))
    6660              :     {
    6661      1712401 :     case INSN:
    6662      1712401 :       new_rtx = emit_insn_after (copy_insn (PATTERN (insn)), after);
    6663      1712401 :       break;
    6664              : 
    6665       500852 :     case JUMP_INSN:
    6666       500852 :       new_rtx = emit_jump_insn_after (copy_insn (PATTERN (insn)), after);
    6667       500852 :       set_crossing_jump (new_rtx, CROSSING_JUMP_P (insn));
    6668       500852 :       break;
    6669              : 
    6670      1408529 :     case DEBUG_INSN:
    6671      1408529 :       new_rtx = emit_debug_insn_after (copy_insn (PATTERN (insn)), after);
    6672      1408529 :       break;
    6673              : 
    6674        24710 :     case CALL_INSN:
    6675        24710 :       new_rtx = emit_call_insn_after (copy_insn (PATTERN (insn)), after);
    6676        24710 :       if (CALL_INSN_FUNCTION_USAGE (insn))
    6677        22346 :         CALL_INSN_FUNCTION_USAGE (new_rtx)
    6678        22346 :           = copy_insn (CALL_INSN_FUNCTION_USAGE (insn));
    6679        24710 :       CALL_INSN_ABI_ID (new_rtx) = CALL_INSN_ABI_ID (insn);
    6680        24710 :       SIBLING_CALL_P (new_rtx) = SIBLING_CALL_P (insn);
    6681        24710 :       RTL_CONST_CALL_P (new_rtx) = RTL_CONST_CALL_P (insn);
    6682        24710 :       RTL_PURE_CALL_P (new_rtx) = RTL_PURE_CALL_P (insn);
    6683        24710 :       RTL_LOOPING_CONST_OR_PURE_CALL_P (new_rtx)
    6684        24710 :         = RTL_LOOPING_CONST_OR_PURE_CALL_P (insn);
    6685        24710 :       break;
    6686              : 
    6687            0 :     default:
    6688            0 :       gcc_unreachable ();
    6689              :     }
    6690              : 
    6691              :   /* Update LABEL_NUSES.  */
    6692      3646492 :   if (NONDEBUG_INSN_P (insn))
    6693      2237963 :     mark_jump_label (PATTERN (new_rtx), new_rtx, 0);
    6694              : 
    6695      3646492 :   INSN_LOCATION (new_rtx) = INSN_LOCATION (insn);
    6696              : 
    6697              :   /* If the old insn is frame related, then so is the new one.  This is
    6698              :      primarily needed for IA-64 unwind info which marks epilogue insns,
    6699              :      which may be duplicated by the basic block reordering code.  */
    6700      3646492 :   RTX_FRAME_RELATED_P (new_rtx) = RTX_FRAME_RELATED_P (insn);
    6701              : 
    6702              :   /* Locate the end of existing REG_NOTES in NEW_RTX.  */
    6703      3646492 :   rtx *ptail = &REG_NOTES (new_rtx);
    6704      3646501 :   while (*ptail != NULL_RTX)
    6705            9 :     ptail = &XEXP (*ptail, 1);
    6706              : 
    6707              :   /* Copy all REG_NOTES except REG_LABEL_OPERAND since mark_jump_label
    6708              :      will make them.  REG_LABEL_TARGETs are created there too, but are
    6709              :      supposed to be sticky, so we copy them.  */
    6710      5961608 :   for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
    6711      2315116 :     if (REG_NOTE_KIND (link) != REG_LABEL_OPERAND)
    6712              :       {
    6713      2313991 :         *ptail = duplicate_reg_note (link);
    6714      2313991 :         ptail = &XEXP (*ptail, 1);
    6715              :       }
    6716              : 
    6717      3646492 :   INSN_CODE (new_rtx) = INSN_CODE (insn);
    6718      3646492 :   return new_rtx;
    6719              : }
    6720              : 
    6721              : static GTY((deletable)) rtx hard_reg_clobbers [NUM_MACHINE_MODES][FIRST_PSEUDO_REGISTER];
    6722              : rtx
    6723      4653509 : gen_hard_reg_clobber (machine_mode mode, unsigned int regno)
    6724              : {
    6725      4653509 :   if (hard_reg_clobbers[mode][regno])
    6726              :     return hard_reg_clobbers[mode][regno];
    6727              :   else
    6728       196478 :     return (hard_reg_clobbers[mode][regno] =
    6729       392956 :             gen_rtx_CLOBBER (VOIDmode, gen_rtx_REG (mode, regno)));
    6730              : }
    6731              : 
    6732              : location_t prologue_location;
    6733              : location_t epilogue_location;
    6734              : 
    6735              : /* Hold current location information and last location information, so the
    6736              :    datastructures are built lazily only when some instructions in given
    6737              :    place are needed.  */
    6738              : static location_t curr_location;
    6739              : 
    6740              : /* Allocate insn location datastructure.  */
    6741              : void
    6742      1759257 : insn_locations_init (void)
    6743              : {
    6744      1759257 :   prologue_location = epilogue_location = 0;
    6745      1759257 :   curr_location = UNKNOWN_LOCATION;
    6746      1759257 : }
    6747              : 
    6748              : /* At the end of emit stage, clear current location.  */
    6749              : void
    6750      1519810 : insn_locations_finalize (void)
    6751              : {
    6752      1519810 :   epilogue_location = curr_location;
    6753      1519810 :   curr_location = UNKNOWN_LOCATION;
    6754      1519810 : }
    6755              : 
    6756              : /* Set current location.  */
    6757              : void
    6758    157394548 : set_curr_insn_location (location_t location)
    6759              : {
    6760    157394548 :   curr_location = location;
    6761    157394548 : }
    6762              : 
    6763              : /* Get current location.  */
    6764              : location_t
    6765    272427266 : curr_insn_location (void)
    6766              : {
    6767    272427266 :   return curr_location;
    6768              : }
    6769              : 
    6770              : /* Set the location of the insn chain starting at INSN to LOC.  */
    6771              : void
    6772      3676601 : set_insn_locations (rtx_insn *insn, location_t loc)
    6773              : {
    6774     16496660 :   while (insn)
    6775              :     {
    6776     12820059 :       if (INSN_P (insn))
    6777     10847547 :         INSN_LOCATION (insn) = loc;
    6778     12820059 :       insn = NEXT_INSN (insn);
    6779              :     }
    6780      3676601 : }
    6781              : 
    6782              : /* Return lexical scope block insn belongs to.  */
    6783              : tree
    6784     50051560 : insn_scope (const rtx_insn *insn)
    6785              : {
    6786     50051560 :   return LOCATION_BLOCK (INSN_LOCATION (insn));
    6787              : }
    6788              : 
    6789              : /* Return line number of the statement that produced this insn.  */
    6790              : int
    6791            0 : insn_line (const rtx_insn *insn)
    6792              : {
    6793            0 :   return LOCATION_LINE (INSN_LOCATION (insn));
    6794              : }
    6795              : 
    6796              : /* Return source file of the statement that produced this insn.  */
    6797              : const char *
    6798            0 : insn_file (const rtx_insn *insn)
    6799              : {
    6800            0 :   return LOCATION_FILE (INSN_LOCATION (insn));
    6801              : }
    6802              : 
    6803              : /* Return expanded location of the statement that produced this insn.  */
    6804              : expanded_location
    6805     86030901 : insn_location (const rtx_insn *insn)
    6806              : {
    6807     86030901 :   return expand_location (INSN_LOCATION (insn));
    6808              : }
    6809              : 
    6810              : /* Return true if memory model MODEL requires a pre-operation (release-style)
    6811              :    barrier or a post-operation (acquire-style) barrier.  While not universal,
    6812              :    this function matches behavior of several targets.  */
    6813              : 
    6814              : bool
    6815            0 : need_atomic_barrier_p (enum memmodel model, bool pre)
    6816              : {
    6817            0 :   switch (model & MEMMODEL_BASE_MASK)
    6818              :     {
    6819              :     case MEMMODEL_RELAXED:
    6820              :     case MEMMODEL_CONSUME:
    6821              :       return false;
    6822            0 :     case MEMMODEL_RELEASE:
    6823            0 :       return pre;
    6824            0 :     case MEMMODEL_ACQUIRE:
    6825            0 :       return !pre;
    6826            0 :     case MEMMODEL_ACQ_REL:
    6827            0 :     case MEMMODEL_SEQ_CST:
    6828            0 :       return true;
    6829            0 :     default:
    6830            0 :       gcc_unreachable ();
    6831              :     }
    6832              : }
    6833              : 
    6834              : /* Return a constant shift amount for shifting a value of mode MODE
    6835              :    by VALUE bits.  */
    6836              : 
    6837              : rtx
    6838    216260435 : gen_int_shift_amount (machine_mode, poly_int64 value)
    6839              : {
    6840              :   /* Use a 64-bit mode, to avoid any truncation.
    6841              : 
    6842              :      ??? Perhaps this should be automatically derived from the .md files
    6843              :      instead, or perhaps have a target hook.  */
    6844    216260435 :   scalar_int_mode shift_mode = (BITS_PER_UNIT == 8
    6845              :                                 ? DImode
    6846              :                                 : int_mode_for_size (64, 0).require ());
    6847    216260435 :   return gen_int_mode (value, shift_mode);
    6848              : }
    6849              : 
    6850              : namespace {
    6851              : /* Helper class for expanding an rtx using the encoding generated by
    6852              :    genemit.cc.  The code needs to be kept in sync with there.  */
    6853              : 
    6854              : class rtx_expander
    6855              : {
    6856              : public:
    6857              :   rtx_expander (const uint8_t *, rtx *);
    6858              : 
    6859              :   rtx get_rtx ();
    6860              :   rtvec get_rtvec ();
    6861              :   void expand_seq ();
    6862              : 
    6863              : protected:
    6864              :   uint64_t get_uint ();
    6865     86687596 :   machine_mode get_mode () { return machine_mode (get_uint ()); }
    6866              :   char *get_string ();
    6867              :   rtx get_shared_operand ();
    6868              :   rtx get_unshared_operand ();
    6869              : 
    6870              :   rtx get_rtx (expand_opcode);
    6871              :   rtx get_rtx (rtx_code, machine_mode);
    6872              : 
    6873              :   /* Points to the first unread byte.  */
    6874              :   const uint8_t *m_seq;
    6875              : 
    6876              :   /* The operands passed to the gen_* function.  */
    6877              :   rtx *m_operands;
    6878              : 
    6879              :   /* A bitmap of operands that have already been used to replace a
    6880              :      MATCH_OPERAND or MATCH_DUP.  In order to ensure correct sharing,
    6881              :      further replacements need to use a copy of the operand, rather than
    6882              :      the original rtx.  */
    6883              :   bbitmap<MAX_RECOG_OPERANDS> m_used;
    6884              : };
    6885              : }
    6886              : 
    6887     25639794 : rtx_expander::rtx_expander (const uint8_t *seq, rtx *operands)
    6888     25639794 :   : m_seq (seq), m_operands (operands), m_used ()
    6889            0 : {}
    6890              : 
    6891              : /* Read and return the next encoded "BEB128" integer.  */
    6892              : 
    6893              : inline uint64_t
    6894    250524043 : rtx_expander::get_uint ()
    6895              : {
    6896    250524043 :   const uint8_t *seq = m_seq;
    6897    250524043 :   uint64_t res = 0;
    6898    252759739 :   do
    6899    252759739 :     res = (res << 7) | (*seq & 127);
    6900    252759739 :   while (*seq++ >= 128);
    6901    250524043 :   m_seq = seq;
    6902    250524043 :   return res;
    6903              : }
    6904              : 
    6905              : /* Read an operand number and return the associated operand rtx,
    6906              :    without copying it.  */
    6907              : 
    6908              : rtx
    6909       892912 : rtx_expander::get_shared_operand ()
    6910              : {
    6911       892912 :   return m_operands[get_uint ()];
    6912              : }
    6913              : 
    6914              : /* Read an operand number and return a correctly-shared instance of
    6915              :    the associated operand rtx.  This can be either the original rtx
    6916              :    or a copy.  */
    6917              : 
    6918              : rtx
    6919     46012218 : rtx_expander::get_unshared_operand ()
    6920              : {
    6921     46012218 :   auto opno = get_uint ();
    6922     46012218 :   auto mask = m_used.from_index (opno);
    6923     46012218 :   if (m_used & mask)
    6924      2995351 :     return copy_rtx (m_operands[opno]);
    6925              : 
    6926     43016867 :   m_used |= mask;
    6927     43016867 :   return m_operands[opno];
    6928              : }
    6929              : 
    6930              : /* Read an encoded rtx.  */
    6931              : 
    6932              : rtx
    6933    130860247 : rtx_expander::get_rtx ()
    6934              : {
    6935    130860247 :   auto FIRST_CODE = (unsigned) expand_opcode::FIRST_CODE;
    6936    130860247 :   auto opcode = get_uint ();
    6937    130860247 :   if (opcode < FIRST_CODE)
    6938     51558639 :     return get_rtx (expand_opcode (opcode));
    6939     79301608 :   return get_rtx (rtx_code (opcode - FIRST_CODE), NUM_MACHINE_MODES);
    6940              : }
    6941              : 
    6942              : /* Read an encoded rtx that starts with the given opcode.  */
    6943              : 
    6944              : rtx
    6945     51558639 : rtx_expander::get_rtx (expand_opcode opcode)
    6946              : {
    6947     51558639 :   switch (opcode)
    6948              :     {
    6949              :     case expand_opcode::NO_RTX:
    6950              :       return NULL_RTX;
    6951              : 
    6952     46012218 :     case expand_opcode::MATCH_OPERAND:
    6953     46012218 :       return get_unshared_operand ();
    6954              : 
    6955        42856 :     case expand_opcode::MATCH_OPERATOR_WITH_MODE:
    6956        42856 :       {
    6957        42856 :         auto mode = get_mode ();
    6958        42856 :         auto op = get_shared_operand ();
    6959        42856 :         return get_rtx (GET_CODE (op), mode);
    6960              :       }
    6961              : 
    6962       848247 :     case expand_opcode::MATCH_OPERATOR:
    6963       848247 :       {
    6964       848247 :         auto op = get_shared_operand ();
    6965       848247 :         return get_rtx (GET_CODE (op), GET_MODE (op));
    6966              :       }
    6967              : 
    6968         1809 :     case expand_opcode::MATCH_PARALLEL:
    6969         1809 :       return get_shared_operand ();
    6970              : 
    6971      4653509 :     case expand_opcode::CLOBBER_REG:
    6972      4653509 :       {
    6973      4653509 :         auto mode = get_mode ();
    6974      4653509 :         auto regno = get_uint ();
    6975      4653509 :         return gen_hard_reg_clobber (mode, regno);
    6976              :       }
    6977              : 
    6978              :     case expand_opcode::FIRST_CODE:
    6979              :       break;
    6980              :     }
    6981            0 :   gcc_unreachable ();
    6982              : }
    6983              : 
    6984              : /* Read the rest of an rtx of code CODE.  If such rtxes are not always
    6985              :    VOIDmode, MODE is the mode that the rtx should have, or NUM_MACHINE_MODES
    6986              :    if the mode is encoded at the current iterator position.  */
    6987              : 
    6988              : rtx
    6989     80192711 : rtx_expander::get_rtx (rtx_code code, machine_mode mode)
    6990              : {
    6991     80192711 :   switch (code)
    6992              :     {
    6993              :       /* Please keep the cases below in sync with gengenrtl.cc:special_rtx.  */
    6994              : 
    6995            0 :     case EXPR_LIST:
    6996            0 :     case INSN_LIST:
    6997            0 :     case INSN:
    6998            0 :       gcc_unreachable ();
    6999              : 
    7000      4909320 :     case CONST_INT:
    7001      9818640 :       return GEN_INT (get_uint ());
    7002              : 
    7003      1943141 :     case REG:
    7004      1943141 :       if (mode == NUM_MACHINE_MODES)
    7005      3886282 :         mode = get_mode ();
    7006      3886282 :       return gen_rtx_REG (mode, get_uint ());
    7007              : 
    7008        52243 :     case SUBREG:
    7009        52243 :       {
    7010        52243 :         if (mode == NUM_MACHINE_MODES)
    7011       104486 :           mode = get_mode ();
    7012        52243 :         auto reg = get_rtx ();
    7013        52243 :         auto byte = get_uint ();
    7014        52243 :         return gen_rtx_SUBREG (mode, reg, byte);
    7015              :       }
    7016              : 
    7017      2515253 :     case MEM:
    7018      2515253 :       if (mode == NUM_MACHINE_MODES)
    7019      5030506 :         mode = get_mode ();
    7020      2515253 :       return gen_rtx_MEM (mode, get_rtx ());
    7021              : 
    7022      9683421 :     case PC:
    7023      9683421 :       return pc_rtx;
    7024              : 
    7025            0 :     case RETURN:
    7026            0 :       return ret_rtx;
    7027              : 
    7028      1617284 :     case SIMPLE_RETURN:
    7029      1617284 :       return simple_return_rtx;
    7030              : 
    7031         1926 :     case CONST_VECTOR:
    7032         1926 :       if (mode == NUM_MACHINE_MODES)
    7033         3852 :         mode = get_mode ();
    7034         1926 :       return gen_rtx_CONST_VECTOR (mode, get_rtvec ());
    7035              : 
    7036              :       /* Please keep the cases below in sync with
    7037              :          gengenrtl.cc:excluded_rtx.  */
    7038              : 
    7039            0 :     case VAR_LOCATION:
    7040            0 :       gcc_unreachable ();
    7041              : 
    7042            0 :     case CONST_DOUBLE:
    7043              :       /* genemit.cc only accepts zero const_doubles.  */
    7044            0 :       if (mode == NUM_MACHINE_MODES)
    7045            0 :         mode = get_mode ();
    7046            0 :       return CONST0_RTX (mode);
    7047              : 
    7048            0 :     case CONST_WIDE_INT:
    7049            0 :     case CONST_POLY_INT:
    7050            0 :     case CONST_FIXED:
    7051            0 :       gcc_unreachable ();
    7052              : 
    7053     59470123 :     default:
    7054     59470123 :       break;
    7055              :     }
    7056              : 
    7057     59470123 :   rtx x = rtx_alloc (code);
    7058     59470123 :   if (!always_void_p (code))
    7059              :     {
    7060     35025973 :       if (mode == NUM_MACHINE_MODES)
    7061     68269740 :         mode = get_mode ();
    7062     35025973 :       PUT_MODE_RAW (x, mode);
    7063              :     }
    7064              : 
    7065     59470123 :   const char *fmt = GET_RTX_FORMAT (code);
    7066    153695604 :   for (unsigned int i = 0; fmt[i]; ++i)
    7067     94225481 :     switch (fmt[i])
    7068              :       {
    7069              :         /* Please keep these cases in sync with
    7070              :            gengenrtl.cc:type_from_format.  */
    7071              : 
    7072      1860266 :       case 'i':
    7073      1860266 :         XINT (x, i) = get_uint ();
    7074      1860266 :         break;
    7075              : 
    7076            0 :       case 'L':
    7077            0 :       case 'w':
    7078            0 :       case 'p':
    7079            0 :       case 's':
    7080            0 :         gcc_unreachable ();
    7081              : 
    7082     84807511 :       case 'e':  case 'u':
    7083     84807511 :         XEXP (x, i) = get_rtx ();
    7084     84807511 :         break;
    7085              : 
    7086      7557704 :       case 'E':
    7087      7557704 :         XVEC (x, i) = get_rtvec ();
    7088      7557704 :         break;
    7089              : 
    7090            0 :       case 't':
    7091            0 :       case 'B':
    7092            0 :       default:
    7093            0 :         gcc_unreachable ();
    7094              :       }
    7095              : 
    7096              :   return x;
    7097              : }
    7098              : 
    7099              : /* Read an encoded rtvec.  */
    7100              : 
    7101              : rtvec
    7102      7559630 : rtx_expander::get_rtvec ()
    7103              : {
    7104      7559630 :   unsigned int len = get_uint ();
    7105      7559630 :   rtvec v = rtvec_alloc (len);
    7106     31911594 :   for (unsigned int i = 0; i < len; ++i)
    7107     16792334 :     RTVEC_ELT (v, i) = get_rtx ();
    7108      7559630 :   return v;
    7109              : }
    7110              : 
    7111              : /* Read and emit an encoded sequence of instructions.  */
    7112              : 
    7113              : void
    7114      8436759 : rtx_expander::expand_seq ()
    7115              : {
    7116      8436759 :   unsigned int len = get_uint ();
    7117     17926630 :   for (unsigned int i = 0; i < len; ++i)
    7118      9489871 :     emit (get_rtx (), i < len - 1);
    7119      8436759 : }
    7120              : 
    7121              : /* Read an rtx from the bytecode in SEQ, which was generated by genemit.cc.
    7122              :    Replace operand placeholders with the values given in OPERANDS.  */
    7123              : 
    7124              : rtx
    7125     17203035 : expand_rtx (const uint8_t *seq, rtx *operands)
    7126              : {
    7127     17203035 :   return rtx_expander (seq, operands).get_rtx ();
    7128              : }
    7129              : 
    7130              : /* Read and emit a sequence of instructions from the bytecode in SEQ,
    7131              :    which was generated by genemit.cc.  Replace operand placeholders with
    7132              :    the values given in OPERANDS.  */
    7133              : 
    7134              : rtx_insn *
    7135      8436759 : complete_seq (const uint8_t *seq, rtx *operands)
    7136              : {
    7137      8436759 :   rtx_expander (seq, operands).expand_seq ();
    7138      8436759 :   return end_sequence ();
    7139              : }
    7140              : 
    7141              : /* Note in the dump file that WHAT, which names a define_split or a
    7142              :    define_peephole2 and where it came from, is being applied.  genemit.cc
    7143              :    emits a call to this rather than the test and the fprintf, so that the
    7144              :    dump is written out once instead of once per pattern.  */
    7145              : 
    7146              : void
    7147      8594520 : note_split (const char *what)
    7148              : {
    7149      8594520 :   if (dump_file)
    7150          117 :     fprintf (dump_file, "Splitting with %s\n", what);
    7151      8594520 : }
    7152              : 
    7153              : /* Initialize fields of rtl_data related to stack alignment.  */
    7154              : 
    7155              : void
    7156      1518011 : rtl_data::init_stack_alignment ()
    7157              : {
    7158      1518011 :   stack_alignment_needed = STACK_BOUNDARY;
    7159      1518011 :   max_used_stack_slot_alignment = STACK_BOUNDARY;
    7160      1518011 :   stack_alignment_estimated = 0;
    7161      1518011 :   preferred_stack_boundary = STACK_BOUNDARY;
    7162      1518011 : }
    7163              : 
    7164              : 
    7165              : #include "gt-emit-rtl.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.