LCOV - code coverage report
Current view: top level - gcc/config/i386 - i386-features.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 88.9 % 2847 2531
Test Date: 2026-11-14 16:39:50 Functions: 98.9 % 95 94
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Copyright (C) 1988-2026 Free Software Foundation, Inc.
       2              : 
       3              : This file is part of GCC.
       4              : 
       5              : GCC is free software; you can redistribute it and/or modify
       6              : it under the terms of the GNU General Public License as published by
       7              : the Free Software Foundation; either version 3, or (at your option)
       8              : any later version.
       9              : 
      10              : GCC is distributed in the hope that it will be useful,
      11              : but WITHOUT ANY WARRANTY; without even the implied warranty of
      12              : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
      13              : GNU General Public License for more details.
      14              : 
      15              : You should have received a copy of the GNU General Public License
      16              : along with GCC; see the file COPYING3.  If not see
      17              : <http://www.gnu.org/licenses/>.  */
      18              : 
      19              : #define IN_TARGET_CODE 1
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "memmodel.h"
      28              : #include "gimple.h"
      29              : #include "cfghooks.h"
      30              : #include "cfgloop.h"
      31              : #include "df.h"
      32              : #include "tm_p.h"
      33              : #include "stringpool.h"
      34              : #include "expmed.h"
      35              : #include "optabs.h"
      36              : #include "regs.h"
      37              : #include "emit-rtl.h"
      38              : #include "recog.h"
      39              : #include "cgraph.h"
      40              : #include "diagnostic.h"
      41              : #include "cfgbuild.h"
      42              : #include "alias.h"
      43              : #include "fold-const.h"
      44              : #include "attribs.h"
      45              : #include "calls.h"
      46              : #include "stor-layout.h"
      47              : #include "varasm.h"
      48              : #include "output.h"
      49              : #include "insn-attr.h"
      50              : #include "flags.h"
      51              : #include "except.h"
      52              : #include "explow.h"
      53              : #include "expr.h"
      54              : #include "cfgrtl.h"
      55              : #include "common/common-target.h"
      56              : #include "langhooks.h"
      57              : #include "reload.h"
      58              : #include "gimplify.h"
      59              : #include "dwarf2.h"
      60              : #include "tm-constrs.h"
      61              : #include "cselib.h"
      62              : #include "sched-int.h"
      63              : #include "opts.h"
      64              : #include "tree-pass.h"
      65              : #include "context.h"
      66              : #include "pass_manager.h"
      67              : #include "target-globals.h"
      68              : #include "gimple-iterator.h"
      69              : #include "shrink-wrap.h"
      70              : #include "builtins.h"
      71              : #include "rtl-iter.h"
      72              : #include "tree-iterator.h"
      73              : #include "dbgcnt.h"
      74              : #include "case-cfn-macros.h"
      75              : #include "dojump.h"
      76              : #include "fold-const-call.h"
      77              : #include "tree-vrp.h"
      78              : #include "tree-ssanames.h"
      79              : #include "selftest.h"
      80              : #include "selftest-rtl.h"
      81              : #include "print-rtl.h"
      82              : #include "intl.h"
      83              : #include "ifcvt.h"
      84              : #include "symbol-summary.h"
      85              : #include "sreal.h"
      86              : #include "ipa-cp.h"
      87              : #include "ipa-prop.h"
      88              : #include "ipa-fnsummary.h"
      89              : #include "wide-int-bitmask.h"
      90              : #include "tree-vector-builder.h"
      91              : #include "debug.h"
      92              : #include "dwarf2out.h"
      93              : #include "function-abi.h"
      94              : #include "i386-builtins.h"
      95              : #include "i386-features.h"
      96              : #include "i386-expand.h"
      97              : 
      98              : const char * const xlogue_layout::STUB_BASE_NAMES[XLOGUE_STUB_COUNT] = {
      99              :   "savms64",
     100              :   "resms64",
     101              :   "resms64x",
     102              :   "savms64f",
     103              :   "resms64f",
     104              :   "resms64fx"
     105              : };
     106              : 
     107              : const unsigned xlogue_layout::REG_ORDER[xlogue_layout::MAX_REGS] = {
     108              : /* The below offset values are where each register is stored for the layout
     109              :    relative to incoming stack pointer.  The value of each m_regs[].offset will
     110              :    be relative to the incoming base pointer (rax or rsi) used by the stub.
     111              : 
     112              :     s_instances:   0            1               2               3
     113              :     Offset:                                     realigned or    aligned + 8
     114              :     Register       aligned      aligned + 8     aligned w/HFP   w/HFP   */
     115              :     XMM15_REG,  /* 0x10         0x18            0x10            0x18    */
     116              :     XMM14_REG,  /* 0x20         0x28            0x20            0x28    */
     117              :     XMM13_REG,  /* 0x30         0x38            0x30            0x38    */
     118              :     XMM12_REG,  /* 0x40         0x48            0x40            0x48    */
     119              :     XMM11_REG,  /* 0x50         0x58            0x50            0x58    */
     120              :     XMM10_REG,  /* 0x60         0x68            0x60            0x68    */
     121              :     XMM9_REG,   /* 0x70         0x78            0x70            0x78    */
     122              :     XMM8_REG,   /* 0x80         0x88            0x80            0x88    */
     123              :     XMM7_REG,   /* 0x90         0x98            0x90            0x98    */
     124              :     XMM6_REG,   /* 0xa0         0xa8            0xa0            0xa8    */
     125              :     SI_REG,     /* 0xa8         0xb0            0xa8            0xb0    */
     126              :     DI_REG,     /* 0xb0         0xb8            0xb0            0xb8    */
     127              :     BX_REG,     /* 0xb8         0xc0            0xb8            0xc0    */
     128              :     BP_REG,     /* 0xc0         0xc8            N/A             N/A     */
     129              :     R12_REG,    /* 0xc8         0xd0            0xc0            0xc8    */
     130              :     R13_REG,    /* 0xd0         0xd8            0xc8            0xd0    */
     131              :     R14_REG,    /* 0xd8         0xe0            0xd0            0xd8    */
     132              :     R15_REG,    /* 0xe0         0xe8            0xd8            0xe0    */
     133              : };
     134              : 
     135              : /* Instantiate static const values.  */
     136              : const HOST_WIDE_INT xlogue_layout::STUB_INDEX_OFFSET;
     137              : const unsigned xlogue_layout::MIN_REGS;
     138              : const unsigned xlogue_layout::MAX_REGS;
     139              : const unsigned xlogue_layout::MAX_EXTRA_REGS;
     140              : const unsigned xlogue_layout::VARIANT_COUNT;
     141              : const unsigned xlogue_layout::STUB_NAME_MAX_LEN;
     142              : 
     143              : /* Initialize xlogue_layout::s_stub_names to zero.  */
     144              : char xlogue_layout::s_stub_names[2][XLOGUE_STUB_COUNT][VARIANT_COUNT]
     145              :                                 [STUB_NAME_MAX_LEN];
     146              : 
     147              : /* Instantiates all xlogue_layout instances.  */
     148              : const xlogue_layout xlogue_layout::s_instances[XLOGUE_SET_COUNT] = {
     149              :   xlogue_layout (0, false),
     150              :   xlogue_layout (8, false),
     151              :   xlogue_layout (0, true),
     152              :   xlogue_layout (8, true)
     153              : };
     154              : 
     155              : /* Return an appropriate const instance of xlogue_layout based upon values
     156              :    in cfun->machine and crtl.  */
     157              : const class xlogue_layout &
     158        49891 : xlogue_layout::get_instance ()
     159              : {
     160        49891 :   enum xlogue_stub_sets stub_set;
     161        49891 :   bool aligned_plus_8 = cfun->machine->call_ms2sysv_pad_in;
     162              : 
     163        49891 :   if (stack_realign_fp)
     164              :     stub_set = XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
     165        40910 :   else if (frame_pointer_needed)
     166        25246 :     stub_set = aligned_plus_8
     167        31552 :               ? XLOGUE_SET_HFP_ALIGNED_PLUS_8
     168              :               : XLOGUE_SET_HFP_ALIGNED_OR_REALIGN;
     169              :   else
     170         9358 :     stub_set = aligned_plus_8 ? XLOGUE_SET_ALIGNED_PLUS_8 : XLOGUE_SET_ALIGNED;
     171              : 
     172        49891 :   return s_instances[stub_set];
     173              : }
     174              : 
     175              : /* Determine how many clobbered registers can be saved by the stub.
     176              :    Returns the count of registers the stub will save and restore.  */
     177              : unsigned
     178        35225 : xlogue_layout::count_stub_managed_regs ()
     179              : {
     180        35225 :   bool hfp = frame_pointer_needed || stack_realign_fp;
     181        35225 :   unsigned i, count;
     182        35225 :   unsigned regno;
     183              : 
     184        93950 :   for (count = i = MIN_REGS; i < MAX_REGS; ++i)
     185              :     {
     186        92750 :       regno = REG_ORDER[i];
     187        92750 :       if (regno == BP_REG && hfp)
     188        17880 :         continue;
     189        74870 :       if (!ix86_save_reg (regno, false, false))
     190              :         break;
     191        40845 :       ++count;
     192              :     }
     193        35225 :   return count;
     194              : }
     195              : 
     196              : /* Determine if register REGNO is a stub managed register given the
     197              :    total COUNT of stub managed registers.  */
     198              : bool
     199      2642240 : xlogue_layout::is_stub_managed_reg (unsigned regno, unsigned count)
     200              : {
     201      2642240 :   bool hfp = frame_pointer_needed || stack_realign_fp;
     202      2642240 :   unsigned i;
     203              : 
     204     34418160 :   for (i = 0; i < count; ++i)
     205              :     {
     206     32275013 :       gcc_assert (i < MAX_REGS);
     207     32275013 :       if (REG_ORDER[i] == BP_REG && hfp)
     208       510254 :         ++count;
     209     31764759 :       else if (REG_ORDER[i] == regno)
     210              :         return true;
     211              :     }
     212              :   return false;
     213              : }
     214              : 
     215              : /* Constructor for xlogue_layout.  */
     216      1184040 : xlogue_layout::xlogue_layout (HOST_WIDE_INT stack_align_off_in, bool hfp)
     217      1184040 :   : m_hfp (hfp) , m_nregs (hfp ? 17 : 18),
     218      1184040 :     m_stack_align_off_in (stack_align_off_in)
     219              : {
     220      1184040 :   HOST_WIDE_INT offset = stack_align_off_in;
     221      1184040 :   unsigned i, j;
     222              : 
     223     22496760 :   for (i = j = 0; i < MAX_REGS; ++i)
     224              :     {
     225     21312720 :       unsigned regno = REG_ORDER[i];
     226              : 
     227     21312720 :       if (regno == BP_REG && hfp)
     228       592020 :         continue;
     229     20720700 :       if (SSE_REGNO_P (regno))
     230              :         {
     231     11840400 :           offset += 16;
     232              :           /* Verify that SSE regs are always aligned.  */
     233     11840400 :           gcc_assert (!((stack_align_off_in + offset) & 15));
     234              :         }
     235              :       else
     236      8880300 :         offset += 8;
     237              : 
     238     20720700 :       m_regs[j].regno    = regno;
     239     20720700 :       m_regs[j++].offset = offset - STUB_INDEX_OFFSET;
     240              :     }
     241      1184040 :   gcc_assert (j == m_nregs);
     242      1184040 : }
     243              : 
     244              : const char *
     245        14666 : xlogue_layout::get_stub_name (enum xlogue_stub stub,
     246              :                               unsigned n_extra_regs)
     247              : {
     248        14666 :   const int have_avx = TARGET_AVX;
     249        14666 :   char *name = s_stub_names[!!have_avx][stub][n_extra_regs];
     250              : 
     251              :   /* Lazy init */
     252        14666 :   if (!*name)
     253              :     {
     254          362 :       int res = snprintf (name, STUB_NAME_MAX_LEN, "__%s_%s_%u",
     255              :                           (have_avx ? "avx" : "sse"),
     256          181 :                           STUB_BASE_NAMES[stub],
     257              :                           MIN_REGS + n_extra_regs);
     258          181 :       gcc_checking_assert (res < (int)STUB_NAME_MAX_LEN);
     259              :     }
     260              : 
     261        14666 :   return name;
     262              : }
     263              : 
     264              : /* Return rtx of a symbol ref for the entry point (based upon
     265              :    cfun->machine->call_ms2sysv_extra_regs) of the specified stub.  */
     266              : rtx
     267        14666 : xlogue_layout::get_stub_rtx (enum xlogue_stub stub)
     268              : {
     269        14666 :   const unsigned n_extra_regs = cfun->machine->call_ms2sysv_extra_regs;
     270        14666 :   gcc_checking_assert (n_extra_regs <= MAX_EXTRA_REGS);
     271        14666 :   gcc_assert (stub < XLOGUE_STUB_COUNT);
     272        14666 :   gcc_assert (crtl->stack_realign_finalized);
     273              : 
     274        14666 :   return gen_rtx_SYMBOL_REF (Pmode, get_stub_name (stub, n_extra_regs));
     275              : }
     276              : 
     277              : namespace {
     278              : 
     279              : class scalar_chain
     280              : {
     281              :  public:
     282              :   scalar_chain (enum machine_mode, enum machine_mode);
     283              :   virtual ~scalar_chain ();
     284              : 
     285              :   static unsigned max_id;
     286              : 
     287              :   /* Scalar mode.  */
     288              :   enum machine_mode smode;
     289              :   /* Vector mode.  */
     290              :   enum machine_mode vmode;
     291              : 
     292              :   /* ID of a chain.  */
     293              :   unsigned int chain_id;
     294              :   /* A queue of instructions to be included into a chain.  */
     295              :   bitmap queue;
     296              :   /* Instructions included into a chain.  */
     297              :   bitmap insns;
     298              :   /* All registers defined by a chain.  */
     299              :   bitmap defs;
     300              :   /* Registers used in both vector and scalar modes.  */
     301              :   bitmap defs_conv;
     302              : 
     303              :   /* Limit on chain discovery.  */
     304              :   unsigned max_visits;
     305              : 
     306              :   bitmap insns_conv;
     307              :   hash_map<rtx, rtx> defs_map;
     308              :   /* Cost of inserted conversion between ineteger and sse.  */
     309              :   int cost_sse_integer;
     310              :   sreal weighted_cost_sse_integer;
     311              :   auto_vec<rtx_insn *> control_flow_insns;
     312              : 
     313              :   bool build (bitmap candidates, unsigned insn_uid, bitmap disallowed);
     314              :   virtual bool compute_convert_gain () = 0;
     315              :   int convert ();
     316              : 
     317              :  protected:
     318              :   void add_to_queue (unsigned insn_uid);
     319              :   void emit_conversion_insns (rtx_insn *insns, rtx_insn *pos);
     320              :   rtx convert_compare (rtx op1, rtx op2, rtx_insn *insn);
     321              :   void mark_dual_mode_def (df_ref def);
     322              :   void convert_reg (rtx_insn *insn, rtx dst, rtx src);
     323              :   void convert_insn_common (rtx_insn *insn);
     324              :   void make_vector_copies (rtx_insn *, rtx);
     325              :   void convert_registers ();
     326              :   void convert_op (rtx *op, rtx_insn *insn);
     327              : 
     328              :  private:
     329              :   bool add_insn (bitmap candidates, unsigned insn_uid, bitmap disallowed);
     330              :   bool analyze_register_chain (bitmap candidates, df_ref ref,
     331              :                                bitmap disallowed);
     332              :   virtual void convert_insn (rtx_insn *insn) = 0;
     333              : };
     334              : 
     335              : class general_scalar_chain : public scalar_chain
     336              : {
     337              :  public:
     338      6088507 :   general_scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
     339     12177014 :     : scalar_chain (smode_, vmode_) {}
     340              :   bool compute_convert_gain () final override;
     341              : 
     342              :  private:
     343              :   void convert_insn (rtx_insn *insn) final override;
     344              :   int vector_const_cost (rtx exp, basic_block bb);
     345              :   rtx convert_rotate (enum rtx_code, rtx op0, rtx op1, rtx_insn *insn);
     346              : };
     347              : 
     348              : class timode_scalar_chain : public scalar_chain
     349              : {
     350              :  public:
     351      1062312 :   timode_scalar_chain () : scalar_chain (TImode, V1TImode) {}
     352              :   bool compute_convert_gain () final override;
     353              : 
     354              :  private:
     355              :   void fix_debug_reg_uses (rtx reg);
     356              :   void convert_insn (rtx_insn *insn) final override;
     357              : };
     358              : 
     359              : } // anon namespace
     360              : 
     361              : unsigned scalar_chain::max_id = 0;
     362              : 
     363              : namespace {
     364              : 
     365              : /* Initialize new chain.  */
     366              : 
     367      6619663 : scalar_chain::scalar_chain (enum machine_mode smode_, enum machine_mode vmode_)
     368              : {
     369      6619663 :   smode = smode_;
     370      6619663 :   vmode = vmode_;
     371              : 
     372      6619663 :   chain_id = ++max_id;
     373              : 
     374      6619663 :    if (dump_file)
     375          139 :     fprintf (dump_file, "Created a new instruction chain #%d\n", chain_id);
     376              : 
     377      6619663 :   bitmap_obstack_initialize (NULL);
     378      6619663 :   insns = BITMAP_ALLOC (NULL);
     379      6619663 :   defs = BITMAP_ALLOC (NULL);
     380      6619663 :   defs_conv = BITMAP_ALLOC (NULL);
     381      6619663 :   insns_conv = BITMAP_ALLOC (NULL);
     382      6619663 :   queue = NULL;
     383              : 
     384      6619663 :   cost_sse_integer = 0;
     385      6619663 :   weighted_cost_sse_integer = 0 ;
     386      6619663 :   max_visits = x86_stv_max_visits;
     387      6619663 : }
     388              : 
     389              : /* Free chain's data.  */
     390              : 
     391      6619663 : scalar_chain::~scalar_chain ()
     392              : {
     393      6619663 :   BITMAP_FREE (insns);
     394      6619663 :   BITMAP_FREE (defs);
     395      6619663 :   BITMAP_FREE (defs_conv);
     396      6619663 :   BITMAP_FREE (insns_conv);
     397      6619663 :   bitmap_obstack_release (NULL);
     398      6619663 : }
     399              : 
     400              : /* Add instruction into chains' queue.  */
     401              : 
     402              : void
     403      8406250 : scalar_chain::add_to_queue (unsigned insn_uid)
     404              : {
     405      8406250 :   if (!bitmap_set_bit (queue, insn_uid))
     406              :     return;
     407              : 
     408      6444536 :   if (dump_file)
     409          147 :     fprintf (dump_file, "  Adding insn %d into chain's #%d queue\n",
     410              :              insn_uid, chain_id);
     411              : }
     412              : 
     413              : /* For DImode conversion, mark register defined by DEF as requiring
     414              :    conversion.  */
     415              : 
     416              : void
     417      9742832 : scalar_chain::mark_dual_mode_def (df_ref def)
     418              : {
     419      9742832 :   gcc_assert (DF_REF_REG_DEF_P (def));
     420              : 
     421              :   /* Record the def/insn pair so we can later efficiently iterate over
     422              :      the defs to convert on insns not in the chain.  */
     423      9742832 :   bool reg_new = bitmap_set_bit (defs_conv, DF_REF_REGNO (def));
     424      9742832 :   basic_block bb = BLOCK_FOR_INSN (DF_REF_INSN (def));
     425      9742832 :   profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
     426      9742832 :   bool speed_p = optimize_bb_for_speed_p (bb);
     427      9742832 :   int cost = 0;
     428              : 
     429      9742832 :   if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def)))
     430              :     {
     431      2957075 :       if (!bitmap_set_bit (insns_conv, DF_REF_INSN_UID (def))
     432      2957075 :           && !reg_new)
     433      1512282 :         return;
     434              : 
     435              :       /* Cost integer to sse moves.  */
     436      2599006 :       if (speed_p)
     437      2290141 :         cost = COSTS_N_INSNS (ix86_cost->integer_to_sse) / 2;
     438       308865 :       else if (TARGET_64BIT || smode == SImode)
     439              :         cost = COSTS_N_BYTES (4);
     440              :       /* vmovd (4 bytes) + vpinsrd (6 bytes).  */
     441        18767 :       else if (TARGET_SSE4_1)
     442              :         cost = COSTS_N_BYTES (10);
     443              :       /* movd (4 bytes) + movd (4 bytes) + unpckldq (4 bytes).  */
     444              :       else
     445      8230550 :         cost = COSTS_N_BYTES (12);
     446              :     }
     447              :   else
     448              :     {
     449      6785757 :       if (!reg_new)
     450              :         return;
     451              : 
     452              :       /* Cost sse to integer moves.  */
     453      5631544 :       if (speed_p)
     454      5042501 :         cost = COSTS_N_INSNS (ix86_cost->sse_to_integer) / 2;
     455       589043 :       else if (TARGET_64BIT || smode == SImode)
     456              :         cost = COSTS_N_BYTES (4);
     457              :       /* vmovd (4 bytes) + vpextrd (6 bytes).  */
     458         2912 :       else if (TARGET_SSE4_1)
     459              :         cost = COSTS_N_BYTES (10);
     460              :       /* movd (4 bytes) + psrlq (5 bytes) + movd (4 bytes).  */
     461              :       else
     462      8230550 :         cost = COSTS_N_BYTES (13);
     463              :     }
     464              : 
     465      8230550 :   if (speed_p)
     466      7332642 :     weighted_cost_sse_integer += bb->count.to_sreal_scale (entry_count) * cost;
     467              : 
     468      8230550 :   cost_sse_integer += cost;
     469              : 
     470      8230550 :   if (dump_file)
     471          236 :     fprintf (dump_file,
     472              :              "  Mark r%d def in insn %d as requiring both modes in chain #%d\n",
     473          236 :              DF_REF_REGNO (def), DF_REF_INSN_UID (def), chain_id);
     474              : }
     475              : 
     476              : /* Check REF's chain to add new insns into a queue
     477              :    and find registers requiring conversion.  Return true if OK, false
     478              :    if the analysis was aborted.  */
     479              : 
     480              : bool
     481     18424624 : scalar_chain::analyze_register_chain (bitmap candidates, df_ref ref,
     482              :                                       bitmap disallowed)
     483              : {
     484     18424624 :   df_link *chain;
     485     18424624 :   bool mark_def = false;
     486              : 
     487     18424624 :   gcc_checking_assert (bitmap_bit_p (insns, DF_REF_INSN_UID (ref)));
     488              : 
     489     62922364 :   for (chain = DF_REF_CHAIN (ref); chain; chain = chain->next)
     490              :     {
     491     44499579 :       unsigned uid = DF_REF_INSN_UID (chain->ref);
     492              : 
     493     44499579 :       if (!NONDEBUG_INSN_P (DF_REF_INSN (chain->ref)))
     494      7706238 :         continue;
     495              : 
     496     36793341 :       if (--max_visits == 0)
     497              :         return false;
     498              : 
     499     36792820 :       if (!DF_REF_REG_MEM_P (chain->ref))
     500              :         {
     501     30848327 :           if (bitmap_bit_p (insns, uid))
     502      9820892 :             continue;
     503              : 
     504     21027435 :           if (bitmap_bit_p (candidates, uid))
     505              :             {
     506      8406250 :               add_to_queue (uid);
     507      8406250 :               continue;
     508              :             }
     509              : 
     510              :           /* If we run into parts of an aborted chain discovery abort.  */
     511     12621185 :           if (bitmap_bit_p (disallowed, uid))
     512              :             return false;
     513              :         }
     514              : 
     515     18564535 :       if (DF_REF_REG_DEF_P (chain->ref))
     516              :         {
     517              :           /* An asm goto with outputs defines them on all outgoing edges,
     518              :              so a conversion copy can't be emitted after it.  */
     519      2957250 :           if (JUMP_P (DF_REF_INSN (chain->ref)))
     520              :             {
     521          175 :               if (dump_file)
     522            1 :                 fprintf (dump_file,
     523              :                          "  r%d def in jump insn %d, aborting chain\n",
     524              :                          DF_REF_REGNO (chain->ref), uid);
     525              :               return false;
     526              :             }
     527              : 
     528      2957075 :           if (dump_file)
     529          125 :             fprintf (dump_file, "  r%d def in insn %d isn't convertible\n",
     530              :                      DF_REF_REGNO (chain->ref), uid);
     531      2957075 :           mark_dual_mode_def (chain->ref);
     532              :         }
     533              :       else
     534              :         {
     535     15607285 :           if (dump_file)
     536          524 :             fprintf (dump_file, "  r%d use in insn %d isn't convertible\n",
     537              :                      DF_REF_REGNO (chain->ref), uid);
     538              :           mark_def = true;
     539              :         }
     540              :     }
     541              : 
     542     18422785 :   if (mark_def)
     543      6785757 :     mark_dual_mode_def (ref);
     544              : 
     545              :   return true;
     546              : }
     547              : 
     548              : /* Check whether X is a convertible *concatditi_? variant.  X is known
     549              :    to be any_or_plus:TI, i.e. PLUS:TI, IOR:TI or XOR:TI.  */
     550              : 
     551              : static bool
     552        32880 : timode_concatdi_p (rtx x)
     553              : {
     554        32880 :   rtx op0 = XEXP (x, 0);
     555        32880 :   rtx op1 = XEXP (x, 1);
     556              : 
     557        32880 :   if (GET_CODE (op1) == ASHIFT)
     558          959 :     std::swap (op0, op1);
     559              : 
     560        32880 :   return GET_CODE (op0) == ASHIFT
     561        21955 :          && GET_CODE (XEXP (op0, 0)) == ZERO_EXTEND
     562        21955 :          && GET_MODE (XEXP (XEXP (op0, 0), 0)) == DImode
     563        21955 :          && REG_P (XEXP (XEXP (op0, 0), 0))
     564        21804 :          && CONST_INT_P (XEXP (op0, 1))
     565        21804 :          && INTVAL (XEXP (op0, 1)) == 64
     566        21804 :          && GET_CODE (op1) == ZERO_EXTEND
     567        20845 :          && GET_MODE (XEXP (op1, 0)) == DImode
     568        53725 :          && REG_P (XEXP (op1, 0));
     569              : }
     570              : 
     571              : 
     572              : /* Add instruction into a chain.  Return true if OK, false if the search
     573              :    was aborted.  */
     574              : 
     575              : bool
     576     13062319 : scalar_chain::add_insn (bitmap candidates, unsigned int insn_uid,
     577              :                         bitmap disallowed)
     578              : {
     579     13062319 :   if (!bitmap_set_bit (insns, insn_uid))
     580              :     return true;
     581              : 
     582     13062319 :   if (dump_file)
     583          284 :     fprintf (dump_file, "  Adding insn %d to chain #%d\n", insn_uid, chain_id);
     584              : 
     585     13062319 :   rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
     586     13062319 :   rtx def_set = single_set (insn);
     587     13062319 :   if (def_set && REG_P (SET_DEST (def_set))
     588     23102924 :       && !HARD_REGISTER_P (SET_DEST (def_set)))
     589     10015110 :     bitmap_set_bit (defs, REGNO (SET_DEST (def_set)));
     590              : 
     591              :   /* ???  The following is quadratic since analyze_register_chain
     592              :      iterates over all refs to look for dual-mode regs.  Instead this
     593              :      should be done separately for all regs mentioned in the chain once.  */
     594     13062319 :   df_ref ref;
     595     26661095 :   for (ref = DF_INSN_UID_DEFS (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
     596     13599567 :     if (!HARD_REGISTER_P (DF_REF_REG (ref)))
     597     10015110 :       if (!analyze_register_chain (candidates, ref, disallowed))
     598              :         return false;
     599              : 
     600              :   /* The operand(s) of VEC_SELECT, ZERO_EXTEND and similar ops don't need
     601              :      to be converted/convertible.  */
     602     13061528 :   if (def_set)
     603     13061528 :     switch (GET_CODE (SET_SRC (def_set)))
     604              :       {
     605      3920920 :       case REG:
     606      3920920 :         if (HARD_REGISTER_P (SET_SRC (def_set)))
     607              :           return true;
     608              :         break;
     609              :       case VEC_SELECT:
     610              :         return true;
     611        10349 :       case ZERO_EXTEND:
     612        10349 :         if (GET_MODE (XEXP (SET_SRC (def_set), 0)) == DImode)
     613              :           return true;
     614              :         break;
     615      2435022 :       case PLUS:
     616      2435022 :       case IOR:
     617      2435022 :       case XOR:
     618      2435022 :         if (smode == TImode && timode_concatdi_p (SET_SRC (def_set)))
     619              :           return true;
     620              :         break;
     621              :       default:
     622              :         break;
     623              :       }
     624              : 
     625     28614249 :   for (ref = DF_INSN_UID_USES (insn_uid); ref; ref = DF_REF_NEXT_LOC (ref))
     626     15607279 :     if (DF_REF_TYPE (ref) == DF_REF_REG_USE
     627      8409517 :         && !SUBREG_P (DF_REF_REG (ref)))
     628      8409514 :       if (!analyze_register_chain (candidates, ref, disallowed))
     629              :         return false;
     630              : 
     631              :   return true;
     632              : }
     633              : 
     634              : /* Build new chain starting from insn INSN_UID recursively
     635              :    adding all dependent uses and definitions.  Return true if OK, false
     636              :    if the chain discovery was aborted.  */
     637              : 
     638              : bool
     639      6619663 : scalar_chain::build (bitmap candidates, unsigned insn_uid, bitmap disallowed)
     640              : {
     641      6619663 :   queue = BITMAP_ALLOC (NULL);
     642      6619663 :   bitmap_set_bit (queue, insn_uid);
     643              : 
     644      6619663 :   if (dump_file)
     645          139 :     fprintf (dump_file, "Building chain #%d...\n", chain_id);
     646              : 
     647     19680143 :   while (!bitmap_empty_p (queue))
     648              :     {
     649     13062319 :       insn_uid = bitmap_first_set_bit (queue);
     650     13062319 :       bitmap_clear_bit (queue, insn_uid);
     651     13062319 :       bitmap_clear_bit (candidates, insn_uid);
     652     13062319 :       if (!add_insn (candidates, insn_uid, disallowed))
     653              :         {
     654              :           /* If we aborted the search put sofar found insn on the set of
     655              :              disallowed insns so that further searches reaching them also
     656              :              abort and thus we abort the whole but yet undiscovered chain.  */
     657         1839 :           bitmap_ior_into (disallowed, insns);
     658         1839 :           if (dump_file)
     659            3 :             fprintf (dump_file, "Aborted chain #%d discovery\n", chain_id);
     660         1839 :           BITMAP_FREE (queue);
     661         1839 :           return false;
     662              :         }
     663              :     }
     664              : 
     665      6617824 :   if (dump_file)
     666              :     {
     667          136 :       fprintf (dump_file, "Collected chain #%d...\n", chain_id);
     668          136 :       fprintf (dump_file, "  insns: ");
     669          136 :       dump_bitmap (dump_file, insns);
     670          136 :       if (!bitmap_empty_p (defs_conv))
     671              :         {
     672          136 :           bitmap_iterator bi;
     673          136 :           unsigned id;
     674          136 :           const char *comma = "";
     675          136 :           fprintf (dump_file, "  defs to convert: ");
     676          362 :           EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
     677              :             {
     678          226 :               fprintf (dump_file, "%sr%d", comma, id);
     679          226 :               comma = ", ";
     680              :             }
     681          136 :           fprintf (dump_file, "\n");
     682              :         }
     683              :     }
     684              : 
     685      6617824 :   BITMAP_FREE (queue);
     686              : 
     687      6617824 :   return true;
     688              : }
     689              : 
     690              : /* Return a cost of building a vector constant
     691              :    instead of using a scalar one.  */
     692              : 
     693              : int
     694      2706354 : general_scalar_chain::vector_const_cost (rtx exp, basic_block bb)
     695              : {
     696      2706354 :   gcc_assert (CONST_INT_P (exp));
     697              : 
     698      2706354 :   if (standard_sse_constant_p (exp, vmode))
     699       611583 :     return ix86_cost->sse_op;
     700      2094771 :   if (optimize_bb_for_size_p (bb))
     701              :     return COSTS_N_BYTES (8);
     702              :   /* We have separate costs for SImode and DImode, use SImode costs
     703              :      for smaller modes.  */
     704      2471632 :   return COSTS_N_INSNS (ix86_cost->sse_load[smode == DImode ? 1 : 0]) / 2;
     705              : }
     706              : 
     707              : /* Return true if it's cost profitable for chain conversion.  */
     708              : 
     709              : bool
     710      6086668 : general_scalar_chain::compute_convert_gain ()
     711              : {
     712      6086668 :   bitmap_iterator bi;
     713      6086668 :   unsigned insn_uid;
     714      6086668 :   int gain = 0;
     715      6086668 :   sreal weighted_gain = 0;
     716              : 
     717      6086668 :   if (dump_file)
     718          136 :     fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
     719              : 
     720              :   /* SSE costs distinguish between SImode and DImode loads/stores, for
     721              :      int costs factor in the number of GPRs involved.  When supporting
     722              :      smaller modes than SImode the int load/store costs need to be
     723              :      adjusted as well.  */
     724      6086668 :   unsigned sse_cost_idx = smode == DImode ? 1 : 0;
     725      6086668 :   int m = smode == DImode ? (TARGET_64BIT ? 1 : 2) : 1;
     726              : 
     727     18083138 :   EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
     728              :     {
     729     11996470 :       rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
     730     11996470 :       rtx def_set = single_set (insn);
     731     11996470 :       rtx src = SET_SRC (def_set);
     732     11996470 :       rtx dst = SET_DEST (def_set);
     733     11996470 :       basic_block bb = BLOCK_FOR_INSN (insn);
     734     11996470 :       int igain = 0;
     735     11996470 :       profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
     736     11996470 :       bool speed_p = optimize_bb_for_speed_p (bb);
     737     11996470 :       sreal bb_freq = bb->count.to_sreal_scale (entry_count);
     738              : 
     739     11996470 :       if (REG_P (src) && REG_P (dst))
     740              :         {
     741       964941 :           if (!speed_p)
     742              :             /* reg-reg move is 2 bytes, while SSE 3.  */
     743       210937 :             igain += COSTS_N_BYTES (2 * m - 3);
     744              :           else
     745              :             /* Move costs are normalized to reg-reg move having cost 2.  */
     746       754004 :             igain += COSTS_N_INSNS (2 * m - ix86_cost->xmm_move) / 2;
     747              :         }
     748     11031529 :       else if (REG_P (src) && MEM_P (dst))
     749              :         {
     750      2406781 :           if (!speed_p)
     751              :             /* Integer load/store is 3+ bytes and SSE 4+.  */
     752       203013 :             igain += COSTS_N_BYTES (3 * m - 4);
     753              :           else
     754      2203768 :             igain
     755      2203768 :               += COSTS_N_INSNS (m * ix86_cost->int_store[2]
     756              :                                 - ix86_cost->sse_store[sse_cost_idx]) / 2;
     757              :         }
     758      8624748 :       else if (MEM_P (src) && REG_P (dst))
     759              :         {
     760      3872044 :           if (!speed_p)
     761       365627 :             igain += COSTS_N_BYTES (3 * m - 4);
     762              :           else
     763      3506417 :             igain += COSTS_N_INSNS (m * ix86_cost->int_load[2]
     764              :                                     - ix86_cost->sse_load[sse_cost_idx]) / 2;
     765              :         }
     766              :       else
     767              :         {
     768              :           /* For operations on memory operands, include the overhead
     769              :              of explicit load and store instructions.  */
     770      4752704 :           if (MEM_P (dst))
     771              :             {
     772        73276 :               if (!speed_p)
     773              :                 /* ??? This probably should account size difference
     774              :                    of SSE and integer load rather than full SSE load.  */
     775              :                 igain -= COSTS_N_BYTES (8);
     776              :               else
     777              :                 {
     778        63297 :                   int cost = (m * (ix86_cost->int_load[2]
     779        63297 :                                    + ix86_cost->int_store[2])
     780        63297 :                              - (ix86_cost->sse_load[sse_cost_idx] +
     781        63297 :                                 ix86_cost->sse_store[sse_cost_idx]));
     782        63297 :                   igain += COSTS_N_INSNS (cost) / 2;
     783              :                 }
     784              :             }
     785              : 
     786      4752704 :           switch (GET_CODE (src))
     787              :             {
     788       500580 :             case ASHIFT:
     789       500580 :             case ASHIFTRT:
     790       500580 :             case LSHIFTRT:
     791       500580 :               if (m == 2)
     792              :                 {
     793        17906 :                   if (INTVAL (XEXP (src, 1)) >= 32)
     794        12306 :                     igain += ix86_cost->add;
     795              :                   /* Gain for extend highpart case.  */
     796         5600 :                   else if (GET_CODE (XEXP (src, 0)) == ASHIFT)
     797            0 :                     igain += ix86_cost->shift_const - ix86_cost->sse_op;
     798              :                   else
     799         5600 :                     igain += ix86_cost->shift_const;
     800              :                 }
     801              : 
     802       500580 :               igain += ix86_cost->shift_const - ix86_cost->sse_op;
     803              : 
     804       500580 :               if (CONST_INT_P (XEXP (src, 0)))
     805            0 :                 igain -= vector_const_cost (XEXP (src, 0), bb);
     806              :               break;
     807              : 
     808         3942 :             case ROTATE:
     809         3942 :             case ROTATERT:
     810         3942 :               igain += m * ix86_cost->shift_const;
     811         3942 :               if (TARGET_AVX512VL)
     812          204 :                 igain -= ix86_cost->sse_op;
     813         3738 :               else if (smode == DImode)
     814              :                 {
     815          798 :                   int bits = INTVAL (XEXP (src, 1));
     816          798 :                   if ((bits & 0x0f) == 0)
     817          193 :                     igain -= ix86_cost->sse_op;
     818          605 :                   else if ((bits & 0x07) == 0)
     819           31 :                     igain -= 2 * ix86_cost->sse_op;
     820              :                   else
     821          574 :                     igain -= 3 * ix86_cost->sse_op;
     822              :                 }
     823         2940 :               else if (INTVAL (XEXP (src, 1)) == 16)
     824          139 :                 igain -= ix86_cost->sse_op;
     825              :               else
     826         2801 :                 igain -= 2 * ix86_cost->sse_op;
     827              :               break;
     828              : 
     829      2935191 :             case AND:
     830      2935191 :             case IOR:
     831      2935191 :             case XOR:
     832      2935191 :             case PLUS:
     833      2935191 :             case MINUS:
     834      2935191 :               igain += m * ix86_cost->add - ix86_cost->sse_op;
     835              :               /* Additional gain for andnot for targets without BMI.  */
     836      2935191 :               if (GET_CODE (XEXP (src, 0)) == NOT
     837         3145 :                   && !TARGET_BMI)
     838         3136 :                 igain += m * ix86_cost->add;
     839              : 
     840      2935191 :               if (CONST_INT_P (XEXP (src, 0)))
     841            0 :                 igain -= vector_const_cost (XEXP (src, 0), bb);
     842      2935191 :               if (CONST_INT_P (XEXP (src, 1)))
     843      1729881 :                 igain -= vector_const_cost (XEXP (src, 1), bb);
     844      2935191 :               if (MEM_P (XEXP (src, 1)))
     845              :                 {
     846       103098 :                   if (!speed_p)
     847        23970 :                     igain -= COSTS_N_BYTES (m == 2 ? 3 : 5);
     848              :                   else
     849        91108 :                     igain += COSTS_N_INSNS
     850              :                                (m * ix86_cost->int_load[2]
     851              :                                  - ix86_cost->sse_load[sse_cost_idx]) / 2;
     852              :                 }
     853              :               break;
     854              : 
     855        59755 :             case NEG:
     856        59755 :             case NOT:
     857        59755 :               igain -= ix86_cost->sse_op + COSTS_N_INSNS (1);
     858              : 
     859        59755 :               if (GET_CODE (XEXP (src, 0)) != ABS)
     860              :                 {
     861        59755 :                   igain += m * ix86_cost->add;
     862        59755 :                   break;
     863              :                 }
     864              :               /* FALLTHRU */
     865              : 
     866          968 :             case ABS:
     867          968 :             case SMAX:
     868          968 :             case SMIN:
     869          968 :             case UMAX:
     870          968 :             case UMIN:
     871              :               /* We do not have any conditional move cost, estimate it as a
     872              :                  reg-reg move.  Comparisons are costed as adds.  */
     873          968 :               igain += m * (COSTS_N_INSNS (2) + ix86_cost->add);
     874              :               /* Integer SSE ops are all costed the same.  */
     875          968 :               igain -= ix86_cost->sse_op;
     876          968 :               break;
     877              : 
     878            0 :             case COMPARE:
     879            0 :               if (XEXP (src, 1) != const0_rtx)
     880              :                 {
     881              :                   /* cmp vs. pxor;pshufd;ptest.  */
     882            0 :                   igain += COSTS_N_INSNS (m - 3);
     883              :                 }
     884            0 :               else if (GET_CODE (XEXP (src, 0)) != AND)
     885              :                 {
     886              :                   /* test vs. pshufd;ptest.  */
     887            0 :                   igain += COSTS_N_INSNS (m - 2);
     888              :                 }
     889            0 :               else if (GET_CODE (XEXP (XEXP (src, 0), 0)) != NOT)
     890              :                 {
     891              :                   /* and;test vs. pshufd;ptest.  */
     892            0 :                   igain += COSTS_N_INSNS (2 * m - 2);
     893              :                 }
     894            0 :               else if (TARGET_BMI)
     895              :                 {
     896              :                   /* andn;test vs. pandn;pshufd;ptest.  */
     897            0 :                   igain += COSTS_N_INSNS (2 * m - 3);
     898              :                 }
     899              :               else
     900              :                 {
     901              :                   /* not;and;test vs. pandn;pshufd;ptest.  */
     902            0 :                   igain += COSTS_N_INSNS (3 * m - 3);
     903              :                 }
     904              :               break;
     905              : 
     906      1218314 :             case CONST_INT:
     907      1218314 :               if (REG_P (dst))
     908              :                 {
     909      1218314 :                   if (!speed_p)
     910              :                     {
     911              :                       /* xor (2 bytes) vs. xorps (3 bytes).  */
     912       241841 :                       if (src == const0_rtx)
     913       126012 :                         igain -= COSTS_N_BYTES (1);
     914              :                       /* movdi_internal vs. movv2di_internal.  */
     915              :                       /* => mov (5 bytes) vs. movaps (7 bytes).  */
     916       115829 :                       else if (x86_64_immediate_operand (src, SImode))
     917       103614 :                         igain -= COSTS_N_BYTES (2);
     918              :                       else
     919              :                         /* ??? Larger immediate constants are placed in the
     920              :                            constant pool, where the size benefit/impact of
     921              :                            STV conversion is affected by whether and how
     922              :                            often each constant pool entry is shared/reused.
     923              :                            The value below is empirically derived from the
     924              :                            CSiBE benchmark (and the optimal value may drift
     925              :                            over time).  */
     926              :                         igain += COSTS_N_BYTES (0);
     927              :                     }
     928              :                   else
     929              :                     {
     930              :                       /* DImode can be immediate for TARGET_64BIT
     931              :                          and SImode always.  */
     932       976473 :                       igain += m * COSTS_N_INSNS (1);
     933       976473 :                       igain -= vector_const_cost (src, bb);
     934              :                     }
     935              :                 }
     936            0 :               else if (MEM_P (dst))
     937              :                 {
     938            0 :                   igain += (m * ix86_cost->int_store[2]
     939            0 :                             - ix86_cost->sse_store[sse_cost_idx]);
     940            0 :                   igain -= vector_const_cost (src, bb);
     941              :                 }
     942              :               break;
     943              : 
     944        23887 :             case VEC_SELECT:
     945        23887 :               if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
     946              :                 {
     947              :                   // movd (4 bytes) replaced with movdqa (4 bytes).
     948        18911 :                   if (!!speed_p)
     949        17260 :                     igain += COSTS_N_INSNS (ix86_cost->sse_to_integer
     950              :                                             - ix86_cost->xmm_move) / 2;
     951              :                 }
     952              :               else
     953              :                 {
     954              :                   // pshufd; movd replaced with pshufd.
     955         4976 :                   if (!speed_p)
     956          536 :                     igain += COSTS_N_BYTES (4);
     957              :                   else
     958         4440 :                     igain += ix86_cost->sse_to_integer;
     959              :                 }
     960              :               break;
     961              : 
     962        10067 :             case ZERO_EXTEND:
     963              :               /* mov eax (6 bytes) vs movd xmm0 (8 bytes). */
     964              :               /* mov eax; xor edx,edx (7 bytes).  */
     965        10067 :               if (speed_p)
     966         8895 :                 igain += COSTS_N_INSNS (ix86_cost->int_load[2]
     967              :                                         - ix86_cost->sse_load[0]) / 2;
     968              :               else
     969         1218 :                 igain += COSTS_N_BYTES (TARGET_64BIT ? -2 : -1);
     970              :               break;
     971              : 
     972            0 :             default:
     973            0 :               gcc_unreachable ();
     974              :             }
     975              :         }
     976              : 
     977     11994819 :       if (speed_p)
     978     10660108 :         weighted_gain += bb_freq * igain;
     979     11996470 :       gain += igain;
     980              : 
     981     11996470 :       if (igain != 0 && dump_file)
     982              :         {
     983           93 :           fprintf (dump_file, "  Instruction gain %d with bb_freq %.2f for",
     984              :                    igain, bb_freq.to_double ());
     985           93 :           dump_insn_slim (dump_file, insn);
     986              :         }
     987              :     }
     988              : 
     989      6086668 :   if (dump_file)
     990              :     {
     991          136 :       fprintf (dump_file, "  Instruction conversion gain: %d, \n",
     992              :                gain);
     993          136 :       fprintf (dump_file, "  Registers conversion cost: %d\n",
     994              :                cost_sse_integer);
     995          136 :       fprintf (dump_file, "  Weighted instruction conversion gain: %.2f, \n",
     996              :                weighted_gain.to_double ());
     997          136 :       fprintf (dump_file, "  Weighted registers conversion cost: %.2f\n",
     998              :                weighted_cost_sse_integer.to_double ());
     999              :     }
    1000              : 
    1001      6086668 :   if (weighted_gain != weighted_cost_sse_integer)
    1002      4901491 :     return weighted_gain > weighted_cost_sse_integer;
    1003              :   else
    1004      1185177 :     return gain > cost_sse_integer;;
    1005              : }
    1006              : 
    1007              : /* Insert generated conversion instruction sequence INSNS
    1008              :    after instruction AFTER.  New BB may be required in case
    1009              :    instruction has EH region attached.  */
    1010              : 
    1011              : void
    1012        32995 : scalar_chain::emit_conversion_insns (rtx_insn *insns, rtx_insn *after)
    1013              : {
    1014        32995 :   if (!control_flow_insn_p (after))
    1015              :     {
    1016        32787 :       emit_insn_after (insns, after);
    1017        32787 :       return;
    1018              :     }
    1019              : 
    1020          208 :   basic_block bb = BLOCK_FOR_INSN (after);
    1021          208 :   edge e = find_fallthru_edge (bb->succs);
    1022          208 :   gcc_assert (e);
    1023              : 
    1024          208 :   basic_block new_bb = split_edge (e);
    1025          208 :   emit_insn_after (insns, BB_HEAD (new_bb));
    1026              : }
    1027              : 
    1028              : } // anon namespace
    1029              : 
    1030              : /* Generate the canonical SET_SRC to move GPR to a VMODE vector register,
    1031              :    zeroing the upper parts.  */
    1032              : 
    1033              : static rtx
    1034       172732 : gen_gpr_to_xmm_move_src (enum machine_mode vmode, rtx gpr)
    1035              : {
    1036       345464 :   switch (GET_MODE_NUNITS (vmode))
    1037              :     {
    1038           71 :     case 1:
    1039           71 :       return gen_rtx_SUBREG (vmode, gpr, 0);
    1040       172086 :     case 2:
    1041       344172 :       return gen_rtx_VEC_CONCAT (vmode, gpr,
    1042              :                                  CONST0_RTX (GET_MODE_INNER (vmode)));
    1043          575 :     default:
    1044          575 :       return gen_rtx_VEC_MERGE (vmode, gen_rtx_VEC_DUPLICATE (vmode, gpr),
    1045              :                                 CONST0_RTX (vmode), GEN_INT (HOST_WIDE_INT_1U));
    1046              :     }
    1047              : }
    1048              : 
    1049              : /* Make vector copies for all register REGNO definitions
    1050              :    and replace its uses in a chain.  */
    1051              : 
    1052              : void
    1053         8697 : scalar_chain::make_vector_copies (rtx_insn *insn, rtx reg)
    1054              : {
    1055         8697 :   rtx vreg = *defs_map.get (reg);
    1056              : 
    1057         8697 :   start_sequence ();
    1058         8697 :   if (!TARGET_INTER_UNIT_MOVES_TO_VEC)
    1059              :     {
    1060            0 :       rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
    1061            0 :       if (smode == DImode && !TARGET_64BIT)
    1062              :         {
    1063            0 :           emit_move_insn (adjust_address (tmp, SImode, 0),
    1064              :                           gen_rtx_SUBREG (SImode, reg, 0));
    1065            0 :           emit_move_insn (adjust_address (tmp, SImode, 4),
    1066              :                           gen_rtx_SUBREG (SImode, reg, 4));
    1067              :         }
    1068              :       else
    1069            0 :         emit_move_insn (copy_rtx (tmp), reg);
    1070            0 :       emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
    1071              :                               gen_gpr_to_xmm_move_src (vmode, tmp)));
    1072              :     }
    1073         8697 :   else if (!TARGET_64BIT && smode == DImode)
    1074              :     {
    1075         8636 :       if (TARGET_SSE4_1)
    1076              :         {
    1077          280 :           emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
    1078              :                                       CONST0_RTX (V4SImode),
    1079              :                                       gen_rtx_SUBREG (SImode, reg, 0)));
    1080          280 :           emit_insn (gen_sse4_1_pinsrd (gen_rtx_SUBREG (V4SImode, vreg, 0),
    1081              :                                         gen_rtx_SUBREG (V4SImode, vreg, 0),
    1082              :                                         gen_rtx_SUBREG (SImode, reg, 4),
    1083              :                                         GEN_INT (2)));
    1084              :         }
    1085              :       else
    1086              :         {
    1087         8356 :           rtx tmp = gen_reg_rtx (DImode);
    1088         8356 :           emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, vreg, 0),
    1089              :                                       CONST0_RTX (V4SImode),
    1090              :                                       gen_rtx_SUBREG (SImode, reg, 0)));
    1091         8356 :           emit_insn (gen_sse2_loadld (gen_rtx_SUBREG (V4SImode, tmp, 0),
    1092              :                                       CONST0_RTX (V4SImode),
    1093              :                                       gen_rtx_SUBREG (SImode, reg, 4)));
    1094         8356 :           emit_insn (gen_vec_interleave_lowv4si
    1095              :                      (gen_rtx_SUBREG (V4SImode, vreg, 0),
    1096              :                       gen_rtx_SUBREG (V4SImode, vreg, 0),
    1097              :                       gen_rtx_SUBREG (V4SImode, tmp, 0)));
    1098              :         }
    1099              :     }
    1100              :   else
    1101           61 :     emit_insn (gen_rtx_SET (gen_rtx_SUBREG (vmode, vreg, 0),
    1102              :                             gen_gpr_to_xmm_move_src (vmode, reg)));
    1103         8697 :   rtx_insn *seq = end_sequence ();
    1104         8697 :   emit_conversion_insns (seq, insn);
    1105              : 
    1106         8697 :   if (dump_file)
    1107            0 :     fprintf (dump_file,
    1108              :              "  Copied r%d to a vector register r%d for insn %d\n",
    1109            0 :              REGNO (reg), REGNO (vreg), INSN_UID (insn));
    1110         8697 : }
    1111              : 
    1112              : /* Copy the definition SRC of INSN inside the chain to DST for
    1113              :    scalar uses outside of the chain.  */
    1114              : 
    1115              : void
    1116        23556 : scalar_chain::convert_reg (rtx_insn *insn, rtx dst, rtx src)
    1117              : {
    1118        23556 :   start_sequence ();
    1119        23556 :   if (!TARGET_INTER_UNIT_MOVES_FROM_VEC)
    1120              :     {
    1121            0 :       rtx tmp = assign_386_stack_local (smode, SLOT_STV_TEMP);
    1122            0 :       emit_move_insn (tmp, src);
    1123            0 :       if (!TARGET_64BIT && smode == DImode)
    1124              :         {
    1125            0 :           emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
    1126              :                           adjust_address (tmp, SImode, 0));
    1127            0 :           emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
    1128              :                           adjust_address (tmp, SImode, 4));
    1129              :         }
    1130              :       else
    1131            0 :         emit_move_insn (dst, copy_rtx (tmp));
    1132              :     }
    1133        23556 :   else if (!TARGET_64BIT && smode == DImode)
    1134              :     {
    1135        22531 :       if (TARGET_SSE4_1)
    1136              :         {
    1137            0 :           rtx tmp = gen_rtx_PARALLEL (VOIDmode,
    1138              :                                       gen_rtvec (1, const0_rtx));
    1139            0 :           emit_insn
    1140            0 :               (gen_rtx_SET
    1141              :                (gen_rtx_SUBREG (SImode, dst, 0),
    1142              :                 gen_rtx_VEC_SELECT (SImode,
    1143              :                                     gen_rtx_SUBREG (V4SImode, src, 0),
    1144              :                                     tmp)));
    1145              : 
    1146            0 :           tmp = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (1, const1_rtx));
    1147            0 :           emit_insn
    1148            0 :               (gen_rtx_SET
    1149              :                (gen_rtx_SUBREG (SImode, dst, 4),
    1150              :                 gen_rtx_VEC_SELECT (SImode,
    1151              :                                     gen_rtx_SUBREG (V4SImode, src, 0),
    1152              :                                     tmp)));
    1153              :         }
    1154              :       else
    1155              :         {
    1156        22531 :           rtx vcopy = gen_reg_rtx (V2DImode);
    1157        22531 :           emit_move_insn (vcopy, gen_rtx_SUBREG (V2DImode, src, 0));
    1158        22531 :           emit_move_insn (gen_rtx_SUBREG (SImode, dst, 0),
    1159              :                           gen_rtx_SUBREG (SImode, vcopy, 0));
    1160        22531 :           emit_move_insn (vcopy,
    1161              :                           gen_rtx_LSHIFTRT (V2DImode,
    1162              :                                             vcopy, GEN_INT (32)));
    1163        22531 :           emit_move_insn (gen_rtx_SUBREG (SImode, dst, 4),
    1164              :                           gen_rtx_SUBREG (SImode, vcopy, 0));
    1165              :         }
    1166              :     }
    1167              :   else
    1168         1025 :     emit_move_insn (dst, src);
    1169              : 
    1170        23556 :   rtx_insn *seq = end_sequence ();
    1171        23556 :   emit_conversion_insns (seq, insn);
    1172              : 
    1173        23556 :   if (dump_file)
    1174            0 :     fprintf (dump_file,
    1175              :              "  Copied r%d to a scalar register r%d for insn %d\n",
    1176            0 :              REGNO (src), REGNO (dst), INSN_UID (insn));
    1177        23556 : }
    1178              : 
    1179              : /* Helper function to convert immediate constant X to vmode.  */
    1180              : static rtx
    1181        38020 : smode_convert_cst (rtx x, enum machine_mode vmode)
    1182              : {
    1183              :   /* Prefer all ones vector in case of -1.  */
    1184        38020 :   if (constm1_operand (x, GET_MODE (x)))
    1185          625 :     return CONSTM1_RTX (vmode);
    1186              : 
    1187        37395 :   unsigned n = GET_MODE_NUNITS (vmode);
    1188        37395 :   rtx *v = XALLOCAVEC (rtx, n);
    1189        37395 :   v[0] = x;
    1190        43432 :   for (unsigned i = 1; i < n; ++i)
    1191         6037 :     v[i] = const0_rtx;
    1192        37395 :   return gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
    1193              : }
    1194              : 
    1195              : /* Convert operand OP in INSN.  We should handle
    1196              :    memory operands and uninitialized registers.
    1197              :    All other register uses are converted during
    1198              :    registers conversion.  */
    1199              : 
    1200              : void
    1201       239534 : scalar_chain::convert_op (rtx *op, rtx_insn *insn)
    1202              : {
    1203       239534 :   rtx tmp;
    1204              : 
    1205       239534 :   if (GET_MODE (*op) == V1TImode)
    1206              :     return;
    1207              : 
    1208       239243 :   *op = copy_rtx_if_shared (*op);
    1209              : 
    1210       239243 :   if (GET_CODE (*op) == NOT
    1211       239243 :       || GET_CODE (*op) == ASHIFT)
    1212              :     {
    1213         2892 :       convert_op (&XEXP (*op, 0), insn);
    1214         2892 :       PUT_MODE (*op, vmode);
    1215              :     }
    1216              :   else if (MEM_P (*op))
    1217              :     {
    1218       172671 :       rtx_insn *movabs = NULL;
    1219              : 
    1220              :       /* Emit MOVABS to load from a 64-bit absolute address to a GPR.  */
    1221       172671 :       if (!memory_operand (*op, GET_MODE (*op)))
    1222              :         {
    1223            0 :           tmp = gen_reg_rtx (GET_MODE (*op));
    1224            0 :           movabs = emit_insn_before (gen_rtx_SET (tmp, *op), insn);
    1225              : 
    1226            0 :           *op = tmp;
    1227              :         }
    1228              : 
    1229       172671 :       tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (GET_MODE (*op)), 0);
    1230              : 
    1231       172671 :       rtx_insn *eh_insn
    1232       172671 :         = emit_insn_before (gen_rtx_SET (copy_rtx (tmp),
    1233              :                                          gen_gpr_to_xmm_move_src (vmode, *op)),
    1234       172671 :                             insn);
    1235              : 
    1236       172671 :       if (cfun->can_throw_non_call_exceptions)
    1237              :         {
    1238              :           /* Handle REG_EH_REGION note.  */
    1239       168367 :           rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
    1240       168367 :           if (note)
    1241              :             {
    1242         3578 :               if (movabs)
    1243            0 :                 eh_insn = movabs;
    1244         3578 :               control_flow_insns.safe_push (eh_insn);
    1245         3578 :               add_reg_note (eh_insn, REG_EH_REGION, XEXP (note, 0));
    1246              :             }
    1247              :         }
    1248              : 
    1249       172671 :       *op = tmp;
    1250              : 
    1251       172671 :       if (dump_file)
    1252            0 :         fprintf (dump_file, "  Preloading operand for insn %d into r%d\n",
    1253            0 :                  INSN_UID (insn), reg_or_subregno (tmp));
    1254              :     }
    1255              :   else if (REG_P (*op))
    1256        57156 :     *op = gen_rtx_SUBREG (vmode, *op, 0);
    1257              :   else if (CONST_SCALAR_INT_P (*op))
    1258              :     {
    1259         6521 :       rtx vec_cst = smode_convert_cst (*op, vmode);
    1260              : 
    1261         6521 :       if (!standard_sse_constant_p (vec_cst, vmode))
    1262              :         {
    1263         2711 :           start_sequence ();
    1264         2711 :           vec_cst = validize_mem (force_const_mem (vmode, vec_cst));
    1265         2711 :           rtx_insn *seq = end_sequence ();
    1266         2711 :           emit_insn_before (seq, insn);
    1267              :         }
    1268              : 
    1269         6521 :       tmp = gen_rtx_SUBREG (vmode, gen_reg_rtx (smode), 0);
    1270              : 
    1271         6521 :       emit_insn_before (gen_move_insn (copy_rtx (tmp), vec_cst), insn);
    1272         6521 :       *op = tmp;
    1273              :     }
    1274              :   else
    1275              :     {
    1276            0 :       gcc_assert (SUBREG_P (*op));
    1277            3 :       if (GET_MODE (*op) != vmode)
    1278              :         {
    1279            3 :           rtx inner = SUBREG_REG (*op);
    1280            3 :           poly_uint64 byte = SUBREG_BYTE (*op);
    1281            3 :           if (targetm.modes_tieable_p (vmode, GET_MODE (inner))
    1282            3 :               && validate_subreg (vmode, GET_MODE (inner), inner, byte))
    1283            3 :             *op = gen_lowpart (vmode, *op);
    1284              :           else
    1285              :             {
    1286            0 :               tmp = gen_reg_rtx (GET_MODE (*op));
    1287            0 :               emit_insn_before (gen_rtx_SET (tmp, *op), insn);
    1288            0 :               *op = gen_rtx_SUBREG (vmode, tmp, 0);
    1289              :             }
    1290              :         }
    1291              :     }
    1292              : }
    1293              : 
    1294              : /* Convert CCZmode COMPARE to vector mode.  */
    1295              : 
    1296              : rtx
    1297            8 : scalar_chain::convert_compare (rtx op1, rtx op2, rtx_insn *insn)
    1298              : {
    1299            8 :   rtx src, tmp;
    1300              : 
    1301              :   /* Handle any REG_EQUAL notes.  */
    1302            8 :   tmp = find_reg_equal_equiv_note (insn);
    1303            8 :   if (tmp)
    1304              :     {
    1305            1 :       if (GET_CODE (XEXP (tmp, 0)) == COMPARE
    1306            1 :           && GET_MODE (XEXP (tmp, 0)) == CCZmode
    1307            1 :           && REG_P (XEXP (XEXP (tmp, 0), 0)))
    1308              :         {
    1309            1 :           rtx *op = &XEXP (XEXP (tmp, 0), 1);
    1310            1 :           if (CONST_SCALAR_INT_P (*op))
    1311              :             {
    1312            1 :               if (constm1_operand (*op, GET_MODE (*op)))
    1313            0 :                 *op = CONSTM1_RTX (vmode);
    1314              :               else
    1315              :                 {
    1316            1 :                   unsigned n = GET_MODE_NUNITS (vmode);
    1317            1 :                   rtx *v = XALLOCAVEC (rtx, n);
    1318            1 :                   v[0] = *op;
    1319            1 :                   for (unsigned i = 1; i < n; ++i)
    1320            0 :                     v[i] = const0_rtx;
    1321            1 :                   *op = gen_rtx_CONST_VECTOR (vmode, gen_rtvec_v (n, v));
    1322              :                 }
    1323              :               tmp = NULL_RTX;
    1324              :             }
    1325            0 :           else if (REG_P (*op))
    1326              :             tmp = NULL_RTX;
    1327              :         }
    1328              : 
    1329            1 :       if (tmp)
    1330            0 :         remove_note (insn, tmp);
    1331              :     }
    1332              : 
    1333              :   /* Comparison against anything other than zero, requires an XOR.  */
    1334            8 :   if (op2 != const0_rtx)
    1335              :     {
    1336            6 :       convert_op (&op1, insn);
    1337            6 :       convert_op (&op2, insn);
    1338              :       /* If both operands are MEMs, explicitly load the OP1 into TMP.  */
    1339            6 :       if (MEM_P (op1) && MEM_P (op2))
    1340              :         {
    1341            0 :           tmp = gen_reg_rtx (vmode);
    1342            0 :           emit_insn_before (gen_rtx_SET (tmp, op1), insn);
    1343            0 :           src = tmp;
    1344              :         }
    1345              :       else
    1346              :         src = op1;
    1347            6 :       src = gen_rtx_XOR (vmode, src, op2);
    1348              :     }
    1349            2 :   else if (GET_CODE (op1) == AND
    1350            0 :            && GET_CODE (XEXP (op1, 0)) == NOT)
    1351              :     {
    1352            0 :       rtx op11 = XEXP (XEXP (op1, 0), 0);
    1353            0 :       rtx op12 = XEXP (op1, 1);
    1354            0 :       convert_op (&op11, insn);
    1355            0 :       convert_op (&op12, insn);
    1356            0 :       if (!REG_P (op11))
    1357              :         {
    1358            0 :           tmp = gen_reg_rtx (vmode);
    1359            0 :           emit_insn_before (gen_rtx_SET (tmp, op11), insn);
    1360            0 :           op11 = tmp;
    1361              :         }
    1362            0 :       src = gen_rtx_AND (vmode, gen_rtx_NOT (vmode, op11), op12);
    1363            0 :     }
    1364            2 :   else if (GET_CODE (op1) == AND)
    1365              :     {
    1366            0 :       rtx op11 = XEXP (op1, 0);
    1367            0 :       rtx op12 = XEXP (op1, 1);
    1368            0 :       convert_op (&op11, insn);
    1369            0 :       convert_op (&op12, insn);
    1370            0 :       if (!REG_P (op11))
    1371              :         {
    1372            0 :           tmp = gen_reg_rtx (vmode);
    1373            0 :           emit_insn_before (gen_rtx_SET (tmp, op11), insn);
    1374            0 :           op11 = tmp;
    1375              :         }
    1376            0 :       return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, op11, op12),
    1377              :                              UNSPEC_PTEST);
    1378              :     }
    1379              :   else
    1380              :     {
    1381            2 :       convert_op (&op1, insn);
    1382            2 :       src = op1;
    1383              :     }
    1384              : 
    1385            8 :   if (!REG_P (src))
    1386              :     {
    1387            6 :       tmp = gen_reg_rtx (vmode);
    1388            6 :       emit_insn_before (gen_rtx_SET (tmp, src), insn);
    1389            6 :       src = tmp;
    1390              :     }
    1391              : 
    1392            8 :   if (vmode == V2DImode)
    1393              :     {
    1394            0 :       tmp = gen_reg_rtx (vmode);
    1395            0 :       emit_insn_before (gen_vec_interleave_lowv2di (tmp, src, src), insn);
    1396            0 :       src = tmp;
    1397              :     }
    1398            8 :   else if (vmode == V4SImode)
    1399              :     {
    1400            0 :       tmp = gen_reg_rtx (vmode);
    1401            0 :       emit_insn_before (gen_sse2_pshufd (tmp, src, const0_rtx), insn);
    1402            0 :       src = tmp;
    1403              :     }
    1404              : 
    1405            8 :   return gen_rtx_UNSPEC (CCZmode, gen_rtvec (2, src, src), UNSPEC_PTEST);
    1406              : }
    1407              : 
    1408              : /* Helper function for converting INSN to vector mode.  */
    1409              : 
    1410              : void
    1411      1373128 : scalar_chain::convert_insn_common (rtx_insn *insn)
    1412              : {
    1413              :   /* Generate copies for out-of-chain uses of defs and adjust debug uses.  */
    1414      2093317 :   for (df_ref ref = DF_INSN_DEFS (insn); ref; ref = DF_REF_NEXT_LOC (ref))
    1415       720189 :     if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
    1416              :       {
    1417        24976 :         df_link *use;
    1418        49087 :         for (use = DF_REF_CHAIN (ref); use; use = use->next)
    1419        47667 :           if (NONDEBUG_INSN_P (DF_REF_INSN (use->ref))
    1420        47667 :               && (DF_REF_REG_MEM_P (use->ref)
    1421        41719 :                   || !bitmap_bit_p (insns, DF_REF_INSN_UID (use->ref))))
    1422              :             break;
    1423        24976 :         if (use)
    1424        23556 :           convert_reg (insn, DF_REF_REG (ref),
    1425        23556 :                        *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]));
    1426         1420 :         else if (MAY_HAVE_DEBUG_BIND_INSNS)
    1427              :           {
    1428              :             /* If we generated a scalar copy we can leave debug-insns
    1429              :                as-is, if not, we have to adjust them.  */
    1430         1284 :             auto_vec<rtx_insn *, 5> to_reset_debug_insns;
    1431         3920 :             for (use = DF_REF_CHAIN (ref); use; use = use->next)
    1432         2636 :               if (DEBUG_INSN_P (DF_REF_INSN (use->ref)))
    1433              :                 {
    1434          831 :                   rtx_insn *debug_insn = DF_REF_INSN (use->ref);
    1435              :                   /* If there's a reaching definition outside of the
    1436              :                      chain we have to reset.  */
    1437          831 :                   df_link *def;
    1438         2909 :                   for (def = DF_REF_CHAIN (use->ref); def; def = def->next)
    1439         2248 :                     if (!bitmap_bit_p (insns, DF_REF_INSN_UID (def->ref)))
    1440              :                       break;
    1441          831 :                   if (def)
    1442          170 :                     to_reset_debug_insns.safe_push (debug_insn);
    1443              :                   else
    1444              :                     {
    1445          661 :                       *DF_REF_REAL_LOC (use->ref)
    1446          661 :                         = *defs_map.get (regno_reg_rtx [DF_REF_REGNO (ref)]);
    1447          661 :                       df_insn_rescan (debug_insn);
    1448              :                     }
    1449              :                 }
    1450              :             /* Have to do the reset outside of the DF_CHAIN walk to not
    1451              :                disrupt it.  */
    1452         2738 :             while (!to_reset_debug_insns.is_empty ())
    1453              :               {
    1454          170 :                 rtx_insn *debug_insn = to_reset_debug_insns.pop ();
    1455          170 :                 INSN_VAR_LOCATION_LOC (debug_insn) = gen_rtx_UNKNOWN_VAR_LOC ();
    1456          170 :                 df_insn_rescan_debug_internal (debug_insn);
    1457              :               }
    1458         1284 :           }
    1459              :       }
    1460              : 
    1461              :   /* Replace uses in this insn with the defs we use in the chain.  */
    1462      3421591 :   for (df_ref ref = DF_INSN_USES (insn); ref; ref = DF_REF_NEXT_LOC (ref))
    1463      2048463 :     if (!DF_REF_REG_MEM_P (ref))
    1464       731677 :       if (rtx *vreg = defs_map.get (regno_reg_rtx[DF_REF_REGNO (ref)]))
    1465              :         {
    1466              :           /* Also update a corresponding REG_DEAD note.  */
    1467        36247 :           rtx note = find_reg_note (insn, REG_DEAD, DF_REF_REG (ref));
    1468        36247 :           if (note)
    1469        22863 :             XEXP (note, 0) = *vreg;
    1470        36247 :           *DF_REF_REAL_LOC (ref) = *vreg;
    1471              :         }
    1472      1373128 : }
    1473              : 
    1474              : /* Convert INSN which is an SImode or DImode rotation by a constant
    1475              :    to vector mode.  CODE is either ROTATE or ROTATERT with operands
    1476              :    OP0 and OP1.  Returns the SET_SRC of the last instruction in the
    1477              :    resulting sequence, which is emitted before INSN.  */
    1478              : 
    1479              : rtx
    1480           92 : general_scalar_chain::convert_rotate (enum rtx_code code, rtx op0, rtx op1,
    1481              :                                       rtx_insn *insn)
    1482              : {
    1483           92 :   int bits = INTVAL (op1);
    1484           92 :   rtx pat, result;
    1485              : 
    1486           92 :   convert_op (&op0, insn);
    1487           92 :   if (bits == 0)
    1488            0 :     return op0;
    1489              : 
    1490           92 :   if (smode == DImode)
    1491              :     {
    1492           92 :       if (code == ROTATE)
    1493           45 :         bits = 64 - bits;
    1494           92 :       if (bits == 32)
    1495              :         {
    1496            0 :           rtx tmp1 = gen_reg_rtx (V4SImode);
    1497            0 :           pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
    1498              :                                  GEN_INT (225));
    1499            0 :           emit_insn_before (pat, insn);
    1500            0 :           result = gen_lowpart (V2DImode, tmp1);
    1501              :         }
    1502           92 :       else if (TARGET_AVX512VL)
    1503            0 :         result = simplify_gen_binary (code, V2DImode, op0, op1);
    1504           92 :       else if (bits == 16 || bits == 48)
    1505              :         {
    1506            0 :           rtx tmp1 = gen_reg_rtx (V8HImode);
    1507            0 :           pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0),
    1508              :                                   GEN_INT (bits == 16 ? 57 : 147));
    1509            0 :           emit_insn_before (pat, insn);
    1510            0 :           result = gen_lowpart (V2DImode, tmp1);
    1511              :         }
    1512           92 :       else if ((bits & 0x07) == 0)
    1513              :         {
    1514            0 :           rtx tmp1 = gen_reg_rtx (V4SImode);
    1515            0 :           pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
    1516              :                                  GEN_INT (68));
    1517            0 :           emit_insn_before (pat, insn);
    1518            0 :           rtx tmp2 = gen_reg_rtx (V1TImode);
    1519            0 :           pat = gen_sse2_lshrv1ti3 (tmp2, gen_lowpart (V1TImode, tmp1),
    1520              :                                     GEN_INT (bits));
    1521            0 :           emit_insn_before (pat, insn);
    1522            0 :           result = gen_lowpart (V2DImode, tmp2);
    1523              :         }
    1524              :       else
    1525              :         {
    1526           92 :           rtx tmp1 = gen_reg_rtx (V4SImode);
    1527           92 :           pat = gen_sse2_pshufd (tmp1, gen_lowpart (V4SImode, op0),
    1528              :                                  GEN_INT (20));
    1529           92 :           emit_insn_before (pat, insn);
    1530           92 :           rtx tmp2 = gen_reg_rtx (V2DImode);
    1531           92 :           pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
    1532              :                                GEN_INT (bits & 31));
    1533           92 :           emit_insn_before (pat, insn);
    1534           92 :           rtx tmp3 = gen_reg_rtx (V4SImode);
    1535          139 :           pat = gen_sse2_pshufd (tmp3, gen_lowpart (V4SImode, tmp2),
    1536              :                                  GEN_INT (bits > 32 ? 34 : 136));
    1537           92 :           emit_insn_before (pat, insn);
    1538           92 :           result = gen_lowpart (V2DImode, tmp3);
    1539              :         }
    1540              :     }
    1541            0 :   else if (bits == 16)
    1542              :     {
    1543            0 :       rtx tmp1 = gen_reg_rtx (V8HImode);
    1544            0 :       pat = gen_sse2_pshuflw (tmp1, gen_lowpart (V8HImode, op0), GEN_INT (225));
    1545            0 :       emit_insn_before (pat, insn);
    1546            0 :       result = gen_lowpart (V4SImode, tmp1);
    1547              :     }
    1548            0 :   else if (TARGET_AVX512VL)
    1549            0 :     result = simplify_gen_binary (code, V4SImode, op0, op1);
    1550              :   else
    1551              :     {
    1552            0 :       if (code == ROTATE)
    1553            0 :         bits = 32 - bits;
    1554              : 
    1555            0 :       rtx tmp1 = gen_reg_rtx (V4SImode);
    1556            0 :       emit_insn_before (gen_sse2_pshufd (tmp1, op0, GEN_INT (224)), insn);
    1557            0 :       rtx tmp2 = gen_reg_rtx (V2DImode);
    1558            0 :       pat = gen_lshrv2di3 (tmp2, gen_lowpart (V2DImode, tmp1),
    1559              :                            GEN_INT (bits));
    1560            0 :       emit_insn_before (pat, insn);
    1561            0 :       result = gen_lowpart (V4SImode, tmp2);
    1562              :     }
    1563              : 
    1564              :   return result;
    1565              : }
    1566              : 
    1567              : /* Convert INSN to vector mode.  */
    1568              : 
    1569              : void
    1570       407478 : general_scalar_chain::convert_insn (rtx_insn *insn)
    1571              : {
    1572       407478 :   rtx def_set = single_set (insn);
    1573       407478 :   rtx src = SET_SRC (def_set);
    1574       407478 :   rtx dst = SET_DEST (def_set);
    1575       407478 :   rtx subreg;
    1576              : 
    1577       407478 :   if (MEM_P (dst) && !REG_P (src))
    1578              :     {
    1579              :       /* There are no scalar integer instructions and therefore
    1580              :          temporary register usage is required.  */
    1581          742 :       rtx tmp = gen_reg_rtx (smode);
    1582          742 :       emit_conversion_insns (gen_move_insn (dst, tmp), insn);
    1583          742 :       dst = gen_rtx_SUBREG (vmode, tmp, 0);
    1584          742 :     }
    1585       406736 :   else if (REG_P (dst) && GET_MODE (dst) == smode)
    1586              :     {
    1587              :       /* Replace the definition with a SUBREG to the definition we
    1588              :          use inside the chain.  */
    1589       212003 :       rtx *vdef = defs_map.get (dst);
    1590       212003 :       if (vdef)
    1591        24976 :         dst = *vdef;
    1592       212003 :       dst = gen_rtx_SUBREG (vmode, dst, 0);
    1593              :       /* IRA doesn't like to have REG_EQUAL/EQUIV notes when the SET_DEST
    1594              :          is a non-REG_P.  So kill those off.  */
    1595       212003 :       rtx note = find_reg_equal_equiv_note (insn);
    1596       212003 :       if (note)
    1597         9636 :         remove_note (insn, note);
    1598              :     }
    1599              : 
    1600       407478 :   switch (GET_CODE (src))
    1601              :     {
    1602        26599 :     case PLUS:
    1603        26599 :     case MINUS:
    1604        26599 :     case IOR:
    1605        26599 :     case XOR:
    1606        26599 :     case AND:
    1607        26599 :     case SMAX:
    1608        26599 :     case SMIN:
    1609        26599 :     case UMAX:
    1610        26599 :     case UMIN:
    1611        26599 :       convert_op (&XEXP (src, 1), insn);
    1612              :       /* FALLTHRU */
    1613              : 
    1614        33200 :     case ABS:
    1615        33200 :     case ASHIFT:
    1616        33200 :     case ASHIFTRT:
    1617        33200 :     case LSHIFTRT:
    1618        33200 :       convert_op (&XEXP (src, 0), insn);
    1619        33200 :       PUT_MODE (src, vmode);
    1620        33200 :       break;
    1621              : 
    1622           92 :     case ROTATE:
    1623           92 :     case ROTATERT:
    1624           92 :       src = convert_rotate (GET_CODE (src), XEXP (src, 0), XEXP (src, 1),
    1625              :                             insn);
    1626           92 :       break;
    1627              : 
    1628          356 :     case NEG:
    1629          356 :       src = XEXP (src, 0);
    1630              : 
    1631          356 :       if (GET_CODE (src) == ABS)
    1632              :         {
    1633            0 :           src = XEXP (src, 0);
    1634            0 :           convert_op (&src, insn);
    1635            0 :           subreg = gen_reg_rtx (vmode);
    1636            0 :           emit_insn_before (gen_rtx_SET (subreg,
    1637              :                                          gen_rtx_ABS (vmode, src)), insn);
    1638            0 :           src = subreg;
    1639              :         }
    1640              :       else
    1641          356 :         convert_op (&src, insn);
    1642              : 
    1643          356 :       subreg = gen_reg_rtx (vmode);
    1644          356 :       emit_insn_before (gen_move_insn (subreg, CONST0_RTX (vmode)), insn);
    1645          356 :       src = gen_rtx_MINUS (vmode, subreg, src);
    1646          356 :       break;
    1647              : 
    1648          219 :     case NOT:
    1649          219 :       src = XEXP (src, 0);
    1650          219 :       convert_op (&src, insn);
    1651          219 :       subreg = gen_reg_rtx (vmode);
    1652          219 :       emit_insn_before (gen_move_insn (subreg, CONSTM1_RTX (vmode)), insn);
    1653          219 :       src = gen_rtx_XOR (vmode, src, subreg);
    1654          219 :       break;
    1655              : 
    1656       170464 :     case MEM:
    1657       170464 :       if (!REG_P (dst))
    1658       170464 :         convert_op (&src, insn);
    1659              :       break;
    1660              : 
    1661       196259 :     case REG:
    1662       196259 :       if (!MEM_P (dst))
    1663         1526 :         convert_op (&src, insn);
    1664              :       break;
    1665              : 
    1666            0 :     case SUBREG:
    1667            0 :       gcc_assert (GET_MODE (src) == vmode);
    1668              :       break;
    1669              : 
    1670            0 :     case COMPARE:
    1671            0 :       dst = gen_rtx_REG (CCZmode, FLAGS_REG);
    1672            0 :       src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
    1673            0 :       break;
    1674              : 
    1675         3799 :     case CONST_INT:
    1676         3799 :       convert_op (&src, insn);
    1677         3799 :       break;
    1678              : 
    1679         2749 :     case VEC_SELECT:
    1680         2749 :       if (XVECEXP (XEXP (src, 1), 0, 0) == const0_rtx)
    1681         1585 :         src = XEXP (src, 0);
    1682         1164 :       else if (smode == DImode)
    1683              :         {
    1684          792 :           rtx tmp = gen_lowpart (V1TImode, XEXP (src, 0));
    1685          792 :           dst = gen_lowpart (V1TImode, dst);
    1686          792 :           src = gen_rtx_LSHIFTRT (V1TImode, tmp, GEN_INT (64));
    1687              :         }
    1688              :       else
    1689              :         {
    1690          372 :           rtx tmp = XVECEXP (XEXP (src, 1), 0, 0);
    1691          372 :           rtvec vec = gen_rtvec (4, tmp, tmp, tmp, tmp);
    1692          372 :           rtx par = gen_rtx_PARALLEL (VOIDmode, vec);
    1693          372 :           src = gen_rtx_VEC_SELECT (vmode, XEXP (src, 0), par);
    1694              :         }
    1695              :       break;
    1696              : 
    1697          340 :     case ZERO_EXTEND:
    1698              :       /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1.  */
    1699          340 :       src = gen_rtx_VEC_CONCAT (V2DImode, src, const0_rtx);
    1700          340 :       break;
    1701              : 
    1702            0 :     default:
    1703            0 :       gcc_unreachable ();
    1704              :     }
    1705              : 
    1706       407478 :   SET_SRC (def_set) = src;
    1707       407478 :   SET_DEST (def_set) = dst;
    1708              : 
    1709              :   /* Drop possible dead definitions.  */
    1710       407478 :   PATTERN (insn) = def_set;
    1711              : 
    1712       407478 :   INSN_CODE (insn) = -1;
    1713       407478 :   int patt = recog_memoized (insn);
    1714       407478 :   if  (patt == -1)
    1715            0 :     fatal_insn_not_found (insn);
    1716       407478 :   df_insn_rescan (insn);
    1717       407478 : }
    1718              : 
    1719              : /* Helper function to compute gain for loading an immediate constant.
    1720              :    Typically, two movabsq for TImode vs. vmovdqa for V1TImode, but
    1721              :    with numerous special cases.  */
    1722              : 
    1723              : static int
    1724           34 : timode_immed_const_gain (rtx cst, basic_block bb)
    1725              : {
    1726              :   /* movabsq vs. movabsq+vmovq+vunpacklqdq.  */
    1727           34 :   if (CONST_WIDE_INT_P (cst)
    1728           23 :       && CONST_WIDE_INT_NUNITS (cst) == 2
    1729           57 :       && CONST_WIDE_INT_ELT (cst, 0) == CONST_WIDE_INT_ELT (cst, 1))
    1730            0 :     return optimize_bb_for_size_p (bb) ? -COSTS_N_BYTES (9)
    1731              :                                        : -COSTS_N_INSNS (2);
    1732              :   /* 2x movabsq ~ vmovdqa.  */
    1733              :   return 0;
    1734              : }
    1735              : 
    1736              : /* Return true it's cost profitable for for chain conversion.  */
    1737              : 
    1738              : bool
    1739       531156 : timode_scalar_chain::compute_convert_gain ()
    1740              : {
    1741              :   /* Assume that if we have to move TImode values between units,
    1742              :      then transforming this chain isn't worth it.  */
    1743       531156 :   if (cost_sse_integer)
    1744              :     return false;
    1745              : 
    1746       531156 :   bitmap_iterator bi;
    1747       531156 :   unsigned insn_uid;
    1748              : 
    1749              :   /* Split ties to prefer V1TImode when not optimizing for size.  */
    1750       531156 :   int gain = optimize_size ? 0 : 1;
    1751       531156 :   sreal weighted_gain  = 0;
    1752              : 
    1753       531156 :   if (dump_file)
    1754            0 :     fprintf (dump_file, "Computing gain for chain #%d...\n", chain_id);
    1755              : 
    1756      1586613 :   EXECUTE_IF_SET_IN_BITMAP (insns, 0, insn_uid, bi)
    1757              :     {
    1758      1055457 :       rtx_insn *insn = DF_INSN_UID_GET (insn_uid)->insn;
    1759      1055457 :       rtx def_set = single_set (insn);
    1760      1055457 :       rtx src = SET_SRC (def_set);
    1761      1055457 :       rtx dst = SET_DEST (def_set);
    1762      1055457 :       HOST_WIDE_INT op1val;
    1763      1055457 :       basic_block bb = BLOCK_FOR_INSN (insn);
    1764      1055457 :       int scost, vcost;
    1765      1055457 :       int igain = 0;
    1766      1055457 :       profile_count entry_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
    1767      1055457 :       bool speed_p = optimize_bb_for_speed_p (bb);
    1768      1055457 :       sreal bb_freq = bb->count.to_sreal_scale (entry_count);
    1769              : 
    1770      1055457 :       switch (GET_CODE (src))
    1771              :         {
    1772       548522 :         case REG:
    1773       548522 :           if (GENERAL_REGNO_P (REGNO (src)))
    1774              :             {
    1775        24975 :               if (TARGET_SSE4_1)
    1776              :                 /* movq + pinsrq */
    1777           27 :                 igain = speed_p ? -ix86_cost->integer_to_sse
    1778              :                                   - COSTS_N_INSNS (1)
    1779              :                                 : -COSTS_N_BYTES (11);
    1780              :               else
    1781              :                 /* movq + movq + punpcklqdq */
    1782        24948 :                 igain = speed_p ? -ix86_cost->integer_to_sse
    1783              :                                   - COSTS_N_INSNS (2)
    1784              :                                 : -COSTS_N_BYTES (14);
    1785              :             }
    1786       523547 :           else if (GENERAL_REG_P (dst))
    1787              :             {
    1788        25108 :               if (TARGET_SSE4_1)
    1789              :                 /* pextrq + movq */
    1790           27 :                 igain = speed_p ? -ix86_cost->sse_to_integer
    1791              :                                   - COSTS_N_INSNS (1)
    1792              :                                 : -COSTS_N_BYTES (11);
    1793              :               else
    1794              :                 /* movhlps + movq + movq */
    1795        25081 :                 igain = speed_p ? -ix86_cost->sse_to_integer
    1796              :                                   - COSTS_N_INSNS (2)
    1797              :                                 : -COSTS_N_BYTES (13);
    1798              :             }
    1799       498439 :           else if (!speed_p)
    1800        15764 :             igain = MEM_P (dst) ? COSTS_N_BYTES (6) : COSTS_N_BYTES (3);
    1801              :           else
    1802              :             igain = COSTS_N_INSNS (1);
    1803              :           break;
    1804              : 
    1805       456995 :         case MEM:
    1806       456995 :           igain = !speed_p ? COSTS_N_BYTES (7) : COSTS_N_INSNS (1);
    1807              :           break;
    1808              : 
    1809        12756 :         case CONST_INT:
    1810        12756 :           if (MEM_P (dst)
    1811        12756 :               && standard_sse_constant_p (src, V1TImode))
    1812        12228 :             igain = !speed_p ? COSTS_N_BYTES (11) : 1;
    1813              :           break;
    1814              : 
    1815        31876 :         case CONST_WIDE_INT:
    1816              :           /* 2 x mov vs. vmovdqa.  */
    1817        31876 :           if (MEM_P (dst))
    1818        31305 :             igain = !speed_p ? COSTS_N_BYTES (3) : COSTS_N_INSNS (1);
    1819              :           break;
    1820              : 
    1821           78 :         case NOT:
    1822           78 :           if (MEM_P (dst))
    1823        68575 :             igain = -COSTS_N_INSNS (1);
    1824              :           break;
    1825              : 
    1826           69 :         case AND:
    1827           69 :           if (!MEM_P (dst))
    1828              :             {
    1829           58 :               igain = COSTS_N_INSNS (1);
    1830           58 :               if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
    1831              :                 /* One less load.  */
    1832           20 :                 igain += COSTS_N_INSNS (1);
    1833              :             }
    1834           69 :           if (CONST_SCALAR_INT_P (XEXP (src, 1)))
    1835           27 :             igain += timode_immed_const_gain (XEXP (src, 1), bb);
    1836              :           break;
    1837              : 
    1838         4506 :         case XOR:
    1839         4506 :         case IOR:
    1840         4506 :           if (timode_concatdi_p (src))
    1841              :             {
    1842              :               /* vmovq;vpinsrq (11 bytes).  */
    1843         4349 :               igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
    1844              :                               : -COSTS_N_BYTES (11);
    1845              :               break;
    1846              :             }
    1847          157 :           if (!MEM_P (dst))
    1848              :             {
    1849          149 :               igain = COSTS_N_INSNS (1);
    1850          149 :               if (MEM_P (XEXP (src, 0)) || MEM_P (XEXP (src, 1)))
    1851              :                 /* One less load.  */
    1852           48 :                 igain += COSTS_N_INSNS (1);
    1853              :             }
    1854          157 :           if (CONST_SCALAR_INT_P (XEXP (src, 1)))
    1855            7 :             igain += timode_immed_const_gain (XEXP (src, 1), bb);
    1856              :           break;
    1857              : 
    1858            0 :         case PLUS:
    1859            0 :           if (timode_concatdi_p (src))
    1860              :             /* vmovq;vpinsrq (11 bytes).  */
    1861            0 :             igain = speed_p ? -ix86_cost->integer_to_sse - COSTS_N_INSNS (1)
    1862              :                             : -COSTS_N_BYTES (11);
    1863              :           break;
    1864              : 
    1865          230 :         case ASHIFT:
    1866          230 :         case LSHIFTRT:
    1867              :           /* See ix86_expand_v1ti_shift.  */
    1868          230 :           op1val = INTVAL (XEXP (src, 1));
    1869          230 :           if (!speed_p)
    1870              :             {
    1871           21 :               if (op1val == 64 || op1val == 65)
    1872              :                 scost = COSTS_N_BYTES (5);
    1873           13 :               else if (op1val >= 66)
    1874              :                 scost = COSTS_N_BYTES (6);
    1875           13 :               else if (op1val == 1)
    1876              :                 scost = COSTS_N_BYTES (8);
    1877              :               else
    1878              :                 scost = COSTS_N_BYTES (9);
    1879              : 
    1880           19 :               if ((op1val & 7) == 0)
    1881              :                 vcost = COSTS_N_BYTES (5);
    1882           13 :               else if (op1val > 64)
    1883              :                 vcost = COSTS_N_BYTES (10);
    1884              :               else
    1885           13 :                 vcost = TARGET_AVX ? COSTS_N_BYTES (19) : COSTS_N_BYTES (23);
    1886              :             }
    1887              :           else
    1888              :             {
    1889          209 :               scost = COSTS_N_INSNS (2);
    1890          209 :               if ((op1val & 7) == 0)
    1891              :                 vcost = COSTS_N_INSNS (1);
    1892          131 :               else if (op1val > 64)
    1893              :                 vcost = COSTS_N_INSNS (2);
    1894              :               else
    1895          131 :                 vcost = TARGET_AVX ? COSTS_N_INSNS (4) : COSTS_N_INSNS (5);
    1896              :             }
    1897          230 :           igain = scost - vcost;
    1898          230 :           break;
    1899              : 
    1900          124 :         case ASHIFTRT:
    1901              :           /* See ix86_expand_v1ti_ashiftrt.  */
    1902          124 :           op1val = INTVAL (XEXP (src, 1));
    1903          124 :           if (!speed_p)
    1904              :             {
    1905            9 :               if (op1val == 64 || op1val == 127)
    1906              :                 scost = COSTS_N_BYTES (7);
    1907            9 :               else if (op1val == 1)
    1908              :                 scost = COSTS_N_BYTES (8);
    1909            8 :               else if (op1val == 65)
    1910              :                 scost = COSTS_N_BYTES (10);
    1911            8 :               else if (op1val >= 66)
    1912              :                 scost = COSTS_N_BYTES (11);
    1913              :               else
    1914              :                 scost = COSTS_N_BYTES (9);
    1915              : 
    1916            0 :               if (op1val == 127)
    1917              :                 vcost = COSTS_N_BYTES (10);
    1918            9 :               else if (op1val == 64)
    1919              :                 vcost = COSTS_N_BYTES (14);
    1920            9 :               else if (op1val == 96)
    1921              :                 vcost = COSTS_N_BYTES (18);
    1922            9 :               else if (op1val >= 111)
    1923              :                 vcost = COSTS_N_BYTES (15);
    1924            9 :               else if (TARGET_AVX2 && op1val == 32)
    1925              :                 vcost = COSTS_N_BYTES (16);
    1926            9 :               else if (TARGET_SSE4_1 && op1val == 32)
    1927              :                 vcost = COSTS_N_BYTES (20);
    1928            9 :               else if (op1val >= 96)
    1929              :                 vcost = COSTS_N_BYTES (23);
    1930            9 :               else if ((op1val & 7) == 0)
    1931              :                 vcost = COSTS_N_BYTES (28);
    1932            9 :               else if (TARGET_AVX2 && op1val < 32)
    1933              :                 vcost = COSTS_N_BYTES (30);
    1934            9 :               else if (op1val == 1 || op1val >= 64)
    1935              :                 vcost = COSTS_N_BYTES (42);
    1936              :               else
    1937            8 :                 vcost = COSTS_N_BYTES (47);
    1938              :             }
    1939              :           else
    1940              :             {
    1941          115 :               if (op1val >= 65 && op1val <= 126)
    1942              :                 scost = COSTS_N_INSNS (3);
    1943              :               else
    1944          115 :                 scost = COSTS_N_INSNS (2);
    1945              : 
    1946          115 :               if (op1val == 127)
    1947              :                 vcost = COSTS_N_INSNS (2);
    1948          114 :               else if (op1val == 64)
    1949              :                 vcost = COSTS_N_INSNS (3);
    1950          114 :               else if (op1val == 96)
    1951              :                 vcost = COSTS_N_INSNS (3);
    1952          114 :               else if (op1val >= 111)
    1953              :                 vcost = COSTS_N_INSNS (3);
    1954          114 :               else if (TARGET_SSE4_1 && op1val == 32)
    1955              :                 vcost = COSTS_N_INSNS (3);
    1956          114 :               else if (TARGET_SSE4_1
    1957            0 :                        && (op1val == 8 || op1val == 16 || op1val == 24))
    1958              :                 vcost = COSTS_N_INSNS (3);
    1959          114 :               else if (op1val >= 96)
    1960              :                 vcost = COSTS_N_INSNS (4);
    1961          114 :               else if (TARGET_SSE4_1 && (op1val == 28 || op1val == 80))
    1962              :                 vcost = COSTS_N_INSNS (4);
    1963          114 :               else if ((op1val & 7) == 0)
    1964              :                 vcost = COSTS_N_INSNS (5);
    1965          114 :               else if (TARGET_AVX2 && op1val < 32)
    1966              :                 vcost = COSTS_N_INSNS (6);
    1967          114 :               else if (TARGET_SSE4_1 && op1val < 15)
    1968              :                 vcost = COSTS_N_INSNS (6);
    1969          114 :               else if (op1val == 1 || op1val >= 64)
    1970              :                 vcost = COSTS_N_INSNS (8);
    1971              :               else
    1972           17 :                 vcost = COSTS_N_INSNS (9);
    1973              :             }
    1974          124 :           igain = scost - vcost;
    1975          124 :           break;
    1976              : 
    1977            6 :         case ROTATE:
    1978            6 :         case ROTATERT:
    1979              :           /* See ix86_expand_v1ti_rotate.  */
    1980            6 :           op1val = INTVAL (XEXP (src, 1));
    1981            6 :           if (!speed_p)
    1982              :             {
    1983            0 :               scost = COSTS_N_BYTES (13);
    1984            0 :               if ((op1val & 31) == 0)
    1985              :                 vcost = COSTS_N_BYTES (5);
    1986            0 :               else if ((op1val & 7) == 0)
    1987            0 :                 vcost = TARGET_AVX ? COSTS_N_BYTES (13) : COSTS_N_BYTES (18);
    1988            0 :               else if (op1val > 32 && op1val < 96)
    1989              :                 vcost = COSTS_N_BYTES (24);
    1990              :               else
    1991            0 :                 vcost = COSTS_N_BYTES (19);
    1992              :             }
    1993              :           else
    1994              :             {
    1995            6 :               scost = COSTS_N_INSNS (3);
    1996            6 :               if ((op1val & 31) == 0)
    1997              :                 vcost = COSTS_N_INSNS (1);
    1998            4 :               else if ((op1val & 7) == 0)
    1999            1 :                 vcost = TARGET_AVX ? COSTS_N_INSNS (3) : COSTS_N_INSNS (4);
    2000            3 :               else if (op1val > 32 && op1val < 96)
    2001              :                 vcost = COSTS_N_INSNS (5);
    2002              :               else
    2003            3 :                 vcost = COSTS_N_INSNS (4);
    2004              :             }
    2005            6 :           igain = scost - vcost;
    2006            6 :           break;
    2007              : 
    2008           13 :         case COMPARE:
    2009           13 :           if (XEXP (src, 1) == const0_rtx)
    2010              :             {
    2011            2 :               if (GET_CODE (XEXP (src, 0)) == AND)
    2012              :                 /* and;and;or (9 bytes) vs. ptest (5 bytes).  */
    2013              :                 igain = !speed_p ? COSTS_N_BYTES (4) : COSTS_N_INSNS (2);
    2014              :               /* or (3 bytes) vs. ptest (5 bytes).  */
    2015            2 :               else if (!speed_p)
    2016            0 :                 igain = -COSTS_N_BYTES (2);
    2017              :             }
    2018           11 :           else if (XEXP (src, 1) == const1_rtx)
    2019              :             /* and;cmp -1 (7 bytes) vs. pcmpeqd;pxor;ptest (13 bytes).  */
    2020            0 :             igain = !speed_p ? -COSTS_N_BYTES (6) : -COSTS_N_INSNS (1);
    2021              :           break;
    2022              : 
    2023          282 :         case ZERO_EXTEND:
    2024          282 :           if (GET_MODE (XEXP (src, 0)) == DImode)
    2025              :             /* xor (2 bytes) vs. vmovq (5 bytes).  */
    2026          282 :             igain = speed_p ? COSTS_N_INSNS (1) - ix86_cost->sse_to_integer
    2027              :                             : -COSTS_N_BYTES (3);
    2028              :           break;
    2029              : 
    2030              :         default:
    2031              :           break;
    2032              :         }
    2033              : 
    2034      2076679 :       gain += igain;
    2035      1055455 :       if (speed_p)
    2036      1021224 :         weighted_gain += bb_freq * igain;
    2037              : 
    2038      1055457 :       if (igain != 0 && dump_file)
    2039              :         {
    2040            0 :           fprintf (dump_file, "  Instruction gain %d with bb_freq %.2f for ",
    2041              :                    igain, bb_freq.to_double ());
    2042            0 :           dump_insn_slim (dump_file, insn);
    2043              :         }
    2044              :     }
    2045              : 
    2046       531156 :   if (dump_file)
    2047            0 :     fprintf (dump_file, "  Total gain: %d, weighted gain %.2f\n",
    2048              :              gain, weighted_gain.to_double ());
    2049              : 
    2050       531156 :   if (weighted_gain > (sreal) 0)
    2051              :     return true;
    2052              :   else
    2053        56336 :     return gain > 0;
    2054              : }
    2055              : 
    2056              : /* Fix uses of converted REG in debug insns.  */
    2057              : 
    2058              : void
    2059       446604 : timode_scalar_chain::fix_debug_reg_uses (rtx reg)
    2060              : {
    2061       446604 :   if (!flag_var_tracking)
    2062              :     return;
    2063              : 
    2064       391458 :   df_ref ref, next;
    2065       801875 :   for (ref = DF_REG_USE_CHAIN (REGNO (reg)); ref; ref = next)
    2066              :     {
    2067       410417 :       rtx_insn *insn = DF_REF_INSN (ref);
    2068              :       /* Make sure the next ref is for a different instruction,
    2069              :          so that we're not affected by the rescan.  */
    2070       410417 :       next = DF_REF_NEXT_REG (ref);
    2071       410417 :       while (next && DF_REF_INSN (next) == insn)
    2072            0 :         next = DF_REF_NEXT_REG (next);
    2073              : 
    2074       410417 :       if (DEBUG_INSN_P (insn))
    2075              :         {
    2076              :           /* It may be a debug insn with a TImode variable in
    2077              :              register.  */
    2078              :           bool changed = false;
    2079          246 :           for (; ref != next; ref = DF_REF_NEXT_REG (ref))
    2080              :             {
    2081          123 :               rtx *loc = DF_REF_LOC (ref);
    2082          123 :               if (REG_P (*loc) && GET_MODE (*loc) == V1TImode)
    2083              :                 {
    2084          114 :                   *loc = gen_rtx_SUBREG (TImode, *loc, 0);
    2085          114 :                   changed = true;
    2086              :                 }
    2087              :             }
    2088          123 :           if (changed)
    2089          114 :             df_insn_rescan (insn);
    2090              :         }
    2091              :     }
    2092              : }
    2093              : 
    2094              : /* Convert SRC, a *concatditi3 pattern, into a vec_concatv2di instruction.
    2095              :    Insert this before INSN, and return the result as a V1TImode subreg.  */
    2096              : 
    2097              : static rtx
    2098          283 : timode_convert_concatdi (rtx src, rtx_insn *insn)
    2099              : {
    2100          283 :   rtx hi, lo;
    2101          283 :   rtx tmp = gen_reg_rtx (V2DImode);
    2102          283 :   if (GET_CODE (XEXP (src, 0)) == ASHIFT)
    2103              :     {
    2104          283 :       hi = XEXP (XEXP (XEXP (src, 0), 0), 0);
    2105          283 :       lo = XEXP (XEXP (src, 1), 0);
    2106              :     }
    2107              :   else
    2108              :     {
    2109            0 :       hi = XEXP (XEXP (XEXP (src, 1), 0), 0);
    2110            0 :       lo = XEXP (XEXP (src, 0), 0);
    2111              :     }
    2112          283 :   emit_insn_before (gen_vec_concatv2di (tmp, lo, hi), insn);
    2113          283 :   return gen_rtx_SUBREG (V1TImode, tmp, 0);
    2114              : }
    2115              : 
    2116              : /* Convert INSN from TImode to V1T1mode.  */
    2117              : 
    2118              : void
    2119       965650 : timode_scalar_chain::convert_insn (rtx_insn *insn)
    2120              : {
    2121       965650 :   rtx def_set = single_set (insn);
    2122       965650 :   rtx src = SET_SRC (def_set);
    2123       965650 :   rtx dst = SET_DEST (def_set);
    2124       965650 :   rtx tmp;
    2125              : 
    2126       965650 :   switch (GET_CODE (dst))
    2127              :     {
    2128       447125 :     case REG:
    2129       447125 :       if (GET_MODE (dst) == TImode)
    2130              :         {
    2131       446531 :           if (!HARD_REGISTER_NUM_P (REGNO (dst)))
    2132              :             {
    2133       446018 :               PUT_MODE (dst, V1TImode);
    2134       446018 :               fix_debug_reg_uses (dst);
    2135              :             }
    2136          513 :           else if (!GENERAL_REGNO_P (REGNO (dst)))
    2137          359 :             dst = gen_raw_REG (V1TImode, REGNO (dst));
    2138              :         }
    2139       447125 :       if (GET_MODE (dst) == V1TImode)
    2140              :         {
    2141              :           /* It might potentially be helpful to convert REG_EQUAL notes,
    2142              :              but for now we just remove them.  */
    2143       446963 :           rtx note = find_reg_equal_equiv_note (insn);
    2144       446963 :           if (note)
    2145          483 :             remove_note (insn, note);
    2146              :         }
    2147              :       break;
    2148       518525 :     case MEM:
    2149       518525 :       PUT_MODE (dst, V1TImode);
    2150       518525 :       break;
    2151              : 
    2152            0 :     default:
    2153            0 :       gcc_unreachable ();
    2154              :     }
    2155              : 
    2156       965650 :   switch (GET_CODE (src))
    2157              :     {
    2158       475840 :     case REG:
    2159       475840 :       if (GET_MODE (src) == TImode)
    2160              :         {
    2161          814 :           if (GENERAL_REGNO_P (REGNO (src)))
    2162              :             {
    2163          223 :               rtx lo = gen_reg_rtx (DImode);
    2164          223 :               rtx hi = gen_reg_rtx (DImode);
    2165          223 :               emit_insn_before (gen_rtx_SET (lo, gen_lowpart (DImode, src)),
    2166              :                                 insn);
    2167          223 :               emit_insn_before (gen_rtx_SET (hi, gen_highpart (DImode, src)),
    2168              :                                 insn);
    2169          223 :               src = gen_reg_rtx (V2DImode);
    2170          223 :               emit_insn_before (gen_vec_concatv2di (src, lo, hi), insn);
    2171          223 :               src = gen_lowpart (V1TImode, src);
    2172              :             }
    2173          591 :           else if (!HARD_REGISTER_NUM_P (REGNO (src)))
    2174              :             {
    2175          586 :               PUT_MODE (src, V1TImode);
    2176          586 :               fix_debug_reg_uses (src);
    2177              :             }
    2178              :           else
    2179            5 :             src = gen_raw_REG (V1TImode, REGNO (src));
    2180              :         }
    2181       475840 :       if (GENERAL_REG_P (dst))
    2182              :         {
    2183          154 :           rtx tmp = gen_reg_rtx (V2DImode);
    2184          154 :           src = gen_lowpart (V2DImode, src);
    2185          154 :           emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2186              :           /* Extracting hi before lo helps register allocation.  */
    2187          154 :           rtx hi = gen_reg_rtx (DImode);
    2188          154 :           rtx lo = gen_reg_rtx (DImode);
    2189          154 :           emit_insn_before (gen_vec_extractv2didi (hi, tmp, const1_rtx), insn);
    2190          154 :           emit_insn_before (gen_vec_extractv2didi (lo, tmp, const0_rtx), insn);
    2191              : 
    2192              :           /* Construct *concatditi3 pattern from lo and hi.  */
    2193          154 :           hi = gen_rtx_ZERO_EXTEND (TImode, hi);
    2194          154 :           hi = gen_rtx_ASHIFT (TImode, hi, GEN_INT (64));
    2195          154 :           lo = gen_rtx_ZERO_EXTEND (TImode, lo);
    2196          154 :           src = gen_rtx_PLUS (TImode, hi, lo);
    2197              :         }
    2198              :       break;
    2199              : 
    2200       444732 :     case MEM:
    2201       444732 :       PUT_MODE (src, V1TImode);
    2202       444732 :       break;
    2203              : 
    2204        31693 :     case CONST_WIDE_INT:
    2205        31693 :       if (NONDEBUG_INSN_P (insn))
    2206              :         {
    2207              :           /* Since there are no instructions to store 128-bit constant,
    2208              :              temporary register usage is required.  */
    2209        31693 :           bool use_move;
    2210        31693 :           start_sequence ();
    2211        31693 :           tmp = ix86_convert_const_wide_int_to_broadcast (TImode, src);
    2212        31693 :           if (tmp)
    2213              :             {
    2214          194 :               src = lowpart_subreg (V1TImode, tmp, TImode);
    2215          194 :               use_move = true;
    2216              :             }
    2217              :           else
    2218              :             {
    2219        31499 :               src = smode_convert_cst (src, V1TImode);
    2220        31499 :               src = validize_mem (force_const_mem (V1TImode, src));
    2221        31499 :               use_move = MEM_P (dst);
    2222              :             }
    2223        31693 :           rtx_insn *seq = end_sequence ();
    2224        31693 :           if (seq)
    2225          195 :             emit_insn_before (seq, insn);
    2226        31693 :           if (use_move)
    2227              :             {
    2228        31306 :               tmp = gen_reg_rtx (V1TImode);
    2229        31306 :               emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2230        31306 :               src = tmp;
    2231              :             }
    2232              :         }
    2233              :       break;
    2234              : 
    2235        12756 :     case CONST_INT:
    2236        12756 :       switch (standard_sse_constant_p (src, TImode))
    2237              :         {
    2238        12528 :         case 1:
    2239        12528 :           src = CONST0_RTX (GET_MODE (dst));
    2240        12528 :           break;
    2241          228 :         case 2:
    2242          228 :           src = CONSTM1_RTX (GET_MODE (dst));
    2243          228 :           break;
    2244            0 :         default:
    2245            0 :           gcc_unreachable ();
    2246              :         }
    2247        12756 :       if (MEM_P (dst))
    2248              :         {
    2249        12228 :           tmp = gen_reg_rtx (V1TImode);
    2250        12228 :           emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2251        12228 :           src = tmp;
    2252              :         }
    2253              :       break;
    2254              : 
    2255           41 :     case AND:
    2256           41 :       if (GET_CODE (XEXP (src, 0)) == NOT)
    2257              :         {
    2258            0 :           convert_op (&XEXP (XEXP (src, 0), 0), insn);
    2259            0 :           convert_op (&XEXP (src, 1), insn);
    2260            0 :           PUT_MODE (XEXP (src, 0), V1TImode);
    2261            0 :           PUT_MODE (src, V1TImode);
    2262            0 :           break;
    2263              :         }
    2264           41 :       convert_op (&XEXP (src, 0), insn);
    2265           41 :       convert_op (&XEXP (src, 1), insn);
    2266           41 :       PUT_MODE (src, V1TImode);
    2267           41 :       if (MEM_P (dst))
    2268              :         {
    2269           10 :           tmp = gen_reg_rtx (V1TImode);
    2270           10 :           emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2271           10 :           src = tmp;
    2272              :         }
    2273              :       break;
    2274              : 
    2275          393 :     case XOR:
    2276          393 :     case IOR:
    2277          393 :       if (timode_concatdi_p (src))
    2278              :         {
    2279          283 :           src = timode_convert_concatdi (src, insn);
    2280          283 :           break;
    2281              :         }
    2282          110 :       convert_op (&XEXP (src, 0), insn);
    2283          110 :       convert_op (&XEXP (src, 1), insn);
    2284          110 :       PUT_MODE (src, V1TImode);
    2285          110 :       if (MEM_P (dst))
    2286              :         {
    2287            8 :           tmp = gen_reg_rtx (V1TImode);
    2288            8 :           emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2289            8 :           src = tmp;
    2290              :         }
    2291              :       break;
    2292              : 
    2293            3 :     case NOT:
    2294            3 :       src = XEXP (src, 0);
    2295            3 :       convert_op (&src, insn);
    2296            3 :       tmp = gen_reg_rtx (V1TImode);
    2297            3 :       emit_insn_before (gen_move_insn (tmp, CONSTM1_RTX (V1TImode)), insn);
    2298            3 :       src = gen_rtx_XOR (V1TImode, src, tmp);
    2299            3 :       if (MEM_P (dst))
    2300              :         {
    2301            0 :           tmp = gen_reg_rtx (V1TImode);
    2302            0 :           emit_insn_before (gen_rtx_SET (tmp, src), insn);
    2303            0 :           src = tmp;
    2304              :         }
    2305              :       break;
    2306              : 
    2307            8 :     case COMPARE:
    2308            8 :       dst = gen_rtx_REG (CCZmode, FLAGS_REG);
    2309            8 :       src = convert_compare (XEXP (src, 0), XEXP (src, 1), insn);
    2310            8 :       break;
    2311              : 
    2312           68 :     case ASHIFT:
    2313           68 :     case LSHIFTRT:
    2314           68 :     case ASHIFTRT:
    2315           68 :     case ROTATERT:
    2316           68 :     case ROTATE:
    2317           68 :       convert_op (&XEXP (src, 0), insn);
    2318           68 :       PUT_MODE (src, V1TImode);
    2319           68 :       break;
    2320              : 
    2321          116 :     case ZERO_EXTEND:
    2322          116 :       if (GET_MODE (XEXP (src, 0)) == DImode)
    2323              :         {
    2324              :           /* Convert to *vec_concatv2di_0.  */
    2325          116 :           rtx tmp = gen_reg_rtx (V2DImode);
    2326          116 :           rtx pat = gen_rtx_VEC_CONCAT (V2DImode, XEXP (src, 0), const0_rtx);
    2327          116 :           emit_insn_before (gen_move_insn (tmp, pat), insn);
    2328          116 :           src = gen_rtx_SUBREG (vmode, tmp, 0);
    2329              :         }
    2330              :       else
    2331            0 :         gcc_unreachable ();
    2332          116 :       break;
    2333              : 
    2334            0 :     case PLUS:
    2335            0 :       if (timode_concatdi_p (src))
    2336            0 :         src = timode_convert_concatdi (src, insn);
    2337              :       else
    2338            0 :         gcc_unreachable ();
    2339            0 :       break;
    2340              : 
    2341            0 :     default:
    2342            0 :       gcc_unreachable ();
    2343              :     }
    2344              : 
    2345       965650 :   SET_SRC (def_set) = src;
    2346       965650 :   SET_DEST (def_set) = dst;
    2347              : 
    2348              :   /* Drop possible dead definitions.  */
    2349       965650 :   PATTERN (insn) = def_set;
    2350              : 
    2351       965650 :   INSN_CODE (insn) = -1;
    2352       965650 :   recog_memoized (insn);
    2353       965650 :   df_insn_rescan (insn);
    2354       965650 : }
    2355              : 
    2356              : /* Generate copies from defs used by the chain but not defined therein.
    2357              :    Also populates defs_map which is used later by convert_insn.  */
    2358              : 
    2359              : void
    2360       662752 : scalar_chain::convert_registers ()
    2361              : {
    2362       662752 :   bitmap_iterator bi;
    2363       662752 :   unsigned id;
    2364       689838 :   EXECUTE_IF_SET_IN_BITMAP (defs_conv, 0, id, bi)
    2365              :     {
    2366        27086 :       rtx chain_reg = gen_reg_rtx (smode);
    2367        27086 :       defs_map.put (regno_reg_rtx[id], chain_reg);
    2368              :     }
    2369       671449 :   EXECUTE_IF_SET_IN_BITMAP (insns_conv, 0, id, bi)
    2370        20952 :     for (df_ref ref = DF_INSN_UID_DEFS (id); ref; ref = DF_REF_NEXT_LOC (ref))
    2371        12255 :       if (bitmap_bit_p (defs_conv, DF_REF_REGNO (ref)))
    2372         8697 :         make_vector_copies (DF_REF_INSN (ref), DF_REF_REAL_REG (ref));
    2373       662752 : }
    2374              : 
    2375              : /* Convert whole chain creating required register
    2376              :    conversions and copies.  */
    2377              : 
    2378              : int
    2379       662752 : scalar_chain::convert ()
    2380              : {
    2381       662752 :   bitmap_iterator bi;
    2382       662752 :   unsigned id;
    2383       662752 :   int converted_insns = 0;
    2384              : 
    2385       662752 :   if (!dbg_cnt (stv_conversion))
    2386              :     return 0;
    2387              : 
    2388       662752 :   if (dump_file)
    2389            0 :     fprintf (dump_file, "Converting chain #%d...\n", chain_id);
    2390              : 
    2391       662752 :   convert_registers ();
    2392              : 
    2393      2035880 :   EXECUTE_IF_SET_IN_BITMAP (insns, 0, id, bi)
    2394              :     {
    2395      1373128 :       rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
    2396      1373128 :       convert_insn_common (insn);
    2397      1373128 :       convert_insn (insn);
    2398      1373128 :       converted_insns++;
    2399              :     }
    2400              : 
    2401              :   return converted_insns;
    2402              : }
    2403              : 
    2404              : /* Return the SET expression if INSN doesn't reference hard register.
    2405              :    Return NULL if INSN uses or defines a hard register, excluding
    2406              :    pseudo register pushes, hard register uses in a memory address,
    2407              :    clobbers and flags definitions.  */
    2408              : 
    2409              : static rtx
    2410    355249558 : pseudo_reg_set (rtx_insn *insn)
    2411              : {
    2412    355249558 :   rtx set = single_set (insn);
    2413    355249558 :   if (!set)
    2414              :     return NULL;
    2415              : 
    2416              :   /* Check pseudo register push first. */
    2417    139424763 :   machine_mode mode = TARGET_64BIT ? TImode : DImode;
    2418    139424763 :   if (REG_P (SET_SRC (set))
    2419     39496911 :       && !HARD_REGISTER_P (SET_SRC (set))
    2420    170297958 :       && push_operand (SET_DEST (set), mode))
    2421              :     return set;
    2422              : 
    2423    139171574 :   df_ref ref;
    2424    225570832 :   FOR_EACH_INSN_DEF (ref, insn)
    2425    124536789 :     if (HARD_REGISTER_P (DF_REF_REAL_REG (ref))
    2426     67041406 :         && !DF_REF_FLAGS_IS_SET (ref, DF_REF_MUST_CLOBBER)
    2427    176438611 :         && DF_REF_REGNO (ref) != FLAGS_REG)
    2428              :       return NULL;
    2429              : 
    2430    193202744 :   FOR_EACH_INSN_USE (ref, insn)
    2431    118262522 :     if (!DF_REF_REG_MEM_P (ref) && HARD_REGISTER_P (DF_REF_REAL_REG (ref)))
    2432              :       return NULL;
    2433              : 
    2434              :   return set;
    2435              : }
    2436              : 
    2437              : /* Return true if the register REG is defined in a single DEF chain.
    2438              :    If it is defined in more than one DEF chains, we may not be able
    2439              :    to convert it in all chains.  */
    2440              : 
    2441              : static bool
    2442      1312672 : single_def_chain_p (rtx reg)
    2443              : {
    2444      1312672 :   df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
    2445      1312672 :   if (!ref)
    2446              :     return false;
    2447      1312641 :   return DF_REF_NEXT_REG (ref) == nullptr;
    2448              : }
    2449              : 
    2450              : /* Check if comparison INSN may be transformed into vector comparison.
    2451              :    Currently we transform equality/inequality checks which look like:
    2452              :    (set (reg:CCZ 17 flags) (compare:CCZ (reg:TI x) (reg:TI y)))  */
    2453              : 
    2454              : static bool
    2455     13126547 : convertible_comparison_p (rtx_insn *insn, enum machine_mode mode)
    2456              : {
    2457     14544269 :   if (mode != (TARGET_64BIT ? TImode : DImode))
    2458              :     return false;
    2459              : 
    2460      4813732 :   if (!TARGET_SSE4_1)
    2461              :     return false;
    2462              : 
    2463       166688 :   rtx def_set = single_set (insn);
    2464              : 
    2465       166688 :   gcc_assert (def_set);
    2466              : 
    2467       166688 :   rtx src = SET_SRC (def_set);
    2468       166688 :   rtx dst = SET_DEST (def_set);
    2469              : 
    2470       166688 :   gcc_assert (GET_CODE (src) == COMPARE);
    2471              : 
    2472       166688 :   if (!REG_P (dst)
    2473       166688 :       || REGNO (dst) != FLAGS_REG
    2474       333376 :       || GET_MODE (dst) != CCZmode)
    2475              :     return false;
    2476              : 
    2477       116234 :   rtx op1 = XEXP (src, 0);
    2478       116234 :   rtx op2 = XEXP (src, 1);
    2479              : 
    2480              :   /* *cmp<dwi>_doubleword.  */
    2481       116234 :   if (general_operand (op1, mode)
    2482       116234 :       && general_operand (op2, mode))
    2483              :     return true;
    2484              : 
    2485              :   /* *testti_doubleword.  */
    2486       116171 :   if (op2 == const0_rtx
    2487        40019 :       && GET_CODE (op1) == AND
    2488          142 :       && REG_P (XEXP (op1, 0)))
    2489              :     {
    2490          142 :       rtx op12 = XEXP (op1, 1);
    2491          142 :       return GET_MODE (XEXP (op1, 0)) == TImode
    2492          142 :              && (CONST_SCALAR_INT_P (op12)
    2493            0 :                  || ((REG_P (op12) || MEM_P (op12))
    2494            0 :                      && GET_MODE (op12) == TImode));
    2495              :     }
    2496              : 
    2497              :   /* *test<dwi>_not_doubleword.  */
    2498       116029 :   if (op2 == const0_rtx
    2499        39877 :       && GET_CODE (op1) == AND
    2500            0 :       && GET_CODE (XEXP (op1, 0)) == NOT)
    2501              :     {
    2502            0 :       rtx op11 = XEXP (XEXP (op1, 0), 0);
    2503            0 :       rtx op12 = XEXP (op1, 1);
    2504            0 :       return (REG_P (op11) || MEM_P (op11))
    2505            0 :              && (REG_P (op12) || MEM_P (op12))
    2506            0 :              && GET_MODE (op11) == mode
    2507            0 :              && GET_MODE (op12) == mode;
    2508              :     }
    2509              : 
    2510              :   return false;
    2511              : }
    2512              : 
    2513              : /* The general version of scalar_to_vector_candidate_p.  */
    2514              : 
    2515              : static bool
    2516    247978531 : general_scalar_to_vector_candidate_p (rtx_insn *insn, enum machine_mode mode)
    2517              : {
    2518    247978531 :   rtx def_set = pseudo_reg_set (insn);
    2519              : 
    2520    247978531 :   if (!def_set)
    2521              :     return false;
    2522              : 
    2523     50800179 :   rtx src = SET_SRC (def_set);
    2524     50800179 :   rtx dst = SET_DEST (def_set);
    2525              : 
    2526     50800179 :   if (GET_CODE (src) == COMPARE)
    2527      9021676 :     return convertible_comparison_p (insn, mode);
    2528              : 
    2529              :   /* We are interested in "mode" only.  */
    2530     41778503 :   if ((GET_MODE (src) != mode
    2531     28489031 :        && !CONST_INT_P (src))
    2532     18693177 :       || GET_MODE (dst) != mode)
    2533              :     return false;
    2534              : 
    2535     15669382 :   if (!REG_P (dst) && !MEM_P (dst))
    2536              :     return false;
    2537              : 
    2538     15407023 :   switch (GET_CODE (src))
    2539              :     {
    2540       557892 :     case ASHIFT:
    2541       557892 :     case LSHIFTRT:
    2542       557892 :     case ASHIFTRT:
    2543       557892 :     case ROTATE:
    2544       557892 :     case ROTATERT:
    2545       557892 :       if (!CONST_INT_P (XEXP (src, 1))
    2546      1077894 :           || !IN_RANGE (INTVAL (XEXP (src, 1)), 0, GET_MODE_BITSIZE (mode)-1))
    2547              :         return false;
    2548              : 
    2549              :       /* Check for extend highpart case.  */
    2550       520002 :       if (mode != DImode
    2551       383295 :           || GET_CODE (src) != ASHIFTRT
    2552        88884 :           || GET_CODE (XEXP (src, 0)) != ASHIFT)
    2553              :         break;
    2554              : 
    2555      3807554 :       src = XEXP (src, 0);
    2556              :       break;
    2557              : 
    2558       102396 :     case SMAX:
    2559       102396 :     case SMIN:
    2560       102396 :     case UMAX:
    2561       102396 :     case UMIN:
    2562       102396 :       if ((mode == DImode && !TARGET_AVX512VL)
    2563        18005 :           || (mode == SImode && !TARGET_SSE4_1))
    2564              :         return false;
    2565              :       /* Fallthru.  */
    2566              : 
    2567      3319372 :     case AND:
    2568      3319372 :     case IOR:
    2569      3319372 :     case XOR:
    2570      3319372 :     case PLUS:
    2571      3319372 :     case MINUS:
    2572      3319372 :       if (!REG_P (XEXP (src, 1))
    2573              :           && !MEM_P (XEXP (src, 1))
    2574              :           && !CONST_INT_P (XEXP (src, 1)))
    2575              :         return false;
    2576              : 
    2577      3225819 :       if (GET_MODE (XEXP (src, 1)) != mode
    2578      1853808 :           && !CONST_INT_P (XEXP (src, 1)))
    2579              :         return false;
    2580              : 
    2581              :       /* Check for andnot case.  */
    2582      3225819 :       if (GET_CODE (src) != AND
    2583       183811 :           || GET_CODE (XEXP (src, 0)) != NOT)
    2584              :         break;
    2585              : 
    2586      3807554 :       src = XEXP (src, 0);
    2587              :       /* FALLTHRU */
    2588              : 
    2589              :     case NOT:
    2590              :       break;
    2591              : 
    2592        32943 :     case NEG:
    2593              :       /* Check for nabs case.  */
    2594        32943 :       if (GET_CODE (XEXP (src, 0)) != ABS)
    2595              :         break;
    2596              : 
    2597              :       src = XEXP (src, 0);
    2598              :       /* FALLTHRU */
    2599              : 
    2600         3786 :     case ABS:
    2601         3786 :       if ((mode == DImode && !TARGET_AVX512VL)
    2602         1400 :           || (mode == SImode && !TARGET_SSSE3))
    2603              :         return false;
    2604              :       break;
    2605              : 
    2606              :     case REG:
    2607              :       return true;
    2608              : 
    2609      6173253 :     case MEM:
    2610      6173253 :     case CONST_INT:
    2611      6173253 :       return REG_P (dst);
    2612              : 
    2613        39913 :     case VEC_SELECT:
    2614              :       /* Excluding MEM_P (dst) avoids interfering with vpextr[dq].  */
    2615        39913 :       return REG_P (dst)
    2616        33297 :              && REG_P (XEXP (src, 0))
    2617        42569 :              && GET_MODE (XEXP (src, 0)) == (mode == DImode ? V2DImode
    2618              :                                                             : V4SImode)
    2619        23898 :              && GET_CODE (XEXP (src, 1)) == PARALLEL
    2620        23898 :              && XVECLEN (XEXP (src, 1), 0) == 1
    2621        63811 :              && CONST_INT_P (XVECEXP (XEXP (src, 1), 0, 0));
    2622              : 
    2623       321740 :     case ZERO_EXTEND:
    2624              :       /* *zero_extendsidi2 becomes *vec_setv2di_0_zero_extendsi_1.  */
    2625       321740 :       return mode == DImode
    2626       226806 :              && REG_P (dst)
    2627       223972 :              && GET_MODE (XEXP (src, 0)) == SImode
    2628       506247 :              && MEM_P (XEXP (src, 0));
    2629              : 
    2630              :     default:
    2631              :       return false;
    2632              :     }
    2633              : 
    2634      3807554 :   if (!REG_P (XEXP (src, 0))
    2635              :       && !MEM_P (XEXP (src, 0))
    2636              :       && !CONST_INT_P (XEXP (src, 0)))
    2637              :     return false;
    2638              : 
    2639      3509518 :   if (GET_MODE (XEXP (src, 0)) != mode
    2640            0 :       && !CONST_INT_P (XEXP (src, 0)))
    2641              :     return false;
    2642              : 
    2643              :   return true;
    2644              : }
    2645              : 
    2646              : /* Check for a suitable TImode memory operand.  */
    2647              : 
    2648              : static bool
    2649         1642 : timode_mem_p (rtx x)
    2650              : {
    2651         1642 :   return MEM_P (x)
    2652         1642 :          && (TARGET_SSE_UNALIGNED_LOAD_OPTIMAL
    2653            0 :              || !misaligned_operand (x, TImode));
    2654              : }
    2655              : 
    2656              : /* The TImode version of scalar_to_vector_candidate_p.  */
    2657              : 
    2658              : static bool
    2659    107271027 : timode_scalar_to_vector_candidate_p (rtx_insn *insn)
    2660              : {
    2661    107271027 :   rtx def_set = pseudo_reg_set (insn);
    2662              : 
    2663              :   /* We allow two exceptions to the pseudo registers only rule.
    2664              :      Setting a hard register from a pseudo, and setting a pseudo
    2665              :      from a hard register.  */
    2666    107271027 :   if (!def_set)
    2667              :     {
    2668     82877795 :       def_set = single_set (insn);
    2669     82877795 :       if (def_set)
    2670              :         {
    2671     18572852 :           rtx src = SET_SRC (def_set);
    2672     18572852 :           rtx dst = SET_DEST (def_set);
    2673     18572852 :           if (GET_MODE (dst) == TImode
    2674       225067 :               && REG_P (src) && REG_P (dst))
    2675              :             {
    2676       102996 :               if (HARD_REGISTER_P (dst)
    2677        53471 :                   && !HARD_REGISTER_P (src)
    2678       156467 :                   && single_def_chain_p (src))
    2679              :                 return true;
    2680        73477 :               if (HARD_REGISTER_P (src)
    2681        49525 :                   && !HARD_REGISTER_P (dst)
    2682       123002 :                   && single_def_chain_p (dst))
    2683              :                 return true;
    2684              :             }
    2685              :         }
    2686              :       return false;
    2687              :     }
    2688              : 
    2689     24393232 :   rtx src = SET_SRC (def_set);
    2690     24393232 :   rtx dst = SET_DEST (def_set);
    2691              : 
    2692     24393232 :   if (GET_CODE (src) == COMPARE)
    2693      4104871 :     return convertible_comparison_p (insn, TImode);
    2694              : 
    2695     20288361 :   if (GET_MODE (dst) != TImode
    2696      1256616 :       || (GET_MODE (src) != TImode
    2697        63644 :           && !CONST_SCALAR_INT_P (src)))
    2698              :     return false;
    2699              : 
    2700      1256616 :   if (!REG_P (dst) && !MEM_P (dst))
    2701              :     return false;
    2702              : 
    2703      1255163 :   if (MEM_P (dst)
    2704       556590 :       && misaligned_operand (dst, TImode)
    2705      1586758 :       && !TARGET_SSE_UNALIGNED_STORE_OPTIMAL)
    2706              :     return false;
    2707              : 
    2708      1255158 :   if (REG_P (dst) && !single_def_chain_p (dst))
    2709              :     return false;
    2710              : 
    2711      1095574 :   switch (GET_CODE (src))
    2712              :     {
    2713       511103 :     case REG:
    2714       511103 :       return single_def_chain_p (src);
    2715              : 
    2716              :     case CONST_WIDE_INT:
    2717              :       return true;
    2718              : 
    2719        15123 :     case CONST_INT:
    2720              :       /* ??? Verify performance impact before enabling CONST_INT for
    2721              :          __int128 store.  */
    2722        15123 :       return standard_sse_constant_p (src, TImode);
    2723              : 
    2724       467463 :     case MEM:
    2725              :       /* Memory must be aligned or unaligned load is optimal.  */
    2726       467463 :       return (REG_P (dst)
    2727       467463 :               && (!misaligned_operand (src, TImode)
    2728       164128 :                   || TARGET_SSE_UNALIGNED_LOAD_OPTIMAL));
    2729              : 
    2730         3849 :     case AND:
    2731         3849 :       if (!MEM_P (dst)
    2732         3808 :           && GET_CODE (XEXP (src, 0)) == NOT
    2733            0 :           && REG_P (XEXP (XEXP (src, 0), 0))
    2734         3849 :           && (REG_P (XEXP (src, 1))
    2735            0 :               || CONST_SCALAR_INT_P (XEXP (src, 1))
    2736            0 :               || timode_mem_p (XEXP (src, 1))))
    2737              :         return true;
    2738         3849 :       return (REG_P (XEXP (src, 0))
    2739           46 :               || timode_mem_p (XEXP (src, 0)))
    2740         3895 :              && (REG_P (XEXP (src, 1))
    2741         1499 :                  || CONST_SCALAR_INT_P (XEXP (src, 1))
    2742           45 :                  || timode_mem_p (XEXP (src, 1)));
    2743              : 
    2744        15831 :     case IOR:
    2745        15831 :     case XOR:
    2746        15831 :       if (timode_concatdi_p (src))
    2747              :         return true;
    2748         4071 :       return (REG_P (XEXP (src, 0))
    2749         1467 :               || timode_mem_p (XEXP (src, 0)))
    2750         4088 :              && (REG_P (XEXP (src, 1))
    2751          314 :                  || CONST_SCALAR_INT_P (XEXP (src, 1))
    2752           68 :                  || timode_mem_p (XEXP (src, 1)));
    2753              : 
    2754          524 :     case NOT:
    2755          524 :       return REG_P (XEXP (src, 0)) || timode_mem_p (XEXP (src, 0));
    2756              : 
    2757        11846 :     case ASHIFT:
    2758        11846 :     case LSHIFTRT:
    2759        11846 :     case ASHIFTRT:
    2760        11846 :     case ROTATERT:
    2761        11846 :     case ROTATE:
    2762              :       /* Handle shifts/rotates by integer constants between 0 and 127.  */
    2763        11846 :       return REG_P (XEXP (src, 0))
    2764        11810 :              && CONST_INT_P (XEXP (src, 1))
    2765        23288 :              && (INTVAL (XEXP (src, 1)) & ~0x7f) == 0;
    2766              : 
    2767         7644 :     case PLUS:
    2768         7644 :       return timode_concatdi_p (src);
    2769              : 
    2770         4195 :     case ZERO_EXTEND:
    2771         4195 :       return REG_P (XEXP (src, 0))
    2772         4195 :              && GET_MODE (XEXP (src, 0)) == DImode;
    2773              : 
    2774              :     default:
    2775              :       return false;
    2776              :     }
    2777              : }
    2778              : 
    2779              : /* For a register REGNO, scan instructions for its defs and uses.
    2780              :    Put REGNO in REGS if a def or use isn't in CANDIDATES.  */
    2781              : 
    2782              : static void
    2783      1291030 : timode_check_non_convertible_regs (bitmap candidates, bitmap regs,
    2784              :                                    unsigned int regno)
    2785              : {
    2786              :   /* Do nothing if REGNO is already in REGS or is a hard reg.  */
    2787      1291030 :   if (bitmap_bit_p (regs, regno)
    2788      1291030 :       || HARD_REGISTER_NUM_P (regno))
    2789              :     return;
    2790              : 
    2791      1282379 :   for (df_ref def = DF_REG_DEF_CHAIN (regno);
    2792      2551766 :        def;
    2793      1269387 :        def = DF_REF_NEXT_REG (def))
    2794              :     {
    2795      1282359 :       if (!bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
    2796              :         {
    2797        12972 :           if (dump_file)
    2798            0 :             fprintf (dump_file,
    2799              :                      "r%d has non convertible def in insn %d\n",
    2800            0 :                      regno, DF_REF_INSN_UID (def));
    2801              : 
    2802        12972 :           bitmap_set_bit (regs, regno);
    2803        12972 :           break;
    2804              :         }
    2805              :     }
    2806              : 
    2807      1282379 :   for (df_ref ref = DF_REG_USE_CHAIN (regno);
    2808      2841340 :        ref;
    2809      1558961 :        ref = DF_REF_NEXT_REG (ref))
    2810              :     {
    2811              :       /* Debug instructions are skipped.  */
    2812      1631738 :       if (NONDEBUG_INSN_P (DF_REF_INSN (ref))
    2813      1631738 :           && !bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
    2814              :         {
    2815        72777 :           if (dump_file)
    2816            0 :             fprintf (dump_file,
    2817              :                      "r%d has non convertible use in insn %d\n",
    2818            0 :                      regno, DF_REF_INSN_UID (ref));
    2819              : 
    2820        72777 :           bitmap_set_bit (regs, regno);
    2821        72777 :           break;
    2822              :         }
    2823              :     }
    2824              : }
    2825              : 
    2826              : /* For a given bitmap of insn UIDs scans all instructions and
    2827              :    remove insn from CANDIDATES in case it has both convertible
    2828              :    and not convertible definitions.
    2829              : 
    2830              :    All insns in a bitmap are conversion candidates according to
    2831              :    scalar_to_vector_candidate_p.  Currently it implies all insns
    2832              :    are single_set.  */
    2833              : 
    2834              : static void
    2835       850474 : timode_remove_non_convertible_regs (bitmap candidates)
    2836              : {
    2837       850474 :   bitmap_iterator bi;
    2838       850474 :   unsigned id;
    2839       850474 :   bitmap regs = BITMAP_ALLOC (NULL);
    2840       872318 :   bool changed;
    2841              : 
    2842       872318 :   do {
    2843       872318 :     changed = false;
    2844      2184916 :     EXECUTE_IF_SET_IN_BITMAP (candidates, 0, id, bi)
    2845              :       {
    2846      1312598 :         rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
    2847      1312598 :         df_ref ref;
    2848              : 
    2849      2053103 :         FOR_EACH_INSN_DEF (ref, insn)
    2850       740505 :           if (!DF_REF_REG_MEM_P (ref)
    2851       740505 :               && GET_MODE (DF_REF_REG (ref)) == TImode)
    2852       647334 :             timode_check_non_convertible_regs (candidates, regs,
    2853              :                                                DF_REF_REGNO (ref));
    2854              : 
    2855      3232632 :         FOR_EACH_INSN_USE (ref, insn)
    2856      1920034 :           if (DF_REF_TYPE (ref) == DF_REF_REG_USE
    2857       781291 :               && GET_MODE (DF_REF_REG (ref)) == TImode
    2858       643701 :               && !SUBREG_P (DF_REF_REG (ref)))
    2859       643696 :             timode_check_non_convertible_regs (candidates, regs,
    2860              :                                                DF_REF_REGNO (ref));
    2861              :       }
    2862              : 
    2863      1063785 :     EXECUTE_IF_SET_IN_BITMAP (regs, 0, id, bi)
    2864              :       {
    2865       191467 :         for (df_ref def = DF_REG_DEF_CHAIN (id);
    2866       392949 :              def;
    2867       201482 :              def = DF_REF_NEXT_REG (def))
    2868       201482 :           if (bitmap_bit_p (candidates, DF_REF_INSN_UID (def)))
    2869              :             {
    2870        57914 :               if (dump_file)
    2871            0 :                 fprintf (dump_file, "Removing insn %d from candidates list\n",
    2872            0 :                          DF_REF_INSN_UID (def));
    2873              : 
    2874        57914 :               bitmap_clear_bit (candidates, DF_REF_INSN_UID (def));
    2875        57914 :               changed = true;
    2876              :             }
    2877              : 
    2878       191467 :         for (df_ref ref = DF_REG_USE_CHAIN (id);
    2879       592033 :              ref;
    2880       400566 :              ref = DF_REF_NEXT_REG (ref))
    2881       400566 :           if (bitmap_bit_p (candidates, DF_REF_INSN_UID (ref)))
    2882              :             {
    2883        18428 :               if (dump_file)
    2884            0 :                 fprintf (dump_file, "Removing insn %d from candidates list\n",
    2885            0 :                          DF_REF_INSN_UID (ref));
    2886              : 
    2887        18428 :               bitmap_clear_bit (candidates, DF_REF_INSN_UID (ref));
    2888        18428 :               changed = true;
    2889              :             }
    2890              :       }
    2891              :   } while (changed);
    2892              : 
    2893       850474 :   BITMAP_FREE (regs);
    2894       850474 : }
    2895              : 
    2896              : /* Main STV pass function.  Find and convert scalar
    2897              :    instructions into vector mode when profitable.  */
    2898              : 
    2899              : static unsigned int
    2900      1827391 : convert_scalars_to_vector (bool timode_p)
    2901              : {
    2902      1827391 :   basic_block bb;
    2903      1827391 :   int converted_insns = 0;
    2904      1827391 :   auto_vec<rtx_insn *> control_flow_insns;
    2905              : 
    2906      1827391 :   bitmap_obstack_initialize (NULL);
    2907      1827391 :   const machine_mode cand_mode[3] = { SImode, DImode, TImode };
    2908      1827391 :   const machine_mode cand_vmode[3] = { V4SImode, V2DImode, V1TImode };
    2909      5482173 :   bitmap_head candidates[3];  /* { SImode, DImode, TImode } */
    2910      7309564 :   for (unsigned i = 0; i < 3; ++i)
    2911      5482173 :     bitmap_initialize (&candidates[i], &bitmap_default_obstack);
    2912              : 
    2913      1827391 :   calculate_dominance_info (CDI_DOMINATORS);
    2914      1827391 :   df_set_flags (DF_DEFER_INSN_RESCAN | DF_RD_PRUNE_DEAD_DEFS);
    2915      1827391 :   df_chain_add_problem (DF_DU_CHAIN | DF_UD_CHAIN);
    2916      1827391 :   df_analyze ();
    2917              : 
    2918              :   /* Find all instructions we want to convert into vector mode.  */
    2919      1827391 :   if (dump_file)
    2920           45 :     fprintf (dump_file, "Searching for mode conversion candidates...\n");
    2921              : 
    2922     20258896 :   FOR_EACH_BB_FN (bb, cfun)
    2923              :     {
    2924     18431505 :       rtx_insn *insn;
    2925    251480844 :       FOR_BB_INSNS (bb, insn)
    2926    233049339 :         if (timode_p
    2927    233049339 :             && timode_scalar_to_vector_candidate_p (insn))
    2928              :           {
    2929      1131799 :             if (dump_file)
    2930            0 :               fprintf (dump_file, "  insn %d is marked as a TImode candidate\n",
    2931            0 :                        INSN_UID (insn));
    2932              : 
    2933      1131799 :             bitmap_set_bit (&candidates[2], INSN_UID (insn));
    2934              :           }
    2935    231917540 :         else if (!timode_p)
    2936              :           {
    2937              :             /* Check {SI,DI}mode.  */
    2938    361749981 :             for (unsigned i = 0; i <= 1; ++i)
    2939    247978531 :               if (general_scalar_to_vector_candidate_p (insn, cand_mode[i]))
    2940              :                 {
    2941     12006862 :                   if (dump_file)
    2942          568 :                     fprintf (dump_file, "  insn %d is marked as a %s candidate\n",
    2943          284 :                              INSN_UID (insn), i == 0 ? "SImode" : "DImode");
    2944              : 
    2945     12006862 :                   bitmap_set_bit (&candidates[i], INSN_UID (insn));
    2946     12006862 :                   break;
    2947              :                 }
    2948              :           }
    2949              :     }
    2950              : 
    2951      1827391 :   if (timode_p)
    2952       850474 :     timode_remove_non_convertible_regs (&candidates[2]);
    2953              : 
    2954      5783191 :   for (unsigned i = 0; i <= 2; ++i)
    2955      4610345 :     if (!bitmap_empty_p (&candidates[i]))
    2956              :       break;
    2957      3955800 :     else if (i == 2 && dump_file)
    2958           23 :       fprintf (dump_file, "There are no candidates for optimization.\n");
    2959              : 
    2960      7309564 :   for (unsigned i = 0; i <= 2; ++i)
    2961              :     {
    2962      5482173 :       auto_bitmap disallowed;
    2963      5482173 :       bitmap_tree_view (&candidates[i]);
    2964     17584009 :       while (!bitmap_empty_p (&candidates[i]))
    2965              :         {
    2966      6619663 :           unsigned uid = bitmap_first_set_bit (&candidates[i]);
    2967      6619663 :           scalar_chain *chain;
    2968              : 
    2969      6619663 :           if (cand_mode[i] == TImode)
    2970       531156 :             chain = new timode_scalar_chain;
    2971              :           else
    2972      6088507 :             chain = new general_scalar_chain (cand_mode[i], cand_vmode[i]);
    2973              : 
    2974              :           /* Find instructions chain we want to convert to vector mode.
    2975              :              Check all uses and definitions to estimate all required
    2976              :              conversions.  */
    2977      6619663 :           if (chain->build (&candidates[i], uid, disallowed))
    2978              :             {
    2979      6617824 :               if (chain->compute_convert_gain ())
    2980       662752 :                 converted_insns += chain->convert ();
    2981      5955072 :               else if (dump_file)
    2982          136 :                 fprintf (dump_file, "Chain #%d conversion is not profitable\n",
    2983              :                          chain->chain_id);
    2984              :             }
    2985              : 
    2986      6619663 :           rtx_insn* iter_insn;
    2987      6619663 :           unsigned int ii;
    2988      6626791 :           FOR_EACH_VEC_ELT (chain->control_flow_insns, ii, iter_insn)
    2989         3578 :             control_flow_insns.safe_push (iter_insn);
    2990              : 
    2991      6619663 :           delete chain;
    2992              :         }
    2993      5482173 :     }
    2994              : 
    2995      1827391 :   if (dump_file)
    2996           45 :     fprintf (dump_file, "Total insns converted: %d\n", converted_insns);
    2997              : 
    2998      7309564 :   for (unsigned i = 0; i <= 2; ++i)
    2999      5482173 :     bitmap_release (&candidates[i]);
    3000      1827391 :   bitmap_obstack_release (NULL);
    3001      1827391 :   df_process_deferred_rescans ();
    3002              : 
    3003              :   /* Conversion means we may have 128bit register spills/fills
    3004              :      which require aligned stack.  */
    3005      1827391 :   if (converted_insns)
    3006              :     {
    3007       107476 :       if (crtl->stack_alignment_needed < 128)
    3008         2466 :         crtl->stack_alignment_needed = 128;
    3009       107476 :       if (crtl->stack_alignment_estimated < 128)
    3010          227 :         crtl->stack_alignment_estimated = 128;
    3011              : 
    3012       107476 :       crtl->stack_realign_needed
    3013       107476 :         = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
    3014       107476 :       crtl->stack_realign_tried = crtl->stack_realign_needed;
    3015              : 
    3016       107476 :       crtl->stack_realign_processed = true;
    3017              : 
    3018       107476 :       if (!crtl->drap_reg)
    3019              :         {
    3020       107286 :           rtx drap_rtx = targetm.calls.get_drap_rtx ();
    3021              : 
    3022              :           /* stack_realign_drap and drap_rtx must match.  */
    3023       107286 :           gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
    3024              : 
    3025              :           /* Do nothing if NULL is returned,
    3026              :              which means DRAP is not needed.  */
    3027       107286 :           if (drap_rtx != NULL)
    3028              :             {
    3029            0 :               crtl->args.internal_arg_pointer = drap_rtx;
    3030              : 
    3031              :               /* Call fixup_tail_calls to clean up
    3032              :                  REG_EQUIV note if DRAP is needed. */
    3033            0 :               fixup_tail_calls ();
    3034              :             }
    3035              :         }
    3036              : 
    3037              :       /* Fix up DECL_RTL/DECL_INCOMING_RTL of arguments.  */
    3038       107476 :       if (TARGET_64BIT)
    3039        68827 :         for (tree parm = DECL_ARGUMENTS (current_function_decl);
    3040       188473 :              parm; parm = DECL_CHAIN (parm))
    3041              :           {
    3042       119646 :             if (TYPE_MODE (TREE_TYPE (parm)) != TImode)
    3043       103530 :               continue;
    3044        16116 :             if (DECL_RTL_SET_P (parm)
    3045        32232 :                 && GET_MODE (DECL_RTL (parm)) == V1TImode)
    3046              :               {
    3047          621 :                 rtx r = DECL_RTL (parm);
    3048          621 :                 if (REG_P (r))
    3049          621 :                   SET_DECL_RTL (parm, gen_rtx_SUBREG (TImode, r, 0));
    3050              :               }
    3051        16116 :             if (DECL_INCOMING_RTL (parm)
    3052        16116 :                 && GET_MODE (DECL_INCOMING_RTL (parm)) == V1TImode)
    3053              :               {
    3054            0 :                 rtx r = DECL_INCOMING_RTL (parm);
    3055            0 :                 if (REG_P (r))
    3056            0 :                   DECL_INCOMING_RTL (parm) = gen_rtx_SUBREG (TImode, r, 0);
    3057              :               }
    3058              :           }
    3059              : 
    3060      1828519 :       if (!control_flow_insns.is_empty ())
    3061              :         {
    3062         1128 :           free_dominance_info (CDI_DOMINATORS);
    3063              : 
    3064         1128 :           unsigned int i;
    3065         1128 :           rtx_insn* insn;
    3066         5834 :           FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
    3067         3578 :             if (control_flow_insn_p (insn))
    3068              :               {
    3069              :                 /* Split the block after insn.  There will be a fallthru
    3070              :                    edge, which is OK so we keep it.  We have to create
    3071              :                    the exception edges ourselves.  */
    3072         3578 :                 bb = BLOCK_FOR_INSN (insn);
    3073         3578 :                 split_block (bb, insn);
    3074         3578 :                 rtl_make_eh_edge (NULL, bb, BB_END (bb));
    3075              :               }
    3076              :         }
    3077              :     }
    3078              : 
    3079      1827391 :   return 0;
    3080      1827391 : }
    3081              : 
    3082              : static unsigned int
    3083        76680 : rest_of_handle_insert_vzeroupper (void)
    3084              : {
    3085              :   /* vzeroupper instructions are inserted immediately after reload and
    3086              :      postreload_cse to clean up after it a little bit to account for possible
    3087              :      spills from 256bit or 512bit registers.  The pass reuses mode switching
    3088              :      infrastructure by re-running mode insertion pass, so disable entities
    3089              :      that have already been processed.  */
    3090       536760 :   for (int i = 0; i < MAX_386_ENTITIES; i++)
    3091       460080 :     ix86_optimize_mode_switching[i] = 0;
    3092              : 
    3093        76680 :   ix86_optimize_mode_switching[AVX_U128] = 1;
    3094              : 
    3095              :   /* Call optimize_mode_switching.  */
    3096        76680 :   g->get_passes ()->execute_pass_mode_switching ();
    3097              : 
    3098              :   /* LRA removes all REG_DEAD/REG_UNUSED notes and normally they
    3099              :      reappear in the IL only at the start of pass_rtl_dse2, which does
    3100              :      df_note_add_problem (); df_analyze ();
    3101              :      The vzeroupper is scheduled after postreload_cse pass and mode
    3102              :      switching computes the notes as well, the problem is that e.g.
    3103              :      pass_gcse2 doesn't maintain the notes, see PR113059 and
    3104              :      PR112760.  Remove the notes now to restore status quo ante
    3105              :      until we figure out how to maintain the notes or what else
    3106              :      to do.  */
    3107        76680 :   basic_block bb;
    3108        76680 :   rtx_insn *insn;
    3109       410728 :   FOR_EACH_BB_FN (bb, cfun)
    3110      4297242 :     FOR_BB_INSNS (bb, insn)
    3111      3963194 :       if (NONDEBUG_INSN_P (insn))
    3112              :         {
    3113      2121222 :           rtx *pnote = &REG_NOTES (insn);
    3114      3945736 :           while (*pnote != 0)
    3115              :             {
    3116      1824514 :               if (REG_NOTE_KIND (*pnote) == REG_DEAD
    3117       834904 :                   || REG_NOTE_KIND (*pnote) == REG_UNUSED)
    3118      1304223 :                 *pnote = XEXP (*pnote, 1);
    3119              :               else
    3120       520291 :                 pnote = &XEXP (*pnote, 1);
    3121              :             }
    3122              :         }
    3123              : 
    3124        76680 :   df_remove_problem (df_note);
    3125        76680 :   df_analyze ();
    3126        76680 :   return 0;
    3127              : }
    3128              : 
    3129              : namespace {
    3130              : 
    3131              : const pass_data pass_data_insert_vzeroupper =
    3132              : {
    3133              :   RTL_PASS, /* type */
    3134              :   "vzeroupper", /* name */
    3135              :   OPTGROUP_NONE, /* optinfo_flags */
    3136              :   TV_MACH_DEP, /* tv_id */
    3137              :   0, /* properties_required */
    3138              :   0, /* properties_provided */
    3139              :   0, /* properties_destroyed */
    3140              :   0, /* todo_flags_start */
    3141              :   TODO_df_finish, /* todo_flags_finish */
    3142              : };
    3143              : 
    3144              : class pass_insert_vzeroupper : public rtl_opt_pass
    3145              : {
    3146              : public:
    3147       297180 :   pass_insert_vzeroupper(gcc::context *ctxt)
    3148       594360 :     : rtl_opt_pass(pass_data_insert_vzeroupper, ctxt)
    3149              :   {}
    3150              : 
    3151              :   /* opt_pass methods: */
    3152      1517135 :   bool gate (function *) final override
    3153              :     {
    3154      1517135 :       return TARGET_AVX && TARGET_VZEROUPPER;
    3155              :     }
    3156              : 
    3157        76680 :   unsigned int execute (function *) final override
    3158              :     {
    3159        76680 :       return rest_of_handle_insert_vzeroupper ();
    3160              :     }
    3161              : 
    3162              : }; // class pass_insert_vzeroupper
    3163              : 
    3164              : const pass_data pass_data_stv =
    3165              : {
    3166              :   RTL_PASS, /* type */
    3167              :   "stv", /* name */
    3168              :   OPTGROUP_NONE, /* optinfo_flags */
    3169              :   TV_MACH_DEP, /* tv_id */
    3170              :   0, /* properties_required */
    3171              :   0, /* properties_provided */
    3172              :   0, /* properties_destroyed */
    3173              :   0, /* todo_flags_start */
    3174              :   TODO_df_finish, /* todo_flags_finish */
    3175              : };
    3176              : 
    3177              : class pass_stv : public rtl_opt_pass
    3178              : {
    3179              : public:
    3180       594360 :   pass_stv (gcc::context *ctxt)
    3181              :     : rtl_opt_pass (pass_data_stv, ctxt),
    3182      1188720 :       timode_p (false)
    3183              :   {}
    3184              : 
    3185              :   /* opt_pass methods: */
    3186      3034270 :   bool gate (function *) final override
    3187              :     {
    3188      1517135 :       return ((!timode_p || TARGET_64BIT)
    3189      4424641 :               && TARGET_STV && TARGET_SSE2 && optimize > 1);
    3190              :     }
    3191              : 
    3192      1827391 :   unsigned int execute (function *) final override
    3193              :     {
    3194      1827391 :       return convert_scalars_to_vector (timode_p);
    3195              :     }
    3196              : 
    3197       297180 :   opt_pass *clone () final override
    3198              :     {
    3199       297180 :       return new pass_stv (m_ctxt);
    3200              :     }
    3201              : 
    3202       594360 :   void set_pass_param (unsigned int n, bool param) final override
    3203              :     {
    3204       594360 :       gcc_assert (n == 0);
    3205       594360 :       timode_p = param;
    3206       594360 :     }
    3207              : 
    3208              : private:
    3209              :   bool timode_p;
    3210              : }; // class pass_stv
    3211              : 
    3212              : } // anon namespace
    3213              : 
    3214              : rtl_opt_pass *
    3215       297180 : make_pass_insert_vzeroupper (gcc::context *ctxt)
    3216              : {
    3217       297180 :   return new pass_insert_vzeroupper (ctxt);
    3218              : }
    3219              : 
    3220              : rtl_opt_pass *
    3221       297180 : make_pass_stv (gcc::context *ctxt)
    3222              : {
    3223       297180 :   return new pass_stv (ctxt);
    3224              : }
    3225              : 
    3226              : /* Inserting ENDBR and pseudo patchable-area instructions.  */
    3227              : 
    3228              : static void
    3229       199164 : rest_of_insert_endbr_and_patchable_area (bool need_endbr,
    3230              :                                          unsigned int patchable_area_size)
    3231              : {
    3232       199164 :   rtx endbr;
    3233       199164 :   rtx_insn *insn;
    3234       199164 :   rtx_insn *endbr_insn = NULL;
    3235       199164 :   basic_block bb;
    3236              : 
    3237       199164 :   if (need_endbr)
    3238              :     {
    3239              :       /* Currently emit EB if it's a tracking function, i.e. 'nocf_check'
    3240              :          is absent among function attributes.  Later an optimization will
    3241              :          be introduced to make analysis if an address of a static function
    3242              :          is taken.  A static function whose address is not taken will get
    3243              :          a nocf_check attribute.  This will allow to reduce the number of
    3244              :          EB.  */
    3245       199119 :       if (!lookup_attribute ("nocf_check",
    3246       199119 :                              TYPE_ATTRIBUTES (TREE_TYPE (cfun->decl)))
    3247       199119 :           && (lookup_attribute ("cf_check", DECL_ATTRIBUTES (cfun->decl))
    3248       199090 :               || (!flag_manual_endbr
    3249       199088 :                   && (!cgraph_node::get (cfun->decl)->only_called_directly_p ()
    3250        28445 :                       || ix86_cmodel == CM_LARGE
    3251        28444 :                       || ix86_cmodel == CM_LARGE_PIC
    3252        28443 :                       || flag_force_indirect_call
    3253        28443 :                       || (TARGET_DLLIMPORT_DECL_ATTRIBUTES
    3254              :                           && DECL_DLLIMPORT_P (cfun->decl))))))
    3255              :         {
    3256       170657 :           if (crtl->profile && flag_fentry)
    3257              :             {
    3258              :               /* Queue ENDBR insertion to x86_function_profiler.
    3259              :                  NB: Any patchable-area insn will be inserted after
    3260              :                  ENDBR.  */
    3261            6 :               cfun->machine->insn_queued_at_entrance = TYPE_ENDBR;
    3262              :             }
    3263              :           else
    3264              :             {
    3265       170651 :               endbr = gen_nop_endbr ();
    3266       170651 :               bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
    3267       170651 :               rtx_insn *insn = BB_HEAD (bb);
    3268       170651 :               endbr_insn = emit_insn_before (endbr, insn);
    3269              :             }
    3270              :         }
    3271              :     }
    3272              : 
    3273       199164 :   if (patchable_area_size)
    3274              :     {
    3275           51 :       if (crtl->profile && flag_fentry)
    3276              :         {
    3277              :           /* Queue patchable-area insertion to x86_function_profiler.
    3278              :              NB: If there is a queued ENDBR, x86_function_profiler
    3279              :              will also handle patchable-area.  */
    3280            2 :           if (!cfun->machine->insn_queued_at_entrance)
    3281            1 :             cfun->machine->insn_queued_at_entrance = TYPE_PATCHABLE_AREA;
    3282              :         }
    3283              :       else
    3284              :         {
    3285           49 :           rtx patchable_area
    3286           49 :             = gen_patchable_area (GEN_INT (patchable_area_size),
    3287           49 :                                   GEN_INT (crtl->patch_area_entry == 0));
    3288           49 :           if (endbr_insn)
    3289            3 :             emit_insn_after (patchable_area, endbr_insn);
    3290              :           else
    3291              :             {
    3292           46 :               bb = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb;
    3293           46 :               insn = BB_HEAD (bb);
    3294           46 :               emit_insn_before (patchable_area, insn);
    3295              :             }
    3296              :         }
    3297              :     }
    3298              : 
    3299       199164 :   if (!need_endbr)
    3300              :     return;
    3301              : 
    3302       199119 :   bb = 0;
    3303      4171384 :   FOR_EACH_BB_FN (bb, cfun)
    3304              :     {
    3305     77901117 :       for (insn = BB_HEAD (bb); insn != NEXT_INSN (BB_END (bb));
    3306     73928852 :            insn = NEXT_INSN (insn))
    3307              :         {
    3308     73928852 :           if (auto call_insn = dyn_cast<rtx_call_insn *> (insn))
    3309              :             {
    3310      1492732 :               need_endbr = find_reg_note (call_insn, REG_SETJMP, NULL) != NULL;
    3311      1492732 :               if (!need_endbr && !SIBLING_CALL_P (call_insn))
    3312              :                 {
    3313      1443327 :                   rtx call = get_call_rtx_from (call_insn);
    3314      1443327 :                   rtx fnaddr = XEXP (call, 0);
    3315      1443327 :                   tree fndecl = NULL_TREE;
    3316              : 
    3317              :                   /* Also generate ENDBRANCH for non-tail call which
    3318              :                      may return via indirect branch.  */
    3319      1443327 :                   if (SYMBOL_REF_P (XEXP (fnaddr, 0)))
    3320      1384431 :                     fndecl = SYMBOL_REF_DECL (XEXP (fnaddr, 0));
    3321      1384431 :                   if (fndecl == NULL_TREE)
    3322        59264 :                     fndecl = MEM_EXPR (fnaddr);
    3323        59264 :                   if (fndecl
    3324      1440864 :                       && TREE_CODE (TREE_TYPE (fndecl)) != FUNCTION_TYPE
    3325       600602 :                       && TREE_CODE (TREE_TYPE (fndecl)) != METHOD_TYPE)
    3326              :                     fndecl = NULL_TREE;
    3327      1443327 :                   if (fndecl && TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
    3328              :                     {
    3329      1398312 :                       tree fntype = TREE_TYPE (fndecl);
    3330      1398312 :                       if (lookup_attribute ("indirect_return",
    3331      1398312 :                                             TYPE_ATTRIBUTES (fntype)))
    3332              :                         need_endbr = true;
    3333              :                     }
    3334              :                 }
    3335      1492720 :               if (!need_endbr)
    3336      1492712 :                 continue;
    3337              :               /* Generate ENDBRANCH after CALL, which can return more than
    3338              :                  twice, setjmp-like functions.  */
    3339              : 
    3340           20 :               endbr = gen_nop_endbr ();
    3341           20 :               emit_insn_after_setloc (endbr, call_insn,
    3342           20 :                                      INSN_LOCATION (call_insn));
    3343           20 :               continue;
    3344           20 :             }
    3345              : 
    3346     72436120 :           if (JUMP_P (insn) && flag_cet_switch)
    3347              :             {
    3348           15 :               rtx target = JUMP_LABEL (insn);
    3349           15 :               if (target == NULL_RTX || ANY_RETURN_P (target))
    3350            6 :                 continue;
    3351              : 
    3352              :               /* Check the jump is a switch table.  */
    3353            9 :               rtx_insn *label = as_a<rtx_insn *> (target);
    3354            9 :               rtx_insn *table = next_insn (label);
    3355            9 :               if (table == NULL_RTX || !JUMP_TABLE_DATA_P (table))
    3356            5 :                 continue;
    3357              : 
    3358              :               /* For the indirect jump find out all places it jumps and insert
    3359              :                  ENDBRANCH there.  It should be done under a special flag to
    3360              :                  control ENDBRANCH generation for switch stmts.  */
    3361            4 :               edge_iterator ei;
    3362            4 :               edge e;
    3363            4 :               basic_block dest_blk;
    3364              : 
    3365           42 :               FOR_EACH_EDGE (e, ei, bb->succs)
    3366              :                 {
    3367           38 :                   rtx_insn *insn;
    3368              : 
    3369           38 :                   dest_blk = e->dest;
    3370           38 :                   insn = BB_HEAD (dest_blk);
    3371           38 :                   gcc_assert (LABEL_P (insn));
    3372              : 
    3373              :                   /* Reuse existing ENDBR.  */
    3374           38 :                   rtx_insn *next = next_nonnote_nondebug_insn (insn);
    3375           38 :                   if (next && NONJUMP_INSN_P (next)
    3376           30 :                       && GET_CODE (PATTERN (next)) == UNSPEC_VOLATILE
    3377           46 :                       && XINT (PATTERN (next), 1) == UNSPECV_NOP_ENDBR)
    3378            8 :                     continue;
    3379              : 
    3380           30 :                   endbr = gen_nop_endbr ();
    3381           30 :                   emit_insn_after (endbr, insn);
    3382              :                 }
    3383            4 :               continue;
    3384            4 :             }
    3385              : 
    3386     72436105 :           if (LABEL_P (insn) && LABEL_PRESERVE_P (insn))
    3387              :             {
    3388       154564 :               endbr = gen_nop_endbr ();
    3389       154564 :               emit_insn_after (endbr, insn);
    3390       154564 :               continue;
    3391              :             }
    3392              :         }
    3393              :     }
    3394              : 
    3395              :   return;
    3396              : }
    3397              : 
    3398              : namespace {
    3399              : 
    3400              : const pass_data pass_data_insert_endbr_and_patchable_area =
    3401              : {
    3402              :   RTL_PASS, /* type.  */
    3403              :   "endbr_and_patchable_area", /* name.  */
    3404              :   OPTGROUP_NONE, /* optinfo_flags.  */
    3405              :   TV_MACH_DEP, /* tv_id.  */
    3406              :   0, /* properties_required.  */
    3407              :   0, /* properties_provided.  */
    3408              :   0, /* properties_destroyed.  */
    3409              :   0, /* todo_flags_start.  */
    3410              :   0, /* todo_flags_finish.  */
    3411              : };
    3412              : 
    3413              : class pass_insert_endbr_and_patchable_area : public rtl_opt_pass
    3414              : {
    3415              : public:
    3416       297180 :   pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
    3417       594360 :     : rtl_opt_pass (pass_data_insert_endbr_and_patchable_area, ctxt)
    3418              :   {}
    3419              : 
    3420              :   /* opt_pass methods: */
    3421      1517135 :   bool gate (function *) final override
    3422              :     {
    3423      1517135 :       need_endbr = (flag_cf_protection & CF_BRANCH) != 0;
    3424      1517135 :       patchable_area_size = crtl->patch_area_size - crtl->patch_area_entry;
    3425      1517135 :       return need_endbr || patchable_area_size;
    3426              :     }
    3427              : 
    3428       199164 :   unsigned int execute (function *) final override
    3429              :     {
    3430       199164 :       timevar_push (TV_MACH_DEP);
    3431       199164 :       rest_of_insert_endbr_and_patchable_area (need_endbr,
    3432              :                                                patchable_area_size);
    3433       199164 :       timevar_pop (TV_MACH_DEP);
    3434       199164 :       return 0;
    3435              :     }
    3436              : 
    3437              : private:
    3438              :   bool need_endbr;
    3439              :   unsigned int patchable_area_size;
    3440              : }; // class pass_insert_endbr_and_patchable_area
    3441              : 
    3442              : } // anon namespace
    3443              : 
    3444              : rtl_opt_pass *
    3445       297180 : make_pass_insert_endbr_and_patchable_area (gcc::context *ctxt)
    3446              : {
    3447       297180 :   return new pass_insert_endbr_and_patchable_area (ctxt);
    3448              : }
    3449              : 
    3450              : bool
    3451      6119607 : ix86_rpad_gate ()
    3452              : {
    3453      6119607 :   return (TARGET_AVX
    3454       392767 :           && TARGET_SSE_PARTIAL_REG_DEPENDENCY
    3455       294848 :           && TARGET_SSE_MATH
    3456       294554 :           && optimize
    3457      6408958 :           && optimize_function_for_speed_p (cfun));
    3458              : }
    3459              : 
    3460              : enum x86_cse_kind
    3461              : {
    3462              :   X86_CSE_CONST0_VECTOR,
    3463              :   X86_CSE_CONSTM1_VECTOR,
    3464              :   X86_CSE_CONST_VECTOR,
    3465              :   X86_CSE_VEC_DUP,
    3466              :   X86_CSE_TLS_GD,
    3467              :   X86_CSE_TLS_LD_BASE,
    3468              :   X86_CSE_TLSDESC
    3469              : };
    3470              : 
    3471       156570 : struct redundant_pattern
    3472              : {
    3473              :   /* Bitmap of basic blocks with broadcast instructions.  */
    3474              :   auto_bitmap bbs;
    3475              :   /* Bitmap of broadcast instructions.  */
    3476              :   auto_bitmap insns;
    3477              :   /* The broadcast inner scalar.  */
    3478              :   rtx val;
    3479              :   /* The actual redundant source value for UNSPEC_TLSDESC.  */
    3480              :   rtx tlsdesc_val;
    3481              :   /* The inner scalar mode.  */
    3482              :   machine_mode mode;
    3483              :   /* The destination mode which can be changed to the integer mode of
    3484              :      the same time.  */
    3485              :   machine_mode dest_mode;
    3486              :   /* The instruction which sets the inner scalar.  Nullptr if the inner
    3487              :      scalar is applied to the whole function, instead of within the same
    3488              :      block.  */
    3489              :   rtx_insn *def_insn;
    3490              :   /* The widest broadcast source.  */
    3491              :   rtx broadcast_source;
    3492              :   /* The widest broadcast register.  */
    3493              :   rtx broadcast_reg;
    3494              :   /* The basic block of the broadcast instruction.  */
    3495              :   basic_block bb;
    3496              :   /* The number of broadcast instructions with the same inner scalar.  */
    3497              :   unsigned HOST_WIDE_INT count;
    3498              :   /* The threshold of broadcast instructions with the same inner
    3499              :      scalar.  */
    3500              :   unsigned int threshold;
    3501              :   /* The widest broadcast size in bytes.  */
    3502              :   unsigned int size;
    3503              :   /* Load kind.  */
    3504              :   x86_cse_kind kind;
    3505              : };
    3506              : 
    3507              : /* Generate a vector set, DEST = SRC, at entry of the nearest dominator
    3508              :    for basic block map BBS, which is in the fake loop that contains the
    3509              :    whole function, so that there is only a single vector set in the
    3510              :    whole function.  If not nullptr, LOAD is a pointer to the load.  */
    3511              : 
    3512              : static void
    3513        45437 : ix86_place_single_vector_set (rtx dest, rtx src, bitmap bbs,
    3514              :                               redundant_pattern *load = nullptr)
    3515              : {
    3516        45437 :   basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
    3517              :   /* For X86_CSE_VEC_DUP and X86_CSE_CONST_VECTOR, don't place the vector
    3518              :      set outside of the loop to avoid extra spills.  */
    3519        45437 :   if (!load
    3520        44409 :       || (load->kind != X86_CSE_VEC_DUP
    3521        44409 :           && load->kind != X86_CSE_CONST_VECTOR))
    3522              :     {
    3523        24959 :       while (bb->loop_father->latch
    3524        24959 :              != EXIT_BLOCK_PTR_FOR_FN (cfun))
    3525         1482 :         bb = get_immediate_dominator (CDI_DOMINATORS,
    3526              :                                       bb->loop_father->header);
    3527              :     }
    3528              : 
    3529        45437 :   if (CONST_INT_P (src))
    3530        11347 :     dest = gen_rtx_SUBREG (load->dest_mode, dest, 0);
    3531        34090 :   else if (CONST_VECTOR_P (src))
    3532              :     {
    3533              :       /* The only possible CONST_VECTORs of SRC are CONST0_RTX and
    3534              :          CONSTM1_RTX.  Otherwise,
    3535              : 
    3536              :          rtx set = gen_rtx_SET (dest, src);
    3537              : 
    3538              :          won't be a valid instruction.  CONST0_RTX always works.  It
    3539              :          can comes from:
    3540              : 
    3541              :          1. remove_partial_avx_dependency with LOAD == NULL.
    3542              :          2. X86_CSE_VEC_DUP with
    3543              : 
    3544              :          (insn 48 58 16 3 (set (reg:V4HI 123)
    3545              :                 (const_vector:V4HI [
    3546              :                         (const_int 0 [0]) repeated x4
    3547              :                   ])) 2065 {*movv4hi_internal} (nil))
    3548              : 
    3549              :          3. X86_CSE_CONST0_VECTOR.
    3550              :        */
    3551        23477 :       machine_mode mode = GET_MODE (dest);
    3552        23477 :       if (!(src == CONST0_RTX (mode)
    3553         1697 :             || (src == CONSTM1_RTX (mode)
    3554         1697 :                 && load->kind == X86_CSE_CONSTM1_VECTOR)))
    3555            0 :         gcc_unreachable ();
    3556              :     }
    3557        45437 :   rtx set = gen_rtx_SET (dest, src);
    3558              : 
    3559        45437 :   rtx_insn *insn = BB_HEAD (bb);
    3560       184844 :   while (insn && !NONDEBUG_INSN_P (insn))
    3561              :     {
    3562       139411 :       if (insn == BB_END (bb))
    3563              :         {
    3564              :           insn = NULL;
    3565              :           break;
    3566              :         }
    3567       139407 :       insn = NEXT_INSN (insn);
    3568              :     }
    3569              : 
    3570        45437 :   rtx_insn *set_insn;
    3571        45437 :   if (insn == BB_HEAD (bb))
    3572              :     {
    3573            0 :       set_insn = emit_insn_before (set, insn);
    3574            0 :       if (dump_file)
    3575              :         {
    3576            0 :           fprintf (dump_file, "\nPlace:\n\n");
    3577            0 :           print_rtl_single (dump_file, set_insn);
    3578            0 :           fprintf (dump_file, "\nbefore:\n\n");
    3579            0 :           print_rtl_single (dump_file, insn);
    3580            0 :           fprintf (dump_file, "\n");
    3581              :         }
    3582              :     }
    3583              :   else
    3584              :     {
    3585        45437 :       rtx_insn *after = insn ? PREV_INSN (insn) : BB_END (bb);
    3586        45437 :       set_insn = emit_insn_after (set, after);
    3587        45437 :       if (dump_file)
    3588              :         {
    3589            0 :           fprintf (dump_file, "\nPlace:\n\n");
    3590            0 :           print_rtl_single (dump_file, set_insn);
    3591            0 :           fprintf (dump_file, "\nafter:\n\n");
    3592            0 :           print_rtl_single (dump_file, after);
    3593            0 :           fprintf (dump_file, "\n");
    3594              :         }
    3595              :     }
    3596              : 
    3597        45437 :   if (load && load->kind == X86_CSE_VEC_DUP)
    3598              :     {
    3599              :       /* Get the source from LOAD as (reg:SI 99) in
    3600              : 
    3601              :          (vec_duplicate:V4SI (reg:SI 99))
    3602              : 
    3603              :        */
    3604        10613 :       rtx inner_scalar = load->val;
    3605              :       /* Set the source in (vec_duplicate:V4SI (reg:SI 99)).  */
    3606        10613 :       rtx reg = XEXP (src, 0);
    3607        10613 :       machine_mode reg_mode = GET_MODE (reg);
    3608        10613 :       if (reg_mode != GET_MODE (inner_scalar))
    3609              :         {
    3610        10475 :           if (REG_P (inner_scalar) || MEM_P (inner_scalar))
    3611            0 :             inner_scalar = gen_rtx_SUBREG (reg_mode, inner_scalar, 0);
    3612        10475 :           else if (!SCALAR_INT_MODE_P (reg_mode))
    3613              :             {
    3614              :               /* For non-int load with integer constant, generate
    3615              : 
    3616              :                  (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
    3617              :                       (const_int 1313486336 [0x4e4a3600]))
    3618              : 
    3619              :                */
    3620            1 :               gcc_assert (CONST_INT_P (inner_scalar));
    3621            1 :               unsigned int bits = GET_MODE_BITSIZE (reg_mode);
    3622            1 :               machine_mode mode = int_mode_for_size (bits, 0).require ();
    3623            1 :               reg = gen_rtx_SUBREG (mode, reg, 0);
    3624              :             }
    3625              :         }
    3626        10613 :       rtx set = gen_rtx_SET (reg, inner_scalar);
    3627        10613 :       insn = emit_insn_before (set, set_insn);
    3628        10613 :       if (dump_file)
    3629              :         {
    3630            0 :           fprintf (dump_file, "\nAdd:\n\n");
    3631            0 :           print_rtl_single (dump_file, insn);
    3632            0 :           fprintf (dump_file, "\nbefore:\n\n");
    3633            0 :           print_rtl_single (dump_file, set_insn);
    3634            0 :           fprintf (dump_file, "\n");
    3635              :         }
    3636              :     }
    3637        45437 : }
    3638              : 
    3639              : /* At entry of the nearest common dominator for basic blocks with
    3640              :    conversions/rcp/sqrt/rsqrt/round, generate a single
    3641              :         vxorps %xmmN, %xmmN, %xmmN
    3642              :    for all
    3643              :         vcvtss2sd  op, %xmmN, %xmmX
    3644              :         vcvtsd2ss  op, %xmmN, %xmmX
    3645              :         vcvtsi2ss  op, %xmmN, %xmmX
    3646              :         vcvtsi2sd  op, %xmmN, %xmmX
    3647              : 
    3648              :    NB: We want to generate only a single vxorps to cover the whole
    3649              :    function.  The LCM algorithm isn't appropriate here since it may
    3650              :    place a vxorps inside the loop.  */
    3651              : 
    3652              : static unsigned int
    3653        34333 : remove_partial_avx_dependency (void)
    3654              : {
    3655        34333 :   timevar_push (TV_MACH_DEP);
    3656              : 
    3657        34333 :   bitmap_obstack_initialize (NULL);
    3658        34333 :   bitmap convert_bbs = BITMAP_ALLOC (NULL);
    3659              : 
    3660        34333 :   basic_block bb;
    3661        34333 :   rtx_insn *insn, *set_insn;
    3662        34333 :   rtx set;
    3663        34333 :   rtx v4sf_const0 = NULL_RTX;
    3664              : 
    3665        34333 :   auto_vec<rtx_insn *> control_flow_insns;
    3666              : 
    3667              :   /* We create invalid RTL initially so defer rescans.  */
    3668        34333 :   df_set_flags (DF_DEFER_INSN_RESCAN);
    3669              : 
    3670       313703 :   FOR_EACH_BB_FN (bb, cfun)
    3671              :     {
    3672      3508123 :       FOR_BB_INSNS (bb, insn)
    3673              :         {
    3674      3228753 :           if (!NONDEBUG_INSN_P (insn))
    3675      1432602 :             continue;
    3676              : 
    3677      1796151 :           set = single_set (insn);
    3678      1796151 :           if (!set)
    3679        72005 :             continue;
    3680              : 
    3681      1724146 :           if (get_attr_avx_partial_xmm_update (insn)
    3682              :               != AVX_PARTIAL_XMM_UPDATE_TRUE)
    3683      1720953 :             continue;
    3684              : 
    3685              :           /* Convert PARTIAL_XMM_UPDATE_TRUE insns, DF -> SF, SF -> DF,
    3686              :              SI -> SF, SI -> DF, DI -> SF, DI -> DF, sqrt, rsqrt, rcp,
    3687              :              round, to vec_dup and vec_merge with subreg.  */
    3688         3193 :           rtx src = SET_SRC (set);
    3689         3193 :           rtx dest = SET_DEST (set);
    3690         3193 :           machine_mode dest_mode = GET_MODE (dest);
    3691         3193 :           bool convert_p = false;
    3692         3193 :           switch (GET_CODE (src))
    3693              :             {
    3694         3128 :             case FLOAT:
    3695         3128 :             case FLOAT_EXTEND:
    3696         3128 :             case FLOAT_TRUNCATE:
    3697         3128 :             case UNSIGNED_FLOAT:
    3698         3128 :               convert_p = true;
    3699         3128 :               break;
    3700              :             default:
    3701              :               break;
    3702              :             }
    3703              : 
    3704              :           /* Only handle conversion here.  */
    3705         3128 :           machine_mode src_mode
    3706         3128 :             = convert_p ? GET_MODE (XEXP (src, 0)) : VOIDmode;
    3707         3128 :           switch (src_mode)
    3708              :             {
    3709          153 :             case E_SFmode:
    3710          153 :             case E_DFmode:
    3711          153 :               if (TARGET_USE_VECTOR_FP_CONVERTS
    3712          147 :                   || !TARGET_SSE_PARTIAL_REG_FP_CONVERTS_DEPENDENCY)
    3713            8 :                 continue;
    3714              :               break;
    3715         2975 :             case E_SImode:
    3716         2975 :             case E_DImode:
    3717         2975 :               if (TARGET_USE_VECTOR_CONVERTS
    3718         2963 :                   || !TARGET_SSE_PARTIAL_REG_CONVERTS_DEPENDENCY)
    3719           14 :                 continue;
    3720              :               break;
    3721           65 :             case E_VOIDmode:
    3722           65 :               gcc_assert (!convert_p);
    3723              :               break;
    3724            0 :             default:
    3725            0 :               gcc_unreachable ();
    3726              :             }
    3727              : 
    3728         3171 :           if (!v4sf_const0)
    3729         1028 :             v4sf_const0 = gen_reg_rtx (V4SFmode);
    3730              : 
    3731         3171 :           rtx zero;
    3732         3171 :           machine_mode dest_vecmode;
    3733         3171 :           switch (dest_mode)
    3734              :             {
    3735           50 :             case E_HFmode:
    3736           50 :               dest_vecmode = V8HFmode;
    3737           50 :               zero = gen_rtx_SUBREG (V8HFmode, v4sf_const0, 0);
    3738           50 :               break;
    3739              :             case E_SFmode:
    3740              :               dest_vecmode = V4SFmode;
    3741              :               zero = v4sf_const0;
    3742              :               break;
    3743         1186 :             case E_DFmode:
    3744         1186 :               dest_vecmode = V2DFmode;
    3745         1186 :               zero = gen_rtx_SUBREG (V2DFmode, v4sf_const0, 0);
    3746         1186 :               break;
    3747            0 :             default:
    3748            0 :               gcc_unreachable ();
    3749              :             }
    3750              : 
    3751              :           /* Change source to vector mode.  */
    3752         3171 :           src = gen_rtx_VEC_DUPLICATE (dest_vecmode, src);
    3753         3171 :           src = gen_rtx_VEC_MERGE (dest_vecmode, src, zero,
    3754              :                                    GEN_INT (HOST_WIDE_INT_1U));
    3755              :           /* Change destination to vector mode.  */
    3756         3171 :           rtx vec = gen_reg_rtx (dest_vecmode);
    3757              :           /* Generate an XMM vector SET.  */
    3758         3171 :           set = gen_rtx_SET (vec, src);
    3759         3171 :           set_insn = emit_insn_before (set, insn);
    3760              : 
    3761         3171 :           if (cfun->can_throw_non_call_exceptions)
    3762              :             {
    3763              :               /* Handle REG_EH_REGION note.  */
    3764            0 :               rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
    3765            0 :               if (note)
    3766              :                 {
    3767            0 :                   control_flow_insns.safe_push (set_insn);
    3768            0 :                   add_reg_note (set_insn, REG_EH_REGION, XEXP (note, 0));
    3769              :                 }
    3770              :             }
    3771              : 
    3772         3171 :           src = gen_rtx_SUBREG (dest_mode, vec, 0);
    3773         3171 :           set = gen_rtx_SET (dest, src);
    3774              : 
    3775              :           /* Drop possible dead definitions.  */
    3776         3171 :           PATTERN (insn) = set;
    3777              : 
    3778         3171 :           INSN_CODE (insn) = -1;
    3779         3171 :           recog_memoized (insn);
    3780         3171 :           df_insn_rescan (insn);
    3781         3171 :           bitmap_set_bit (convert_bbs, bb->index);
    3782              :         }
    3783              :     }
    3784              : 
    3785        34333 :   if (v4sf_const0)
    3786              :     {
    3787              :       /* (Re-)discover loops so that bb->loop_father can be used in the
    3788              :          analysis below.  */
    3789         1028 :       calculate_dominance_info (CDI_DOMINATORS);
    3790         1028 :       loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
    3791              : 
    3792         1028 :       ix86_place_single_vector_set (v4sf_const0,
    3793              :                                     CONST0_RTX (V4SFmode),
    3794              :                                     convert_bbs);
    3795              : 
    3796         1028 :       loop_optimizer_finalize ();
    3797              : 
    3798        34333 :       if (!control_flow_insns.is_empty ())
    3799              :         {
    3800            0 :           free_dominance_info (CDI_DOMINATORS);
    3801              : 
    3802            0 :           unsigned int i;
    3803            0 :           FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
    3804            0 :             if (control_flow_insn_p (insn))
    3805              :               {
    3806              :                 /* Split the block after insn.  There will be a fallthru
    3807              :                    edge, which is OK so we keep it.  We have to create
    3808              :                    the exception edges ourselves.  */
    3809            0 :                 bb = BLOCK_FOR_INSN (insn);
    3810            0 :                 split_block (bb, insn);
    3811            0 :                 rtl_make_eh_edge (NULL, bb, BB_END (bb));
    3812              :               }
    3813              :         }
    3814              :     }
    3815              : 
    3816        34333 :   df_process_deferred_rescans ();
    3817        34333 :   df_clear_flags (DF_DEFER_INSN_RESCAN);
    3818        34333 :   bitmap_obstack_release (NULL);
    3819        34333 :   BITMAP_FREE (convert_bbs);
    3820              : 
    3821        34333 :   timevar_pop (TV_MACH_DEP);
    3822        34333 :   return 0;
    3823        34333 : }
    3824              : 
    3825              : namespace {
    3826              : 
    3827              : const pass_data pass_data_remove_partial_avx_dependency =
    3828              : {
    3829              :   RTL_PASS, /* type */
    3830              :   "rpad", /* name */
    3831              :   OPTGROUP_NONE, /* optinfo_flags */
    3832              :   TV_MACH_DEP, /* tv_id */
    3833              :   0, /* properties_required */
    3834              :   0, /* properties_provided */
    3835              :   0, /* properties_destroyed */
    3836              :   0, /* todo_flags_start */
    3837              :   0, /* todo_flags_finish */
    3838              : };
    3839              : 
    3840              : class pass_remove_partial_avx_dependency : public rtl_opt_pass
    3841              : {
    3842              : public:
    3843       297180 :   pass_remove_partial_avx_dependency (gcc::context *ctxt)
    3844       594360 :     : rtl_opt_pass (pass_data_remove_partial_avx_dependency, ctxt)
    3845              :   {}
    3846              : 
    3847              :   /* opt_pass methods: */
    3848      1517135 :   bool gate (function *) final override
    3849              :     {
    3850      1517135 :       return ix86_rpad_gate ();
    3851              :     }
    3852              : 
    3853        34333 :   unsigned int execute (function *) final override
    3854              :     {
    3855        34333 :       return remove_partial_avx_dependency ();
    3856              :     }
    3857              : }; // class pass_rpad
    3858              : 
    3859              : } // anon namespace
    3860              : 
    3861              : rtl_opt_pass *
    3862       297180 : make_pass_remove_partial_avx_dependency (gcc::context *ctxt)
    3863              : {
    3864       297180 :   return new pass_remove_partial_avx_dependency (ctxt);
    3865              : }
    3866              : 
    3867              : /* Return a machine mode suitable for vector SIZE with SMODE inner
    3868              :    mode.  */
    3869              : 
    3870              : static machine_mode
    3871        66760 : ix86_get_vector_cse_mode (unsigned int size, machine_mode smode)
    3872              : {
    3873              :   /* Use the inner scalar mode of vector broadcast source in:
    3874              : 
    3875              :      (set (reg:V8DF 394)
    3876              :           (vec_duplicate:V8DF (reg:V2DF 190 [ alpha ])))
    3877              : 
    3878              :      to compute the vector mode for broadcast from vector source.
    3879              :    */
    3880        66760 :   if (VECTOR_MODE_P (smode))
    3881        32275 :     smode = GET_MODE_INNER (smode);
    3882        66760 :   scalar_mode s_mode = as_a <scalar_mode> (smode);
    3883       133520 :   poly_uint64 nunits = size / GET_MODE_SIZE (smode);
    3884        66760 :   machine_mode mode = mode_for_vector (s_mode, nunits).require ();
    3885        66760 :   return mode;
    3886              : }
    3887              : 
    3888              : /* Replace the source operand of instructions in VECTOR_INSNS with
    3889              :    VECTOR_CONST in VECTOR_MODE.  */
    3890              : 
    3891              : static void
    3892        66117 : replace_vector_const (machine_mode vector_mode, rtx vector_const,
    3893              :                       auto_bitmap &vector_insns,
    3894              :                       machine_mode scalar_mode)
    3895              : {
    3896        66117 :   bitmap_iterator bi;
    3897        66117 :   unsigned int id;
    3898              : 
    3899       232594 :   EXECUTE_IF_SET_IN_BITMAP (vector_insns, 0, id, bi)
    3900              :     {
    3901       166477 :       rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
    3902              : 
    3903              :       /* Get the single SET instruction.  */
    3904       166477 :       rtx set = single_set (insn);
    3905       166477 :       rtx src = SET_SRC (set);
    3906       166477 :       rtx dest = SET_DEST (set);
    3907       166477 :       machine_mode mode = GET_MODE (dest);
    3908              : 
    3909       166477 :       rtx replace;
    3910              :       /* Replace the source operand with VECTOR_CONST.  */
    3911       166477 :       if (SUBREG_P (src)
    3912       166477 :           || mode == vector_mode
    3913        61344 :           || CONST_INT_P (vector_const))
    3914              :         replace = vector_const;
    3915              :       else
    3916              :         {
    3917        61337 :           unsigned int size = GET_MODE_SIZE (mode);
    3918        61337 :           if (size < ix86_regmode_natural_size (mode))
    3919              :             {
    3920              :               /* If the mode size is smaller than its natural size,
    3921              :                  first insert an extra move with a QI vector SUBREG
    3922              :                  of the same size to avoid validate_subreg failure.  */
    3923          643 :               machine_mode vmode
    3924          643 :                 = ix86_get_vector_cse_mode (size, scalar_mode);
    3925          643 :               rtx vreg;
    3926          643 :               if (mode == vmode)
    3927              :                 vreg = vector_const;
    3928              :               else
    3929              :                 {
    3930          135 :                   vreg = gen_reg_rtx (vmode);
    3931          135 :                   rtx vsubreg = gen_rtx_SUBREG (vmode, vector_const, 0);
    3932          135 :                   rtx pat = gen_rtx_SET (vreg, vsubreg);
    3933          135 :                   rtx_insn *vinsn = emit_insn_before (pat, insn);
    3934          135 :                   if (dump_file)
    3935              :                     {
    3936            0 :                       fprintf (dump_file, "\nInsert an extra move:\n\n");
    3937            0 :                       print_rtl_single (dump_file, vinsn);
    3938            0 :                       fprintf (dump_file, "\nbefore:\n\n");
    3939            0 :                       print_rtl_single (dump_file, insn);
    3940            0 :                       fprintf (dump_file, "\n");
    3941              :                     }
    3942              :                 }
    3943          643 :               replace = gen_rtx_SUBREG (mode, vreg, 0);
    3944              :             }
    3945              :           else
    3946        60694 :             replace = gen_rtx_SUBREG (mode, vector_const, 0);
    3947              :         }
    3948              : 
    3949       166477 :       if (dump_file)
    3950              :         {
    3951            0 :           fprintf (dump_file, "\nReplace:\n\n");
    3952            0 :           print_rtl_single (dump_file, insn);
    3953              :         }
    3954       166477 :       SET_SRC (set) = replace;
    3955       166477 :       if (CONST_INT_P (replace))
    3956              :         {
    3957        24430 :           dest = gen_lowpart (scalar_mode, dest);
    3958        24430 :           SET_DEST (set) = dest;
    3959              :         }
    3960              :       /* Drop possible dead definitions.  */
    3961       166477 :       PATTERN (insn) = set;
    3962       166477 :       INSN_CODE (insn) = -1;
    3963       166477 :       recog_memoized (insn);
    3964              :       /* Add a REG_EQUAL note with the original CONST_VECTOR so that LRA
    3965              :          can rematerialize it instead of spilling DEST.  */
    3966       166477 :       if (REG_P (dest) && !CONST_INT_P (replace) && CONST_VECTOR_P (src))
    3967        90256 :         set_unique_reg_note (insn, REG_EQUAL, copy_rtx (src));
    3968       166477 :       if (dump_file)
    3969              :         {
    3970            0 :           fprintf (dump_file, "\nwith:\n\n");
    3971            0 :           print_rtl_single (dump_file, insn);
    3972            0 :           fprintf (dump_file, "\n");
    3973              :         }
    3974       166477 :       df_insn_rescan (insn);
    3975              :     }
    3976        66117 : }
    3977              : 
    3978              : /* Return the inner scalar if OP is a broadcast, else return nullptr.  */
    3979              : 
    3980              : static rtx
    3981      2220655 : ix86_broadcast_inner (rtx op, machine_mode mode,
    3982              :                       machine_mode *scalar_mode_p,
    3983              :                       x86_cse_kind *kind_p, rtx_insn **insn_p)
    3984              : {
    3985      2220655 :   switch (standard_sse_constant_p (op, mode))
    3986              :     {
    3987       119749 :     case 1:
    3988       119749 :       *scalar_mode_p = QImode;
    3989       119749 :       *kind_p = X86_CSE_CONST0_VECTOR;
    3990       119749 :       *insn_p = nullptr;
    3991       119749 :       return const0_rtx;
    3992        12914 :     case 2:
    3993        12914 :       *scalar_mode_p = QImode;
    3994        12914 :       *kind_p = X86_CSE_CONSTM1_VECTOR;
    3995        12914 :       *insn_p = nullptr;
    3996        12914 :       return constm1_rtx;
    3997      2087992 :     default:
    3998      2087992 :       break;
    3999              :     }
    4000              : 
    4001      2087992 :   mode = GET_MODE (op);
    4002      2087992 :   int nunits = GET_MODE_NUNITS (mode);
    4003      2087992 :   if (nunits < 2)
    4004              :     return nullptr;
    4005              : 
    4006      1581095 :   bool const_vector_p = CONST_VECTOR_P (op);
    4007      1581095 :   bool duplicated = GET_CODE (op) == VEC_DUPLICATE;
    4008      1581095 :   rtx orig_op = op;
    4009      1581095 :   if (!const_vector_p)
    4010              :     {
    4011              :       /* Check CONST_VECTOR in REG_EQUAL note.  */
    4012      1581075 :       rtx equal = find_reg_equal_equiv_note (*insn_p);
    4013      1581075 :       if (equal)
    4014              :         {
    4015       386208 :           equal = XEXP (equal, 0);
    4016       386208 :           const_vector_p = CONST_VECTOR_P (equal);
    4017              :           /* Use CONST_VECTOR in REG_EQUAL note.  */
    4018       386208 :           if (const_vector_p)
    4019              :             {
    4020              :               /* Handle REG_EQUAL note in:
    4021              : 
    4022              :                  (insn 7 5 12 2 (set (subreg:V8SI (reg:V4DI 100) 0)
    4023              :                         (vec_duplicate:V8SI (reg:SI 102)))
    4024              :                     (expr_list:REG_DEAD (reg:SI 102)
    4025              :                        (expr_list:REG_EQUAL (const_vector:V4DI [
    4026              :                           (const_int -1 [0xffffffffffffffff]) repeated x4]) (nil))))
    4027              : 
    4028              :                  NB: Don't treat it as CONST_VECTOR since EQUAL isn't
    4029              :                  supported by ISAs as in gcc.target/i386/pr40957.c.  */
    4030       273651 :               if (GET_MODE (equal) != mode)
    4031              :                 const_vector_p = false;
    4032              :               else
    4033      1581095 :                 op = equal;
    4034              :             }
    4035              :         }
    4036              :     }
    4037              : 
    4038      1581095 :   machine_mode inner_mode = GET_MODE_INNER (mode);
    4039              : 
    4040      1581095 :   if (const_vector_p)
    4041              :     {
    4042       547268 :       bool int_load_p = GET_MODE_SIZE (mode) <= UNITS_PER_WORD;
    4043       273634 :       *kind_p = X86_CSE_CONST_VECTOR;
    4044       273634 :       if (int_load_p)
    4045              :         {
    4046              :           /* This CONST_VECTOR load can be converted to constant
    4047              :              integer load.  */
    4048        36420 :           *scalar_mode_p = mode;
    4049        36420 :           *insn_p = nullptr;
    4050        36420 :           return op;
    4051              :         }
    4052              : 
    4053              :       /* This CONST_VECTOR is wider than the integer register.  */
    4054       237214 :       rtx first = XVECEXP (op, 0, 0);
    4055              : 
    4056       237214 :       if (duplicated)
    4057              :         {
    4058              :           /* Check if CONST_VECTOR in REG_EQUAL note is duplicated in
    4059              : 
    4060              :              (insn 10 7 12 2 (set (reg:V8SI 128)
    4061              :                 (vec_duplicate:V8SI (vec_select:V2SI (reg:V4SI 180)
    4062              :                         (parallel [(const_int 0 [0])
    4063              :                                    (const_int 1 [0x1])]))))
    4064              :                   (expr_list:REG_EQUAL (const_vector:V8SI [
    4065              :                     (const_int 0 [0])
    4066              :                     (const_int 34 [0x22])
    4067              :                     (const_int 0 [0])
    4068              :                     (const_int 34 [0x22])
    4069              :                     (const_int 0 [0])
    4070              :                     (const_int 34 [0x22])
    4071              :                     (const_int 0 [0])
    4072              :                     (const_int 34 [0x22])])(nil)))
    4073              : 
    4074              :            */
    4075              : 
    4076       205236 :           bool duplicated_const_vector = true;
    4077       205236 :           for (int i = 1; i < nunits; ++i)
    4078              :             {
    4079       133820 :               rtx tmp = XVECEXP (op, 0, i);
    4080       133820 :               if (!rtx_equal_p (tmp, first))
    4081              :                 {
    4082              :                   duplicated_const_vector = false;
    4083              :                   break;
    4084              :                 }
    4085              :             }
    4086              : 
    4087        71432 :           if (duplicated_const_vector)
    4088              :             {
    4089        71416 :               bool const_double_p = CONST_DOUBLE_P (first);
    4090              :               /* Force the floating point constant to memory.  */
    4091        71416 :               if (const_double_p)
    4092         1257 :                 first = validize_mem (force_const_mem (inner_mode, first));
    4093              : 
    4094        71416 :               if (const_double_p || CONST_INT_P (first))
    4095              :                 {
    4096              :                   /* Handle
    4097              : 
    4098              :                      (insn 7 6 8 2 (set (reg:V4SF 99)
    4099              :                           (vec_duplicate:V4SF (mem/u/c:SF (symbol_ref/u:DI ("*.LC2") [flags 0x2]) [0  S4 A32])))
    4100              :                         (expr_list:REG_EQUAL (const_vector:V4SF [
    4101              :                            (const_double:SF 3.4e+1 [0x0.88p+6]) repeated x4]) (nil)))
    4102              : 
    4103              :                      and
    4104              : 
    4105              :                      (insn 14 15 16 3 (set (reg:V4SI 116)
    4106              :                           (vec_duplicate:V4SI (reg:SI 117)))
    4107              :                        (expr_list:REG_EQUAL (const_vector:V4SI [
    4108              :                           (const_int 34 [0x22]) repeated x4]) (nil)))
    4109              : 
    4110              :                    */
    4111        71416 :                   *kind_p = X86_CSE_VEC_DUP;
    4112        71416 :                   *insn_p = nullptr;
    4113        71416 :                   *scalar_mode_p = inner_mode;
    4114        71416 :                   return first;
    4115              :                 }
    4116              :             }
    4117              : 
    4118              :           op = orig_op;
    4119              :         }
    4120              :       else
    4121              :         {
    4122              :           /* Only native CONST_VECTOR is allowed.  */
    4123       165782 :           if (orig_op != op)
    4124              :             return nullptr;
    4125              : 
    4126              :           /* Check if VEC_DUPLICATE can be used.  */
    4127           48 :           for (int i = 1; i < nunits; ++i)
    4128              :             {
    4129           48 :               rtx tmp = XVECEXP (op, 0, i);
    4130              :               /* Vector duplicate value.  */
    4131           48 :               if (!rtx_equal_p (tmp, first))
    4132              :                 return nullptr;
    4133              :             }
    4134              : 
    4135              :           /* Use the inner mode to handle
    4136              :              (const_vector:V2QI [(const_int 0 [0]) repeated x2])
    4137              :            */
    4138            0 :           *scalar_mode_p = inner_mode;
    4139            0 :           *insn_p = nullptr;
    4140            0 :           return first;
    4141              :         }
    4142              :     }
    4143              : 
    4144      1307477 :   if (!duplicated)
    4145              :     return nullptr;
    4146              : 
    4147        23493 :   *kind_p = X86_CSE_VEC_DUP;
    4148              : 
    4149              :   /* Only
    4150              : 
    4151              :      (vec_duplicate:V4SI (reg:SI 99))
    4152              :      (vec_duplicate:V2DF (mem/u/c:DF (symbol_ref/u:DI ("*.LC1") [flags 0x2]) [0 S8 A64]))
    4153              : 
    4154              :      are supported.  Set OP to the broadcast source by default.  */
    4155        23493 :   op = XEXP (op, 0);
    4156        23493 :   rtx reg = op;
    4157        23493 :   if (SUBREG_P (op)
    4158          414 :       && SUBREG_BYTE (op) == 0
    4159        23907 :       && !paradoxical_subreg_p (op))
    4160          414 :     reg = SUBREG_REG (op);
    4161        23493 :   if (!REG_P (reg))
    4162              :     {
    4163         2354 :       if (MEM_P (op)
    4164         2084 :           && SYMBOL_REF_P (XEXP (op, 0))
    4165         2629 :           && CONSTANT_POOL_ADDRESS_P (XEXP (op, 0)))
    4166              :         {
    4167              :           /* Handle constant broadcast from memory.  */
    4168           10 :           *scalar_mode_p = inner_mode;
    4169           10 :           *insn_p = nullptr;
    4170           10 :           return op;
    4171              :         }
    4172              :       return nullptr;
    4173              :     }
    4174              : 
    4175        21139 :   machine_mode orig_mode = mode;
    4176        21139 :   mode = GET_MODE (op);
    4177              : 
    4178              :   /* Only single def chain is supported.  */
    4179        21139 :   df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
    4180        21139 :   if (!ref
    4181        21139 :       || DF_REF_IS_ARTIFICIAL (ref)
    4182        21139 :       || DF_REF_NEXT_REG (ref) != nullptr)
    4183              :     return nullptr;
    4184              : 
    4185        15459 :   rtx_insn *insn = DF_REF_INSN (ref);
    4186              : 
    4187              :   /* Skip JUMP which happens with asm goto.  */
    4188        15459 :   if (JUMP_P (insn))
    4189              :     return nullptr;
    4190              : 
    4191        15457 :   rtx set = single_set (insn);
    4192        15457 :   if (!set)
    4193              :     return nullptr;
    4194              : 
    4195        15447 :   rtx src = SET_SRC (set);
    4196              : 
    4197        15447 :   if (CONST_INT_P (src))
    4198              :     {
    4199              :       /* Handle sequences like
    4200              : 
    4201              :          (set (subreg:SI (reg/v:SF 105 [ f ]) 0)
    4202              :               (const_int 0 [0]))
    4203              :          (set (reg:V4SF 110)
    4204              :               (vec_duplicate:V4SF (reg/v:SF 105 [ f ])))
    4205              : 
    4206              :          and
    4207              : 
    4208              :          (set (reg:SI 99)
    4209              :                (const_int 34 [0x22]))
    4210              :          (set (reg:V4SI 98)
    4211              :                (vec_duplicate:V4SI (reg:SI 99)))
    4212              : 
    4213              :          Set *INSN_P to nullptr and return SET_SRC if SET_SRC is an
    4214              :          integer constant.  */
    4215          231 :       op = src;
    4216          231 :       if (SCALAR_INT_MODE_P (mode) && mode != GET_MODE (reg))
    4217            0 :         op = gen_int_mode (INTVAL (src), mode);
    4218          231 :       if (op == const0_rtx)
    4219              :         {
    4220            4 :            if (standard_sse_constant_p (CONST0_RTX (orig_mode),
    4221              :                                         orig_mode) == 1)
    4222              :              {
    4223            4 :                *scalar_mode_p = QImode;
    4224            4 :                *kind_p = X86_CSE_CONST0_VECTOR;
    4225            4 :                *insn_p = nullptr;
    4226            4 :                return const0_rtx;
    4227              :              }
    4228            0 :            op = CONST0_RTX (mode);
    4229              :         }
    4230          227 :       else if (op == constm1_rtx
    4231          227 :                && standard_sse_constant_p (CONSTM1_RTX (orig_mode),
    4232              :                                            orig_mode) == 2)
    4233              :         {
    4234            0 :           *scalar_mode_p = QImode;
    4235            0 :           *kind_p = X86_CSE_CONSTM1_VECTOR;
    4236            0 :           *insn_p = nullptr;
    4237            0 :           return constm1_rtx;
    4238              :         }
    4239              : 
    4240              :       /* Check if we can convert:
    4241              : 
    4242              :          (insn 14 465 412 3 (set (reg:SI 507 [ j_lsm.26 ])
    4243              :                 (const_int 2 [0x2])) "foo.c":10:12 discrim 2 100 {*movsi_internal} (nil))
    4244              :          ...
    4245              :          (insn 518 507 434 16 (set (reg:V2SI 493)
    4246              :                 (vec_duplicate:V2SI (reg:SI 507 [ j_lsm.26 ]))) 2395 {*vec_dupv2si} (nil))
    4247              : 
    4248              :          to constant integer load:
    4249              : 
    4250              :          (insn 566 55 56 6 (set (subreg:DI (reg:V2SI 517) 0)
    4251              :                 (const_int 8589934594 [0x200000002])) -1 (nil))
    4252              :          ...
    4253              :          (insn 518 507 434 16 (set (reg:V2SI 493)
    4254              :                 (reg:V2SI 517)) 2066 {*movv2si_internal} (nil))
    4255              : 
    4256              :        */
    4257          454 :       if (GET_MODE_SIZE (orig_mode) <= UNITS_PER_WORD)
    4258           17 :         *kind_p = X86_CSE_CONST_VECTOR;
    4259              : 
    4260          227 :       *insn_p = nullptr;
    4261              :     }
    4262              :   else
    4263              :     {
    4264              :       /* Handle sequences like
    4265              : 
    4266              :          (set (reg:QI 105 [ c ])
    4267              :               (reg:QI 5 di [ c ]))
    4268              :          (set (reg:V64QI 102 [ _1 ])
    4269              :               (vec_duplicate:V64QI (reg:QI 105 [ c ])))
    4270              : 
    4271              :          (set (reg/v:SI 116 [ argc ])
    4272              :               (mem/c:SI (reg:SI 135) [2 argc+0 S4 A32]))
    4273              :          (set (reg:V4SI 119 [ _45 ])
    4274              :               (vec_duplicate:V4SI (reg/v:SI 116 [ argc ])))
    4275              : 
    4276              :          (set (reg:SI 98 [ _1 ])
    4277              :               (sign_extend:SI (reg:QI 106 [ c ])))
    4278              :          (set (reg:V16SI 103 [ _2 ])
    4279              :                (vec_duplicate:V16SI (reg:SI 98 [ _1 ])))
    4280              : 
    4281              :          (set (reg:SI 102 [ cost ])
    4282              :               (mem/c:SI (symbol_ref:DI ("cost") [flags 0x40])))
    4283              :          (set (reg:V4HI 103 [ _16 ])
    4284              :               (vec_duplicate:V4HI (subreg:HI (reg:SI 102 [ cost ]) 0)))
    4285              : 
    4286              :          (set (subreg:SI (reg/v:HI 107 [ cr_val ]) 0)
    4287              :               (ashift:SI (reg:SI 158)
    4288              :                          (subreg:QI (reg:SI 156 [ _2 ]) 0)))
    4289              :          (set (reg:V16HI 183 [ _61 ])
    4290              :               (vec_duplicate:V16HI (reg/v:HI 107 [ cr_val ])))
    4291              : 
    4292              :          Set *INSN_P to INSN and return the broadcast source otherwise.  */
    4293        15216 :       *insn_p = insn;
    4294              :     }
    4295              : 
    4296        15443 :   *scalar_mode_p = mode;
    4297        15443 :   return op;
    4298              : }
    4299              : 
    4300              : /* Replace CALL instruction in TLS_CALL_INSNS with SET from SRC and
    4301              :    put the updated instruction in UPDATED_TLS_INSNS.  */
    4302              : 
    4303              : static void
    4304          323 : replace_tls_call (rtx src, auto_bitmap &tls_call_insns,
    4305              :                   auto_bitmap &updated_tls_insns)
    4306              : {
    4307          323 :   bitmap_iterator bi;
    4308          323 :   unsigned int id;
    4309              : 
    4310         1763 :   EXECUTE_IF_SET_IN_BITMAP (tls_call_insns, 0, id, bi)
    4311              :     {
    4312         1440 :       rtx_insn *insn = DF_INSN_UID_GET (id)->insn;
    4313              : 
    4314              :       /* If this isn't a CALL, only GNU2 TLS implicit CALL patterns are
    4315              :          allowed.  */
    4316         1440 :       if (!CALL_P (insn))
    4317              :         {
    4318           47 :           attr_tls64 tls64 = get_attr_tls64 (insn);
    4319           47 :           if (tls64 != TLS64_CALL && tls64 != TLS64_COMBINE)
    4320            0 :             gcc_unreachable ();
    4321              :         }
    4322              : 
    4323         1440 :       rtx pat = PATTERN (insn);
    4324         1440 :       gcc_assert (GET_CODE (pat) == PARALLEL);
    4325         1440 :       rtx set = XVECEXP (pat, 0, 0);
    4326         1440 :       gcc_assert (GET_CODE (set) == SET);
    4327         1440 :       rtx dest = SET_DEST (set);
    4328              : 
    4329         1440 :       set = gen_rtx_SET (dest, src);
    4330         1440 :       rtx_insn *set_insn = emit_insn_after (set, insn);
    4331         1440 :       if (recog_memoized (set_insn) < 0)
    4332            0 :         gcc_unreachable ();
    4333              : 
    4334              :       /* Put SET_INSN in UPDATED_TLS_INSNS.  */
    4335         1440 :       bitmap_set_bit (updated_tls_insns, INSN_UID (set_insn));
    4336              : 
    4337         1440 :       if (dump_file)
    4338              :         {
    4339            0 :           fprintf (dump_file, "\nReplace:\n\n");
    4340            0 :           print_rtl_single (dump_file, insn);
    4341            0 :           fprintf (dump_file, "\nwith:\n\n");
    4342            0 :           print_rtl_single (dump_file, set_insn);
    4343            0 :           fprintf (dump_file, "\n");
    4344              :         }
    4345              : 
    4346              :       /* Delete the CALL insn.  */
    4347         1440 :       delete_insn (insn);
    4348              : 
    4349         1440 :       df_insn_rescan (set_insn);
    4350              :     }
    4351          323 : }
    4352              : 
    4353              : /* Return the basic block which dominates all basic blocks which set
    4354              :    hard register REGNO used in basic block BB.  */
    4355              : 
    4356              : static basic_block
    4357            2 : ix86_get_dominator_for_reg (unsigned int regno, basic_block bb)
    4358              : {
    4359            2 :   basic_block set_bb;
    4360            2 :   auto_bitmap set_bbs;
    4361              : 
    4362              :   /* Get all BBs which set REGNO and dominate the current BB from all
    4363              :      DEFs of REGNO.  */
    4364            2 :   for (df_ref def = DF_REG_DEF_CHAIN (regno);
    4365           18 :        def;
    4366           16 :        def = DF_REF_NEXT_REG (def))
    4367           16 :     if (!DF_REF_IS_ARTIFICIAL (def)
    4368           16 :         && !DF_REF_FLAGS_IS_SET (def, DF_REF_MAY_CLOBBER)
    4369            6 :         && !DF_REF_FLAGS_IS_SET (def, DF_REF_MUST_CLOBBER))
    4370              :       {
    4371            4 :         set_bb = DF_REF_BB (def);
    4372            4 :         if (dominated_by_p (CDI_DOMINATORS, bb, set_bb))
    4373            2 :           bitmap_set_bit (set_bbs, set_bb->index);
    4374              :       }
    4375              : 
    4376            2 :   bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
    4377            2 :   return bb;
    4378            2 : }
    4379              : 
    4380              : /* Mark FLAGS register as live in DATA, a bitmap of live caller-saved
    4381              :    registers, if DEST is FLAGS register.  */
    4382              : 
    4383              : static void
    4384          388 : ix86_check_flags_reg (rtx dest, const_rtx x, void *data)
    4385              : {
    4386          388 :   if (GET_CODE (x) == CLOBBER)
    4387              :     return;
    4388              : 
    4389          381 :   auto_bitmap *live_caller_saved_regs = (auto_bitmap *) data;
    4390          381 :   if (REG_P (dest) && REGNO (dest) == FLAGS_REG)
    4391            0 :     bitmap_set_bit (*live_caller_saved_regs, FLAGS_REG);
    4392              : }
    4393              : 
    4394              : /* Emit a TLS_SET instruction of KIND in basic block BB.   Store the
    4395              :    insertion point in *BEFORE_P for emit_insn_before or in *AFTER_P
    4396              :    for emit_insn_after.  UPDATED_GNU_TLS_INSNS contains instructions
    4397              :    which replace the GNU TLS instructions.  UPDATED_GNU2_TLS_INSNS
    4398              :    contains instructions which replace the GNU2 TLS instructions.  */
    4399              : 
    4400              : static rtx_insn *
    4401          323 : ix86_emit_tls_call (rtx tls_set, x86_cse_kind kind, basic_block bb,
    4402              :                     rtx_insn **before_p, rtx_insn **after_p,
    4403              :                     auto_bitmap &updated_gnu_tls_insns,
    4404              :                     auto_bitmap &updated_gnu2_tls_insns)
    4405              : {
    4406          325 :   rtx_insn *tls_insn;
    4407              : 
    4408          325 :   do
    4409              :     {
    4410          325 :       rtx_insn *insn = BB_HEAD (bb);
    4411         1296 :       while (insn && !NONDEBUG_INSN_P (insn))
    4412              :         {
    4413          975 :           if (insn == BB_END (bb))
    4414              :             {
    4415              :               /* This must be the beginning basic block:
    4416              : 
    4417              :                  (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
    4418              :                  (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
    4419              : 
    4420              :                  or a basic block with only a label:
    4421              : 
    4422              :                  (code_label 78 11 77 3 14 (nil) [1 uses])
    4423              :                  (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
    4424              : 
    4425              :                  or a basic block with only a debug marker:
    4426              : 
    4427              :                  (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
    4428              :                  (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
    4429              :                  (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
    4430              : 
    4431              :                  or a basic block with only deleted instructions:
    4432              : 
    4433              :                  (code_label 348 23 349 45 3 (nil) [0 uses])
    4434              :                  (note 349 348 436 45 [bb 45] NOTE_INSN_BASIC_BLOCK)
    4435              :                  (note 436 349 362 45 NOTE_INSN_DELETED)
    4436              : 
    4437              :                */
    4438            4 :               gcc_assert (DEBUG_INSN_P (insn)
    4439              :                           || (NOTE_P (insn)
    4440              :                               && ((NOTE_KIND (insn)
    4441              :                                    == NOTE_INSN_FUNCTION_BEG)
    4442              :                                   || (NOTE_KIND (insn)
    4443              :                                       == NOTE_INSN_DELETED)
    4444              :                                   || (NOTE_KIND (insn)
    4445              :                                       == NOTE_INSN_BASIC_BLOCK))));
    4446              :               insn = NULL;
    4447              :               break;
    4448              :             }
    4449          971 :           insn = NEXT_INSN (insn);
    4450              :         }
    4451              : 
    4452              :       /* TLS_GD and TLS_LD_BASE instructions are normal functions which
    4453              :          clobber caller-saved registers.  TLSDESC instructions only
    4454              :          clobber FLAGS.  If any registers clobbered by TLS instructions
    4455              :          are live in this basic block, we must insert TLS instructions
    4456              :          after all live registers clobbered are dead.  */
    4457              : 
    4458          325 :       auto_bitmap live_caller_saved_regs;
    4459          650 :       bitmap in = df_live ? DF_LIVE_IN (bb) : DF_LR_IN (bb);
    4460              : 
    4461          325 :       if (bitmap_bit_p (in, FLAGS_REG))
    4462            4 :         bitmap_set_bit (live_caller_saved_regs, FLAGS_REG);
    4463              : 
    4464          325 :       unsigned int i;
    4465              : 
    4466              :       /* Get all live caller-saved registers for TLS_GD and TLS_LD_BASE
    4467              :          instructions.  */
    4468          325 :       if (kind != X86_CSE_TLSDESC)
    4469              :         {
    4470          303 :           const predefined_function_abi &tls_abi
    4471          303 :             = ix86_tls_get_addr_abi ();
    4472        29088 :           for (i = 0; i < FIRST_PSEUDO_REGISTER; i++)
    4473        28482 :             if (tls_abi.clobbers_full_reg_p (i)
    4474        25758 :                 && !fixed_regs[i]
    4475        40953 :                 && bitmap_bit_p (in, i))
    4476          350 :               bitmap_set_bit (live_caller_saved_regs, i);
    4477          303 :           if (df_live && crtl->drap_reg)
    4478              :             {
    4479              :               /* DRAP has no reaching definition at this point, so df_live drops
    4480              :                  it above.  Its hard register can also go dead mid-function once
    4481              :                  copied elsewhere (e.g. right after the prologue), so query
    4482              :                  DF_LR_IN per-block rather than treating it as live whenever
    4483              :                  crtl->drap_reg is set.  */
    4484            1 :               i = REGNO (crtl->drap_reg);
    4485            2 :               if (bitmap_bit_p (DF_LR_IN (bb), i))
    4486            1 :                 bitmap_set_bit (live_caller_saved_regs, i);
    4487              :             }
    4488              :         }
    4489              : 
    4490          325 :       if (bitmap_empty_p (live_caller_saved_regs))
    4491              :         {
    4492           85 :           if (insn == BB_HEAD (bb))
    4493              :             {
    4494            0 :               *before_p = insn;
    4495            0 :               tls_insn = emit_insn_before (tls_set, insn);
    4496              :             }
    4497              :           else
    4498              :             {
    4499              :               /* Emit the TLS call after NOTE_INSN_FUNCTION_BEG in the
    4500              :                  beginning basic block:
    4501              : 
    4502              :                  (note 4 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
    4503              :                  (note 2 4 26 2 NOTE_INSN_FUNCTION_BEG)
    4504              : 
    4505              :                  or after NOTE_INSN_BASIC_BLOCK in a basic block with
    4506              :                  only a label:
    4507              : 
    4508              :                  (code_label 78 11 77 3 14 (nil) [1 uses])
    4509              :                  (note 77 78 54 3 [bb 3] NOTE_INSN_BASIC_BLOCK)
    4510              : 
    4511              :                  or after debug marker in a basic block with only a
    4512              :                  debug marker:
    4513              : 
    4514              :                  (note 3 0 2 2 [bb 2] NOTE_INSN_BASIC_BLOCK)
    4515              :                  (note 2 3 5 2 NOTE_INSN_FUNCTION_BEG)
    4516              :                  (debug_insn 5 2 16 2 (debug_marker) "x.c":6:3 -1 (nil))
    4517              : 
    4518              :                */
    4519           85 :               insn = insn ? PREV_INSN (insn) : BB_END (bb);
    4520           85 :               *after_p = insn;
    4521           85 :               tls_insn = emit_insn_after (tls_set, insn);
    4522              :             }
    4523              :           return tls_insn;
    4524              :         }
    4525              : 
    4526          240 :       bool repeat = false;
    4527              : 
    4528              :       /* Search for REG_DEAD notes in this basic block.  */
    4529          675 :       FOR_BB_INSNS (bb, insn)
    4530              :         {
    4531          675 :           if (!NONDEBUG_INSN_P (insn))
    4532          290 :             continue;
    4533              : 
    4534              :           /* NB: Conditional jump is the only instruction which reads
    4535              :              flags register and changes control flow.  We can never
    4536              :              place the TLS call after unconditional jump.  */
    4537          385 :           if (JUMP_P (insn))
    4538              :             {
    4539              :               /* This must be a conditional jump.  */
    4540            2 :               rtx label = JUMP_LABEL (insn);
    4541            2 :               if (label == nullptr
    4542            2 :                   || ANY_RETURN_P (label)
    4543            2 :                   || !(LABEL_P (label) || SYMBOL_REF_P (label)))
    4544            0 :                 gcc_unreachable ();
    4545              : 
    4546              :               /* Place the call before all FLAGS_REG setting BBs since
    4547              :                  we can't place a call before nor after a conditional
    4548              :                  jump.  */
    4549            2 :               bb = ix86_get_dominator_for_reg (FLAGS_REG, bb);
    4550              : 
    4551              :               /* Start over again.  */
    4552            2 :               repeat = true;
    4553            2 :               break;
    4554              :             }
    4555              : 
    4556          383 :           if (bitmap_bit_p (updated_gnu_tls_insns, INSN_UID (insn)))
    4557              :             {
    4558              :               /* Insert the __tls_get_addr call before INSN which
    4559              :                  replaces a __tls_get_addr call.  */
    4560            1 :               *before_p = insn;
    4561            1 :               tls_insn = emit_insn_before (tls_set, insn);
    4562            1 :               return tls_insn;
    4563              :             }
    4564              : 
    4565          382 :           if (bitmap_bit_p (updated_gnu2_tls_insns, INSN_UID (insn)))
    4566              :             {
    4567              :               /* Mark FLAGS register as dead since FLAGS register
    4568              :                  would be clobbered by the GNU2 TLS instruction.  */
    4569            1 :               bitmap_clear_bit (live_caller_saved_regs, FLAGS_REG);
    4570            1 :               continue;
    4571              :             }
    4572              : 
    4573              :           /* Check if FLAGS register is live.  */
    4574          381 :           note_stores (insn, ix86_check_flags_reg,
    4575              :                        &live_caller_saved_regs);
    4576              : 
    4577          381 :           rtx link;
    4578          522 :           for (link = REG_NOTES (insn); link; link = XEXP (link, 1))
    4579          378 :             if ((REG_NOTE_KIND (link) == REG_DEAD
    4580            9 :                  || (REG_NOTE_KIND (link) == REG_UNUSED
    4581            7 :                      && REGNO (XEXP (link, 0)) == FLAGS_REG))
    4582          385 :                 && REG_P (XEXP (link, 0)))
    4583              :               {
    4584              :                 /* Mark the live caller-saved register as dead.  */
    4585          757 :                 for (i = REGNO (XEXP (link, 0));
    4586          757 :                      i < END_REGNO (XEXP (link, 0));
    4587              :                      i++)
    4588          381 :                   if (i < FIRST_PSEUDO_REGISTER)
    4589          358 :                     bitmap_clear_bit (live_caller_saved_regs, i);
    4590              : 
    4591          376 :                 if (bitmap_empty_p (live_caller_saved_regs))
    4592              :                   {
    4593          237 :                     *after_p = insn;
    4594          237 :                     tls_insn = emit_insn_after (tls_set, insn);
    4595          237 :                     return tls_insn;
    4596              :                   }
    4597              :               }
    4598              :         }
    4599              : 
    4600              :       /* NB: Start over again for conditional jump.  */
    4601            2 :       if (repeat)
    4602            2 :         continue;
    4603              : 
    4604            0 :       gcc_assert (!bitmap_empty_p (live_caller_saved_regs));
    4605              : 
    4606              :       /* If any live caller-saved registers aren't dead at the end of
    4607              :          this basic block, get the basic block which dominates all
    4608              :          basic blocks which set the remaining live registers.  */
    4609            0 :       auto_bitmap set_bbs;
    4610            0 :       bitmap_iterator bi;
    4611            0 :       unsigned int id;
    4612            0 :       EXECUTE_IF_SET_IN_BITMAP (live_caller_saved_regs, 0, id, bi)
    4613              :         {
    4614            0 :           basic_block set_bb = ix86_get_dominator_for_reg (id, bb);
    4615            0 :           bitmap_set_bit (set_bbs, set_bb->index);
    4616              :         }
    4617            0 :       bb = nearest_common_dominator_for_set (CDI_DOMINATORS, set_bbs);
    4618            2 :     }
    4619              :   while (true);
    4620              : }
    4621              : 
    4622              : /* Generate a TLS call of KIND with VAL and copy the call result to DEST,
    4623              :    at entry of the nearest dominator for basic block map BBS, which is in
    4624              :    the fake loop that contains the whole function, so that there is only
    4625              :    a single TLS CALL of KIND with VAL in the whole function.
    4626              :    UPDATED_GNU_TLS_INSNS contains instructions which replace the GNU TLS
    4627              :    instructions.  UPDATED_GNU2_TLS_INSNS contains instructions which
    4628              :    replace the GNU2 TLS instructions.  If TLSDESC_SET isn't nullptr,
    4629              :    insert it before the TLS call.  */
    4630              : 
    4631              : static void
    4632          323 : ix86_place_single_tls_call (rtx dest, rtx val, x86_cse_kind kind,
    4633              :                             auto_bitmap &bbs,
    4634              :                             auto_bitmap &updated_gnu_tls_insns,
    4635              :                             auto_bitmap &updated_gnu2_tls_insns,
    4636              :                             rtx tlsdesc_set = nullptr)
    4637              : {
    4638          323 :   basic_block bb = nearest_common_dominator_for_set (CDI_DOMINATORS, bbs);
    4639          323 :   while (bb->loop_father->latch
    4640          333 :          != EXIT_BLOCK_PTR_FOR_FN (cfun))
    4641           10 :     bb = get_immediate_dominator (CDI_DOMINATORS,
    4642              :                                   bb->loop_father->header);
    4643              : 
    4644          323 :   rtx rax = nullptr, rdi;
    4645          323 :   rtx eqv = nullptr;
    4646          323 :   rtx caddr;
    4647          323 :   rtx set;
    4648          323 :   rtx clob;
    4649          323 :   rtx symbol;
    4650          323 :   rtx tls;
    4651              : 
    4652          323 :   switch (kind)
    4653              :     {
    4654          265 :     case X86_CSE_TLS_GD:
    4655          265 :       rax = gen_rtx_REG (Pmode, AX_REG);
    4656          265 :       rdi = gen_rtx_REG (Pmode, DI_REG);
    4657          265 :       caddr = ix86_tls_get_addr ();
    4658              : 
    4659          265 :       symbol = XVECEXP (val, 0, 0);
    4660          265 :       tls = gen_tls_global_dynamic_64 (Pmode, rax, symbol, caddr, rdi);
    4661          265 :       CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
    4662              : 
    4663          265 :       if (GET_MODE (symbol) != Pmode)
    4664            0 :         symbol = gen_rtx_ZERO_EXTEND (Pmode, symbol);
    4665              :       eqv = symbol;
    4666              :       break;
    4667              : 
    4668           37 :     case X86_CSE_TLS_LD_BASE:
    4669           37 :       rax = gen_rtx_REG (Pmode, AX_REG);
    4670           37 :       rdi = gen_rtx_REG (Pmode, DI_REG);
    4671           37 :       caddr = ix86_tls_get_addr ();
    4672              : 
    4673           37 :       tls = gen_tls_local_dynamic_base_64 (Pmode, rax, caddr, rdi);
    4674           37 :       CALL_INSN_ABI_ID (tls) = ix86_tls_get_addr_abi ().id ();
    4675              : 
    4676              :       /* Attach a unique REG_EQUAL to DEST, to allow the RTL optimizers
    4677              :          to share the LD_BASE result with other LD model accesses.  */
    4678           37 :       eqv = gen_rtx_UNSPEC (Pmode, gen_rtvec (1, const0_rtx),
    4679              :                             UNSPEC_TLS_LD_BASE);
    4680              : 
    4681           37 :       break;
    4682              : 
    4683           21 :     case X86_CSE_TLSDESC:
    4684           21 :       set = gen_rtx_SET (dest, val);
    4685           21 :       clob = gen_rtx_CLOBBER (VOIDmode,
    4686              :                               gen_rtx_REG (CCmode, FLAGS_REG));
    4687           21 :       tls = gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, set, clob));
    4688           21 :       break;
    4689              : 
    4690            0 :     default:
    4691            0 :       gcc_unreachable ();
    4692              :     }
    4693              : 
    4694              :   /* Emit the TLS CALL insn.  */
    4695          323 :   rtx_insn *before = nullptr;
    4696          323 :   rtx_insn *after = nullptr;
    4697          323 :   rtx_insn *tls_insn = ix86_emit_tls_call (tls, kind, bb, &before,
    4698              :                                            &after,
    4699              :                                            updated_gnu_tls_insns,
    4700              :                                            updated_gnu2_tls_insns);
    4701              : 
    4702          323 :   rtx_insn *tlsdesc_insn = nullptr;
    4703          323 :   if (tlsdesc_set)
    4704              :     {
    4705           16 :       rtx dest = copy_rtx (SET_DEST (tlsdesc_set));
    4706           16 :       rtx src = copy_rtx (SET_SRC (tlsdesc_set));
    4707           16 :       tlsdesc_set = gen_rtx_SET (dest, src);
    4708           16 :       tlsdesc_insn = emit_insn_before (tlsdesc_set, tls_insn);
    4709              :     }
    4710              : 
    4711          323 :   if (kind != X86_CSE_TLSDESC)
    4712              :     {
    4713          302 :       RTL_CONST_CALL_P (tls_insn) = 1;
    4714              : 
    4715              :       /* Indicate that this function can't jump to non-local gotos.  */
    4716          302 :       make_reg_eh_region_note_nothrow_nononlocal (tls_insn);
    4717              :     }
    4718              : 
    4719          323 :   if (recog_memoized (tls_insn) < 0)
    4720            0 :     gcc_unreachable ();
    4721              : 
    4722          323 :   if (dump_file)
    4723              :     {
    4724            0 :       if (after)
    4725              :         {
    4726            0 :           fprintf (dump_file, "\nPlace:\n\n");
    4727            0 :           if (tlsdesc_insn)
    4728            0 :             print_rtl_single (dump_file, tlsdesc_insn);
    4729            0 :           print_rtl_single (dump_file, tls_insn);
    4730            0 :           fprintf (dump_file, "\nafter:\n\n");
    4731            0 :           print_rtl_single (dump_file, after);
    4732            0 :           fprintf (dump_file, "\n");
    4733              :         }
    4734              :       else
    4735              :         {
    4736            0 :           fprintf (dump_file, "\nPlace:\n\n");
    4737            0 :           if (tlsdesc_insn)
    4738            0 :             print_rtl_single (dump_file, tlsdesc_insn);
    4739            0 :           print_rtl_single (dump_file, tls_insn);
    4740            0 :           fprintf (dump_file, "\nbefore:\n\n");
    4741            0 :           print_rtl_single (dump_file, before);
    4742            0 :           fprintf (dump_file, "\n");
    4743              :         }
    4744              :     }
    4745              : 
    4746          323 :   if (kind != X86_CSE_TLSDESC)
    4747              :     {
    4748              :       /* Copy RAX to DEST.  */
    4749          302 :       set = gen_rtx_SET (dest, rax);
    4750          302 :       rtx_insn *set_insn = emit_insn_after (set, tls_insn);
    4751          302 :       set_dst_reg_note (set_insn, REG_EQUAL, copy_rtx (eqv), dest);
    4752          302 :       if (dump_file)
    4753              :         {
    4754            0 :           fprintf (dump_file, "\nPlace:\n\n");
    4755            0 :           print_rtl_single (dump_file, set_insn);
    4756            0 :           fprintf (dump_file, "\nafter:\n\n");
    4757            0 :           print_rtl_single (dump_file, tls_insn);
    4758            0 :           fprintf (dump_file, "\n");
    4759              :         }
    4760              :     }
    4761          323 : }
    4762              : 
    4763              : namespace {
    4764              : 
    4765              : const pass_data pass_data_x86_cse =
    4766              : {
    4767              :   RTL_PASS, /* type */
    4768              :   "x86_cse", /* name */
    4769              :   OPTGROUP_NONE, /* optinfo_flags */
    4770              :   TV_MACH_DEP, /* tv_id */
    4771              :   0, /* properties_required */
    4772              :   0, /* properties_provided */
    4773              :   0, /* properties_destroyed */
    4774              :   0, /* todo_flags_start */
    4775              :   0, /* todo_flags_finish */
    4776              : };
    4777              : 
    4778              : class pass_x86_cse : public rtl_opt_pass
    4779              : {
    4780              : public:
    4781       297180 :   pass_x86_cse (gcc::context *ctxt)
    4782       594360 :     : rtl_opt_pass (pass_data_x86_cse, ctxt)
    4783              :   {}
    4784              : 
    4785              :   /* opt_pass methods: */
    4786      1517135 :   bool gate (function *fun) final override
    4787              :     {
    4788      1517135 :       return optimize && optimize_function_for_speed_p (fun);
    4789              :     }
    4790              : 
    4791       998209 :   unsigned int execute (function *) final override
    4792              :     {
    4793       998209 :       return x86_cse ();
    4794              :     }
    4795              : 
    4796              : private:
    4797              :   /* The redundant source value.  */
    4798              :   rtx val;
    4799              :   /* The actual redundant source value for UNSPEC_TLSDESC.  */
    4800              :   rtx tlsdesc_val;
    4801              :   /* The instruction which defines the redundant value.  */
    4802              :   rtx_insn *def_insn;
    4803              :   /* Mode of the destination of the candidate redundant instruction.  */
    4804              :   machine_mode mode;
    4805              :   /* Mode of the source of the candidate redundant instruction.  */
    4806              :   machine_mode scalar_mode;
    4807              :   /* The classification of the candidate redundant instruction.  */
    4808              :   x86_cse_kind kind;
    4809              : 
    4810              :   unsigned int x86_cse (void);
    4811              :   bool candidate_gnu_tls_p (rtx_insn *, attr_tls64);
    4812              :   bool candidate_gnu2_tls_p (rtx, attr_tls64);
    4813              :   bool candidate_vector_p (rtx, rtx_insn *);
    4814              :   rtx_insn *tls_set_insn_from_symbol (const_rtx, const_rtx);
    4815              : }; // class pass_x86_cse
    4816              : 
    4817              : /* Return the instruction which sets REG from TLS_SYMBOL.  */
    4818              : 
    4819              : rtx_insn *
    4820           42 : pass_x86_cse::tls_set_insn_from_symbol (const_rtx reg,
    4821              :                                         const_rtx tls_symbol)
    4822              : {
    4823           42 :   rtx_insn *set_insn = nullptr;
    4824           42 :   for (df_ref ref = DF_REG_DEF_CHAIN (REGNO (reg));
    4825          111 :        ref;
    4826           69 :        ref = DF_REF_NEXT_REG (ref))
    4827              :     {
    4828           69 :       if (DF_REF_IS_ARTIFICIAL (ref))
    4829              :         return nullptr;
    4830              : 
    4831           69 :       set_insn = DF_REF_INSN (ref);
    4832           69 :       if (get_attr_tls64 (set_insn) != TLS64_LEA)
    4833              :         return nullptr;
    4834              : 
    4835           69 :       rtx tls_set = PATTERN (set_insn);
    4836           69 :       rtx tls_src = XVECEXP (SET_SRC (tls_set), 0, 0);
    4837           69 :       if (!rtx_equal_p (tls_symbol, tls_src))
    4838              :         return nullptr;
    4839              :     }
    4840              : 
    4841              :   return set_insn;
    4842              : }
    4843              : 
    4844              : /* Return true and output def_insn, val, mode, scalar_mode and kind if
    4845              :    INSN is UNSPEC_TLS_GD or UNSPEC_TLS_LD_BASE.  */
    4846              : 
    4847              : bool
    4848         2211 : pass_x86_cse::candidate_gnu_tls_p (rtx_insn *insn, attr_tls64 tls64)
    4849              : {
    4850         2211 :   if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
    4851              :     return false;
    4852              : 
    4853              :   /* Record the redundant TLS CALLs for 64-bit:
    4854              : 
    4855              :      (parallel [
    4856              :         (set (reg:DI 0 ax)
    4857              :              (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
    4858              :                       (const_int 0 [0])))
    4859              :         (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
    4860              :                     (reg/f:DI 7 sp)] UNSPEC_TLS_GD)
    4861              :         (clobber (reg:DI 5 di))])
    4862              : 
    4863              : 
    4864              :      and
    4865              : 
    4866              :      (parallel [
    4867              :         (set (reg:DI 0 ax)
    4868              :              (call:DI (mem:QI (symbol_ref:DI ("__tls_get_addr")))
    4869              :                       (const_int 0 [0])))
    4870              :         (unspec:DI [(reg/f:DI 7 sp)] UNSPEC_TLS_LD_BASE)])
    4871              : 
    4872              :    */
    4873              : 
    4874         2037 :   rtx pat = PATTERN (insn);
    4875         2037 :   rtx set = XVECEXP (pat, 0, 0);
    4876         2037 :   gcc_assert (GET_CODE (set) == SET);
    4877         2037 :   rtx dest = SET_DEST (set);
    4878         2037 :   scalar_mode = mode = GET_MODE (dest);
    4879         2037 :   val = XVECEXP (pat, 0, 1);
    4880         2037 :   gcc_assert (GET_CODE (val) == UNSPEC);
    4881              : 
    4882         2037 :   if (tls64 == TLS64_GD)
    4883              :     kind = X86_CSE_TLS_GD;
    4884              :   else
    4885          115 :     kind = X86_CSE_TLS_LD_BASE;
    4886              : 
    4887         2037 :   def_insn = nullptr;
    4888         2037 :   return true;
    4889              : }
    4890              : 
    4891              : /* Return true and output def_insn, val, mode, scalar_mode and kind if
    4892              :    SET is UNSPEC_TLSDESC.  */
    4893              : 
    4894              : bool
    4895           56 : pass_x86_cse::candidate_gnu2_tls_p (rtx set, attr_tls64 tls64)
    4896              : {
    4897           56 :   if (!TARGET_64BIT || !cfun->machine->tls_descriptor_call_multiple_p)
    4898              :     return false;
    4899              : 
    4900           54 :   rtx tls_symbol;
    4901           54 :   rtx_insn *set_insn;
    4902           54 :   rtx src = SET_SRC (set);
    4903           54 :   val = src;
    4904           54 :   tlsdesc_val = src;
    4905           54 :   kind = X86_CSE_TLSDESC;
    4906              : 
    4907           54 :   if (tls64 == TLS64_COMBINE)
    4908              :     {
    4909              :       /* Record 64-bit TLS64_COMBINE:
    4910              : 
    4911              :          (set (reg/f:DI 104)
    4912              :               (plus:DI (unspec:DI [
    4913              :                           (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
    4914              :                           (reg:DI 114)
    4915              :                           (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
    4916              :                        (const:DI (unspec:DI [
    4917              :                                     (symbol_ref:DI ("e") [flags 0x1a])
    4918              :                                   ] UNSPEC_DTPOFF))))
    4919              : 
    4920              :          (set (reg/f:DI 104)
    4921              :               (plus:DI (unspec:DI [
    4922              :                           (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
    4923              :                           (unspec:DI [
    4924              :                              (symbol_ref:DI ("_TLS_MODULE_BASE_") [flags 0x10])
    4925              :                           ] UNSPEC_TLSDESC)
    4926              :                           (reg/f:DI 7 sp)] UNSPEC_TLSDESC)
    4927              :                        (const:DI (unspec:DI [
    4928              :                                     (symbol_ref:DI ("e") [flags 0x1a])
    4929              :                                  ] UNSPEC_DTPOFF))))
    4930              :      */
    4931              : 
    4932           12 :       scalar_mode = mode = GET_MODE (src);
    4933              : 
    4934              :       /* Since the first operand of PLUS in the source TLS_COMBINE
    4935              :          pattern is unused, use the second operand of PLUS:
    4936              : 
    4937              :          (const:DI (unspec:DI [
    4938              :                       (symbol_ref:DI ("e") [flags 0x1a])
    4939              :                    ] UNSPEC_DTPOFF))
    4940              : 
    4941              :          as VAL to check if 2 TLS_COMBINE patterns have the same
    4942              :          source.  */
    4943           12 :       val = XEXP (src, 1);
    4944           12 :       gcc_assert (GET_CODE (val) == CONST
    4945              :                   && GET_CODE (XEXP (val, 0)) == UNSPEC
    4946              :                       && XINT (XEXP (val, 0), 1) == UNSPEC_DTPOFF
    4947              :                       && SYMBOL_REF_P (XVECEXP (XEXP (val, 0), 0, 0)));
    4948           12 :       def_insn = nullptr;
    4949           12 :       return true;
    4950              :     }
    4951              : 
    4952              :   /* Record 64-bit TLS_CALL:
    4953              : 
    4954              :      (set (reg:DI 101)
    4955              :           (unspec:DI [(symbol_ref:DI ("foo") [flags 0x50])
    4956              :                       (reg:DI 112)
    4957              :                       (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
    4958              : 
    4959              :    */
    4960              : 
    4961           42 :   gcc_assert (GET_CODE (src) == UNSPEC);
    4962           42 :   tls_symbol = XVECEXP (src, 0, 0);
    4963           42 :   src = XVECEXP (src, 0, 1);
    4964           42 :   scalar_mode = mode = GET_MODE (src);
    4965           42 :   gcc_assert (REG_P (src));
    4966              : 
    4967              :   /* All definitions of reg:DI 129 in
    4968              : 
    4969              :      (set (reg:DI 110)
    4970              :           (unspec:DI [(symbol_ref:DI ("foo"))
    4971              :                       (reg:DI 129)
    4972              :                       (reg/f:DI 7 sp)] UNSPEC_TLSDESC))
    4973              : 
    4974              :      should have the same source as in
    4975              : 
    4976              :      (set (reg:DI 129)
    4977              :           (unspec:DI [(symbol_ref:DI ("foo"))] UNSPEC_TLSDESC))
    4978              : 
    4979              :    */
    4980              : 
    4981           42 :   set_insn = tls_set_insn_from_symbol (src, tls_symbol);
    4982           42 :   if (!set_insn)
    4983              :     return false;
    4984              : 
    4985              :   /* Use TLS_SYMBOL as VAL to check if 2 patterns have the same source.  */
    4986           42 :   val = tls_symbol;
    4987           42 :   def_insn = set_insn;
    4988           42 :   return true;
    4989              : }
    4990              : 
    4991              : /* Return true and output def_insn, val, mode, scalar_mode and kind if
    4992              :   INSN is a vector broadcast instruction.  */
    4993              : 
    4994              : bool
    4995     51538833 : pass_x86_cse::candidate_vector_p (rtx set, rtx_insn *insn)
    4996              : {
    4997     51538833 :   rtx src = SET_SRC (set);
    4998     51538833 :   rtx dest = SET_DEST (set);
    4999     51538833 :   mode = GET_MODE (dest);
    5000              :   /* Skip non-vector instruction.  */
    5001     51538833 :   if (!VECTOR_MODE_P (mode))
    5002              :     return false;
    5003              : 
    5004              :   /* Skip non-vector load instruction.  */
    5005      3704852 :   if (!REG_P (dest) && !SUBREG_P (dest))
    5006              :     return false;
    5007              : 
    5008      2220655 :   def_insn = insn;
    5009      2220655 :   val = ix86_broadcast_inner (src, mode, &scalar_mode, &kind,
    5010              :                               &def_insn);
    5011      2220655 :   return val ? true : false;
    5012              : }
    5013              : 
    5014              : /* At entry of the nearest common dominator for basic blocks with
    5015              : 
    5016              :    1. Vector CONST0_RTX patterns.
    5017              :    2. Vector CONSTM1_RTX patterns.
    5018              :    3. Vector broadcast patterns.
    5019              :    4. UNSPEC_TLS_GD patterns.
    5020              :    5. UNSPEC_TLS_LD_BASE patterns.
    5021              :    6. UNSPEC_TLSDESC patterns.
    5022              : 
    5023              :    generate a single pattern whose destination is used to replace the
    5024              :    source in all identical patterns.
    5025              : 
    5026              :    NB: We want to generate a pattern, which is executed only once, to
    5027              :    cover the whole function.  The LCM algorithm isn't appropriate here
    5028              :    since it may place a pattern inside the loop.  */
    5029              : 
    5030              : unsigned int
    5031       998209 : pass_x86_cse::x86_cse (void)
    5032              : {
    5033       998209 :   timevar_push (TV_MACH_DEP);
    5034              : 
    5035       998209 :   auto_vec<redundant_pattern *> loads;
    5036       998209 :   redundant_pattern *load;
    5037       998209 :   basic_block bb;
    5038       998209 :   rtx_insn *insn;
    5039       998209 :   unsigned int i;
    5040       998209 :   auto_bitmap updated_gnu_tls_insns;
    5041       998209 :   auto_bitmap updated_gnu2_tls_insns;
    5042       998209 :   auto_bitmap call_bbs;
    5043              : 
    5044       998209 :   df_set_flags (DF_DEFER_INSN_RESCAN);
    5045              : 
    5046       998209 :   bool recursive_call_p = cfun->machine->recursive_function;
    5047              : 
    5048     11219938 :   FOR_EACH_BB_FN (bb, cfun)
    5049              :     {
    5050    138089399 :       FOR_BB_INSNS (bb, insn)
    5051              :         {
    5052    127867670 :           if (!NONDEBUG_INSN_P (insn))
    5053     72564163 :             continue;
    5054              : 
    5055     55303507 :           bool matched = false;
    5056              :           /* Remove redundant patterns if there are more than 2 of
    5057              :              them.  */
    5058     55303507 :           unsigned int threshold = 2;
    5059              : 
    5060     55303507 :           bool call_p = CALL_P (insn);
    5061     55303507 :           rtx set = single_set (insn);
    5062     55303507 :           if (!set && !call_p)
    5063      1113504 :             continue;
    5064              : 
    5065     54190003 :           tlsdesc_val = nullptr;
    5066              : 
    5067     54190003 :           attr_tls64 tls64 = get_attr_tls64 (insn);
    5068              : 
    5069              :           /* NB: TLS calls preserve all registers.  */
    5070     54190003 :           if (call_p && tls64 == TLS64_NONE)
    5071      4584388 :             bitmap_set_bit (call_bbs, BLOCK_FOR_INSN (insn)->index);
    5072              : 
    5073     54190003 :           switch (tls64)
    5074              :             {
    5075         2211 :             case TLS64_GD:
    5076         2211 :             case TLS64_LD_BASE:
    5077              :               /* Verify UNSPEC_TLS_GD and UNSPEC_TLS_LD_BASE.  */
    5078         2211 :               if (candidate_gnu_tls_p (insn, tls64))
    5079              :                 break;
    5080          174 :               continue;
    5081              : 
    5082           56 :             case TLS64_CALL:
    5083           56 :             case TLS64_COMBINE:
    5084              :               /* Verify UNSPEC_TLSDESC.  */
    5085           56 :               if (candidate_gnu2_tls_p (set, tls64))
    5086              :                 break;
    5087            2 :               continue;
    5088              : 
    5089           36 :             case TLS64_LEA:
    5090              :               /* Skip TLS64_LEA.  */
    5091           36 :               continue;
    5092              : 
    5093     54187700 :             case TLS64_NONE:
    5094     54187700 :               if (!set)
    5095      2648867 :                 continue;
    5096              : 
    5097              :               /* Check for vector broadcast.  */
    5098     51538833 :               if (candidate_vector_p (set, insn))
    5099              :                 break;
    5100     51282877 :               continue;
    5101              :             }
    5102              : 
    5103              :           /* Check if there is a matching redundant load.   */
    5104       658868 :           FOR_EACH_VEC_ELT (loads, i, load)
    5105       442284 :             if (load->val
    5106       442284 :                 && load->kind == kind
    5107       292304 :                 && load->mode == scalar_mode
    5108       260087 :                 && (load->bb == bb
    5109       198813 :                     || (kind != X86_CSE_VEC_DUP
    5110       198813 :                         && kind != X86_CSE_CONST_VECTOR)
    5111              :                     /* Non all 0s/1s vector load must be in the same
    5112              :                        basic block if it is in a recursive call.  */
    5113       135050 :                     || !recursive_call_p)
    5114       700192 :                 && rtx_equal_p (load->val, val))
    5115              :               {
    5116              :                 /* Record instruction.  */
    5117       101477 :                 bitmap_set_bit (load->insns, INSN_UID (insn));
    5118              : 
    5119              :                 /* Record the maximum vector size.  */
    5120       101477 :                 if (kind <= X86_CSE_VEC_DUP
    5121       201837 :                     && load->size < GET_MODE_SIZE (mode))
    5122          836 :                   load->size = GET_MODE_SIZE (mode);
    5123              : 
    5124              :                 /* Record the basic block.  */
    5125       101477 :                 bitmap_set_bit (load->bbs, bb->index);
    5126              : 
    5127              :                 /* Increment the count.  */
    5128       101477 :                 load->count++;
    5129              : 
    5130       101477 :                 matched = true;
    5131       101477 :                 break;
    5132              :               }
    5133              : 
    5134       258047 :           if (matched)
    5135       101477 :             continue;
    5136              : 
    5137              :           /* We see this instruction the first time.  Record the
    5138              :              redundant source value, its mode, the destination size,
    5139              :              instruction which defines the redundant source value,
    5140              :              instruction basic block and the instruction kind.  */
    5141       156570 :           load = new redundant_pattern;
    5142              : 
    5143              :           /* Convert CONST_VECTOR load no larger than integer register
    5144              :              to constant integer load even if there is no redundant
    5145              :              CONST_VECTOR load.  */
    5146       156570 :           if (CONST_VECTOR_P (val))
    5147        32274 :             threshold = 1;
    5148              : 
    5149       156570 :           load->val = copy_rtx (val);
    5150       156570 :           if (tlsdesc_val)
    5151           28 :             load->tlsdesc_val = copy_rtx (tlsdesc_val);
    5152              :           else
    5153              :             load->tlsdesc_val = nullptr;
    5154       156570 :           load->mode = scalar_mode;
    5155       156570 :           load->dest_mode = mode;
    5156       156570 :           load->size = GET_MODE_SIZE (mode);
    5157       156570 :           load->def_insn = def_insn;
    5158       156570 :           load->count = 1;
    5159       156570 :           load->threshold = threshold;
    5160       156570 :           load->bb = BLOCK_FOR_INSN (insn);
    5161       156570 :           load->kind = kind;
    5162              : 
    5163       156570 :           bitmap_set_bit (load->insns, INSN_UID (insn));
    5164       156570 :           bitmap_set_bit (load->bbs, bb->index);
    5165              : 
    5166       156570 :           loads.safe_push (load);
    5167              :         }
    5168              :     }
    5169              : 
    5170              :   bool replaced = false;
    5171      1251335 :   FOR_EACH_VEC_ELT (loads, i, load)
    5172       156570 :     if (load->count >= load->threshold)
    5173              :       {
    5174        66440 :         machine_mode mode;
    5175        66440 :         rtx reg, broadcast_reg;
    5176        66440 :         rtx broadcast_source = nullptr;
    5177        66440 :         replaced = true;
    5178        66440 :         switch (load->kind)
    5179              :           {
    5180          323 :           case X86_CSE_TLS_GD:
    5181          323 :           case X86_CSE_TLS_LD_BASE:
    5182          323 :           case X86_CSE_TLSDESC:
    5183          323 :             broadcast_reg = gen_reg_rtx (load->mode);
    5184          323 :             replace_tls_call (broadcast_reg, load->insns,
    5185          323 :                               (load->kind == X86_CSE_TLSDESC
    5186              :                                ? updated_gnu2_tls_insns
    5187              :                                : updated_gnu_tls_insns));
    5188          323 :             load->broadcast_reg = broadcast_reg;
    5189          323 :             break;
    5190              : 
    5191        11391 :           case X86_CSE_VEC_DUP:
    5192        11391 :             if (CONST_INT_P (load->val)
    5193        10475 :                 && (load->val == CONST0_RTX (load->mode)
    5194        10500 :                     || load->size <= UNITS_PER_WORD))
    5195              :               {
    5196              :                 /* Generate CONST_VECTOR load.  */
    5197        32277 :               case X86_CSE_CONST_VECTOR:
    5198        32277 :                 mode = ix86_get_vector_cse_mode (load->size,
    5199              :                                                  load->mode);
    5200              : 
    5201        32277 :                 if (CONST_VECTOR_P (load->val))
    5202              :                   broadcast_source = load->val;
    5203            3 :                 else if (load->val == CONST0_RTX (load->mode))
    5204            0 :                   broadcast_source = CONST0_RTX (mode);
    5205            3 :                 else if (load->val == CONSTM1_RTX (load->mode))
    5206            0 :                   broadcast_source = CONSTM1_RTX (mode);
    5207              :                 else
    5208              :                   {
    5209            3 :                     int nunits = GET_MODE_NUNITS (mode);
    5210            3 :                     rtvec v = rtvec_alloc (nunits);
    5211           12 :                     for (int j = 0; j < nunits ; j++)
    5212            6 :                       RTVEC_ELT (v, j) = load->val;
    5213            3 :                     if (CONST_INT_P (load->val)
    5214            3 :                         && GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
    5215              :                       {
    5216            1 :                         gcc_assert (load->size <= UNITS_PER_WORD);
    5217            1 :                         machine_mode imode = GET_MODE_INNER (mode);
    5218            1 :                         class scalar_mode i_mode
    5219            1 :                           = int_mode_for_mode (imode).require ();
    5220            1 :                         mode = mode_for_vector (i_mode,
    5221            1 :                                                 nunits).require ();
    5222              :                       }
    5223              : 
    5224            3 :                     broadcast_source = gen_rtx_CONST_VECTOR (mode, v);
    5225              :                   }
    5226              : 
    5227              :                 /* NB: Zero CONST_VECTOR load works for MMX and XMM
    5228              :                    registers.  */
    5229        33687 :                 if (load->size <= UNITS_PER_WORD)
    5230              :                   {
    5231              :                     /* Convert CONST_VECTOR load no larger than integer
    5232              :                        register:
    5233              : 
    5234              :                        (set (reg:V2SI 106)
    5235              :                             (const_vector:V2SI [(const_int 1 [1]) repeated x2]))
    5236              : 
    5237              :                        to constant integer load:
    5238              : 
    5239              :                        (set (subreg:DI (reg:V2SI 106 [ _20 ]) 0)
    5240              :                             (const_int 4294967297 [0x100000001]))
    5241              :                        */
    5242        32277 :                     machine_mode int_mode
    5243        32277 :                       = int_mode_for_mode (mode).require ();
    5244        32277 :                     load->dest_mode = int_mode;
    5245        32277 :                     broadcast_source = simplify_subreg (int_mode,
    5246              :                                                         broadcast_source,
    5247              :                                                         mode, 0);
    5248        32277 :                     gcc_assert (broadcast_source != nullptr);
    5249              : 
    5250        32277 :                     bool keep_const_int_load = false;
    5251        32277 :                     if (!bitmap_empty_p (call_bbs))
    5252              :                       {
    5253        28853 :                         bitmap_iterator bi;
    5254        28853 :                         unsigned int id;
    5255        38274 :                         EXECUTE_IF_SET_IN_BITMAP (load->bbs, 0, id, bi)
    5256        30351 :                           if (bitmap_bit_p (call_bbs, id))
    5257              :                             {
    5258              :                               /* NB: Constant integer load is faster
    5259              :                                  than save and restore an integer
    5260              :                                  register when crossing a function call.
    5261              :                                */
    5262              :                               keep_const_int_load = true;
    5263              :                               break;
    5264              :                             }
    5265              :                       }
    5266              : 
    5267        28853 :                     if (keep_const_int_load)
    5268              :                       {
    5269              :                         /* Keep constant integer load.  */
    5270        20930 :                         replace_vector_const (mode, broadcast_source,
    5271        20930 :                                               load->insns, int_mode);
    5272        20930 :                         load->broadcast_source = nullptr;
    5273        20930 :                         load->broadcast_reg = nullptr;
    5274              :                       }
    5275              :                     else
    5276              :                       {
    5277        11347 :                         broadcast_reg = gen_reg_rtx (mode);
    5278        11347 :                         reg = gen_reg_rtx (load->mode);
    5279        11347 :                         replace_vector_const (mode, broadcast_reg,
    5280        11347 :                                               load->insns, load->mode);
    5281        11347 :                         load->broadcast_source = broadcast_source;
    5282        11347 :                         load->broadcast_reg = broadcast_reg;
    5283              :                       }
    5284              :                     break;
    5285              :                   }
    5286              :               }
    5287              :             /* FALLTHRU */
    5288              : 
    5289        33840 :           case X86_CSE_CONST0_VECTOR:
    5290        33840 :           case X86_CSE_CONSTM1_VECTOR:
    5291        33840 :             mode = ix86_get_vector_cse_mode (load->size, load->mode);
    5292        33840 :             broadcast_reg = gen_reg_rtx (mode);
    5293        33840 :             if (load->def_insn)
    5294              :               {
    5295              :                 /* Replace redundant vector loads with a single vector
    5296              :                    load in the same basic block.  */
    5297          778 :                 reg = load->val;
    5298          778 :                 if (load->mode != GET_MODE (reg))
    5299            0 :                   reg = gen_rtx_SUBREG (load->mode, reg, 0);
    5300          778 :                 broadcast_source = gen_rtx_VEC_DUPLICATE (mode, reg);
    5301              :               }
    5302              :             else
    5303              :               /* This is a constant integer/double vector.  If the
    5304              :                  inner scalar is 0 or -1, set vector to CONST0_RTX
    5305              :                  or CONSTM1_RTX directly.  */
    5306        33062 :               switch (load->kind)
    5307              :                 {
    5308        20752 :                 case X86_CSE_CONST0_VECTOR:
    5309        20752 :                   broadcast_source = CONST0_RTX (mode);
    5310        20752 :                   break;
    5311         1697 :                 case X86_CSE_CONSTM1_VECTOR:
    5312         1697 :                   broadcast_source = CONSTM1_RTX (mode);
    5313         1697 :                   break;
    5314        10613 :                 case X86_CSE_CONST_VECTOR:
    5315        10613 :                 case X86_CSE_VEC_DUP:
    5316        10613 :                   if (!broadcast_source)
    5317              :                     {
    5318        10613 :                       reg = gen_reg_rtx (load->mode);
    5319        10613 :                       broadcast_source = gen_rtx_VEC_DUPLICATE (mode,
    5320              :                                                                 reg);
    5321              :                     }
    5322              :                   break;
    5323            0 :                 default:
    5324            0 :                   gcc_unreachable ();
    5325              :                 }
    5326        33840 :             replace_vector_const (mode, broadcast_reg, load->insns,
    5327              :                                   load->mode);
    5328        33840 :             load->broadcast_source = broadcast_source;
    5329        33840 :             load->broadcast_reg = broadcast_reg;
    5330        33840 :             break;
    5331              :           }
    5332              :       }
    5333              : 
    5334       998209 :   if (replaced)
    5335              :     {
    5336        43210 :       auto_vec<rtx_insn *> control_flow_insns;
    5337              : 
    5338              :       /* (Re-)discover loops so that bb->loop_father can be used in the
    5339              :          analysis below.  */
    5340        43210 :       calculate_dominance_info (CDI_DOMINATORS);
    5341        43210 :       loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
    5342              : 
    5343       216492 :       FOR_EACH_VEC_ELT (loads, i, load)
    5344        86862 :         if (load->count >= load->threshold)
    5345              :           {
    5346        66440 :             rtx set;
    5347        66440 :             if (load->def_insn)
    5348          794 :               switch (load->kind)
    5349              :                 {
    5350           16 :                 case X86_CSE_TLSDESC:
    5351           16 :                   ix86_place_single_tls_call (load->broadcast_reg,
    5352              :                                               load->tlsdesc_val,
    5353              :                                               load->kind,
    5354           16 :                                               load->bbs,
    5355              :                                               updated_gnu_tls_insns,
    5356              :                                               updated_gnu2_tls_insns,
    5357           16 :                                               PATTERN (load->def_insn));
    5358           16 :                   break;
    5359          778 :                 case X86_CSE_VEC_DUP:
    5360              :                   /* Insert a broadcast after the original scalar
    5361              :                      definition.  */
    5362          778 :                   set = gen_rtx_SET (load->broadcast_reg,
    5363              :                                      load->broadcast_source);
    5364          778 :                   insn = emit_insn_after (set, load->def_insn);
    5365              : 
    5366          778 :                   if (cfun->can_throw_non_call_exceptions)
    5367              :                     {
    5368              :                       /* Handle REG_EH_REGION note in DEF_INSN.  */
    5369            4 :                       rtx note = find_reg_note (load->def_insn,
    5370              :                                                 REG_EH_REGION, nullptr);
    5371            4 :                       if (note)
    5372              :                         {
    5373            1 :                           control_flow_insns.safe_push (load->def_insn);
    5374            1 :                           add_reg_note (insn, REG_EH_REGION,
    5375              :                                         XEXP (note, 0));
    5376              :                         }
    5377              :                     }
    5378              : 
    5379          778 :                   if (dump_file)
    5380              :                     {
    5381            0 :                       fprintf (dump_file, "\nAdd:\n\n");
    5382            0 :                       print_rtl_single (dump_file, insn);
    5383            0 :                       fprintf (dump_file, "\nafter:\n\n");
    5384            0 :                       print_rtl_single (dump_file, load->def_insn);
    5385            0 :                       fprintf (dump_file, "\n");
    5386              :                     }
    5387              :                   break;
    5388            0 :                 default:
    5389            0 :                   gcc_unreachable ();
    5390              :                 }
    5391              :             else
    5392        65646 :               switch (load->kind)
    5393              :                 {
    5394          307 :                 case X86_CSE_TLS_GD:
    5395          307 :                 case X86_CSE_TLS_LD_BASE:
    5396          307 :                 case X86_CSE_TLSDESC:
    5397          307 :                   ix86_place_single_tls_call (load->broadcast_reg,
    5398              :                                               (load->kind == X86_CSE_TLSDESC
    5399              :                                                ? load->tlsdesc_val
    5400              :                                                : load->val),
    5401              :                                               load->kind,
    5402          307 :                                               load->bbs,
    5403              :                                               updated_gnu_tls_insns,
    5404              :                                               updated_gnu2_tls_insns);
    5405          307 :                   break;
    5406        42890 :                 case X86_CSE_CONST_VECTOR:
    5407        42890 :                 case X86_CSE_VEC_DUP:
    5408              :                   /* Keep redundant constant integer load.  */
    5409        42890 :                   if (!load->broadcast_reg)
    5410              :                     break;
    5411              :                   /* FALLTHRU */
    5412        44409 :                 case X86_CSE_CONST0_VECTOR:
    5413        44409 :                 case X86_CSE_CONSTM1_VECTOR:
    5414        44409 :                   ix86_place_single_vector_set (load->broadcast_reg,
    5415              :                                                 load->broadcast_source,
    5416              :                                                 load->bbs,
    5417              :                                                 load);
    5418        44409 :                   break;
    5419              :                 }
    5420              :           }
    5421              : 
    5422        43210 :       loop_optimizer_finalize ();
    5423              : 
    5424        43211 :       if (!control_flow_insns.is_empty ())
    5425              :         {
    5426            1 :           free_dominance_info (CDI_DOMINATORS);
    5427              : 
    5428            3 :           FOR_EACH_VEC_ELT (control_flow_insns, i, insn)
    5429            1 :             if (control_flow_insn_p (insn))
    5430              :               {
    5431              :                 /* Split the block after insn.  There will be a fallthru
    5432              :                    edge, which is OK so we keep it.  We have to create
    5433              :                    the exception edges ourselves.  */
    5434            1 :                 bb = BLOCK_FOR_INSN (insn);
    5435            1 :                 split_block (bb, insn);
    5436            1 :                 rtl_make_eh_edge (NULL, bb, BB_END (bb));
    5437              :               }
    5438              :         }
    5439              : 
    5440        43210 :       df_process_deferred_rescans ();
    5441        43210 :     }
    5442              : 
    5443      1251335 :   FOR_EACH_VEC_ELT (loads, i, load)
    5444       313140 :     delete load;
    5445              : 
    5446       998209 :   df_clear_flags (DF_DEFER_INSN_RESCAN);
    5447              : 
    5448       998209 :   timevar_pop (TV_MACH_DEP);
    5449       998209 :   return 0;
    5450       998209 : }
    5451              : 
    5452              : } // anon namespace
    5453              : 
    5454              : rtl_opt_pass *
    5455       297180 : make_pass_x86_cse (gcc::context *ctxt)
    5456              : {
    5457       297180 :   return new pass_x86_cse (ctxt);
    5458              : }
    5459              : 
    5460              : /* Convert legacy instructions that clobbers EFLAGS to APX_NF
    5461              :    instructions when there are no flag set between a flag
    5462              :    producer and user.  */
    5463              : 
    5464              : static unsigned int
    5465          384 : ix86_apx_nf_convert (void)
    5466              : {
    5467          384 :   timevar_push (TV_MACH_DEP);
    5468              : 
    5469          384 :   basic_block bb;
    5470          384 :   rtx_insn *insn;
    5471          384 :   hash_map <rtx_insn *, rtx> converting_map;
    5472          384 :   auto_vec <rtx_insn *> current_convert_list;
    5473              : 
    5474          384 :   bool converting_seq = false;
    5475          384 :   rtx cc = gen_rtx_REG (CCmode, FLAGS_REG);
    5476              : 
    5477          842 :   FOR_EACH_BB_FN (bb, cfun)
    5478              :     {
    5479              :       /* Reset conversion for each bb.  */
    5480          458 :       converting_seq = false;
    5481         5352 :       FOR_BB_INSNS (bb, insn)
    5482              :         {
    5483         4894 :           if (!NONDEBUG_INSN_P (insn))
    5484         4277 :             continue;
    5485              : 
    5486         3891 :           if (recog_memoized (insn) < 0)
    5487          346 :             continue;
    5488              : 
    5489              :           /* Convert candidate insns after cstore, which should
    5490              :              satisfy the two conditions:
    5491              :              1. Is not flag user or producer, only clobbers
    5492              :              FLAGS_REG.
    5493              :              2. Have corresponding nf pattern.  */
    5494              : 
    5495         3545 :           rtx pat = PATTERN (insn);
    5496              : 
    5497              :           /* Starting conversion at first cstorecc.  */
    5498         3545 :           rtx set = NULL_RTX;
    5499         3545 :           if (!converting_seq
    5500         2954 :               && (set = single_set (insn))
    5501         2876 :               && ix86_comparison_operator (SET_SRC (set), VOIDmode)
    5502          131 :               && reg_overlap_mentioned_p (cc, SET_SRC (set))
    5503         3673 :               && !reg_overlap_mentioned_p (cc, SET_DEST (set)))
    5504              :             {
    5505          128 :               converting_seq = true;
    5506          128 :               current_convert_list.truncate (0);
    5507              :             }
    5508              :           /* Terminate at the next explicit flag set.  */
    5509         3417 :           else if (reg_set_p (cc, pat)
    5510         3417 :                    && GET_CODE (set_of (cc, pat)) != CLOBBER)
    5511              :             converting_seq = false;
    5512              : 
    5513         3306 :           if (!converting_seq)
    5514         2928 :             continue;
    5515              : 
    5516          617 :           if (get_attr_has_nf (insn)
    5517          617 :               && GET_CODE (pat) == PARALLEL)
    5518              :             {
    5519              :               /* Record the insn to candidate map.  */
    5520           74 :               current_convert_list.safe_push (insn);
    5521           74 :               converting_map.put (insn, pat);
    5522              :             }
    5523              :           /* If the insn clobbers flags but has no nf_attr,
    5524              :              revoke all previous candidates.  */
    5525          543 :           else if (!get_attr_has_nf (insn)
    5526          542 :                    && reg_set_p (cc, pat)
    5527          546 :                    && GET_CODE (set_of (cc, pat)) == CLOBBER)
    5528              :             {
    5529            3 :               for (auto item : current_convert_list)
    5530            0 :                 converting_map.remove (item);
    5531            3 :               converting_seq = false;
    5532              :             }
    5533              :         }
    5534              :     }
    5535              : 
    5536          384 :   if (!converting_map.is_empty ())
    5537              :     {
    5538           88 :       for (auto iter = converting_map.begin ();
    5539          102 :            iter != converting_map.end (); ++iter)
    5540              :         {
    5541           74 :           rtx_insn *replace = (*iter).first;
    5542           74 :           rtx pat = (*iter).second;
    5543           74 :           int i, n = 0, len = XVECLEN (pat, 0);
    5544           74 :           rtx *new_elems = XALLOCAVEC (rtx, len);
    5545           74 :           rtx new_pat;
    5546          222 :           for (i = 0; i < len; i++)
    5547              :             {
    5548          148 :               rtx temp = XVECEXP (pat, 0, i);
    5549          222 :               if (! (GET_CODE (temp) == CLOBBER
    5550           74 :                      && reg_overlap_mentioned_p (cc,
    5551           74 :                                                  XEXP (temp, 0))))
    5552              :                 {
    5553           74 :                   new_elems[n] = temp;
    5554           74 :                   n++;
    5555              :                 }
    5556              :             }
    5557              : 
    5558           74 :           if (n == 1)
    5559           74 :             new_pat = new_elems[0];
    5560              :           else
    5561            0 :             new_pat =
    5562            0 :               gen_rtx_PARALLEL (VOIDmode,
    5563              :                                 gen_rtvec_v (n,
    5564              :                                              new_elems));
    5565              : 
    5566           74 :           PATTERN (replace) = new_pat;
    5567           74 :           INSN_CODE (replace) = -1;
    5568           74 :           recog_memoized (replace);
    5569           74 :           df_insn_rescan (replace);
    5570              :         }
    5571              :     }
    5572              : 
    5573          384 :   timevar_pop (TV_MACH_DEP);
    5574          384 :   return 0;
    5575          384 : }
    5576              : 
    5577              : 
    5578              : namespace {
    5579              : 
    5580              : const pass_data pass_data_apx_nf_convert =
    5581              : {
    5582              :   RTL_PASS, /* type */
    5583              :   "apx_nfcvt", /* name */
    5584              :   OPTGROUP_NONE, /* optinfo_flags */
    5585              :   TV_MACH_DEP, /* tv_id */
    5586              :   0, /* properties_required */
    5587              :   0, /* properties_provided */
    5588              :   0, /* properties_destroyed */
    5589              :   0, /* todo_flags_start */
    5590              :   0, /* todo_flags_finish */
    5591              : };
    5592              : 
    5593              : class pass_apx_nf_convert : public rtl_opt_pass
    5594              : {
    5595              : public:
    5596       297180 :   pass_apx_nf_convert (gcc::context *ctxt)
    5597       594360 :     : rtl_opt_pass (pass_data_apx_nf_convert, ctxt)
    5598              :   {}
    5599              : 
    5600              :   /* opt_pass methods: */
    5601      1517135 :   bool gate (function *) final override
    5602              :     {
    5603      1517135 :       return (TARGET_APX_NF
    5604          480 :               && optimize
    5605      1517603 :               && optimize_function_for_speed_p (cfun));
    5606              :     }
    5607              : 
    5608          384 :   unsigned int execute (function *) final override
    5609              :     {
    5610          384 :       return ix86_apx_nf_convert ();
    5611              :     }
    5612              : }; // class pass_apx_nf_convert
    5613              : 
    5614              : } // anon namespace
    5615              : 
    5616              : rtl_opt_pass *
    5617       297180 : make_pass_apx_nf_convert (gcc::context *ctxt)
    5618              : {
    5619       297180 :   return new pass_apx_nf_convert (ctxt);
    5620              : }
    5621              : 
    5622              : /* When a hot loop can be fit into one cacheline,
    5623              :    force align the loop without considering the max skip.  */
    5624              : static void
    5625       997723 : ix86_align_loops ()
    5626              : {
    5627       997723 :   basic_block bb;
    5628              : 
    5629              :   /* Don't do this when we don't know cache line size.  */
    5630       997723 :   if (ix86_cost->prefetch_block == 0)
    5631            9 :     return;
    5632              : 
    5633       997714 :   loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
    5634       997714 :   profile_count count_threshold = cfun->cfg->count_max / param_align_threshold;
    5635     11666475 :   FOR_EACH_BB_FN (bb, cfun)
    5636              :     {
    5637     10668761 :       rtx_insn *label = BB_HEAD (bb);
    5638     10668761 :       bool has_fallthru = 0;
    5639     10668761 :       edge e;
    5640     10668761 :       edge_iterator ei;
    5641              : 
    5642     10668761 :       if (!LABEL_P (label))
    5643      5432103 :         continue;
    5644              : 
    5645      5241561 :       profile_count fallthru_count = profile_count::zero ();
    5646      5241561 :       profile_count branch_count = profile_count::zero ();
    5647              : 
    5648     15263565 :       FOR_EACH_EDGE (e, ei, bb->preds)
    5649              :         {
    5650     10022004 :           if (e->flags & EDGE_FALLTHRU)
    5651      2530366 :             has_fallthru = 1, fallthru_count += e->count ();
    5652              :           else
    5653      7491638 :             branch_count += e->count ();
    5654              :         }
    5655              : 
    5656      5241561 :       if (!fallthru_count.initialized_p () || !branch_count.initialized_p ())
    5657         4903 :         continue;
    5658              : 
    5659      5236658 :       if (bb->loop_father
    5660      5236658 :           && bb->loop_father->latch != EXIT_BLOCK_PTR_FOR_FN (cfun)
    5661      6612635 :           && (has_fallthru
    5662      1375977 :               ? (!(single_succ_p (bb)
    5663       153655 :                    && single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun))
    5664       947588 :                  && optimize_bb_for_speed_p (bb)
    5665       865054 :                  && branch_count + fallthru_count > count_threshold
    5666       738680 :                  && (branch_count > fallthru_count * param_align_loop_iterations))
    5667              :               /* In case there'no fallthru for the loop.
    5668              :                  Nops inserted won't be executed.  */
    5669       428389 :               : (branch_count > count_threshold
    5670       142735 :                  || (bb->count > bb->prev_bb->count * 10
    5671        12697 :                      && (bb->prev_bb->count
    5672      5249355 :                          <= ENTRY_BLOCK_PTR_FOR_FN (cfun)->count / 2)))))
    5673              :         {
    5674       563376 :           rtx_insn* insn, *end_insn;
    5675       563376 :           HOST_WIDE_INT size = 0;
    5676       563376 :           bool padding_p = true;
    5677       563376 :           basic_block tbb = bb;
    5678       563376 :           unsigned cond_branch_num = 0;
    5679       563376 :           bool detect_tight_loop_p = false;
    5680              : 
    5681       878136 :           for (unsigned int i = 0; i != bb->loop_father->num_nodes;
    5682       314760 :                i++, tbb = tbb->next_bb)
    5683              :             {
    5684              :               /* Only handle continuous cfg layout. */
    5685       878136 :               if (bb->loop_father != tbb->loop_father)
    5686              :                 {
    5687              :                   padding_p = false;
    5688              :                   break;
    5689              :                 }
    5690              : 
    5691     10776915 :               FOR_BB_INSNS (tbb, insn)
    5692              :                 {
    5693     10101367 :                   if (!NONDEBUG_INSN_P (insn))
    5694      5882757 :                     continue;
    5695      4218610 :                   size += ix86_min_insn_size (insn);
    5696              : 
    5697              :                   /* We don't know size of inline asm.
    5698              :                      Don't align loop for call.  */
    5699      4218610 :                   if (asm_noperands (PATTERN (insn)) >= 0
    5700      4218610 :                       || CALL_P (insn))
    5701              :                     {
    5702              :                       size = -1;
    5703              :                       break;
    5704              :                     }
    5705              :                 }
    5706              : 
    5707       835812 :               if (size == -1 || size > ix86_cost->prefetch_block)
    5708              :                 {
    5709              :                   padding_p = false;
    5710              :                   break;
    5711              :                 }
    5712              : 
    5713      1482509 :               FOR_EACH_EDGE (e, ei, tbb->succs)
    5714              :                 {
    5715              :                   /* It could be part of the loop.  */
    5716      1024006 :                   if (e->dest == bb)
    5717              :                     {
    5718              :                       detect_tight_loop_p = true;
    5719              :                       break;
    5720              :                     }
    5721              :                 }
    5722              : 
    5723       649359 :               if (detect_tight_loop_p)
    5724              :                 break;
    5725              : 
    5726       458503 :               end_insn = BB_END (tbb);
    5727       458503 :               if (JUMP_P (end_insn))
    5728              :                 {
    5729              :                   /* For decoded icache:
    5730              :                      1. Up to two branches are allowed per Way.
    5731              :                      2. A non-conditional branch is the last micro-op in a Way.
    5732              :                   */
    5733       372326 :                   if (onlyjump_p (end_insn)
    5734       372326 :                       && (any_uncondjump_p (end_insn)
    5735       309144 :                           || single_succ_p (tbb)))
    5736              :                     {
    5737              :                       padding_p = false;
    5738              :                       break;
    5739              :                     }
    5740       309144 :                   else if (++cond_branch_num >= 2)
    5741              :                     {
    5742              :                       padding_p = false;
    5743              :                       break;
    5744              :                     }
    5745              :                 }
    5746              : 
    5747              :             }
    5748              : 
    5749       563376 :           if (padding_p && detect_tight_loop_p)
    5750              :             {
    5751       381712 :               emit_insn_before (gen_max_skip_align (GEN_INT (ceil_log2 (size)),
    5752              :                                                     GEN_INT (0)), label);
    5753              :               /* End of function.  */
    5754       190856 :               if (!tbb || tbb == EXIT_BLOCK_PTR_FOR_FN (cfun))
    5755              :                 break;
    5756              :               /* Skip bb which already fits into one cacheline.  */
    5757              :               bb = tbb;
    5758              :             }
    5759              :         }
    5760              :     }
    5761              : 
    5762       997714 :   loop_optimizer_finalize ();
    5763       997714 :   free_dominance_info (CDI_DOMINATORS);
    5764              : }
    5765              : 
    5766              : namespace {
    5767              : 
    5768              : const pass_data pass_data_align_tight_loops =
    5769              : {
    5770              :   RTL_PASS, /* type */
    5771              :   "align_tight_loops", /* name */
    5772              :   OPTGROUP_NONE, /* optinfo_flags */
    5773              :   TV_MACH_DEP, /* tv_id */
    5774              :   0, /* properties_required */
    5775              :   0, /* properties_provided */
    5776              :   0, /* properties_destroyed */
    5777              :   0, /* todo_flags_start */
    5778              :   0, /* todo_flags_finish */
    5779              : };
    5780              : 
    5781              : class pass_align_tight_loops : public rtl_opt_pass
    5782              : {
    5783              : public:
    5784       297180 :   pass_align_tight_loops (gcc::context *ctxt)
    5785       594360 :     : rtl_opt_pass (pass_data_align_tight_loops, ctxt)
    5786              :   {}
    5787              : 
    5788              :   /* opt_pass methods: */
    5789      1517135 :   bool gate (function *) final override
    5790              :     {
    5791      1517135 :       return TARGET_ALIGN_TIGHT_LOOPS
    5792      1516642 :              && optimize
    5793      2581609 :              && optimize_function_for_speed_p (cfun);
    5794              :     }
    5795              : 
    5796       997723 :   unsigned int execute (function *) final override
    5797              :     {
    5798       997723 :       timevar_push (TV_MACH_DEP);
    5799              : #ifdef ASM_OUTPUT_MAX_SKIP_ALIGN
    5800       997723 :       ix86_align_loops ();
    5801              : #endif
    5802       997723 :       timevar_pop (TV_MACH_DEP);
    5803       997723 :       return 0;
    5804              :     }
    5805              : }; // class pass_align_tight_loops
    5806              : 
    5807              : } // anon namespace
    5808              : 
    5809              : rtl_opt_pass *
    5810       297180 : make_pass_align_tight_loops (gcc::context *ctxt)
    5811              : {
    5812       297180 :   return new pass_align_tight_loops (ctxt);
    5813              : }
    5814              : 
    5815              : /* This compares the priority of target features in function DECL1
    5816              :    and DECL2.  It returns positive value if DECL1 is higher priority,
    5817              :    negative value if DECL2 is higher priority and 0 if they are the
    5818              :    same.  */
    5819              : 
    5820              : int
    5821         5830 : ix86_compare_version_priority (tree decl1, tree decl2)
    5822              : {
    5823         5830 :   unsigned int priority1 = get_builtin_code_for_version (decl1, NULL);
    5824         5830 :   unsigned int priority2 = get_builtin_code_for_version (decl2, NULL);
    5825              : 
    5826         5830 :   return (int)priority1 - (int)priority2;
    5827              : }
    5828              : 
    5829              : /* V1 and V2 point to function versions with different priorities
    5830              :    based on the target ISA.  This function compares their priorities.  */
    5831              : 
    5832              : static int
    5833         6882 : feature_compare (const void *v1, const void *v2)
    5834              : {
    5835         6882 :   typedef struct _function_version_info
    5836              :     {
    5837              :       tree version_decl;
    5838              :       tree predicate_chain;
    5839              :       unsigned int dispatch_priority;
    5840              :     } function_version_info;
    5841              : 
    5842         6882 :   const function_version_info c1 = *(const function_version_info *)v1;
    5843         6882 :   const function_version_info c2 = *(const function_version_info *)v2;
    5844         6882 :   return (c2.dispatch_priority - c1.dispatch_priority);
    5845              : }
    5846              : 
    5847              : /* This adds a condition to the basic_block NEW_BB in function FUNCTION_DECL
    5848              :    to return a pointer to VERSION_DECL if the outcome of the expression
    5849              :    formed by PREDICATE_CHAIN is true.  This function will be called during
    5850              :    version dispatch to decide which function version to execute.  It returns
    5851              :    the basic block at the end, to which more conditions can be added.  */
    5852              : 
    5853              : static basic_block
    5854          848 : add_condition_to_bb (tree function_decl, tree version_decl,
    5855              :                      tree predicate_chain, basic_block new_bb)
    5856              : {
    5857          848 :   gimple *return_stmt;
    5858          848 :   tree convert_expr, result_var;
    5859          848 :   gimple *convert_stmt;
    5860          848 :   gimple *call_cond_stmt;
    5861          848 :   gimple *if_else_stmt;
    5862              : 
    5863          848 :   basic_block bb1, bb2, bb3;
    5864          848 :   edge e12, e23;
    5865              : 
    5866          848 :   tree cond_var, and_expr_var = NULL_TREE;
    5867          848 :   gimple_seq gseq;
    5868              : 
    5869          848 :   tree predicate_decl, predicate_arg;
    5870              : 
    5871          848 :   push_cfun (DECL_STRUCT_FUNCTION (function_decl));
    5872              : 
    5873          848 :   gcc_assert (new_bb != NULL);
    5874          848 :   gseq = bb_seq (new_bb);
    5875              : 
    5876              : 
    5877          848 :   convert_expr = build1 (CONVERT_EXPR, ptr_type_node,
    5878              :                          build_fold_addr_expr (version_decl));
    5879          848 :   result_var = create_tmp_var (ptr_type_node);
    5880          848 :   convert_stmt = gimple_build_assign (result_var, convert_expr);
    5881          848 :   return_stmt = gimple_build_return (result_var);
    5882              : 
    5883          848 :   if (predicate_chain == NULL_TREE)
    5884              :     {
    5885          204 :       gimple_seq_add_stmt (&gseq, convert_stmt);
    5886          204 :       gimple_seq_add_stmt (&gseq, return_stmt);
    5887          204 :       set_bb_seq (new_bb, gseq);
    5888          204 :       gimple_set_bb (convert_stmt, new_bb);
    5889          204 :       gimple_set_bb (return_stmt, new_bb);
    5890          204 :       pop_cfun ();
    5891          204 :       return new_bb;
    5892              :     }
    5893              : 
    5894         1327 :   while (predicate_chain != NULL)
    5895              :     {
    5896          683 :       cond_var = create_tmp_var (integer_type_node);
    5897          683 :       predicate_decl = TREE_PURPOSE (predicate_chain);
    5898          683 :       predicate_arg = TREE_VALUE (predicate_chain);
    5899          683 :       call_cond_stmt = gimple_build_call (predicate_decl, 1, predicate_arg);
    5900          683 :       gimple_call_set_lhs (call_cond_stmt, cond_var);
    5901              : 
    5902          683 :       gimple_set_block (call_cond_stmt, DECL_INITIAL (function_decl));
    5903          683 :       gimple_set_bb (call_cond_stmt, new_bb);
    5904          683 :       gimple_seq_add_stmt (&gseq, call_cond_stmt);
    5905              : 
    5906          683 :       predicate_chain = TREE_CHAIN (predicate_chain);
    5907              : 
    5908          683 :       if (and_expr_var == NULL)
    5909              :         and_expr_var = cond_var;
    5910              :       else
    5911              :         {
    5912           39 :           gimple *assign_stmt;
    5913              :           /* Use MIN_EXPR to check if any integer is zero?.
    5914              :              and_expr_var = min_expr <cond_var, and_expr_var>  */
    5915           39 :           assign_stmt = gimple_build_assign (and_expr_var,
    5916              :                           build2 (MIN_EXPR, integer_type_node,
    5917              :                                   cond_var, and_expr_var));
    5918              : 
    5919           39 :           gimple_set_block (assign_stmt, DECL_INITIAL (function_decl));
    5920           39 :           gimple_set_bb (assign_stmt, new_bb);
    5921           39 :           gimple_seq_add_stmt (&gseq, assign_stmt);
    5922              :         }
    5923              :     }
    5924              : 
    5925          644 :   if_else_stmt = gimple_build_cond (GT_EXPR, and_expr_var,
    5926              :                                     integer_zero_node,
    5927              :                                     NULL_TREE, NULL_TREE);
    5928          644 :   gimple_set_block (if_else_stmt, DECL_INITIAL (function_decl));
    5929          644 :   gimple_set_bb (if_else_stmt, new_bb);
    5930          644 :   gimple_seq_add_stmt (&gseq, if_else_stmt);
    5931              : 
    5932          644 :   gimple_seq_add_stmt (&gseq, convert_stmt);
    5933          644 :   gimple_seq_add_stmt (&gseq, return_stmt);
    5934          644 :   set_bb_seq (new_bb, gseq);
    5935              : 
    5936          644 :   bb1 = new_bb;
    5937          644 :   e12 = split_block (bb1, if_else_stmt);
    5938          644 :   bb2 = e12->dest;
    5939          644 :   e12->flags &= ~EDGE_FALLTHRU;
    5940          644 :   e12->flags |= EDGE_TRUE_VALUE;
    5941              : 
    5942          644 :   e23 = split_block (bb2, return_stmt);
    5943              : 
    5944          644 :   gimple_set_bb (convert_stmt, bb2);
    5945          644 :   gimple_set_bb (return_stmt, bb2);
    5946              : 
    5947          644 :   bb3 = e23->dest;
    5948          644 :   make_edge (bb1, bb3, EDGE_FALSE_VALUE);
    5949              : 
    5950          644 :   remove_edge (e23);
    5951          644 :   make_edge (bb2, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
    5952              : 
    5953          644 :   pop_cfun ();
    5954              : 
    5955          644 :   return bb3;
    5956              : }
    5957              : 
    5958              : /* This function generates the dispatch function for
    5959              :    multi-versioned functions.  DISPATCH_DECL is the function which will
    5960              :    contain the dispatch logic.  FNDECLS are the function choices for
    5961              :    dispatch, and is a tree chain.  EMPTY_BB is the basic block pointer
    5962              :    in DISPATCH_DECL in which the dispatch code is generated.  */
    5963              : 
    5964              : static int
    5965          204 : dispatch_function_versions (tree dispatch_decl,
    5966              :                             void *fndecls_p,
    5967              :                             basic_block *empty_bb)
    5968              : {
    5969          204 :   tree default_decl;
    5970          204 :   gimple *ifunc_cpu_init_stmt;
    5971          204 :   gimple_seq gseq;
    5972          204 :   int ix;
    5973          204 :   tree ele;
    5974          204 :   vec<tree> *fndecls;
    5975          204 :   unsigned int num_versions = 0;
    5976          204 :   unsigned int actual_versions = 0;
    5977          204 :   unsigned int i;
    5978              : 
    5979          204 :   struct _function_version_info
    5980              :     {
    5981              :       tree version_decl;
    5982              :       tree predicate_chain;
    5983              :       unsigned int dispatch_priority;
    5984              :     }*function_version_info;
    5985              : 
    5986          204 :   gcc_assert (dispatch_decl != NULL
    5987              :               && fndecls_p != NULL
    5988              :               && empty_bb != NULL);
    5989              : 
    5990              :   /*fndecls_p is actually a vector.  */
    5991          204 :   fndecls = static_cast<vec<tree> *> (fndecls_p);
    5992              : 
    5993              :   /* At least one more version other than the default.  */
    5994          204 :   num_versions = fndecls->length ();
    5995          204 :   gcc_assert (num_versions >= 2);
    5996              : 
    5997          204 :   function_version_info = (struct _function_version_info *)
    5998          204 :     XNEWVEC (struct _function_version_info, (num_versions - 1));
    5999              : 
    6000              :   /* The first version in the vector is the default decl.  */
    6001          204 :   default_decl = (*fndecls)[0];
    6002              : 
    6003          204 :   push_cfun (DECL_STRUCT_FUNCTION (dispatch_decl));
    6004              : 
    6005          204 :   gseq = bb_seq (*empty_bb);
    6006              :   /* Function version dispatch is via IFUNC.  IFUNC resolvers fire before
    6007              :      constructors, so explicitly call __builtin_cpu_init here.  */
    6008          204 :   ifunc_cpu_init_stmt
    6009          204 :     = gimple_build_call_vec (get_ix86_builtin (IX86_BUILTIN_CPU_INIT), vNULL);
    6010          204 :   gimple_seq_add_stmt (&gseq, ifunc_cpu_init_stmt);
    6011          204 :   gimple_set_bb (ifunc_cpu_init_stmt, *empty_bb);
    6012          204 :   set_bb_seq (*empty_bb, gseq);
    6013              : 
    6014          204 :   pop_cfun ();
    6015              : 
    6016              : 
    6017         1209 :   for (ix = 1; fndecls->iterate (ix, &ele); ++ix)
    6018              :     {
    6019          801 :       tree version_decl = ele;
    6020          801 :       tree predicate_chain = NULL_TREE;
    6021          801 :       unsigned int priority;
    6022              :       /* Get attribute string, parse it and find the right predicate decl.
    6023              :          The predicate function could be a lengthy combination of many
    6024              :          features, like arch-type and various isa-variants.  */
    6025          801 :       priority = get_builtin_code_for_version (version_decl,
    6026              :                                                &predicate_chain);
    6027              : 
    6028          801 :       if (predicate_chain == NULL_TREE)
    6029          157 :         continue;
    6030              : 
    6031          644 :       function_version_info [actual_versions].version_decl = version_decl;
    6032          644 :       function_version_info [actual_versions].predicate_chain
    6033          644 :          = predicate_chain;
    6034          644 :       function_version_info [actual_versions].dispatch_priority = priority;
    6035          644 :       actual_versions++;
    6036              :     }
    6037              : 
    6038              :   /* Sort the versions according to descending order of dispatch priority.  The
    6039              :      priority is based on the ISA.  This is not a perfect solution.  There
    6040              :      could still be ambiguity.  If more than one function version is suitable
    6041              :      to execute,  which one should be dispatched?  In future, allow the user
    6042              :      to specify a dispatch  priority next to the version.  */
    6043          204 :   qsort (function_version_info, actual_versions,
    6044              :          sizeof (struct _function_version_info), feature_compare);
    6045              : 
    6046         1052 :   for  (i = 0; i < actual_versions; ++i)
    6047          644 :     *empty_bb = add_condition_to_bb (dispatch_decl,
    6048              :                                      function_version_info[i].version_decl,
    6049          644 :                                      function_version_info[i].predicate_chain,
    6050              :                                      *empty_bb);
    6051              : 
    6052              :   /* dispatch default version at the end.  */
    6053          204 :   *empty_bb = add_condition_to_bb (dispatch_decl, default_decl,
    6054              :                                    NULL, *empty_bb);
    6055              : 
    6056          204 :   free (function_version_info);
    6057          204 :   return 0;
    6058              : }
    6059              : 
    6060              : /* This function changes the assembler name for functions that are
    6061              :    versions.  If DECL is a function version and has a "target"
    6062              :    attribute, it appends the attribute string to its assembler name.  */
    6063              : 
    6064              : static tree
    6065         1127 : ix86_mangle_function_version_assembler_name (tree decl, tree id)
    6066              : {
    6067         1127 :   tree version_attr;
    6068         1127 :   char *attr_str;
    6069              : 
    6070         1127 :   if (DECL_DECLARED_INLINE_P (decl)
    6071         1176 :       && lookup_attribute ("gnu_inline",
    6072           49 :                            DECL_ATTRIBUTES (decl)))
    6073            0 :     error_at (DECL_SOURCE_LOCATION (decl),
    6074              :               "function versions cannot be marked as %<gnu_inline%>,"
    6075              :               " bodies have to be generated");
    6076              : 
    6077         1127 :   if (DECL_VIRTUAL_P (decl)
    6078         2254 :       || DECL_VINDEX (decl))
    6079            0 :     sorry ("virtual function multiversioning not supported");
    6080              : 
    6081         1127 :   version_attr = lookup_attribute ("target", DECL_ATTRIBUTES (decl));
    6082              : 
    6083              :   /* target attribute string cannot be NULL.  */
    6084         1127 :   gcc_assert (version_attr != NULL_TREE);
    6085              : 
    6086         1127 :   attr_str = sorted_attr_string (TREE_VALUE (version_attr));
    6087              : 
    6088              :   /* Allow assembler name to be modified if already set.  */
    6089         1127 :   if (DECL_ASSEMBLER_NAME_SET_P (decl))
    6090         1112 :     SET_DECL_RTL (decl, NULL);
    6091              : 
    6092         1127 :   tree ret = clone_identifier (id, attr_str, true);
    6093              : 
    6094         1127 :   XDELETEVEC (attr_str);
    6095              : 
    6096         1127 :   return ret;
    6097              : }
    6098              : 
    6099              : tree
    6100    505736639 : ix86_mangle_decl_assembler_name (tree decl, tree id)
    6101              : {
    6102              :   /* For function version, add the target suffix to the assembler name.  */
    6103    505736639 :   if (TREE_CODE (decl) == FUNCTION_DECL)
    6104              :     {
    6105    469535133 :       cgraph_node *node = cgraph_node::get (decl);
    6106              :       /* Mangle all versions when annotated with target_clones, but only
    6107              :          non-default versions when annotated with target attributes.  */
    6108    469535133 :       if (DECL_FUNCTION_VERSIONED (decl)
    6109    469535133 :           && (node->is_target_clone
    6110         1092 :               || !is_function_default_version (node->decl)))
    6111         1127 :         id = ix86_mangle_function_version_assembler_name (decl, id);
    6112              :       /* Mangle the dispatched symbol but only in the case of target clones.  */
    6113    469534006 :       else if (node && node->dispatcher_function && !node->is_target_clone)
    6114          117 :         id = clone_identifier (id, "ifunc");
    6115     67398264 :       else if (node && node->dispatcher_resolver_function)
    6116          204 :         id = clone_identifier (id, "resolver");
    6117              :     }
    6118              : #ifdef SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME
    6119              :   id = SUBTARGET_MANGLE_DECL_ASSEMBLER_NAME (decl, id);
    6120              : #endif
    6121              : 
    6122    505736639 :   return id;
    6123              : }
    6124              : 
    6125              : /* Make a dispatcher declaration for the multi-versioned function DECL.
    6126              :    Calls to DECL function will be replaced with calls to the dispatcher
    6127              :    by the front-end.  Returns the decl of the dispatcher function.  */
    6128              : 
    6129              : tree
    6130          330 : ix86_get_function_versions_dispatcher (void *decl)
    6131              : {
    6132          330 :   tree fn = (tree) decl;
    6133          330 :   struct cgraph_node *node = NULL;
    6134          330 :   struct cgraph_node *default_node = NULL;
    6135          330 :   struct cgraph_function_version_info *node_v = NULL;
    6136              : 
    6137          330 :   tree dispatch_decl = NULL;
    6138              : 
    6139          330 :   struct cgraph_function_version_info *default_version_info = NULL;
    6140              : 
    6141          660 :   gcc_assert (fn != NULL && DECL_FUNCTION_VERSIONED (fn));
    6142              : 
    6143          330 :   node = cgraph_node::get (fn);
    6144          330 :   gcc_assert (node != NULL);
    6145              : 
    6146          330 :   node_v = node->function_version ();
    6147          330 :   gcc_assert (node_v != NULL);
    6148              : 
    6149          330 :   if (node_v->dispatcher_resolver != NULL)
    6150              :     return node_v->dispatcher_resolver;
    6151              : 
    6152              :   /* The default node is always the beginning of the chain.  */
    6153              :   default_version_info = node_v;
    6154          681 :   while (default_version_info->prev != NULL)
    6155              :     default_version_info = default_version_info->prev;
    6156          216 :   default_node = default_version_info->this_node;
    6157              : 
    6158              :   /* If there is no default node, just return NULL.  */
    6159          216 :   if (!is_function_default_version (default_node->decl))
    6160              :     return NULL;
    6161              : 
    6162              : #if defined (ASM_OUTPUT_TYPE_DIRECTIVE)
    6163          207 :   if (targetm.has_ifunc_p ())
    6164              :     {
    6165          207 :       struct cgraph_function_version_info *it_v = NULL;
    6166              : 
    6167              :       /* Right now, the dispatching is done via ifunc.  */
    6168          207 :       dispatch_decl = make_dispatcher_decl (default_node->decl);
    6169              : 
    6170              :       /* Set the dispatcher for all the versions.  */
    6171          207 :       it_v = default_version_info;
    6172         1425 :       while (it_v != NULL)
    6173              :         {
    6174         1011 :           it_v->dispatcher_resolver = dispatch_decl;
    6175         1011 :           it_v = it_v->next;
    6176              :         }
    6177              :     }
    6178              :   else
    6179              : #endif
    6180              :     {
    6181            0 :       error_at (DECL_SOURCE_LOCATION (default_node->decl),
    6182              :                 "multiversioning needs %<ifunc%> which is not supported "
    6183              :                 "on this target");
    6184              :     }
    6185              : 
    6186              :   return dispatch_decl;
    6187              : }
    6188              : 
    6189              : /* Make the resolver function decl to dispatch the versions of
    6190              :    a multi-versioned function,  DEFAULT_DECL.  IFUNC_ALIAS_DECL is
    6191              :    ifunc alias that will point to the created resolver.  Create an
    6192              :    empty basic block in the resolver and store the pointer in
    6193              :    EMPTY_BB.  Return the decl of the resolver function.  */
    6194              : 
    6195              : static tree
    6196          204 : make_resolver_func (const tree default_decl,
    6197              :                     const tree ifunc_alias_decl,
    6198              :                     basic_block *empty_bb)
    6199              : {
    6200          204 :   tree decl, type, t;
    6201              : 
    6202              :   /* The resolver function should return a (void *). */
    6203          204 :   type = build_function_type_list (ptr_type_node, NULL_TREE);
    6204              : 
    6205          204 :   cgraph_node *node = cgraph_node::get (default_decl);
    6206          204 :   gcc_assert (node && node->function_version ());
    6207              : 
    6208          204 :   decl = build_fn_decl (IDENTIFIER_POINTER (DECL_NAME (default_decl)), type);
    6209              : 
    6210              :   /* Set the assembler name to prevent cgraph_node attempting to mangle.  */
    6211          204 :   SET_DECL_ASSEMBLER_NAME (decl, DECL_ASSEMBLER_NAME (default_decl));
    6212              : 
    6213          204 :   cgraph_node *resolver_node = cgraph_node::get_create (decl);
    6214          204 :   resolver_node->dispatcher_resolver_function = true;
    6215              : 
    6216          204 :   if (node->is_target_clone)
    6217           90 :     resolver_node->is_target_clone = true;
    6218              : 
    6219          204 :   tree id = ix86_mangle_decl_assembler_name
    6220          204 :     (decl, node->function_version ()->assembler_name);
    6221          204 :   symtab->change_decl_assembler_name (decl, id);
    6222              : 
    6223          204 :   DECL_NAME (decl) = DECL_NAME (default_decl);
    6224          204 :   TREE_USED (decl) = 1;
    6225          204 :   DECL_ARTIFICIAL (decl) = 1;
    6226          204 :   DECL_IGNORED_P (decl) = 1;
    6227          204 :   TREE_PUBLIC (decl) = 0;
    6228          204 :   DECL_UNINLINABLE (decl) = 1;
    6229              : 
    6230              :   /* Resolver is not external, body is generated.  */
    6231          204 :   DECL_EXTERNAL (decl) = 0;
    6232          204 :   DECL_EXTERNAL (ifunc_alias_decl) = 0;
    6233              : 
    6234          204 :   DECL_CONTEXT (decl) = NULL_TREE;
    6235          204 :   DECL_INITIAL (decl) = make_node (BLOCK);
    6236          204 :   DECL_STATIC_CONSTRUCTOR (decl) = 0;
    6237              : 
    6238          204 :   if (DECL_COMDAT_GROUP (default_decl)
    6239          204 :       || TREE_PUBLIC (default_decl))
    6240              :     {
    6241              :       /* In this case, each translation unit with a call to this
    6242              :          versioned function will put out a resolver.  Ensure it
    6243              :          is comdat to keep just one copy.  */
    6244          179 :       DECL_COMDAT (decl) = 1;
    6245          179 :       make_decl_one_only (decl, DECL_ASSEMBLER_NAME (decl));
    6246              :     }
    6247              :   else
    6248           25 :     TREE_PUBLIC (ifunc_alias_decl) = 0;
    6249              : 
    6250              :   /* Build result decl and add to function_decl. */
    6251          204 :   t = build_decl (UNKNOWN_LOCATION, RESULT_DECL, NULL_TREE, ptr_type_node);
    6252          204 :   DECL_CONTEXT (t) = decl;
    6253          204 :   DECL_ARTIFICIAL (t) = 1;
    6254          204 :   DECL_IGNORED_P (t) = 1;
    6255          204 :   DECL_RESULT (decl) = t;
    6256              : 
    6257          204 :   gimplify_function_tree (decl);
    6258          204 :   push_cfun (DECL_STRUCT_FUNCTION (decl));
    6259          204 :   *empty_bb = init_lowered_empty_function (decl, false,
    6260              :                                            profile_count::uninitialized ());
    6261              : 
    6262          204 :   cgraph_node::add_new_function (decl, true);
    6263          204 :   symtab->call_cgraph_insertion_hooks (cgraph_node::get_create (decl));
    6264              : 
    6265          204 :   pop_cfun ();
    6266              : 
    6267          204 :   gcc_assert (ifunc_alias_decl != NULL);
    6268              :   /* Mark ifunc_alias_decl as "ifunc" with resolver as resolver_name.  */
    6269          204 :   DECL_ATTRIBUTES (ifunc_alias_decl)
    6270          204 :     = make_attribute ("ifunc", IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)),
    6271          204 :                       DECL_ATTRIBUTES (ifunc_alias_decl));
    6272              : 
    6273              :   /* Create the alias for dispatch to resolver here.  */
    6274          204 :   cgraph_node::create_same_body_alias (ifunc_alias_decl, decl);
    6275          204 :   return decl;
    6276              : }
    6277              : 
    6278              : /* Generate the dispatching code body to dispatch multi-versioned function
    6279              :    DECL.  The target hook is called to process the "target" attributes and
    6280              :    provide the code to dispatch the right function at run-time.  NODE points
    6281              :    to the dispatcher decl whose body will be created.  */
    6282              : 
    6283              : tree
    6284          204 : ix86_generate_version_dispatcher_body (void *node_p)
    6285              : {
    6286          204 :   tree resolver_decl;
    6287          204 :   basic_block empty_bb;
    6288          204 :   tree default_ver_decl;
    6289          204 :   struct cgraph_node *versn;
    6290          204 :   struct cgraph_node *node;
    6291              : 
    6292          204 :   struct cgraph_function_version_info *node_version_info = NULL;
    6293          204 :   struct cgraph_function_version_info *versn_info = NULL;
    6294              : 
    6295          204 :   node = (cgraph_node *)node_p;
    6296              : 
    6297          204 :   node_version_info = node->function_version ();
    6298          204 :   gcc_assert (node->dispatcher_function
    6299              :               && node_version_info != NULL);
    6300              : 
    6301          204 :   if (node_version_info->dispatcher_resolver)
    6302              :     return node_version_info->dispatcher_resolver;
    6303              : 
    6304              :   /* The first version in the chain corresponds to the default version.  */
    6305          204 :   default_ver_decl = node_version_info->next->this_node->decl;
    6306              : 
    6307              :   /* node is going to be an alias, so remove the finalized bit.  */
    6308          204 :   node->definition = false;
    6309              : 
    6310          204 :   resolver_decl = make_resolver_func (default_ver_decl,
    6311              :                                       node->decl, &empty_bb);
    6312              : 
    6313          204 :   node_version_info->dispatcher_resolver = resolver_decl;
    6314              : 
    6315          204 :   push_cfun (DECL_STRUCT_FUNCTION (resolver_decl));
    6316              : 
    6317          204 :   auto_vec<tree, 2> fn_ver_vec;
    6318              : 
    6319         1209 :   for (versn_info = node_version_info->next; versn_info;
    6320         1005 :        versn_info = versn_info->next)
    6321              :     {
    6322         1005 :       versn = versn_info->this_node;
    6323              :       /* Check for virtual functions here again, as by this time it should
    6324              :          have been determined if this function needs a vtable index or
    6325              :          not.  This happens for methods in derived classes that override
    6326              :          virtual methods in base classes but are not explicitly marked as
    6327              :          virtual.  */
    6328         1005 :       if (DECL_VIRTUAL_P (versn->decl))
    6329            0 :         sorry ("virtual function multiversioning not supported");
    6330              : 
    6331         1005 :       fn_ver_vec.safe_push (versn->decl);
    6332              :     }
    6333              : 
    6334          204 :   dispatch_function_versions (resolver_decl, &fn_ver_vec, &empty_bb);
    6335          204 :   cgraph_edge::rebuild_edges ();
    6336          204 :   pop_cfun ();
    6337          204 :   return resolver_decl;
    6338          204 : }
    6339              : 
    6340              : 
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.